<commit_before>package terraform

import (
	"fmt"
	"strings"
	"testing"
	"time"

	"github.com/hashicorp/go-version"
	"github.com/hashicorp/terraform/flatmap"
)

func TestNewContextRequiredVersion(t *testing.T) {
	cases := []struct {
		Name    string
		Module  string
		Version string
		Value   string
		Err     bool
	}{
		{
			"no requirement",
			"",
			"0.1.0",
			"",
			false,
		},

		{
			"doesn't match",
			"",
			"0.1.0",
			"> 0.6.0",
			true,
		},

		{
			"matches",
			"",
			"0.7.0",
			"> 0.6.0",
			false,
		},

		{
			"module matches",
			"context-required-version-module",
			"0.5.0",
			"",
			false,
		},

		{
			"module doesn't match",
			"context-required-version-module",
			"0.4.0",
			"",
			true,
		},
	}

	for i, tc := range cases {
		t.Run(fmt.Sprintf("%d-%s", i, tc.Name), func(t *testing.T) {
			// Reset the version for the tests
			old := SemVersion
			SemVersion = version.Must(version.NewVersion(tc.Version))
			defer func() { SemVersion = old }()

			name := "context-required-version"
			if tc.Module != "" {
				name = tc.Module
			}
			mod := testModule(t, name)
			if tc.Value != "" {
				mod.Config().Terraform.RequiredVersion = tc.Value
			}
			_, err := NewContext(&ContextOpts{
				Module: mod,
			})
			if (err != nil) != tc.Err {
				t.Fatalf("err: %s", err)
			}
			if err != nil {
				return
			}
		})
	}
}

func TestNewContextState(t *testing.T) {
	cases := map[string]struct {
		Input *ContextOpts
		Err   bool
	}{
		"empty TFVersion": {
			&ContextOpts{
				State: &State{},
			},
			false,
		},

		"past TFVersion": {
			&ContextOpts{
				State: &State{TFVersion: "0.1.2"},
			},
			false,
		},

		"equal TFVersion": {
			&ContextOpts{
				State: &State{TFVersion: Version},
			},
			false,
		},

		"future TFVersion": {
			&ContextOpts{
				State: &State{TFVersion: "99.99.99"},
			},
			true,
		},

		"future TFVersion, allowed": {
			&ContextOpts{
				State:              &State{TFVersion: "99.99.99"},
				StateFutureAllowed: true,
			},
			false,
		},
	}

	for k, tc := range cases {
		ctx, err := NewContext(tc.Input)
		if (err != nil) != tc.Err {
			t.Fatalf("%s: err: %s", k, err)
		}
		if err != nil {
			continue
		}

		// Version should always be set to our current
		if ctx.state.TFVersion != Version {
			t.Fatalf("%s: state not set to current version", k)
		}
	}
}

func testContext2(t *testing.T, opts *ContextOpts) *Context {
	// Enable the shadow graph
	opts.Shadow = true

	ctx, err := NewContext(opts)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	return ctx
}

func testDataApplyFn(
	info *InstanceInfo,
	d *InstanceDiff) (*InstanceState, error) {
	return testApplyFn(info, new(InstanceState), d)
}

func testDataDiffFn(
	info *InstanceInfo,
	c *ResourceConfig) (*InstanceDiff, error) {
	return testDiffFn(info, new(InstanceState), c)
}

func testApplyFn(
	info *InstanceInfo,
	s *InstanceState,
	d *InstanceDiff) (*InstanceState, error) {
	if d.Destroy {
		return nil, nil
	}

	id := "foo"
	if idAttr, ok := d.Attributes["id"]; ok && !idAttr.NewComputed {
		id = idAttr.New
	}

	result := &InstanceState{
		ID:         id,
		Attributes: make(map[string]string),
	}

	// Copy all the prior attributes
	for k, v := range s.Attributes {
		result.Attributes[k] = v
	}

	if d != nil {
		result = result.MergeDiff(d)
	}
	return result, nil
}

func testDiffFn(
	info *InstanceInfo,
	s *InstanceState,
	c *ResourceConfig) (*InstanceDiff, error) {
	diff := new(InstanceDiff)
	diff.Attributes = make(map[string]*ResourceAttrDiff)

	if s != nil {
		diff.DestroyTainted = s.Tainted
	}

	for k, v := range c.Raw {
		if _, ok := v.(string); !ok {
			continue
		}

		// Ignore __-prefixed keys since they're used for magic
		if k[0] == '_' && k[1] == '_' {
			continue
		}

		if k == "nil" {
			return nil, nil
		}

		// This key is used for other purposes
		if k == "compute_value" {
			continue
		}

		if k == "compute" {
			attrDiff := &ResourceAttrDiff{
				Old:         "",
				New:         "",
				NewComputed: true,
			}

			if cv, ok := c.Config["compute_value"]; ok {
				if cv.(string) == "1" {
					attrDiff.NewComputed = false
					attrDiff.New = fmt.Sprintf("computed_%s", v.(string))
				}
			}

			diff.Attributes[v.(string)] = attrDiff
			continue
		}

		// If this key is not computed, then look it up in the
		// cleaned config.
		found := false
		for _, ck := range c.ComputedKeys {
			if ck == k {
				found = true
				break
			}
		}
		if !found {
			v = c.Config[k]
		}

		for k, attrDiff := range testFlatAttrDiffs(k, v) {
			if k == "require_new" {
				attrDiff.RequiresNew = true
			}
			if _, ok := c.Raw["__"+k+"_requires_new"]; ok {
				attrDiff.RequiresNew = true
			}
			diff.Attributes[k] = attrDiff
		}
	}

	for _, k := range c.ComputedKeys {
		diff.Attributes[k] = &ResourceAttrDiff{
			Old:         "",
			NewComputed: true,
		}
	}

	// If we recreate this resource because it's tainted, we keep all attrs
	if !diff.RequiresNew() {
		for k, v := range diff.Attributes {
			if v.NewComputed {
				continue
			}

			old, ok := s.Attributes[k]
			if !ok {
				continue
			}

			if old == v.New {
				delete(diff.Attributes, k)
			}
		}
	}

	if !diff.Empty() {
		diff.Attributes["type"] = &ResourceAttrDiff{
			Old: "",
			New: info.Type,
		}
	}

	return diff, nil
}

// generate ResourceAttrDiffs for nested data structures in tests
func testFlatAttrDiffs(k string, i interface{}) map[string]*ResourceAttrDiff {
	diffs := make(map[string]*ResourceAttrDiff)
	// check for strings and empty containers first
	switch t := i.(type) {
	case string:
		diffs[k] = &ResourceAttrDiff{New: t}
		return diffs
	case map[string]interface{}:
		if len(t) == 0 {
			diffs[k] = &ResourceAttrDiff{New: ""}
			return diffs
		}
	case []interface{}:
		if len(t) == 0 {
			diffs[k] = &ResourceAttrDiff{New: ""}
			return diffs
		}
	}

	flat := flatmap.Flatten(map[string]interface{}{k: i})

	for k, v := range flat {
		attrDiff := &ResourceAttrDiff{
			Old: "",
			New: v,
		}
		diffs[k] = attrDiff
	}

	return diffs
}

func testProvider(prefix string) *MockResourceProvider {
	p := new(MockResourceProvider)
	p.RefreshFn = func(info *InstanceInfo, s *InstanceState) (*InstanceState, error) {
		return s, nil
	}
	p.ResourcesReturn = []ResourceType{
		ResourceType{
			Name: fmt.Sprintf("%s_instance", prefix),
		},
	}

	return p
}

func testProvisioner() *MockResourceProvisioner {
	p := new(MockResourceProvisioner)
	return p
}

func checkStateString(t *testing.T, state *State, expected string) {
	actual := strings.TrimSpace(state.String())
	expected = strings.TrimSpace(expected)

	if actual != expected {
		t.Fatalf("state does not match! actual:\n%s\n\nexpected:\n%s", actual, expected)
	}
}

func resourceState(resourceType, resourceID string) *ResourceState {
	return &ResourceState{
		Type: resourceType,
		Primary: &InstanceState{
			ID: resourceID,
		},
	}
}

// Test helper that gives a function 3 seconds to finish, assumes deadlock and
// fails test if it does not.
func testCheckDeadlock(t *testing.T, f func()) {
	timeout := make(chan bool, 1)
	done := make(chan bool, 1)
	go func() {
		time.Sleep(3 * time.Second)
		timeout <- true
	}()
	go func(f func(), done chan bool) {
		defer func() { done <- true }()
		f()
	}(f, done)
	select {
	case <-timeout:
		t.Fatalf("timed out! probably deadlock")
	case <-done:
		// ok
	}
}

const testContextGraph = `
root: root
aws_instance.bar
  aws_instance.bar -> provider.aws
aws_instance.foo
  aws_instance.foo -> provider.aws
provider.aws
root
  root -> aws_instance.bar
  root -> aws_instance.foo
`

const testContextRefreshModuleStr = `
aws_instance.web: (tainted)
  ID = bar

module.child:
  aws_instance.web:
    ID = new
`

const testContextRefreshOutputStr = `
aws_instance.web:
  ID = foo
  foo = bar

Outputs:

foo = bar
`

const testContextRefreshOutputPartialStr = `
<no state>
`

const testContextRefreshTaintedStr = `
aws_instance.web: (tainted)
  ID = foo
`
<commit_msg>Allow test fixtures with non-string attributes<commit_after>package terraform

import (
	"fmt"
	"strings"
	"testing"
	"time"

	"github.com/hashicorp/go-version"
	"github.com/hashicorp/terraform/flatmap"
)

func TestNewContextRequiredVersion(t *testing.T) {
	cases := []struct {
		Name    string
		Module  string
		Version string
		Value   string
		Err     bool
	}{
		{
			"no requirement",
			"",
			"0.1.0",
			"",
			false,
		},

		{
			"doesn't match",
			"",
			"0.1.0",
			"> 0.6.0",
			true,
		},

		{
			"matches",
			"",
			"0.7.0",
			"> 0.6.0",
			false,
		},

		{
			"module matches",
			"context-required-version-module",
			"0.5.0",
			"",
			false,
		},

		{
			"module doesn't match",
			"context-required-version-module",
			"0.4.0",
			"",
			true,
		},
	}

	for i, tc := range cases {
		t.Run(fmt.Sprintf("%d-%s", i, tc.Name), func(t *testing.T) {
			// Reset the version for the tests
			old := SemVersion
			SemVersion = version.Must(version.NewVersion(tc.Version))
			defer func() { SemVersion = old }()

			name := "context-required-version"
			if tc.Module != "" {
				name = tc.Module
			}
			mod := testModule(t, name)
			if tc.Value != "" {
				mod.Config().Terraform.RequiredVersion = tc.Value
			}
			_, err := NewContext(&ContextOpts{
				Module: mod,
			})
			if (err != nil) != tc.Err {
				t.Fatalf("err: %s", err)
			}
			if err != nil {
				return
			}
		})
	}
}

func TestNewContextState(t *testing.T) {
	cases := map[string]struct {
		Input *ContextOpts
		Err   bool
	}{
		"empty TFVersion": {
			&ContextOpts{
				State: &State{},
			},
			false,
		},

		"past TFVersion": {
			&ContextOpts{
				State: &State{TFVersion: "0.1.2"},
			},
			false,
		},

		"equal TFVersion": {
			&ContextOpts{
				State: &State{TFVersion: Version},
			},
			false,
		},

		"future TFVersion": {
			&ContextOpts{
				State: &State{TFVersion: "99.99.99"},
			},
			true,
		},

		"future TFVersion, allowed": {
			&ContextOpts{
				State:              &State{TFVersion: "99.99.99"},
				StateFutureAllowed: true,
			},
			false,
		},
	}

	for k, tc := range cases {
		ctx, err := NewContext(tc.Input)
		if (err != nil) != tc.Err {
			t.Fatalf("%s: err: %s", k, err)
		}
		if err != nil {
			continue
		}

		// Version should always be set to our current
		if ctx.state.TFVersion != Version {
			t.Fatalf("%s: state not set to current version", k)
		}
	}
}

func testContext2(t *testing.T, opts *ContextOpts) *Context {
	// Enable the shadow graph
	opts.Shadow = true

	ctx, err := NewContext(opts)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	return ctx
}

func testDataApplyFn(
	info *InstanceInfo,
	d *InstanceDiff) (*InstanceState, error) {
	return testApplyFn(info, new(InstanceState), d)
}

func testDataDiffFn(
	info *InstanceInfo,
	c *ResourceConfig) (*InstanceDiff, error) {
	return testDiffFn(info, new(InstanceState), c)
}

func testApplyFn(
	info *InstanceInfo,
	s *InstanceState,
	d *InstanceDiff) (*InstanceState, error) {
	if d.Destroy {
		return nil, nil
	}

	id := "foo"
	if idAttr, ok := d.Attributes["id"]; ok && !idAttr.NewComputed {
		id = idAttr.New
	}

	result := &InstanceState{
		ID:         id,
		Attributes: make(map[string]string),
	}

	// Copy all the prior attributes
	for k, v := range s.Attributes {
		result.Attributes[k] = v
	}

	if d != nil {
		result = result.MergeDiff(d)
	}
	return result, nil
}

func testDiffFn(
	info *InstanceInfo,
	s *InstanceState,
	c *ResourceConfig) (*InstanceDiff, error) {
	diff := new(InstanceDiff)
	diff.Attributes = make(map[string]*ResourceAttrDiff)

	if s != nil {
		diff.DestroyTainted = s.Tainted
	}

	for k, v := range c.Raw {
		// Ignore __-prefixed keys since they're used for magic
		if k[0] == '_' && k[1] == '_' {
			continue
		}

		if k == "nil" {
			return nil, nil
		}

		// This key is used for other purposes
		if k == "compute_value" {
			continue
		}

		if k == "compute" {
			attrDiff := &ResourceAttrDiff{
				Old:         "",
				New:         "",
				NewComputed: true,
			}

			if cv, ok := c.Config["compute_value"]; ok {
				if cv.(string) == "1" {
					attrDiff.NewComputed = false
					attrDiff.New = fmt.Sprintf("computed_%s", v.(string))
				}
			}

			diff.Attributes[v.(string)] = attrDiff
			continue
		}

		// If this key is not computed, then look it up in the
		// cleaned config.
		found := false
		for _, ck := range c.ComputedKeys {
			if ck == k {
				found = true
				break
			}
		}
		if !found {
			v = c.Config[k]
		}

		for k, attrDiff := range testFlatAttrDiffs(k, v) {
			if k == "require_new" {
				attrDiff.RequiresNew = true
			}
			if _, ok := c.Raw["__"+k+"_requires_new"]; ok {
				attrDiff.RequiresNew = true
			}
			diff.Attributes[k] = attrDiff
		}
	}

	for _, k := range c.ComputedKeys {
		diff.Attributes[k] = &ResourceAttrDiff{
			Old:         "",
			NewComputed: true,
		}
	}

	// If we recreate this resource because it's tainted, we keep all attrs
	if !diff.RequiresNew() {
		for k, v := range diff.Attributes {
			if v.NewComputed {
				continue
			}

			old, ok := s.Attributes[k]
			if !ok {
				continue
			}

			if old == v.New {
				delete(diff.Attributes, k)
			}
		}
	}

	if !diff.Empty() {
		diff.Attributes["type"] = &ResourceAttrDiff{
			Old: "",
			New: info.Type,
		}
	}

	return diff, nil
}

// generate ResourceAttrDiffs for nested data structures in tests
func testFlatAttrDiffs(k string, i interface{}) map[string]*ResourceAttrDiff {
	diffs := make(map[string]*ResourceAttrDiff)
	// check for strings and empty containers first
	switch t := i.(type) {
	case string:
		diffs[k] = &ResourceAttrDiff{New: t}
		return diffs
	case map[string]interface{}:
		if len(t) == 0 {
			diffs[k] = &ResourceAttrDiff{New: ""}
			return diffs
		}
	case []interface{}:
		if len(t) == 0 {
			diffs[k] = &ResourceAttrDiff{New: ""}
			return diffs
		}
	}

	flat := flatmap.Flatten(map[string]interface{}{k: i})

	for k, v := range flat {
		attrDiff := &ResourceAttrDiff{
			Old: "",
			New: v,
		}
		diffs[k] = attrDiff
	}

	return diffs
}

func testProvider(prefix string) *MockResourceProvider {
	p := new(MockResourceProvider)
	p.RefreshFn = func(info *InstanceInfo, s *InstanceState) (*InstanceState, error) {
		return s, nil
	}
	p.ResourcesReturn = []ResourceType{
		ResourceType{
			Name: fmt.Sprintf("%s_instance", prefix),
		},
	}

	return p
}

func testProvisioner() *MockResourceProvisioner {
	p := new(MockResourceProvisioner)
	return p
}

func checkStateString(t *testing.T, state *State, expected string) {
	actual := strings.TrimSpace(state.String())
	expected = strings.TrimSpace(expected)

	if actual != expected {
		t.Fatalf("state does not match! actual:\n%s\n\nexpected:\n%s", actual, expected)
	}
}

func resourceState(resourceType, resourceID string) *ResourceState {
	return &ResourceState{
		Type: resourceType,
		Primary: &InstanceState{
			ID: resourceID,
		},
	}
}

// Test helper that gives a function 3 seconds to finish, assumes deadlock and
// fails test if it does not.
func testCheckDeadlock(t *testing.T, f func()) {
	timeout := make(chan bool, 1)
	done := make(chan bool, 1)
	go func() {
		time.Sleep(3 * time.Second)
		timeout <- true
	}()
	go func(f func(), done chan bool) {
		defer func() { done <- true }()
		f()
	}(f, done)
	select {
	case <-timeout:
		t.Fatalf("timed out! probably deadlock")
	case <-done:
		// ok
	}
}

const testContextGraph = `
root: root
aws_instance.bar
  aws_instance.bar -> provider.aws
aws_instance.foo
  aws_instance.foo -> provider.aws
provider.aws
root
  root -> aws_instance.bar
  root -> aws_instance.foo
`

const testContextRefreshModuleStr = `
aws_instance.web: (tainted)
  ID = bar

module.child:
  aws_instance.web:
    ID = new
`

const testContextRefreshOutputStr = `
aws_instance.web:
  ID = foo
  foo = bar

Outputs:

foo = bar
`

const testContextRefreshOutputPartialStr = `
<no state>
`

const testContextRefreshTaintedStr = `
aws_instance.web: (tainted)
  ID = foo
`
<|endoftext|>


<commit_before>// Copyright 2016 Bob Ziuchkovski. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package writ_test

import (
	"github.com/ziuchkovski/writ"
	"os"
	"runtime"
)

type Config struct {
	help      bool
	verbosity int

	// A hidden flag
	tolerance float32

	// A dynamically added option for Mac OS
	useQuartz bool
}

// This example demonstrates explicit Command and Option construction
// without the use of writ.New()
func Example_explicit() {
	config := &Config{}
	cmd := &writ.Command{Name: "explicit"}
	cmd.Options = []*writ.Option{
		&writ.Option{
			Names:       []string{"h", "help"},
			Description: "Display this help text and exit",
			Decoder:     writ.NewFlagDecoder(&config.help),
			Flag:        true,
		},
		&writ.Option{
			Names:       []string{"v"},
			Description: "Increase verbosity; may be specified more than once",
			Decoder:     writ.NewFlagAccumulator(&config.verbosity),
			Flag:        true,
			Plural:      true,
		},
		&writ.Option{
			Names:       []string{"t", "tolerance"},
			Description: "Set the tolerance level (from 0.0 - 1.0)",
			Placeholder: "DECIMAL",
			Decoder:     writ.NewOptionDecoder(&config.tolerance),
		},
	}

	cmd.Help = writ.Help{
		Usage:  "Usage: explicit [OPTION]...",
		Header: "Explicit demonstrates explicit commands and options",
		Footer: "This method is flexible but more verbose than using writ.New()",
	}
	general := cmd.GroupOptions("help", "v")
	general.Header = "General Options:"
	cmd.Help.OptionGroups = append(cmd.Help.OptionGroups, general)

	if runtime.GOOS == "darwin" {
		cmd.Options = append(cmd.Options, &writ.Option{
			Names:       []string{"use-quartz"},
			Description: "Use Quartz display on Mac",
			Decoder:     writ.NewFlagDecoder(&config.useQuartz),
			Flag:        true,
		})
		platform := cmd.GroupOptions("use-quartz")
		platform.Header = "Platform Options:"
		cmd.Help.OptionGroups = append(cmd.Help.OptionGroups, platform)
	}

	// Decode the options
	_, _, err := cmd.Decode(os.Args[1:])
	if err != nil || config.help {
		cmd.ExitHelp(err)
	}
}
<commit_msg>Minor simplifications to example_explicit_test.go<commit_after>// Copyright 2016 Bob Ziuchkovski. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package writ_test

import (
	"github.com/ziuchkovski/writ"
	"os"
	"runtime"
)

type Config struct {
	help      bool
	verbosity int

	// A hidden flag
	tolerance float32

	// A dynamically added option for Mac OS
	useQuartz bool
}

// This example demonstrates explicit Command and Option construction
// without the use of writ.New()
func Example_explicit() {
	config := &Config{}
	cmd := &writ.Command{Name: "explicit"}
	cmd.Options = []*writ.Option{
		{
			Names:       []string{"h", "help"},
			Description: "Display this help text and exit",
			Decoder:     writ.NewFlagDecoder(&config.help),
			Flag:        true,
		},
		{
			Names:       []string{"v"},
			Description: "Increase verbosity; may be specified more than once",
			Decoder:     writ.NewFlagAccumulator(&config.verbosity),
			Flag:        true,
			Plural:      true,
		},
		{
			Names:       []string{"t", "tolerance"},
			Description: "Set the tolerance level (from 0.0 - 1.0)",
			Placeholder: "DECIMAL",
			Decoder:     writ.NewOptionDecoder(&config.tolerance),
		},
	}

	cmd.Help = writ.Help{
		Usage:  "Usage: explicit [OPTION]...",
		Header: "Explicit demonstrates explicit commands and options",
		Footer: "This method is flexible but more verbose than using writ.New()",
	}
	general := cmd.GroupOptions("help", "v")
	general.Header = "General Options:"
	cmd.Help.OptionGroups = append(cmd.Help.OptionGroups, general)

	if runtime.GOOS == "darwin" {
		cmd.Options = append(cmd.Options, &writ.Option{
			Names:       []string{"use-quartz"},
			Description: "Use Quartz display on Mac",
			Decoder:     writ.NewFlagDecoder(&config.useQuartz),
			Flag:        true,
		})
		platform := cmd.GroupOptions("use-quartz")
		platform.Header = "Platform Options:"
		cmd.Help.OptionGroups = append(cmd.Help.OptionGroups, platform)
	}

	// Decode the options
	_, _, err := cmd.Decode(os.Args[1:])
	if err != nil || config.help {
		cmd.ExitHelp(err)
	}
}
<|endoftext|>


<commit_before>package reddit

import (
	"net/http"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/clientcredentials"
)

var oauthScopes = []string{
	"identity",
	"read",
	"privatemessages",
	"submit",
	"history",
}

type appClient struct {
	baseClient
	cfg    clientConfig
	cli    *http.Client
	expiry time.Time
}

func (a *appClient) Do(req *http.Request) ([]byte, error) {
	if time.Until(a.expiry) < time.Minute*5 {
		if err := a.authorize(); err != nil {
			return nil, err
		}
	}

	return a.baseClient.Do(req)
}

func (a *appClient) authorize() error {
	ctx := context.WithValue(oauth2.NoContext, oauth2.HTTPClient, a.cli)

	if a.cfg.app.Username == "" || a.cfg.app.Password == "" {
		a.baseClient.cli = a.clientCredentialsClient(ctx)
		return nil
	}

	cfg := &oauth2.Config{
		ClientID:     a.cfg.app.ID,
		ClientSecret: a.cfg.app.Secret,
		Endpoint:     oauth2.Endpoint{TokenURL: a.cfg.app.tokenURL},
		Scopes:       oauthScopes,
	}

	token, err := cfg.PasswordCredentialsToken(
		ctx,
		a.cfg.app.Username,
		a.cfg.app.Password,
	)
	if err != nil {
		return err
	}

	a.baseClient.cli = cfg.Client(ctx, token)
	a.expiry = token.Expiry
	return err
}

func (a *appClient) clientCredentialsClient(ctx context.Context) *http.Client {
	cfg := &clientcredentials.Config{
		ClientID:     a.cfg.app.ID,
		ClientSecret: a.cfg.app.Secret,
		TokenURL:     a.cfg.app.tokenURL,
		Scopes:       oauthScopes,
	}

	return cfg.Client(ctx)
}

func newAppClient(c clientConfig) (*appClient, error) {
	var client *http.Client
	if c.client == nil {
		client = clientWithAgent(c.agent)
	} else {
		client = patchWithAgent(client, c.agent)
	}

	a := &appClient{
		cli: client,
		cfg: c,
	}
	return a, a.authorize()
}
<commit_msg>pass correct client to patchWithAgent in newAppClient (#77)<commit_after>package reddit

import (
	"net/http"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/clientcredentials"
)

var oauthScopes = []string{
	"identity",
	"read",
	"privatemessages",
	"submit",
	"history",
}

type appClient struct {
	baseClient
	cfg    clientConfig
	cli    *http.Client
	expiry time.Time
}

func (a *appClient) Do(req *http.Request) ([]byte, error) {
	if time.Until(a.expiry) < time.Minute*5 {
		if err := a.authorize(); err != nil {
			return nil, err
		}
	}

	return a.baseClient.Do(req)
}

func (a *appClient) authorize() error {
	ctx := context.WithValue(oauth2.NoContext, oauth2.HTTPClient, a.cli)

	if a.cfg.app.Username == "" || a.cfg.app.Password == "" {
		a.baseClient.cli = a.clientCredentialsClient(ctx)
		return nil
	}

	cfg := &oauth2.Config{
		ClientID:     a.cfg.app.ID,
		ClientSecret: a.cfg.app.Secret,
		Endpoint:     oauth2.Endpoint{TokenURL: a.cfg.app.tokenURL},
		Scopes:       oauthScopes,
	}

	token, err := cfg.PasswordCredentialsToken(
		ctx,
		a.cfg.app.Username,
		a.cfg.app.Password,
	)
	if err != nil {
		return err
	}

	a.baseClient.cli = cfg.Client(ctx, token)
	a.expiry = token.Expiry
	return err
}

func (a *appClient) clientCredentialsClient(ctx context.Context) *http.Client {
	cfg := &clientcredentials.Config{
		ClientID:     a.cfg.app.ID,
		ClientSecret: a.cfg.app.Secret,
		TokenURL:     a.cfg.app.tokenURL,
		Scopes:       oauthScopes,
	}

	return cfg.Client(ctx)
}

func newAppClient(c clientConfig) (*appClient, error) {
	var client *http.Client
	if c.client == nil {
		client = clientWithAgent(c.agent)
	} else {
		client = patchWithAgent(c.client, c.agent)
	}

	a := &appClient{
		cli: client,
		cfg: c,
	}
	return a, a.authorize()
}
<|endoftext|>


<commit_before>package pixur

import (
	"database/sql"
	"fmt"
	"os"
	"testing"

	ptest "pixur.org/pixur/testing"
)

var (
	testDB *sql.DB
)

func TestMain(m *testing.M) {
	db, err := ptest.GetDB()

	if err != nil {
		fmt.Println("error getting db", err)
		os.Exit(1)
	}
	defer ptest.CleanUp()
	testDB = db

	if err := createTables(db); err != nil {
		fmt.Println("error creating db tables", err)
		os.Exit(1)
	}

	os.Exit(m.Run())
}
<commit_msg>Standardize how setup and teardown methods are called.  The idea is to be like Junit4's BeforeClass<commit_after>package pixur

import (
	"database/sql"
	"fmt"
	"os"
	"testing"

	ptest "pixur.org/pixur/testing"
)

var (
	testDB         *sql.DB
	_testSetups    []func() error
	_testTearDowns []func() error
)

func BeforeAfterTestSuite(before func() error) {
	_testSetups = append(_testSetups, before)
}

func AfterTestSuite(after func() error) {
	_testTearDowns = append(_testTearDowns, after)
}

func init() {
	BeforeAfterTestSuite(func() error {
		db, err := ptest.GetDB()
		if err != nil {
			return err
		}
		AfterTestSuite(func() error {
			ptest.CleanUp()
			return nil
		})
		testDB = db
		if err := createTables(db); err != nil {
			return err
		}
		return nil
	})
}

func runTests(m *testing.M) int {
	defer func() {
		for _, after := range _testTearDowns {
			if err := after(); err != nil {
				fmt.Println("Error in teardown", err)
			}
		}
	}()

	for _, before := range _testSetups {
		if err := before(); err != nil {
			fmt.Println("Error in test setup", err)
			return 1
		}
	}

	return m.Run()
}

func TestMain(m *testing.M) {
	os.Exit(runTests(m))
}
<|endoftext|>


<commit_before>// Package reflectx implements extensions to the standard reflect lib suitable
// for implementing marshaling and unmarshaling packages.  The main Mapper type
// allows for Go-compatible named atribute access, including accessing embedded
// struct attributes and the ability to use  functions and struct tags to
// customize field names.
//
package reflectx

import (
	"fmt"
	"reflect"
	"runtime"
	"strings"
	"sync"
)

// A FieldInfo is a collection of metadata about a struct field.
type FieldInfo struct {
	Index    []int
	Path     string
	Field    reflect.StructField
	Zero     reflect.Value
	Name     string
	Options  map[string]string
	Embedded bool
}

// A StructMap is an index of field metadata for a struct.
type StructMap struct {
	Index []*FieldInfo
	Paths map[string]*FieldInfo
	Names map[string]*FieldInfo
}

// GetByPath returns a *FieldInfo for a given string path.
func (f StructMap) GetByPath(path string) *FieldInfo {
	return f.Paths[path]
}

// GetByTraversal returns a *FieldInfo for a given integer path.  It is
// analagous to reflect.FieldByIndex.
func (f StructMap) GetByTraversal(index []int) *FieldInfo {
	// XXX: this function has a lot worse complexity than the one it replaces.
	// It should work exactly as FieldByIndexes does below, but instead of
	// actually traversing a tree it linearly checks a big list of all possible
	// traversals, which doesn't feel right.  Ideally, StructMap.Index could
	// be changed to be a Tree, but I'm not sure that's possible since it's
	// exposed.
	n := len(index)
	for _, fi := range f.Index {
		if len(fi.Index) != n {
			continue
		}
		same := true
		for i, v := range fi.Index {
			if index[i] != v {
				same = false
				break
			}
		}
		if same {
			return fi
		}
	}
	return nil
}

// Mapper is a general purpose mapper of names to struct fields.  A Mapper
// behaves like most marshallers, optionally obeying a field tag for name
// mapping and a function to provide a basic mapping of fields to names.
type Mapper struct {
	cache      map[reflect.Type]*StructMap
	tagName    string
	tagMapFunc func(string) string
	mapFunc    func(string) string
	mutex      sync.Mutex
}

// NewMapper returns a new mapper which optionally obeys the field tag given
// by tagName.  If tagName is the empty string, it is ignored.
func NewMapper(tagName string) *Mapper {
	return &Mapper{
		cache:   make(map[reflect.Type]*StructMap),
		tagName: tagName,
	}
}

// NewMapperTagFunc returns a new mapper which contains a mapper for field names
// AND a mapper for tag values.  This is useful for tags like json which can
// have values like "name,omitempty".
func NewMapperTagFunc(tagName string, mapFunc, tagMapFunc func(string) string) *Mapper {
	return &Mapper{
		cache:      make(map[reflect.Type]*StructMap),
		tagName:    tagName,
		mapFunc:    mapFunc,
		tagMapFunc: tagMapFunc,
	}
}

// NewMapperFunc returns a new mapper which optionally obeys a field tag and
// a struct field name mapper func given by f.  Tags will take precedence, but
// for any other field, the mapped name will be f(field.Name)
func NewMapperFunc(tagName string, f func(string) string) *Mapper {
	return &Mapper{
		cache:   make(map[reflect.Type]*StructMap),
		tagName: tagName,
		mapFunc: f,
	}
}

// TypeMap returns a mapping of field strings to int slices representing
// the traversal down the struct to reach the field.
func (m *Mapper) TypeMap(t reflect.Type) *StructMap {
	m.mutex.Lock()
	mapping, ok := m.cache[t]
	if !ok {
		mapping = getMapping(t, m.tagName, m.mapFunc, m.tagMapFunc)
		m.cache[t] = mapping
	}
	m.mutex.Unlock()
	return mapping
}

// FieldMap returns the mapper's mapping of field names to reflect values.  Panics
// if v's Kind is not Struct, or v is not Indirectable to a struct kind.
func (m *Mapper) FieldMap(v reflect.Value) map[string]reflect.Value {
	v = reflect.Indirect(v)
	mustBe(v, reflect.Struct)

	r := map[string]reflect.Value{}
	tm := m.TypeMap(v.Type())
	for tagName, fi := range tm.Names {
		r[tagName] = FieldByIndexes(v, fi.Index)
	}
	return r
}

// FieldByName returns a field by the its mapped name as a reflect.Value.
// Panics if v's Kind is not Struct or v is not Indirectable to a struct Kind.
// Returns zero Value if the name is not found.
func (m *Mapper) FieldByName(v reflect.Value, name string) reflect.Value {
	v = reflect.Indirect(v)
	mustBe(v, reflect.Struct)

	tm := m.TypeMap(v.Type())
	fi, ok := tm.Names[name]
	if !ok {
		return v
	}
	return FieldByIndexes(v, fi.Index)
}

// FieldsByName returns a slice of values corresponding to the slice of names
// for the value.  Panics if v's Kind is not Struct or v is not Indirectable
// to a struct Kind.  Returns zero Value for each name not found.
func (m *Mapper) FieldsByName(v reflect.Value, names []string) []reflect.Value {
	v = reflect.Indirect(v)
	mustBe(v, reflect.Struct)

	tm := m.TypeMap(v.Type())
	vals := make([]reflect.Value, 0, len(names))
	for _, name := range names {
		fi, ok := tm.Names[name]
		if !ok {
			vals = append(vals, *new(reflect.Value))
		} else {
			vals = append(vals, FieldByIndexes(v, fi.Index))
		}
	}
	return vals
}

// TraversalsByName returns a slice of int slices which represent the struct
// traversals for each mapped name.  Panics if t is not a struct or Indirectable
// to a struct.  Returns empty int slice for each name not found.
func (m *Mapper) TraversalsByName(t reflect.Type, names []string) [][]int {
	t = Deref(t)
	mustBe(t, reflect.Struct)
	tm := m.TypeMap(t)

	r := make([][]int, 0, len(names))
	for _, name := range names {
		fi, ok := tm.Names[name]
		if !ok {
			r = append(r, []int{})
		} else {
			r = append(r, fi.Index)
		}
	}
	return r
}

// FieldByIndexes returns a value for a particular struct traversal.
func FieldByIndexes(v reflect.Value, indexes []int) reflect.Value {
	for _, i := range indexes {
		v = reflect.Indirect(v).Field(i)
		// if this is a pointer, it's possible it is nil
		if v.Kind() == reflect.Ptr && v.IsNil() {
			alloc := reflect.New(Deref(v.Type()))
			v.Set(alloc)
		}
		if v.Kind() == reflect.Map && v.IsNil() {
			v.Set(reflect.MakeMap(v.Type()))
		}
	}
	return v
}

// FieldByIndexesReadOnly returns a value for a particular struct traversal,
// but is not concerned with allocating nil pointers because the value is
// going to be used for reading and not setting.
func FieldByIndexesReadOnly(v reflect.Value, indexes []int) reflect.Value {
	for _, i := range indexes {
		v = reflect.Indirect(v).Field(i)
	}
	return v
}

// Deref is Indirect for reflect.Types
func Deref(t reflect.Type) reflect.Type {
	if t.Kind() == reflect.Ptr {
		t = t.Elem()
	}
	return t
}

// -- helpers & utilities --

type kinder interface {
	Kind() reflect.Kind
}

// mustBe checks a value against a kind, panicing with a reflect.ValueError
// if the kind isn't that which is required.
func mustBe(v kinder, expected reflect.Kind) {
	k := v.Kind()
	if k != expected {
		panic(&reflect.ValueError{Method: methodName(), Kind: k})
	}
}

// methodName is returns the caller of the function calling methodName
func methodName() string {
	pc, _, _, _ := runtime.Caller(2)
	f := runtime.FuncForPC(pc)
	if f == nil {
		return "unknown method"
	}
	return f.Name()
}

type typeQueue struct {
	t  reflect.Type
	fi *FieldInfo
	pp string // Parent path
}

// A copying append that creates a new slice each time.
func apnd(is []int, i int) []int {
	x := make([]int, len(is)+1)
	for p, n := range is {
		x[p] = n
	}
	x[len(x)-1] = i
	return x
}

// getMapping returns a mapping for the t type, using the tagName, mapFunc and
// tagMapFunc to determine the canonical names of fields.
func getMapping(t reflect.Type, tagName string, mapFunc, tagMapFunc func(string) string) *StructMap {
	m := []*FieldInfo{}

	queue := []typeQueue{}
	queue = append(queue, typeQueue{Deref(t), &FieldInfo{}, ""})

	for len(queue) != 0 {
		// pop the first item off of the queue
		tq := queue[0]
		queue = queue[1:]

		// iterate through all of its fields
		for fieldPos := 0; fieldPos < tq.t.NumField(); fieldPos++ {
			f := tq.t.Field(fieldPos)

			fi := &FieldInfo{}
			fi.Field = f
			fi.Zero = reflect.New(f.Type).Elem()
			fi.Options = map[string]string{}

			var tag, name string
			if tagName != "" && strings.Contains(string(f.Tag), tagName+":") {
				tag = f.Tag.Get(tagName)
				name = tag
			} else {
				if mapFunc != nil {
					name = mapFunc(f.Name)
				}
			}

			parts := strings.Split(name, ",")
			if len(parts) > 1 {
				name = parts[0]
				for _, opt := range parts[1:] {
					kv := strings.Split(opt, "=")
					if len(kv) > 1 {
						fi.Options[kv[0]] = kv[1]
					} else {
						fi.Options[kv[0]] = ""
					}
				}
			}

			if tagMapFunc != nil {
				tag = tagMapFunc(tag)
			}

			fi.Name = name

			if tq.pp == "" || (tq.pp == "" && tag == "") {
				fi.Path = fi.Name
			} else {
				fi.Path = fmt.Sprintf("%s.%s", tq.pp, fi.Name)
			}

			// if the name is "-", disabled via a tag, skip it
			if name == "-" {
				continue
			}

			// skip unexported fields
			if len(f.PkgPath) != 0 {
				continue
			}

			// bfs search of anonymous embedded structs
			if f.Anonymous {
				pp := tq.pp
				if tag != "" {
					pp = fi.Path
				}

				fiq := *fi
				fiq.Index = apnd(tq.fi.Index, fieldPos)
				queue = append(queue, typeQueue{Deref(f.Type), &fiq, pp})
				fi.Embedded = true
			} else if fi.Zero.Kind() == reflect.Struct {
				fiq := *fi
				fiq.Index = apnd(tq.fi.Index, fieldPos)
				queue = append(queue, typeQueue{Deref(f.Type), &fiq, fiq.Path})
			}

			fiq := *fi
			fiq.Index = apnd(tq.fi.Index, fieldPos)
			m = append(m, &fiq)
		}
	}

	flds := &StructMap{Index: m, Paths: map[string]*FieldInfo{}, Names: map[string]*FieldInfo{}}
	for _, fi := range flds.Index {
		flds.Paths[fi.Path] = fi
		if fi.Name != "" && !fi.Embedded {
			flds.Names[fi.Path] = fi
		}
	}

	return flds
}
<commit_msg>implement a btree of fieldinfos representing the "hierarchy" of embedded structs<commit_after>// Package reflectx implements extensions to the standard reflect lib suitable
// for implementing marshaling and unmarshaling packages.  The main Mapper type
// allows for Go-compatible named atribute access, including accessing embedded
// struct attributes and the ability to use  functions and struct tags to
// customize field names.
//
package reflectx

import (
	"fmt"
	"reflect"
	"runtime"
	"strings"
	"sync"
)

// A FieldInfo is a collection of metadata about a struct field.
type FieldInfo struct {
	Index    []int
	Path     string
	Field    reflect.StructField
	Zero     reflect.Value
	Name     string
	Options  map[string]string
	Embedded bool
	Children []*FieldInfo
	Parent   *FieldInfo
}

// A StructMap is an index of field metadata for a struct.
type StructMap struct {
	Tree  *FieldInfo
	Index []*FieldInfo
	Paths map[string]*FieldInfo
	Names map[string]*FieldInfo
}

// GetByPath returns a *FieldInfo for a given string path.
func (f StructMap) GetByPath(path string) *FieldInfo {
	return f.Paths[path]
}

// GetByTraversal returns a *FieldInfo for a given integer path.  It is
// analagous to reflect.FieldByIndex.
func (f StructMap) GetByTraversal(index []int) *FieldInfo {
	// XXX: this function has a lot worse complexity than the one it replaces.
	// It should work exactly as FieldByIndexes does below, but instead of
	// actually traversing a tree it linearly checks a big list of all possible
	// traversals, which doesn't feel right.  Ideally, StructMap.Index could
	// be changed to be a Tree, but I'm not sure that's possible since it's
	// exposed.
	if len(index) == 0 {
		return nil
	}

	tree := f.Tree
	for _, i := range index {
		if i >= len(tree.Children) || tree.Children[i] == nil {
			return nil
		}
		tree = tree.Children[i]
	}
	return tree
}

// Mapper is a general purpose mapper of names to struct fields.  A Mapper
// behaves like most marshallers, optionally obeying a field tag for name
// mapping and a function to provide a basic mapping of fields to names.
type Mapper struct {
	cache      map[reflect.Type]*StructMap
	tagName    string
	tagMapFunc func(string) string
	mapFunc    func(string) string
	mutex      sync.Mutex
}

// NewMapper returns a new mapper which optionally obeys the field tag given
// by tagName.  If tagName is the empty string, it is ignored.
func NewMapper(tagName string) *Mapper {
	return &Mapper{
		cache:   make(map[reflect.Type]*StructMap),
		tagName: tagName,
	}
}

// NewMapperTagFunc returns a new mapper which contains a mapper for field names
// AND a mapper for tag values.  This is useful for tags like json which can
// have values like "name,omitempty".
func NewMapperTagFunc(tagName string, mapFunc, tagMapFunc func(string) string) *Mapper {
	return &Mapper{
		cache:      make(map[reflect.Type]*StructMap),
		tagName:    tagName,
		mapFunc:    mapFunc,
		tagMapFunc: tagMapFunc,
	}
}

// NewMapperFunc returns a new mapper which optionally obeys a field tag and
// a struct field name mapper func given by f.  Tags will take precedence, but
// for any other field, the mapped name will be f(field.Name)
func NewMapperFunc(tagName string, f func(string) string) *Mapper {
	return &Mapper{
		cache:   make(map[reflect.Type]*StructMap),
		tagName: tagName,
		mapFunc: f,
	}
}

// TypeMap returns a mapping of field strings to int slices representing
// the traversal down the struct to reach the field.
func (m *Mapper) TypeMap(t reflect.Type) *StructMap {
	m.mutex.Lock()
	mapping, ok := m.cache[t]
	if !ok {
		mapping = getMapping(t, m.tagName, m.mapFunc, m.tagMapFunc)
		m.cache[t] = mapping
	}
	m.mutex.Unlock()
	return mapping
}

// FieldMap returns the mapper's mapping of field names to reflect values.  Panics
// if v's Kind is not Struct, or v is not Indirectable to a struct kind.
func (m *Mapper) FieldMap(v reflect.Value) map[string]reflect.Value {
	v = reflect.Indirect(v)
	mustBe(v, reflect.Struct)

	r := map[string]reflect.Value{}
	tm := m.TypeMap(v.Type())
	for tagName, fi := range tm.Names {
		r[tagName] = FieldByIndexes(v, fi.Index)
	}
	return r
}

// FieldByName returns a field by the its mapped name as a reflect.Value.
// Panics if v's Kind is not Struct or v is not Indirectable to a struct Kind.
// Returns zero Value if the name is not found.
func (m *Mapper) FieldByName(v reflect.Value, name string) reflect.Value {
	v = reflect.Indirect(v)
	mustBe(v, reflect.Struct)

	tm := m.TypeMap(v.Type())
	fi, ok := tm.Names[name]
	if !ok {
		return v
	}
	return FieldByIndexes(v, fi.Index)
}

// FieldsByName returns a slice of values corresponding to the slice of names
// for the value.  Panics if v's Kind is not Struct or v is not Indirectable
// to a struct Kind.  Returns zero Value for each name not found.
func (m *Mapper) FieldsByName(v reflect.Value, names []string) []reflect.Value {
	v = reflect.Indirect(v)
	mustBe(v, reflect.Struct)

	tm := m.TypeMap(v.Type())
	vals := make([]reflect.Value, 0, len(names))
	for _, name := range names {
		fi, ok := tm.Names[name]
		if !ok {
			vals = append(vals, *new(reflect.Value))
		} else {
			vals = append(vals, FieldByIndexes(v, fi.Index))
		}
	}
	return vals
}

// TraversalsByName returns a slice of int slices which represent the struct
// traversals for each mapped name.  Panics if t is not a struct or Indirectable
// to a struct.  Returns empty int slice for each name not found.
func (m *Mapper) TraversalsByName(t reflect.Type, names []string) [][]int {
	t = Deref(t)
	mustBe(t, reflect.Struct)
	tm := m.TypeMap(t)

	r := make([][]int, 0, len(names))
	for _, name := range names {
		fi, ok := tm.Names[name]
		if !ok {
			r = append(r, []int{})
		} else {
			r = append(r, fi.Index)
		}
	}
	return r
}

// FieldByIndexes returns a value for a particular struct traversal.
func FieldByIndexes(v reflect.Value, indexes []int) reflect.Value {
	for _, i := range indexes {
		v = reflect.Indirect(v).Field(i)
		// if this is a pointer, it's possible it is nil
		if v.Kind() == reflect.Ptr && v.IsNil() {
			alloc := reflect.New(Deref(v.Type()))
			v.Set(alloc)
		}
		if v.Kind() == reflect.Map && v.IsNil() {
			v.Set(reflect.MakeMap(v.Type()))
		}
	}
	return v
}

// FieldByIndexesReadOnly returns a value for a particular struct traversal,
// but is not concerned with allocating nil pointers because the value is
// going to be used for reading and not setting.
func FieldByIndexesReadOnly(v reflect.Value, indexes []int) reflect.Value {
	for _, i := range indexes {
		v = reflect.Indirect(v).Field(i)
	}
	return v
}

// Deref is Indirect for reflect.Types
func Deref(t reflect.Type) reflect.Type {
	if t.Kind() == reflect.Ptr {
		t = t.Elem()
	}
	return t
}

// -- helpers & utilities --

type kinder interface {
	Kind() reflect.Kind
}

// mustBe checks a value against a kind, panicing with a reflect.ValueError
// if the kind isn't that which is required.
func mustBe(v kinder, expected reflect.Kind) {
	k := v.Kind()
	if k != expected {
		panic(&reflect.ValueError{Method: methodName(), Kind: k})
	}
}

// methodName is returns the caller of the function calling methodName
func methodName() string {
	pc, _, _, _ := runtime.Caller(2)
	f := runtime.FuncForPC(pc)
	if f == nil {
		return "unknown method"
	}
	return f.Name()
}

type typeQueue struct {
	t  reflect.Type
	fi *FieldInfo
	pp string // Parent path
}

// A copying append that creates a new slice each time.
func apnd(is []int, i int) []int {
	x := make([]int, len(is)+1)
	for p, n := range is {
		x[p] = n
	}
	x[len(x)-1] = i
	return x
}

// getMapping returns a mapping for the t type, using the tagName, mapFunc and
// tagMapFunc to determine the canonical names of fields.
func getMapping(t reflect.Type, tagName string, mapFunc, tagMapFunc func(string) string) *StructMap {
	m := []*FieldInfo{}

	root := &FieldInfo{}
	queue := []typeQueue{}
	queue = append(queue, typeQueue{Deref(t), root, ""})

	for len(queue) != 0 {
		// pop the first item off of the queue
		tq := queue[0]
		queue = queue[1:]
		tq.fi.Children = make([]*FieldInfo, tq.t.NumField())

		// iterate through all of its fields
		for fieldPos := 0; fieldPos < tq.t.NumField(); fieldPos++ {
			f := tq.t.Field(fieldPos)

			fi := FieldInfo{}
			fi.Field = f
			fi.Zero = reflect.New(f.Type).Elem()
			fi.Options = map[string]string{}

			var tag, name string
			if tagName != "" && strings.Contains(string(f.Tag), tagName+":") {
				tag = f.Tag.Get(tagName)
				name = tag
			} else {
				if mapFunc != nil {
					name = mapFunc(f.Name)
				}
			}

			parts := strings.Split(name, ",")
			if len(parts) > 1 {
				name = parts[0]
				for _, opt := range parts[1:] {
					kv := strings.Split(opt, "=")
					if len(kv) > 1 {
						fi.Options[kv[0]] = kv[1]
					} else {
						fi.Options[kv[0]] = ""
					}
				}
			}

			if tagMapFunc != nil {
				tag = tagMapFunc(tag)
			}

			fi.Name = name

			if tq.pp == "" || (tq.pp == "" && tag == "") {
				fi.Path = fi.Name
			} else {
				fi.Path = fmt.Sprintf("%s.%s", tq.pp, fi.Name)
			}

			// if the name is "-", disabled via a tag, skip it
			if name == "-" {
				continue
			}

			// skip unexported fields
			if len(f.PkgPath) != 0 {
				continue
			}

			// bfs search of anonymous embedded structs
			if f.Anonymous {
				pp := tq.pp
				if tag != "" {
					pp = fi.Path
				}

				fi.Embedded = true
				fi.Index = apnd(tq.fi.Index, fieldPos)
				fi.Children = make([]*FieldInfo, Deref(f.Type).NumField())
				queue = append(queue, typeQueue{Deref(f.Type), &fi, pp})
			} else if fi.Zero.Kind() == reflect.Struct {
				fi.Index = apnd(tq.fi.Index, fieldPos)
				fi.Children = make([]*FieldInfo, Deref(f.Type).NumField())
				queue = append(queue, typeQueue{Deref(f.Type), &fi, fi.Path})
			}

			fi.Index = apnd(tq.fi.Index, fieldPos)
			fi.Parent = tq.fi
			tq.fi.Children[fieldPos] = &fi
			m = append(m, &fi)
		}
	}

	flds := &StructMap{Index: m, Tree: root, Paths: map[string]*FieldInfo{}, Names: map[string]*FieldInfo{}}
	for _, fi := range flds.Index {
		flds.Paths[fi.Path] = fi
		if fi.Name != "" && !fi.Embedded {
			flds.Names[fi.Path] = fi
		}
	}

	return flds
}
<|endoftext|>


<commit_before>package core

import (
	"fmt"
	"strings"
	"time"

	"github.com/af83/edwig/audit"
	"github.com/af83/edwig/logger"
	"github.com/af83/edwig/model"
	"github.com/af83/edwig/siri"
)

type SIRIGeneralMessageSubscriber interface {
	model.Stopable
	model.Startable
}

type GMSubscriber struct {
	model.ClockConsumer

	connector *SIRIGeneralMessageSubscriptionCollector
}

type GeneralMessageSubscriber struct {
	GMSubscriber

	stop chan struct{}
}

type FakeGeneralMessageSubscriber struct {
	GMSubscriber
}

func NewFakeGeneralMessageSubscriber(connector *SIRIGeneralMessageSubscriptionCollector) SIRIGeneralMessageSubscriber {
	subscriber := &FakeGeneralMessageSubscriber{}
	subscriber.connector = connector
	return subscriber
}

func (subscriber *FakeGeneralMessageSubscriber) Start() {
	subscriber.prepareSIRIGeneralMessageSubscriptionRequest()
}

func (subscriber *FakeGeneralMessageSubscriber) Stop() {}

func NewSIRIGeneralMessageSubscriber(connector *SIRIGeneralMessageSubscriptionCollector) SIRIGeneralMessageSubscriber {
	subscriber := &GeneralMessageSubscriber{}
	subscriber.connector = connector
	return subscriber
}

func (subscriber *GeneralMessageSubscriber) Start() {
	logger.Log.Debugf("Start GeneralMessageSubscriber")

	subscriber.stop = make(chan struct{})
	go subscriber.run()
}

func (subscriber *GeneralMessageSubscriber) run() {
	c := subscriber.Clock().After(5 * time.Second)

	for {
		select {
		case <-subscriber.stop:
			return
		case <-c:
			logger.Log.Debugf("SIRIGeneralMessageSubscriber visit")

			subscriber.prepareSIRIGeneralMessageSubscriptionRequest()

			c = subscriber.Clock().After(5 * time.Second)
		}
	}
}

func (subscriber *GeneralMessageSubscriber) Stop() {
	if subscriber.stop != nil {
		close(subscriber.stop)
	}
}

func (subscriber *GMSubscriber) prepareSIRIGeneralMessageSubscriptionRequest() {
	subscription, ok := subscriber.connector.partner.Subscriptions().FindByKind("GeneralMessageCollect")
	if !ok {
		logger.Log.Debugf("GeneralMessageSubscriber visit without a subscription")
		return
	}
	lineRefList := []string{}

	linesToRequest := make(map[string]*model.ObjectID)
	for _, resource := range subscription.ResourcesByObjectID() {
		if resource.SubscribedAt.IsZero() {
			messageIdentifier := subscriber.connector.SIRIPartner().IdentifierGenerator("message_identifier").NewMessageIdentifier()
			linesToRequest[messageIdentifier] = resource.Reference.ObjectId
		}
	}

	if len(linesToRequest) == 0 {
		return
	}

	logStashEvent := subscriber.newLogStashEvent()
	defer audit.CurrentLogStash().WriteEvent(logStashEvent)

	gmRequest := &siri.SIRIGeneralMessageSubscriptionRequest{
		ConsumerAddress:   subscriber.connector.Partner().Address(),
		MessageIdentifier: subscriber.connector.SIRIPartner().IdentifierGenerator("message_identifier").NewMessageIdentifier(),
		RequestorRef:      subscriber.connector.SIRIPartner().RequestorRef(),
		RequestTimestamp:  subscriber.Clock().Now(),
	}

	for messageIdentifier, lineObjectid := range linesToRequest {
		entry := &siri.SIRIGeneralMessageSubscriptionRequestEntry{
			SubscriberRef:          subscriber.connector.SIRIPartner().RequestorRef(),
			SubscriptionIdentifier: string(subscription.Id()),
			InitialTerminationTime: subscriber.Clock().Now().Add(48 * time.Hour),
		}
		entry.MessageIdentifier = messageIdentifier
		entry.RequestTimestamp = subscriber.Clock().Now()
		entry.LineRef = []string{lineObjectid.Value()}

		lineRefList = append(lineRefList, lineObjectid.Value())
		gmRequest.Entries = append(gmRequest.Entries, entry)
	}

	logStashEvent["lineRefs"] = strings.Join(lineRefList, ", ")
	logSIRIGeneralMessageSubscriptionRequest(logStashEvent, gmRequest)

	response, err := subscriber.connector.SIRIPartner().SOAPClient().GeneralMessageSubscription(gmRequest)
	if err != nil {
		logger.Log.Debugf("Error while subscribing: %v", err)
		logStashEvent["status"] = "false"
		logStashEvent["response"] = fmt.Sprintf("Error during GeneralMessageSubscriptionRequest: %v", err)
		for _, lineObjectid := range linesToRequest {
			resource := subscription.Resource(*lineObjectid)
			resource.RetryCount++
		}
		return
	}

	logStashEvent["response"] = response.RawXML()

	for _, responseStatus := range response.ResponseStatus() {
		lineObjectid, ok := linesToRequest[responseStatus.RequestMessageRef()]
		if !ok {
			logger.Log.Debugf("ResponseStatus RequestMessageRef unknown: %v", responseStatus.RequestMessageRef())
			continue
		}
		delete(linesToRequest, responseStatus.RequestMessageRef()) // See #4691
		resource := subscription.Resource(*lineObjectid)
		if resource == nil { // Should never happen
			logger.Log.Debugf("Response for unknown subscription resource %v", lineObjectid.String())
			continue
		}
		if !responseStatus.Status() {
			logger.Log.Debugf("Subscription status false for line %v: %v %v ", lineObjectid.Value(), responseStatus.ErrorType(), responseStatus.ErrorText())
			resource.RetryCount++
			continue
		}
		resource.SubscribedAt = subscriber.Clock().Now()
		resource.RetryCount = 0
	}
	// Should not happen but see #4691
	if len(linesToRequest) == 0 {
		return
	}
	for _, notInResponseObjectid := range linesToRequest {
		resource := subscription.Resource(*notInResponseObjectid)
		if resource == nil { // Should never happen
			logger.Log.Debugf("Can't increment RetryCount for unknown subscription resource %v", notInResponseObjectid.String())
			continue
		}
		resource.RetryCount++
	}
}

func (smb *GMSubscriber) newLogStashEvent() audit.LogStashEvent {
	event := smb.connector.partner.NewLogStashEvent()
	event["connector"] = "GeneralMessageSubscriptionCollector"
	return event
}

func logSIRIGeneralMessageSubscriptionRequest(logStashEvent audit.LogStashEvent, request *siri.SIRIGeneralMessageSubscriptionRequest) {
	logStashEvent["type"] = "GeneralMessageSubscriptionRequest"
	logStashEvent["consumerAddress"] = request.ConsumerAddress
	logStashEvent["messageIdentifier"] = request.MessageIdentifier
	logStashEvent["requestorRef"] = request.RequestorRef
	logStashEvent["requestTimestamp"] = request.RequestTimestamp.String()
	xml, err := request.BuildXML()
	if err != nil {
		logStashEvent["requestXML"] = fmt.Sprintf("%v", err)
		return
	}
	logStashEvent["requestXML"] = xml
}
<commit_msg>Check retry count in GM subscriber<commit_after>package core

import (
	"fmt"
	"strings"
	"time"

	"github.com/af83/edwig/audit"
	"github.com/af83/edwig/logger"
	"github.com/af83/edwig/model"
	"github.com/af83/edwig/siri"
)

type SIRIGeneralMessageSubscriber interface {
	model.Stopable
	model.Startable
}

type GMSubscriber struct {
	model.ClockConsumer

	connector *SIRIGeneralMessageSubscriptionCollector
}

type GeneralMessageSubscriber struct {
	GMSubscriber

	stop chan struct{}
}

type FakeGeneralMessageSubscriber struct {
	GMSubscriber
}

func NewFakeGeneralMessageSubscriber(connector *SIRIGeneralMessageSubscriptionCollector) SIRIGeneralMessageSubscriber {
	subscriber := &FakeGeneralMessageSubscriber{}
	subscriber.connector = connector
	return subscriber
}

func (subscriber *FakeGeneralMessageSubscriber) Start() {
	subscriber.prepareSIRIGeneralMessageSubscriptionRequest()
}

func (subscriber *FakeGeneralMessageSubscriber) Stop() {}

func NewSIRIGeneralMessageSubscriber(connector *SIRIGeneralMessageSubscriptionCollector) SIRIGeneralMessageSubscriber {
	subscriber := &GeneralMessageSubscriber{}
	subscriber.connector = connector
	return subscriber
}

func (subscriber *GeneralMessageSubscriber) Start() {
	logger.Log.Debugf("Start GeneralMessageSubscriber")

	subscriber.stop = make(chan struct{})
	go subscriber.run()
}

func (subscriber *GeneralMessageSubscriber) run() {
	c := subscriber.Clock().After(5 * time.Second)

	for {
		select {
		case <-subscriber.stop:
			return
		case <-c:
			logger.Log.Debugf("SIRIGeneralMessageSubscriber visit")

			subscriber.prepareSIRIGeneralMessageSubscriptionRequest()

			c = subscriber.Clock().After(5 * time.Second)
		}
	}
}

func (subscriber *GeneralMessageSubscriber) Stop() {
	if subscriber.stop != nil {
		close(subscriber.stop)
	}
}

func (subscriber *GMSubscriber) prepareSIRIGeneralMessageSubscriptionRequest() {
	subscription, ok := subscriber.connector.partner.Subscriptions().FindByKind("GeneralMessageCollect")
	if !ok {
		logger.Log.Debugf("GeneralMessageSubscriber visit without a subscription")
		return
	}
	lineRefList := []string{}

	linesToRequest := make(map[string]*model.ObjectID)
	for _, resource := range subscription.ResourcesByObjectID() {
		if resource.SubscribedAt.IsZero() && resource.RetryCount <= 10 {
			messageIdentifier := subscriber.connector.SIRIPartner().IdentifierGenerator("message_identifier").NewMessageIdentifier()
			linesToRequest[messageIdentifier] = resource.Reference.ObjectId
		}
	}

	if len(linesToRequest) == 0 {
		return
	}

	logStashEvent := subscriber.newLogStashEvent()
	defer audit.CurrentLogStash().WriteEvent(logStashEvent)

	gmRequest := &siri.SIRIGeneralMessageSubscriptionRequest{
		ConsumerAddress:   subscriber.connector.Partner().Address(),
		MessageIdentifier: subscriber.connector.SIRIPartner().IdentifierGenerator("message_identifier").NewMessageIdentifier(),
		RequestorRef:      subscriber.connector.SIRIPartner().RequestorRef(),
		RequestTimestamp:  subscriber.Clock().Now(),
	}

	for messageIdentifier, lineObjectid := range linesToRequest {
		entry := &siri.SIRIGeneralMessageSubscriptionRequestEntry{
			SubscriberRef:          subscriber.connector.SIRIPartner().RequestorRef(),
			SubscriptionIdentifier: string(subscription.Id()),
			InitialTerminationTime: subscriber.Clock().Now().Add(48 * time.Hour),
		}
		entry.MessageIdentifier = messageIdentifier
		entry.RequestTimestamp = subscriber.Clock().Now()
		entry.LineRef = []string{lineObjectid.Value()}

		lineRefList = append(lineRefList, lineObjectid.Value())
		gmRequest.Entries = append(gmRequest.Entries, entry)
	}

	logStashEvent["lineRefs"] = strings.Join(lineRefList, ", ")
	logSIRIGeneralMessageSubscriptionRequest(logStashEvent, gmRequest)

	response, err := subscriber.connector.SIRIPartner().SOAPClient().GeneralMessageSubscription(gmRequest)
	if err != nil {
		logger.Log.Debugf("Error while subscribing: %v", err)
		logStashEvent["status"] = "false"
		logStashEvent["response"] = fmt.Sprintf("Error during GeneralMessageSubscriptionRequest: %v", err)
		for _, lineObjectid := range linesToRequest {
			resource := subscription.Resource(*lineObjectid)
			resource.RetryCount++
		}
		return
	}

	logStashEvent["response"] = response.RawXML()

	for _, responseStatus := range response.ResponseStatus() {
		lineObjectid, ok := linesToRequest[responseStatus.RequestMessageRef()]
		if !ok {
			logger.Log.Debugf("ResponseStatus RequestMessageRef unknown: %v", responseStatus.RequestMessageRef())
			continue
		}
		delete(linesToRequest, responseStatus.RequestMessageRef()) // See #4691
		resource := subscription.Resource(*lineObjectid)
		if resource == nil { // Should never happen
			logger.Log.Debugf("Response for unknown subscription resource %v", lineObjectid.String())
			continue
		}
		if !responseStatus.Status() {
			logger.Log.Debugf("Subscription status false for line %v: %v %v ", lineObjectid.Value(), responseStatus.ErrorType(), responseStatus.ErrorText())
			resource.RetryCount++
			continue
		}
		resource.SubscribedAt = subscriber.Clock().Now()
		resource.RetryCount = 0
	}
	// Should not happen but see #4691
	if len(linesToRequest) == 0 {
		return
	}
	for _, notInResponseObjectid := range linesToRequest {
		resource := subscription.Resource(*notInResponseObjectid)
		if resource == nil { // Should never happen
			logger.Log.Debugf("Can't increment RetryCount for unknown subscription resource %v", notInResponseObjectid.String())
			continue
		}
		resource.RetryCount++
	}
}

func (smb *GMSubscriber) newLogStashEvent() audit.LogStashEvent {
	event := smb.connector.partner.NewLogStashEvent()
	event["connector"] = "GeneralMessageSubscriptionCollector"
	return event
}

func logSIRIGeneralMessageSubscriptionRequest(logStashEvent audit.LogStashEvent, request *siri.SIRIGeneralMessageSubscriptionRequest) {
	logStashEvent["type"] = "GeneralMessageSubscriptionRequest"
	logStashEvent["consumerAddress"] = request.ConsumerAddress
	logStashEvent["messageIdentifier"] = request.MessageIdentifier
	logStashEvent["requestorRef"] = request.RequestorRef
	logStashEvent["requestTimestamp"] = request.RequestTimestamp.String()
	xml, err := request.BuildXML()
	if err != nil {
		logStashEvent["requestXML"] = fmt.Sprintf("%v", err)
		return
	}
	logStashEvent["requestXML"] = xml
}
<|endoftext|>


<commit_before>/*
Pacakge apachelog provides an Apache log parser.
*/
package apachelog
<commit_msg>apachelog: fix typo in package documentation<commit_after>/*
Package apachelog provides an Apache log parser.
*/
package apachelog
<|endoftext|>


<commit_before>package render

import (
	"image"
	"image/draw"

	"github.com/oakmound/oak/alg/floatgeom"
	"github.com/oakmound/oak/physics"
)

// Composite Types, distinct from Compound Types,
// Display all of their parts at the same time,
// and respect the positions and layers of their
// parts.
type Composite struct {
	LayeredPoint
	rs []Modifiable
}

//NewComposite creates a Composite
func NewComposite(sl []Modifiable) *Composite {
	cs := new(Composite)
	cs.LayeredPoint = NewLayeredPoint(0, 0, 0)
	cs.rs = sl
	return cs
}

//AppendOffset adds a new offset modifiable to the composite
func (cs *Composite) AppendOffset(r Modifiable, v physics.Vector) {
	r.SetPos(v.X(), v.Y())
	cs.rs = append(cs.rs, r)
}

//Append adds a renderable as is to the composite
func (cs *Composite) Append(r Modifiable) {
	cs.rs = append(cs.rs, r)
}

// Prepend adds a new renderable to the front of the CompositeR.
func (cs *Composite) Prepend(r Modifiable) {
	cs.rs = append([]Modifiable{r}, cs.rs...)
}

// Add places a renderable at a certain point in the composites renderable slice
func (cs *Composite) Add(i int, r Modifiable) {
	cs.rs[i] = r
}

//AddOffset offsets all renderables in the composite by a vector
func (cs *Composite) AddOffset(i int, v physics.Vector) {
	if i < len(cs.rs) {
		cs.rs[i].SetPos(v.X(), v.Y())
	}
}

//SetOffsets applies the initial offsets to the entire Composite
func (cs *Composite) SetOffsets(vs []physics.Vector) {
	for i, v := range vs {
		if i < len(cs.rs) {
			cs.rs[i].SetPos(v.X(), v.Y())
		}
	}
}

//Get returns a renderable at the given index within the composite
func (cs *Composite) Get(i int) Modifiable {
	return cs.rs[i]
}

//DrawOffset draws the Composite with some offset from its logical position (and therefore sub renderables logical positions).
func (cs *Composite) DrawOffset(buff draw.Image, xOff, yOff float64) {
	for _, c := range cs.rs {
		c.DrawOffset(buff, cs.X()+xOff, cs.Y()+yOff)
	}
}

//Draw draws the Composite at its logical position
func (cs *Composite) Draw(buff draw.Image) {
	for _, c := range cs.rs {
		c.DrawOffset(buff, cs.X(), cs.Y())
	}
}

//UnDraw stops the composite from being drawn
func (cs *Composite) UnDraw() {
	cs.layer = Undraw
	for _, c := range cs.rs {
		c.UnDraw()
	}
}

//GetRGBA does not work on a composite and therefore returns nil
func (cs *Composite) GetRGBA() *image.RGBA {
	return nil
}

//AlwaysDirty shows that the Composite always needs updating
func (cs *Composite) AlwaysDirty() bool {
	return true
}

//Modify applies modifications to the composite
func (cs *Composite) Modify(ms ...Modification) Modifiable {
	for _, r := range cs.rs {
		r.Modify(ms...)
	}
	return cs
}

//Copy makes a new Composite with the same renderables
func (cs *Composite) Copy() Modifiable {
	cs2 := new(Composite)
	cs2.layer = cs.layer
	cs2.Vector = cs.Vector
	cs2.rs = make([]Modifiable, len(cs.rs))
	for i, v := range cs.rs {
		cs2.rs[i] = v.Copy()
	}
	return cs2
}

func (cs *Composite) String() string {
	s := "Composite{"
	for _, v := range cs.rs {
		s += v.String() + "\n"
	}
	s += "}"
	return s
}

// ToSprite draws the elements of this composite to a new buffer,
// returning a static sprite element
func (cs *Composite) ToSprite() *Sprite {
	w, h := 0.0, 0.0
	for _, v := range cs.rs {
		x := v.GetX()
		y := v.GetY()
		w2, h2 := v.GetDims()
		if float64(w2)+x > w {
			w = float64(w2) + x
		}
		if float64(h2)+y > h {
			h = float64(h2) + y
		}
	}
	rgba := image.NewRGBA(image.Rect(0, 0, int(w), int(h)))
	cs.Draw(rgba)
	return NewSprite(cs.X(), cs.Y(), rgba)
}

//CompositeR keeps track of a set of renderables at a location
type CompositeR struct {
	LayeredPoint
	toPush []Renderable
	rs     []Renderable
}

//NewCompositeR creates a new CompositeR from a slice of renderables
func NewCompositeR(sl []Renderable) *CompositeR {
	cs := new(CompositeR)
	cs.LayeredPoint = NewLayeredPoint(0, 0, 0)
	cs.toPush = make([]Renderable, 0)
	cs.rs = sl
	return cs
}

//AppendOffset adds a new renderable to CompositeR with an offset
func (cs *CompositeR) AppendOffset(r Renderable, v physics.Vector) {
	r.SetPos(v.X(), v.Y())
	cs.rs = append(cs.rs, r)
}

// Append adds a new renderable to the end of the CompositeR.
func (cs *CompositeR) Append(r Renderable) {
	cs.rs = append(cs.rs, r)
}

// Prepend adds a new renderable to the front of the CompositeR.
func (cs *CompositeR) Prepend(r Renderable) {
	cs.rs = append([]Renderable{r}, cs.rs...)
}

// Len returns the number of renderables in this composite.
func (cs *CompositeR) Len() int {
	return len(cs.rs)
}

// Add places a renderable at a certain point in the composites renderable slice
func (cs *CompositeR) Add(i int, r Renderable) {
	cs.rs[i] = r
}

// SetOffsets sets all renderables in CompositeR to the passed in Vector positions positions
func (cs *CompositeR) SetOffsets(ps ...floatgeom.Point2) {
	for i, p := range ps {
		if i < len(cs.rs) {
			cs.rs[i].SetPos(p.X(), p.Y())
		}
	}
}

// DrawOffset Draws the CompositeR with an offset from its logical location.
func (cs *CompositeR) DrawOffset(buff draw.Image, xOff, yOff float64) {
	for _, c := range cs.rs {
		c.DrawOffset(buff, cs.X()+xOff, cs.Y()+yOff)
	}
}

// Draw draws the CompositeR at its logical location and therefore its consituent renderables as well
func (cs *CompositeR) Draw(buff draw.Image) {
	for _, c := range cs.rs {
		c.DrawOffset(buff, cs.X(), cs.Y())
	}
}

// UnDraw undraws the CompositeR and its consituent renderables
func (cs *CompositeR) UnDraw() {
	cs.layer = Undraw
	for _, c := range cs.rs {
		c.UnDraw()
	}
}

// GetRGBA always returns nil from Composites
func (cs *CompositeR) GetRGBA() *image.RGBA {
	return nil
}

// Get returns renderable from a given index in CompositeR
func (cs *CompositeR) Get(i int) Renderable {
	return cs.rs[i]
}

// Add stages a renderable to be added to the Composite at the next PreDraw
func (cs *CompositeR) Add(r Renderable, _ ...int) Renderable {
	cs.toPush = append(cs.toPush, r)
	return r
}

//Replace updates a renderable in the CompositeR to the new Renderable
func (cs *CompositeR) Replace(r1, r2 Renderable, i int) {
	cs.Add(r2, i)
	r1.UnDraw()
}

//AddOffset adds an offset to a given renderable of the slice
func (cs *CompositeR) AddOffset(i int, v physics.Vector) {
	if i < len(cs.rs) {
		cs.rs[i].SetPos(v.X(), v.Y())
	}
}

//PreDraw updates the CompositeR with the new renderables to add. This helps keep consistency and mitigates the threat of unsafe operations.
func (cs *CompositeR) PreDraw() {
	push := cs.toPush
	cs.toPush = []Renderable{}
	cs.rs = append(cs.rs, push...)
}

// Copy returns a new composite with the same length slice of renderables but no actual renderables...
// CompositeRs cannot have their internal elements copied,
// as renderables cannot be copied.
func (cs *CompositeR) Copy() Addable {
	cs2 := new(CompositeR)
	cs2.LayeredPoint = cs.LayeredPoint
	cs2.rs = make([]Renderable, len(cs.rs))
	return cs2
}

//SetOffsets sets all renderables in CompositeR to the passed in Vector positions positions
func (cs *CompositeR) SetOffsets(ps []physics.Vector) {
	for i, p := range ps {
		if i < len(cs.rs) {
			cs.rs[i].SetPos(p.X(), p.Y())
		}
	}
}

//Get returns renderable at given location in CompositeR
func (cs *CompositeR) Get(i int) Renderable {
	return cs.rs[i]
}

//DrawOffset Draws the CompositeR with an offset from its logical location.
func (cs *CompositeR) DrawOffset(buff draw.Image, xOff, yOff float64) {
	for _, c := range cs.rs {
		c.DrawOffset(buff, cs.X()+xOff, cs.Y()+yOff)
	}
}
func (cs *CompositeR) draw(world draw.Image, viewPos image.Point, screenW, screenH int) {
	realLength := len(cs.rs)
	for i := 0; i < realLength; i++ {
		r := cs.rs[i]
		for (r == nil || r.GetLayer() == Undraw) && realLength > i {
			cs.rs[i], cs.rs[realLength-1] = cs.rs[realLength-1], cs.rs[i]
			realLength--
			r = cs.rs[i]
		}
		if realLength == i {
			break
		}
		x := int(r.GetX())
		y := int(r.GetY())
		x2 := x
		y2 := y
		w, h := r.GetDims()
		x += w
		y += h
		if x > viewPos.X && y > viewPos.Y &&
			x2 < viewPos.X+screenW && y2 < viewPos.Y+screenH {

			if InDrawPolygon(x, y, x2, y2) {
				r.DrawOffset(world, float64(-viewPos.X), float64(-viewPos.Y))
			}
		}
	}
	cs.rs = cs.rs[0:realLength]
}

//Draw draws the CompositeR at its logical location and therefore its consituent renderables as well
func (cs *CompositeR) Draw(buff draw.Image) {
	for _, c := range cs.rs {
		c.DrawOffset(buff, cs.X(), cs.Y())
	}
}

//UnDraw undraws the CompositeR and therefore its consituent renderables as well
func (cs *CompositeR) UnDraw() {
	cs.layer = Undraw
	for _, c := range cs.rs {
		c.UnDraw()
	}
}

//GetRGBA does not work on composites and returns nil
func (cs *CompositeR) GetRGBA() *image.RGBA {
	return nil
}

//AlwaysDirty notes that CompositeR is alwaysdirty
func (cs *CompositeR) AlwaysDirty() bool {
	return true
}

func (cs *CompositeR) String() string {
	s := "CompositeR{"
	for _, v := range cs.rs {
		s += v.String() + "\n"
	}
	s += "}"
	return s
}
<commit_msg>Give Composites Len()<commit_after>package render

import (
	"image"
	"image/draw"

	"github.com/oakmound/oak/alg/floatgeom"
	"github.com/oakmound/oak/physics"
)

// Composite Types, distinct from Compound Types,
// Display all of their parts at the same time,
// and respect the positions and layers of their
// parts.
type Composite struct {
	LayeredPoint
	rs []Modifiable
}

//NewComposite creates a Composite
func NewComposite(sl []Modifiable) *Composite {
	cs := new(Composite)
	cs.LayeredPoint = NewLayeredPoint(0, 0, 0)
	cs.rs = sl
	return cs
}

//AppendOffset adds a new offset modifiable to the composite
func (cs *Composite) AppendOffset(r Modifiable, v physics.Vector) {
	r.SetPos(v.X(), v.Y())
	cs.rs = append(cs.rs, r)
}

//Append adds a renderable as is to the composite
func (cs *Composite) Append(r Modifiable) {
	cs.rs = append(cs.rs, r)
}

// Prepend adds a new renderable to the front of the CompositeR.
func (cs *Composite) Prepend(r Modifiable) {
	cs.rs = append([]Modifiable{r}, cs.rs...)
}

// Add places a renderable at a certain point in the composites renderable slice
func (cs *Composite) Add(i int, r Modifiable) {
	cs.rs[i] = r
}

// Len returns the number of renderables in this composite.
func (cs *Composite) Len() int {
	return len(cs.rs)
}

//AddOffset offsets all renderables in the composite by a vector
func (cs *Composite) AddOffset(i int, v physics.Vector) {
	if i < len(cs.rs) {
		cs.rs[i].SetPos(v.X(), v.Y())
	}
}

//SetOffsets applies the initial offsets to the entire Composite
func (cs *Composite) SetOffsets(vs []physics.Vector) {
	for i, v := range vs {
		if i < len(cs.rs) {
			cs.rs[i].SetPos(v.X(), v.Y())
		}
	}
}

//Get returns a renderable at the given index within the composite
func (cs *Composite) Get(i int) Modifiable {
	return cs.rs[i]
}

//DrawOffset draws the Composite with some offset from its logical position (and therefore sub renderables logical positions).
func (cs *Composite) DrawOffset(buff draw.Image, xOff, yOff float64) {
	for _, c := range cs.rs {
		c.DrawOffset(buff, cs.X()+xOff, cs.Y()+yOff)
	}
}

//Draw draws the Composite at its logical position
func (cs *Composite) Draw(buff draw.Image) {
	for _, c := range cs.rs {
		c.DrawOffset(buff, cs.X(), cs.Y())
	}
}

//UnDraw stops the composite from being drawn
func (cs *Composite) UnDraw() {
	cs.layer = Undraw
	for _, c := range cs.rs {
		c.UnDraw()
	}
}

//GetRGBA does not work on a composite and therefore returns nil
func (cs *Composite) GetRGBA() *image.RGBA {
	return nil
}

//AlwaysDirty shows that the Composite always needs updating
func (cs *Composite) AlwaysDirty() bool {
	return true
}

//Modify applies modifications to the composite
func (cs *Composite) Modify(ms ...Modification) Modifiable {
	for _, r := range cs.rs {
		r.Modify(ms...)
	}
	return cs
}

//Copy makes a new Composite with the same renderables
func (cs *Composite) Copy() Modifiable {
	cs2 := new(Composite)
	cs2.layer = cs.layer
	cs2.Vector = cs.Vector
	cs2.rs = make([]Modifiable, len(cs.rs))
	for i, v := range cs.rs {
		cs2.rs[i] = v.Copy()
	}
	return cs2
}

func (cs *Composite) String() string {
	s := "Composite{"
	for _, v := range cs.rs {
		s += v.String() + "\n"
	}
	s += "}"
	return s
}

// ToSprite draws the elements of this composite to a new buffer,
// returning a static sprite element
func (cs *Composite) ToSprite() *Sprite {
	w, h := 0.0, 0.0
	for _, v := range cs.rs {
		x := v.GetX()
		y := v.GetY()
		w2, h2 := v.GetDims()
		if float64(w2)+x > w {
			w = float64(w2) + x
		}
		if float64(h2)+y > h {
			h = float64(h2) + y
		}
	}
	rgba := image.NewRGBA(image.Rect(0, 0, int(w), int(h)))
	cs.Draw(rgba)
	return NewSprite(cs.X(), cs.Y(), rgba)
}

//CompositeR keeps track of a set of renderables at a location
type CompositeR struct {
	LayeredPoint
	toPush []Renderable
	rs     []Renderable
}

//NewCompositeR creates a new CompositeR from a slice of renderables
func NewCompositeR(sl []Renderable) *CompositeR {
	cs := new(CompositeR)
	cs.LayeredPoint = NewLayeredPoint(0, 0, 0)
	cs.toPush = make([]Renderable, 0)
	cs.rs = sl
	return cs
}

//AppendOffset adds a new renderable to CompositeR with an offset
func (cs *CompositeR) AppendOffset(r Renderable, v physics.Vector) {
	r.SetPos(v.X(), v.Y())
	cs.rs = append(cs.rs, r)
}

// Append adds a new renderable to the end of the CompositeR.
func (cs *CompositeR) Append(r Renderable) {
	cs.rs = append(cs.rs, r)
}

// Prepend adds a new renderable to the front of the CompositeR.
func (cs *CompositeR) Prepend(r Renderable) {
	cs.rs = append([]Renderable{r}, cs.rs...)
}

// Len returns the number of renderables in this composite.
func (cs *CompositeR) Len() int {
	return len(cs.rs)
}

// Add places a renderable at a certain point in the composites renderable slice
func (cs *CompositeR) Add(i int, r Renderable) {
	cs.rs[i] = r
}

// SetOffsets sets all renderables in CompositeR to the passed in Vector positions positions
func (cs *CompositeR) SetOffsets(ps ...floatgeom.Point2) {
	for i, p := range ps {
		if i < len(cs.rs) {
			cs.rs[i].SetPos(p.X(), p.Y())
		}
	}
}

// DrawOffset Draws the CompositeR with an offset from its logical location.
func (cs *CompositeR) DrawOffset(buff draw.Image, xOff, yOff float64) {
	for _, c := range cs.rs {
		c.DrawOffset(buff, cs.X()+xOff, cs.Y()+yOff)
	}
}

// Draw draws the CompositeR at its logical location and therefore its consituent renderables as well
func (cs *CompositeR) Draw(buff draw.Image) {
	for _, c := range cs.rs {
		c.DrawOffset(buff, cs.X(), cs.Y())
	}
}

// UnDraw undraws the CompositeR and its consituent renderables
func (cs *CompositeR) UnDraw() {
	cs.layer = Undraw
	for _, c := range cs.rs {
		c.UnDraw()
	}
}

// GetRGBA always returns nil from Composites
func (cs *CompositeR) GetRGBA() *image.RGBA {
	return nil
}

// Get returns renderable from a given index in CompositeR
func (cs *CompositeR) Get(i int) Renderable {
	return cs.rs[i]
}

// Add stages a renderable to be added to the Composite at the next PreDraw
func (cs *CompositeR) Add(r Renderable, _ ...int) Renderable {
	cs.toPush = append(cs.toPush, r)
	return r
}

//Replace updates a renderable in the CompositeR to the new Renderable
func (cs *CompositeR) Replace(r1, r2 Renderable, i int) {
	cs.Add(r2, i)
	r1.UnDraw()
}

//AddOffset adds an offset to a given renderable of the slice
func (cs *CompositeR) AddOffset(i int, v physics.Vector) {
	if i < len(cs.rs) {
		cs.rs[i].SetPos(v.X(), v.Y())
	}
}

//PreDraw updates the CompositeR with the new renderables to add. This helps keep consistency and mitigates the threat of unsafe operations.
func (cs *CompositeR) PreDraw() {
	push := cs.toPush
	cs.toPush = []Renderable{}
	cs.rs = append(cs.rs, push...)
}

// Copy returns a new composite with the same length slice of renderables but no actual renderables...
// CompositeRs cannot have their internal elements copied,
// as renderables cannot be copied.
func (cs *CompositeR) Copy() Addable {
	cs2 := new(CompositeR)
	cs2.LayeredPoint = cs.LayeredPoint
	cs2.rs = make([]Renderable, len(cs.rs))
	return cs2
}

//SetOffsets sets all renderables in CompositeR to the passed in Vector positions positions
func (cs *CompositeR) SetOffsets(ps []physics.Vector) {
	for i, p := range ps {
		if i < len(cs.rs) {
			cs.rs[i].SetPos(p.X(), p.Y())
		}
	}
}

//Get returns renderable at given location in CompositeR
func (cs *CompositeR) Get(i int) Renderable {
	return cs.rs[i]
}

//DrawOffset Draws the CompositeR with an offset from its logical location.
func (cs *CompositeR) DrawOffset(buff draw.Image, xOff, yOff float64) {
	for _, c := range cs.rs {
		c.DrawOffset(buff, cs.X()+xOff, cs.Y()+yOff)
	}
}
func (cs *CompositeR) draw(world draw.Image, viewPos image.Point, screenW, screenH int) {
	realLength := len(cs.rs)
	for i := 0; i < realLength; i++ {
		r := cs.rs[i]
		for (r == nil || r.GetLayer() == Undraw) && realLength > i {
			cs.rs[i], cs.rs[realLength-1] = cs.rs[realLength-1], cs.rs[i]
			realLength--
			r = cs.rs[i]
		}
		if realLength == i {
			break
		}
		x := int(r.GetX())
		y := int(r.GetY())
		x2 := x
		y2 := y
		w, h := r.GetDims()
		x += w
		y += h
		if x > viewPos.X && y > viewPos.Y &&
			x2 < viewPos.X+screenW && y2 < viewPos.Y+screenH {

			if InDrawPolygon(x, y, x2, y2) {
				r.DrawOffset(world, float64(-viewPos.X), float64(-viewPos.Y))
			}
		}
	}
	cs.rs = cs.rs[0:realLength]
}

//Draw draws the CompositeR at its logical location and therefore its consituent renderables as well
func (cs *CompositeR) Draw(buff draw.Image) {
	for _, c := range cs.rs {
		c.DrawOffset(buff, cs.X(), cs.Y())
	}
}

//UnDraw undraws the CompositeR and therefore its consituent renderables as well
func (cs *CompositeR) UnDraw() {
	cs.layer = Undraw
	for _, c := range cs.rs {
		c.UnDraw()
	}
}

//GetRGBA does not work on composites and returns nil
func (cs *CompositeR) GetRGBA() *image.RGBA {
	return nil
}

//AlwaysDirty notes that CompositeR is alwaysdirty
func (cs *CompositeR) AlwaysDirty() bool {
	return true
}

func (cs *CompositeR) String() string {
	s := "CompositeR{"
	for _, v := range cs.rs {
		s += v.String() + "\n"
	}
	s += "}"
	return s
}
<|endoftext|>


<commit_before>package consumergroup

import (
	"errors"
	"fmt"
	"math"
	"sort"
	"sync"
	"time"

	"github.com/Shopify/sarama"
	"github.com/samuel/go-zookeeper/zk"
)

var (
	DiscardCommit = errors.New("sarama: commit discarded")
	NoCheckout    = errors.New("sarama: not checkout")
)

const (
	REBALANCE_START uint8 = iota + 1
	REBALANCE_OK
	REBALANCE_ERROR
)

type ConsumerGroupConfig struct {
	// The Zookeeper timeout
	ZookeeperTimeout time.Duration

	// The preempt interval when listening to a single partition of a topic.
	// After this interval, the current offset will be committed to Zookeeper,
	// and a different partition will be checked out to consume next.
	CheckoutInterval time.Duration

	KafkaClientConfig   *sarama.ClientConfig   // This will be passed to Sarama when creating a new Client
	KafkaConsumerConfig *sarama.ConsumerConfig // This will be passed to Sarama when creating a new Consumer
}

func NewConsumerGroupConfig() *ConsumerGroupConfig {
	return &ConsumerGroupConfig{
		ZookeeperTimeout:    1 * time.Second,
		CheckoutInterval:    1 * time.Second,
		KafkaClientConfig:   sarama.NewClientConfig(),
		KafkaConsumerConfig: sarama.NewConsumerConfig(),
	}
}

func (cgc *ConsumerGroupConfig) Validate() error {
	if cgc.ZookeeperTimeout <= 0 {
		return errors.New("ZookeeperTimeout should have a duration > 0")
	}

	if cgc.KafkaClientConfig == nil {
		return errors.New("KafkaClientConfig is not set!")
	} else if err := cgc.KafkaClientConfig.Validate(); err != nil {
		return err
	}

	if cgc.KafkaConsumerConfig == nil {
		return errors.New("KafkaConsumerConfig is not set!")
	} else if err := cgc.KafkaConsumerConfig.Validate(); err != nil {
		return err
	}

	return nil
}

// A ConsumerGroup operates on all partitions of a single topic. The goal is to ensure
// each topic message is consumed only once, no matter of the number of consumer instances within
// a cluster, as described in: http://kafka.apache.org/documentation.html#distributionimpl.
//
// The ConsumerGroup internally creates multiple Consumer instances. It uses Zookkeper
// and follows a simple consumer rebalancing algorithm which allows all the consumers
// in a group to come into consensus on which consumer is consuming which partitions. Each
// ConsumerGroup can 'claim' 0-n partitions and will consume their messages until another
// ConsumerGroup instance with the same name joins or leaves the cluster.
//
// Unlike stated in the Kafka documentation, consumer rebalancing is *only* triggered on each
// addition or removal of consumers within the same group, while the addition of broker nodes
// and/or partition *does currently not trigger* a rebalancing cycle.
type ConsumerGroup struct {
	id, name, topic string

	config *ConsumerGroupConfig

	client *sarama.Client
	zoo    *ZK
	claims []PartitionConsumer
	wg     *sync.WaitGroup

	zkchange <-chan zk.Event
	claimed  chan *PartitionConsumer
	listener chan *Notification

	events chan *sarama.ConsumerEvent

	checkout, force, stopper chan bool
}

// Connects to a consumer group, using Zookeeper for auto-discovery
func JoinConsumerGroup(name string, topic string, zookeeper []string, config *ConsumerGroupConfig) (cg *ConsumerGroup, err error) {

	if config == nil {
		config = NewConsumerGroupConfig()
	}

	if err := config.Validate(); err != nil {
		return nil, err
	}

	if len(zookeeper) == 0 {
		return nil, errors.New("You need to provide at least one zookeeper node address!")
	}

	var zk *ZK
	if zk, err = NewZK(zookeeper, config.ZookeeperTimeout); err != nil {
		return nil, err
	}

	var kafkaBrokers []string
	if kafkaBrokers, err = zk.Brokers(); err != nil {
		return
	}

	var client *sarama.Client
	if client, err = sarama.NewClient(name, kafkaBrokers, config.KafkaClientConfig); err != nil {
		return
	}

	return NewConsumerGroup(client, zk, name, topic, nil, config)
}

// NewConsumerGroup creates a new consumer group for a given topic.
//
// You MUST call Close() on a consumer to avoid leaks, it will not be garbage-collected automatically when
// it passes out of scope (this is in addition to calling Close on the underlying client, which is still necessary).
func NewConsumerGroup(client *sarama.Client, zoo *ZK, name string, topic string, listener chan *Notification, config *ConsumerGroupConfig) (group *ConsumerGroup, err error) {

	// Validate configuration
	if err = config.Validate(); err != nil {
		return
	} else if topic == "" {
		return nil, sarama.ConfigurationError("Empty topic")
	} else if name == "" {
		return nil, sarama.ConfigurationError("Empty name")
	}

	// Register consumer group
	if err = zoo.RegisterGroup(name); err != nil {
		return
	}

	var consumerID string
	consumerID, err = generateConsumerID()
	if err != nil {
		return
	}

	group = &ConsumerGroup{
		id:    consumerID,
		name:  name,
		topic: topic,

		config:   config,
		client:   client,
		zoo:      zoo,
		claims:   make([]PartitionConsumer, 0),
		listener: listener,

		stopper:  make(chan bool),
		checkout: make(chan bool),
		force:    make(chan bool),
		claimed:  make(chan *PartitionConsumer),

		events: make(chan *sarama.ConsumerEvent),
	}

	group.wg = &sync.WaitGroup{}

	// Register itself with zookeeper
	if err = zoo.RegisterConsumer(group.name, group.id, group.topic); err != nil {
		return nil, err
	}

	group.wg.Add(2)
	go group.signalLoop()
	go group.eventLoop()
	return group, nil
}

// Checkout applies a callback function to a single partition consumer.
// The latest consumer offset is automatically comitted to zookeeper if successful.
// The callback may return a DiscardCommit error to skip the commit silently.
// Returns an error if any, but may also return a NoCheckout error to indicate
// that no partition was available. You should add an artificial delay keep your CPU cool.
func (cg *ConsumerGroup) Checkout(callback func(*PartitionConsumer) error) error {
	select {
	case <-cg.stopper:
		return NoCheckout
	default:
		cg.checkout <- true
	}

	var claimed *PartitionConsumer
	select {
	case claimed = <-cg.claimed:
	case <-cg.stopper:
		return NoCheckout
	}

	if claimed == nil {
		return NoCheckout
	}

	err := callback(claimed)
	if err == DiscardCommit {
		err = nil
	} else if err == nil && claimed.offset > 0 {
		sarama.Logger.Printf("Committing partition %d offset %d", claimed.partition, claimed.offset)
		err = cg.Commit(claimed.partition, claimed.offset)
	}
	return err
}

func (cg *ConsumerGroup) Stream() <-chan *sarama.ConsumerEvent {
	return cg.events
}

func (cg *ConsumerGroup) eventLoop() {
	defer cg.wg.Done()
EventLoop:
	for {
		select {
		case <-cg.stopper:
			close(cg.events)
			break EventLoop

		default:
			cg.Checkout(func(pc *PartitionConsumer) error {
				sarama.Logger.Printf("Start consuming partition %d...", pc.partition)

				partitionEvents := make(chan *sarama.ConsumerEvent)
				partitionStopper := make(chan bool)

				go func() {
					if err := pc.Fetch(partitionEvents, pc.group.config.CheckoutInterval, cg.stopper); err != nil {
						panic(fmt.Sprintf("Fetch failed: %s", err))
					}

					close(partitionStopper)
				}()

				for {
					select {
					case <-partitionStopper:
						return nil

					case event, ok := <-partitionEvents:
						if ok {
							cg.events <- event
						} else {
							return errors.New("Failed to read event from channel!")
						}
					}
				}
			})
		}

	}
}

// Commit manually commits an offset for a partition
func (cg *ConsumerGroup) Commit(partition int32, offset int64) error {
	return cg.zoo.Commit(cg.name, cg.topic, partition, offset)
}

// Offset manually retrives an offset for a partition
func (cg *ConsumerGroup) Offset(partition int32) (int64, error) {
	return cg.zoo.Offset(cg.name, cg.topic, partition)
}

// Claims returns the claimed partitions
func (cg *ConsumerGroup) Claims() []int32 {
	res := make([]int32, 0, len(cg.claims))
	for _, claim := range cg.claims {
		res = append(res, claim.partition)
	}
	return res
}

// Close closes the consumer group
func (cg *ConsumerGroup) Close() error {
	close(cg.stopper)
	cg.wg.Wait()
	return nil
}

// Background signal loop
func (cg *ConsumerGroup) signalLoop() {
	defer cg.wg.Done()
	defer cg.releaseClaims()
	for {
		// If we have no zk handle, rebalance
		if cg.zkchange == nil {
			if err := cg.rebalance(); err != nil && cg.listener != nil {
				select {
				case cg.listener <- &Notification{Type: REBALANCE_ERROR, Src: cg, Err: err}:
				case <-cg.stopper:
					return
				}
			}
		}

		// If rebalance failed, check if we had a stop signal, then try again
		if cg.zkchange == nil {
			select {
			case <-cg.stopper:
				return
			case <-time.After(time.Millisecond):
				// Continue
			}
			continue
		}

		// If rebalance worked, wait for a stop signal or a zookeeper change or a fetch-request
		select {
		case <-cg.stopper:
			return
		case <-cg.force:
			cg.zkchange = nil
		case <-cg.zkchange:
			cg.zkchange = nil
		case <-cg.checkout:
			select {
			case cg.claimed <- cg.nextConsumer():
			case <-cg.stopper:
				return
			}
		}
	}
}

/**********************************************************************
 * PRIVATE
 **********************************************************************/

// Checkout a claimed partition consumer
func (cg *ConsumerGroup) nextConsumer() *PartitionConsumer {
	if len(cg.claims) < 1 {
		return nil
	}

	shift := cg.claims[0]
	cg.claims = append(cg.claims[1:], shift)
	return &shift
}

// Start a rebalance cycle
func (cg *ConsumerGroup) rebalance() (err error) {
	var cids []string
	var pids []int32

	if cg.listener != nil {
		cg.listener <- &Notification{Type: REBALANCE_START, Src: cg}
	}

	// Fetch a list of consumers and listen for changes
	if cids, cg.zkchange, err = cg.zoo.Consumers(cg.name); err != nil {
		cg.zkchange = nil
		return
	}

	// Fetch a list of partition IDs
	if pids, err = cg.client.Partitions(cg.topic); err != nil {
		cg.zkchange = nil
		return
	}

	// Get leaders for each partition ID
	parts := make(partitionSlice, len(pids))
	for i, pid := range pids {
		var broker *sarama.Broker
		if broker, err = cg.client.Leader(cg.topic, pid); err != nil {
			cg.zkchange = nil
			return
		}
		defer broker.Close()
		parts[i] = partitionLeader{id: pid, leader: broker.Addr()}
	}

	if err = cg.makeClaims(cids, parts); err != nil {
		cg.zkchange = nil
		cg.releaseClaims()
		return
	}
	return
}

func (cg *ConsumerGroup) makeClaims(cids []string, parts partitionSlice) error {
	cg.releaseClaims()

	for _, part := range cg.claimRange(cids, parts) {
		pc, err := NewPartitionConsumer(cg, part.id)
		if err != nil {
			return err
		}

		err = cg.zoo.Claim(cg.name, cg.topic, pc.partition, cg.id)
		if err != nil {
			return err
		}

		cg.claims = append(cg.claims, *pc)
	}

	if cg.listener != nil {
		cg.listener <- &Notification{Type: REBALANCE_OK, Src: cg}
	}
	return nil
}

// Determine the partititons dumber to claim
func (cg *ConsumerGroup) claimRange(cids []string, parts partitionSlice) partitionSlice {
	sort.Strings(cids)
	sort.Sort(parts)

	cpos := sort.SearchStrings(cids, cg.id)
	clen := len(cids)
	plen := len(parts)
	if cpos >= clen || cpos >= plen {
		return make(partitionSlice, 0)
	}

	step := int(math.Ceil(float64(plen) / float64(clen)))
	if step < 1 {
		step = 1
	}

	last := (cpos + 1) * step
	if last > plen {
		last = plen
	}
	return parts[cpos*step : last]
}

// Releases all claims
func (cg *ConsumerGroup) releaseClaims() {
	for _, pc := range cg.claims {
		sarama.Logger.Printf("Releasing claim for partition %d...\n", pc.partition)
		pc.Close()
		cg.zoo.Release(cg.name, cg.topic, pc.partition, cg.id)
	}
	cg.claims = cg.claims[:0]
}
<commit_msg>Beter deadlock fix<commit_after>package consumergroup

import (
	"errors"
	"fmt"
	"math"
	"sort"
	"sync"
	"time"

	"github.com/Shopify/sarama"
	"github.com/samuel/go-zookeeper/zk"
)

var (
	DiscardCommit = errors.New("sarama: commit discarded")
	NoCheckout    = errors.New("sarama: not checkout")
)

const (
	REBALANCE_START uint8 = iota + 1
	REBALANCE_OK
	REBALANCE_ERROR
)

type ConsumerGroupConfig struct {
	// The Zookeeper timeout
	ZookeeperTimeout time.Duration

	// The preempt interval when listening to a single partition of a topic.
	// After this interval, the current offset will be committed to Zookeeper,
	// and a different partition will be checked out to consume next.
	CheckoutInterval time.Duration

	KafkaClientConfig   *sarama.ClientConfig   // This will be passed to Sarama when creating a new Client
	KafkaConsumerConfig *sarama.ConsumerConfig // This will be passed to Sarama when creating a new Consumer
}

func NewConsumerGroupConfig() *ConsumerGroupConfig {
	return &ConsumerGroupConfig{
		ZookeeperTimeout:    1 * time.Second,
		CheckoutInterval:    1 * time.Second,
		KafkaClientConfig:   sarama.NewClientConfig(),
		KafkaConsumerConfig: sarama.NewConsumerConfig(),
	}
}

func (cgc *ConsumerGroupConfig) Validate() error {
	if cgc.ZookeeperTimeout <= 0 {
		return errors.New("ZookeeperTimeout should have a duration > 0")
	}

	if cgc.KafkaClientConfig == nil {
		return errors.New("KafkaClientConfig is not set!")
	} else if err := cgc.KafkaClientConfig.Validate(); err != nil {
		return err
	}

	if cgc.KafkaConsumerConfig == nil {
		return errors.New("KafkaConsumerConfig is not set!")
	} else if err := cgc.KafkaConsumerConfig.Validate(); err != nil {
		return err
	}

	return nil
}

// A ConsumerGroup operates on all partitions of a single topic. The goal is to ensure
// each topic message is consumed only once, no matter of the number of consumer instances within
// a cluster, as described in: http://kafka.apache.org/documentation.html#distributionimpl.
//
// The ConsumerGroup internally creates multiple Consumer instances. It uses Zookkeper
// and follows a simple consumer rebalancing algorithm which allows all the consumers
// in a group to come into consensus on which consumer is consuming which partitions. Each
// ConsumerGroup can 'claim' 0-n partitions and will consume their messages until another
// ConsumerGroup instance with the same name joins or leaves the cluster.
//
// Unlike stated in the Kafka documentation, consumer rebalancing is *only* triggered on each
// addition or removal of consumers within the same group, while the addition of broker nodes
// and/or partition *does currently not trigger* a rebalancing cycle.
type ConsumerGroup struct {
	id, name, topic string

	config *ConsumerGroupConfig

	client *sarama.Client
	zoo    *ZK
	claims []PartitionConsumer
	wg     *sync.WaitGroup

	zkchange <-chan zk.Event
	claimed  chan *PartitionConsumer
	listener chan *Notification

	events chan *sarama.ConsumerEvent

	checkout, force, stopper chan bool
}

// Connects to a consumer group, using Zookeeper for auto-discovery
func JoinConsumerGroup(name string, topic string, zookeeper []string, config *ConsumerGroupConfig) (cg *ConsumerGroup, err error) {

	if config == nil {
		config = NewConsumerGroupConfig()
	}

	if err := config.Validate(); err != nil {
		return nil, err
	}

	if len(zookeeper) == 0 {
		return nil, errors.New("You need to provide at least one zookeeper node address!")
	}

	var zk *ZK
	if zk, err = NewZK(zookeeper, config.ZookeeperTimeout); err != nil {
		return nil, err
	}

	var kafkaBrokers []string
	if kafkaBrokers, err = zk.Brokers(); err != nil {
		return
	}

	var client *sarama.Client
	if client, err = sarama.NewClient(name, kafkaBrokers, config.KafkaClientConfig); err != nil {
		return
	}

	return NewConsumerGroup(client, zk, name, topic, nil, config)
}

// NewConsumerGroup creates a new consumer group for a given topic.
//
// You MUST call Close() on a consumer to avoid leaks, it will not be garbage-collected automatically when
// it passes out of scope (this is in addition to calling Close on the underlying client, which is still necessary).
func NewConsumerGroup(client *sarama.Client, zoo *ZK, name string, topic string, listener chan *Notification, config *ConsumerGroupConfig) (group *ConsumerGroup, err error) {

	// Validate configuration
	if err = config.Validate(); err != nil {
		return
	} else if topic == "" {
		return nil, sarama.ConfigurationError("Empty topic")
	} else if name == "" {
		return nil, sarama.ConfigurationError("Empty name")
	}

	// Register consumer group
	if err = zoo.RegisterGroup(name); err != nil {
		return
	}

	var consumerID string
	consumerID, err = generateConsumerID()
	if err != nil {
		return
	}

	group = &ConsumerGroup{
		id:    consumerID,
		name:  name,
		topic: topic,

		config:   config,
		client:   client,
		zoo:      zoo,
		claims:   make([]PartitionConsumer, 0),
		listener: listener,

		stopper:  make(chan bool),
		checkout: make(chan bool),
		force:    make(chan bool),
		claimed:  make(chan *PartitionConsumer),

		events: make(chan *sarama.ConsumerEvent),
	}

	group.wg = &sync.WaitGroup{}

	// Register itself with zookeeper
	if err = zoo.RegisterConsumer(group.name, group.id, group.topic); err != nil {
		return nil, err
	}

	group.wg.Add(2)
	go group.signalLoop()
	go group.eventLoop()
	return group, nil
}

// Checkout applies a callback function to a single partition consumer.
// The latest consumer offset is automatically comitted to zookeeper if successful.
// The callback may return a DiscardCommit error to skip the commit silently.
// Returns an error if any, but may also return a NoCheckout error to indicate
// that no partition was available. You should add an artificial delay keep your CPU cool.
func (cg *ConsumerGroup) Checkout(callback func(*PartitionConsumer) error) error {
	select {
	case <-cg.stopper:
		return NoCheckout
	case cg.checkout <- true:
	}

	var claimed *PartitionConsumer
	select {
	case claimed = <-cg.claimed:
	case <-cg.stopper:
		return NoCheckout
	}

	if claimed == nil {
		return NoCheckout
	}

	err := callback(claimed)
	if err == DiscardCommit {
		err = nil
	} else if err == nil && claimed.offset > 0 {
		sarama.Logger.Printf("Committing partition %d offset %d", claimed.partition, claimed.offset)
		err = cg.Commit(claimed.partition, claimed.offset)
	}
	return err
}

func (cg *ConsumerGroup) Stream() <-chan *sarama.ConsumerEvent {
	return cg.events
}

func (cg *ConsumerGroup) eventLoop() {
	defer cg.wg.Done()
EventLoop:
	for {
		select {
		case <-cg.stopper:
			close(cg.events)
			break EventLoop

		default:
			cg.Checkout(func(pc *PartitionConsumer) error {
				sarama.Logger.Printf("Start consuming partition %d...", pc.partition)

				partitionEvents := make(chan *sarama.ConsumerEvent)
				partitionStopper := make(chan bool)

				go func() {
					if err := pc.Fetch(partitionEvents, pc.group.config.CheckoutInterval, cg.stopper); err != nil {
						panic(fmt.Sprintf("Fetch failed: %s", err))
					}

					close(partitionStopper)
				}()

				for {
					select {
					case <-partitionStopper:
						return nil

					case event, ok := <-partitionEvents:
						if ok {
							cg.events <- event
						} else {
							return errors.New("Failed to read event from channel!")
						}
					}
				}
			})
		}

	}
}

// Commit manually commits an offset for a partition
func (cg *ConsumerGroup) Commit(partition int32, offset int64) error {
	return cg.zoo.Commit(cg.name, cg.topic, partition, offset)
}

// Offset manually retrives an offset for a partition
func (cg *ConsumerGroup) Offset(partition int32) (int64, error) {
	return cg.zoo.Offset(cg.name, cg.topic, partition)
}

// Claims returns the claimed partitions
func (cg *ConsumerGroup) Claims() []int32 {
	res := make([]int32, 0, len(cg.claims))
	for _, claim := range cg.claims {
		res = append(res, claim.partition)
	}
	return res
}

// Close closes the consumer group
func (cg *ConsumerGroup) Close() error {
	close(cg.stopper)
	cg.wg.Wait()
	return nil
}

// Background signal loop
func (cg *ConsumerGroup) signalLoop() {
	defer cg.wg.Done()
	defer cg.releaseClaims()
	for {
		// If we have no zk handle, rebalance
		if cg.zkchange == nil {
			if err := cg.rebalance(); err != nil && cg.listener != nil {
				select {
				case cg.listener <- &Notification{Type: REBALANCE_ERROR, Src: cg, Err: err}:
				case <-cg.stopper:
					return
				}
			}
		}

		// If rebalance failed, check if we had a stop signal, then try again
		if cg.zkchange == nil {
			select {
			case <-cg.stopper:
				return
			case <-time.After(time.Millisecond):
				// Continue
			}
			continue
		}

		// If rebalance worked, wait for a stop signal or a zookeeper change or a fetch-request
		select {
		case <-cg.stopper:
			return
		case <-cg.force:
			cg.zkchange = nil
		case <-cg.zkchange:
			cg.zkchange = nil
		case <-cg.checkout:
			select {
			case cg.claimed <- cg.nextConsumer():
			case <-cg.stopper:
				return
			}
		}
	}
}

/**********************************************************************
 * PRIVATE
 **********************************************************************/

// Checkout a claimed partition consumer
func (cg *ConsumerGroup) nextConsumer() *PartitionConsumer {
	if len(cg.claims) < 1 {
		return nil
	}

	shift := cg.claims[0]
	cg.claims = append(cg.claims[1:], shift)
	return &shift
}

// Start a rebalance cycle
func (cg *ConsumerGroup) rebalance() (err error) {
	var cids []string
	var pids []int32

	if cg.listener != nil {
		cg.listener <- &Notification{Type: REBALANCE_START, Src: cg}
	}

	// Fetch a list of consumers and listen for changes
	if cids, cg.zkchange, err = cg.zoo.Consumers(cg.name); err != nil {
		cg.zkchange = nil
		return
	}

	// Fetch a list of partition IDs
	if pids, err = cg.client.Partitions(cg.topic); err != nil {
		cg.zkchange = nil
		return
	}

	// Get leaders for each partition ID
	parts := make(partitionSlice, len(pids))
	for i, pid := range pids {
		var broker *sarama.Broker
		if broker, err = cg.client.Leader(cg.topic, pid); err != nil {
			cg.zkchange = nil
			return
		}
		defer broker.Close()
		parts[i] = partitionLeader{id: pid, leader: broker.Addr()}
	}

	if err = cg.makeClaims(cids, parts); err != nil {
		cg.zkchange = nil
		cg.releaseClaims()
		return
	}
	return
}

func (cg *ConsumerGroup) makeClaims(cids []string, parts partitionSlice) error {
	cg.releaseClaims()

	for _, part := range cg.claimRange(cids, parts) {
		pc, err := NewPartitionConsumer(cg, part.id)
		if err != nil {
			return err
		}

		err = cg.zoo.Claim(cg.name, cg.topic, pc.partition, cg.id)
		if err != nil {
			return err
		}

		cg.claims = append(cg.claims, *pc)
	}

	if cg.listener != nil {
		cg.listener <- &Notification{Type: REBALANCE_OK, Src: cg}
	}
	return nil
}

// Determine the partititons dumber to claim
func (cg *ConsumerGroup) claimRange(cids []string, parts partitionSlice) partitionSlice {
	sort.Strings(cids)
	sort.Sort(parts)

	cpos := sort.SearchStrings(cids, cg.id)
	clen := len(cids)
	plen := len(parts)
	if cpos >= clen || cpos >= plen {
		return make(partitionSlice, 0)
	}

	step := int(math.Ceil(float64(plen) / float64(clen)))
	if step < 1 {
		step = 1
	}

	last := (cpos + 1) * step
	if last > plen {
		last = plen
	}
	return parts[cpos*step : last]
}

// Releases all claims
func (cg *ConsumerGroup) releaseClaims() {
	for _, pc := range cg.claims {
		sarama.Logger.Printf("Releasing claim for partition %d...\n", pc.partition)
		pc.Close()
		cg.zoo.Release(cg.name, cg.topic, pc.partition, cg.id)
	}
	cg.claims = cg.claims[:0]
}
<|endoftext|>