<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package heap

import (
	"testing"
	"container/vector"
)

type myHeap struct {
	// A vector.Vector implements sort.Interface except for Less,
	// and it implements Push and Pop as required for heap.Interface.
	vector.Vector
}

func (h *myHeap) Less(i, j int) bool { return h.At(i).(int) < h.At(j).(int) }

func (h *myHeap) verify(t *testing.T, i int) {
	n := h.Len()
	j1 := 2*i + 1
	j2 := 2*i + 2
	if j1 < n {
		if h.Less(j1, i) {
			t.Errorf("heap invariant invalidated [%d] = %d > [%d] = %d", i, h.At(i), j1, h.At(j1))
			return
		}
		h.verify(t, j1)
	}
	if j2 < n {
		if h.Less(j2, i) {
			t.Errorf("heap invariant invalidated [%d] = %d > [%d] = %d", i, h.At(i), j1, h.At(j2))
			return
		}
		h.verify(t, j2)
	}
}

func TestInit0(t *testing.T) {
	h := new(myHeap)
	for i := 20; i > 0; i-- {
		h.Push(0) // all elements are the same
	}
	Init(h)
	h.verify(t, 0)

	for i := 1; h.Len() > 0; i++ {
		x := Pop(h).(int)
		h.verify(t, 0)
		if x != 0 {
			t.Errorf("%d.th pop got %d; want %d", i, x, 0)
		}
	}
}

func TestInit1(t *testing.T) {
	h := new(myHeap)
	for i := 20; i > 0; i-- {
		h.Push(i) // all elements are different
	}
	Init(h)
	h.verify(t, 0)

	for i := 1; h.Len() > 0; i++ {
		x := Pop(h).(int)
		h.verify(t, 0)
		if x != i {
			t.Errorf("%d.th pop got %d; want %d", i, x, i)
		}
	}
}

func Test(t *testing.T) {
	h := new(myHeap)
	h.verify(t, 0)

	for i := 20; i > 10; i-- {
		h.Push(i)
	}
	Init(h)
	h.verify(t, 0)

	for i := 10; i > 0; i-- {
		Push(h, i)
		h.verify(t, 0)
	}

	for i := 1; h.Len() > 0; i++ {
		x := Pop(h).(int)
		if i < 20 {
			Push(h, 20+i)
		}
		h.verify(t, 0)
		if x != i {
			t.Errorf("%d.th pop got %d; want %d", i, x, i)
		}
	}
}

func TestRemove0(t *testing.T) {
	h := new(myHeap)
	for i := 0; i < 10; i++ {
		h.Push(i)
	}
	h.verify(t, 0)

	for h.Len() > 0 {
		i := h.Len() - 1
		x := Remove(h, i).(int)
		if x != i {
			t.Errorf("Remove(%d) got %d; want %d", i, x, i)
		}
		h.verify(t, 0)
	}
}

func TestRemove1(t *testing.T) {
	h := new(myHeap)
	for i := 0; i < 10; i++ {
		h.Push(i)
	}
	h.verify(t, 0)

	for i := 0; h.Len() > 0; i++ {
		x := Remove(h, 0).(int)
		if x != i {
			t.Errorf("Remove(0) got %d; want %d", x, i)
		}
		h.verify(t, 0)
	}
}

func TestRemove2(t *testing.T) {
	N := 10

	h := new(myHeap)
	for i := 0; i < N; i++ {
		h.Push(i)
	}
	h.verify(t, 0)

	m := make(map[int]bool)
	for h.Len() > 0 {
		m[Remove(h, (h.Len()-1)/2).(int)] = true
		h.verify(t, 0)
	}

	if len(m) != N {
		t.Errorf("len(m) = %d; want %d", len(m), N)
	}
	for i := 0; i < len(m); i++ {
		if !m[i] {
			t.Errorf("m[%d] doesn't exist", i)
		}
	}
}
<commit_msg>container/heap: fix circular dependency in test<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package heap_test

import (
	"testing"
	"container/vector"
	. "container/heap"
)

type myHeap struct {
	// A vector.Vector implements sort.Interface except for Less,
	// and it implements Push and Pop as required for heap.Interface.
	vector.Vector
}

func (h *myHeap) Less(i, j int) bool { return h.At(i).(int) < h.At(j).(int) }

func (h *myHeap) verify(t *testing.T, i int) {
	n := h.Len()
	j1 := 2*i + 1
	j2 := 2*i + 2
	if j1 < n {
		if h.Less(j1, i) {
			t.Errorf("heap invariant invalidated [%d] = %d > [%d] = %d", i, h.At(i), j1, h.At(j1))
			return
		}
		h.verify(t, j1)
	}
	if j2 < n {
		if h.Less(j2, i) {
			t.Errorf("heap invariant invalidated [%d] = %d > [%d] = %d", i, h.At(i), j1, h.At(j2))
			return
		}
		h.verify(t, j2)
	}
}

func TestInit0(t *testing.T) {
	h := new(myHeap)
	for i := 20; i > 0; i-- {
		h.Push(0) // all elements are the same
	}
	Init(h)
	h.verify(t, 0)

	for i := 1; h.Len() > 0; i++ {
		x := Pop(h).(int)
		h.verify(t, 0)
		if x != 0 {
			t.Errorf("%d.th pop got %d; want %d", i, x, 0)
		}
	}
}

func TestInit1(t *testing.T) {
	h := new(myHeap)
	for i := 20; i > 0; i-- {
		h.Push(i) // all elements are different
	}
	Init(h)
	h.verify(t, 0)

	for i := 1; h.Len() > 0; i++ {
		x := Pop(h).(int)
		h.verify(t, 0)
		if x != i {
			t.Errorf("%d.th pop got %d; want %d", i, x, i)
		}
	}
}

func Test(t *testing.T) {
	h := new(myHeap)
	h.verify(t, 0)

	for i := 20; i > 10; i-- {
		h.Push(i)
	}
	Init(h)
	h.verify(t, 0)

	for i := 10; i > 0; i-- {
		Push(h, i)
		h.verify(t, 0)
	}

	for i := 1; h.Len() > 0; i++ {
		x := Pop(h).(int)
		if i < 20 {
			Push(h, 20+i)
		}
		h.verify(t, 0)
		if x != i {
			t.Errorf("%d.th pop got %d; want %d", i, x, i)
		}
	}
}

func TestRemove0(t *testing.T) {
	h := new(myHeap)
	for i := 0; i < 10; i++ {
		h.Push(i)
	}
	h.verify(t, 0)

	for h.Len() > 0 {
		i := h.Len() - 1
		x := Remove(h, i).(int)
		if x != i {
			t.Errorf("Remove(%d) got %d; want %d", i, x, i)
		}
		h.verify(t, 0)
	}
}

func TestRemove1(t *testing.T) {
	h := new(myHeap)
	for i := 0; i < 10; i++ {
		h.Push(i)
	}
	h.verify(t, 0)

	for i := 0; h.Len() > 0; i++ {
		x := Remove(h, 0).(int)
		if x != i {
			t.Errorf("Remove(0) got %d; want %d", x, i)
		}
		h.verify(t, 0)
	}
}

func TestRemove2(t *testing.T) {
	N := 10

	h := new(myHeap)
	for i := 0; i < N; i++ {
		h.Push(i)
	}
	h.verify(t, 0)

	m := make(map[int]bool)
	for h.Len() > 0 {
		m[Remove(h, (h.Len()-1)/2).(int)] = true
		h.verify(t, 0)
	}

	if len(m) != N {
		t.Errorf("len(m) = %d; want %d", len(m), N)
	}
	for i := 0; i < len(m); i++ {
		if !m[i] {
			t.Errorf("m[%d] doesn't exist", i)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gitiles

import (
	"fmt"
	"net/url"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/golang/protobuf/proto"
	"golang.org/x/net/context"
	"google.golang.org/api/pubsub/v1"

	"go.chromium.org/luci/common/api/gitiles"
	"go.chromium.org/luci/common/data/stringset"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	gitilespb "go.chromium.org/luci/common/proto/gitiles"
	"go.chromium.org/luci/common/sync/parallel"
	"go.chromium.org/luci/config/validation"
	"go.chromium.org/luci/server/auth"

	api "go.chromium.org/luci/scheduler/api/scheduler/v1"
	"go.chromium.org/luci/scheduler/appengine/internal"
	"go.chromium.org/luci/scheduler/appengine/messages"
	"go.chromium.org/luci/scheduler/appengine/task"
)

// defaultMaxTriggersPerInvocation limits number of triggers emitted per one
// invocation.
const defaultMaxTriggersPerInvocation = 100

// TaskManager implements task.Manager interface for tasks defined with
// GitilesTask proto message.
type TaskManager struct {
	mockGitilesClient        gitilespb.GitilesClient // Used for testing only.
	maxTriggersPerInvocation int                     // Avoid choking on DS or runtime limits.
}

// Name is part of Manager interface.
func (m TaskManager) Name() string {
	return "gitiles"
}

// ProtoMessageType is part of Manager interface.
func (m TaskManager) ProtoMessageType() proto.Message {
	return (*messages.GitilesTask)(nil)
}

// Traits is part of Manager interface.
func (m TaskManager) Traits() task.Traits {
	return task.Traits{
		Multistage: false, // we don't use task.StatusRunning state
	}
}

// ValidateProtoMessage is part of Manager interface.
func (m TaskManager) ValidateProtoMessage(c *validation.Context, msg proto.Message) {
	cfg, ok := msg.(*messages.GitilesTask)
	if !ok {
		c.Errorf("wrong type %T, expecting *messages.GitilesTask", msg)
		return
	}

	// Validate 'repo' field.
	c.Enter("repo")
	if cfg.Repo == "" {
		c.Errorf("field 'repository' is required")
	} else {
		u, err := url.Parse(cfg.Repo)
		if err != nil {
			c.Errorf("invalid URL %q: %s", cfg.Repo, err)
		} else if !u.IsAbs() {
			c.Errorf("not an absolute url: %q", cfg.Repo)
		}
	}
	c.Exit()

	c.Enter("refs")
	for _, ref := range cfg.Refs {
		if !strings.HasPrefix(ref, "refs/") {
			c.Errorf("ref must start with 'refs/' not %q", ref)
		}
		cnt := strings.Count(ref, "*")
		if cnt > 1 || (cnt == 1 && !strings.HasSuffix(ref, "/*")) {
			c.Errorf("only trailing (e.g. refs/blah/*) globs are supported, not %q", ref)
		}
	}
	c.Exit()
}

// LaunchTask is part of Manager interface.
func (m TaskManager) LaunchTask(c context.Context, ctl task.Controller) error {
	cfg := ctl.Task().(*messages.GitilesTask)
	ctl.DebugLog("Repo: %s, Refs: %s", cfg.Repo, cfg.Refs)
	repoURL, err := url.Parse(cfg.Repo)
	if err != nil {
		return err
	}

	refs, err := m.fetchRefsState(c, ctl, cfg, repoURL)
	if err != nil {
		ctl.DebugLog("Error fetching state of the world: %s", err)
		return err
	}

	refs.pruneKnown(ctl)
	leftToProcess := m.emitTriggersRefAtATime(c, ctl, cfg.Repo, refs)

	if refs.changed == 0 {
		ctl.DebugLog("No changes detected")
		ctl.State().Status = task.StatusSucceeded
		return nil
	}
	ctl.DebugLog("%d changed refs processed, %d refs not yet examined", refs.changed, leftToProcess)
	// Force save to ensure triggers are actually emitted.
	if err := ctl.Save(c); err != nil {
		// At this point, triggers have not been sent, so bail now and don't save
		// the refs' heads newest values.
		return err
	}
	if err := saveState(c, ctl.JobID(), repoURL, refs.known); err != nil {
		return err
	}
	ctl.DebugLog("Saved %d known refs", len(refs.known))
	ctl.State().Status = task.StatusSucceeded
	return nil
}

// AbortTask is part of Manager interface.
func (m TaskManager) AbortTask(c context.Context, ctl task.Controller) error {
	return nil
}

// HandleNotification is part of Manager interface.
func (m TaskManager) HandleNotification(c context.Context, ctl task.Controller, msg *pubsub.PubsubMessage) error {
	return errors.New("not implemented")
}

// HandleTimer is part of Manager interface.
func (m TaskManager) HandleTimer(c context.Context, ctl task.Controller, name string, payload []byte) error {
	return errors.New("not implemented")
}

func (m TaskManager) fetchRefsState(c context.Context, ctl task.Controller, cfg *messages.GitilesTask, repoURL *url.URL) (*refsState, error) {
	refs := &refsState{}
	refs.watched.init(cfg.GetRefs())
	return refs, parallel.FanOutIn(func(work chan<- func() error) {
		work <- func() (loadErr error) {
			refs.known, loadErr = loadState(c, ctl.JobID(), repoURL)
			return
		}
		work <- func() (refsErr error) {
			// Merge getRefsTips here.
			refs.current, refsErr = m.getRefsTips(c, ctl, cfg.Repo, refs.watched)
			return
		}
	})
}

// getRefsTips returns tip for each ref being watched.
func (m TaskManager) getRefsTips(c context.Context, ctl task.Controller, repoURL string, watched watchedRefs) (map[string]string, error) {
	host, project, err := gitiles.ParseRepoURL(repoURL)
	if err != nil {
		return nil, errors.Annotate(err, "invalid repo URL %q", repoURL).Err()
	}

	g, err := m.getGitilesClient(c, ctl, host)
	if err != nil {
		return nil, err
	}

	// Query gitiles for each namespace in parallel.
	var wg sync.WaitGroup
	var lock sync.Mutex
	errs := []error{}
	allTips := map[string]string{}
	// Group all refs by their namespace to reduce # of RPCs.
	for _, wrs := range watched.namespaces {
		wg.Add(1)
		go func(wrs *watchedRefNamespace) {
			defer wg.Done()
			res, err := g.Refs(c, &gitilespb.RefsRequest{
				Project:  project,
				RefsPath: wrs.namespace,
			})
			lock.Lock()
			defer lock.Unlock()
			if err != nil {
				ctl.DebugLog("failed to fetch %q namespace tips for %q: %q", wrs.namespace, err)
				errs = append(errs, err)
				return
			}
			for ref, tip := range res.Revisions {
				if watched.hasRef(ref) {
					allTips[ref] = tip
				}
			}
		}(wrs)
	}
	wg.Wait()
	if len(errs) > 0 {
		return nil, errors.NewMultiError(errs...)
	}
	return allTips, nil
}

// emitTriggersRefAtATime processes refs one a time and emits triggers if ref
// changed. Limits number of triggers emitted and so may stop early.
//
// Returns how many refs were not examined.
func (m TaskManager) emitTriggersRefAtATime(c context.Context, ctl task.Controller, repo string, refs *refsState) (left int) {
	maxTriggersPerInvocation := m.maxTriggersPerInvocation
	if maxTriggersPerInvocation == 0 {
		maxTriggersPerInvocation = defaultMaxTriggersPerInvocation
	}
	emittedTriggers := 0
	// Note, that refs.current contain only watched refs (see getRefsTips).
	// For determinism, sort refs by name.
	sortedRefs := make([]string, 0, len(refs.current))
	for ref := range refs.current {
		sortedRefs = append(sortedRefs, ref)
	}
	sort.Strings(sortedRefs)
	for i, ref := range sortedRefs {
		// TODO(tandrii): enforce limit on emitted triggers here.
		emittedTriggers += m.emitTriggersForRef(c, ctl, repo, ref, refs)
		// Safeguard against too many changes such as the first run after
		// config change to watch many more refs than before.
		if emittedTriggers >= maxTriggersPerInvocation {
			ctl.DebugLog("Emitted %d triggers, postponing the rest", emittedTriggers)
			left = len(sortedRefs) - i - 1
			return left
		}
	}
	left = 0
	return
}

// emitTriggersForRef emits triggers for 1 ref if there are any changes and
// returns number of triggers emitted.
func (m TaskManager) emitTriggersForRef(c context.Context, ctl task.Controller, repo, ref string, refs *refsState) int {
	newHead := refs.current[ref]
	switch oldHead, existed := refs.known[ref]; {
	case !existed:
		ctl.DebugLog("Ref %s is new: %s", ref, newHead)
	case oldHead != newHead:
		ctl.DebugLog("Ref %s updated: %s => %s", ref, oldHead, newHead)
	default:
		return 0 // no change
	}
	refs.known[ref] = newHead
	refs.changed++
	// TODO(tandrii): actually look at commits between current and previously
	// known tips of each ref.
	// In current (v1) engine, all triggers emitted around the same time will
	// result in just 1 invocation of each triggered job. Therefore,
	// passing just HEAD's revision is good enough.
	// For the same reason, only 1 of the refs will actually be processed if
	// several refs changed at the same time.
	ctl.EmitTrigger(c, &internal.Trigger{
		Id:    fmt.Sprintf("%s/+/%s@%s", repo, ref, newHead),
		Title: newHead,
		Url:   fmt.Sprintf("%s/+/%s", repo, newHead),
		Payload: &internal.Trigger_Gitiles{
			Gitiles: &api.GitilesTrigger{Repo: repo, Ref: ref, Revision: newHead},
		},
	})
	return 1
}

func (m TaskManager) getGitilesClient(c context.Context, ctl task.Controller, host string) (gitilespb.GitilesClient, error) {
	if m.mockGitilesClient != nil {
		// Used for testing only.
		logging.Infof(c, "using mockGitilesClient")
		return m.mockGitilesClient, nil
	}

	httpClient, err := ctl.GetClient(c, time.Minute, auth.WithScopes(gitiles.OAuthScope))
	if err != nil {
		return nil, err
	}

	return gitiles.NewRESTClient(httpClient, host, true)
}

type refsState struct {
	watched watchedRefs
	known   map[string]string // HEADs we saw before
	current map[string]string // HEADs available now
	changed int
}

func (s *refsState) pruneKnown(ctl task.Controller) {
	for ref := range s.known {
		switch {
		case !s.watched.hasRef(ref):
			ctl.DebugLog("Ref %s is no longer watched", ref)
			delete(s.known, ref)
			s.changed++
		case s.current[ref] == "":
			ctl.DebugLog("Ref %s deleted", ref)
			delete(s.known, ref)
			s.changed++
		}
	}
}

func (s *refsState) sortedCurrentRefNames() []string {
	sortedRefs := make([]string, 0, len(s.current))
	for ref := range s.current {
		sortedRefs = append(sortedRefs, ref)
	}
	sort.Strings(sortedRefs)
	return sortedRefs
}

type watchedRefNamespace struct {
	namespace    string // no trailing "/".
	allChildren  bool   // if true, someChildren is ignored.
	someChildren stringset.Set
}

func (w watchedRefNamespace) hasSuffix(suffix string) bool {
	switch {
	case suffix == "*":
		panic(fmt.Errorf("watchedRefNamespace membership should only be checked for refs, not ref glob %s", suffix))
	case w.allChildren:
		return true
	case w.someChildren == nil:
		return false
	default:
		return w.someChildren.Has(suffix)
	}
}

func (w *watchedRefNamespace) addSuffix(suffix string) {
	switch {
	case w.allChildren:
		return
	case suffix == "*":
		w.allChildren = true
		w.someChildren = nil
		return
	case w.someChildren == nil:
		w.someChildren = stringset.New(1)
	}
	w.someChildren.Add(suffix)
}

type watchedRefs struct {
	namespaces map[string]*watchedRefNamespace
}

func (w *watchedRefs) init(refsConfig []string) {
	w.namespaces = map[string]*watchedRefNamespace{}
	for _, ref := range refsConfig {
		ns, suffix := splitRef(ref)
		if _, exists := w.namespaces[ns]; !exists {
			w.namespaces[ns] = &watchedRefNamespace{namespace: ns}
		}
		w.namespaces[ns].addSuffix(suffix)
	}
}

func (w *watchedRefs) hasRef(ref string) bool {
	ns, suffix := splitRef(ref)
	if wrn, exists := w.namespaces[ns]; exists {
		return wrn.hasSuffix(suffix)
	}
	return false
}

func splitRef(s string) (string, string) {
	if i := strings.LastIndex(s, "/"); i <= 0 {
		return s, ""
	} else {
		return s[:i], s[i+1:]
	}
}
<commit_msg>[scheduler] prepare to share gitilesClient to fetch refs and commits.<commit_after>// Copyright 2016 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gitiles

import (
	"fmt"
	"net/url"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/golang/protobuf/proto"
	"golang.org/x/net/context"
	"google.golang.org/api/pubsub/v1"

	"go.chromium.org/luci/common/api/gitiles"
	"go.chromium.org/luci/common/data/stringset"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	gitilespb "go.chromium.org/luci/common/proto/gitiles"
	"go.chromium.org/luci/common/sync/parallel"
	"go.chromium.org/luci/config/validation"
	"go.chromium.org/luci/server/auth"

	api "go.chromium.org/luci/scheduler/api/scheduler/v1"
	"go.chromium.org/luci/scheduler/appengine/internal"
	"go.chromium.org/luci/scheduler/appengine/messages"
	"go.chromium.org/luci/scheduler/appengine/task"
)

// defaultMaxTriggersPerInvocation limits number of triggers emitted per one
// invocation.
const defaultMaxTriggersPerInvocation = 100

// TaskManager implements task.Manager interface for tasks defined with
// GitilesTask proto message.
type TaskManager struct {
	mockGitilesClient        gitilespb.GitilesClient // Used for testing only.
	maxTriggersPerInvocation int                     // Avoid choking on DS or runtime limits.
}

// Name is part of Manager interface.
func (m TaskManager) Name() string {
	return "gitiles"
}

// ProtoMessageType is part of Manager interface.
func (m TaskManager) ProtoMessageType() proto.Message {
	return (*messages.GitilesTask)(nil)
}

// Traits is part of Manager interface.
func (m TaskManager) Traits() task.Traits {
	return task.Traits{
		Multistage: false, // we don't use task.StatusRunning state
	}
}

// ValidateProtoMessage is part of Manager interface.
func (m TaskManager) ValidateProtoMessage(c *validation.Context, msg proto.Message) {
	cfg, ok := msg.(*messages.GitilesTask)
	if !ok {
		c.Errorf("wrong type %T, expecting *messages.GitilesTask", msg)
		return
	}

	// Validate 'repo' field.
	c.Enter("repo")
	if cfg.Repo == "" {
		c.Errorf("field 'repository' is required")
	} else {
		u, err := url.Parse(cfg.Repo)
		if err != nil {
			c.Errorf("invalid URL %q: %s", cfg.Repo, err)
		} else if !u.IsAbs() {
			c.Errorf("not an absolute url: %q", cfg.Repo)
		}
	}
	c.Exit()

	c.Enter("refs")
	for _, ref := range cfg.Refs {
		if !strings.HasPrefix(ref, "refs/") {
			c.Errorf("ref must start with 'refs/' not %q", ref)
		}
		cnt := strings.Count(ref, "*")
		if cnt > 1 || (cnt == 1 && !strings.HasSuffix(ref, "/*")) {
			c.Errorf("only trailing (e.g. refs/blah/*) globs are supported, not %q", ref)
		}
	}
	c.Exit()
}

// LaunchTask is part of Manager interface.
func (m TaskManager) LaunchTask(c context.Context, ctl task.Controller) error {
	cfg := ctl.Task().(*messages.GitilesTask)
	ctl.DebugLog("Repo: %s, Refs: %s", cfg.Repo, cfg.Refs)

	g, err := m.getGitilesClient(c, ctl, cfg)
	if err != nil {
		return err
	}
	// TODO(tandrii): use g.host, g.project for saving/loading state
	// instead of repoURL.
	repoURL, err := url.Parse(cfg.Repo)
	if err != nil {
		return err
	}

	refs, err := m.fetchRefsState(c, ctl, cfg, g, repoURL)
	if err != nil {
		ctl.DebugLog("Error fetching state of the world: %s", err)
		return err
	}

	refs.pruneKnown(ctl)
	leftToProcess := m.emitTriggersRefAtATime(c, ctl, cfg.Repo, refs)

	switch {
	case leftToProcess == 0 && refs.changed == 0:
		ctl.DebugLog("No changes detected")
		ctl.State().Status = task.StatusSucceeded
		return nil
	case leftToProcess == 0:
		ctl.DebugLog("All %d changed refs processed", refs.changed)
	default:
		ctl.DebugLog("%d changed refs processed, %d refs not yet examined", refs.changed, leftToProcess)
	}
	// Force save to ensure triggers are actually emitted.
	if err := ctl.Save(c); err != nil {
		// At this point, triggers have not been sent, so bail now and don't save
		// the refs' heads newest values.
		return err
	}
	if err := saveState(c, ctl.JobID(), repoURL, refs.known); err != nil {
		return err
	}
	ctl.DebugLog("Saved %d known refs", len(refs.known))
	ctl.State().Status = task.StatusSucceeded
	return nil
}

// AbortTask is part of Manager interface.
func (m TaskManager) AbortTask(c context.Context, ctl task.Controller) error {
	return nil
}

// HandleNotification is part of Manager interface.
func (m TaskManager) HandleNotification(c context.Context, ctl task.Controller, msg *pubsub.PubsubMessage) error {
	return errors.New("not implemented")
}

// HandleTimer is part of Manager interface.
func (m TaskManager) HandleTimer(c context.Context, ctl task.Controller, name string, payload []byte) error {
	return errors.New("not implemented")
}

func (m TaskManager) fetchRefsState(c context.Context, ctl task.Controller, cfg *messages.GitilesTask, g *gitilesClient, repoURL *url.URL) (*refsState, error) {
	refs := &refsState{}
	refs.watched.init(cfg.GetRefs())
	return refs, parallel.FanOutIn(func(work chan<- func() error) {
		work <- func() (loadErr error) {
			refs.known, loadErr = loadState(c, ctl.JobID(), repoURL)
			return
		}
		work <- func() (refsErr error) {
			// Merge getRefsTips here.
			refs.current, refsErr = m.getRefsTips(c, ctl, g, refs.watched)
			return
		}
	})
}

// getRefsTips returns tip for each ref being watched.
func (m TaskManager) getRefsTips(c context.Context, ctl task.Controller, g *gitilesClient, watched watchedRefs) (map[string]string, error) {
	// Query gitiles for each namespace in parallel.
	var wg sync.WaitGroup
	var lock sync.Mutex
	errs := []error{}
	allTips := map[string]string{}
	// Group all refs by their namespace to reduce # of RPCs.
	for _, wrs := range watched.namespaces {
		wg.Add(1)
		go func(wrs *watchedRefNamespace) {
			defer wg.Done()
			res, err := g.Refs(c, &gitilespb.RefsRequest{
				Project:  g.project,
				RefsPath: wrs.namespace,
			})
			lock.Lock()
			defer lock.Unlock()
			if err != nil {
				ctl.DebugLog("failed to fetch %q namespace tips for %q: %q", wrs.namespace, err)
				errs = append(errs, err)
				return
			}
			for ref, tip := range res.Revisions {
				if watched.hasRef(ref) {
					allTips[ref] = tip
				}
			}
		}(wrs)
	}
	wg.Wait()
	if len(errs) > 0 {
		return nil, errors.NewMultiError(errs...)
	}
	return allTips, nil
}

// emitTriggersRefAtATime processes refs one a time and emits triggers if ref
// changed. Limits number of triggers emitted and so may stop early.
//
// Returns how many refs were not examined.
func (m TaskManager) emitTriggersRefAtATime(c context.Context, ctl task.Controller, repo string, refs *refsState) (left int) {
	maxTriggersPerInvocation := m.maxTriggersPerInvocation
	if maxTriggersPerInvocation == 0 {
		maxTriggersPerInvocation = defaultMaxTriggersPerInvocation
	}
	emittedTriggers := 0
	// Note, that refs.current contain only watched refs (see getRefsTips).
	// For determinism, sort refs by name.
	sortedRefs := make([]string, 0, len(refs.current))
	for ref := range refs.current {
		sortedRefs = append(sortedRefs, ref)
	}
	sort.Strings(sortedRefs)
	for i, ref := range sortedRefs {
		// TODO(tandrii): enforce limit on emitted triggers here.
		emittedTriggers += m.emitTriggersForRef(c, ctl, repo, ref, refs)
		// Safeguard against too many changes such as the first run after
		// config change to watch many more refs than before.
		if emittedTriggers >= maxTriggersPerInvocation {
			ctl.DebugLog("Emitted %d triggers, postponing the rest", emittedTriggers)
			left = len(sortedRefs) - i - 1
			return left
		}
	}
	left = 0
	return
}

// emitTriggersForRef emits triggers for 1 ref if there are any changes and
// returns number of triggers emitted.
func (m TaskManager) emitTriggersForRef(c context.Context, ctl task.Controller, repo, ref string, refs *refsState) int {
	newHead := refs.current[ref]
	switch oldHead, existed := refs.known[ref]; {
	case !existed:
		ctl.DebugLog("Ref %s is new: %s", ref, newHead)
	case oldHead != newHead:
		ctl.DebugLog("Ref %s updated: %s => %s", ref, oldHead, newHead)
	default:
		return 0 // no change
	}
	refs.known[ref] = newHead
	refs.changed++
	// TODO(tandrii): actually look at commits between current and previously
	// known tips of each ref.
	// In current (v1) engine, all triggers emitted around the same time will
	// result in just 1 invocation of each triggered job. Therefore,
	// passing just HEAD's revision is good enough.
	// For the same reason, only 1 of the refs will actually be processed if
	// several refs changed at the same time.
	ctl.EmitTrigger(c, &internal.Trigger{
		Id:    fmt.Sprintf("%s/+/%s@%s", repo, ref, newHead),
		Title: newHead,
		Url:   fmt.Sprintf("%s/+/%s", repo, newHead),
		Payload: &internal.Trigger_Gitiles{
			Gitiles: &api.GitilesTrigger{Repo: repo, Ref: ref, Revision: newHead},
		},
	})
	return 1
}

func (m TaskManager) getGitilesClient(c context.Context, ctl task.Controller, cfg *messages.GitilesTask) (*gitilesClient, error) {
	host, project, err := gitiles.ParseRepoURL(cfg.Repo)
	if err != nil {
		return nil, errors.Annotate(err, "invalid repo URL %q", cfg.Repo).Err()
	}
	r := &gitilesClient{host: host, project: project}

	if m.mockGitilesClient != nil {
		// Used for testing only.
		logging.Infof(c, "using mockGitilesClient")
		r.GitilesClient = m.mockGitilesClient
		return r, nil
	}

	httpClient, err := ctl.GetClient(c, time.Minute, auth.WithScopes(gitiles.OAuthScope))
	if err != nil {
		return nil, err
	}
	if r.GitilesClient, err = gitiles.NewRESTClient(httpClient, host, true); err != nil {
		return nil, err
	}
	return r, nil
}

// gitilesClient embeds GitilesClient with useful metadata.
type gitilesClient struct {
	gitilespb.GitilesClient
	host    string // Gitiles host
	project string // Gitiles project
}

type refsState struct {
	watched watchedRefs
	known   map[string]string // HEADs we saw before
	current map[string]string // HEADs available now
	changed int
}

func (s *refsState) pruneKnown(ctl task.Controller) {
	for ref := range s.known {
		switch {
		case !s.watched.hasRef(ref):
			ctl.DebugLog("Ref %s is no longer watched", ref)
			delete(s.known, ref)
			s.changed++
		case s.current[ref] == "":
			ctl.DebugLog("Ref %s deleted", ref)
			delete(s.known, ref)
			s.changed++
		}
	}
}

func (s *refsState) sortedCurrentRefNames() []string {
	sortedRefs := make([]string, 0, len(s.current))
	for ref := range s.current {
		sortedRefs = append(sortedRefs, ref)
	}
	sort.Strings(sortedRefs)
	return sortedRefs
}

type watchedRefNamespace struct {
	namespace    string // no trailing "/".
	allChildren  bool   // if true, someChildren is ignored.
	someChildren stringset.Set
}

func (w watchedRefNamespace) hasSuffix(suffix string) bool {
	switch {
	case suffix == "*":
		panic(fmt.Errorf("watchedRefNamespace membership should only be checked for refs, not ref glob %s", suffix))
	case w.allChildren:
		return true
	case w.someChildren == nil:
		return false
	default:
		return w.someChildren.Has(suffix)
	}
}

func (w *watchedRefNamespace) addSuffix(suffix string) {
	switch {
	case w.allChildren:
		return
	case suffix == "*":
		w.allChildren = true
		w.someChildren = nil
		return
	case w.someChildren == nil:
		w.someChildren = stringset.New(1)
	}
	w.someChildren.Add(suffix)
}

type watchedRefs struct {
	namespaces map[string]*watchedRefNamespace
}

func (w *watchedRefs) init(refsConfig []string) {
	w.namespaces = map[string]*watchedRefNamespace{}
	for _, ref := range refsConfig {
		ns, suffix := splitRef(ref)
		if _, exists := w.namespaces[ns]; !exists {
			w.namespaces[ns] = &watchedRefNamespace{namespace: ns}
		}
		w.namespaces[ns].addSuffix(suffix)
	}
}

func (w *watchedRefs) hasRef(ref string) bool {
	ns, suffix := splitRef(ref)
	if wrn, exists := w.namespaces[ns]; exists {
		return wrn.hasSuffix(suffix)
	}
	return false
}

func splitRef(s string) (string, string) {
	if i := strings.LastIndex(s, "/"); i <= 0 {
		return s, ""
	} else {
		return s[:i], s[i+1:]
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sdb

import (
	"github.com/jacobsa/aws/exp/sdb/mock"
	. "github.com/jacobsa/ogletest"
)

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// A common helper class.
type simpleDBTest struct {
	c *fakeConn
	db SimpleDB
}

func (t *simpleDBTest) SetUp(i *TestInfo) {
	var err error

	t.c = &fakeConn{}

	t.db, err = newSimpleDB(t.c)
	AssertEq(nil, err)
}

////////////////////////////////////////////////////////////////////////
// OpenDomain
////////////////////////////////////////////////////////////////////////

type OpenDomainTest struct {
	simpleDBTest

	name string

	domain Domain
	err error
}

func init() { RegisterTestSuite(&OpenDomainTest{}) }

func (t *OpenDomainTest) SetUp(i *TestInfo) {
	// Call common setup code.
	t.simpleDBTest.SetUp(i)

	// Make the request legal by default.
	t.name = "foo"
}

func (t *OpenDomainTest) callDB() {
	t.domain, t.err = t.db.OpenDomain(t.name)
}

func (t *OpenDomainTest) NameIsEmpty() {
	ExpectFalse(true, "TODO")
}

func (t *OpenDomainTest) NameIsInvalid() {
	ExpectFalse(true, "TODO")
}

func (t *OpenDomainTest) CallsConn() {
	ExpectFalse(true, "TODO")
}

func (t *OpenDomainTest) ConnReturnsError() {
	ExpectFalse(true, "TODO")
}

func (t *OpenDomainTest) CallsFactoryFuncAndReturnsResult() {
	ExpectFalse(true, "TODO")
}

////////////////////////////////////////////////////////////////////////
// DeleteDomain
////////////////////////////////////////////////////////////////////////

type DeleteDomainTest struct {
	simpleDBTest

	domain Domain
	err error
}

func init() { RegisterTestSuite(&DeleteDomainTest{}) }

func (t *DeleteDomainTest) SetUp(i *TestInfo) {
	// Call common setup code.
	t.simpleDBTest.SetUp(i)

	// Set up a fake named domain.
	domain := mock_sdb.NewMockDomain(i.MockController, "domain")
	ExpectCall(domain, "Name")().
		WillRepeatedly(oglemock.Return("some_domain"))

	t.domain = domain
}

func (t *DeleteDomainTest) callDB() {
	t.err = t.db.DeleteDomain(t.domain)
}

func (t *DeleteDomainTest) DoesFoo() {
	ExpectFalse(true, "TODO")
}
<commit_msg>Fixed build errors.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sdb

import (
	. "github.com/jacobsa/ogletest"
)

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// A common helper class.
type simpleDBTest struct {
	c *fakeConn
	db SimpleDB
}

func (t *simpleDBTest) SetUp(i *TestInfo) {
	var err error

	t.c = &fakeConn{}

	t.db, err = newSimpleDB(t.c)
	AssertEq(nil, err)
}

type fakeDomain struct {
	name string
}

func (d *fakeDomain) Name() string {
	return d.name
}

func (d *fakeDomain) PutAttributes(
		item ItemName,
		updates []PutUpdate,
		preconditions []Precondition) error {
	panic("Unsupported")
}

func (d *fakeDomain) BatchPutAttributes(updateMap map[ItemName][]PutUpdate) error {
	panic("Unsupported")
}

func (d *fakeDomain) DeleteAttributes(
		item ItemName,
		deletes []DeleteUpdate,
		preconditions []Precondition) error {
	panic("Unsupported")
}

func (d *fakeDomain) BatchDeleteAttributes(deleteMap map[ItemName][]DeleteUpdate) error {
	panic("Unsupported")
}

func (d *fakeDomain) GetAttributes(
		item ItemName,
		constistentRead bool,
		attrNames []string) (attrs []Attribute, err error) {
	panic("Unsupported")
}

////////////////////////////////////////////////////////////////////////
// OpenDomain
////////////////////////////////////////////////////////////////////////

type OpenDomainTest struct {
	simpleDBTest

	name string

	domain Domain
	err error
}

func init() { RegisterTestSuite(&OpenDomainTest{}) }

func (t *OpenDomainTest) SetUp(i *TestInfo) {
	// Call common setup code.
	t.simpleDBTest.SetUp(i)

	// Make the request legal by default.
	t.name = "foo"
}

func (t *OpenDomainTest) callDB() {
	t.domain, t.err = t.db.OpenDomain(t.name)
}

func (t *OpenDomainTest) NameIsEmpty() {
	ExpectFalse(true, "TODO")
}

func (t *OpenDomainTest) NameIsInvalid() {
	ExpectFalse(true, "TODO")
}

func (t *OpenDomainTest) CallsConn() {
	ExpectFalse(true, "TODO")
}

func (t *OpenDomainTest) ConnReturnsError() {
	ExpectFalse(true, "TODO")
}

func (t *OpenDomainTest) CallsFactoryFuncAndReturnsResult() {
	ExpectFalse(true, "TODO")
}

////////////////////////////////////////////////////////////////////////
// DeleteDomain
////////////////////////////////////////////////////////////////////////

type DeleteDomainTest struct {
	simpleDBTest

	domain Domain
	err error
}

func init() { RegisterTestSuite(&DeleteDomainTest{}) }

func (t *DeleteDomainTest) SetUp(i *TestInfo) {
	// Call common setup code.
	t.simpleDBTest.SetUp(i)

	// Set up a fake named domain.
	t.domain = &fakeDomain{"some_domain"}
}

func (t *DeleteDomainTest) callDB() {
	t.err = t.db.DeleteDomain(t.domain)
}

func (t *DeleteDomainTest) DoesFoo() {
	ExpectFalse(true, "TODO")
}
<|endoftext|>


<commit_before>package obj

import (
	"bytes"
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudfront/sign"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/aws/aws-sdk-go/service/storagegateway"
	"github.com/cenkalti/backoff"
	"go.pedge.io/lion"
)

type amazonClient struct {
	bucket       string
	distribution string
	s3           *s3.S3
	uploader     *s3manager.Uploader
}

func newAmazonClient(bucket string, distribution string, id string, secret string, token string, region string) (*amazonClient, error) {
	session := session.New(&aws.Config{
		Credentials: credentials.NewStaticCredentials(id, secret, token),
		Region:      aws.String(region),
	})
	return &amazonClient{
		bucket:       bucket,
		distribution: strings.TrimSpace(distribution),
		s3:           s3.New(session),
		uploader:     s3manager.NewUploader(session),
	}, nil
}

func (c *amazonClient) Writer(name string) (io.WriteCloser, error) {
	return newBackoffWriteCloser(c, newWriter(c, name)), nil
}

func (c *amazonClient) Walk(name string, fn func(name string) error) error {
	var fnErr error
	if err := c.s3.ListObjectsPages(
		&s3.ListObjectsInput{
			Bucket: aws.String(c.bucket),
			Prefix: aws.String(name),
		},
		func(listObjectsOutput *s3.ListObjectsOutput, lastPage bool) bool {
			for _, object := range listObjectsOutput.Contents {
				if err := fn(*object.Key); err != nil {
					fnErr = err
					return false
				}
			}
			return true
		},
	); err != nil {
		return err
	}
	return fnErr
}

func (c *amazonClient) Reader(name string, offset uint64, size uint64) (io.ReadCloser, error) {
	byteRange := byteRange(offset, size)
	if byteRange != "" {
		byteRange = fmt.Sprintf("bytes=%s", byteRange)
	}
	fmt.Printf("in amazon.Reader()\n")
	var reader io.ReadCloser
	if c.distribution != "" {
		var resp *http.Response
		var connErr error
		url := fmt.Sprintf("http://%v.cloudfront.net/%v", c.distribution, name)

		fmt.Println("Checking for cloudfront private key")
		rawCloudfrontPrivateKey, err := ioutil.ReadFile("/amazon-secret/cloudfrontPrivateKey")
		if err == nil {
			// If cloudfront security credentials are present, use them
			fmt.Printf("got cf private key secret: (%v)\n", string(rawCloudfrontPrivateKey))

			rawCloudfrontKeyPairId, err := ioutil.ReadFile("/amazon-secret/cloudfrontKeyPairId")
			if err != nil {
				return nil, fmt.Errorf("cloudfront private key provided, but missing cloudfront key pair id")
			}
			fmt.Printf("keypair id: %v\n", string(rawCloudfrontKeyPairId))
			/*
					fmt.Printf("got cf keypaird id (%v)\n", string(rawCloudfrontKeyPairId))
					decodedCloudfrontKeyPairId, err := base64.StdEncoding.DecodeString(string(rawCloudfrontKeyPairId))
					if err != nil {
						return nil, err
					}
					decodedCloudfrontKeyPairId = bytes.TrimSpace(decodedCloudfrontKeyPairId)
					fmt.Printf("decoded keypair id (%v)\n", string(decodedCloudfrontKeyPairId))

				decodedCloudfrontPrivateKey, err := base64.StdEncoding.DecodeString(string(rawCloudfrontPrivateKey))
				if err != nil {
					return nil, err
				}
				decodedCloudfrontPrivateKey = bytes.TrimSpace(decodedCloudfrontPrivateKey)
				fmt.Printf("decoded private key (%v)\n", string(decodedCloudfrontPrivateKey))
			*/
			//			block, _ := pem.Decode(bytes.TrimSpace(decodedCloudfrontPrivateKey))
			block, _ := pem.Decode(bytes.TrimSpace(rawCloudfrontPrivateKey))
			if block == nil || block.Type != "RSA PRIVATE KEY" {
				return nil, fmt.Errorf("block undefined or wrong type: type is (%v) should be (RSA PRIVATE KEY)", block.Type)
			}

			cloudfrontPrivateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes)
			if err != nil {
				return nil, err
			}
			signer := sign.NewURLSigner(string(rawCloudfrontKeyPairId), cloudfrontPrivateKey)
			signedURL, err := signer.Sign(url, time.Now().Add(1*time.Hour))
			fmt.Printf("orig url (%v), signed url (%v)\n", url, signedURL)
			if err != nil {
				return nil, err
			}
			url = signedURL
		}
		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			return nil, err
		}
		req.Header.Add("Range", byteRange)

		backoff.RetryNotify(func() error {
			resp, connErr = http.DefaultClient.Do(req)
			if connErr != nil && isNetRetryable(connErr) {
				return connErr
			}
			return nil
		}, backoff.NewExponentialBackOff(), func(err error, d time.Duration) {
			lion.Infof("Error connecting to (%v); retrying in %s: %#v", url, d, err)
		})
		fmt.Printf("connErr (%v), resp (%v)\n", connErr, resp)
		if connErr != nil {
			return nil, connErr
		}
		fmt.Printf("resp status code %v %v\n", resp.StatusCode, resp.Status)
		if resp.StatusCode >= 300 {
			// Cloudfront returns 200s, and 206s as success codes
			return nil, fmt.Errorf("cloudfront returned HTTP error code %v for url %v", resp.Status, url)
		}
		reader = resp.Body
	} else {
		getObjectOutput, err := c.s3.GetObject(&s3.GetObjectInput{
			Bucket: aws.String(c.bucket),
			Key:    aws.String(name),
			Range:  aws.String(byteRange),
		})
		if err != nil {
			return nil, err
		}
		reader = getObjectOutput.Body
	}
	return newBackoffReadCloser(c, reader), nil
}

func (c *amazonClient) Delete(name string) error {
	_, err := c.s3.DeleteObject(&s3.DeleteObjectInput{
		Bucket: aws.String(c.bucket),
		Key:    aws.String(name),
	})
	return err
}

func (c *amazonClient) Exists(name string) bool {
	_, err := c.s3.HeadObject(&s3.HeadObjectInput{
		Bucket: aws.String(c.bucket),
		Key:    aws.String(name),
	})
	return err == nil
}

func (c *amazonClient) isRetryable(err error) (retVal bool) {
	if strings.Contains(err.Error(), "unexpected EOF") {
		return true
	}

	awsErr, ok := err.(awserr.Error)
	if !ok {
		return false
	}
	for _, c := range []string{
		storagegateway.ErrorCodeServiceUnavailable,
		storagegateway.ErrorCodeInternalError,
		storagegateway.ErrorCodeGatewayInternalError,
	} {
		if c == awsErr.Code() {
			return true
		}
	}
	return false
}

func (c *amazonClient) IsIgnorable(err error) bool {
	return false
}

func (c *amazonClient) IsNotExist(err error) bool {
	if c.distribution != "" {
		// cloudfront returns forbidden error for nonexisting data
		if strings.Contains(err.Error(), "error code 403") {
			return true
		}
	}
	awsErr, ok := err.(awserr.Error)
	if !ok {
		return false
	}
	if awsErr.Code() == "NoSuchKey" {
		return true
	}
	return false
}

type amazonWriter struct {
	errChan chan error
	pipe    *io.PipeWriter
}

func newWriter(client *amazonClient, name string) *amazonWriter {
	reader, writer := io.Pipe()
	w := &amazonWriter{
		errChan: make(chan error),
		pipe:    writer,
	}
	go func() {
		_, err := client.uploader.Upload(&s3manager.UploadInput{
			Body:            reader,
			Bucket:          aws.String(client.bucket),
			Key:             aws.String(name),
			ContentEncoding: aws.String("application/octet-stream"),
		})
		w.errChan <- err
	}()
	return w
}

func (w *amazonWriter) Write(p []byte) (int, error) {
	return w.pipe.Write(p)
}

func (w *amazonWriter) Close() error {
	if err := w.pipe.Close(); err != nil {
		return err
	}
	return <-w.errChan
}
<commit_msg>Try trimming the space<commit_after>package obj

import (
	"bytes"
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudfront/sign"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/aws/aws-sdk-go/service/storagegateway"
	"github.com/cenkalti/backoff"
	"go.pedge.io/lion"
)

type amazonClient struct {
	bucket       string
	distribution string
	s3           *s3.S3
	uploader     *s3manager.Uploader
}

func newAmazonClient(bucket string, distribution string, id string, secret string, token string, region string) (*amazonClient, error) {
	session := session.New(&aws.Config{
		Credentials: credentials.NewStaticCredentials(id, secret, token),
		Region:      aws.String(region),
	})
	return &amazonClient{
		bucket:       bucket,
		distribution: strings.TrimSpace(distribution),
		s3:           s3.New(session),
		uploader:     s3manager.NewUploader(session),
	}, nil
}

func (c *amazonClient) Writer(name string) (io.WriteCloser, error) {
	return newBackoffWriteCloser(c, newWriter(c, name)), nil
}

func (c *amazonClient) Walk(name string, fn func(name string) error) error {
	var fnErr error
	if err := c.s3.ListObjectsPages(
		&s3.ListObjectsInput{
			Bucket: aws.String(c.bucket),
			Prefix: aws.String(name),
		},
		func(listObjectsOutput *s3.ListObjectsOutput, lastPage bool) bool {
			for _, object := range listObjectsOutput.Contents {
				if err := fn(*object.Key); err != nil {
					fnErr = err
					return false
				}
			}
			return true
		},
	); err != nil {
		return err
	}
	return fnErr
}

func (c *amazonClient) Reader(name string, offset uint64, size uint64) (io.ReadCloser, error) {
	byteRange := byteRange(offset, size)
	if byteRange != "" {
		byteRange = fmt.Sprintf("bytes=%s", byteRange)
	}
	fmt.Printf("in amazon.Reader()\n")
	var reader io.ReadCloser
	if c.distribution != "" {
		var resp *http.Response
		var connErr error
		url := fmt.Sprintf("http://%v.cloudfront.net/%v", c.distribution, name)

		fmt.Println("Checking for cloudfront private key")
		rawCloudfrontPrivateKey, err := ioutil.ReadFile("/amazon-secret/cloudfrontPrivateKey")
		if err == nil {
			// If cloudfront security credentials are present, use them
			fmt.Printf("got cf private key secret: (%v)\n", string(rawCloudfrontPrivateKey))

			rawCloudfrontKeyPairId, err := ioutil.ReadFile("/amazon-secret/cloudfrontKeyPairId")
			if err != nil {
				return nil, fmt.Errorf("cloudfront private key provided, but missing cloudfront key pair id")
			}
			fmt.Printf("keypair id: %v\n", string(rawCloudfrontKeyPairId))
			/*
					fmt.Printf("got cf keypaird id (%v)\n", string(rawCloudfrontKeyPairId))
					decodedCloudfrontKeyPairId, err := base64.StdEncoding.DecodeString(string(rawCloudfrontKeyPairId))
					if err != nil {
						return nil, err
					}
					decodedCloudfrontKeyPairId = bytes.TrimSpace(decodedCloudfrontKeyPairId)
					fmt.Printf("decoded keypair id (%v)\n", string(decodedCloudfrontKeyPairId))

				decodedCloudfrontPrivateKey, err := base64.StdEncoding.DecodeString(string(rawCloudfrontPrivateKey))
				if err != nil {
					return nil, err
				}
				decodedCloudfrontPrivateKey = bytes.TrimSpace(decodedCloudfrontPrivateKey)
				fmt.Printf("decoded private key (%v)\n", string(decodedCloudfrontPrivateKey))
			*/
			//			block, _ := pem.Decode(bytes.TrimSpace(decodedCloudfrontPrivateKey))
			block, _ := pem.Decode(bytes.TrimSpace(rawCloudfrontPrivateKey))
			if block == nil || block.Type != "RSA PRIVATE KEY" {
				return nil, fmt.Errorf("block undefined or wrong type: type is (%v) should be (RSA PRIVATE KEY)", block.Type)
			}

			cloudfrontPrivateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes)
			if err != nil {
				return nil, err
			}
			signer := sign.NewURLSigner(string(rawCloudfrontKeyPairId), cloudfrontPrivateKey)
			signedURL, err := signer.Sign(url, time.Now().Add(1*time.Hour))
			fmt.Printf("orig url (%v), signed url (%v)\n", url, signedURL)
			if err != nil {
				return nil, err
			}
			url = strings.TrimSpace(signedURL)
		}
		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			return nil, err
		}
		req.Header.Add("Range", byteRange)

		backoff.RetryNotify(func() error {
			resp, connErr = http.DefaultClient.Do(req)
			if connErr != nil && isNetRetryable(connErr) {
				return connErr
			}
			return nil
		}, backoff.NewExponentialBackOff(), func(err error, d time.Duration) {
			lion.Infof("Error connecting to (%v); retrying in %s: %#v", url, d, err)
		})
		fmt.Printf("connErr (%v), resp (%v)\n", connErr, resp)
		if connErr != nil {
			return nil, connErr
		}
		fmt.Printf("resp status code %v %v\n", resp.StatusCode, resp.Status)
		if resp.StatusCode >= 300 {
			// Cloudfront returns 200s, and 206s as success codes
			return nil, fmt.Errorf("cloudfront returned HTTP error code %v for url %v", resp.Status, url)
		}
		reader = resp.Body
	} else {
		getObjectOutput, err := c.s3.GetObject(&s3.GetObjectInput{
			Bucket: aws.String(c.bucket),
			Key:    aws.String(name),
			Range:  aws.String(byteRange),
		})
		if err != nil {
			return nil, err
		}
		reader = getObjectOutput.Body
	}
	return newBackoffReadCloser(c, reader), nil
}

func (c *amazonClient) Delete(name string) error {
	_, err := c.s3.DeleteObject(&s3.DeleteObjectInput{
		Bucket: aws.String(c.bucket),
		Key:    aws.String(name),
	})
	return err
}

func (c *amazonClient) Exists(name string) bool {
	_, err := c.s3.HeadObject(&s3.HeadObjectInput{
		Bucket: aws.String(c.bucket),
		Key:    aws.String(name),
	})
	return err == nil
}

func (c *amazonClient) isRetryable(err error) (retVal bool) {
	if strings.Contains(err.Error(), "unexpected EOF") {
		return true
	}

	awsErr, ok := err.(awserr.Error)
	if !ok {
		return false
	}
	for _, c := range []string{
		storagegateway.ErrorCodeServiceUnavailable,
		storagegateway.ErrorCodeInternalError,
		storagegateway.ErrorCodeGatewayInternalError,
	} {
		if c == awsErr.Code() {
			return true
		}
	}
	return false
}

func (c *amazonClient) IsIgnorable(err error) bool {
	return false
}

func (c *amazonClient) IsNotExist(err error) bool {
	if c.distribution != "" {
		// cloudfront returns forbidden error for nonexisting data
		if strings.Contains(err.Error(), "error code 403") {
			return true
		}
	}
	awsErr, ok := err.(awserr.Error)
	if !ok {
		return false
	}
	if awsErr.Code() == "NoSuchKey" {
		return true
	}
	return false
}

type amazonWriter struct {
	errChan chan error
	pipe    *io.PipeWriter
}

func newWriter(client *amazonClient, name string) *amazonWriter {
	reader, writer := io.Pipe()
	w := &amazonWriter{
		errChan: make(chan error),
		pipe:    writer,
	}
	go func() {
		_, err := client.uploader.Upload(&s3manager.UploadInput{
			Body:            reader,
			Bucket:          aws.String(client.bucket),
			Key:             aws.String(name),
			ContentEncoding: aws.String("application/octet-stream"),
		})
		w.errChan <- err
	}()
	return w
}

func (w *amazonWriter) Write(p []byte) (int, error) {
	return w.pipe.Write(p)
}

func (w *amazonWriter) Close() error {
	if err := w.pipe.Close(); err != nil {
		return err
	}
	return <-w.errChan
}
<|endoftext|>


<commit_before>package worker

import (
	"fmt"
	"io"
	"sort"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pps"
)

// DatumIterator is an interface which allows you to iterate through the datums
// for a job. A datum iterator keeps track of which datum it is on, which can be Reset()
// The intended use is by using this pattern `for di.Next() { ... datum := di.Datum() ... }`
// Note that since you start the loop by a call to Next(), the datum iterator's location starts at -1
type DatumIterator interface {
	Reset()
	Len() int
	Next() bool
	Datum() []*Input
	DatumN(int) []*Input
}

type pfsDatumIterator struct {
	inputs   []*Input
	location int
}

func newPFSDatumIterator(pachClient *client.APIClient, input *pps.PFSInput) (DatumIterator, error) {
	result := &pfsDatumIterator{}
	// make sure it gets initialized properly
	result.Reset()
	if input.Commit == "" {
		// this can happen if a pipeline with multiple inputs has been triggered
		// before all commits have inputs
		return result, nil
	}
	fs, err := pachClient.GlobFileStream(pachClient.Ctx(), &pfs.GlobFileRequest{
		Commit:  client.NewCommit(input.Repo, input.Commit),
		Pattern: input.Glob,
	})
	if err != nil {
		return nil, err
	}
	for {
		fileInfo, err := fs.Recv()
		if err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}
		result.inputs = append(result.inputs, &Input{
			FileInfo:   fileInfo,
			Name:       input.Name,
			Lazy:       input.Lazy,
			Branch:     input.Branch,
			EmptyFiles: input.EmptyFiles,
		})
	}
	// We sort the inputs so that the order is deterministic. Note that it's
	// not possible for 2 inputs to have the same path so this is guaranteed to
	// produce a deterministic order.
	sort.Slice(result.inputs, func(i, j int) bool {
		// We sort by descending size first because it can boost performance to
		// process the biggest datums first.
		if result.inputs[i].FileInfo.SizeBytes != result.inputs[j].FileInfo.SizeBytes {
			return result.inputs[i].FileInfo.SizeBytes > result.inputs[j].FileInfo.SizeBytes
		}
		return result.inputs[i].FileInfo.File.Path < result.inputs[j].FileInfo.File.Path
	})
	return result, nil
}

func (d *pfsDatumIterator) Reset() {
	d.location = -1
}

func (d *pfsDatumIterator) Len() int {
	return len(d.inputs)
}

func (d *pfsDatumIterator) Datum() []*Input {
	return []*Input{d.inputs[d.location]}
}

func (d *pfsDatumIterator) DatumN(n int) []*Input {
	if n > d.location {
		d.Reset()
	}
	for d.location != n {
		d.Next()
	}
	return d.Datum()
}

func (d *pfsDatumIterator) Next() bool {
	d.location++
	return d.location < len(d.inputs)
}

type unionDatumIterator struct {
	iterators []DatumIterator
	unionIdx  int
	location  int
}

func newUnionDatumIterator(pachClient *client.APIClient, union []*pps.Input) (DatumIterator, error) {
	result := &unionDatumIterator{}
	defer result.Reset()
	for _, input := range union {
		datumIterator, err := NewDatumIterator(pachClient, input)
		if err != nil {
			return nil, err
		}
		result.iterators = append(result.iterators, datumIterator)
	}
	return result, nil
}

func (d *unionDatumIterator) Reset() {
	for _, input := range d.iterators {
		input.Reset()
	}
	d.unionIdx = 0
	d.location = -1
}

func (d *unionDatumIterator) Len() int {
	result := 0
	for _, datumIterator := range d.iterators {
		result += datumIterator.Len()
	}
	return result
}

func (d *unionDatumIterator) Next() bool {
	d.location++
	if d.unionIdx >= len(d.iterators) {
		return false
	}
	if !d.iterators[d.unionIdx].Next() {
		d.unionIdx++
		return d.Next()
	}
	return true
}

func (d *unionDatumIterator) Datum() []*Input {
	return d.iterators[d.unionIdx].Datum()
}

func (d *unionDatumIterator) DatumN(n int) []*Input {
	if n > d.location {
		d.Reset()
	}
	for d.location != n {
		d.Next()
	}
	return d.Datum()
}

type crossDatumIterator struct {
	iterators []DatumIterator
	started   bool
	location  int
}

func newCrossDatumIterator(pachClient *client.APIClient, cross []*pps.Input) (DatumIterator, error) {
	result := &crossDatumIterator{}
	defer result.Reset()
	for _, iterator := range cross {
		datumIterator, err := NewDatumIterator(pachClient, iterator)
		if err != nil {
			return nil, err
		}
		result.iterators = append(result.iterators, datumIterator)
	}
	result.location = -1
	return result, nil
}

func (d *crossDatumIterator) Reset() {
	inhabited := len(d.iterators) > 0
	for _, iterators := range d.iterators {
		iterators.Reset()
		if !iterators.Next() {
			inhabited = false
		}
	}
	if !inhabited {
		d.iterators = nil
	}
	d.started = !inhabited
}

func (d *crossDatumIterator) Len() int {
	if len(d.iterators) == 0 {
		return 0
	}
	result := d.iterators[0].Len()
	for i := 1; i < len(d.iterators); i++ {
		result *= d.iterators[i].Len()
	}
	return result
}

func (d *crossDatumIterator) Next() bool {
	d.location++
	if !d.started {
		d.started = true
		return true
	}
	for _, input := range d.iterators {
		// if we're at the end of the "row"
		if !input.Next() {
			// we reset the "row"
			input.Reset()
			// and start it back up
			input.Next()
			// after resetting this "row", start iterating through the next "row"
		} else {
			return true
		}
	}
	return false
}

func (d *crossDatumIterator) Datum() []*Input {
	var result []*Input
	for _, datumIterator := range d.iterators {
		result = append(result, datumIterator.Datum()...)
	}
	sortInputs(result)
	return result
}

func (d *crossDatumIterator) DatumN(n int) []*Input {
	if n > d.location {
		d.Reset()
	}
	for d.location != n {
		d.Next()
	}
	return d.Datum()
}

type joinDatumIterator struct {
	datums   [][]*Input
	location int
}

func newJoinDatumIterator(pachClient *client.APIClient, join []*pps.Input) (DatumIterator, error) {
	cross, err := newCrossDatumIterator(pachClient, join)
	if err != nil {
		return nil, err
	}
	result := &joinDatumIterator{}
	for cross.Next() {
		tuple := cross.Datum()
		count := make(map[string]int)
		for _, input := range tuple {
			count[input.MatchOn]++
		}
		if len(count) == 1 {
			result.datums = append(result.datums, tuple)
		}
	}
	result.location = -1
	return result, nil
}

func (d *joinDatumIterator) Reset() {
	d.location = -1
}

func (d *joinDatumIterator) Len() int {
	return len(d.datums)
}

func (d *joinDatumIterator) Next() bool {
	d.location++
	return d.location < len(d.datums)
}

func (d *joinDatumIterator) Datum() []*Input {
	var result []*Input
	for _, datum := range d.datums[d.location] {
		result = append(result, datum)
	}
	sortInputs(result)
	return result
}

func (d *joinDatumIterator) DatumN(n int) []*Input {
	if n > d.location {
		d.Reset()
	}
	for d.location != n {
		d.Next()
	}
	return d.Datum()
}

type gitDatumIterator struct {
	inputs   []*Input
	location int
}

func newGitDatumIterator(pachClient *client.APIClient, input *pps.GitInput) (DatumIterator, error) {
	result := &gitDatumIterator{}
	defer result.Reset()
	if input.Commit == "" {
		// this can happen if a pipeline with multiple inputs has been triggered
		// before all commits have inputs
		return result, nil
	}
	fileInfo, err := pachClient.InspectFile(input.Name, input.Commit, "/commit.json")
	if err != nil {
		return nil, err
	}
	result.inputs = append(
		result.inputs,
		&Input{
			FileInfo: fileInfo,
			Name:     input.Name,
			Branch:   input.Branch,
			GitURL:   input.URL,
		},
	)
	return result, nil
}

func (d *gitDatumIterator) Reset() {
	d.location = -1
}

func (d *gitDatumIterator) Len() int {
	return len(d.inputs)
}

func (d *gitDatumIterator) Datum() []*Input {
	return []*Input{d.inputs[d.location]}
}

func (d *gitDatumIterator) Next() bool {
	d.location++
	return d.location < len(d.inputs)
}

func (d *gitDatumIterator) DatumN(n int) []*Input {
	if n > d.location {
		d.Reset()
	}
	for d.location != n {
		d.Next()
	}
	return d.Datum()
}

func newCronDatumIterator(pachClient *client.APIClient, input *pps.CronInput) (DatumIterator, error) {
	return newPFSDatumIterator(pachClient, &pps.PFSInput{
		Name:   input.Name,
		Repo:   input.Repo,
		Branch: "master",
		Commit: input.Commit,
		Glob:   "/*",
	})
}

// NewDatumIterator creates a datumIterator for an input.
func NewDatumIterator(pachClient *client.APIClient, input *pps.Input) (DatumIterator, error) {
	switch {
	case input.Pfs != nil:
		return newPFSDatumIterator(pachClient, input.Pfs)
	case input.Union != nil:
		return newUnionDatumIterator(pachClient, input.Union)
	case input.Cross != nil:
		return newCrossDatumIterator(pachClient, input.Cross)
	case input.Cron != nil:
		return newCronDatumIterator(pachClient, input.Cron)
	case input.Git != nil:
		return newGitDatumIterator(pachClient, input.Git)
	}
	return nil, fmt.Errorf("unrecognized input type")
}

func sortInputs(inputs []*Input) {
	sort.Slice(inputs, func(i, j int) bool {
		return inputs[i].Name < inputs[j].Name
	})
}
<commit_msg>replace glob on joins<commit_after>package worker

import (
	"fmt"
	"io"
	"sort"

	"github.com/pachyderm/glob"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pps"
)

// DatumIterator is an interface which allows you to iterate through the datums
// for a job. A datum iterator keeps track of which datum it is on, which can be Reset()
// The intended use is by using this pattern `for di.Next() { ... datum := di.Datum() ... }`
// Note that since you start the loop by a call to Next(), the datum iterator's location starts at -1
type DatumIterator interface {
	Reset()
	Len() int
	Next() bool
	Datum() []*Input
	DatumN(int) []*Input
}

type pfsDatumIterator struct {
	inputs   []*Input
	location int
}

func newPFSDatumIterator(pachClient *client.APIClient, input *pps.PFSInput) (DatumIterator, error) {
	result := &pfsDatumIterator{}
	// make sure it gets initialized properly
	result.Reset()
	if input.Commit == "" {
		// this can happen if a pipeline with multiple inputs has been triggered
		// before all commits have inputs
		return result, nil
	}
	fs, err := pachClient.GlobFileStream(pachClient.Ctx(), &pfs.GlobFileRequest{
		Commit:  client.NewCommit(input.Repo, input.Commit),
		Pattern: input.Glob,
	})
	if err != nil {
		return nil, err
	}
	for {
		fileInfo, err := fs.Recv()
		if err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}
		g := glob.MustCompile(input.Glob, '/')
		matchOn := g.Replace(fileInfo.File.Path, input.MatchOn)
		result.inputs = append(result.inputs, &Input{
			FileInfo:   fileInfo,
			MatchOn:    matchOn,
			Name:       input.Name,
			Lazy:       input.Lazy,
			Branch:     input.Branch,
			EmptyFiles: input.EmptyFiles,
		})
	}
	// We sort the inputs so that the order is deterministic. Note that it's
	// not possible for 2 inputs to have the same path so this is guaranteed to
	// produce a deterministic order.
	sort.Slice(result.inputs, func(i, j int) bool {
		// We sort by descending size first because it can boost performance to
		// process the biggest datums first.
		if result.inputs[i].FileInfo.SizeBytes != result.inputs[j].FileInfo.SizeBytes {
			return result.inputs[i].FileInfo.SizeBytes > result.inputs[j].FileInfo.SizeBytes
		}
		return result.inputs[i].FileInfo.File.Path < result.inputs[j].FileInfo.File.Path
	})
	return result, nil
}

func (d *pfsDatumIterator) Reset() {
	d.location = -1
}

func (d *pfsDatumIterator) Len() int {
	return len(d.inputs)
}

func (d *pfsDatumIterator) Datum() []*Input {
	return []*Input{d.inputs[d.location]}
}

func (d *pfsDatumIterator) DatumN(n int) []*Input {
	if n > d.location {
		d.Reset()
	}
	for d.location != n {
		d.Next()
	}
	return d.Datum()
}

func (d *pfsDatumIterator) Next() bool {
	d.location++
	return d.location < len(d.inputs)
}

type unionDatumIterator struct {
	iterators []DatumIterator
	unionIdx  int
	location  int
}

func newUnionDatumIterator(pachClient *client.APIClient, union []*pps.Input) (DatumIterator, error) {
	result := &unionDatumIterator{}
	defer result.Reset()
	for _, input := range union {
		datumIterator, err := NewDatumIterator(pachClient, input)
		if err != nil {
			return nil, err
		}
		result.iterators = append(result.iterators, datumIterator)
	}
	return result, nil
}

func (d *unionDatumIterator) Reset() {
	for _, input := range d.iterators {
		input.Reset()
	}
	d.unionIdx = 0
	d.location = -1
}

func (d *unionDatumIterator) Len() int {
	result := 0
	for _, datumIterator := range d.iterators {
		result += datumIterator.Len()
	}
	return result
}

func (d *unionDatumIterator) Next() bool {
	d.location++
	if d.unionIdx >= len(d.iterators) {
		return false
	}
	if !d.iterators[d.unionIdx].Next() {
		d.unionIdx++
		return d.Next()
	}
	return true
}

func (d *unionDatumIterator) Datum() []*Input {
	return d.iterators[d.unionIdx].Datum()
}

func (d *unionDatumIterator) DatumN(n int) []*Input {
	if n > d.location {
		d.Reset()
	}
	for d.location != n {
		d.Next()
	}
	return d.Datum()
}

type crossDatumIterator struct {
	iterators []DatumIterator
	started   bool
	location  int
}

func newCrossDatumIterator(pachClient *client.APIClient, cross []*pps.Input) (DatumIterator, error) {
	result := &crossDatumIterator{}
	defer result.Reset()
	for _, iterator := range cross {
		datumIterator, err := NewDatumIterator(pachClient, iterator)
		if err != nil {
			return nil, err
		}
		result.iterators = append(result.iterators, datumIterator)
	}
	result.location = -1
	return result, nil
}

func (d *crossDatumIterator) Reset() {
	inhabited := len(d.iterators) > 0
	for _, iterators := range d.iterators {
		iterators.Reset()
		if !iterators.Next() {
			inhabited = false
		}
	}
	if !inhabited {
		d.iterators = nil
	}
	d.started = !inhabited
}

func (d *crossDatumIterator) Len() int {
	if len(d.iterators) == 0 {
		return 0
	}
	result := d.iterators[0].Len()
	for i := 1; i < len(d.iterators); i++ {
		result *= d.iterators[i].Len()
	}
	return result
}

func (d *crossDatumIterator) Next() bool {
	d.location++
	if !d.started {
		d.started = true
		return true
	}
	for _, input := range d.iterators {
		// if we're at the end of the "row"
		if !input.Next() {
			// we reset the "row"
			input.Reset()
			// and start it back up
			input.Next()
			// after resetting this "row", start iterating through the next "row"
		} else {
			return true
		}
	}
	return false
}

func (d *crossDatumIterator) Datum() []*Input {
	var result []*Input
	for _, datumIterator := range d.iterators {
		result = append(result, datumIterator.Datum()...)
	}
	sortInputs(result)
	return result
}

func (d *crossDatumIterator) DatumN(n int) []*Input {
	if n > d.location {
		d.Reset()
	}
	for d.location != n {
		d.Next()
	}
	return d.Datum()
}

type joinDatumIterator struct {
	datums   [][]*Input
	location int
}

func newJoinDatumIterator(pachClient *client.APIClient, join []*pps.Input) (DatumIterator, error) {
	cross, err := newCrossDatumIterator(pachClient, join)
	if err != nil {
		return nil, err
	}
	result := &joinDatumIterator{}
	for cross.Next() {
		tuple := cross.Datum()
		count := make(map[string]int)
		for _, input := range tuple {
			count[input.MatchOn]++
		}
		if len(count) == 1 {
			result.datums = append(result.datums, tuple)
		}
	}
	result.location = -1
	return result, nil
}

func (d *joinDatumIterator) Reset() {
	d.location = -1
}

func (d *joinDatumIterator) Len() int {
	return len(d.datums)
}

func (d *joinDatumIterator) Next() bool {
	d.location++
	return d.location < len(d.datums)
}

func (d *joinDatumIterator) Datum() []*Input {
	var result []*Input
	for _, datum := range d.datums[d.location] {
		result = append(result, datum)
	}
	sortInputs(result)
	return result
}

func (d *joinDatumIterator) DatumN(n int) []*Input {
	if n > d.location {
		d.Reset()
	}
	for d.location != n {
		d.Next()
	}
	return d.Datum()
}

type gitDatumIterator struct {
	inputs   []*Input
	location int
}

func newGitDatumIterator(pachClient *client.APIClient, input *pps.GitInput) (DatumIterator, error) {
	result := &gitDatumIterator{}
	defer result.Reset()
	if input.Commit == "" {
		// this can happen if a pipeline with multiple inputs has been triggered
		// before all commits have inputs
		return result, nil
	}
	fileInfo, err := pachClient.InspectFile(input.Name, input.Commit, "/commit.json")
	if err != nil {
		return nil, err
	}
	result.inputs = append(
		result.inputs,
		&Input{
			FileInfo: fileInfo,
			Name:     input.Name,
			Branch:   input.Branch,
			GitURL:   input.URL,
		},
	)
	return result, nil
}

func (d *gitDatumIterator) Reset() {
	d.location = -1
}

func (d *gitDatumIterator) Len() int {
	return len(d.inputs)
}

func (d *gitDatumIterator) Datum() []*Input {
	return []*Input{d.inputs[d.location]}
}

func (d *gitDatumIterator) Next() bool {
	d.location++
	return d.location < len(d.inputs)
}

func (d *gitDatumIterator) DatumN(n int) []*Input {
	if n > d.location {
		d.Reset()
	}
	for d.location != n {
		d.Next()
	}
	return d.Datum()
}

func newCronDatumIterator(pachClient *client.APIClient, input *pps.CronInput) (DatumIterator, error) {
	return newPFSDatumIterator(pachClient, &pps.PFSInput{
		Name:   input.Name,
		Repo:   input.Repo,
		Branch: "master",
		Commit: input.Commit,
		Glob:   "/*",
	})
}

// NewDatumIterator creates a datumIterator for an input.
func NewDatumIterator(pachClient *client.APIClient, input *pps.Input) (DatumIterator, error) {
	switch {
	case input.Pfs != nil:
		return newPFSDatumIterator(pachClient, input.Pfs)
	case input.Union != nil:
		return newUnionDatumIterator(pachClient, input.Union)
	case input.Cross != nil:
		return newCrossDatumIterator(pachClient, input.Cross)
	case input.Cron != nil:
		return newCronDatumIterator(pachClient, input.Cron)
	case input.Git != nil:
		return newGitDatumIterator(pachClient, input.Git)
	}
	return nil, fmt.Errorf("unrecognized input type")
}

func sortInputs(inputs []*Input) {
	sort.Slice(inputs, func(i, j int) bool {
		return inputs[i].Name < inputs[j].Name
	})
}
<|endoftext|>


<commit_before><commit_msg>Added another TODO<commit_after><|endoftext|>


<commit_before>package render

import (
	"errors"
	"image"
	"image/draw"

	"github.com/oakmound/oak/dlog"
)

var (
	// GlobalDrawStack is the stack that all draw calls are parsed through.
	GlobalDrawStack = &DrawStack{
		as: []Addable{NewHeap(false)},
	}
	initialDrawStack = GlobalDrawStack
)

//The DrawStack is a stack with a safe adding mechanism that creates isolation between draw steps via predraw
type DrawStack struct {
	as     []Addable
	toPush []Addable
	toPop  int
}

// An Addable manages Renderables
type Addable interface {
	PreDraw()
	Add(Renderable, int) Renderable
	Replace(Renderable, Renderable, int)
	Copy() Addable
	draw(draw.Image, image.Point, int, int)
}

//SetDrawStack takes in a set of Addables which act as the set of Drawstacks available
func SetDrawStack(as ...Addable) {
	GlobalDrawStack = &DrawStack{as: as}
	initialDrawStack = GlobalDrawStack.Copy()
	dlog.Info("Global", GlobalDrawStack)
	dlog.Info("Initial", initialDrawStack)
}

//ResetDrawStack resets the Global stack back to the initial stack
func ResetDrawStack() {
	GlobalDrawStack = initialDrawStack.Copy()
}

//Draw actively draws the onto the actual screen
func (ds *DrawStack) Draw(world draw.Image, view image.Point, w, h int) {

	for _, a := range ds.as {
		// If we had concurrent operations, we'd do it here
		// in that case each draw call would return to us something
		// to composite onto the window / world
		a.draw(world, view, w, h)
	}
}

//Draw accesses the global draw stack
func Draw(r Renderable, l int) (Renderable, error) {
	if r == nil {
		dlog.Error("Tried to draw nil")
		return nil, errors.New("Tried to draw nil")
	}
	// If there's only one element, l refers to the layer
	// within that element.
	if len(GlobalDrawStack.as) == 1 {
		return GlobalDrawStack.as[0].Add(r, l), nil

		// Otherwise, l refers to the index within the DrawStack.
	}
	if l < 0 || l >= len(GlobalDrawStack.as) {
		dlog.Error("Layer", l, "does not exist on global draw stack")
		return nil, errors.New("Layer does not exist on stack")
	}
	return GlobalDrawStack.as[l].Add(r, r.GetLayer()), nil
}

// ReplaceDraw will undraw r1 and draw r2 after the next draw frame
// Useful for not working
func ReplaceDraw(r1, r2 Renderable, stackLayer, layer int) {
	if r1 == nil || r2 == nil {
		dlog.Error("Tried to draw nil")
		return
	}
	if stackLayer < 0 || stackLayer >= len(GlobalDrawStack.as) {
		dlog.Error("Layer", stackLayer, "does not exist on global draw stack")
		return
	}
	r2.SetLayer(layer)
	GlobalDrawStack.as[stackLayer].Replace(r1, r2, layer)
}

//Push appends an addable to the draw stack during the next predraw
func (ds *DrawStack) Push(a Addable) {
	ds.toPush = append(ds.toPush, a)

}

//Pop increments the pop counter
func (ds *DrawStack) Pop() {
	ds.toPop++
}

//PreDraw takes care of popping and pushing onto the stack. This helps safegaurd against operations taking place in the middle of a draw
func (ds *DrawStack) PreDraw() {
	if ds.toPop > 0 {
		ds.as = ds.as[0 : len(ds.as)-ds.toPop]
		ds.toPop = 0
	}
	if len(ds.toPush) > 0 {
		ds.as = append(ds.as, ds.toPush...)
		// Should use two toPush lists, for this and
		// draw heaps, so this call won't ever drop anything
		ds.toPush = []Addable{}
	}
	for _, a := range ds.as {
		a.PreDraw()
	}
}

//Copy creates a new deep copy of a Drawstack
func (ds *DrawStack) Copy() *DrawStack {
	ds2 := new(DrawStack)
	ds2.as = make([]Addable, len(ds.as))
	for i, a := range ds.as {
		ds2.as[i] = a.Copy()
	}
	ds2.toPop = ds.toPop
	ds2.toPush = ds.toPush
	return ds2
}

//PreDraw tries to reset the GlobalDrawStack or performs the GlobalDrawStack's predraw functions
func PreDraw() {
	if resetDraw {
		ResetDrawStack()
		resetDraw = false
	} else {
		GlobalDrawStack.PreDraw()
	}
}
<commit_msg>Small doc changes<commit_after>package render

import (
	"errors"
	"image"
	"image/draw"

	"github.com/oakmound/oak/dlog"
)

var (
	// GlobalDrawStack is the stack that all draw calls are parsed through.
	GlobalDrawStack = &DrawStack{
		as: []Addable{NewHeap(false)},
	}
	initialDrawStack = GlobalDrawStack
)

//The DrawStack is a stack with a safe adding mechanism that creates isolation between draw steps via predraw
type DrawStack struct {
	as     []Addable
	toPush []Addable
	toPop  int
}

// An Addable manages Renderables
type Addable interface {
	PreDraw()
	Add(Renderable, int) Renderable
	Replace(Renderable, Renderable, int)
	Copy() Addable
	draw(draw.Image, image.Point, int, int)
}

//SetDrawStack takes in a set of Addables which act as the set of Drawstacks available
func SetDrawStack(as ...Addable) {
	GlobalDrawStack = &DrawStack{as: as}
	initialDrawStack = GlobalDrawStack.Copy()
	dlog.Info("Global draw stack", GlobalDrawStack)
	dlog.Info("Initial draw stack", initialDrawStack)
}

//ResetDrawStack resets the Global stack back to the initial stack
func ResetDrawStack() {
	GlobalDrawStack = initialDrawStack.Copy()
}

//Draw actively draws the onto the actual screen
func (ds *DrawStack) Draw(world draw.Image, view image.Point, w, h int) {
	for _, a := range ds.as {
		// If we had concurrent operations, we'd do it here
		// in that case each draw call would return to us something
		// to composite onto the window / world
		a.draw(world, view, w, h)
	}
}

//Draw accesses the global draw stack
func Draw(r Renderable, l int) (Renderable, error) {
	if r == nil {
		dlog.Error("Tried to draw nil")
		return nil, errors.New("Tried to draw nil")
	}
	// If there's only one element, l refers to the layer
	// within that element.
	if len(GlobalDrawStack.as) == 1 {
		return GlobalDrawStack.as[0].Add(r, l), nil

		// Otherwise, l refers to the index within the DrawStack.
	}
	if l < 0 || l >= len(GlobalDrawStack.as) {
		dlog.Error("Layer", l, "does not exist on global draw stack")
		return nil, errors.New("Layer does not exist on stack")
	}
	return GlobalDrawStack.as[l].Add(r, r.GetLayer()), nil
}

// ReplaceDraw will undraw r1 and draw r2 after the next draw frame
// Useful for not working
func ReplaceDraw(r1, r2 Renderable, stackLayer, layer int) {
	if r1 == nil || r2 == nil {
		dlog.Error("Tried to draw nil")
		return
	}
	if stackLayer < 0 || stackLayer >= len(GlobalDrawStack.as) {
		dlog.Error("Layer", stackLayer, "does not exist on global draw stack")
		return
	}
	r2.SetLayer(layer)
	GlobalDrawStack.as[stackLayer].Replace(r1, r2, layer)
}

// Push appends an addable to the draw stack during the next PreDraw.
func (ds *DrawStack) Push(a Addable) {
	ds.toPush = append(ds.toPush, a)

}

// Pop pops an element from the stack at the next PreDraw call.
func (ds *DrawStack) Pop() {
	ds.toPop++
}

// PreDraw performs whatever processes need to occur before this can be
// drawn. In the case of the stack, it enacts previous Push and Pop calls,
// and signals to elements on the stack to also prepare to be drawn.
func (ds *DrawStack) PreDraw() {
	if ds.toPop > 0 {
		ds.as = ds.as[0 : len(ds.as)-ds.toPop]
		ds.toPop = 0
	}
	if len(ds.toPush) > 0 {
		ds.as = append(ds.as, ds.toPush...)
		// Should use two toPush lists, for this and
		// draw heaps, so this call won't ever drop anything
		ds.toPush = []Addable{}
	}
	for _, a := range ds.as {
		a.PreDraw()
	}
}

// Copy creates a new deep copy of a Drawstack
func (ds *DrawStack) Copy() *DrawStack {
	ds2 := new(DrawStack)
	ds2.as = make([]Addable, len(ds.as))
	for i, a := range ds.as {
		ds2.as[i] = a.Copy()
	}
	ds2.toPop = ds.toPop
	ds2.toPush = ds.toPush
	return ds2
}

// PreDraw tries to reset the GlobalDrawStack or performs the GlobalDrawStack's predraw functions
func PreDraw() {
	if resetDraw {
		ResetDrawStack()
		resetDraw = false
	} else {
		GlobalDrawStack.PreDraw()
	}
}
<|endoftext|>


<commit_before>package aws

import (
	"bytes"
	"fmt"
	"log"

	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/mitchellh/goamz/elb"
)

func resourceAwsElb() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsElbCreate,
		Read:   resourceAwsElbRead,
		Update: resourceAwsElbUpdate,
		Delete: resourceAwsElbDelete,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"internal": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				ForceNew: true,
				Computed: true,
			},

			"availability_zones": &schema.Schema{
				Type:     schema.TypeList,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Optional: true,
				ForceNew: true,
			},

			"instances": &schema.Schema{
				Type:     schema.TypeSet,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Optional: true,
				Set: func(v interface{}) int {
					return hashcode.String(v.(string))
				},
			},

			// TODO: could be not ForceNew
			"security_groups": &schema.Schema{
				Type:     schema.TypeList,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Optional: true,
				ForceNew: true,
				Computed: true,
			},

			// TODO: could be not ForceNew
			"subnets": &schema.Schema{
				Type:     schema.TypeList,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Optional: true,
				ForceNew: true,
				Computed: true,
			},

			// TODO: could be not ForceNew
			"listener": &schema.Schema{
				Type:     schema.TypeSet,
				Required: true,
				ForceNew: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"instance_port": &schema.Schema{
							Type:     schema.TypeInt,
							Required: true,
						},

						"instance_protocol": &schema.Schema{
							Type:     schema.TypeString,
							Required: true,
						},

						"lb_port": &schema.Schema{
							Type:     schema.TypeInt,
							Required: true,
						},

						"lb_protocol": &schema.Schema{
							Type:     schema.TypeString,
							Required: true,
						},

						"ssl_certificate_id": &schema.Schema{
							Type:     schema.TypeString,
							Optional: true,
						},
					},
				},
				Set: resourceAwsElbListenerHash,
			},

			// TODO: could be not ForceNew
			"health_check": &schema.Schema{
				Type:     schema.TypeSet,
				Optional: true,
				ForceNew: true,
				Computed: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"healthy_threshold": &schema.Schema{
							Type:     schema.TypeInt,
							Required: true,
						},

						"unhealthy_threshold": &schema.Schema{
							Type:     schema.TypeInt,
							Required: true,
						},

						"target": &schema.Schema{
							Type:     schema.TypeString,
							Required: true,
						},

						"interval": &schema.Schema{
							Type:     schema.TypeInt,
							Required: true,
						},

						"timeout": &schema.Schema{
							Type:     schema.TypeInt,
							Required: true,
						},
					},
				},
				Set: resourceAwsElbHealthCheckHash,
			},

			"dns_name": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceAwsElbHealthCheckHash(v interface{}) int {
	var buf bytes.Buffer
	m := v.(map[string]interface{})
	buf.WriteString(fmt.Sprintf("%d-", m["healthy_threshold"].(int)))
	buf.WriteString(fmt.Sprintf("%d-", m["unhealthy_threshold"].(int)))
	buf.WriteString(fmt.Sprintf("%s-", m["target"].(string)))
	buf.WriteString(fmt.Sprintf("%d-", m["interval"].(int)))
	buf.WriteString(fmt.Sprintf("%d-", m["timeout"].(int)))

	return hashcode.String(buf.String())
}

func resourceAwsElbListenerHash(v interface{}) int {
	var buf bytes.Buffer
	m := v.(map[string]interface{})
	buf.WriteString(fmt.Sprintf("%d-", m["instance_port"].(int)))
	buf.WriteString(fmt.Sprintf("%s-", m["instance_protocol"].(string)))
	buf.WriteString(fmt.Sprintf("%d-", m["lb_port"].(int)))
	buf.WriteString(fmt.Sprintf("%s-", m["lb_protocol"].(string)))

	if v, ok := m["ssl_certificate_id"]; ok {
		buf.WriteString(fmt.Sprintf("%s-", v.(string)))
	}

	return hashcode.String(buf.String())
}

func resourceAwsElbCreate(d *schema.ResourceData, meta interface{}) error {
	p := meta.(*ResourceProvider)
	elbconn := p.elbconn

	// Expand the "listener" set to goamz compat []elb.Listener
	listeners, err := expandListeners(d.Get("listener").(*schema.Set).List())
	if err != nil {
		return err
	}

	// Provision the elb
	elbOpts := &elb.CreateLoadBalancer{
		LoadBalancerName: d.Get("name").(string),
		Listeners:        listeners,
		Internal:         d.Get("internal").(bool),
	}

	if v, ok := d.GetOk("availability_zones"); ok {
		elbOpts.AvailZone = expandStringList(v.([]interface{}))
	}

	if v, ok := d.GetOk("security_groups"); ok {
		elbOpts.SecurityGroups = expandStringList(v.([]interface{}))
	}

	if v, ok := d.GetOk("subnets"); ok {
		elbOpts.Subnets = expandStringList(v.([]interface{}))
	}

	log.Printf("[DEBUG] ELB create configuration: %#v", elbOpts)
	if _, err := elbconn.CreateLoadBalancer(elbOpts); err != nil {
		return fmt.Errorf("Error creating ELB: %s", err)
	}

	// Assign the elb's unique identifier for use later
	d.SetId(d.Get("name").(string))
	log.Printf("[INFO] ELB ID: %s", d.Id())

	// Enable partial mode and record what we set
	d.Partial(true)
	d.SetPartial("name")
	d.SetPartial("internal")
	d.SetPartial("availability_zones")
	d.SetPartial("security_groups")
	d.SetPartial("subnets")

	if d.HasChange("health_check") {
		vs := d.Get("health_check").(*schema.Set).List()
		if len(vs) > 0 {
			check := vs[0].(map[string]interface{})

			configureHealthCheckOpts := elb.ConfigureHealthCheck{
				LoadBalancerName: d.Id(),
				Check: elb.HealthCheck{
					HealthyThreshold:   int64(check["healthy_threshold"].(int)),
					UnhealthyThreshold: int64(check["unhealthy_threshold"].(int)),
					Interval:           int64(check["interval"].(int)),
					Target:             check["target"].(string),
					Timeout:            int64(check["timeout"].(int)),
				},
			}

			_, err = elbconn.ConfigureHealthCheck(&configureHealthCheckOpts)
			if err != nil {
				return fmt.Errorf("Failure configuring health check: %s", err)
			}
		}
	}

	return resourceAwsElbUpdate(d, meta)
}

func resourceAwsElbUpdate(d *schema.ResourceData, meta interface{}) error {
	p := meta.(*ResourceProvider)
	elbconn := p.elbconn

	d.Partial(true)

	// If we currently have instances, or did have instances,
	// we want to figure out what to add and remove from the load
	// balancer
	if d.HasChange("instances") {
		o, n := d.GetChange("instances")
		os := o.(*schema.Set)
		ns := n.(*schema.Set)
		remove := expandStringList(os.Difference(ns).List())
		add := expandStringList(ns.Difference(os).List())

		if len(add) > 0 {
			registerInstancesOpts := elb.RegisterInstancesWithLoadBalancer{
				LoadBalancerName: d.Id(),
				Instances:        add,
			}

			_, err := elbconn.RegisterInstancesWithLoadBalancer(&registerInstancesOpts)
			if err != nil {
				return fmt.Errorf("Failure registering instances: %s", err)
			}
		}
		if len(remove) > 0 {
			deRegisterInstancesOpts := elb.DeregisterInstancesFromLoadBalancer{
				LoadBalancerName: d.Id(),
				Instances:        remove,
			}

			_, err := elbconn.DeregisterInstancesFromLoadBalancer(&deRegisterInstancesOpts)
			if err != nil {
				return fmt.Errorf("Failure deregistering instances: %s", err)
			}
		}

		d.SetPartial("instances")
	}

	d.Partial(false)
	return nil
}

func resourceAwsElbDelete(d *schema.ResourceData, meta interface{}) error {
	p := meta.(*ResourceProvider)
	elbconn := p.elbconn

	log.Printf("[INFO] Deleting ELB: %s", d.Id())

	// Destroy the load balancer
	deleteElbOpts := elb.DeleteLoadBalancer{
		LoadBalancerName: d.Id(),
	}
	if _, err := elbconn.DeleteLoadBalancer(&deleteElbOpts); err != nil {
		return fmt.Errorf("Error deleting ELB: %s", err)
	}

	return nil
}

func resourceAwsElbRead(d *schema.ResourceData, meta interface{}) error {
	p := meta.(*ResourceProvider)
	elbconn := p.elbconn

	// Retrieve the ELB properties for updating the state
	describeElbOpts := &elb.DescribeLoadBalancer{
		Names: []string{d.Id()},
	}

	describeResp, err := elbconn.DescribeLoadBalancers(describeElbOpts)
	if err != nil {
		return fmt.Errorf("Error retrieving ELB: %s", err)
	}
	if len(describeResp.LoadBalancers) != 1 {
		return fmt.Errorf("Unable to find ELB: %#v", describeResp.LoadBalancers)
	}

	lb := describeResp.LoadBalancers[0]

	d.Set("name", lb.LoadBalancerName)
	d.Set("dns_name", lb.DNSName)
	d.Set("internal", lb.Scheme == "internal")
	d.Set("instances", flattenInstances(lb.Instances))
	d.Set("security_groups", lb.SecurityGroups)
	d.Set("subnets", lb.Subnets)

	// There's only one health check, so save that to state as we
	// currently can
	if lb.HealthCheck.Target != "" {
		d.Set("health_check", flattenHealthCheck(lb.HealthCheck))
	}

	return nil
}
<commit_msg>providers/aws: handle case where ELB is deleted manually [GH-304]<commit_after>package aws

import (
	"bytes"
	"fmt"
	"log"

	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/mitchellh/goamz/elb"
)

func resourceAwsElb() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsElbCreate,
		Read:   resourceAwsElbRead,
		Update: resourceAwsElbUpdate,
		Delete: resourceAwsElbDelete,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"internal": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				ForceNew: true,
				Computed: true,
			},

			"availability_zones": &schema.Schema{
				Type:     schema.TypeList,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Optional: true,
				ForceNew: true,
			},

			"instances": &schema.Schema{
				Type:     schema.TypeSet,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Optional: true,
				Set: func(v interface{}) int {
					return hashcode.String(v.(string))
				},
			},

			// TODO: could be not ForceNew
			"security_groups": &schema.Schema{
				Type:     schema.TypeList,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Optional: true,
				ForceNew: true,
				Computed: true,
			},

			// TODO: could be not ForceNew
			"subnets": &schema.Schema{
				Type:     schema.TypeList,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Optional: true,
				ForceNew: true,
				Computed: true,
			},

			// TODO: could be not ForceNew
			"listener": &schema.Schema{
				Type:     schema.TypeSet,
				Required: true,
				ForceNew: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"instance_port": &schema.Schema{
							Type:     schema.TypeInt,
							Required: true,
						},

						"instance_protocol": &schema.Schema{
							Type:     schema.TypeString,
							Required: true,
						},

						"lb_port": &schema.Schema{
							Type:     schema.TypeInt,
							Required: true,
						},

						"lb_protocol": &schema.Schema{
							Type:     schema.TypeString,
							Required: true,
						},

						"ssl_certificate_id": &schema.Schema{
							Type:     schema.TypeString,
							Optional: true,
						},
					},
				},
				Set: resourceAwsElbListenerHash,
			},

			// TODO: could be not ForceNew
			"health_check": &schema.Schema{
				Type:     schema.TypeSet,
				Optional: true,
				ForceNew: true,
				Computed: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"healthy_threshold": &schema.Schema{
							Type:     schema.TypeInt,
							Required: true,
						},

						"unhealthy_threshold": &schema.Schema{
							Type:     schema.TypeInt,
							Required: true,
						},

						"target": &schema.Schema{
							Type:     schema.TypeString,
							Required: true,
						},

						"interval": &schema.Schema{
							Type:     schema.TypeInt,
							Required: true,
						},

						"timeout": &schema.Schema{
							Type:     schema.TypeInt,
							Required: true,
						},
					},
				},
				Set: resourceAwsElbHealthCheckHash,
			},

			"dns_name": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceAwsElbHealthCheckHash(v interface{}) int {
	var buf bytes.Buffer
	m := v.(map[string]interface{})
	buf.WriteString(fmt.Sprintf("%d-", m["healthy_threshold"].(int)))
	buf.WriteString(fmt.Sprintf("%d-", m["unhealthy_threshold"].(int)))
	buf.WriteString(fmt.Sprintf("%s-", m["target"].(string)))
	buf.WriteString(fmt.Sprintf("%d-", m["interval"].(int)))
	buf.WriteString(fmt.Sprintf("%d-", m["timeout"].(int)))

	return hashcode.String(buf.String())
}

func resourceAwsElbListenerHash(v interface{}) int {
	var buf bytes.Buffer
	m := v.(map[string]interface{})
	buf.WriteString(fmt.Sprintf("%d-", m["instance_port"].(int)))
	buf.WriteString(fmt.Sprintf("%s-", m["instance_protocol"].(string)))
	buf.WriteString(fmt.Sprintf("%d-", m["lb_port"].(int)))
	buf.WriteString(fmt.Sprintf("%s-", m["lb_protocol"].(string)))

	if v, ok := m["ssl_certificate_id"]; ok {
		buf.WriteString(fmt.Sprintf("%s-", v.(string)))
	}

	return hashcode.String(buf.String())
}

func resourceAwsElbCreate(d *schema.ResourceData, meta interface{}) error {
	p := meta.(*ResourceProvider)
	elbconn := p.elbconn

	// Expand the "listener" set to goamz compat []elb.Listener
	listeners, err := expandListeners(d.Get("listener").(*schema.Set).List())
	if err != nil {
		return err
	}

	// Provision the elb
	elbOpts := &elb.CreateLoadBalancer{
		LoadBalancerName: d.Get("name").(string),
		Listeners:        listeners,
		Internal:         d.Get("internal").(bool),
	}

	if v, ok := d.GetOk("availability_zones"); ok {
		elbOpts.AvailZone = expandStringList(v.([]interface{}))
	}

	if v, ok := d.GetOk("security_groups"); ok {
		elbOpts.SecurityGroups = expandStringList(v.([]interface{}))
	}

	if v, ok := d.GetOk("subnets"); ok {
		elbOpts.Subnets = expandStringList(v.([]interface{}))
	}

	log.Printf("[DEBUG] ELB create configuration: %#v", elbOpts)
	if _, err := elbconn.CreateLoadBalancer(elbOpts); err != nil {
		return fmt.Errorf("Error creating ELB: %s", err)
	}

	// Assign the elb's unique identifier for use later
	d.SetId(d.Get("name").(string))
	log.Printf("[INFO] ELB ID: %s", d.Id())

	// Enable partial mode and record what we set
	d.Partial(true)
	d.SetPartial("name")
	d.SetPartial("internal")
	d.SetPartial("availability_zones")
	d.SetPartial("security_groups")
	d.SetPartial("subnets")

	if d.HasChange("health_check") {
		vs := d.Get("health_check").(*schema.Set).List()
		if len(vs) > 0 {
			check := vs[0].(map[string]interface{})

			configureHealthCheckOpts := elb.ConfigureHealthCheck{
				LoadBalancerName: d.Id(),
				Check: elb.HealthCheck{
					HealthyThreshold:   int64(check["healthy_threshold"].(int)),
					UnhealthyThreshold: int64(check["unhealthy_threshold"].(int)),
					Interval:           int64(check["interval"].(int)),
					Target:             check["target"].(string),
					Timeout:            int64(check["timeout"].(int)),
				},
			}

			_, err = elbconn.ConfigureHealthCheck(&configureHealthCheckOpts)
			if err != nil {
				return fmt.Errorf("Failure configuring health check: %s", err)
			}
		}
	}

	return resourceAwsElbUpdate(d, meta)
}

func resourceAwsElbUpdate(d *schema.ResourceData, meta interface{}) error {
	p := meta.(*ResourceProvider)
	elbconn := p.elbconn

	d.Partial(true)

	// If we currently have instances, or did have instances,
	// we want to figure out what to add and remove from the load
	// balancer
	if d.HasChange("instances") {
		o, n := d.GetChange("instances")
		os := o.(*schema.Set)
		ns := n.(*schema.Set)
		remove := expandStringList(os.Difference(ns).List())
		add := expandStringList(ns.Difference(os).List())

		if len(add) > 0 {
			registerInstancesOpts := elb.RegisterInstancesWithLoadBalancer{
				LoadBalancerName: d.Id(),
				Instances:        add,
			}

			_, err := elbconn.RegisterInstancesWithLoadBalancer(&registerInstancesOpts)
			if err != nil {
				return fmt.Errorf("Failure registering instances: %s", err)
			}
		}
		if len(remove) > 0 {
			deRegisterInstancesOpts := elb.DeregisterInstancesFromLoadBalancer{
				LoadBalancerName: d.Id(),
				Instances:        remove,
			}

			_, err := elbconn.DeregisterInstancesFromLoadBalancer(&deRegisterInstancesOpts)
			if err != nil {
				return fmt.Errorf("Failure deregistering instances: %s", err)
			}
		}

		d.SetPartial("instances")
	}

	d.Partial(false)
	return nil
}

func resourceAwsElbDelete(d *schema.ResourceData, meta interface{}) error {
	p := meta.(*ResourceProvider)
	elbconn := p.elbconn

	log.Printf("[INFO] Deleting ELB: %s", d.Id())

	// Destroy the load balancer
	deleteElbOpts := elb.DeleteLoadBalancer{
		LoadBalancerName: d.Id(),
	}
	if _, err := elbconn.DeleteLoadBalancer(&deleteElbOpts); err != nil {
		return fmt.Errorf("Error deleting ELB: %s", err)
	}

	return nil
}

func resourceAwsElbRead(d *schema.ResourceData, meta interface{}) error {
	p := meta.(*ResourceProvider)
	elbconn := p.elbconn

	// Retrieve the ELB properties for updating the state
	describeElbOpts := &elb.DescribeLoadBalancer{
		Names: []string{d.Id()},
	}

	describeResp, err := elbconn.DescribeLoadBalancers(describeElbOpts)
	if err != nil {
		if ec2err, ok := err.(*elb.Error); ok && ec2err.Code == "LoadBalancerNotFound" {
			// The ELB is gone now, so just remove it from the state
			d.SetId("")
			return nil
		}

		return fmt.Errorf("Error retrieving ELB: %s", err)
	}
	if len(describeResp.LoadBalancers) != 1 {
		return fmt.Errorf("Unable to find ELB: %#v", describeResp.LoadBalancers)
	}

	lb := describeResp.LoadBalancers[0]

	d.Set("name", lb.LoadBalancerName)
	d.Set("dns_name", lb.DNSName)
	d.Set("internal", lb.Scheme == "internal")
	d.Set("instances", flattenInstances(lb.Instances))
	d.Set("security_groups", lb.SecurityGroups)
	d.Set("subnets", lb.Subnets)

	// There's only one health check, so save that to state as we
	// currently can
	if lb.HealthCheck.Target != "" {
		d.Set("health_check", flattenHealthCheck(lb.HealthCheck))
	}

	return nil
}
<|endoftext|>


<commit_before><commit_msg>Adds tests for fieldIndex and fieldRange types.<commit_after><|endoftext|>