<commit_before>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package buildstore

import (
	"strings"
	"time"

	"golang.org/x/net/context"

	"go.chromium.org/gae/service/datastore"
	"go.chromium.org/luci/common/auth/identity"
	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/sync/parallel"
	"go.chromium.org/luci/milo/api/buildbot"
	"go.chromium.org/luci/milo/common"
	"go.chromium.org/luci/server/auth"
)

// Master is buildbot.Master plus extra storage-level information.
type Master struct {
	buildbot.Master
	Internal bool
	Modified time.Time
}

// GetMaster fetches a master.
//
// If refreshState is true, refreshes individual builds from the datastore
// and the list of Cached builds.
//
// If any of the master's builders is emulated, the returned Master
// does not have any slave information or pending build states.
//
// Does not check access.
func GetMaster(c context.Context, name string, refreshState bool) (*Master, error) {
	entity := masterEntity{Name: name}
	err := datastore.Get(c, &entity)
	if err == datastore.ErrNoSuchEntity {
		return nil, errors.New("master not found", common.CodeNotFound)
	}
	if err != nil {
		return nil, err
	}

	m, err := entity.decode()
	if err != nil {
		return nil, err
	}
	if !refreshState {
		return m, nil
	}

	emulation := EmulationEnabled(c)
	if emulation {
		// Emulation does not support this Slaves field.
		m.Slaves = nil
		for _, b := range m.Builders {
			b.Slaves = nil
			b.PendingBuildStates = nil
		}
	} else {
		var refreshBuilds []*buildEntity
		for _, slave := range m.Slaves {
			for _, b := range slave.Runningbuilds {
				refreshBuilds = append(refreshBuilds, (*buildEntity)(b))
			}
		}
		if err = datastore.Get(c, refreshBuilds); err != nil {
			return nil, errors.Annotate(err, "refresh builds").Err()
		}
	}

	// Inject cached builds information.
	return m, parallel.FanOutIn(func(work chan<- func() error) {
		for builderName, builder := range m.Builders {
			builderName := builderName
			builder := builder
			work <- func() error {
				// Get the most recent 50 buildNums on the builder to simulate what the
				// cachedBuilds field looks like from the real buildbot master json.
				q := Query{
					Master:   name,
					Builder:  builderName,
					Finished: Yes,
					Limit:    50,

					NoAnnotationFetch: true,
					KeyOnly:           true,
				}
				res, err := GetBuilds(c, q)
				if err != nil {
					return err
				}
				builder.CachedBuilds = make([]int, len(res.Builds))
				for i, b := range res.Builds {
					builder.CachedBuilds[i] = b.Number
				}

				if emulation {
					// builder.PendingBuilds and builder.CurrentBuilds
					// contain only buildbot data. Add LUCI data.

					// This will load both Buildbot and LUCI builds, merged.
					q := Query{
						Master:   name,
						Builder:  builderName,
						Finished: No,

						// we will ignore buildbot builds
						KeyOnly:           true,
						NoAnnotationFetch: true,
					}
					res, err := GetBuilds(c, q)
					if err != nil {
						return err
					}

					// Pending buildbot builds are "build requests".
					// They did not turn into builds yet and we don't know if
					// they will be come experimental or not. The moment one
					// turns into a build, it will excluded from
					// builder.PendingBuilds and will possibly not included
					// in CurrentBuilds in case it is experimental.
					// Thus, not zeroing builder.PendingBuilds.
					builder.CurrentBuilds = nil
					for _, b := range res.Builds {
						if b.Times.Start.IsZero() {
							builder.PendingBuilds++
						} else {
							builder.CurrentBuilds = append(builder.CurrentBuilds, b.Number)
						}
					}
				}

				return nil
			}
		}
	})
}

// AllMasters returns all buildbot masters.
func AllMasters(c context.Context, checkAccess bool) ([]*Master, error) {
	const batchSize = int32(500)
	masters := make([]*Master, 0, batchSize)
	q := datastore.NewQuery(masterKind)

	// note: avoid calling isAllowedInternal is checkAccess is false.
	// checkAccess is usually false in cron jobs where there is no auth state,
	// so isAllowedInternal call would unconditionally log an error.
	allowInternal := !checkAccess || isAllowedInternal(c)
	err := datastore.RunBatch(c, batchSize, q, func(e *masterEntity) error {
		if allowInternal || !e.Internal {
			m, err := e.decode()
			if err != nil {
				return err
			}
			masters = append(masters, m)
		}
		return nil
	})
	return masters, err
}

// GetPendingCounts returns numbers of pending builds in builders.
// builders must be a list of slash-separated master, builder names.
func GetPendingCounts(c context.Context, builders []string) ([]int, error) {
	entities := make([]builderEntity, len(builders))
	for i, b := range builders {
		parts := strings.SplitN(b, "/", 2)
		if len(parts) < 2 {
			return nil, errors.Reason("builder does not have a slash: %q", b).Err()
		}
		entities[i].MasterKey = datastore.MakeKey(c, masterKind, parts[0])
		entities[i].Name = parts[1]
	}
	if err := datastore.Get(c, entities); err != nil {
		for _, e := range err.(errors.MultiError) {
			if e != nil && e != datastore.ErrNoSuchEntity {
				return nil, err
			}
		}
	}

	counts := make([]int, len(entities))
	for i, e := range entities {
		counts[i] = e.PendingCount
	}
	return counts, nil
}

// PutPendingCount persists number of pending builds to a builder.
// Useful for testing.
func PutPendingCount(c context.Context, master, builder string, count int) error {
	return datastore.Put(c, &builderEntity{
		MasterKey:    datastore.MakeKey(c, masterKind, master),
		Name:         builder,
		PendingCount: count,
	})
}

// ExpireCallback is called when a build is marked as expired.
type ExpireCallback func(b *buildbot.Build, reason string)

// SaveMaster persists the master in the storage.
//
// Expires all incomplete builds in the datastore associated with this master
// and
//   - associated with builders not declared in master.Builders
//   - OR or not "current" from this master's perspective and >=20min stale.
func SaveMaster(c context.Context, master *buildbot.Master,
	internal bool, expireCallback ExpireCallback) error {

	entity := &masterEntity{
		Name:     master.Name,
		Internal: internal,
		Modified: clock.Now(c).UTC(),
	}
	toPut := []interface{}{entity}
	for builderName, builder := range master.Builders {
		// Trim out extra info in the "Changes" portion of the pending build state,
		// we don't actually need comments, files, and properties
		for _, pbs := range builder.PendingBuildStates {
			for i := range pbs.Source.Changes {
				c := &pbs.Source.Changes[i]
				c.Comments = ""
				c.Files = nil
				c.Properties = nil
			}
		}
		toPut = append(toPut, &builderEntity{
			MasterKey:    datastore.KeyForObj(c, entity),
			Name:         builderName,
			PendingCount: builder.PendingBuilds,
		})
	}
	publicTag := &masterPublic{Name: master.Name}
	if internal {
		// do the deletion immediately so that the 'public' bit is removed from
		// datastore before any internal details are actually written to datastore.
		if err := datastore.Delete(c, publicTag); err != nil && err != datastore.ErrNoSuchEntity {
			return err
		}
	} else {
		toPut = append(toPut, publicTag)
	}

	var err error
	entity.Data, err = encode(master)
	if err != nil {
		return err
	}
	logging.Debugf(c, "Length of gzipped master data: %d", len(entity.Data))
	if len(entity.Data) > maxDataSize {
		return errors.Reason("master data is %d bytes, which is more than %d limit", len(entity.Data), maxDataSize).
			Tag(TooBigTag).
			Err()
	}
	if err := datastore.Put(c, toPut); err != nil {
		return err
	}

	return cleanUpExpiredBuilds(c, master, expireCallback)
}

func cleanUpExpiredBuilds(c context.Context, master *buildbot.Master, expiredCallback ExpireCallback) error {
	q := datastore.NewQuery(buildKind).
		Eq("master", master.Name).
		Eq("finished", false)
	return parallel.WorkPool(10, func(work chan<- func() error) {
		err := datastore.Run(c, q, func(b *buildEntity) {
			now := clock.Now(c)
			reason := ""
			if builder, ok := master.Builders[b.Buildername]; !ok {
				reason = "builder removed"
			} else {
				for _, bnum := range builder.CurrentBuilds {
					if b.Number == bnum {
						return
					}
				}
				// This build is not among master's current builds.
				// Expire builds after 20 minutes of not getting data.
				if now.Sub(b.TimeStamp.Time) > 20*time.Minute {
					reason = "build stale"
				}
			}
			if reason != "" {
				b := (*buildbot.Build)(b)
				work <- func() error {
					err := expireBuild(c, b)
					if err != nil {
						return err
					}
					if expiredCallback != nil {
						expiredCallback(b, reason)
					}
					return nil
				}
			}
		})
		if err != nil {
			// use the existing channel for returning errors
			work <- func() error { return err }
		}
	})
}

// expireBuild marks a build as finished and expired.
func expireBuild(c context.Context, b *buildbot.Build) error {
	if !b.TimeStamp.IsZero() {
		b.Times.Finish = b.TimeStamp
	} else {
		b.Times.Finish.Time = clock.Now(c)
	}
	b.Finished = true
	b.Results = buildbot.Exception
	b.Currentstep = nil
	b.Text = append(b.Text, "Build expired on Milo")
	_, err := SaveBuild(c, b)
	return err
}

// masterPublic is a datastore entity that exists for public builtbot masters,
// and not for internal masters. It's used for ACL checks.
type masterPublic struct {
	_kind string `gae:"$kind,buildbotMasterPublic"`
	Name  string `gae:"$id"`
}

// isAllowedInternal returns true if the current user has access to internal
// data. In case of an error, logs it and returns false to prevent from
// sniffing based on internal errors.
func isAllowedInternal(c context.Context) bool {
	settings := common.GetSettings(c)
	if settings.Buildbot.InternalReader == "" {
		return false
	}
	allowed, err := auth.IsMember(c, settings.Buildbot.InternalReader)
	if err != nil {
		logging.WithError(err).Errorf(c, "IsMember(%q) failed", settings.Buildbot.InternalReader)
		allowed = false
	}
	return allowed
}

// CanAccessMaster returns nil if the currently logged in user can see the
// masters, or if the given master is a known public master,
// otherwise an error.
func CanAccessMaster(c context.Context, name string) error {
	if ex, err := datastore.Exists(c, &masterPublic{Name: name}); err == nil && ex.Get(0) {
		// It exists => it is public
		return nil
	}

	if isAllowedInternal(c) {
		return nil
	}

	code := common.CodeNotFound
	if auth.CurrentUser(c).Identity == identity.AnonymousIdentity {
		code = common.CodeUnauthorized
	}
	// Act like master does not exist.
	return errors.Reason("master %q not found", name).Tag(code).Err()
}

var masterKind = "buildbotMasterEntry"

// masterEntity is a datastore entity containing marshaled and compressed
// buildbot master json.
type masterEntity struct {
	_kind string `gae:"$kind,buildbotMasterEntry"`
	// Name of the buildbot master, without "master." prefix.
	Name string `gae:"$id"`
	// Internal indicates whether the master is internal.
	// This value must by synced with the existance of masterPublic entity.
	// FIXME: the masterPublic entity should have been in the same entity group.
	Internal bool
	// Data is the json serialized and gzipped buildbot.Master.
	Data []byte `gae:",noindex"`
	// Modified is when this entry was last modified.
	Modified time.Time
}

func (m *masterEntity) decode() (*Master, error) {
	res := Master{
		Internal: m.Internal,
		Modified: m.Modified,
	}
	return &res, decode(&res.Master, m.Data)
}

// builderEntity is a child of masterEntity, specific to a Builder.
type builderEntity struct {
	_kind        string         `gae:"$kind,buildbotBuilder"`
	Name         string         `gae:"$id"` // builder name
	MasterKey    *datastore.Key `gae:"$parent"`
	PendingCount int
}
<commit_msg>[milo] make CurrentBuilds empty array<commit_after>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package buildstore

import (
	"strings"
	"time"

	"golang.org/x/net/context"

	"go.chromium.org/gae/service/datastore"
	"go.chromium.org/luci/common/auth/identity"
	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/sync/parallel"
	"go.chromium.org/luci/milo/api/buildbot"
	"go.chromium.org/luci/milo/common"
	"go.chromium.org/luci/server/auth"
)

// Master is buildbot.Master plus extra storage-level information.
type Master struct {
	buildbot.Master
	Internal bool
	Modified time.Time
}

// GetMaster fetches a master.
//
// If refreshState is true, refreshes individual builds from the datastore
// and the list of Cached builds.
//
// If any of the master's builders is emulated, the returned Master
// does not have any slave information or pending build states.
//
// Does not check access.
func GetMaster(c context.Context, name string, refreshState bool) (*Master, error) {
	entity := masterEntity{Name: name}
	err := datastore.Get(c, &entity)
	if err == datastore.ErrNoSuchEntity {
		return nil, errors.New("master not found", common.CodeNotFound)
	}
	if err != nil {
		return nil, err
	}

	m, err := entity.decode()
	if err != nil {
		return nil, err
	}
	if !refreshState {
		return m, nil
	}

	emulation := EmulationEnabled(c)
	if emulation {
		// Emulation does not support this Slaves field.
		m.Slaves = nil
		for _, b := range m.Builders {
			b.Slaves = nil
			b.PendingBuildStates = nil
		}
	} else {
		var refreshBuilds []*buildEntity
		for _, slave := range m.Slaves {
			for _, b := range slave.Runningbuilds {
				refreshBuilds = append(refreshBuilds, (*buildEntity)(b))
			}
		}
		if err = datastore.Get(c, refreshBuilds); err != nil {
			return nil, errors.Annotate(err, "refresh builds").Err()
		}
	}

	// Inject cached builds information.
	return m, parallel.FanOutIn(func(work chan<- func() error) {
		for builderName, builder := range m.Builders {
			builderName := builderName
			builder := builder
			work <- func() error {
				// Get the most recent 50 buildNums on the builder to simulate what the
				// cachedBuilds field looks like from the real buildbot master json.
				q := Query{
					Master:   name,
					Builder:  builderName,
					Finished: Yes,
					Limit:    50,

					NoAnnotationFetch: true,
					KeyOnly:           true,
				}
				res, err := GetBuilds(c, q)
				if err != nil {
					return err
				}
				builder.CachedBuilds = make([]int, len(res.Builds))
				for i, b := range res.Builds {
					builder.CachedBuilds[i] = b.Number
				}

				if emulation {
					// builder.PendingBuilds and builder.CurrentBuilds
					// contain only buildbot data. Add LUCI data.

					// This will load both Buildbot and LUCI builds, merged.
					q := Query{
						Master:   name,
						Builder:  builderName,
						Finished: No,

						// we will ignore buildbot builds
						KeyOnly:           true,
						NoAnnotationFetch: true,
					}
					res, err := GetBuilds(c, q)
					if err != nil {
						return err
					}

					// Pending buildbot builds are "build requests".
					// They did not turn into builds yet and we don't know if
					// they will be come experimental or not. The moment one
					// turns into a build, it will excluded from
					// builder.PendingBuilds and will possibly not included
					// in CurrentBuilds in case it is experimental.
					// Thus, not zeroing builder.PendingBuilds.

					// In contrast to Golang, nil in JSON is not a valid array.
					// so do not set CurrentBuilds to nil.
					builder.CurrentBuilds = make([]int, 0, len(res.Builds))
					for _, b := range res.Builds {
						if b.Times.Start.IsZero() {
							builder.PendingBuilds++
						} else {
							builder.CurrentBuilds = append(builder.CurrentBuilds, b.Number)
						}
					}
				}

				return nil
			}
		}
	})
}

// AllMasters returns all buildbot masters.
func AllMasters(c context.Context, checkAccess bool) ([]*Master, error) {
	const batchSize = int32(500)
	masters := make([]*Master, 0, batchSize)
	q := datastore.NewQuery(masterKind)

	// note: avoid calling isAllowedInternal is checkAccess is false.
	// checkAccess is usually false in cron jobs where there is no auth state,
	// so isAllowedInternal call would unconditionally log an error.
	allowInternal := !checkAccess || isAllowedInternal(c)
	err := datastore.RunBatch(c, batchSize, q, func(e *masterEntity) error {
		if allowInternal || !e.Internal {
			m, err := e.decode()
			if err != nil {
				return err
			}
			masters = append(masters, m)
		}
		return nil
	})
	return masters, err
}

// GetPendingCounts returns numbers of pending builds in builders.
// builders must be a list of slash-separated master, builder names.
func GetPendingCounts(c context.Context, builders []string) ([]int, error) {
	entities := make([]builderEntity, len(builders))
	for i, b := range builders {
		parts := strings.SplitN(b, "/", 2)
		if len(parts) < 2 {
			return nil, errors.Reason("builder does not have a slash: %q", b).Err()
		}
		entities[i].MasterKey = datastore.MakeKey(c, masterKind, parts[0])
		entities[i].Name = parts[1]
	}
	if err := datastore.Get(c, entities); err != nil {
		for _, e := range err.(errors.MultiError) {
			if e != nil && e != datastore.ErrNoSuchEntity {
				return nil, err
			}
		}
	}

	counts := make([]int, len(entities))
	for i, e := range entities {
		counts[i] = e.PendingCount
	}
	return counts, nil
}

// PutPendingCount persists number of pending builds to a builder.
// Useful for testing.
func PutPendingCount(c context.Context, master, builder string, count int) error {
	return datastore.Put(c, &builderEntity{
		MasterKey:    datastore.MakeKey(c, masterKind, master),
		Name:         builder,
		PendingCount: count,
	})
}

// ExpireCallback is called when a build is marked as expired.
type ExpireCallback func(b *buildbot.Build, reason string)

// SaveMaster persists the master in the storage.
//
// Expires all incomplete builds in the datastore associated with this master
// and
//   - associated with builders not declared in master.Builders
//   - OR or not "current" from this master's perspective and >=20min stale.
func SaveMaster(c context.Context, master *buildbot.Master,
	internal bool, expireCallback ExpireCallback) error {

	entity := &masterEntity{
		Name:     master.Name,
		Internal: internal,
		Modified: clock.Now(c).UTC(),
	}
	toPut := []interface{}{entity}
	for builderName, builder := range master.Builders {
		// Trim out extra info in the "Changes" portion of the pending build state,
		// we don't actually need comments, files, and properties
		for _, pbs := range builder.PendingBuildStates {
			for i := range pbs.Source.Changes {
				c := &pbs.Source.Changes[i]
				c.Comments = ""
				c.Files = nil
				c.Properties = nil
			}
		}
		toPut = append(toPut, &builderEntity{
			MasterKey:    datastore.KeyForObj(c, entity),
			Name:         builderName,
			PendingCount: builder.PendingBuilds,
		})
	}
	publicTag := &masterPublic{Name: master.Name}
	if internal {
		// do the deletion immediately so that the 'public' bit is removed from
		// datastore before any internal details are actually written to datastore.
		if err := datastore.Delete(c, publicTag); err != nil && err != datastore.ErrNoSuchEntity {
			return err
		}
	} else {
		toPut = append(toPut, publicTag)
	}

	var err error
	entity.Data, err = encode(master)
	if err != nil {
		return err
	}
	logging.Debugf(c, "Length of gzipped master data: %d", len(entity.Data))
	if len(entity.Data) > maxDataSize {
		return errors.Reason("master data is %d bytes, which is more than %d limit", len(entity.Data), maxDataSize).
			Tag(TooBigTag).
			Err()
	}
	if err := datastore.Put(c, toPut); err != nil {
		return err
	}

	return cleanUpExpiredBuilds(c, master, expireCallback)
}

func cleanUpExpiredBuilds(c context.Context, master *buildbot.Master, expiredCallback ExpireCallback) error {
	q := datastore.NewQuery(buildKind).
		Eq("master", master.Name).
		Eq("finished", false)
	return parallel.WorkPool(10, func(work chan<- func() error) {
		err := datastore.Run(c, q, func(b *buildEntity) {
			now := clock.Now(c)
			reason := ""
			if builder, ok := master.Builders[b.Buildername]; !ok {
				reason = "builder removed"
			} else {
				for _, bnum := range builder.CurrentBuilds {
					if b.Number == bnum {
						return
					}
				}
				// This build is not among master's current builds.
				// Expire builds after 20 minutes of not getting data.
				if now.Sub(b.TimeStamp.Time) > 20*time.Minute {
					reason = "build stale"
				}
			}
			if reason != "" {
				b := (*buildbot.Build)(b)
				work <- func() error {
					err := expireBuild(c, b)
					if err != nil {
						return err
					}
					if expiredCallback != nil {
						expiredCallback(b, reason)
					}
					return nil
				}
			}
		})
		if err != nil {
			// use the existing channel for returning errors
			work <- func() error { return err }
		}
	})
}

// expireBuild marks a build as finished and expired.
func expireBuild(c context.Context, b *buildbot.Build) error {
	if !b.TimeStamp.IsZero() {
		b.Times.Finish = b.TimeStamp
	} else {
		b.Times.Finish.Time = clock.Now(c)
	}
	b.Finished = true
	b.Results = buildbot.Exception
	b.Currentstep = nil
	b.Text = append(b.Text, "Build expired on Milo")
	_, err := SaveBuild(c, b)
	return err
}

// masterPublic is a datastore entity that exists for public builtbot masters,
// and not for internal masters. It's used for ACL checks.
type masterPublic struct {
	_kind string `gae:"$kind,buildbotMasterPublic"`
	Name  string `gae:"$id"`
}

// isAllowedInternal returns true if the current user has access to internal
// data. In case of an error, logs it and returns false to prevent from
// sniffing based on internal errors.
func isAllowedInternal(c context.Context) bool {
	settings := common.GetSettings(c)
	if settings.Buildbot.InternalReader == "" {
		return false
	}
	allowed, err := auth.IsMember(c, settings.Buildbot.InternalReader)
	if err != nil {
		logging.WithError(err).Errorf(c, "IsMember(%q) failed", settings.Buildbot.InternalReader)
		allowed = false
	}
	return allowed
}

// CanAccessMaster returns nil if the currently logged in user can see the
// masters, or if the given master is a known public master,
// otherwise an error.
func CanAccessMaster(c context.Context, name string) error {
	if ex, err := datastore.Exists(c, &masterPublic{Name: name}); err == nil && ex.Get(0) {
		// It exists => it is public
		return nil
	}

	if isAllowedInternal(c) {
		return nil
	}

	code := common.CodeNotFound
	if auth.CurrentUser(c).Identity == identity.AnonymousIdentity {
		code = common.CodeUnauthorized
	}
	// Act like master does not exist.
	return errors.Reason("master %q not found", name).Tag(code).Err()
}

var masterKind = "buildbotMasterEntry"

// masterEntity is a datastore entity containing marshaled and compressed
// buildbot master json.
type masterEntity struct {
	_kind string `gae:"$kind,buildbotMasterEntry"`
	// Name of the buildbot master, without "master." prefix.
	Name string `gae:"$id"`
	// Internal indicates whether the master is internal.
	// This value must by synced with the existance of masterPublic entity.
	// FIXME: the masterPublic entity should have been in the same entity group.
	Internal bool
	// Data is the json serialized and gzipped buildbot.Master.
	Data []byte `gae:",noindex"`
	// Modified is when this entry was last modified.
	Modified time.Time
}

func (m *masterEntity) decode() (*Master, error) {
	res := Master{
		Internal: m.Internal,
		Modified: m.Modified,
	}
	return &res, decode(&res.Master, m.Data)
}

// builderEntity is a child of masterEntity, specific to a Builder.
type builderEntity struct {
	_kind        string         `gae:"$kind,buildbotBuilder"`
	Name         string         `gae:"$id"` // builder name
	MasterKey    *datastore.Key `gae:"$parent"`
	PendingCount int
}
<|endoftext|>


<commit_before><commit_msg>Fix go1.12 typename in afpacket.<commit_after><|endoftext|>


<commit_before>package apidsl

import (
	"fmt"
	"mime"
	"strings"

	"github.com/goadesign/goa/design"
	"github.com/goadesign/goa/dslengine"
)

// Counter used to create unique media type names for identifier-less media types.
var mediaTypeCount int

// MediaType implements the media type definition apidsl. A media type definition describes the
// representation of a resource used in a response body. This includes listing all the *potential*
// resource attributes that can appear in the body. Views specify which of the attributes are
// actually rendered so that the same media type definition may represent multiple rendering of a
// given resource representation.
//
// All media types must define a view named "default". This view is used to render the media type in
// response bodies when no other view is specified.
//
// A media type definition may also define links to other media types. This is done by first
// defining an attribute for the linked-to media type and then referring to that attribute in the
// Links apidsl. Views may then elect to render one or the other or both. Links are rendered using the
// special "link" view. Media types that are linked to must define that view. Here is an example
// showing all the possible media type sub-definitions:
//
//	MediaType("application/vnd.goa.example.bottle", func() {
//		Description("A bottle of wine")
//		TypeName("BottleMedia") 		// Optionally override the default generated name
//		Attributes(func() {
//			Attribute("id", Integer, "ID of bottle")
//			Attribute("href", String, "API href of bottle")
//			Attribute("account", Account, "Owner account")
//			Attribute("origin", Origin, "Details on wine origin")
//			Links(func() {
//				Link("account")		// Defines a link to the Account media type
//				Link("origin", "tiny")	// Overrides the default view used to render links
//			})
//			Required("id", "href")
//		})
//		View("default", func() {
//			Attribute("id")
//			Attribute("href")
//			Attribute("links")	// Default view renders links
//		})
//		View("extended", func() {
//			Attribute("id")
//			Attribute("href")
//			Attribute("account")	// Extended view renders account inline
//			Attribute("origin")	// Extended view renders origin inline
//			Attribute("links")	// Extended view also renders links
//		})
// 	})
//
// This function returns the media type definition so it can be referred to throughout the apidsl.
func MediaType(identifier string, apidsl func()) *design.MediaTypeDefinition {
	if design.Design.MediaTypes == nil {
		design.Design.MediaTypes = make(map[string]*design.MediaTypeDefinition)
	}

	if !dslengine.IsTopLevelDefinition() {
		dslengine.IncompatibleDSL()
		return nil
	}

	// Validate Media Type
	identifier, params, err := mime.ParseMediaType(identifier)
	if err != nil {
		dslengine.ReportError("invalid media type identifier %#v: %s",
			identifier, err)
		// We don't return so that other errors may be
		// captured in this one run.
		identifier = "plain/text"
	}
	canonicalID := design.CanonicalIdentifier(identifier)
	// Validate that media type identifier doesn't clash
	if _, ok := design.Design.MediaTypes[canonicalID]; ok {
		dslengine.ReportError("media type %#v is defined twice", identifier)
		return nil
	}
	parts := strings.Split(identifier, "+")
	// Make sure it has the `+json` suffix (TBD update when goa supports other encodings)
	if len(parts) > 1 {
		parts = parts[1:]
		found := false
		for _, part := range parts {
			if part == "json" {
				found = true
				break
			}
		}
		if !found {
			identifier += "+json"
		}
	}
	identifier = mime.FormatMediaType(identifier, params)
	// Concoct a Go type name from the identifier, should it be possible to set it in the apidsl?
	// pros: control the type name generated, cons: not needed in apidsl, adds one more thing to worry about
	lastPart := identifier
	lastPartIndex := strings.LastIndex(identifier, "/")
	if lastPartIndex > -1 {
		lastPart = identifier[lastPartIndex+1:]
	}
	plusIndex := strings.Index(lastPart, "+")
	if plusIndex > 0 {
		lastPart = lastPart[:plusIndex]
	}
	lastPart = strings.TrimPrefix(lastPart, "vnd.")
	elems := strings.Split(lastPart, ".")
	for i, e := range elems {
		elems[i] = strings.Title(e)
	}
	typeName := strings.Join(elems, "")
	if typeName == "" {
		mediaTypeCount++
		typeName = fmt.Sprintf("MediaType%d", mediaTypeCount)
	}
	// Now save the type in the API media types map
	mt := design.NewMediaTypeDefinition(typeName, identifier, apidsl)
	design.Design.MediaTypes[canonicalID] = mt
	return mt

}

// Media sets a response media type by name or by reference using a value returned by MediaType:
//
//	Response("NotFound", func() {
//		Status(404)
//		Media("application/json")
//	})
//
// Media can be used inside Response or ResponseTemplate.
func Media(val interface{}) {
	if r, ok := responseDefinition(); ok {
		if m, ok := val.(*design.MediaTypeDefinition); ok {
			if m != nil {
				r.MediaType = m.Identifier
			}
		} else if identifier, ok := val.(string); ok {
			r.MediaType = identifier
		} else {
			dslengine.ReportError("media type must be a string or a pointer to MediaTypeDefinition, got %#v", val)
		}
	}
}

// Reference sets a type or media type reference. The value itself can be a type or a media type.
// The reference type attributes define the default properties for attributes with the same name in
// the type using the reference. So for example if a type is defined as such:
//
//	var Bottle = Type("bottle", func() {
//		Attribute("name", func() {
//			MinLength(3)
//		})
//		Attribute("vintage", Integer, func() {
//			Minimum(1970)
//		})
//		Attribute("somethingelse")
//	})
//
// Declaring the following media type:
//
//	var BottleMedia = MediaType("vnd.goa.bottle", func() {
//		Reference(Bottle)
//		Attributes(func() {
//			Attribute("id", Integer)
//			Attribute("name")
//			Attribute("vintage")
//		})
//	})
//
// defines the "name" and "vintage" attributes with the same type and validations as defined in
// the Bottle type.
func Reference(t design.DataType) {
	switch def := dslengine.CurrentDefinition().(type) {
	case *design.MediaTypeDefinition:
		def.Reference = t
	case *design.AttributeDefinition:
		def.Reference = t
	default:
		dslengine.IncompatibleDSL()
	}
}

// TypeName makes it possible to set the Go struct name for a type or media type in the generated
// code. By default goagen uses the name (type) or identifier (media type) given in the apidsl and
// computes a valid Go identifier from it. This function makes it possible to override that and
// provide a custom name. name must be a valid Go identifier.
func TypeName(name string) {
	switch def := dslengine.CurrentDefinition().(type) {
	case *design.MediaTypeDefinition:
		def.TypeName = name
	case *design.UserTypeDefinition:
		def.TypeName = name
	default:
		dslengine.IncompatibleDSL()
	}
}

// View adds a new view to a media type. A view has a name and lists attributes that are
// rendered when the view is used to produce a response. The attribute names must appear in the
// media type definition. If an attribute is itself a media type then the view may specify which
// view to use when rendering the attribute using the View function in the View apidsl. If not
// specified then the view named "default" is used. Examples:
//
//	View("default", func() {
//		Attribute("id")		// "id" and "name" must be media type attributes
//		Attribute("name")
//	})
//
//	View("extended", func() {
//		Attribute("id")
//		Attribute("name")
//		Attribute("origin", func() {
//			View("extended")	// Use view "extended" to render attribute "origin"
//		})
//	})
func View(name string, apidsl ...func()) {
	switch def := dslengine.CurrentDefinition().(type) {
	case *design.MediaTypeDefinition:
		mt := def

		if !mt.Type.IsObject() && !mt.Type.IsArray() {
			dslengine.ReportError("cannot define view on non object and non collection media types")
			return
		}
		if mt.Views == nil {
			mt.Views = make(map[string]*design.ViewDefinition)
		} else {
			if _, ok := mt.Views[name]; ok {
				dslengine.ReportError("multiple definitions for view %#v in media type %#v", name, mt.TypeName)
				return
			}
		}
		at := &design.AttributeDefinition{}
		ok := false
		if len(apidsl) > 0 {
			ok = dslengine.Execute(apidsl[0], at)
		} else if mt.Type.IsArray() {
			// inherit view from collection element if present
			elem := mt.Type.ToArray().ElemType
			if elem != nil {
				if pa, ok2 := elem.Type.(*design.MediaTypeDefinition); ok2 {
					if v, ok2 := pa.Views[name]; ok2 {
						at = v.AttributeDefinition
						ok = true
					} else {
						dslengine.ReportError("unknown view %#v", name)
						return
					}
				}
			}
		}
		if ok {
			if at.Type == nil || !at.Type.IsObject() {
				dslengine.ReportError("invalid view DSL")
				return
			}
			o := at.Type.ToObject()
			if o != nil {
				mto := mt.Type.ToObject()
				if mto == nil {
					mto = mt.Type.ToArray().ElemType.Type.ToObject()
				}
				for n, cat := range o {
					if existing, ok := mto[n]; ok {
						dup := design.DupAtt(existing)
						dup.View = cat.View
						o[n] = dup
					} else if n != "links" {
						dslengine.ReportError("unknown attribute %#v", n)
					}
				}
			}
			mt.Views[name] = &design.ViewDefinition{
				AttributeDefinition: at,
				Name:                name,
				Parent:              mt,
			}
		}

	case *design.AttributeDefinition:
		def.View = name

	default:
		dslengine.IncompatibleDSL()
	}
}

// Attributes implements the media type attributes apidsl. See MediaType.
func Attributes(apidsl func()) {
	if mt, ok := mediaTypeDefinition(); ok {
		dslengine.Execute(apidsl, mt)
	}
}

// Links implements the media type links apidsl. See MediaType.
func Links(apidsl func()) {
	if mt, ok := mediaTypeDefinition(); ok {
		dslengine.Execute(apidsl, mt)
	}
}

// Link adds a link to a media type. At the minimum a link has a name corresponding to one of the
// media type attribute names. A link may also define the view used to render the linked-to
// attribute. The default view used to render links is "link". Examples:
//
//	Link("origin")		// Use the "link" view of the "origin" attribute
//	Link("account", "tiny")	// Use the "tiny" view of the "account" attribute
func Link(name string, view ...string) {
	if mt, ok := mediaTypeDefinition(); ok {
		if mt.Links == nil {
			mt.Links = make(map[string]*design.LinkDefinition)
		} else {
			if _, ok := mt.Links[name]; ok {
				dslengine.ReportError("duplicate definition for link %#v", name)
				return
			}
		}
		link := &design.LinkDefinition{Name: name, Parent: mt}
		if len(view) > 1 {
			dslengine.ReportError("invalid syntax in Link definition for %#v, allowed syntax is Link(name) or Link(name, view)", name)
		}
		if len(view) > 0 {
			link.View = view[0]
		} else {
			link.View = "link"
		}
		mt.Links[name] = link
	}
}

// CollectionOf creates a collection media type from its element media type. A collection media
// type represents the content of responses that return a collection of resources such as "list"
// actions. This function can be called from any place where a media type can be used.
// The resulting media type identifier is built from the element media type by appending the media
// type parameter "type" with value "collection".
func CollectionOf(v interface{}, apidsl ...func()) *design.MediaTypeDefinition {
	var m *design.MediaTypeDefinition
	var ok bool
	m, ok = v.(*design.MediaTypeDefinition)
	if !ok {
		if id, ok := v.(string); ok {
			m = design.Design.MediaTypes[design.CanonicalIdentifier(id)]
		}
	}
	if m == nil {
		dslengine.ReportError("invalid CollectionOf argument: not a media type and not a known media type identifier")
		return nil
	}
	id := m.Identifier
	mediatype, params, err := mime.ParseMediaType(id)
	if err != nil {
		dslengine.ReportError("invalid media type identifier %#v: %s", id, err)
		return nil
	}
	hasType := false
	for param := range params {
		if param == "type" {
			hasType = true
			break
		}
	}
	if !hasType {
		params["type"] = "collection"
	}
	id = mime.FormatMediaType(mediatype, params)
	typeName := m.TypeName + "Collection"
	if mt, ok := design.GeneratedMediaTypes[typeName]; ok {
		// Already have a type for this collection, reuse it.
		return mt
	}
	mt := design.NewMediaTypeDefinition(typeName, id, func() {
		if mt, ok := mediaTypeDefinition(); ok {
			mt.TypeName = typeName
			mt.AttributeDefinition = &design.AttributeDefinition{Type: ArrayOf(m)}
			if len(apidsl) > 0 {
				dslengine.Execute(apidsl[0], mt)
			}
			if mt.Views == nil {
				// If the apidsl didn't create any views (or there is no apidsl at all)
				// then inherit the views from the collection element.
				mt.Views = make(map[string]*design.ViewDefinition)
				for n, v := range m.Views {
					mt.Views[n] = v
				}
			}
		}
	})
	// Do not execute the apidsl right away, will be done last to make sure the element apidsl has run
	// first.
	design.GeneratedMediaTypes[typeName] = mt
	return mt
}
<commit_msg>Fix cyclomatic complexity<commit_after>package apidsl

import (
	"fmt"
	"mime"
	"strings"

	"github.com/goadesign/goa/design"
	"github.com/goadesign/goa/dslengine"
)

// Counter used to create unique media type names for identifier-less media types.
var mediaTypeCount int

// MediaType implements the media type definition apidsl. A media type definition describes the
// representation of a resource used in a response body. This includes listing all the *potential*
// resource attributes that can appear in the body. Views specify which of the attributes are
// actually rendered so that the same media type definition may represent multiple rendering of a
// given resource representation.
//
// All media types must define a view named "default". This view is used to render the media type in
// response bodies when no other view is specified.
//
// A media type definition may also define links to other media types. This is done by first
// defining an attribute for the linked-to media type and then referring to that attribute in the
// Links apidsl. Views may then elect to render one or the other or both. Links are rendered using the
// special "link" view. Media types that are linked to must define that view. Here is an example
// showing all the possible media type sub-definitions:
//
//	MediaType("application/vnd.goa.example.bottle", func() {
//		Description("A bottle of wine")
//		TypeName("BottleMedia") 		// Optionally override the default generated name
//		Attributes(func() {
//			Attribute("id", Integer, "ID of bottle")
//			Attribute("href", String, "API href of bottle")
//			Attribute("account", Account, "Owner account")
//			Attribute("origin", Origin, "Details on wine origin")
//			Links(func() {
//				Link("account")		// Defines a link to the Account media type
//				Link("origin", "tiny")	// Overrides the default view used to render links
//			})
//			Required("id", "href")
//		})
//		View("default", func() {
//			Attribute("id")
//			Attribute("href")
//			Attribute("links")	// Default view renders links
//		})
//		View("extended", func() {
//			Attribute("id")
//			Attribute("href")
//			Attribute("account")	// Extended view renders account inline
//			Attribute("origin")	// Extended view renders origin inline
//			Attribute("links")	// Extended view also renders links
//		})
// 	})
//
// This function returns the media type definition so it can be referred to throughout the apidsl.
func MediaType(identifier string, apidsl func()) *design.MediaTypeDefinition {
	if design.Design.MediaTypes == nil {
		design.Design.MediaTypes = make(map[string]*design.MediaTypeDefinition)
	}

	if !dslengine.IsTopLevelDefinition() {
		dslengine.IncompatibleDSL()
		return nil
	}

	// Validate Media Type
	identifier, params, err := mime.ParseMediaType(identifier)
	if err != nil {
		dslengine.ReportError("invalid media type identifier %#v: %s",
			identifier, err)
		// We don't return so that other errors may be
		// captured in this one run.
		identifier = "plain/text"
	}
	canonicalID := design.CanonicalIdentifier(identifier)
	// Validate that media type identifier doesn't clash
	if _, ok := design.Design.MediaTypes[canonicalID]; ok {
		dslengine.ReportError("media type %#v is defined twice", identifier)
		return nil
	}
	parts := strings.Split(identifier, "+")
	// Make sure it has the `+json` suffix (TBD update when goa supports other encodings)
	if len(parts) > 1 {
		parts = parts[1:]
		found := false
		for _, part := range parts {
			if part == "json" {
				found = true
				break
			}
		}
		if !found {
			identifier += "+json"
		}
	}
	identifier = mime.FormatMediaType(identifier, params)
	// Concoct a Go type name from the identifier, should it be possible to set it in the apidsl?
	// pros: control the type name generated, cons: not needed in apidsl, adds one more thing to worry about
	lastPart := identifier
	lastPartIndex := strings.LastIndex(identifier, "/")
	if lastPartIndex > -1 {
		lastPart = identifier[lastPartIndex+1:]
	}
	plusIndex := strings.Index(lastPart, "+")
	if plusIndex > 0 {
		lastPart = lastPart[:plusIndex]
	}
	lastPart = strings.TrimPrefix(lastPart, "vnd.")
	elems := strings.Split(lastPart, ".")
	for i, e := range elems {
		elems[i] = strings.Title(e)
	}
	typeName := strings.Join(elems, "")
	if typeName == "" {
		mediaTypeCount++
		typeName = fmt.Sprintf("MediaType%d", mediaTypeCount)
	}
	// Now save the type in the API media types map
	mt := design.NewMediaTypeDefinition(typeName, identifier, apidsl)
	design.Design.MediaTypes[canonicalID] = mt
	return mt

}

// Media sets a response media type by name or by reference using a value returned by MediaType:
//
//	Response("NotFound", func() {
//		Status(404)
//		Media("application/json")
//	})
//
// Media can be used inside Response or ResponseTemplate.
func Media(val interface{}) {
	if r, ok := responseDefinition(); ok {
		if m, ok := val.(*design.MediaTypeDefinition); ok {
			if m != nil {
				r.MediaType = m.Identifier
			}
		} else if identifier, ok := val.(string); ok {
			r.MediaType = identifier
		} else {
			dslengine.ReportError("media type must be a string or a pointer to MediaTypeDefinition, got %#v", val)
		}
	}
}

// Reference sets a type or media type reference. The value itself can be a type or a media type.
// The reference type attributes define the default properties for attributes with the same name in
// the type using the reference. So for example if a type is defined as such:
//
//	var Bottle = Type("bottle", func() {
//		Attribute("name", func() {
//			MinLength(3)
//		})
//		Attribute("vintage", Integer, func() {
//			Minimum(1970)
//		})
//		Attribute("somethingelse")
//	})
//
// Declaring the following media type:
//
//	var BottleMedia = MediaType("vnd.goa.bottle", func() {
//		Reference(Bottle)
//		Attributes(func() {
//			Attribute("id", Integer)
//			Attribute("name")
//			Attribute("vintage")
//		})
//	})
//
// defines the "name" and "vintage" attributes with the same type and validations as defined in
// the Bottle type.
func Reference(t design.DataType) {
	switch def := dslengine.CurrentDefinition().(type) {
	case *design.MediaTypeDefinition:
		def.Reference = t
	case *design.AttributeDefinition:
		def.Reference = t
	default:
		dslengine.IncompatibleDSL()
	}
}

// TypeName makes it possible to set the Go struct name for a type or media type in the generated
// code. By default goagen uses the name (type) or identifier (media type) given in the apidsl and
// computes a valid Go identifier from it. This function makes it possible to override that and
// provide a custom name. name must be a valid Go identifier.
func TypeName(name string) {
	switch def := dslengine.CurrentDefinition().(type) {
	case *design.MediaTypeDefinition:
		def.TypeName = name
	case *design.UserTypeDefinition:
		def.TypeName = name
	default:
		dslengine.IncompatibleDSL()
	}
}

// View adds a new view to a media type. A view has a name and lists attributes that are
// rendered when the view is used to produce a response. The attribute names must appear in the
// media type definition. If an attribute is itself a media type then the view may specify which
// view to use when rendering the attribute using the View function in the View apidsl. If not
// specified then the view named "default" is used. Examples:
//
//	View("default", func() {
//		Attribute("id")		// "id" and "name" must be media type attributes
//		Attribute("name")
//	})
//
//	View("extended", func() {
//		Attribute("id")
//		Attribute("name")
//		Attribute("origin", func() {
//			View("extended")	// Use view "extended" to render attribute "origin"
//		})
//	})
func View(name string, apidsl ...func()) {
	switch def := dslengine.CurrentDefinition().(type) {
	case *design.MediaTypeDefinition:
		mt := def

		if !mt.Type.IsObject() && !mt.Type.IsArray() {
			dslengine.ReportError("cannot define view on non object and non collection media types")
			return
		}
		if mt.Views == nil {
			mt.Views = make(map[string]*design.ViewDefinition)
		} else {
			if _, ok := mt.Views[name]; ok {
				dslengine.ReportError("multiple definitions for view %#v in media type %#v", name, mt.TypeName)
				return
			}
		}
		at := &design.AttributeDefinition{}
		ok := false
		if len(apidsl) > 0 {
			ok = dslengine.Execute(apidsl[0], at)
		} else if mt.Type.IsArray() {
			// inherit view from collection element if present
			elem := mt.Type.ToArray().ElemType
			if elem != nil {
				if pa, ok2 := elem.Type.(*design.MediaTypeDefinition); ok2 {
					if v, ok2 := pa.Views[name]; ok2 {
						at = v.AttributeDefinition
						ok = true
					} else {
						dslengine.ReportError("unknown view %#v", name)
						return
					}
				}
			}
		}
		if ok {
			view, err := buildView(name, mt, at)
			if err != nil {
				dslengine.ReportError(err.Error())
				return
			}
			mt.Views[name] = view
		}

	case *design.AttributeDefinition:
		def.View = name

	default:
		dslengine.IncompatibleDSL()
	}
}

// buildView builds a view definition given an attribute and a corresponding media type.
func buildView(name string, mt *design.MediaTypeDefinition, at *design.AttributeDefinition) (*design.ViewDefinition, error) {
	if at.Type == nil || !at.Type.IsObject() {
		return nil, fmt.Errorf("invalid view DSL")
	}
	o := at.Type.ToObject()
	if o != nil {
		mto := mt.Type.ToObject()
		if mto == nil {
			mto = mt.Type.ToArray().ElemType.Type.ToObject()
		}
		for n, cat := range o {
			if existing, ok := mto[n]; ok {
				dup := design.DupAtt(existing)
				dup.View = cat.View
				o[n] = dup
			} else if n != "links" {
				return nil, fmt.Errorf("unknown attribute %#v", n)
			}
		}
	}
	return &design.ViewDefinition{
		AttributeDefinition: at,
		Name:                name,
		Parent:              mt,
	}, nil
}

// Attributes implements the media type attributes apidsl. See MediaType.
func Attributes(apidsl func()) {
	if mt, ok := mediaTypeDefinition(); ok {
		dslengine.Execute(apidsl, mt)
	}
}

// Links implements the media type links apidsl. See MediaType.
func Links(apidsl func()) {
	if mt, ok := mediaTypeDefinition(); ok {
		dslengine.Execute(apidsl, mt)
	}
}

// Link adds a link to a media type. At the minimum a link has a name corresponding to one of the
// media type attribute names. A link may also define the view used to render the linked-to
// attribute. The default view used to render links is "link". Examples:
//
//	Link("origin")		// Use the "link" view of the "origin" attribute
//	Link("account", "tiny")	// Use the "tiny" view of the "account" attribute
func Link(name string, view ...string) {
	if mt, ok := mediaTypeDefinition(); ok {
		if mt.Links == nil {
			mt.Links = make(map[string]*design.LinkDefinition)
		} else {
			if _, ok := mt.Links[name]; ok {
				dslengine.ReportError("duplicate definition for link %#v", name)
				return
			}
		}
		link := &design.LinkDefinition{Name: name, Parent: mt}
		if len(view) > 1 {
			dslengine.ReportError("invalid syntax in Link definition for %#v, allowed syntax is Link(name) or Link(name, view)", name)
		}
		if len(view) > 0 {
			link.View = view[0]
		} else {
			link.View = "link"
		}
		mt.Links[name] = link
	}
}

// CollectionOf creates a collection media type from its element media type. A collection media
// type represents the content of responses that return a collection of resources such as "list"
// actions. This function can be called from any place where a media type can be used.
// The resulting media type identifier is built from the element media type by appending the media
// type parameter "type" with value "collection".
func CollectionOf(v interface{}, apidsl ...func()) *design.MediaTypeDefinition {
	var m *design.MediaTypeDefinition
	var ok bool
	m, ok = v.(*design.MediaTypeDefinition)
	if !ok {
		if id, ok := v.(string); ok {
			m = design.Design.MediaTypes[design.CanonicalIdentifier(id)]
		}
	}
	if m == nil {
		dslengine.ReportError("invalid CollectionOf argument: not a media type and not a known media type identifier")
		return nil
	}
	id := m.Identifier
	mediatype, params, err := mime.ParseMediaType(id)
	if err != nil {
		dslengine.ReportError("invalid media type identifier %#v: %s", id, err)
		return nil
	}
	hasType := false
	for param := range params {
		if param == "type" {
			hasType = true
			break
		}
	}
	if !hasType {
		params["type"] = "collection"
	}
	id = mime.FormatMediaType(mediatype, params)
	typeName := m.TypeName + "Collection"
	if mt, ok := design.GeneratedMediaTypes[typeName]; ok {
		// Already have a type for this collection, reuse it.
		return mt
	}
	mt := design.NewMediaTypeDefinition(typeName, id, func() {
		if mt, ok := mediaTypeDefinition(); ok {
			mt.TypeName = typeName
			mt.AttributeDefinition = &design.AttributeDefinition{Type: ArrayOf(m)}
			if len(apidsl) > 0 {
				dslengine.Execute(apidsl[0], mt)
			}
			if mt.Views == nil {
				// If the apidsl didn't create any views (or there is no apidsl at all)
				// then inherit the views from the collection element.
				mt.Views = make(map[string]*design.ViewDefinition)
				for n, v := range m.Views {
					mt.Views[n] = v
				}
			}
		}
	})
	// Do not execute the apidsl right away, will be done last to make sure the element apidsl has run
	// first.
	design.GeneratedMediaTypes[typeName] = mt
	return mt
}
<|endoftext|>


<commit_before>package gitnotify

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	yaml "gopkg.in/yaml.v2"
)

// Setting is the data structure that has all the details
//  data/$provider/$username/settings.yml
type Setting struct {
	Version `yaml:"version"`
	Repos   []*Repo                 `yaml:"repos"`
	Orgs    []*Organisation         `yaml:"orgs"`
	Auth    *Authentication         `yaml:"auth"`
	User    *UserNotification       `yaml:"user_notification"`
	Info    map[string]*Information `yaml:"fetched_info"`
}

func (c *Setting) usersEmail() string {
	if c.User.Email == "" {
		return c.Auth.Email
	}
	return c.User.Email
}

func (c *Setting) usersName() string {
	if c.User.Name == "" {
		return c.Auth.Name
	}
	return c.User.Name
}

func (c *Setting) anyValidNotifications() bool {
	return config.isEmailSetup() || c.User.isValidWebhook()
}

// UserNotification is the customization/scheduling is provided for user
// NOTE: this can be an array of notifications like email, Webhook options.
// not gonna make the change to the array
type UserNotification struct {
	Email     string `yaml:"email"`
	Name      string `yaml:"name"`
	Disabled  bool   `yaml:"disabled"`
	Frequency `yaml:",inline"`

	WebhookURL  string `yaml:"webhook_url"`
	WebhookType string `yaml:"webhook_type"`
}

func (u *UserNotification) isValidWebhook() bool {
	return StringIn(config.WebhookIntegrations, u.WebhookType) && u.WebhookURL != ""
}

// Frequency is the cron format along with a TimeZone to process
// Minute, Monthday and Month cannot be controlled. Consider them to be '*'
type Frequency struct {
	TimeZone     string `yaml:"tz"`
	TimeZoneName string `yaml:"tzname"`
	// Minute string // 0-59 allowed
	Hour string `yaml:"hour"` // Hour - 0-23 allowed
	// MonthDay string // 1-31 allowed. Ignore if you want to use weekday vs weekend
	// Month - cannot be set
	WeekDay string `yaml:"weekday"` // 0-6 to point SUN-SAT
}

// Information has the type and contains either Org or Repo Information.
// This is more of an interface/abstract role based on type
type Information struct {
	Repo RepoInformation `yaml:",inline"`
	Org  OrgInformation  `yaml:",inline"`
	Type string          `yaml:"type"` // type is either Org or Repo
}

// UnmarshalYAML overrides the default unmarshaling logic
func (i *Information) UnmarshalYAML(unmarshal func(interface{}) error) error {
	var infoType struct {
		Type string `yaml:"type"`
	}
	unmarshal(&infoType)
	if infoType.Type == "repo" || infoType.Type == "" {
		var r RepoInformation
		unmarshal(&r)
		*i = Information{Repo: r, Type: "repo"}
	} else if infoType.Type == "org" {
		var r OrgInformation
		unmarshal(&r)
		*i = Information{Org: r, Type: "org"}
	}
	return nil
}

func (i *Information) getType() interface{} {
	if i.Type == "repo" || i.Type == "" {
		return i.Repo
	} else if i.Type == "org" {
		return i.Org
	}
	return nil
}

// OrgInformation is info for org and ref_type
type OrgInformation struct {
	OrgType string   `yaml:"org_type,omitempty"`
	Repos   []string `yaml:"repos,omitempty,flow"`
}

// RepoInformation is all the information fetched from remote location, updated and saved
// contains the fetched_info
type RepoInformation struct {
	Tags     []string       `yaml:"tags,omitempty,flow"`
	Branches []string       `yaml:"branches,omitempty,flow"`
	Commits  LocalCommitRef `yaml:"commits,omitempty"`
}

func newRepoInformation() *Information {
	i := &Information{}
	i.Repo = RepoInformation{}
	i.Repo.Commits = make(LocalCommitRef)
	return i
}

// LocalCommitRef is of the form map[BranchName] = "1234567890abcdef"
type LocalCommitRef map[string]string

// Version of the structure
type Version string

// Organisation is a user/org that is being tracked
type Organisation struct {
	Name     string `yaml:"name"`
	Type     string `yaml:"type"`
	Provider string
}

// Repo is a repository that is being tracked
type Repo struct {
	Repo            string      `yaml:"repo"`
	NamedReferences []reference `yaml:"commits"`
	Branches        bool        `yaml:"new_branches"`
	Tags            bool        `yaml:"new_tags"`
	Provider        string
}
type reference string

func (c *Setting) String() string {
	arr := make([]string, len(c.Repos))
	for i, repo := range c.Repos {
		arr[i] = fmt.Sprint(repo)
	}
	return strings.Join(arr, "\n")
}

func (r *Repo) String() string {
	return fmt.Sprintf("repo: %s, references: %v, branches: %t, tags: %t", r.Repo, r.NamedReferences, r.Branches, r.Tags)
}

func (x reference) String() string { return fmt.Sprintf("%s", string(x)) }

// read setting from file into memory
func (c *Setting) load(settingFile string) error {

	if _, err := os.Stat(settingFile); os.IsNotExist(err) {
		return nil
	}

	data, err := ioutil.ReadFile(settingFile)
	if os.IsNotExist(err) {
		return err
	}

	err = yaml.Unmarshal(data, c)

	// set provider at repo level
	if c.Auth.Provider != "" {
		for _, repo := range c.Repos {
			repo.Provider = c.Auth.Provider
		}
	}

	if err != nil {
		return err
	}
	if c.Info == nil {
		c.Info = make(map[string]*Information)
	}

	if c.User == nil {
		c.User = new(UserNotification)
	}

	return nil
}

// persists setting into file
func (c *Setting) save(settingFile string) error {
	out, err := yaml.Marshal(c)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(settingFile, out, 0600)
}
<commit_msg>fix for missing CommitRef panics<commit_after>package gitnotify

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	yaml "gopkg.in/yaml.v2"
)

// Setting is the data structure that has all the details
//  data/$provider/$username/settings.yml
type Setting struct {
	Version `yaml:"version"`
	Repos   []*Repo                 `yaml:"repos"`
	Orgs    []*Organisation         `yaml:"orgs"`
	Auth    *Authentication         `yaml:"auth"`
	User    *UserNotification       `yaml:"user_notification"`
	Info    map[string]*Information `yaml:"fetched_info"`
}

func (c *Setting) usersEmail() string {
	if c.User.Email == "" {
		return c.Auth.Email
	}
	return c.User.Email
}

func (c *Setting) usersName() string {
	if c.User.Name == "" {
		return c.Auth.Name
	}
	return c.User.Name
}

func (c *Setting) anyValidNotifications() bool {
	return config.isEmailSetup() || c.User.isValidWebhook()
}

// UserNotification is the customization/scheduling is provided for user
// NOTE: this can be an array of notifications like email, Webhook options.
// not gonna make the change to the array
type UserNotification struct {
	Email     string `yaml:"email"`
	Name      string `yaml:"name"`
	Disabled  bool   `yaml:"disabled"`
	Frequency `yaml:",inline"`

	WebhookURL  string `yaml:"webhook_url"`
	WebhookType string `yaml:"webhook_type"`
}

func (u *UserNotification) isValidWebhook() bool {
	return StringIn(config.WebhookIntegrations, u.WebhookType) && u.WebhookURL != ""
}

// Frequency is the cron format along with a TimeZone to process
// Minute, Monthday and Month cannot be controlled. Consider them to be '*'
type Frequency struct {
	TimeZone     string `yaml:"tz"`
	TimeZoneName string `yaml:"tzname"`
	// Minute string // 0-59 allowed
	Hour string `yaml:"hour"` // Hour - 0-23 allowed
	// MonthDay string // 1-31 allowed. Ignore if you want to use weekday vs weekend
	// Month - cannot be set
	WeekDay string `yaml:"weekday"` // 0-6 to point SUN-SAT
}

// Information has the type and contains either Org or Repo Information.
// This is more of an interface/abstract role based on type
type Information struct {
	Repo RepoInformation `yaml:",inline"`
	Org  OrgInformation  `yaml:",inline"`
	Type string          `yaml:"type"` // type is either Org or Repo
}

// UnmarshalYAML overrides the default unmarshaling logic
func (i *Information) UnmarshalYAML(unmarshal func(interface{}) error) error {
	var infoType struct {
		Type string `yaml:"type"`
	}
	unmarshal(&infoType)
	if infoType.Type == "repo" || infoType.Type == "" {
		var r RepoInformation
		unmarshal(&r)
		// TODO - add test case
		//   hashicorp/terraform:
		//     type: repo
		//    // commits is missing
		if r.Commits == nil {
			r.Commits = make(LocalCommitRef)
		}
		*i = Information{Repo: r, Type: "repo"}
	} else if infoType.Type == "org" {
		var r OrgInformation
		unmarshal(&r)
		*i = Information{Org: r, Type: "org"}
	}
	return nil
}

func (i *Information) getType() interface{} {
	if i.Type == "repo" || i.Type == "" {
		return i.Repo
	} else if i.Type == "org" {
		return i.Org
	}
	return nil
}

// OrgInformation is info for org and ref_type
type OrgInformation struct {
	OrgType string   `yaml:"org_type,omitempty"`
	Repos   []string `yaml:"repos,omitempty,flow"`
}

// RepoInformation is all the information fetched from remote location, updated and saved
// contains the fetched_info
type RepoInformation struct {
	Tags     []string       `yaml:"tags,omitempty,flow"`
	Branches []string       `yaml:"branches,omitempty,flow"`
	Commits  LocalCommitRef `yaml:"commits,omitempty"`
}

func newRepoInformation() *Information {
	i := &Information{}
	i.Repo = RepoInformation{}
	i.Repo.Commits = make(LocalCommitRef)
	return i
}

// LocalCommitRef is of the form map[BranchName] = "1234567890abcdef"
type LocalCommitRef map[string]string

// Version of the structure
type Version string

// Organisation is a user/org that is being tracked
type Organisation struct {
	Name     string `yaml:"name"`
	Type     string `yaml:"type"`
	Provider string
}

// Repo is a repository that is being tracked
type Repo struct {
	Repo            string      `yaml:"repo"`
	NamedReferences []reference `yaml:"commits"`
	Branches        bool        `yaml:"new_branches"`
	Tags            bool        `yaml:"new_tags"`
	Provider        string
}
type reference string

func (c *Setting) String() string {
	arr := make([]string, len(c.Repos))
	for i, repo := range c.Repos {
		arr[i] = fmt.Sprint(repo)
	}
	return strings.Join(arr, "\n")
}

func (r *Repo) String() string {
	return fmt.Sprintf("repo: %s, references: %v, branches: %t, tags: %t", r.Repo, r.NamedReferences, r.Branches, r.Tags)
}

func (x reference) String() string { return fmt.Sprintf("%s", string(x)) }

// read setting from file into memory
func (c *Setting) load(settingFile string) error {

	if _, err := os.Stat(settingFile); os.IsNotExist(err) {
		return nil
	}

	data, err := ioutil.ReadFile(settingFile)
	if os.IsNotExist(err) {
		return err
	}

	err = yaml.Unmarshal(data, c)

	// set provider at repo level
	if c.Auth.Provider != "" {
		for _, repo := range c.Repos {
			repo.Provider = c.Auth.Provider
		}
	}

	if err != nil {
		return err
	}
	if c.Info == nil {
		c.Info = make(map[string]*Information)
	}

	if c.User == nil {
		c.User = new(UserNotification)
	}

	return nil
}

// persists setting into file
func (c *Setting) save(settingFile string) error {
	out, err := yaml.Marshal(c)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(settingFile, out, 0600)
}
<|endoftext|>


<commit_before><commit_msg>No command eval should disallow commands above working dir<commit_after><|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package netchan

import (
	"fmt"
	"testing"
)

type value struct {
	i int
	s string
}

const count = 10

func exportSend(exp *Exporter, t *testing.T) {
	ch := make(chan value)
	err := exp.Export("exportedSend", ch, Send, new(value))
	if err != nil {
		t.Fatal("exportSend:", err)
	}
	for i := 0; i < count; i++ {
		ch <- value{23 + i, "hello"}
	}
}

func exportReceive(exp *Exporter, t *testing.T) {
	ch := make(chan value)
	err := exp.Export("exportedRecv", ch, Recv, new(value))
	if err != nil {
		t.Fatal("exportReceive:", err)
	}
	for i := 0; i < count; i++ {
		v := <-ch
		fmt.Printf("%v\n", v)
		if v.i != 45+i || v.s != "hello" {
			t.Errorf("export Receive: bad value: expected 4%d, hello; got %+v", 45+i, v)
		}
	}
}

func importReceive(imp *Importer, t *testing.T) {
	ch := make(chan value)
	err := imp.ImportNValues("exportedSend", ch, Recv, new(value), count)
	if err != nil {
		t.Fatal("importReceive:", err)
	}
	for i := 0; i < count; i++ {
		v := <-ch
		fmt.Printf("%v\n", v)
		if v.i != 23+i || v.s != "hello" {
			t.Errorf("importReceive: bad value: expected %d, hello; got %+v", 23+i, v)
		}
	}
}

func importSend(imp *Importer, t *testing.T) {
	ch := make(chan value)
	err := imp.ImportNValues("exportedRecv", ch, Send, new(value), count)
	if err != nil {
		t.Fatal("importSend:", err)
	}
	for i := 0; i < count; i++ {
		ch <- value{45 + i, "hello"}
	}
}

func TestExportSendImportReceive(t *testing.T) {
	exp, err := NewExporter("tcp", ":0")
	if err != nil {
		t.Fatal("new exporter:", err)
	}
	imp, err := NewImporter("tcp", exp.Addr().String())
	if err != nil {
		t.Fatal("new importer:", err)
	}
	go exportSend(exp, t)
	importReceive(imp, t)
}

func TestExportReceiveImportSend(t *testing.T) {
	exp, err := NewExporter("tcp", ":0")
	if err != nil {
		t.Fatal("new exporter:", err)
	}
	imp, err := NewImporter("tcp", exp.Addr().String())
	if err != nil {
		t.Fatal("new importer:", err)
	}
	go importSend(imp, t)
	exportReceive(exp, t)
}
<commit_msg>netchan: be less chatty during gotest<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package netchan

import "testing"

type value struct {
	i int
	s string
}

const count = 10

func exportSend(exp *Exporter, t *testing.T) {
	ch := make(chan value)
	err := exp.Export("exportedSend", ch, Send, new(value))
	if err != nil {
		t.Fatal("exportSend:", err)
	}
	for i := 0; i < count; i++ {
		ch <- value{23 + i, "hello"}
	}
}

func exportReceive(exp *Exporter, t *testing.T) {
	ch := make(chan value)
	err := exp.Export("exportedRecv", ch, Recv, new(value))
	if err != nil {
		t.Fatal("exportReceive:", err)
	}
	for i := 0; i < count; i++ {
		v := <-ch
		if v.i != 45+i || v.s != "hello" {
			t.Errorf("export Receive: bad value: expected 4%d, hello; got %+v", 45+i, v)
		}
	}
}

func importReceive(imp *Importer, t *testing.T) {
	ch := make(chan value)
	err := imp.ImportNValues("exportedSend", ch, Recv, new(value), count)
	if err != nil {
		t.Fatal("importReceive:", err)
	}
	for i := 0; i < count; i++ {
		v := <-ch
		if v.i != 23+i || v.s != "hello" {
			t.Errorf("importReceive: bad value: expected %d, hello; got %+v", 23+i, v)
		}
	}
}

func importSend(imp *Importer, t *testing.T) {
	ch := make(chan value)
	err := imp.ImportNValues("exportedRecv", ch, Send, new(value), count)
	if err != nil {
		t.Fatal("importSend:", err)
	}
	for i := 0; i < count; i++ {
		ch <- value{45 + i, "hello"}
	}
}

func TestExportSendImportReceive(t *testing.T) {
	exp, err := NewExporter("tcp", ":0")
	if err != nil {
		t.Fatal("new exporter:", err)
	}
	imp, err := NewImporter("tcp", exp.Addr().String())
	if err != nil {
		t.Fatal("new importer:", err)
	}
	go exportSend(exp, t)
	importReceive(imp, t)
}

func TestExportReceiveImportSend(t *testing.T) {
	exp, err := NewExporter("tcp", ":0")
	if err != nil {
		t.Fatal("new exporter:", err)
	}
	imp, err := NewImporter("tcp", exp.Addr().String())
	if err != nil {
		t.Fatal("new importer:", err)
	}
	go importSend(imp, t)
	exportReceive(exp, t)
}
<|endoftext|>


<commit_before>package mechturk

// NOTE: This code has been adapted from the generated output
// of gowsdl (github.com/hooklift/gowsdl). See comments in wsdltypes.go
// re possible changes to gowsdl. Eventual goal is to auto-generate
// code that can be auto-generated, but for now I'm just focussing on
// getting some code to work.

import "github.com/jjeffery/mturk/soap"

type MTurk struct {
	client *soap.Client
}

func NewMTurk(url string, tls bool, auth *soap.BasicAuth) *MTurk {
	if url == "" {
		url = ""
	}
	client := soap.NewClient(url, tls, auth)

	return &MTurk{
		client: client,
	}
}

func (service *MTurk) CreateHIT(request *CreateHIT) (*CreateHITResponse, error) {
	response := new(CreateHITResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) RegisterHITType(request *RegisterHITType) (*RegisterHITTypeResponse, error) {
	response := new(RegisterHITTypeResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) SetHITTypeNotification(request *SetHITTypeNotification) (*SetHITTypeNotificationResponse, error) {
	response := new(SetHITTypeNotificationResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) SendTestEventNotification(request *SendTestEventNotification) (*SendTestEventNotificationResponse, error) {
	response := new(SendTestEventNotificationResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) DisposeHIT(request *DisposeHIT) (*DisposeHITResponse, error) {
	response := new(DisposeHITResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) DisableHIT(request *DisableHIT) (*DisableHITResponse, error) {
	response := new(DisableHITResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) GetHIT(request *GetHIT) (*GetHITResponse, error) {
	response := new(GetHITResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) GetAssignment(request *GetAssignment) (*GetAssignmentResponse, error) {
	response := new(GetAssignmentResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) GetReviewResultsForHIT(request *GetReviewResultsForHIT) (*GetReviewResultsForHITResponse, error) {
	response := new(GetReviewResultsForHITResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) GetReviewableHITs(request *GetReviewableHITs) (*GetReviewableHITsResponse, error) {
	response := new(GetReviewableHITsResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) GetHITsForQualificationType(request *GetHITsForQualificationType) (*GetHITsForQualificationTypeResponse, error) {
	response := new(GetHITsForQualificationTypeResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) GetQualificationsForQualificationType(request *GetQualificationsForQualificationType) (*GetQualificationsForQualificationTypeResponse, error) {
	response := new(GetQualificationsForQualificationTypeResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) SetHITAsReviewing(request *SetHITAsReviewing) (*SetHITAsReviewingResponse, error) {
	response := new(SetHITAsReviewingResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) ExtendHIT(request *ExtendHIT) (*ExtendHITResponse, error) {
	response := new(ExtendHITResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) ForceExpireHIT(request *ForceExpireHIT) (*ForceExpireHITResponse, error) {
	response := new(ForceExpireHITResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) ApproveAssignment(request *ApproveAssignment) (*ApproveAssignmentResponse, error) {
	response := new(ApproveAssignmentResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) RejectAssignment(request *RejectAssignment) (*RejectAssignmentResponse, error) {
	response := new(RejectAssignmentResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) ApproveRejectedAssignment(request *ApproveRejectedAssignment) (*ApproveRejectedAssignmentResponse, error) {
	response := new(ApproveRejectedAssignmentResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) GetAssignmentsForHIT(request *GetAssignmentsForHIT) (*GetAssignmentsForHITResponse, error) {
	response := new(GetAssignmentsForHITResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) GetFileUploadURL(request *GetFileUploadURL) (*GetFileUploadURLResponse, error) {
	response := new(GetFileUploadURLResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) SearchHITs(request *SearchHITs) (*SearchHITsResponse, error) {
	response := new(SearchHITsResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) GrantBonus(request *GrantBonus) (*GrantBonusResponse, error) {
	response := new(GrantBonusResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) GetBonusPayments(request *GetBonusPayments) (*GetBonusPaymentsResponse, error) {
	response := new(GetBonusPaymentsResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) ChangeHITTypeOfHIT(request *ChangeHITTypeOfHIT) (*ChangeHITTypeOfHITResponse, error) {
	response := new(ChangeHITTypeOfHITResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) CreateQualificationType(request *CreateQualificationType) (*CreateQualificationTypeResponse, error) {
	response := new(CreateQualificationTypeResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) GrantQualification(request *GrantQualification) (*GrantQualificationResponse, error) {
	response := new(GrantQualificationResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) AssignQualification(request *AssignQualification) (*AssignQualificationResponse, error) {
	response := new(AssignQualificationResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) RevokeQualification(request *RevokeQualification) (*RevokeQualificationResponse, error) {
	response := new(RevokeQualificationResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) GetQualificationType(request *GetQualificationType) (*GetQualificationTypeResponse, error) {
	response := new(GetQualificationTypeResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) GetQualificationRequests(request *GetQualificationRequests) (*GetQualificationRequestsResponse, error) {
	response := new(GetQualificationRequestsResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) RejectQualificationRequest(request *RejectQualificationRequest) (*RejectQualificationRequestResponse, error) {
	response := new(RejectQualificationRequestResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) UpdateQualificationType(request *UpdateQualificationType) (*UpdateQualificationTypeResponse, error) {
	response := new(UpdateQualificationTypeResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) SearchQualificationTypes(request *SearchQualificationTypes) (*SearchQualificationTypesResponse, error) {
	response := new(SearchQualificationTypesResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) GetQualificationScore(request *GetQualificationScore) (*GetQualificationScoreResponse, error) {
	response := new(GetQualificationScoreResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) UpdateQualificationScore(request *UpdateQualificationScore) (*UpdateQualificationScoreResponse, error) {
	response := new(UpdateQualificationScoreResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) DisposeQualificationType(request *DisposeQualificationType) (*DisposeQualificationTypeResponse, error) {
	response := new(DisposeQualificationTypeResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) GetRequesterStatistic(request *GetRequesterStatistic) (*GetRequesterStatisticResponse, error) {
	response := new(GetRequesterStatisticResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) GetRequesterWorkerStatistic(request *GetRequesterWorkerStatistic) (*GetRequesterWorkerStatisticResponse, error) {
	response := new(GetRequesterWorkerStatisticResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) NotifyWorkers(request *NotifyWorkers) (*NotifyWorkersResponse, error) {
	response := new(NotifyWorkersResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) GetAccountBalance(request *GetAccountBalance) (*GetAccountBalanceResponse, error) {
	response := new(GetAccountBalanceResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) GetBlockedWorkers(request *GetBlockedWorkers) (*GetBlockedWorkersResponse, error) {
	response := new(GetBlockedWorkersResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) BlockWorker(request *BlockWorker) (*BlockWorkerResponse, error) {
	response := new(BlockWorkerResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) UnblockWorker(request *UnblockWorker) (*UnblockWorkerResponse, error) {
	response := new(UnblockWorkerResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (service *MTurk) Help(request *Help) (*HelpResponse, error) {
	response := new(HelpResponse)
	err := service.client.Call("http://soap.amazon.com", request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}
<commit_msg>Remove _operations.go<commit_after><|endoftext|>


<commit_before>// Package aws provides core functionality for making requests to AWS services.
package aws

// SDKName is the name of this AWS SDK
const SDKName = "aws-sdk-go"

// SDKVersion is the version of this SDK
const SDKVersion = "1.1.12"
<commit_msg>Tag release v1.1.13<commit_after>// Package aws provides core functionality for making requests to AWS services.
package aws

// SDKName is the name of this AWS SDK
const SDKName = "aws-sdk-go"

// SDKVersion is the version of this SDK
const SDKVersion = "1.1.13"
<|endoftext|>


<commit_before>// Package aws provides core functionality for making requests to AWS services.
package aws

// SDKName is the name of this AWS SDK
const SDKName = "aws-sdk-go"

// SDKVersion is the version of this SDK
const SDKVersion = "0.9.17"
<commit_msg>Tag release vrelease-v0.10.0<commit_after>// Package aws provides core functionality for making requests to AWS services.
package aws

// SDKName is the name of this AWS SDK
const SDKName = "aws-sdk-go"

// SDKVersion is the version of this SDK
const SDKVersion = "release-v0.10.0"
<|endoftext|>


<commit_before>// Copyright (c) 2016, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package sh

import (
	"reflect"
	"strings"
	"testing"
)

var tests = []struct {
	in   string
	want []node
}{
	{
		in:   "",
		want: nil,
	},
	{
		in:   "# comment",
		want: nil,
	},
	{
		in: "foo",
		want: []node{
			command{args: []lit{"foo"}},
		},
	},
	{
		in: "foo; bar;",
		want: []node{
			command{args: []lit{"foo"}},
			command{args: []lit{"bar"}},
		},
	},
	{
		in: "# comment\nfoo",
		want: []node{
			command{args: []lit{"foo"}},
		},
	},
	{
		in: "foo arg1 arg2",
		want: []node{
			command{args: []lit{"foo", "arg1", "arg2"}},
		},
	},
	{
		in: "( foo; )",
		want: []node{
			subshell{stmts: []node{
				command{args: []lit{"foo"}},
			}},
		},
	},
	{
		in: "{ foo; }",
		want: []node{
			block{stmts: []node{
				command{args: []lit{"foo"}},
			}},
		},
	},
	{
		in: "if foo; then bar; fi",
		want: []node{ifStmt{
			cond: command{args: []lit{"foo"}},
			thenStmts: []node{
				command{args: []lit{"bar"}},
			}},
		},
	},
	{
		in: "if foo; then bar; else pass; fi",
		want: []node{ifStmt{
			cond: command{args: []lit{"foo"}},
			thenStmts: []node{
				command{args: []lit{"bar"}},
			},
			elseStmts: []node{
				command{args: []lit{"pass"}},
			}},
		},
	},
	{
		in: "if a; then a; elif b; then b; elif c; then c; else d; fi",
		want: []node{ifStmt{
			cond: command{args: []lit{"a"}},
			thenStmts: []node{
				command{args: []lit{"a"}},
			},
			elifs: []node{
				elif{cond: command{args: []lit{"b"}},
					thenStmts: []node{
						command{args: []lit{"b"}},
					}},
				elif{cond: command{args: []lit{"c"}},
					thenStmts: []node{
						command{args: []lit{"c"}},
					}},
			},
			elseStmts: []node{
				command{args: []lit{"d"}},
			}},
		},
	},
	{
		in: "while foo; do bar; done",
		want: []node{whileStmt{
			cond: command{args: []lit{"foo"}},
			doStmts: []node{
				command{args: []lit{"bar"}},
			}},
		},
	},
	{
		in: "echo ' ' \"foo bar\"",
		want: []node{
			command{args: []lit{"echo", "' '", "\"foo bar\""}},
		},
	},
	{
		in: "$a ${b} s{s s=s",
		want: []node{
			command{args: []lit{"$a", "${b}", "s{s", "s=s"}},
		},
	},
	{
		in: "foo && bar",
		want: []node{binaryExpr{
			op: "&&",
			X:  command{args: []lit{"foo"}},
			Y:  command{args: []lit{"bar"}},
		}},
	},
	{
		in: "foo && bar || else",
		want: []node{binaryExpr{
			op: "&&",
			X:  command{args: []lit{"foo"}},
			Y: binaryExpr{
				op: "||",
				X:  command{args: []lit{"bar"}},
				Y:  command{args: []lit{"else"}},
			},
		}},
	},
}

func TestParseAST(t *testing.T) {
	for _, c := range tests {
		r := strings.NewReader(c.in)
		got, err := parse(r, "")
		if err != nil {
			t.Fatalf("Unexpected error in %q: %v", c.in, err)
		}
		want := prog{
			stmts: c.want,
		}
		if !reflect.DeepEqual(got, want) {
			t.Fatalf("AST mismatch in %q\nwant: %s\ngot:  %s\ndumps:\n%#v\n%#v",
				c.in, want.String(), got.String(), want, got)
		}
	}
}
<commit_msg>ast_test: use multiple equivalent inputs<commit_after>// Copyright (c) 2016, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package sh

import (
	"reflect"
	"strings"
	"testing"
)

var tests = []struct {
	ins  []string
	want []node
}{
	{
		ins:  []string{"", " ", "\n"},
		want: nil,
	},
	{
		ins:  []string{"#a", "# a b", "#a\n#b"},
		want: nil,
	},
	{
		ins: []string{"foo", "foo ", " foo"},
		want: []node{
			command{args: []lit{"foo"}},
		},
	},
	{
		ins: []string{"foo; bar", "foo; bar;", "\nfoo\nbar\n"},
		want: []node{
			command{args: []lit{"foo"}},
			command{args: []lit{"bar"}},
		},
	},
	{
		ins: []string{"# comment\nfoo"},
		want: []node{
			command{args: []lit{"foo"}},
		},
	},
	{
		ins: []string{"foo arg1 arg2"},
		want: []node{
			command{args: []lit{"foo", "arg1", "arg2"}},
		},
	},
	{
		ins: []string{"( foo; )"},
		want: []node{
			subshell{stmts: []node{
				command{args: []lit{"foo"}},
			}},
		},
	},
	{
		ins: []string{"{ foo; }"},
		want: []node{
			block{stmts: []node{
				command{args: []lit{"foo"}},
			}},
		},
	},
	{
		ins: []string{"if foo; then bar; fi"},
		want: []node{ifStmt{
			cond: command{args: []lit{"foo"}},
			thenStmts: []node{
				command{args: []lit{"bar"}},
			}},
		},
	},
	{
		ins: []string{"if foo; then bar; else pass; fi"},
		want: []node{ifStmt{
			cond: command{args: []lit{"foo"}},
			thenStmts: []node{
				command{args: []lit{"bar"}},
			},
			elseStmts: []node{
				command{args: []lit{"pass"}},
			}},
		},
	},
	{
		ins: []string{"if a; then a; elif b; then b; elif c; then c; else d; fi"},
		want: []node{ifStmt{
			cond: command{args: []lit{"a"}},
			thenStmts: []node{
				command{args: []lit{"a"}},
			},
			elifs: []node{
				elif{cond: command{args: []lit{"b"}},
					thenStmts: []node{
						command{args: []lit{"b"}},
					}},
				elif{cond: command{args: []lit{"c"}},
					thenStmts: []node{
						command{args: []lit{"c"}},
					}},
			},
			elseStmts: []node{
				command{args: []lit{"d"}},
			}},
		},
	},
	{
		ins: []string{"while foo; do bar; done"},
		want: []node{whileStmt{
			cond: command{args: []lit{"foo"}},
			doStmts: []node{
				command{args: []lit{"bar"}},
			}},
		},
	},
	{
		ins: []string{"echo ' ' \"foo bar\""},
		want: []node{
			command{args: []lit{"echo", "' '", "\"foo bar\""}},
		},
	},
	{
		ins: []string{"$a ${b} s{s s=s"},
		want: []node{
			command{args: []lit{"$a", "${b}", "s{s", "s=s"}},
		},
	},
	{
		ins: []string{"foo && bar"},
		want: []node{binaryExpr{
			op: "&&",
			X:  command{args: []lit{"foo"}},
			Y:  command{args: []lit{"bar"}},
		}},
	},
	{
		ins: []string{"foo && bar || else"},
		want: []node{binaryExpr{
			op: "&&",
			X:  command{args: []lit{"foo"}},
			Y: binaryExpr{
				op: "||",
				X:  command{args: []lit{"bar"}},
				Y:  command{args: []lit{"else"}},
			},
		}},
	},
}

func TestParseAST(t *testing.T) {
	for _, c := range tests {
		want := prog{
			stmts: c.want,
		}
		for _, in := range c.ins {
			r := strings.NewReader(in)
			got, err := parse(r, "")
			if err != nil {
				t.Fatalf("Unexpected error in %q: %v", in, err)
			}
			if !reflect.DeepEqual(got, want) {
				t.Fatalf("AST mismatch in %q\nwant: %s\ngot:  %s\ndumps:\n%#v\n%#v",
					in, want.String(), got.String(), want, got)
			}
		}
	}
}
<|endoftext|>


<commit_before>package astitodo

import (
	"go/ast"
	"go/parser"
	"go/token"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

// Vars
var (
	regexpAssignee = regexp.MustCompile("^\\([\\w \\._\\+\\-@]+\\)")
)

// TODOs represents a set of todos
type TODOs []*TODO

// TODO represents a todo
type TODO struct {
	Assignee string
	Filename string
	Line     int
	Message  []string
}

// Extract walks through an input path and extracts TODOs from all files it encounters
func Extract(path string, excludedPaths ...string) (todos TODOs, err error) {
	err = todos.extract(path, excludedPaths...)
	return
}

func (todos *TODOs) extract(path string, excludedPaths ...string) error {
	return filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
		// Skip excluded paths
		for _, p := range excludedPaths {
			if p == path {
				if info.IsDir() {
					return filepath.SkipDir
				} else {
					return nil
				}
			}
		}

		// Skip vendor and all directories beginning with a .
		if info.IsDir() && (info.Name() == "vendor" || (len(info.Name()) > 1 && info.Name()[0] == '.')) {
			return filepath.SkipDir
		}

		// Only process go files
		if !info.IsDir() && filepath.Ext(path) != ".go" {
			return nil
		}

		// Everything is fine here, extract if path is a file
		if !info.IsDir() {
			if err = todos.extractFile(path); err != nil {
				return err
			}
		}
		return nil
	})
}

func (todos *TODOs) extractFile(filename string) (err error) {
	// Parse file and create the AST
	var fset = token.NewFileSet()
	var f *ast.File
	if f, err = parser.ParseFile(fset, filename, nil, parser.ParseComments); err != nil {
		return
	}

	// Loop in comment groups
	for _, cg := range f.Comments {
		// Loop in comments
		var todo *TODO
		var TODOFound bool
		for _, c := range cg.List {
			// Loop in lines
			for i, l := range strings.Split(c.Text, "\n") {
				// Init text
				var t = strings.TrimSpace(l)
				if strings.HasPrefix(t, "//") || strings.HasPrefix(t, "/*") || strings.HasPrefix(t, "*/") {
					t = strings.TrimSpace(t[2:])
				}

				// To do found
				if strings.HasPrefix(strings.ToLower(t), "todo") {
					// Init to do
					todo = &TODO{Filename: filename, Line: fset.Position(c.Slash).Line + i}
					t = strings.TrimSpace(t[4:])

					// Look for assignee
					if todo.Assignee = regexpAssignee.FindString(t); todo.Assignee != "" {
						t = strings.TrimSpace(t[len(todo.Assignee):])
						todo.Assignee = todo.Assignee[1 : len(todo.Assignee)-1]
					}

					// Append text
					todo.Message = append(todo.Message, t)
					*todos = append(*todos, todo)
					TODOFound = true
				} else if TODOFound && len(t) > 0 {
					todo.Message = append(todo.Message, t)
				} else {
					TODOFound = false
				}
			}
		}
	}
	return
}
<commit_msg>Minor correction<commit_after>package astitodo

import (
	"go/ast"
	"go/parser"
	"go/token"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

// Vars
var (
	regexpAssignee = regexp.MustCompile("^\\([\\w \\._\\+\\-@]+\\)")
)

// TODOs represents a set of todos
type TODOs []*TODO

// TODO represents a todo
type TODO struct {
	Assignee string
	Filename string
	Line     int
	Message  []string
}

// Extract walks through an input path and extracts TODOs from all files it encounters
func Extract(path string, excludedPaths ...string) (todos TODOs, err error) {
	err = todos.extract(path, excludedPaths...)
	return
}

func (todos *TODOs) extract(path string, excludedPaths ...string) error {
	return filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
		// Skip excluded paths
		for _, p := range excludedPaths {
			if p == path {
				if info.IsDir() {
					return filepath.SkipDir
				}
				return nil
			}
		}

		// Skip vendor and all directories beginning with a .
		if info.IsDir() && (info.Name() == "vendor" || (len(info.Name()) > 1 && info.Name()[0] == '.')) {
			return filepath.SkipDir
		}

		// Only process go files
		if !info.IsDir() && filepath.Ext(path) != ".go" {
			return nil
		}

		// Everything is fine here, extract if path is a file
		if !info.IsDir() {
			if err = todos.extractFile(path); err != nil {
				return err
			}
		}
		return nil
	})
}

func (todos *TODOs) extractFile(filename string) (err error) {
	// Parse file and create the AST
	var fset = token.NewFileSet()
	var f *ast.File
	if f, err = parser.ParseFile(fset, filename, nil, parser.ParseComments); err != nil {
		return
	}

	// Loop in comment groups
	for _, cg := range f.Comments {
		// Loop in comments
		var todo *TODO
		var TODOFound bool
		for _, c := range cg.List {
			// Loop in lines
			for i, l := range strings.Split(c.Text, "\n") {
				// Init text
				var t = strings.TrimSpace(l)
				if strings.HasPrefix(t, "//") || strings.HasPrefix(t, "/*") || strings.HasPrefix(t, "*/") {
					t = strings.TrimSpace(t[2:])
				}

				// To do found
				if strings.HasPrefix(strings.ToLower(t), "todo") {
					// Init to do
					todo = &TODO{Filename: filename, Line: fset.Position(c.Slash).Line + i}
					t = strings.TrimSpace(t[4:])

					// Look for assignee
					if todo.Assignee = regexpAssignee.FindString(t); todo.Assignee != "" {
						t = strings.TrimSpace(t[len(todo.Assignee):])
						todo.Assignee = todo.Assignee[1 : len(todo.Assignee)-1]
					}

					// Append text
					todo.Message = append(todo.Message, t)
					*todos = append(*todos, todo)
					TODOFound = true
				} else if TODOFound && len(t) > 0 {
					todo.Message = append(todo.Message, t)
				} else {
					TODOFound = false
				}
			}
		}
	}
	return
}
<|endoftext|>