<commit_before>/*                          _       _
 *__      _____  __ ___   ___  __ _| |_ ___
 *\ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
 * \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
 *  \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
 *
 * Copyright © 2016 - 2019 Weaviate. All rights reserved.
 * LICENSE: https://github.com/creativesoftwarefdn/weaviate/blob/develop/LICENSE.md
 * DESIGN & CONCEPT: Bob van Luijt (@bobvanluijt)
 * CONTACT: hello@creativesoftwarefdn.org
 */

// Package aggregate provides the local aggregate graphql endpoint for Weaviate
package aggregate

import (
	"fmt"

	"github.com/creativesoftwarefdn/weaviate/telemetry"

	"github.com/creativesoftwarefdn/weaviate/config"
	"github.com/creativesoftwarefdn/weaviate/database/schema"
	"github.com/creativesoftwarefdn/weaviate/database/schema/kind"
	"github.com/creativesoftwarefdn/weaviate/graphqlapi/local/common_filters"
	"github.com/graphql-go/graphql"
	"github.com/graphql-go/graphql/language/ast"
)

// GroupedByFieldName is a special graphQL field that appears alongside the
// to-be-aggregated props, but doesn't require any processing by the connectors
// itself, as it just displays meta info about the overall aggregation.
const GroupedByFieldName = "groupedBy"

// Resolver is a local interface that can be composed with other interfaces to
// form the overall GraphQL API main interface. All data-base connectors that
// want to support the GetMeta feature must implement this interface.
type Resolver interface {
	LocalAggregate(info *Params) (interface{}, error)
}

// RequestsLog is a local abstraction on the RequestsLog that needs to be
// provided to the graphQL API in order to log Local.Get queries.
type RequestsLog interface {
	Register(requestType string, identifier string)
}

// Params to describe the Local->GetMeta->Kind->Class query. Will be passed to
// the individual connector methods responsible for resolving the GetMeta
// query.
type Params struct {
	Kind       kind.Kind
	Filters    *common_filters.LocalFilter
	Analytics  common_filters.AnalyticsProps
	ClassName  schema.ClassName
	Properties []Property
	GroupBy    *common_filters.Path
}

// Aggregator is the desired computation that the database connector
// should perform on this property
type Aggregator string

const (
	// Count the occurence of this property
	Count Aggregator = "count"

	// Sum of all the values of the prop (i.e. sum of all Ints or Numbers)
	Sum Aggregator = "sum"

	// Mean calculates the mean of an Int or Number
	Mean Aggregator = "mean"

	// Mode calculates the mode (most occurring value) of an Int or Number
	Mode Aggregator = "mode"

	// Median calculates the median (most occurring value) of an Int or Number
	Median Aggregator = "median"

	// Maximum selects the maximum value of an Int or Number
	Maximum Aggregator = "maximum"

	// Minimum selects the maximum value of an Int or Number
	Minimum Aggregator = "minimum"
)

// Property is any property of a class that we want to retrieve meta
// information about
type Property struct {
	Name        schema.PropertyName
	Aggregators []Aggregator
}

func makeResolveClass(kind kind.Kind) graphql.FieldResolveFn {
	return func(p graphql.ResolveParams) (interface{}, error) {
		className := schema.ClassName(p.Info.FieldName)
		source, ok := p.Source.(map[string]interface{})
		if !ok {
			return nil, fmt.Errorf("expected source to be a map, but was %t", p.Source)
		}

		resolver, ok := source["Resolver"].(Resolver)
		if !ok {
			return nil, fmt.Errorf("expected source to contain a usable Resolver, but was %t", p.Source)
		}

		cfg, ok := source["Config"].(config.Environment)
		if !ok {
			return nil, fmt.Errorf("expected source to contain a config, but was %t", p.Source)
		}

		// There can only be exactly one ast.Field; it is the class name.
		if len(p.Info.FieldASTs) != 1 {
			panic("Only one Field expected here")
		}

		selections := p.Info.FieldASTs[0].SelectionSet
		properties, err := extractProperties(selections)
		if err != nil {
			return nil, fmt.Errorf("could not extract properties for class '%s': %s", className, err)
		}

		groupBy, err := extractGroupBy(p.Args, p.Info.FieldName)
		if err != nil {
			return nil, fmt.Errorf("could not extract groupBy path: %s", err)
		}

		filters, err := common_filters.ExtractFilters(p.Args, p.Info.FieldName)
		if err != nil {
			return nil, fmt.Errorf("could not extract filters: %s", err)
		}

		requestsLog, ok := source["RequestsLog"].(RequestsLog)
		if !ok {
			return nil, fmt.Errorf("expected source to contain a usable RequestsLog, but was %t", p.Source)
		}
		for _, property := range properties {
			for _, aggregator := range property.Aggregators {
				serviceID := fmt.Sprintf("%s[%s]", telemetry.LocalQuery, aggregator)
				go func() {
					requestsLog.Register(telemetry.TypeGQL, serviceID)
				}()

			}
		analytics, err := common_filters.ExtractAnalyticsProps(p.Args, cfg.AnalyticsEngine)
		if err != nil {
			return nil, fmt.Errorf("could not extract filters: %s", err)
		}

		params := &Params{
			Kind:       kind,
			Filters:    filters,
			ClassName:  className,
			Properties: properties,
			GroupBy:    groupBy,
			Analytics:  analytics,
		}

		return resolver.LocalAggregate(params)
	}
}

func extractProperties(selections *ast.SelectionSet) ([]Property, error) {
	properties := []Property{}

	for _, selection := range selections.Selections {
		field := selection.(*ast.Field)
		name := field.Name.Value
		if name == GroupedByFieldName {
			// in the graphQL API we show the "groupedBy" field alongside various
			// properties, however, we don't have to include it here, as we don't
			// wont to perform aggregations on it.
			// If we didn't exclude it we'd run into errors down the line, because
			// the connector would look for a "groupedBy" prop on the specific class
			// which doesn't exist.

			continue
		}

		property := Property{Name: schema.PropertyName(name)}
		aggregators, err := extractAggregators(field.SelectionSet)
		if err != nil {
			return nil, err
		}

		property.Aggregators = aggregators
		properties = append(properties, property)
	}

	return properties, nil
}

func extractAggregators(selections *ast.SelectionSet) ([]Aggregator, error) {
	analyses := []Aggregator{}
	for _, selection := range selections.Selections {
		field := selection.(*ast.Field)
		name := field.Name.Value
		property, err := parseAnalysisProp(name)
		if err != nil {
			return nil, err
		}

		analyses = append(analyses, property)
	}

	return analyses, nil
}

func parseAnalysisProp(name string) (Aggregator, error) {
	switch name {
	case string(Mean):
		return Mean, nil
	case string(Median):
		return Median, nil
	case string(Mode):
		return Mode, nil
	case string(Maximum):
		return Maximum, nil
	case string(Minimum):
		return Minimum, nil
	case string(Count):
		return Count, nil
	case string(Sum):
		return Sum, nil
	default:
		return "", fmt.Errorf("unrecognized aggregator prop '%s'", name)
	}
}

func extractGroupBy(args map[string]interface{}, rootClass string) (*common_filters.Path, error) {
	groupBy, ok := args["groupBy"]
	if !ok {
		return nil, fmt.Errorf("no groupBy present in args")
	}

	pathSegments, ok := groupBy.([]interface{})
	if !ok {
		return nil, fmt.Errorf("no groupBy must be a list, instead got: %#v", groupBy)
	}

	return common_filters.ParsePath(pathSegments, rootClass)
}
<commit_msg>gh-803&gh-699: bugfix<commit_after>/*                          _       _
 *__      _____  __ ___   ___  __ _| |_ ___
 *\ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
 * \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
 *  \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
 *
 * Copyright © 2016 - 2019 Weaviate. All rights reserved.
 * LICENSE: https://github.com/creativesoftwarefdn/weaviate/blob/develop/LICENSE.md
 * DESIGN & CONCEPT: Bob van Luijt (@bobvanluijt)
 * CONTACT: hello@creativesoftwarefdn.org
 */

// Package aggregate provides the local aggregate graphql endpoint for Weaviate
package aggregate

import (
	"fmt"

	"github.com/creativesoftwarefdn/weaviate/telemetry"

	"github.com/creativesoftwarefdn/weaviate/config"
	"github.com/creativesoftwarefdn/weaviate/database/schema"
	"github.com/creativesoftwarefdn/weaviate/database/schema/kind"
	"github.com/creativesoftwarefdn/weaviate/graphqlapi/local/common_filters"
	"github.com/graphql-go/graphql"
	"github.com/graphql-go/graphql/language/ast"
)

// GroupedByFieldName is a special graphQL field that appears alongside the
// to-be-aggregated props, but doesn't require any processing by the connectors
// itself, as it just displays meta info about the overall aggregation.
const GroupedByFieldName = "groupedBy"

// Resolver is a local interface that can be composed with other interfaces to
// form the overall GraphQL API main interface. All data-base connectors that
// want to support the GetMeta feature must implement this interface.
type Resolver interface {
	LocalAggregate(info *Params) (interface{}, error)
}

// RequestsLog is a local abstraction on the RequestsLog that needs to be
// provided to the graphQL API in order to log Local.Get queries.
type RequestsLog interface {
	Register(requestType string, identifier string)
}

// Params to describe the Local->GetMeta->Kind->Class query. Will be passed to
// the individual connector methods responsible for resolving the GetMeta
// query.
type Params struct {
	Kind       kind.Kind
	Filters    *common_filters.LocalFilter
	Analytics  common_filters.AnalyticsProps
	ClassName  schema.ClassName
	Properties []Property
	GroupBy    *common_filters.Path
}

// Aggregator is the desired computation that the database connector
// should perform on this property
type Aggregator string

const (
	// Count the occurence of this property
	Count Aggregator = "count"

	// Sum of all the values of the prop (i.e. sum of all Ints or Numbers)
	Sum Aggregator = "sum"

	// Mean calculates the mean of an Int or Number
	Mean Aggregator = "mean"

	// Mode calculates the mode (most occurring value) of an Int or Number
	Mode Aggregator = "mode"

	// Median calculates the median (most occurring value) of an Int or Number
	Median Aggregator = "median"

	// Maximum selects the maximum value of an Int or Number
	Maximum Aggregator = "maximum"

	// Minimum selects the maximum value of an Int or Number
	Minimum Aggregator = "minimum"
)

// Property is any property of a class that we want to retrieve meta
// information about
type Property struct {
	Name        schema.PropertyName
	Aggregators []Aggregator
}

func makeResolveClass(kind kind.Kind) graphql.FieldResolveFn {
	return func(p graphql.ResolveParams) (interface{}, error) {
		className := schema.ClassName(p.Info.FieldName)
		source, ok := p.Source.(map[string]interface{})
		if !ok {
			return nil, fmt.Errorf("expected source to be a map, but was %t", p.Source)
		}

		resolver, ok := source["Resolver"].(Resolver)
		if !ok {
			return nil, fmt.Errorf("expected source to contain a usable Resolver, but was %t", p.Source)
		}

		cfg, ok := source["Config"].(config.Environment)
		if !ok {
			return nil, fmt.Errorf("expected source to contain a config, but was %t", p.Source)
		}

		// There can only be exactly one ast.Field; it is the class name.
		if len(p.Info.FieldASTs) != 1 {
			panic("Only one Field expected here")
		}

		selections := p.Info.FieldASTs[0].SelectionSet
		properties, err := extractProperties(selections)
		if err != nil {
			return nil, fmt.Errorf("could not extract properties for class '%s': %s", className, err)
		}

		groupBy, err := extractGroupBy(p.Args, p.Info.FieldName)
		if err != nil {
			return nil, fmt.Errorf("could not extract groupBy path: %s", err)
		}

		filters, err := common_filters.ExtractFilters(p.Args, p.Info.FieldName)
		if err != nil {
			return nil, fmt.Errorf("could not extract filters: %s", err)
		}

		requestsLog, ok := source["RequestsLog"].(RequestsLog)
		if !ok {
			return nil, fmt.Errorf("expected source to contain a usable RequestsLog, but was %t", p.Source)
		}
		for _, property := range properties {
			for _, aggregator := range property.Aggregators {
				serviceID := fmt.Sprintf("%s[%s]", telemetry.LocalQuery, aggregator)
				go func() {
					requestsLog.Register(telemetry.TypeGQL, serviceID)
				}()

			}
		}
		analytics, err := common_filters.ExtractAnalyticsProps(p.Args, cfg.AnalyticsEngine)
		if err != nil {
			return nil, fmt.Errorf("could not extract filters: %s", err)
		}

		params := &Params{
			Kind:       kind,
			Filters:    filters,
			ClassName:  className,
			Properties: properties,
			GroupBy:    groupBy,
			Analytics:  analytics,
		}

		return resolver.LocalAggregate(params)
	}
}

func extractProperties(selections *ast.SelectionSet) ([]Property, error) {
	properties := []Property{}

	for _, selection := range selections.Selections {
		field := selection.(*ast.Field)
		name := field.Name.Value
		if name == GroupedByFieldName {
			// in the graphQL API we show the "groupedBy" field alongside various
			// properties, however, we don't have to include it here, as we don't
			// wont to perform aggregations on it.
			// If we didn't exclude it we'd run into errors down the line, because
			// the connector would look for a "groupedBy" prop on the specific class
			// which doesn't exist.

			continue
		}

		property := Property{Name: schema.PropertyName(name)}
		aggregators, err := extractAggregators(field.SelectionSet)
		if err != nil {
			return nil, err
		}

		property.Aggregators = aggregators
		properties = append(properties, property)
	}

	return properties, nil
}

func extractAggregators(selections *ast.SelectionSet) ([]Aggregator, error) {
	analyses := []Aggregator{}
	for _, selection := range selections.Selections {
		field := selection.(*ast.Field)
		name := field.Name.Value
		property, err := parseAnalysisProp(name)
		if err != nil {
			return nil, err
		}

		analyses = append(analyses, property)
	}

	return analyses, nil
}

func parseAnalysisProp(name string) (Aggregator, error) {
	switch name {
	case string(Mean):
		return Mean, nil
	case string(Median):
		return Median, nil
	case string(Mode):
		return Mode, nil
	case string(Maximum):
		return Maximum, nil
	case string(Minimum):
		return Minimum, nil
	case string(Count):
		return Count, nil
	case string(Sum):
		return Sum, nil
	default:
		return "", fmt.Errorf("unrecognized aggregator prop '%s'", name)
	}
}

func extractGroupBy(args map[string]interface{}, rootClass string) (*common_filters.Path, error) {
	groupBy, ok := args["groupBy"]
	if !ok {
		return nil, fmt.Errorf("no groupBy present in args")
	}

	pathSegments, ok := groupBy.([]interface{})
	if !ok {
		return nil, fmt.Errorf("no groupBy must be a list, instead got: %#v", groupBy)
	}

	return common_filters.ParsePath(pathSegments, rootClass)
}
<|endoftext|>


<commit_before>// Package s3 implements Storage based on an S3 bucket.
package s3

import (
	"bytes"
	"context"
	"crypto/tls"
	"fmt"
	"io"
	"net/http"
	"strings"
	"time"

	"github.com/minio/minio-go/v7"
	"github.com/minio/minio-go/v7/pkg/credentials"
	"github.com/pkg/errors"

	"github.com/kopia/kopia/internal/clock"
	"github.com/kopia/kopia/internal/gather"
	"github.com/kopia/kopia/internal/iocopy"
	"github.com/kopia/kopia/repo/blob"
	"github.com/kopia/kopia/repo/blob/retrying"
)

const (
	s3storageType   = "s3"
	latestVersionID = ""
)

type s3Storage struct {
	Options

	cli *minio.Client

	storageConfig *StorageConfig
}

func (s *s3Storage) GetCapacity(ctx context.Context) (blob.Capacity, error) {
	return blob.Capacity{}, blob.ErrNotAVolume
}

func (s *s3Storage) GetBlob(ctx context.Context, b blob.ID, offset, length int64, output blob.OutputBuffer) error {
	return s.getBlobWithVersion(ctx, b, latestVersionID, offset, length, output)
}

// getBlobWithVersion returns full or partial contents of a blob with given ID and version.
func (s *s3Storage) getBlobWithVersion(ctx context.Context, b blob.ID, version string, offset, length int64, output blob.OutputBuffer) error {
	output.Reset()

	attempt := func() error {
		opt := minio.GetObjectOptions{VersionID: version}

		if length > 0 {
			if err := opt.SetRange(offset, offset+length-1); err != nil {
				return errors.Wrap(blob.ErrInvalidRange, "unable to set range")
			}
		}

		if length == 0 {
			// zero-length ranges require special handling, set non-zero range and
			// we won't be trying to read the response anyway.
			if err := opt.SetRange(0, 1); err != nil {
				return errors.Wrap(blob.ErrInvalidRange, "unable to set range")
			}
		}

		o, err := s.cli.GetObject(ctx, s.BucketName, s.getObjectNameString(b), opt)
		if err != nil {
			return errors.Wrap(err, "GetObject")
		}

		defer o.Close() //nolint:errcheck

		if length == 0 {
			return nil
		}

		// nolint:wrapcheck
		return iocopy.JustCopy(output, o)
	}

	if err := attempt(); err != nil {
		return translateError(err)
	}

	// nolint:wrapcheck
	return blob.EnsureLengthExactly(output.Length(), length)
}

func isInvalidCredentials(err error) bool {
	return err != nil && strings.Contains(err.Error(), blob.InvalidCredentialsErrStr)
}

func translateError(err error) error {
	var me minio.ErrorResponse

	if isInvalidCredentials(err) {
		return blob.ErrInvalidCredentials
	}

	if errors.As(err, &me) {
		switch me.StatusCode {
		case http.StatusOK:
			return nil

		case http.StatusNotFound:
			return blob.ErrBlobNotFound

		case http.StatusRequestedRangeNotSatisfiable:
			return blob.ErrInvalidRange
		}
	}

	return err
}

func (s *s3Storage) GetMetadata(ctx context.Context, b blob.ID) (blob.Metadata, error) {
	vm, err := s.getVersionMetadata(ctx, b, "")

	return vm.Metadata, err
}

func (s *s3Storage) getVersionMetadata(ctx context.Context, b blob.ID, version string) (versionMetadata, error) {
	opts := minio.GetObjectOptions{
		VersionID: version,
	}

	oi, err := s.cli.StatObject(ctx, s.BucketName, s.getObjectNameString(b), opts)
	if err != nil {
		return versionMetadata{}, errors.Wrap(translateError(err), "StatObject")
	}

	return infoToVersionMetadata(s.Prefix, &oi), nil
}

func (s *s3Storage) PutBlob(ctx context.Context, b blob.ID, data blob.Bytes, opts blob.PutOptions) error {
	switch {
	case opts.DoNotRecreate:
		return errors.Wrap(blob.ErrUnsupportedPutBlobOption, "do-not-recreate")
	case !opts.SetModTime.IsZero():
		return blob.ErrSetTimeUnsupported
	}

	_, err := s.putBlob(ctx, b, data, opts)

	if opts.GetModTime != nil {
		bm, err2 := s.GetMetadata(ctx, b)
		if err2 != nil {
			return err2
		}

		*opts.GetModTime = bm.Timestamp
	}

	return err
}

func (s *s3Storage) putBlob(ctx context.Context, b blob.ID, data blob.Bytes, opts blob.PutOptions) (versionMetadata, error) {
	var (
		storageClass    = s.storageConfig.getStorageClassForBlobID(b)
		retentionMode   minio.RetentionMode
		retainUntilDate time.Time
	)

	if opts.RetentionPeriod != 0 {
		retentionMode = minio.RetentionMode(opts.RetentionMode)
		if !retentionMode.IsValid() {
			return versionMetadata{}, errors.Errorf("invalid retention mode: %q", opts.RetentionMode)
		}

		retainUntilDate = clock.Now().Add(opts.RetentionPeriod).UTC()
	}

	uploadInfo, err := s.cli.PutObject(ctx, s.BucketName, s.getObjectNameString(b), data.Reader(), int64(data.Length()), minio.PutObjectOptions{
		ContentType:      "application/x-kopia",
		DisableMultipart: true,
		// The Content-MD5 header is required for any request to upload an object
		// with a retention period configured using Amazon S3 Object Lock.
		// Unconditionally computing the content MD5, potentially incurring
		// a slightly higher CPU overhead.
		SendContentMd5:  true,
		StorageClass:    storageClass,
		RetainUntilDate: retainUntilDate,
		Mode:            retentionMode,
	})

	if isInvalidCredentials(err) {
		return versionMetadata{}, blob.ErrInvalidCredentials
	}

	var er minio.ErrorResponse

	if errors.As(err, &er) && er.Code == "InvalidRequest" && strings.Contains(strings.ToLower(er.Message), "content-md5") {
		return versionMetadata{}, err // nolint:wrapcheck
	}

	if errors.Is(err, io.EOF) && uploadInfo.Size == 0 {
		// special case empty stream
		_, err = s.cli.PutObject(ctx, s.BucketName, s.getObjectNameString(b), bytes.NewBuffer(nil), 0, minio.PutObjectOptions{
			ContentType:     "application/x-kopia",
			StorageClass:    storageClass,
			RetainUntilDate: retainUntilDate,
			Mode:            retentionMode,
		})
	}

	if err != nil {
		return versionMetadata{}, err // nolint:wrapcheck
	}

	return versionMetadata{
		Metadata: blob.Metadata{
			BlobID:    b,
			Length:    uploadInfo.Size,
			Timestamp: uploadInfo.LastModified,
		},
		Version: uploadInfo.VersionID,
	}, nil
}

func (s *s3Storage) DeleteBlob(ctx context.Context, b blob.ID) error {
	err := translateError(s.cli.RemoveObject(ctx, s.BucketName, s.getObjectNameString(b), minio.RemoveObjectOptions{}))
	if errors.Is(err, blob.ErrBlobNotFound) {
		return nil
	}

	return err
}

func (s *s3Storage) getObjectNameString(b blob.ID) string {
	return s.Prefix + string(b)
}

func (s *s3Storage) ListBlobs(ctx context.Context, prefix blob.ID, callback func(blob.Metadata) error) error {
	ctx, cancel := context.WithCancel(ctx)

	defer cancel()

	oi := s.cli.ListObjects(ctx, s.BucketName, minio.ListObjectsOptions{
		Prefix: s.getObjectNameString(prefix),
	})
	for o := range oi {
		if err := o.Err; err != nil {
			if isInvalidCredentials(err) {
				return blob.ErrInvalidCredentials
			}

			return err
		}

		bm := blob.Metadata{
			BlobID:    blob.ID(o.Key[len(s.Prefix):]),
			Length:    o.Size,
			Timestamp: o.LastModified,
		}

		if bm.BlobID == ConfigName {
			continue
		}

		if err := callback(bm); err != nil {
			return err
		}
	}

	return nil
}

func (s *s3Storage) ConnectionInfo() blob.ConnectionInfo {
	return blob.ConnectionInfo{
		Type:   s3storageType,
		Config: &s.Options,
	}
}

func (s *s3Storage) Close(ctx context.Context) error {
	return nil
}

func (s *s3Storage) String() string {
	return fmt.Sprintf("s3://%v/%v", s.BucketName, s.Prefix)
}

func (s *s3Storage) DisplayName() string {
	return fmt.Sprintf("S3: %v %v", s.Endpoint, s.BucketName)
}

func (s *s3Storage) FlushCaches(ctx context.Context) error {
	return nil
}

func getCustomTransport(insecureSkipVerify bool) (transport *http.Transport) {
	// nolint:gosec
	customTransport := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: insecureSkipVerify}}
	return customTransport
}

// New creates new S3-backed storage with specified options:
//
// - the 'BucketName' field is required and all other parameters are optional.
func New(ctx context.Context, opt *Options) (blob.Storage, error) {
	st, err := newStorage(ctx, opt)
	if err != nil {
		return nil, err
	}

	s, err := maybePointInTimeStore(ctx, st, opt.PointInTime)
	if err != nil {
		return nil, err
	}

	return retrying.NewWrapper(s), nil
}

func newStorage(ctx context.Context, opt *Options) (*s3Storage, error) {
	return newStorageWithCredentials(ctx, credentials.NewStaticV4(opt.AccessKeyID, opt.SecretAccessKey, opt.SessionToken), opt)
}

func newStorageWithCredentials(ctx context.Context, creds *credentials.Credentials, opt *Options) (*s3Storage, error) {
	if opt.BucketName == "" {
		return nil, errors.New("bucket name must be specified")
	}

	minioOpts := &minio.Options{
		Creds:  creds,
		Secure: !opt.DoNotUseTLS,
		Region: opt.Region,
	}

	if opt.DoNotVerifyTLS {
		minioOpts.Transport = getCustomTransport(true)
	}

	cli, err := minio.New(opt.Endpoint, minioOpts)
	if err != nil {
		return nil, errors.Wrap(err, "unable to create client")
	}

	ok, err := cli.BucketExists(ctx, opt.BucketName)
	if err != nil {
		return nil, errors.Wrapf(err, "unable to determine if bucket %q exists", opt.BucketName)
	}

	if !ok {
		return nil, errors.Errorf("bucket %q does not exist", opt.BucketName)
	}

	s := s3Storage{
		Options:       *opt,
		cli:           cli,
		storageConfig: &StorageConfig{},
	}

	var scOutput gather.WriteBuffer

	if getBlobErr := s.GetBlob(ctx, ConfigName, 0, -1, &scOutput); getBlobErr == nil {
		if scErr := s.storageConfig.Load(scOutput.Bytes().Reader()); scErr != nil {
			return nil, errors.Wrapf(scErr, "error parsing storage config for bucket %q", opt.BucketName)
		}
	} else if !errors.Is(getBlobErr, blob.ErrBlobNotFound) {
		return nil, errors.Wrapf(getBlobErr, "error retrieving storage config from bucket %q", opt.BucketName)
	}

	return &s, nil
}

func init() {
	blob.AddSupportedStorage(
		s3storageType,
		func() interface{} {
			return &Options{}
		},
		func(ctx context.Context, o interface{}, isCreate bool) (blob.Storage, error) {
			return New(ctx, o.(*Options)) // nolint:forcetypeassert
		})
}
<commit_msg>Add comment explaining disabling multipart upload (#1944)<commit_after>// Package s3 implements Storage based on an S3 bucket.
package s3

import (
	"bytes"
	"context"
	"crypto/tls"
	"fmt"
	"io"
	"net/http"
	"strings"
	"time"

	"github.com/minio/minio-go/v7"
	"github.com/minio/minio-go/v7/pkg/credentials"
	"github.com/pkg/errors"

	"github.com/kopia/kopia/internal/clock"
	"github.com/kopia/kopia/internal/gather"
	"github.com/kopia/kopia/internal/iocopy"
	"github.com/kopia/kopia/repo/blob"
	"github.com/kopia/kopia/repo/blob/retrying"
)

const (
	s3storageType   = "s3"
	latestVersionID = ""
)

type s3Storage struct {
	Options

	cli *minio.Client

	storageConfig *StorageConfig
}

func (s *s3Storage) GetCapacity(ctx context.Context) (blob.Capacity, error) {
	return blob.Capacity{}, blob.ErrNotAVolume
}

func (s *s3Storage) GetBlob(ctx context.Context, b blob.ID, offset, length int64, output blob.OutputBuffer) error {
	return s.getBlobWithVersion(ctx, b, latestVersionID, offset, length, output)
}

// getBlobWithVersion returns full or partial contents of a blob with given ID and version.
func (s *s3Storage) getBlobWithVersion(ctx context.Context, b blob.ID, version string, offset, length int64, output blob.OutputBuffer) error {
	output.Reset()

	attempt := func() error {
		opt := minio.GetObjectOptions{VersionID: version}

		if length > 0 {
			if err := opt.SetRange(offset, offset+length-1); err != nil {
				return errors.Wrap(blob.ErrInvalidRange, "unable to set range")
			}
		}

		if length == 0 {
			// zero-length ranges require special handling, set non-zero range and
			// we won't be trying to read the response anyway.
			if err := opt.SetRange(0, 1); err != nil {
				return errors.Wrap(blob.ErrInvalidRange, "unable to set range")
			}
		}

		o, err := s.cli.GetObject(ctx, s.BucketName, s.getObjectNameString(b), opt)
		if err != nil {
			return errors.Wrap(err, "GetObject")
		}

		defer o.Close() //nolint:errcheck

		if length == 0 {
			return nil
		}

		// nolint:wrapcheck
		return iocopy.JustCopy(output, o)
	}

	if err := attempt(); err != nil {
		return translateError(err)
	}

	// nolint:wrapcheck
	return blob.EnsureLengthExactly(output.Length(), length)
}

func isInvalidCredentials(err error) bool {
	return err != nil && strings.Contains(err.Error(), blob.InvalidCredentialsErrStr)
}

func translateError(err error) error {
	var me minio.ErrorResponse

	if isInvalidCredentials(err) {
		return blob.ErrInvalidCredentials
	}

	if errors.As(err, &me) {
		switch me.StatusCode {
		case http.StatusOK:
			return nil

		case http.StatusNotFound:
			return blob.ErrBlobNotFound

		case http.StatusRequestedRangeNotSatisfiable:
			return blob.ErrInvalidRange
		}
	}

	return err
}

func (s *s3Storage) GetMetadata(ctx context.Context, b blob.ID) (blob.Metadata, error) {
	vm, err := s.getVersionMetadata(ctx, b, "")

	return vm.Metadata, err
}

func (s *s3Storage) getVersionMetadata(ctx context.Context, b blob.ID, version string) (versionMetadata, error) {
	opts := minio.GetObjectOptions{
		VersionID: version,
	}

	oi, err := s.cli.StatObject(ctx, s.BucketName, s.getObjectNameString(b), opts)
	if err != nil {
		return versionMetadata{}, errors.Wrap(translateError(err), "StatObject")
	}

	return infoToVersionMetadata(s.Prefix, &oi), nil
}

func (s *s3Storage) PutBlob(ctx context.Context, b blob.ID, data blob.Bytes, opts blob.PutOptions) error {
	switch {
	case opts.DoNotRecreate:
		return errors.Wrap(blob.ErrUnsupportedPutBlobOption, "do-not-recreate")
	case !opts.SetModTime.IsZero():
		return blob.ErrSetTimeUnsupported
	}

	_, err := s.putBlob(ctx, b, data, opts)

	if opts.GetModTime != nil {
		bm, err2 := s.GetMetadata(ctx, b)
		if err2 != nil {
			return err2
		}

		*opts.GetModTime = bm.Timestamp
	}

	return err
}

func (s *s3Storage) putBlob(ctx context.Context, b blob.ID, data blob.Bytes, opts blob.PutOptions) (versionMetadata, error) {
	var (
		storageClass    = s.storageConfig.getStorageClassForBlobID(b)
		retentionMode   minio.RetentionMode
		retainUntilDate time.Time
	)

	if opts.RetentionPeriod != 0 {
		retentionMode = minio.RetentionMode(opts.RetentionMode)
		if !retentionMode.IsValid() {
			return versionMetadata{}, errors.Errorf("invalid retention mode: %q", opts.RetentionMode)
		}

		retainUntilDate = clock.Now().Add(opts.RetentionPeriod).UTC()
	}

	uploadInfo, err := s.cli.PutObject(ctx, s.BucketName, s.getObjectNameString(b), data.Reader(), int64(data.Length()), minio.PutObjectOptions{
		ContentType: "application/x-kopia",
		// Kopia already splits snapshot contents into small blobs to improve
		// upload throughput. There is no need for further splitting
		// through multipart uploads.
		DisableMultipart: true,
		// The Content-MD5 header is required for any request to upload an object
		// with a retention period configured using Amazon S3 Object Lock.
		// Unconditionally computing the content MD5, potentially incurring
		// a slightly higher CPU overhead.
		SendContentMd5:  true,
		StorageClass:    storageClass,
		RetainUntilDate: retainUntilDate,
		Mode:            retentionMode,
	})

	if isInvalidCredentials(err) {
		return versionMetadata{}, blob.ErrInvalidCredentials
	}

	var er minio.ErrorResponse

	if errors.As(err, &er) && er.Code == "InvalidRequest" && strings.Contains(strings.ToLower(er.Message), "content-md5") {
		return versionMetadata{}, err // nolint:wrapcheck
	}

	if errors.Is(err, io.EOF) && uploadInfo.Size == 0 {
		// special case empty stream
		_, err = s.cli.PutObject(ctx, s.BucketName, s.getObjectNameString(b), bytes.NewBuffer(nil), 0, minio.PutObjectOptions{
			ContentType:     "application/x-kopia",
			StorageClass:    storageClass,
			RetainUntilDate: retainUntilDate,
			Mode:            retentionMode,
		})
	}

	if err != nil {
		return versionMetadata{}, err // nolint:wrapcheck
	}

	return versionMetadata{
		Metadata: blob.Metadata{
			BlobID:    b,
			Length:    uploadInfo.Size,
			Timestamp: uploadInfo.LastModified,
		},
		Version: uploadInfo.VersionID,
	}, nil
}

func (s *s3Storage) DeleteBlob(ctx context.Context, b blob.ID) error {
	err := translateError(s.cli.RemoveObject(ctx, s.BucketName, s.getObjectNameString(b), minio.RemoveObjectOptions{}))
	if errors.Is(err, blob.ErrBlobNotFound) {
		return nil
	}

	return err
}

func (s *s3Storage) getObjectNameString(b blob.ID) string {
	return s.Prefix + string(b)
}

func (s *s3Storage) ListBlobs(ctx context.Context, prefix blob.ID, callback func(blob.Metadata) error) error {
	ctx, cancel := context.WithCancel(ctx)

	defer cancel()

	oi := s.cli.ListObjects(ctx, s.BucketName, minio.ListObjectsOptions{
		Prefix: s.getObjectNameString(prefix),
	})
	for o := range oi {
		if err := o.Err; err != nil {
			if isInvalidCredentials(err) {
				return blob.ErrInvalidCredentials
			}

			return err
		}

		bm := blob.Metadata{
			BlobID:    blob.ID(o.Key[len(s.Prefix):]),
			Length:    o.Size,
			Timestamp: o.LastModified,
		}

		if bm.BlobID == ConfigName {
			continue
		}

		if err := callback(bm); err != nil {
			return err
		}
	}

	return nil
}

func (s *s3Storage) ConnectionInfo() blob.ConnectionInfo {
	return blob.ConnectionInfo{
		Type:   s3storageType,
		Config: &s.Options,
	}
}

func (s *s3Storage) Close(ctx context.Context) error {
	return nil
}

func (s *s3Storage) String() string {
	return fmt.Sprintf("s3://%v/%v", s.BucketName, s.Prefix)
}

func (s *s3Storage) DisplayName() string {
	return fmt.Sprintf("S3: %v %v", s.Endpoint, s.BucketName)
}

func (s *s3Storage) FlushCaches(ctx context.Context) error {
	return nil
}

func getCustomTransport(insecureSkipVerify bool) (transport *http.Transport) {
	// nolint:gosec
	customTransport := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: insecureSkipVerify}}
	return customTransport
}

// New creates new S3-backed storage with specified options:
//
// - the 'BucketName' field is required and all other parameters are optional.
func New(ctx context.Context, opt *Options) (blob.Storage, error) {
	st, err := newStorage(ctx, opt)
	if err != nil {
		return nil, err
	}

	s, err := maybePointInTimeStore(ctx, st, opt.PointInTime)
	if err != nil {
		return nil, err
	}

	return retrying.NewWrapper(s), nil
}

func newStorage(ctx context.Context, opt *Options) (*s3Storage, error) {
	return newStorageWithCredentials(ctx, credentials.NewStaticV4(opt.AccessKeyID, opt.SecretAccessKey, opt.SessionToken), opt)
}

func newStorageWithCredentials(ctx context.Context, creds *credentials.Credentials, opt *Options) (*s3Storage, error) {
	if opt.BucketName == "" {
		return nil, errors.New("bucket name must be specified")
	}

	minioOpts := &minio.Options{
		Creds:  creds,
		Secure: !opt.DoNotUseTLS,
		Region: opt.Region,
	}

	if opt.DoNotVerifyTLS {
		minioOpts.Transport = getCustomTransport(true)
	}

	cli, err := minio.New(opt.Endpoint, minioOpts)
	if err != nil {
		return nil, errors.Wrap(err, "unable to create client")
	}

	ok, err := cli.BucketExists(ctx, opt.BucketName)
	if err != nil {
		return nil, errors.Wrapf(err, "unable to determine if bucket %q exists", opt.BucketName)
	}

	if !ok {
		return nil, errors.Errorf("bucket %q does not exist", opt.BucketName)
	}

	s := s3Storage{
		Options:       *opt,
		cli:           cli,
		storageConfig: &StorageConfig{},
	}

	var scOutput gather.WriteBuffer

	if getBlobErr := s.GetBlob(ctx, ConfigName, 0, -1, &scOutput); getBlobErr == nil {
		if scErr := s.storageConfig.Load(scOutput.Bytes().Reader()); scErr != nil {
			return nil, errors.Wrapf(scErr, "error parsing storage config for bucket %q", opt.BucketName)
		}
	} else if !errors.Is(getBlobErr, blob.ErrBlobNotFound) {
		return nil, errors.Wrapf(getBlobErr, "error retrieving storage config from bucket %q", opt.BucketName)
	}

	return &s, nil
}

func init() {
	blob.AddSupportedStorage(
		s3storageType,
		func() interface{} {
			return &Options{}
		},
		func(ctx context.Context, o interface{}, isCreate bool) (blob.Storage, error) {
			return New(ctx, o.(*Options)) // nolint:forcetypeassert
		})
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
	The unsafe package contains operations that step around the type safety of Go programs.
*/
package unsafe

// ArbitraryType is here for the purposes of documentation only and is not actually
// part of the unsafe package.  It represents the type of an arbitrary Go expression.
type ArbitraryType int

// Pointer represents a pointer to an arbitrary type.  There are three special operations
// available for type Pointer that are not available for other types.
//	1) A pointer value of any type can be converted to a Pointer.
//	2) A Pointer can be converted to a pointer value of any type.
//	3) A uintptr can be converted to a Pointer.
//	4) A Pointer can be converted to a uintptr.
// Pointer therefore allows a program to defeat the type system and read and write
// arbitrary memory. It should be used with extreme care.
type Pointer *ArbitraryType

// Sizeof returns the size in bytes occupied by the value v.  The size is that of the
// "top level" of the value only.  For instance, if v is a slice, it returns the size of
// the slice descriptor, not the size of the memory referenced by the slice.
func Sizeof(v ArbitraryType) int

// Offsetof returns the offset within the struct of the field represented by v,
// which must be of the form struct_value.field.  In other words, it returns the
// number of bytes between the start of the struct and the start of the field.
func Offsetof(v ArbitraryType) int

// Alignof returns the alignment of the value v.  It is the maximum value m such
// that the address of a variable with the type of v will always always be zero mod m.
// If v is of the form obj.f, it returns the alignment of field f within struct object obj.
func Alignof(v ArbitraryType) int

// Typeof returns the type of an interface value, a runtime.Type.
func Typeof(i interface{}) (typ interface{})

// Reflect unpacks an interface value into its type and the address of a copy of the
// internal value.
func Reflect(i interface{}) (typ interface{}, addr uintptr)

// Unreflect inverts Reflect: Given a type and a pointer, it returns an empty interface value
// with those contents.  The typ is assumed to contain a pointer to a runtime type;
// the type information in the interface{} is ignored, so that, for example, both
// *reflect.StructType and *runtime.StructType can be passed for typ.
func Unreflect(typ interface{}, addr uintptr) (ret interface{})

// New allocates and returns a pointer to memory for a new value of the given type.
// The typ is assumed to hold a pointer to a runtime type.
// Callers should use reflect.MakeZero instead of invoking unsafe.New directly.
func New(typ interface{}) Pointer

// NewArray allocates and returns a pointer to an array of n elements of the given type.
// The typ is assumed to hold a pointer to a runtime type.
// Callers should use reflect.MakeSlice instead of invoking unsafe.NewArray directly.
func NewArray(typ interface{}, n int) Pointer
<commit_msg>unsafe: fix the documentation Reflection is about Pointer, not uintptr. Also clarify that Reflect and Unreflect are indirect.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
	The unsafe package contains operations that step around the type safety of Go programs.
*/
package unsafe

// ArbitraryType is here for the purposes of documentation only and is not actually
// part of the unsafe package.  It represents the type of an arbitrary Go expression.
type ArbitraryType int

// Pointer represents a pointer to an arbitrary type.  There are three special operations
// available for type Pointer that are not available for other types.
//	1) A pointer value of any type can be converted to a Pointer.
//	2) A Pointer can be converted to a pointer value of any type.
//	3) A uintptr can be converted to a Pointer.
//	4) A Pointer can be converted to a uintptr.
// Pointer therefore allows a program to defeat the type system and read and write
// arbitrary memory. It should be used with extreme care.
type Pointer *ArbitraryType

// Sizeof returns the size in bytes occupied by the value v.  The size is that of the
// "top level" of the value only.  For instance, if v is a slice, it returns the size of
// the slice descriptor, not the size of the memory referenced by the slice.
func Sizeof(v ArbitraryType) int

// Offsetof returns the offset within the struct of the field represented by v,
// which must be of the form struct_value.field.  In other words, it returns the
// number of bytes between the start of the struct and the start of the field.
func Offsetof(v ArbitraryType) int

// Alignof returns the alignment of the value v.  It is the maximum value m such
// that the address of a variable with the type of v will always always be zero mod m.
// If v is of the form obj.f, it returns the alignment of field f within struct object obj.
func Alignof(v ArbitraryType) int

// Typeof returns the type of an interface value, a runtime.Type.
func Typeof(i interface{}) (typ interface{})

// Reflect unpacks an interface value into its type and the address of a copy of the
// internal value.
func Reflect(i interface{}) (typ interface{}, addr Pointer)

// Unreflect inverts Reflect: Given a type and a pointer to a value, it returns an
// empty interface value with contents the type and the value (not the pointer to
// the value).  The typ is assumed to contain a pointer to a runtime type; the type
// information in the interface{} is ignored, so that, for example, both
// *reflect.StructType and *runtime.StructType can be passed for typ.
func Unreflect(typ interface{}, addr Pointer) (ret interface{})

// New allocates and returns a pointer to memory for a new value of the given type.
// The typ is assumed to hold a pointer to a runtime type.
// Callers should use reflect.MakeZero instead of invoking unsafe.New directly.
func New(typ interface{}) Pointer

// NewArray allocates and returns a pointer to an array of n elements of the given type.
// The typ is assumed to hold a pointer to a runtime type.
// Callers should use reflect.MakeSlice instead of invoking unsafe.NewArray directly.
func NewArray(typ interface{}, n int) Pointer
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
	"github.com/mitchellh/goamz/rds"
)

func TestAccAWSDbSubnetGroup(t *testing.T) {
	var v rds.DBSubnetGroup

	testCheck := func(*terraform.State) error {
		return nil
	}

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckDbSubnetGroupDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccDbSubnetGroupConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckDbSubnetGroupExists(
						"aws_db_subnet_group.foo", &v),
					testCheck,
				),
			},
		},
	})
}

func testAccCheckDbSubnetGroupDestroy(s *terraform.State) error {
	conn := testAccProvider.rdsconn

	for _, rs := range s.Resources {
		if rs.Type != "aws_db_subnet_group" {
			continue
		}

		// Try to find the resource
		resp, err := conn.DescribeDBSubnetGroups(&rds.DescribeDBSubnetGroups{rs.ID})
		if err == nil {
			if len(resp.DBSubnetGroups) > 0 {
				return fmt.Errorf("still exist.")
			}

			return nil
		}

		// Verify the error is what we want
		rdserr, ok := err.(*rds.Error)
		if !ok {
			return err
		}
		if rdserr.Code != "DBSubnetGroupNotFoundFault" {
			return err
		}
	}

	return nil
}

func testAccCheckDbSubnetGroupExists(n string, v *rds.DBSubnetGroup) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.ID == "" {
			return fmt.Errorf("No ID is set")
		}

		conn := testAccProvider.rdsconn
		resp, err := conn.DescribeDBSubnetGroups(&rds.DescribeDBSubnetGroups{rs.ID})
		if err != nil {
			return err
		}
		if len(resp.DBSubnetGroups) == 0 {
			return fmt.Errorf("DbSubnetGroup not found")
		}

		*v = resp.DBSubnetGroups[0]

		return nil
	}
}

const testAccDbSubnetGroupConfig = `
resource "aws_vpc" "foo" {
	cidr_block = "10.1.0.0/16"
}

resource "aws_subnet" "foo" {
	cidr_block = "10.1.1.0/24"
	vpc_id = "${aws_vpc.foo.id}"
}

resource "aws_subnet" "bar" {
	cidr_block = "10.1.2.0/24"
	vpc_id = "${aws_vpc.foo.id}"
}

resource "aws_db_subnet_group" "foo" {
	name = "foo"
	description = "foo description"
	subnet_ids = ["${aws_subnet.foo.id}", "${aws_subnet.bar.id}"]
}
`
<commit_msg>providers/aws: fix some broken tests<commit_after>package aws

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
	"github.com/mitchellh/goamz/rds"
)

func TestAccAWSDbSubnetGroup(t *testing.T) {
	var v rds.DBSubnetGroup

	testCheck := func(*terraform.State) error {
		return nil
	}

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckDbSubnetGroupDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccDbSubnetGroupConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckDbSubnetGroupExists(
						"aws_db_subnet_group.foo", &v),
					testCheck,
				),
			},
		},
	})
}

func testAccCheckDbSubnetGroupDestroy(s *terraform.State) error {
	conn := testAccProvider.rdsconn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_db_subnet_group" {
			continue
		}

		// Try to find the resource
		resp, err := conn.DescribeDBSubnetGroups(&rds.DescribeDBSubnetGroups{rs.Primary.ID})
		if err == nil {
			if len(resp.DBSubnetGroups) > 0 {
				return fmt.Errorf("still exist.")
			}

			return nil
		}

		// Verify the error is what we want
		rdserr, ok := err.(*rds.Error)
		if !ok {
			return err
		}
		if rdserr.Code != "DBSubnetGroupNotFoundFault" {
			return err
		}
	}

	return nil
}

func testAccCheckDbSubnetGroupExists(n string, v *rds.DBSubnetGroup) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No ID is set")
		}

		conn := testAccProvider.rdsconn
		resp, err := conn.DescribeDBSubnetGroups(&rds.DescribeDBSubnetGroups{rs.Primary.ID})
		if err != nil {
			return err
		}
		if len(resp.DBSubnetGroups) == 0 {
			return fmt.Errorf("DbSubnetGroup not found")
		}

		*v = resp.DBSubnetGroups[0]

		return nil
	}
}

const testAccDbSubnetGroupConfig = `
resource "aws_vpc" "foo" {
	cidr_block = "10.1.0.0/16"
}

resource "aws_subnet" "foo" {
	cidr_block = "10.1.1.0/24"
	vpc_id = "${aws_vpc.foo.id}"
}

resource "aws_subnet" "bar" {
	cidr_block = "10.1.2.0/24"
	vpc_id = "${aws_vpc.foo.id}"
}

resource "aws_db_subnet_group" "foo" {
	name = "foo"
	description = "foo description"
	subnet_ids = ["${aws_subnet.foo.id}", "${aws_subnet.bar.id}"]
}
`
<|endoftext|>


<commit_before>package wdclient

import (
	"errors"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"math/rand"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"

	"github.com/chrislusf/seaweedfs/weed/glog"
)

const (
	maxCursorIndex = 4096
)

type HasLookupFileIdFunction interface {
	GetLookupFileIdFunction() LookupFileIdFunctionType
}

type LookupFileIdFunctionType func(fileId string) (targetUrls []string, err error)

type Location struct {
	Url        string `json:"url,omitempty"`
	PublicUrl  string `json:"publicUrl,omitempty"`
	DataCenter string `json:"dataCenter,omitempty"`
	GrpcPort   int    `json:"grpcPort,omitempty"`
}

func (l Location) ServerAddress() pb.ServerAddress {
	return pb.NewServerAddressWithGrpcPort(l.Url, l.GrpcPort)
}

type vidMap struct {
	sync.RWMutex
	vid2Locations map[uint32][]Location
	DataCenter    string
	cursor        int32
}

func newVidMap(dataCenter string) vidMap {
	return vidMap{
		vid2Locations: make(map[uint32][]Location),
		DataCenter:    dataCenter,
		cursor:        -1,
	}
}

func (vc *vidMap) getLocationIndex(length int) (int, error) {
	if length <= 0 {
		return 0, fmt.Errorf("invalid length: %d", length)
	}
	if atomic.LoadInt32(&vc.cursor) == maxCursorIndex {
		atomic.CompareAndSwapInt32(&vc.cursor, maxCursorIndex, -1)
	}
	return int(atomic.AddInt32(&vc.cursor, 1)) % length, nil
}

func (vc *vidMap) LookupVolumeServerUrl(vid string) (serverUrls []string, err error) {
	id, err := strconv.Atoi(vid)
	if err != nil {
		glog.V(1).Infof("Unknown volume id %s", vid)
		return nil, err
	}

	locations, found := vc.GetLocations(uint32(id))
	if !found {
		return nil, fmt.Errorf("volume %d not found", id)
	}
	var sameDcServers, otherDcServers []string
	for _, loc := range locations {
		if vc.DataCenter == "" || loc.DataCenter == "" || vc.DataCenter != loc.DataCenter {
			sameDcServers = append(sameDcServers, loc.Url)
		} else {
			otherDcServers = append(otherDcServers, loc.Url)
		}
	}
	rand.Shuffle(len(sameDcServers), func(i, j int) {
		sameDcServers[i], sameDcServers[j] = sameDcServers[j], sameDcServers[i]
	})
	rand.Shuffle(len(otherDcServers), func(i, j int) {
		otherDcServers[i], otherDcServers[j] = otherDcServers[j], otherDcServers[i]
	})
	serverUrls = append(sameDcServers, otherDcServers...)
	return
}

func (vc *vidMap) GetLookupFileIdFunction() LookupFileIdFunctionType {
	return vc.LookupFileId
}

func (vc *vidMap) LookupFileId(fileId string) (fullUrls []string, err error) {
	parts := strings.Split(fileId, ",")
	if len(parts) != 2 {
		return nil, errors.New("Invalid fileId " + fileId)
	}
	serverUrls, lookupError := vc.LookupVolumeServerUrl(parts[0])
	if lookupError != nil {
		return nil, lookupError
	}
	for _, serverUrl := range serverUrls {
		fullUrls = append(fullUrls, "http://"+serverUrl+"/"+fileId)
	}
	return
}

func (vc *vidMap) GetVidLocations(vid string) (locations []Location, err error) {
	id, err := strconv.Atoi(vid)
	if err != nil {
		glog.V(1).Infof("Unknown volume id %s", vid)
		return nil, fmt.Errorf("Unknown volume id %s", vid)
	}
	foundLocations, found := vc.GetLocations(uint32(id))
	if found {
		return foundLocations, nil
	}
	return nil, fmt.Errorf("volume id %s not found", vid)
}

func (vc *vidMap) GetLocations(vid uint32) (locations []Location, found bool) {
	vc.RLock()
	defer vc.RUnlock()

	locations, found = vc.vid2Locations[vid]
	return
}

func (vc *vidMap) addLocation(vid uint32, location Location) {
	vc.Lock()
	defer vc.Unlock()

	locations, found := vc.vid2Locations[vid]
	if !found {
		vc.vid2Locations[vid] = []Location{location}
		return
	}

	for _, loc := range locations {
		if loc.Url == location.Url {
			return
		}
	}

	vc.vid2Locations[vid] = append(locations, location)

}

func (vc *vidMap) deleteLocation(vid uint32, location Location) {
	vc.Lock()
	defer vc.Unlock()

	locations, found := vc.vid2Locations[vid]
	if !found {
		return
	}

	for i, loc := range locations {
		if loc.Url == location.Url {
			vc.vid2Locations[vid] = append(locations[0:i], locations[i+1:]...)
			break
		}
	}

}
<commit_msg>fix same dc and other dc<commit_after>package wdclient

import (
	"errors"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"math/rand"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"

	"github.com/chrislusf/seaweedfs/weed/glog"
)

const (
	maxCursorIndex = 4096
)

type HasLookupFileIdFunction interface {
	GetLookupFileIdFunction() LookupFileIdFunctionType
}

type LookupFileIdFunctionType func(fileId string) (targetUrls []string, err error)

type Location struct {
	Url        string `json:"url,omitempty"`
	PublicUrl  string `json:"publicUrl,omitempty"`
	DataCenter string `json:"dataCenter,omitempty"`
	GrpcPort   int    `json:"grpcPort,omitempty"`
}

func (l Location) ServerAddress() pb.ServerAddress {
	return pb.NewServerAddressWithGrpcPort(l.Url, l.GrpcPort)
}

type vidMap struct {
	sync.RWMutex
	vid2Locations map[uint32][]Location
	DataCenter    string
	cursor        int32
}

func newVidMap(dataCenter string) vidMap {
	return vidMap{
		vid2Locations: make(map[uint32][]Location),
		DataCenter:    dataCenter,
		cursor:        -1,
	}
}

func (vc *vidMap) getLocationIndex(length int) (int, error) {
	if length <= 0 {
		return 0, fmt.Errorf("invalid length: %d", length)
	}
	if atomic.LoadInt32(&vc.cursor) == maxCursorIndex {
		atomic.CompareAndSwapInt32(&vc.cursor, maxCursorIndex, -1)
	}
	return int(atomic.AddInt32(&vc.cursor, 1)) % length, nil
}

func (vc *vidMap) LookupVolumeServerUrl(vid string) (serverUrls []string, err error) {
	id, err := strconv.Atoi(vid)
	if err != nil {
		glog.V(1).Infof("Unknown volume id %s", vid)
		return nil, err
	}

	locations, found := vc.GetLocations(uint32(id))
	if !found {
		return nil, fmt.Errorf("volume %d not found", id)
	}
	var sameDcServers, otherDcServers []string
	for _, loc := range locations {
		if vc.DataCenter == "" || loc.DataCenter == "" || vc.DataCenter != loc.DataCenter {
			otherDcServers = append(otherDcServers, loc.Url)
		} else {
			sameDcServers = append(sameDcServers, loc.Url)
		}
	}
	rand.Shuffle(len(sameDcServers), func(i, j int) {
		sameDcServers[i], sameDcServers[j] = sameDcServers[j], sameDcServers[i]
	})
	rand.Shuffle(len(otherDcServers), func(i, j int) {
		otherDcServers[i], otherDcServers[j] = otherDcServers[j], otherDcServers[i]
	})
	serverUrls = append(sameDcServers, otherDcServers...)
	return
}

func (vc *vidMap) GetLookupFileIdFunction() LookupFileIdFunctionType {
	return vc.LookupFileId
}

func (vc *vidMap) LookupFileId(fileId string) (fullUrls []string, err error) {
	parts := strings.Split(fileId, ",")
	if len(parts) != 2 {
		return nil, errors.New("Invalid fileId " + fileId)
	}
	serverUrls, lookupError := vc.LookupVolumeServerUrl(parts[0])
	if lookupError != nil {
		return nil, lookupError
	}
	for _, serverUrl := range serverUrls {
		fullUrls = append(fullUrls, "http://"+serverUrl+"/"+fileId)
	}
	return
}

func (vc *vidMap) GetVidLocations(vid string) (locations []Location, err error) {
	id, err := strconv.Atoi(vid)
	if err != nil {
		glog.V(1).Infof("Unknown volume id %s", vid)
		return nil, fmt.Errorf("Unknown volume id %s", vid)
	}
	foundLocations, found := vc.GetLocations(uint32(id))
	if found {
		return foundLocations, nil
	}
	return nil, fmt.Errorf("volume id %s not found", vid)
}

func (vc *vidMap) GetLocations(vid uint32) (locations []Location, found bool) {
	vc.RLock()
	defer vc.RUnlock()

	locations, found = vc.vid2Locations[vid]
	return
}

func (vc *vidMap) addLocation(vid uint32, location Location) {
	vc.Lock()
	defer vc.Unlock()

	locations, found := vc.vid2Locations[vid]
	if !found {
		vc.vid2Locations[vid] = []Location{location}
		return
	}

	for _, loc := range locations {
		if loc.Url == location.Url {
			return
		}
	}

	vc.vid2Locations[vid] = append(locations, location)

}

func (vc *vidMap) deleteLocation(vid uint32, location Location) {
	vc.Lock()
	defer vc.Unlock()

	locations, found := vc.vid2Locations[vid]
	if !found {
		return
	}

	for i, loc := range locations {
		if loc.Url == location.Url {
			vc.vid2Locations[vid] = append(locations[0:i], locations[i+1:]...)
			break
		}
	}

}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package Globals

// The following types should really be in their respective files
// (object.go, type.go, scope.go, package.go, compilation.go, etc.) but
// they refer to each other and we don't know how to handle forward
// declared pointers across packages yet.

// ----------------------------------------------------------------------------

type Type struct
type Scope struct
type Elem struct
type Compilation struct

export type Object struct {
	exported bool;
	pos int;  // source position (< 0 if unknown position)
	kind int;
	ident string;
	typ *Type;  // nil for packages
	pnolev int;  // >= 0: package no., <= 0: function nesting level, 0: global level
}

export type Type struct {
	ref int;  // for exporting only: >= 0 means already exported
	form int;
	flags int;  // channels, functions
	size int;  // in bytes
	len_ int;  // array length, no. of parameters (w/o recv)
	obj *Object;  // primary type object or NULL
	aux *Type;  // alias base type or map key
	elt *Type;  // aliases, arrays, maps, channels, pointers
	scope *Scope;  // forwards, structs, interfaces, functions
}

export type Package struct {
	ref int;  // for exporting only: >= 0 means already exported
	file_name string;
	key string;
	obj *Object;
	scope *Scope;  // holds the (global) objects in this package
}

export type List struct {
	len_ int;
	first, last *Elem;
};

export type Scope struct {
	parent *Scope;
	entries *List;
	// entries *map[string] *Object;  // doesn't work properly
}

export type Flags struct {
	debug bool;
	object_file string;
	update_packages bool;
	print_interface bool;
	verbosity uint;
	sixg bool;

	scan bool;
	parse bool;
	ast bool;
	deps bool;
	token_chan bool;
}

export type Environment struct {
	Error *(comp *Compilation);  // TODO complete this
	Import *(comp *Compilation, pkg_file string) *Package;
	Export *(comp *Compilation, pkg_file string);
	Compile *(comp *Compilation, src_file string);
}

export type Compilation struct {
	// environment
	flags *Flags;
	env *Environment;

	// TODO use open arrays eventually
	pkg_list [256] *Package;  // pkg_list[0] is the current package
	pkg_ref int;
}

export type Expr interface {
	pos() int;  // source position
	typ() *Type;
	// ... more to come here
}

export type Stat interface {
	// ... more to come here
}

// TODO This is hideous! We need to have a decent way to do lists.
// Ideally open arrays that allow '+'.

type Elem struct {
	next *Elem;
	val int;
	str string;
	obj *Object;
	typ *Type;
	expr Expr
}

// ----------------------------------------------------------------------------
// Creation

export var Universe_void_t *Type  // initialized by Universe to Universe.void_t

export func NewObject(pos, kind int, ident string) *Object {
	obj := new(Object);
	obj.exported = false;
	obj.pos = pos;
	obj.kind = kind;
	obj.ident = ident;
	obj.typ = Universe_void_t;
	obj.pnolev = 0;
	return obj;
}

export func NewType(form int) *Type {
	typ := new(Type);
	typ.ref = -1;  // not yet exported
	typ.form = form;
	return typ;
}

export func NewPackage(file_name string, obj *Object, scope *Scope) *Package {
	pkg := new(Package);
	pkg.ref = -1;  // not yet exported
	pkg.file_name = file_name;
	pkg.key = "<the package key>";  // empty key means package forward declaration
	pkg.obj = obj;
	pkg.scope = scope;
	return pkg;
}

export func NewList() *List {
	return new(List);
}

export func NewScope(parent *Scope) *Scope {
	scope := new(Scope);
	scope.parent = parent;
	scope.entries = NewList();
	return scope;
}

// ----------------------------------------------------------------------------
// Object methods

func (obj *Object) Copy() *Object {
	copy := new(Object);
	copy.exported = obj.exported;
	copy.pos = obj.pos;
	copy.kind = obj.kind;
	copy.ident = obj.ident;
	copy.typ = obj.typ;
	copy.pnolev = obj.pnolev;
	return copy;
}

// ----------------------------------------------------------------------------
// List methods

func (L* List) len_() int {
	return L.len_;
}

func (L *List) at(i int) *Elem {
	if i < 0 || L.len_ <= i {
		panic("index out of bounds");
	}

	p := L.first;
	for ; i > 0; i-- {
		p = p.next;
	}

	return p;
}

func (L *List) Clear() {
	L.len_, L.first, L.last = 0, nil, nil;
}

func (L *List) Add() *Elem {
	L.len_++;
	e := new(Elem);
	if L.first == nil {
		L.first = e;
	} else {
		L.last.next = e;
	}
	L.last = e;
	return e;
}

func (L *List) IntAt(i int) int {
	return L.at(i).val;
}

func (L *List) StrAt(i int) string {
	return L.at(i).str;
}

func (L *List) ObjAt(i int) *Object {
	return L.at(i).obj;
}

func (L *List) TypAt(i int) *Type {
	return L.at(i).typ;
}

func (L *List) AddInt(val int) {
	L.Add().val = val;
}

func (L *List) AddStr(str string) {
	L.Add().str = str;
}

func (L *List) AddObj(obj *Object) {
	L.Add().obj = obj;
}

func (L *List) AddTyp(typ *Type) {
	L.Add().typ = typ;
}

func (L *List) AddExpr(expr Expr) {
	L.Add().expr = expr;
}

// ----------------------------------------------------------------------------
// Scope methods

func (scope *Scope) Lookup(ident string) *Object {
	for p := scope.entries.first; p != nil; p = p.next {
		if p.obj.ident == ident {
			return p.obj;
		}
	}
	return nil;
}

func (scope *Scope) Insert(obj *Object) {
	if scope.Lookup(obj.ident) != nil {
		panic("obj already inserted");
	}
	scope.entries.AddObj(obj);
}

func (scope *Scope) InsertImport(obj *Object) *Object {
	 p := scope.Lookup(obj.ident);
	 if p == nil {
		scope.Insert(obj);
		p = obj;
	 }
	 return p;
}

func (scope *Scope) Print() {
	print("scope {");
	for p := scope.entries.first; p != nil; p = p.next {
		print("\n  ", p.obj.ident);
	}
	print("\n}\n");
}

// ----------------------------------------------------------------------------
// Compilation methods

func (C *Compilation) Lookup(file_name string) *Package {
	for i := 0; i < C.pkg_ref; i++ {
		pkg := C.pkg_list[i];
		if pkg.file_name == file_name {
			return pkg;
		}
	}
	return nil;
}

func (C *Compilation) Insert(pkg *Package) {
	if C.Lookup(pkg.file_name) != nil {
		panic("package already inserted");
	}
	pkg.obj.pnolev = C.pkg_ref;
	C.pkg_list[C.pkg_ref] = pkg;
	C.pkg_ref++;
}
<commit_msg>- removed double-declaration of len()<commit_after>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package Globals

// The following types should really be in their respective files
// (object.go, type.go, scope.go, package.go, compilation.go, etc.) but
// they refer to each other and we don't know how to handle forward
// declared pointers across packages yet.

// ----------------------------------------------------------------------------

type Type struct
type Scope struct
type Elem struct
type Compilation struct

export type Object struct {
	exported bool;
	pos int;  // source position (< 0 if unknown position)
	kind int;
	ident string;
	typ *Type;  // nil for packages
	pnolev int;  // >= 0: package no., <= 0: function nesting level, 0: global level
}

export type Type struct {
	ref int;  // for exporting only: >= 0 means already exported
	form int;
	flags int;  // channels, functions
	size int;  // in bytes
	len_ int;  // array length, no. of parameters (w/o recv)
	obj *Object;  // primary type object or NULL
	aux *Type;  // alias base type or map key
	elt *Type;  // aliases, arrays, maps, channels, pointers
	scope *Scope;  // forwards, structs, interfaces, functions
}

export type Package struct {
	ref int;  // for exporting only: >= 0 means already exported
	file_name string;
	key string;
	obj *Object;
	scope *Scope;  // holds the (global) objects in this package
}

export type List struct {
	len_ int;
	first, last *Elem;
};

export type Scope struct {
	parent *Scope;
	entries *List;
	// entries *map[string] *Object;  // doesn't work properly
}

export type Flags struct {
	debug bool;
	object_file string;
	update_packages bool;
	print_interface bool;
	verbosity uint;
	sixg bool;

	scan bool;
	parse bool;
	ast bool;
	deps bool;
	token_chan bool;
}

export type Environment struct {
	Error *(comp *Compilation);  // TODO complete this
	Import *(comp *Compilation, pkg_file string) *Package;
	Export *(comp *Compilation, pkg_file string);
	Compile *(comp *Compilation, src_file string);
}

export type Compilation struct {
	// environment
	flags *Flags;
	env *Environment;

	// TODO use open arrays eventually
	pkg_list [256] *Package;  // pkg_list[0] is the current package
	pkg_ref int;
}

export type Expr interface {
	pos() int;  // source position
	typ() *Type;
	// ... more to come here
}

export type Stat interface {
	// ... more to come here
}

// TODO This is hideous! We need to have a decent way to do lists.
// Ideally open arrays that allow '+'.

type Elem struct {
	next *Elem;
	val int;
	str string;
	obj *Object;
	typ *Type;
	expr Expr
}

// ----------------------------------------------------------------------------
// Creation

export var Universe_void_t *Type  // initialized by Universe to Universe.void_t

export func NewObject(pos, kind int, ident string) *Object {
	obj := new(Object);
	obj.exported = false;
	obj.pos = pos;
	obj.kind = kind;
	obj.ident = ident;
	obj.typ = Universe_void_t;
	obj.pnolev = 0;
	return obj;
}

export func NewType(form int) *Type {
	typ := new(Type);
	typ.ref = -1;  // not yet exported
	typ.form = form;
	return typ;
}

export func NewPackage(file_name string, obj *Object, scope *Scope) *Package {
	pkg := new(Package);
	pkg.ref = -1;  // not yet exported
	pkg.file_name = file_name;
	pkg.key = "<the package key>";  // empty key means package forward declaration
	pkg.obj = obj;
	pkg.scope = scope;
	return pkg;
}

export func NewList() *List {
	return new(List);
}

export func NewScope(parent *Scope) *Scope {
	scope := new(Scope);
	scope.parent = parent;
	scope.entries = NewList();
	return scope;
}

// ----------------------------------------------------------------------------
// Object methods

func (obj *Object) Copy() *Object {
	copy := new(Object);
	copy.exported = obj.exported;
	copy.pos = obj.pos;
	copy.kind = obj.kind;
	copy.ident = obj.ident;
	copy.typ = obj.typ;
	copy.pnolev = obj.pnolev;
	return copy;
}

// ----------------------------------------------------------------------------
// List methods

func (L *List) at(i int) *Elem {
	if i < 0 || L.len_ <= i {
		panic("index out of bounds");
	}

	p := L.first;
	for ; i > 0; i-- {
		p = p.next;
	}

	return p;
}

func (L *List) Clear() {
	L.len_, L.first, L.last = 0, nil, nil;
}

func (L *List) Add() *Elem {
	L.len_++;
	e := new(Elem);
	if L.first == nil {
		L.first = e;
	} else {
		L.last.next = e;
	}
	L.last = e;
	return e;
}

func (L *List) IntAt(i int) int {
	return L.at(i).val;
}

func (L *List) StrAt(i int) string {
	return L.at(i).str;
}

func (L *List) ObjAt(i int) *Object {
	return L.at(i).obj;
}

func (L *List) TypAt(i int) *Type {
	return L.at(i).typ;
}

func (L *List) AddInt(val int) {
	L.Add().val = val;
}

func (L *List) AddStr(str string) {
	L.Add().str = str;
}

func (L *List) AddObj(obj *Object) {
	L.Add().obj = obj;
}

func (L *List) AddTyp(typ *Type) {
	L.Add().typ = typ;
}

func (L *List) AddExpr(expr Expr) {
	L.Add().expr = expr;
}

// ----------------------------------------------------------------------------
// Scope methods

func (scope *Scope) Lookup(ident string) *Object {
	for p := scope.entries.first; p != nil; p = p.next {
		if p.obj.ident == ident {
			return p.obj;
		}
	}
	return nil;
}

func (scope *Scope) Insert(obj *Object) {
	if scope.Lookup(obj.ident) != nil {
		panic("obj already inserted");
	}
	scope.entries.AddObj(obj);
}

func (scope *Scope) InsertImport(obj *Object) *Object {
	 p := scope.Lookup(obj.ident);
	 if p == nil {
		scope.Insert(obj);
		p = obj;
	 }
	 return p;
}

func (scope *Scope) Print() {
	print("scope {");
	for p := scope.entries.first; p != nil; p = p.next {
		print("\n  ", p.obj.ident);
	}
	print("\n}\n");
}

// ----------------------------------------------------------------------------
// Compilation methods

func (C *Compilation) Lookup(file_name string) *Package {
	for i := 0; i < C.pkg_ref; i++ {
		pkg := C.pkg_list[i];
		if pkg.file_name == file_name {
			return pkg;
		}
	}
	return nil;
}

func (C *Compilation) Insert(pkg *Package) {
	if C.Lookup(pkg.file_name) != nil {
		panic("package already inserted");
	}
	pkg.obj.pnolev = C.pkg_ref;
	C.pkg_list[C.pkg_ref] = pkg;
	C.pkg_ref++;
}
<|endoftext|>


<commit_before><commit_msg>fix logsend crasher (#6562)<commit_after><|endoftext|>


<commit_before>package null

import (
	"fmt"

	"github.com/RichardKnop/machinery/v1/backends/iface"
	"github.com/RichardKnop/machinery/v1/common"
	"github.com/RichardKnop/machinery/v1/config"
	"github.com/RichardKnop/machinery/v1/tasks"
)

// ErrGroupNotFound ...
type ErrGroupNotFound struct {
	groupUUID string
}

// NewErrGroupNotFound returns new instance of ErrGroupNotFound
func NewErrGroupNotFound(groupUUID string) ErrGroupNotFound {
	return ErrGroupNotFound{groupUUID: groupUUID}
}

// Error implements error interface
func (e ErrGroupNotFound) Error() string {
	return fmt.Sprintf("Group not found: %v", e.groupUUID)
}

// ErrTasknotFound ...
type ErrTasknotFound struct {
	taskUUID string
}

// NewErrTasknotFound returns new instance of ErrTasknotFound
func NewErrTasknotFound(taskUUID string) ErrTasknotFound {
	return ErrTasknotFound{taskUUID: taskUUID}
}

// Error implements error interface
func (e ErrTasknotFound) Error() string {
	return fmt.Sprintf("Task not found: %v", e.taskUUID)
}

// Backend represents an "eager" in-memory result backend
type Backend struct {
	common.Backend
	groups map[string]struct{}
}

// New creates EagerBackend instance
func New() iface.Backend {
	return &Backend{
		Backend: common.NewBackend(new(config.Config)),
		groups:  make(map[string]struct{}),
	}
}

// InitGroup creates and saves a group meta data object
func (b *Backend) InitGroup(groupUUID string, taskUUIDs []string) error {
	b.groups[groupUUID] = struct{}{}
	return nil
}

// GroupCompleted returns true if all tasks in a group finished
func (b *Backend) GroupCompleted(groupUUID string, groupTaskCount int) (bool, error) {
	_, ok := b.groups[groupUUID]
	if !ok {
		return false, NewErrGroupNotFound(groupUUID)
	}

	return true, nil
}

// GroupTaskStates returns states of all tasks in the group
func (b *Backend) GroupTaskStates(groupUUID string, groupTaskCount int) ([]*tasks.TaskState, error) {
	_, ok := b.groups[groupUUID]
	if !ok {
		return nil, NewErrGroupNotFound(groupUUID)
	}

	ret := make([]*tasks.TaskState, 0, groupTaskCount)
	return ret, nil
}

// TriggerChord flags chord as triggered in the backend storage to make sure
// chord is never trigerred multiple times. Returns a boolean flag to indicate
// whether the worker should trigger chord (true) or no if it has been triggered
// already (false)
func (b *Backend) TriggerChord(groupUUID string) (bool, error) {
	return true, nil
}

// SetStatePending updates task state to PENDING
func (b *Backend) SetStatePending(signature *tasks.Signature) error {
	state := tasks.NewPendingTaskState(signature)
	return b.updateState(state)
}

// SetStateReceived updates task state to RECEIVED
func (b *Backend) SetStateReceived(signature *tasks.Signature) error {
	state := tasks.NewReceivedTaskState(signature)
	return b.updateState(state)
}

// SetStateStarted updates task state to STARTED
func (b *Backend) SetStateStarted(signature *tasks.Signature) error {
	state := tasks.NewStartedTaskState(signature)
	return b.updateState(state)
}

// SetStateRetry updates task state to RETRY
func (b *Backend) SetStateRetry(signature *tasks.Signature) error {
	state := tasks.NewRetryTaskState(signature)
	return b.updateState(state)
}

// SetStateSuccess updates task state to SUCCESS
func (b *Backend) SetStateSuccess(signature *tasks.Signature, results []*tasks.TaskResult) error {
	state := tasks.NewSuccessTaskState(signature, results)
	return b.updateState(state)
}

// SetStateFailure updates task state to FAILURE
func (b *Backend) SetStateFailure(signature *tasks.Signature, err string) error {
	state := tasks.NewFailureTaskState(signature, err)
	return b.updateState(state)
}

// GetState returns the latest task state
func (b *Backend) GetState(taskUUID string) (*tasks.TaskState, error) {
	return nil, NewErrTasknotFound(taskUUID)
}

// PurgeState deletes stored task state
func (b *Backend) PurgeState(taskUUID string) error {
	return NewErrTasknotFound(taskUUID)
}

// PurgeGroupMeta deletes stored group meta data
func (b *Backend) PurgeGroupMeta(groupUUID string) error {
	_, ok := b.groups[groupUUID]
	if !ok {
		return NewErrGroupNotFound(groupUUID)
	}

	return nil
}

func (b *Backend) updateState(s *tasks.TaskState) error {
	// simulate the behavior of json marshal/unmarshal
	return nil
}
<commit_msg>CC - Cleanup some comments in null ResultBackend (#603)<commit_after>package null

import (
	"fmt"

	"github.com/RichardKnop/machinery/v1/backends/iface"
	"github.com/RichardKnop/machinery/v1/common"
	"github.com/RichardKnop/machinery/v1/config"
	"github.com/RichardKnop/machinery/v1/tasks"
)

// ErrGroupNotFound ...
type ErrGroupNotFound struct {
	groupUUID string
}

// NewErrGroupNotFound returns new instance of ErrGroupNotFound
func NewErrGroupNotFound(groupUUID string) ErrGroupNotFound {
	return ErrGroupNotFound{groupUUID: groupUUID}
}

// Error implements error interface
func (e ErrGroupNotFound) Error() string {
	return fmt.Sprintf("Group not found: %v", e.groupUUID)
}

// ErrTasknotFound ...
type ErrTasknotFound struct {
	taskUUID string
}

// NewErrTasknotFound returns new instance of ErrTasknotFound
func NewErrTasknotFound(taskUUID string) ErrTasknotFound {
	return ErrTasknotFound{taskUUID: taskUUID}
}

// Error implements error interface
func (e ErrTasknotFound) Error() string {
	return fmt.Sprintf("Task not found: %v", e.taskUUID)
}

// Backend represents an "null" result backend
type Backend struct {
	common.Backend
	groups map[string]struct{}
}

// New creates NullBackend instance
func New() iface.Backend {
	return &Backend{
		Backend: common.NewBackend(new(config.Config)),
		groups:  make(map[string]struct{}),
	}
}

// InitGroup creates and saves a group meta data object
func (b *Backend) InitGroup(groupUUID string, taskUUIDs []string) error {
	b.groups[groupUUID] = struct{}{}
	return nil
}

// GroupCompleted returns true (always)
func (b *Backend) GroupCompleted(groupUUID string, groupTaskCount int) (bool, error) {
	_, ok := b.groups[groupUUID]
	if !ok {
		return false, NewErrGroupNotFound(groupUUID)
	}

	return true, nil
}

// GroupTaskStates returns null states of all tasks in the group
func (b *Backend) GroupTaskStates(groupUUID string, groupTaskCount int) ([]*tasks.TaskState, error) {
	_, ok := b.groups[groupUUID]
	if !ok {
		return nil, NewErrGroupNotFound(groupUUID)
	}

	ret := make([]*tasks.TaskState, 0, groupTaskCount)
	return ret, nil
}

// TriggerChord returns true (always)
func (b *Backend) TriggerChord(groupUUID string) (bool, error) {
	return true, nil
}

// SetStatePending updates task state to PENDING
func (b *Backend) SetStatePending(signature *tasks.Signature) error {
	state := tasks.NewPendingTaskState(signature)
	return b.updateState(state)
}

// SetStateReceived updates task state to RECEIVED
func (b *Backend) SetStateReceived(signature *tasks.Signature) error {
	state := tasks.NewReceivedTaskState(signature)
	return b.updateState(state)
}

// SetStateStarted updates task state to STARTED
func (b *Backend) SetStateStarted(signature *tasks.Signature) error {
	state := tasks.NewStartedTaskState(signature)
	return b.updateState(state)
}

// SetStateRetry updates task state to RETRY
func (b *Backend) SetStateRetry(signature *tasks.Signature) error {
	state := tasks.NewRetryTaskState(signature)
	return b.updateState(state)
}

// SetStateSuccess updates task state to SUCCESS
func (b *Backend) SetStateSuccess(signature *tasks.Signature, results []*tasks.TaskResult) error {
	state := tasks.NewSuccessTaskState(signature, results)
	return b.updateState(state)
}

// SetStateFailure updates task state to FAILURE
func (b *Backend) SetStateFailure(signature *tasks.Signature, err string) error {
	state := tasks.NewFailureTaskState(signature, err)
	return b.updateState(state)
}

// GetState returns the latest task state
func (b *Backend) GetState(taskUUID string) (*tasks.TaskState, error) {
	return nil, NewErrTasknotFound(taskUUID)
}

// PurgeState deletes stored task state
func (b *Backend) PurgeState(taskUUID string) error {
	return NewErrTasknotFound(taskUUID)
}

// PurgeGroupMeta deletes stored group meta data
func (b *Backend) PurgeGroupMeta(groupUUID string) error {
	_, ok := b.groups[groupUUID]
	if !ok {
		return NewErrGroupNotFound(groupUUID)
	}

	return nil
}

func (b *Backend) updateState(s *tasks.TaskState) error {
	return nil
}
<|endoftext|>


<commit_before>package websocket

import (
	"context"
	"encoding/json"
	"fmt"

	"github.com/bitfinexcom/bitfinex-api-go/pkg/convert"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/fundingcredit"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/fundingloan"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/fundingoffer"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/fundingtrade"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/order"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/position"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/tradeexecutionupdate"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/wallet"
	"github.com/bitfinexcom/bitfinex-api-go/v2"
)

type Heartbeat struct {
	//ChannelIDs []int64
}

func (c *Client) handleChannel(socketId SocketId, msg []byte) error {
	if c.terminal {
		return fmt.Errorf("received a message after close")
	}

	var raw []interface{}
	err := json.Unmarshal(msg, &raw)
	if err != nil {
		return err
	} else if len(raw) < 2 {
		return nil
	}

	chID, ok := raw[0].(float64)
	if !ok {
		return fmt.Errorf("expected message to start with a channel id but got %#v instead", raw[0])
	}

	chanID := int64(chID)
	sub, err := c.subscriptions.lookupBySocketChannelID(chanID, socketId)
	if err != nil {
		// no subscribed channel for message
		return err
	}
	c.subscriptions.heartbeat(chanID)
	if sub.Public {
		switch data := raw[1].(type) {
		case string:
			switch data {
			case "hb":
				// no-op, already updated heartbeat timeout from this event
				return nil
			case "cs":
				if checksum, ok := raw[2].(float64); ok {
					return c.handleChecksumChannel(sub, int(checksum))
				} else {
					c.log.Error("Unable to parse checksum")
				}
			default:
				body := raw[2].([]interface{})
				return c.handlePublicChannel(sub, sub.Request.Channel, data, body, msg)
			}
		case []interface{}:
			return c.handlePublicChannel(sub, sub.Request.Channel, "", data, msg)
		}
	} else {
		return c.handlePrivateChannel(raw)
	}
	return nil
}

func (c *Client) handleChecksumChannel(sub *subscription, checksum int) error {
	symbol := sub.Request.Symbol
	// force to signed integer
	bChecksum := uint32(checksum)
	var orderbook *Orderbook
	c.mtx.Lock()
	if ob, ok := c.orderbooks[symbol]; ok {
		orderbook = ob
	}
	c.mtx.Unlock()
	if orderbook != nil {
		oChecksum := orderbook.Checksum()
		// compare bitfinex checksum with local checksum
		if bChecksum == oChecksum {
			c.log.Debugf("Orderbook '%s' checksum verification successful.", symbol)
		} else {
			c.log.Warningf("Orderbook '%s' checksum is invalid got %d bot got %d. Data Out of sync, reconnecting.",
				symbol, bChecksum, oChecksum)
			err := c.sendUnsubscribeMessage(context.Background(), sub)
			if err != nil {
				return err
			}
			newSub := &SubscriptionRequest{
				SubID:   c.nonce.GetNonce(), // generate new subID
				Event:   sub.Request.Event,
				Channel: sub.Request.Channel,
				Symbol:  sub.Request.Symbol,
			}
			_, err_sub := c.Subscribe(context.Background(), newSub)
			if err_sub != nil {
				c.log.Warningf("could not resubscribe: %s", err_sub.Error())
				return err_sub
			}
		}
	}
	return nil
}

func (c *Client) handlePublicChannel(sub *subscription, channel, objType string, data []interface{}, raw_msg []byte) error {
	// unauthenticated data slice
	// public data is returned as raw interface arrays, use a factory to convert to raw type & publish
	if factory, ok := c.factories[channel]; ok {
		// convert to type array of interfaces
		if len(data) > 0 {
			if _, ok := data[0].([]interface{}); ok {
				interfaceArray := convert.ToInterfaceArray(data)
				// snapshot item
				c.mtx.Lock()
				// lock mutex since its mutates client struct
				msg, err := factory.BuildSnapshot(sub, interfaceArray, raw_msg)
				c.mtx.Unlock()
				if err != nil {
					return err
				}
				if msg != nil {
					c.listener <- msg
				}
			} else {
				// single item
				msg, err := factory.Build(sub, objType, data, raw_msg)
				if err != nil {
					return err
				}
				if msg != nil {
					c.listener <- msg
				}
			}
		}
	} else {
		// factory lookup error
		return fmt.Errorf("could not find public factory for %s channel", channel)
	}
	return nil
}

func (c *Client) handlePrivateChannel(raw []interface{}) error {
	// authenticated data slice, or a heartbeat
	if val, ok := raw[1].(string); ok && val == "hb" {
		chanID, ok := raw[0].(float64)
		if !ok {
			c.log.Warningf("could not find chanID: %#v", raw)
			return nil
		}
		c.handleHeartbeat(int64(chanID))
	} else {
		// raw[2] is data slice
		// authenticated snapshots?
		if len(raw) > 2 {
			if arr, ok := raw[2].([]interface{}); ok {
				obj, err := c.handlePrivateDataMessage(raw[1].(string), arr)
				if err != nil {
					return err
				}
				// private data is returned as strongly typed data, publish directly
				if obj != nil {
					c.listener <- obj
				}
			}
		}
	}
	return nil
}

func (c *Client) handleHeartbeat(chanID int64) {
	c.subscriptions.heartbeat(chanID)
}

type unsubscribeMsg struct {
	Event  string `json:"event"`
	ChanID int64  `json:"chanId"`
}

// public msg: [ChanID, [Data]]
// hb (both): [ChanID, "hb"]
// private update msg: [ChanID, "type", [Data]]
// private snapshot msg: [ChanID, "type", [[Data]]]
func (c *Client) handlePrivateDataMessage(term string, data []interface{}) (ms interface{}, err error) {
	if len(data) == 0 {
		// empty data msg
		return nil, nil
	}

	if term == "hb" { // Heartbeat
		// TODO: Consider adding a switch to enable/disable passing these along.
		return &Heartbeat{}, nil
	}
	/*
		list, ok := data[2].([]interface{})
		if !ok {
			return ms, fmt.Errorf("expected data list in third position but got %#v in %#v", data[2], data)
		}
	*/
	ms = c.convertRaw(term, data)

	return
}

// convertRaw takes a term and the raw data attached to it to try and convert that
// untyped list into a proper type.
func (c *Client) convertRaw(term string, raw []interface{}) interface{} {
	// The things you do to get proper types.
	switch term {
	case "bu":
		o, err := bitfinex.NewBalanceInfoFromRaw(raw)
		if err != nil {
			return err
		}
		bu := bitfinex.BalanceUpdate(*o)
		return &bu
	case "ps":
		o, err := position.SnapshotFromRaw(raw)
		if err != nil {
			return err
		}
		return o
	case "pn":
		o, err := position.FromRaw(raw)
		if err != nil {
			return err
		}
		pn := position.New(*o)
		return &pn
	case "pu":
		o, err := position.FromRaw(raw)
		if err != nil {
			return err
		}
		pu := position.Update(*o)
		return &pu
	case "pc":
		o, err := position.FromRaw(raw)
		if err != nil {
			return err
		}
		pc := position.Cancel(*o)
		return &pc
	case "ws":
		o, err := wallet.SnapshotFromRaw(raw, wallet.FromWsRaw)
		if err != nil {
			return err
		}
		return o
	case "wu":
		o, err := wallet.FromWsRaw(raw)
		if err != nil {
			return err
		}
		wu := wallet.Update(*o)
		return &wu
	case "os":
		o, err := order.SnapshotFromRaw(raw)
		if err != nil {
			return err
		}
		return o
	case "on":
		o, err := order.FromRaw(raw)
		if err != nil {
			return err
		}
		on := order.New(*o)
		return &on
	case "ou":
		o, err := order.FromRaw(raw)
		if err != nil {
			return err
		}
		ou := order.Update(*o)
		return &ou
	case "oc":
		o, err := order.FromRaw(raw)
		if err != nil {
			return err
		}
		oc := order.Cancel(*o)
		return &oc
	case "hts":
		tu, err := tradeexecutionupdate.SnapshotFromRaw(raw)
		if err != nil {
			return err
		}
		hts := tradeexecutionupdate.HistoricalTradeSnapshot(*tu)
		return &hts
	case "te":
		o, err := bitfinex.NewTradeExecutionFromRaw(raw)
		if err != nil {
			return err
		}
		return o
	case "tu":
		tu, err := tradeexecutionupdate.FromRaw(raw)
		if err != nil {
			return err
		}
		return tu
	case "fte":
		o, err := fundingtrade.FromRaw(raw)
		if err != nil {
			return err
		}
		fte := fundingtrade.Execution(*o)
		return &fte
	case "ftu":
		o, err := fundingtrade.FromRaw(raw)
		if err != nil {
			return err
		}
		ftu := fundingtrade.Update(*o)
		return &ftu
	case "hfts":
		fts, err := fundingtrade.SnapshotFromRaw(raw)
		if err != nil {
			return err
		}
		nfts := fundingtrade.HistoricalSnapshot(*fts)
		return &nfts
	case "n":
		o, err := bitfinex.NewNotificationFromRaw(raw)
		if err != nil {
			return err
		}
		return o
	case "fos":
		o, err := fundingoffer.SnapshotFromRaw(raw)
		if err != nil {
			return err
		}
		return o
	case "fon":
		o, err := fundingoffer.FromRaw(raw)
		if err != nil {
			return err
		}
		fon := fundingoffer.New(*o)
		return &fon
	case "fou":
		o, err := fundingoffer.FromRaw(raw)
		if err != nil {
			return err
		}
		fou := fundingoffer.Update(*o)
		return &fou
	case "foc":
		o, err := fundingoffer.FromRaw(raw)
		if err != nil {
			return err
		}
		foc := fundingoffer.Cancel(*o)
		return &foc
	case "fiu":
		o, err := bitfinex.NewFundingInfoFromRaw(raw)
		if err != nil {
			return err
		}
		return o
	case "fcs":
		o, err := fundingcredit.SnapshotFromRaw(raw)
		if err != nil {
			return err
		}
		return o
	case "fcn":
		o, err := fundingcredit.FromRaw(raw)
		if err != nil {
			return err
		}
		fcn := fundingcredit.New(*o)
		return &fcn
	case "fcu":
		o, err := fundingcredit.FromRaw(raw)
		if err != nil {
			return err
		}
		fcu := fundingcredit.Update(*o)
		return &fcu
	case "fcc":
		o, err := fundingcredit.FromRaw(raw)
		if err != nil {
			return err
		}
		fcc := fundingcredit.Cancel(*o)
		return &fcc
	case "fls":
		o, err := fundingloan.SnapshotFromRaw(raw)
		if err != nil {
			return err
		}
		return o
	case "fln":
		o, err := fundingloan.FromRaw(raw)
		if err != nil {
			return err
		}
		fln := fundingloan.New(*o)
		return &fln
	case "flu":
		o, err := fundingloan.FromRaw(raw)
		if err != nil {
			return err
		}
		flu := fundingloan.Update(*o)
		return &flu
	case "flc":
		o, err := fundingloan.FromRaw(raw)
		if err != nil {
			return err
		}
		flc := fundingloan.Cancel(*o)
		return &flc
	//case "uac":
	case "hb":
		return &Heartbeat{}
	case "ats":
		// TODO: Is not in documentation, so figure out what it is.
		return nil
	case "oc-req":
		// TODO
		return nil
	case "on-req":
		// TODO
		return nil
	case "mis": // Should not be sent anymore as of 2017-04-01
		return nil
	case "miu":
		o, err := bitfinex.NewMarginInfoFromRaw(raw)
		if err != nil {
			return err
		}
		// return a strongly typed reference, rather than dereference a generic interface
		// too bad golang doesn't inherit an interface's underlying type when creating a reference to the interface
		if base, ok := o.(*bitfinex.MarginInfoBase); ok {
			return base
		}
		if update, ok := o.(*bitfinex.MarginInfoUpdate); ok {
			return update
		}
		return o // better than nothing
	default:
		c.log.Warningf("unhandled channel data, term: %s", term)
	}

	return fmt.Errorf("term %q not recognized", term)
}
<commit_msg>v2/websocket/channels.go putting new tradeexecution package to work<commit_after>package websocket

import (
	"context"
	"encoding/json"
	"fmt"

	"github.com/bitfinexcom/bitfinex-api-go/pkg/convert"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/fundingcredit"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/fundingloan"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/fundingoffer"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/fundingtrade"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/order"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/position"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/tradeexecution"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/tradeexecutionupdate"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/wallet"
	"github.com/bitfinexcom/bitfinex-api-go/v2"
)

type Heartbeat struct {
	//ChannelIDs []int64
}

func (c *Client) handleChannel(socketId SocketId, msg []byte) error {
	if c.terminal {
		return fmt.Errorf("received a message after close")
	}

	var raw []interface{}
	err := json.Unmarshal(msg, &raw)
	if err != nil {
		return err
	} else if len(raw) < 2 {
		return nil
	}

	chID, ok := raw[0].(float64)
	if !ok {
		return fmt.Errorf("expected message to start with a channel id but got %#v instead", raw[0])
	}

	chanID := int64(chID)
	sub, err := c.subscriptions.lookupBySocketChannelID(chanID, socketId)
	if err != nil {
		// no subscribed channel for message
		return err
	}
	c.subscriptions.heartbeat(chanID)
	if sub.Public {
		switch data := raw[1].(type) {
		case string:
			switch data {
			case "hb":
				// no-op, already updated heartbeat timeout from this event
				return nil
			case "cs":
				if checksum, ok := raw[2].(float64); ok {
					return c.handleChecksumChannel(sub, int(checksum))
				} else {
					c.log.Error("Unable to parse checksum")
				}
			default:
				body := raw[2].([]interface{})
				return c.handlePublicChannel(sub, sub.Request.Channel, data, body, msg)
			}
		case []interface{}:
			return c.handlePublicChannel(sub, sub.Request.Channel, "", data, msg)
		}
	} else {
		return c.handlePrivateChannel(raw)
	}
	return nil
}

func (c *Client) handleChecksumChannel(sub *subscription, checksum int) error {
	symbol := sub.Request.Symbol
	// force to signed integer
	bChecksum := uint32(checksum)
	var orderbook *Orderbook
	c.mtx.Lock()
	if ob, ok := c.orderbooks[symbol]; ok {
		orderbook = ob
	}
	c.mtx.Unlock()
	if orderbook != nil {
		oChecksum := orderbook.Checksum()
		// compare bitfinex checksum with local checksum
		if bChecksum == oChecksum {
			c.log.Debugf("Orderbook '%s' checksum verification successful.", symbol)
		} else {
			c.log.Warningf("Orderbook '%s' checksum is invalid got %d bot got %d. Data Out of sync, reconnecting.",
				symbol, bChecksum, oChecksum)
			err := c.sendUnsubscribeMessage(context.Background(), sub)
			if err != nil {
				return err
			}
			newSub := &SubscriptionRequest{
				SubID:   c.nonce.GetNonce(), // generate new subID
				Event:   sub.Request.Event,
				Channel: sub.Request.Channel,
				Symbol:  sub.Request.Symbol,
			}
			_, err_sub := c.Subscribe(context.Background(), newSub)
			if err_sub != nil {
				c.log.Warningf("could not resubscribe: %s", err_sub.Error())
				return err_sub
			}
		}
	}
	return nil
}

func (c *Client) handlePublicChannel(sub *subscription, channel, objType string, data []interface{}, raw_msg []byte) error {
	// unauthenticated data slice
	// public data is returned as raw interface arrays, use a factory to convert to raw type & publish
	if factory, ok := c.factories[channel]; ok {
		// convert to type array of interfaces
		if len(data) > 0 {
			if _, ok := data[0].([]interface{}); ok {
				interfaceArray := convert.ToInterfaceArray(data)
				// snapshot item
				c.mtx.Lock()
				// lock mutex since its mutates client struct
				msg, err := factory.BuildSnapshot(sub, interfaceArray, raw_msg)
				c.mtx.Unlock()
				if err != nil {
					return err
				}
				if msg != nil {
					c.listener <- msg
				}
			} else {
				// single item
				msg, err := factory.Build(sub, objType, data, raw_msg)
				if err != nil {
					return err
				}
				if msg != nil {
					c.listener <- msg
				}
			}
		}
	} else {
		// factory lookup error
		return fmt.Errorf("could not find public factory for %s channel", channel)
	}
	return nil
}

func (c *Client) handlePrivateChannel(raw []interface{}) error {
	// authenticated data slice, or a heartbeat
	if val, ok := raw[1].(string); ok && val == "hb" {
		chanID, ok := raw[0].(float64)
		if !ok {
			c.log.Warningf("could not find chanID: %#v", raw)
			return nil
		}
		c.handleHeartbeat(int64(chanID))
	} else {
		// raw[2] is data slice
		// authenticated snapshots?
		if len(raw) > 2 {
			if arr, ok := raw[2].([]interface{}); ok {
				obj, err := c.handlePrivateDataMessage(raw[1].(string), arr)
				if err != nil {
					return err
				}
				// private data is returned as strongly typed data, publish directly
				if obj != nil {
					c.listener <- obj
				}
			}
		}
	}
	return nil
}

func (c *Client) handleHeartbeat(chanID int64) {
	c.subscriptions.heartbeat(chanID)
}

type unsubscribeMsg struct {
	Event  string `json:"event"`
	ChanID int64  `json:"chanId"`
}

// public msg: [ChanID, [Data]]
// hb (both): [ChanID, "hb"]
// private update msg: [ChanID, "type", [Data]]
// private snapshot msg: [ChanID, "type", [[Data]]]
func (c *Client) handlePrivateDataMessage(term string, data []interface{}) (ms interface{}, err error) {
	if len(data) == 0 {
		// empty data msg
		return nil, nil
	}

	if term == "hb" { // Heartbeat
		// TODO: Consider adding a switch to enable/disable passing these along.
		return &Heartbeat{}, nil
	}
	/*
		list, ok := data[2].([]interface{})
		if !ok {
			return ms, fmt.Errorf("expected data list in third position but got %#v in %#v", data[2], data)
		}
	*/
	ms = c.convertRaw(term, data)

	return
}

// convertRaw takes a term and the raw data attached to it to try and convert that
// untyped list into a proper type.
func (c *Client) convertRaw(term string, raw []interface{}) interface{} {
	// The things you do to get proper types.
	switch term {
	case "bu":
		o, err := bitfinex.NewBalanceInfoFromRaw(raw)
		if err != nil {
			return err
		}
		bu := bitfinex.BalanceUpdate(*o)
		return &bu
	case "ps":
		o, err := position.SnapshotFromRaw(raw)
		if err != nil {
			return err
		}
		return o
	case "pn":
		o, err := position.FromRaw(raw)
		if err != nil {
			return err
		}
		pn := position.New(*o)
		return &pn
	case "pu":
		o, err := position.FromRaw(raw)
		if err != nil {
			return err
		}
		pu := position.Update(*o)
		return &pu
	case "pc":
		o, err := position.FromRaw(raw)
		if err != nil {
			return err
		}
		pc := position.Cancel(*o)
		return &pc
	case "ws":
		o, err := wallet.SnapshotFromRaw(raw, wallet.FromWsRaw)
		if err != nil {
			return err
		}
		return o
	case "wu":
		o, err := wallet.FromWsRaw(raw)
		if err != nil {
			return err
		}
		wu := wallet.Update(*o)
		return &wu
	case "os":
		o, err := order.SnapshotFromRaw(raw)
		if err != nil {
			return err
		}
		return o
	case "on":
		o, err := order.FromRaw(raw)
		if err != nil {
			return err
		}
		on := order.New(*o)
		return &on
	case "ou":
		o, err := order.FromRaw(raw)
		if err != nil {
			return err
		}
		ou := order.Update(*o)
		return &ou
	case "oc":
		o, err := order.FromRaw(raw)
		if err != nil {
			return err
		}
		oc := order.Cancel(*o)
		return &oc
	case "hts":
		tu, err := tradeexecutionupdate.SnapshotFromRaw(raw)
		if err != nil {
			return err
		}
		hts := tradeexecutionupdate.HistoricalTradeSnapshot(*tu)
		return &hts
	case "te":
		o, err := tradeexecution.FromRaw(raw)
		if err != nil {
			return err
		}
		return o
	case "tu":
		tu, err := tradeexecutionupdate.FromRaw(raw)
		if err != nil {
			return err
		}
		return tu
	case "fte":
		o, err := fundingtrade.FromRaw(raw)
		if err != nil {
			return err
		}
		fte := fundingtrade.Execution(*o)
		return &fte
	case "ftu":
		o, err := fundingtrade.FromRaw(raw)
		if err != nil {
			return err
		}
		ftu := fundingtrade.Update(*o)
		return &ftu
	case "hfts":
		fts, err := fundingtrade.SnapshotFromRaw(raw)
		if err != nil {
			return err
		}
		nfts := fundingtrade.HistoricalSnapshot(*fts)
		return &nfts
	case "n":
		o, err := bitfinex.NewNotificationFromRaw(raw)
		if err != nil {
			return err
		}
		return o
	case "fos":
		o, err := fundingoffer.SnapshotFromRaw(raw)
		if err != nil {
			return err
		}
		return o
	case "fon":
		o, err := fundingoffer.FromRaw(raw)
		if err != nil {
			return err
		}
		fon := fundingoffer.New(*o)
		return &fon
	case "fou":
		o, err := fundingoffer.FromRaw(raw)
		if err != nil {
			return err
		}
		fou := fundingoffer.Update(*o)
		return &fou
	case "foc":
		o, err := fundingoffer.FromRaw(raw)
		if err != nil {
			return err
		}
		foc := fundingoffer.Cancel(*o)
		return &foc
	case "fiu":
		o, err := bitfinex.NewFundingInfoFromRaw(raw)
		if err != nil {
			return err
		}
		return o
	case "fcs":
		o, err := fundingcredit.SnapshotFromRaw(raw)
		if err != nil {
			return err
		}
		return o
	case "fcn":
		o, err := fundingcredit.FromRaw(raw)
		if err != nil {
			return err
		}
		fcn := fundingcredit.New(*o)
		return &fcn
	case "fcu":
		o, err := fundingcredit.FromRaw(raw)
		if err != nil {
			return err
		}
		fcu := fundingcredit.Update(*o)
		return &fcu
	case "fcc":
		o, err := fundingcredit.FromRaw(raw)
		if err != nil {
			return err
		}
		fcc := fundingcredit.Cancel(*o)
		return &fcc
	case "fls":
		o, err := fundingloan.SnapshotFromRaw(raw)
		if err != nil {
			return err
		}
		return o
	case "fln":
		o, err := fundingloan.FromRaw(raw)
		if err != nil {
			return err
		}
		fln := fundingloan.New(*o)
		return &fln
	case "flu":
		o, err := fundingloan.FromRaw(raw)
		if err != nil {
			return err
		}
		flu := fundingloan.Update(*o)
		return &flu
	case "flc":
		o, err := fundingloan.FromRaw(raw)
		if err != nil {
			return err
		}
		flc := fundingloan.Cancel(*o)
		return &flc
	//case "uac":
	case "hb":
		return &Heartbeat{}
	case "ats":
		// TODO: Is not in documentation, so figure out what it is.
		return nil
	case "oc-req":
		// TODO
		return nil
	case "on-req":
		// TODO
		return nil
	case "mis": // Should not be sent anymore as of 2017-04-01
		return nil
	case "miu":
		o, err := bitfinex.NewMarginInfoFromRaw(raw)
		if err != nil {
			return err
		}
		// return a strongly typed reference, rather than dereference a generic interface
		// too bad golang doesn't inherit an interface's underlying type when creating a reference to the interface
		if base, ok := o.(*bitfinex.MarginInfoBase); ok {
			return base
		}
		if update, ok := o.(*bitfinex.MarginInfoUpdate); ok {
			return update
		}
		return o // better than nothing
	default:
		c.log.Warningf("unhandled channel data, term: %s", term)
	}

	return fmt.Errorf("term %q not recognized", term)
}
<|endoftext|>


<commit_before>package random

import (
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceShuffle() *schema.Resource {
	return &schema.Resource{
		Create: CreateShuffle,
		Read:   schema.Noop,
		Delete: schema.RemoveFromState,

		Schema: map[string]*schema.Schema{
			"keepers": {
				Type:     schema.TypeMap,
				Optional: true,
				ForceNew: true,
			},

			"seed": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},

			"input": {
				Type:     schema.TypeList,
				Required: true,
				ForceNew: true,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
			},

			"result": {
				Type:     schema.TypeList,
				Computed: true,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
			},

			"result_count": {
				Type:     schema.TypeInt,
				Optional: true,
				ForceNew: true,
			},
		},
	}
}

func CreateShuffle(d *schema.ResourceData, _ interface{}) error {
	input := d.Get("input").([]interface{})
	seed := d.Get("seed").(string)

	resultCount := d.Get("result_count").(int)
	if resultCount == 0 {
		resultCount = len(input)
	}
	result := make([]interface{}, 0, resultCount)

	rand := NewRand(seed)

	// Keep producing permutations until we fill our result
Batches:
	for {
		perm := rand.Perm(len(input))

		for _, i := range perm {
			result = append(result, input[i])

			if len(result) >= resultCount {
				break Batches
			}
		}
	}

	d.SetId("-")
	d.Set("result", result)

	return nil
}
<commit_msg>Check if random_shuffle is given an empty list<commit_after>package random

import (
	"errors"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceShuffle() *schema.Resource {
	return &schema.Resource{
		Create: CreateShuffle,
		Read:   schema.Noop,
		Delete: schema.RemoveFromState,

		Schema: map[string]*schema.Schema{
			"keepers": {
				Type:     schema.TypeMap,
				Optional: true,
				ForceNew: true,
			},

			"seed": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},

			"input": {
				Type:     schema.TypeList,
				Required: true,
				ForceNew: true,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
			},

			"result": {
				Type:     schema.TypeList,
				Computed: true,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
			},

			"result_count": {
				Type:     schema.TypeInt,
				Optional: true,
				ForceNew: true,
			},
		},
	}
}

func CreateShuffle(d *schema.ResourceData, _ interface{}) error {
	input := d.Get("input").([]interface{})
	seed := d.Get("seed").(string)

	if len(input) < 1 {
		return errors.New("input can't be empty")
	}

	resultCount := d.Get("result_count").(int)
	if resultCount == 0 {
		resultCount = len(input)
	}
	result := make([]interface{}, 0, resultCount)

	rand := NewRand(seed)

	// Keep producing permutations until we fill our result
Batches:
	for {
		perm := rand.Perm(len(input))

		for _, i := range perm {
			result = append(result, input[i])

			if len(result) >= resultCount {
				break Batches
			}
		}
	}

	d.SetId("-")
	d.Set("result", result)

	return nil
}
<|endoftext|>