<commit_before>package broker

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net/url"
	"time"

	"code.cloudfoundry.org/lager"
	"github.com/cloudfoundry-community/go-cfclient"
	"github.com/pivotal-cf/brokerapi"

	"github.com/cloudfoundry-community/s3-broker/awsiam"
	"github.com/cloudfoundry-community/s3-broker/awss3"
)

const instanceIDLogKey = "instance-id"
const bindingIDLogKey = "binding-id"
const detailsLogKey = "details"
const acceptsIncompleteLogKey = "acceptsIncomplete"

var (
	ErrNoClientConfigured = errors.New("This broker is not configured to support binding to additional instances. Contact your Cloud Foundry operator for details.")
)

type S3Broker struct {
	iamPath                      string
	userPrefix                   string
	policyPrefix                 string
	bucketPrefix                 string
	awsPartition                 string
	allowUserProvisionParameters bool
	allowUserUpdateParameters    bool
	allowUserBindParameters      bool
	catalog                      Catalog
	bucket                       awss3.Bucket
	user                         awsiam.User
	cfClient                     *cfclient.Client
	logger                       lager.Logger
}

type CatalogExternal struct {
	Services []brokerapi.Service `json:"services"`
}

type Credentials struct {
	AccessKeyID       string   `json:"access_key_id"`
	SecretAccessKey   string   `json:"secret_access_key"`
	Region            string   `json:"region"`
	Bucket            string   `json:"bucket"`
	AdditionalBuckets []string `json:"additional_buckets"`
}

func New(
	config Config,
	bucket awss3.Bucket,
	user awsiam.User,
	cfClient *cfclient.Client,
	logger lager.Logger,
) *S3Broker {
	return &S3Broker{
		iamPath:                      config.IamPath,
		userPrefix:                   config.UserPrefix,
		policyPrefix:                 config.PolicyPrefix,
		bucketPrefix:                 config.BucketPrefix,
		awsPartition:                 config.AwsPartition,
		allowUserProvisionParameters: config.AllowUserProvisionParameters,
		allowUserUpdateParameters:    config.AllowUserUpdateParameters,
		catalog:                      config.Catalog,
		bucket:                       bucket,
		user:                         user,
		cfClient:                     cfClient,
		logger:                       logger.Session("broker"),
	}
}

func (b *S3Broker) Services(context context.Context) []brokerapi.Service {
	brokerCatalog, err := json.Marshal(b.catalog)
	if err != nil {
		b.logger.Error("marshal-error", err)
		return []brokerapi.Service{}
	}

	apiCatalog := CatalogExternal{}
	if err = json.Unmarshal(brokerCatalog, &apiCatalog); err != nil {
		b.logger.Error("unmarshal-error", err)
		return []brokerapi.Service{}
	}

	return apiCatalog.Services
}

func (b *S3Broker) Provision(
	context context.Context,
	instanceID string,
	details brokerapi.ProvisionDetails,
	asyncAllowed bool,
) (brokerapi.ProvisionedServiceSpec, error) {
	b.logger.Debug("provision", lager.Data{
		instanceIDLogKey:        instanceID,
		detailsLogKey:           details,
		acceptsIncompleteLogKey: asyncAllowed,
	})

	provisionParameters := ProvisionParameters{}
	if b.allowUserProvisionParameters && len(details.RawParameters) > 0 {
		if err := json.Unmarshal(details.RawParameters, &provisionParameters); err != nil {
			return brokerapi.ProvisionedServiceSpec{}, err
		}
	}

	servicePlan, ok := b.catalog.FindServicePlan(details.PlanID)
	if !ok {
		return brokerapi.ProvisionedServiceSpec{}, fmt.Errorf("Service Plan '%s' not found", details.PlanID)
	}

	var err error
	instance := b.createBucket(instanceID, servicePlan, provisionParameters, details)
	if _, err = b.bucket.Create(b.bucketName(instanceID), *instance); err != nil {
		return brokerapi.ProvisionedServiceSpec{}, err
	}

	return brokerapi.ProvisionedServiceSpec{IsAsync: false}, nil
}

func (b *S3Broker) Update(
	context context.Context,
	instanceID string,
	details brokerapi.UpdateDetails,
	asyncAllowed bool,
) (brokerapi.UpdateServiceSpec, error) {
	b.logger.Debug("update", lager.Data{
		instanceIDLogKey:        instanceID,
		detailsLogKey:           details,
		acceptsIncompleteLogKey: asyncAllowed,
	})

	updateParameters := UpdateParameters{}
	if b.allowUserUpdateParameters && len(details.RawParameters) > 0 {
		if err := json.Unmarshal(details.RawParameters, &updateParameters); err != nil {
			return brokerapi.UpdateServiceSpec{}, err
		}
	}

	servicePlan, ok := b.catalog.FindServicePlan(details.PlanID)
	if !ok {
		return brokerapi.UpdateServiceSpec{}, fmt.Errorf("Service Plan '%s' not found", details.PlanID)
	}

	instance := b.modifyBucket(instanceID, servicePlan, updateParameters, details)
	if err := b.bucket.Modify(b.bucketName(instanceID), *instance); err != nil {
		if err == awss3.ErrBucketDoesNotExist {
			return brokerapi.UpdateServiceSpec{}, brokerapi.ErrInstanceDoesNotExist
		}
		return brokerapi.UpdateServiceSpec{}, err
	}

	return brokerapi.UpdateServiceSpec{IsAsync: false}, nil
}

func (b *S3Broker) Deprovision(
	context context.Context,
	instanceID string,
	details brokerapi.DeprovisionDetails,
	asyncAllowed bool,
) (brokerapi.DeprovisionServiceSpec, error) {
	b.logger.Debug("deprovision", lager.Data{
		instanceIDLogKey:        instanceID,
		detailsLogKey:           details,
		acceptsIncompleteLogKey: asyncAllowed,
	})

	if err := b.bucket.Delete(b.bucketName(instanceID)); err != nil {
		if err == awss3.ErrBucketDoesNotExist {
			return brokerapi.DeprovisionServiceSpec{}, brokerapi.ErrInstanceDoesNotExist
		}
		return brokerapi.DeprovisionServiceSpec{}, err
	}

	return brokerapi.DeprovisionServiceSpec{IsAsync: false}, nil
}

func (b *S3Broker) getBucketNames(instanceNames []string, instanceGUID, serviceGUID string) ([]string, error) {
	var bucketNames []string

	instance, err := b.cfClient.ServiceInstanceByGuid(instanceGUID)
	if err != nil {
		return nil, err
	}

	query := url.Values{}
	query.Set("space_guid", instance.SpaceGuid)
	query.Set("plan_guid", serviceGUID)
	instances, err := b.cfClient.ListServiceInstancesByQuery(query)
	if err != nil {
		return nil, err
	}

	instanceGUIDs := make(map[string]string, len(instanceNames))
	for _, instance := range instances {
		instanceGUIDs[instance.Name] = instance.Guid
	}

	for _, instanceName := range instanceNames {
		instanceGUID, ok := instanceGUIDs[instanceName]
		if !ok {
			return nil, fmt.Errorf("Service instance %s not found", instanceName)
		}
		bucketNames = append(bucketNames, b.bucketName(instanceGUID))
	}

	return bucketNames, nil
}

func (b *S3Broker) Bind(
	context context.Context,
	instanceID, bindingID string,
	details brokerapi.BindDetails,
) (brokerapi.Binding, error) {
	b.logger.Debug("bind", lager.Data{
		instanceIDLogKey: instanceID,
		bindingIDLogKey:  bindingID,
		detailsLogKey:    details,
	})

	binding := brokerapi.Binding{}

	var accessKeyID, secretAccessKey string
	var policyARN string
	var err error

	servicePlan, ok := b.catalog.FindServicePlan(details.PlanID)
	if !ok {
		return binding, fmt.Errorf("Service Plan '%s' not found", details.PlanID)
	}

	bindParameters := BindParameters{}
	if len(details.RawParameters) > 0 {
		if err := json.Unmarshal(details.RawParameters, &bindParameters); err != nil {
			return binding, err
		}
	}

	bucketNames := []string{b.bucketName(instanceID)}
	if len(bindParameters.AdditionalInstances) > 0 {
		if b.cfClient == nil {
			return binding, ErrNoClientConfigured
		}
		additionalNames, err := b.getBucketNames(bindParameters.AdditionalInstances, instanceID, details.ServiceID)
		if err != nil {
			return binding, err
		}
		bucketNames = append(bucketNames, additionalNames...)
	}

	credentials := Credentials{AdditionalBuckets: []string{}}
	bucketARNs := make([]string, len(bucketNames))
	detailc, errc := make(chan awss3.BucketDetails), make(chan error)
	for _, bucketName := range bucketNames {
		go func(bucketName string) {
			bucketDetails, err := b.bucket.Describe(bucketName, b.awsPartition)
			if err != nil {
				if err == awss3.ErrBucketDoesNotExist {
					errc <- brokerapi.ErrInstanceDoesNotExist
				}
				errc <- err
			} else {
				detailc <- bucketDetails
			}
		}(bucketName)
	}
	for idx, _ := range bucketNames {
		select {
		case bucketDetails := <-detailc:
			bucketARNs[idx] = bucketDetails.ARN
			if bucketDetails.BucketName == b.bucketName(instanceID) {
				credentials.Bucket = bucketDetails.BucketName
				credentials.Region = bucketDetails.Region
			} else {
				credentials.AdditionalBuckets = append(credentials.AdditionalBuckets, bucketDetails.BucketName)
			}
		case err := <-errc:
			return binding, err
		}
	}

	if _, err = b.user.Create(b.userName(bindingID), b.iamPath); err != nil {
		return binding, err
	}
	defer func() {
		if err != nil {
			if policyARN != "" {
				b.user.DeletePolicy(policyARN)
			}
			if accessKeyID != "" {
				b.user.DeleteAccessKey(b.userName(bindingID), accessKeyID)
			}
			b.user.Delete(b.userName(bindingID))
		}
	}()

	accessKeyID, secretAccessKey, err = b.user.CreateAccessKey(b.userName(bindingID))
	if err != nil {
		return binding, err
	}

	policyARN, err = b.user.CreatePolicy(b.policyName(bindingID), b.iamPath, string(servicePlan.S3Properties.IamPolicy), bucketARNs)
	if err != nil {
		return binding, err
	}

	if err = b.user.AttachUserPolicy(b.userName(bindingID), policyARN); err != nil {
		return binding, err
	}

	credentials.AccessKeyID = accessKeyID
	credentials.SecretAccessKey = secretAccessKey
	binding.Credentials = credentials

	return binding, nil
}

func (b *S3Broker) Unbind(
	context context.Context,
	instanceID, bindingID string,
	details brokerapi.UnbindDetails,
) error {
	b.logger.Debug("unbind", lager.Data{
		instanceIDLogKey: instanceID,
		bindingIDLogKey:  bindingID,
		detailsLogKey:    details,
	})

	accessKeys, err := b.user.ListAccessKeys(b.userName(bindingID))
	if err != nil {
		return err
	}

	for _, accessKey := range accessKeys {
		if err := b.user.DeleteAccessKey(b.userName(bindingID), accessKey); err != nil {
			return err
		}
	}

	userPolicies, err := b.user.ListAttachedUserPolicies(b.userName(bindingID), b.iamPath)
	if err != nil {
		return err
	}

	for _, userPolicy := range userPolicies {
		if err := b.user.DetachUserPolicy(b.userName(bindingID), userPolicy); err != nil {
			return err
		}

		if err := b.user.DeletePolicy(userPolicy); err != nil {
			return err
		}
	}

	if err := b.user.Delete(b.userName(bindingID)); err != nil {
		return err
	}

	return nil
}

func (b *S3Broker) LastOperation(
	context context.Context,
	instanceID, operationData string,
) (brokerapi.LastOperation, error) {
	b.logger.Debug("last-operation", lager.Data{
		instanceIDLogKey: instanceID,
	})

	return brokerapi.LastOperation{}, errors.New("This broker does not support LastOperation")
}

func (b *S3Broker) bucketName(instanceID string) string {
	return fmt.Sprintf("%s-%s", b.bucketPrefix, instanceID)
}

func (b *S3Broker) userName(bindingID string) string {
	return fmt.Sprintf("%s-%s", b.userPrefix, bindingID)
}

func (b *S3Broker) policyName(bindingID string) string {
	return fmt.Sprintf("%s-%s", b.policyPrefix, bindingID)
}

func (b *S3Broker) createBucket(instanceID string, servicePlan ServicePlan, provisionParameters ProvisionParameters, details brokerapi.ProvisionDetails) *awss3.BucketDetails {
	bucketDetails := b.bucketFromPlan(servicePlan)
	bucketDetails.Tags = b.bucketTags("Created", details.ServiceID, details.PlanID, details.OrganizationGUID, details.SpaceGUID)
	bucketDetails.Policy = string(servicePlan.S3Properties.BucketPolicy)
	bucketDetails.AwsPartition = b.awsPartition
	return bucketDetails
}

func (b *S3Broker) modifyBucket(instanceID string, servicePlan ServicePlan, updateParameters UpdateParameters, details brokerapi.UpdateDetails) *awss3.BucketDetails {
	bucketDetails := b.bucketFromPlan(servicePlan)
	bucketDetails.Tags = b.bucketTags("Updated", details.ServiceID, details.PlanID, "", "")
	return bucketDetails
}

func (b *S3Broker) bucketFromPlan(servicePlan ServicePlan) *awss3.BucketDetails {
	bucketDetails := &awss3.BucketDetails{}
	return bucketDetails
}

func (b *S3Broker) bucketTags(action, serviceID, planID, organizationID, spaceID string) map[string]string {
	tags := make(map[string]string)

	tags["Owner"] = "Cloud Foundry"

	tags[action+" by"] = "AWS S3 Service Broker"

	tags[action+" at"] = time.Now().Format(time.RFC822Z)

	if serviceID != "" {
		tags["Service ID"] = serviceID
	}

	if planID != "" {
		tags["Plan ID"] = planID
	}

	if organizationID != "" {
		tags["Organization ID"] = organizationID
	}

	if spaceID != "" {
		tags["Space ID"] = spaceID
	}
	return tags
}
<commit_msg>Include a URI with the S3 bucket credentials<commit_after>package broker

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net/url"
	"time"

	"code.cloudfoundry.org/lager"
	"github.com/cloudfoundry-community/go-cfclient"
	"github.com/pivotal-cf/brokerapi"

	"github.com/cloudfoundry-community/s3-broker/awsiam"
	"github.com/cloudfoundry-community/s3-broker/awss3"
)

const instanceIDLogKey = "instance-id"
const bindingIDLogKey = "binding-id"
const detailsLogKey = "details"
const acceptsIncompleteLogKey = "acceptsIncomplete"

var (
	ErrNoClientConfigured = errors.New("This broker is not configured to support binding to additional instances. Contact your Cloud Foundry operator for details.")
)

type S3Broker struct {
	iamPath                      string
	userPrefix                   string
	policyPrefix                 string
	bucketPrefix                 string
	awsPartition                 string
	allowUserProvisionParameters bool
	allowUserUpdateParameters    bool
	allowUserBindParameters      bool
	catalog                      Catalog
	bucket                       awss3.Bucket
	user                         awsiam.User
	cfClient                     *cfclient.Client
	logger                       lager.Logger
}

type CatalogExternal struct {
	Services []brokerapi.Service `json:"services"`
}

type Credentials struct {
	URI               string   `json:"URI"`
	AccessKeyID       string   `json:"access_key_id"`
	SecretAccessKey   string   `json:"secret_access_key"`
	Region            string   `json:"region"`
	Bucket            string   `json:"bucket"`
	AdditionalBuckets []string `json:"additional_buckets"`
}

func New(
	config Config,
	bucket awss3.Bucket,
	user awsiam.User,
	cfClient *cfclient.Client,
	logger lager.Logger,
) *S3Broker {
	return &S3Broker{
		iamPath:                      config.IamPath,
		userPrefix:                   config.UserPrefix,
		policyPrefix:                 config.PolicyPrefix,
		bucketPrefix:                 config.BucketPrefix,
		awsPartition:                 config.AwsPartition,
		allowUserProvisionParameters: config.AllowUserProvisionParameters,
		allowUserUpdateParameters:    config.AllowUserUpdateParameters,
		catalog:                      config.Catalog,
		bucket:                       bucket,
		user:                         user,
		cfClient:                     cfClient,
		logger:                       logger.Session("broker"),
	}
}

func (b *S3Broker) Services(context context.Context) []brokerapi.Service {
	brokerCatalog, err := json.Marshal(b.catalog)
	if err != nil {
		b.logger.Error("marshal-error", err)
		return []brokerapi.Service{}
	}

	apiCatalog := CatalogExternal{}
	if err = json.Unmarshal(brokerCatalog, &apiCatalog); err != nil {
		b.logger.Error("unmarshal-error", err)
		return []brokerapi.Service{}
	}

	return apiCatalog.Services
}

func (b *S3Broker) Provision(
	context context.Context,
	instanceID string,
	details brokerapi.ProvisionDetails,
	asyncAllowed bool,
) (brokerapi.ProvisionedServiceSpec, error) {
	b.logger.Debug("provision", lager.Data{
		instanceIDLogKey:        instanceID,
		detailsLogKey:           details,
		acceptsIncompleteLogKey: asyncAllowed,
	})

	provisionParameters := ProvisionParameters{}
	if b.allowUserProvisionParameters && len(details.RawParameters) > 0 {
		if err := json.Unmarshal(details.RawParameters, &provisionParameters); err != nil {
			return brokerapi.ProvisionedServiceSpec{}, err
		}
	}

	servicePlan, ok := b.catalog.FindServicePlan(details.PlanID)
	if !ok {
		return brokerapi.ProvisionedServiceSpec{}, fmt.Errorf("Service Plan '%s' not found", details.PlanID)
	}

	var err error
	instance := b.createBucket(instanceID, servicePlan, provisionParameters, details)
	if _, err = b.bucket.Create(b.bucketName(instanceID), *instance); err != nil {
		return brokerapi.ProvisionedServiceSpec{}, err
	}

	return brokerapi.ProvisionedServiceSpec{IsAsync: false}, nil
}

func (b *S3Broker) Update(
	context context.Context,
	instanceID string,
	details brokerapi.UpdateDetails,
	asyncAllowed bool,
) (brokerapi.UpdateServiceSpec, error) {
	b.logger.Debug("update", lager.Data{
		instanceIDLogKey:        instanceID,
		detailsLogKey:           details,
		acceptsIncompleteLogKey: asyncAllowed,
	})

	updateParameters := UpdateParameters{}
	if b.allowUserUpdateParameters && len(details.RawParameters) > 0 {
		if err := json.Unmarshal(details.RawParameters, &updateParameters); err != nil {
			return brokerapi.UpdateServiceSpec{}, err
		}
	}

	servicePlan, ok := b.catalog.FindServicePlan(details.PlanID)
	if !ok {
		return brokerapi.UpdateServiceSpec{}, fmt.Errorf("Service Plan '%s' not found", details.PlanID)
	}

	instance := b.modifyBucket(instanceID, servicePlan, updateParameters, details)
	if err := b.bucket.Modify(b.bucketName(instanceID), *instance); err != nil {
		if err == awss3.ErrBucketDoesNotExist {
			return brokerapi.UpdateServiceSpec{}, brokerapi.ErrInstanceDoesNotExist
		}
		return brokerapi.UpdateServiceSpec{}, err
	}

	return brokerapi.UpdateServiceSpec{IsAsync: false}, nil
}

func (b *S3Broker) Deprovision(
	context context.Context,
	instanceID string,
	details brokerapi.DeprovisionDetails,
	asyncAllowed bool,
) (brokerapi.DeprovisionServiceSpec, error) {
	b.logger.Debug("deprovision", lager.Data{
		instanceIDLogKey:        instanceID,
		detailsLogKey:           details,
		acceptsIncompleteLogKey: asyncAllowed,
	})

	if err := b.bucket.Delete(b.bucketName(instanceID)); err != nil {
		if err == awss3.ErrBucketDoesNotExist {
			return brokerapi.DeprovisionServiceSpec{}, brokerapi.ErrInstanceDoesNotExist
		}
		return brokerapi.DeprovisionServiceSpec{}, err
	}

	return brokerapi.DeprovisionServiceSpec{IsAsync: false}, nil
}

func (b *S3Broker) getBucketURI(credentials Credentials) string {
	var endpoint string
	if credentials.Region == "us-east-1" {
		endpoint = "s3.amazonaws.com"
	} else {
		endpoint = "s3-" + credentials.Region + ".amazonaws.com"
	}
	return fmt.Sprintf("s3://%s:%s@%s/%s", credentials.AccessKeyID, credentials.SecretAccessKey, endpoint, credentials.Bucket)
}

func (b *S3Broker) getBucketNames(instanceNames []string, instanceGUID, serviceGUID string) ([]string, error) {
	var bucketNames []string

	instance, err := b.cfClient.ServiceInstanceByGuid(instanceGUID)
	if err != nil {
		return nil, err
	}

	query := url.Values{}
	query.Set("space_guid", instance.SpaceGuid)
	query.Set("plan_guid", serviceGUID)
	instances, err := b.cfClient.ListServiceInstancesByQuery(query)
	if err != nil {
		return nil, err
	}

	instanceGUIDs := make(map[string]string, len(instanceNames))
	for _, instance := range instances {
		instanceGUIDs[instance.Name] = instance.Guid
	}

	for _, instanceName := range instanceNames {
		instanceGUID, ok := instanceGUIDs[instanceName]
		if !ok {
			return nil, fmt.Errorf("Service instance %s not found", instanceName)
		}
		bucketNames = append(bucketNames, b.bucketName(instanceGUID))
	}

	return bucketNames, nil
}

func (b *S3Broker) Bind(
	context context.Context,
	instanceID, bindingID string,
	details brokerapi.BindDetails,
) (brokerapi.Binding, error) {
	b.logger.Debug("bind", lager.Data{
		instanceIDLogKey: instanceID,
		bindingIDLogKey:  bindingID,
		detailsLogKey:    details,
	})

	binding := brokerapi.Binding{}

	var accessKeyID, secretAccessKey string
	var policyARN string
	var err error

	servicePlan, ok := b.catalog.FindServicePlan(details.PlanID)
	if !ok {
		return binding, fmt.Errorf("Service Plan '%s' not found", details.PlanID)
	}

	bindParameters := BindParameters{}
	if len(details.RawParameters) > 0 {
		if err := json.Unmarshal(details.RawParameters, &bindParameters); err != nil {
			return binding, err
		}
	}

	bucketNames := []string{b.bucketName(instanceID)}
	if len(bindParameters.AdditionalInstances) > 0 {
		if b.cfClient == nil {
			return binding, ErrNoClientConfigured
		}
		additionalNames, err := b.getBucketNames(bindParameters.AdditionalInstances, instanceID, details.ServiceID)
		if err != nil {
			return binding, err
		}
		bucketNames = append(bucketNames, additionalNames...)
	}

	credentials := Credentials{AdditionalBuckets: []string{}}
	bucketARNs := make([]string, len(bucketNames))
	detailc, errc := make(chan awss3.BucketDetails), make(chan error)
	for _, bucketName := range bucketNames {
		go func(bucketName string) {
			bucketDetails, err := b.bucket.Describe(bucketName, b.awsPartition)
			if err != nil {
				if err == awss3.ErrBucketDoesNotExist {
					errc <- brokerapi.ErrInstanceDoesNotExist
				}
				errc <- err
			} else {
				detailc <- bucketDetails
			}
		}(bucketName)
	}
	for idx, _ := range bucketNames {
		select {
		case bucketDetails := <-detailc:
			bucketARNs[idx] = bucketDetails.ARN
			if bucketDetails.BucketName == b.bucketName(instanceID) {
				credentials.Bucket = bucketDetails.BucketName
				credentials.Region = bucketDetails.Region
			} else {
				credentials.AdditionalBuckets = append(credentials.AdditionalBuckets, bucketDetails.BucketName)
			}
		case err := <-errc:
			return binding, err
		}
	}

	if _, err = b.user.Create(b.userName(bindingID), b.iamPath); err != nil {
		return binding, err
	}
	defer func() {
		if err != nil {
			if policyARN != "" {
				b.user.DeletePolicy(policyARN)
			}
			if accessKeyID != "" {
				b.user.DeleteAccessKey(b.userName(bindingID), accessKeyID)
			}
			b.user.Delete(b.userName(bindingID))
		}
	}()

	accessKeyID, secretAccessKey, err = b.user.CreateAccessKey(b.userName(bindingID))
	if err != nil {
		return binding, err
	}

	policyARN, err = b.user.CreatePolicy(b.policyName(bindingID), b.iamPath, string(servicePlan.S3Properties.IamPolicy), bucketARNs)
	if err != nil {
		return binding, err
	}

	if err = b.user.AttachUserPolicy(b.userName(bindingID), policyARN); err != nil {
		return binding, err
	}

	credentials.URI = b.getBucketURI(credentials)
	credentials.AccessKeyID = accessKeyID
	credentials.SecretAccessKey = secretAccessKey
	binding.Credentials = credentials

	return binding, nil
}

func (b *S3Broker) Unbind(
	context context.Context,
	instanceID, bindingID string,
	details brokerapi.UnbindDetails,
) error {
	b.logger.Debug("unbind", lager.Data{
		instanceIDLogKey: instanceID,
		bindingIDLogKey:  bindingID,
		detailsLogKey:    details,
	})

	accessKeys, err := b.user.ListAccessKeys(b.userName(bindingID))
	if err != nil {
		return err
	}

	for _, accessKey := range accessKeys {
		if err := b.user.DeleteAccessKey(b.userName(bindingID), accessKey); err != nil {
			return err
		}
	}

	userPolicies, err := b.user.ListAttachedUserPolicies(b.userName(bindingID), b.iamPath)
	if err != nil {
		return err
	}

	for _, userPolicy := range userPolicies {
		if err := b.user.DetachUserPolicy(b.userName(bindingID), userPolicy); err != nil {
			return err
		}

		if err := b.user.DeletePolicy(userPolicy); err != nil {
			return err
		}
	}

	if err := b.user.Delete(b.userName(bindingID)); err != nil {
		return err
	}

	return nil
}

func (b *S3Broker) LastOperation(
	context context.Context,
	instanceID, operationData string,
) (brokerapi.LastOperation, error) {
	b.logger.Debug("last-operation", lager.Data{
		instanceIDLogKey: instanceID,
	})

	return brokerapi.LastOperation{}, errors.New("This broker does not support LastOperation")
}

func (b *S3Broker) bucketName(instanceID string) string {
	return fmt.Sprintf("%s-%s", b.bucketPrefix, instanceID)
}

func (b *S3Broker) userName(bindingID string) string {
	return fmt.Sprintf("%s-%s", b.userPrefix, bindingID)
}

func (b *S3Broker) policyName(bindingID string) string {
	return fmt.Sprintf("%s-%s", b.policyPrefix, bindingID)
}

func (b *S3Broker) createBucket(instanceID string, servicePlan ServicePlan, provisionParameters ProvisionParameters, details brokerapi.ProvisionDetails) *awss3.BucketDetails {
	bucketDetails := b.bucketFromPlan(servicePlan)
	bucketDetails.Tags = b.bucketTags("Created", details.ServiceID, details.PlanID, details.OrganizationGUID, details.SpaceGUID)
	bucketDetails.Policy = string(servicePlan.S3Properties.BucketPolicy)
	bucketDetails.AwsPartition = b.awsPartition
	return bucketDetails
}

func (b *S3Broker) modifyBucket(instanceID string, servicePlan ServicePlan, updateParameters UpdateParameters, details brokerapi.UpdateDetails) *awss3.BucketDetails {
	bucketDetails := b.bucketFromPlan(servicePlan)
	bucketDetails.Tags = b.bucketTags("Updated", details.ServiceID, details.PlanID, "", "")
	return bucketDetails
}

func (b *S3Broker) bucketFromPlan(servicePlan ServicePlan) *awss3.BucketDetails {
	bucketDetails := &awss3.BucketDetails{}
	return bucketDetails
}

func (b *S3Broker) bucketTags(action, serviceID, planID, organizationID, spaceID string) map[string]string {
	tags := make(map[string]string)

	tags["Owner"] = "Cloud Foundry"

	tags[action+" by"] = "AWS S3 Service Broker"

	tags[action+" at"] = time.Now().Format(time.RFC822Z)

	if serviceID != "" {
		tags["Service ID"] = serviceID
	}

	if planID != "" {
		tags["Plan ID"] = planID
	}

	if organizationID != "" {
		tags["Organization ID"] = organizationID
	}

	if spaceID != "" {
		tags["Space ID"] = spaceID
	}
	return tags
}
<|endoftext|>


<commit_before>package browser

import (
	"fmt"
	"net/url"
	"strings"

	"github.com/moovweb/gokogiri"
	"github.com/moovweb/gokogiri/xml"
)

type Form struct {
	Action string
	Method string

	Inputs []*Input
}

func (f *Form) FillIn(name string, value string) error {
	for _, i := range f.Inputs {
		if i.Name == name {
			i.Value = value
			return nil
		}
	}
	return fmt.Errorf("no input with Name %q found", name)
}

type Input struct {
	Type    string
	Name    string
	Value   string
	Checked bool
}

func loadForms(baseUrl string, b []byte) ([]*Form, error) {
	u, e := url.Parse(baseUrl)
	if e != nil {
		return nil, e
	}
	doc, e := gokogiri.ParseHtml(b)
	if e != nil {
		return nil, e
	}
	formTags, e := doc.Search("//form")
	if e != nil {
		return nil, e
	}
	out := []*Form{}
	for _, node := range formTags {
		action := node.Attr("action")
		if !strings.HasPrefix(action, "http") {
			base := u.Scheme + "://" + u.Host
			if strings.HasPrefix(action, "/") {
				action = base + action
			} else {
				action = base + u.Path + action
			}
		}
		f := &Form{Method: node.Attr("method"), Action: action}
		f.Inputs, e = loadInputs(node)
		if e != nil {
			return nil, e
		}
		out = append(out, f)
	}
	return out, nil
}

func loadInputs(doc xml.Node) ([]*Input, error) {
	nodes, e := doc.Search(".//input")
	if e != nil {
		return nil, e
	}
	out := []*Input{}
	for _, n := range nodes {
		i := &Input{
			Type:    n.Attr("type"),
			Name:    n.Attr("name"),
			Value:   n.Attr("value"),
			Checked: n.Attr("checked") == "checked",
		}
		out = append(out, i)
	}
	return out, nil

}
<commit_msg>fix missing slashes in actions<commit_after>package browser

import (
	"fmt"
	"net/url"
	"strings"

	"github.com/moovweb/gokogiri"
	"github.com/moovweb/gokogiri/xml"
)

type Form struct {
	Action string
	Method string

	Inputs []*Input
}

func (f *Form) FillIn(name string, value string) error {
	for _, i := range f.Inputs {
		if i.Name == name {
			i.Value = value
			return nil
		}
	}
	return fmt.Errorf("no input with Name %q found", name)
}

type Input struct {
	Type    string
	Name    string
	Value   string
	Checked bool
}

func loadForms(baseUrl string, b []byte) ([]*Form, error) {
	u, e := url.Parse(baseUrl)
	if e != nil {
		return nil, e
	}
	doc, e := gokogiri.ParseHtml(b)
	if e != nil {
		return nil, e
	}
	formTags, e := doc.Search("//form")
	if e != nil {
		return nil, e
	}
	out := []*Form{}
	for _, node := range formTags {
		action := node.Attr("action")
		if !strings.HasPrefix(action, "http") {
			base := u.Scheme + "://" + u.Host
			if strings.HasPrefix(action, "/") {
				action = base + action
			} else {
				action = base + u.Path + "/" + strings.TrimPrefix(action, "/")
			}
		}
		f := &Form{Method: node.Attr("method"), Action: action}
		f.Inputs, e = loadInputs(node)
		if e != nil {
			return nil, e
		}
		out = append(out, f)
	}
	return out, nil
}

func loadInputs(doc xml.Node) ([]*Input, error) {
	nodes, e := doc.Search(".//input")
	if e != nil {
		return nil, e
	}
	out := []*Input{}
	for _, n := range nodes {
		i := &Input{
			Type:    n.Attr("type"),
			Name:    n.Attr("name"),
			Value:   n.Attr("value"),
			Checked: n.Attr("checked") == "checked",
		}
		out = append(out, i)
	}
	return out, nil

}
<|endoftext|>


<commit_before>package eveConsumer

import (
	"fmt"
	"log"
)

func (c *EveConsumer) contactSync() {
	rows, err := c.db.Query(
		`SELECT source, group_concat(destination)
			FROM contactSyncs GROUP BY source
		    HAVING max(nextSync) < UTC_TIMESTAMP()`)
	tx, err := c.db.Beginx()
	if err != nil {
		log.Printf("EVEConsumer: Failed starting transaction: %v", err)
		return
	}

	for rows.Next() {
		var (
			source int
			dest   string
		)

		err = rows.Scan(&source, &dest)
		//destinations := strings.Split(dest, ",")
		if err != nil {
			log.Printf("EVEConsumer: Failed Scanning Rows: %v", err)
			return
		}
		char, err := c.eve.GetCharacterInfo(source)
		if err != nil {
			log.Printf("EVEConsumer: Failed getting character info %v", err)
			continue
		}
		fmt.Printf("%+v %+v\n", source, char)

	}
	err = tx.Commit()
}
<commit_msg>update<commit_after>package eveConsumer

import "log"

func (c *EveConsumer) contactSync() {
	rows, err := c.db.Query(
		`SELECT source, group_concat(destination)
			FROM contactSyncs GROUP BY source
		    HAVING max(nextSync) < UTC_TIMESTAMP()`)
	tx, err := c.db.Beginx()
	if err != nil {
		log.Printf("EVEConsumer: Failed starting transaction: %v", err)
		return
	}

	for rows.Next() {
		var (
			source int
			dest   string
		)

		err = rows.Scan(&source, &dest)
		//destinations := strings.Split(dest, ",")
		if err != nil {
			log.Printf("EVEConsumer: Failed Scanning Rows: %v", err)
			return
		}
		char, err := c.eve.GetCharacterInfo(source)
		if err != nil {
			log.Printf("EVEConsumer: Failed getting character info %v", err)
			continue
		}
		if char.AllianceID > 0 {

		}

	}
	err = tx.Commit()
}
<|endoftext|>


<commit_before>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package components

import (
	"encoding/json"
	"github.com/thethingsnetwork/core/utils/pointer"
	. "github.com/thethingsnetwork/core/utils/testing"
	"testing"
	"time"
)

// The broker can handle an uplink packet
func TestMarshaljson(t *testing.T) {
	tests := []struct {
		Metadata  Metadata
		WantError error
		WantJSON  string
	}{
		{ // Basic attributes, uint, string and float64
			Metadata{Chan: pointer.Uint(2), Codr: pointer.String("4/6"), Freq: pointer.Float64(864.125)},
			nil,
			`{"chan":2,"codr":"4/6","freq":864.125}`,
		},

		{ // Basic attributes #2, int and bool
			Metadata{Imme: pointer.Bool(true), Rssi: pointer.Int(-54)},
			nil,
			`{"imme":true,"rssi":-54}`,
		},

		{ // Datr attr, FSK type
			Metadata{Datr: pointer.String("50000"), Modu: pointer.String("FSK")},
			nil,
			`{"modu":"FSK","datr":50000}`,
		},

		{ // Datr attr, lora modulation
			Metadata{Datr: pointer.String("SF7BW125"), Modu: pointer.String("LORA")},
			nil,
			`{"modu":"LORA","datr":"SF7BW125"}`,
		},

		{ // Time attr
			Metadata{Time: pointer.Time(time.Date(2016, 1, 6, 15, 11, 12, 142, time.UTC))},
			nil,
			`{"time":"2016-01-06T15:11:12.000000142Z"}`,
		},

		{ // Mixed
			Metadata{
				Time: pointer.Time(time.Date(2016, 1, 6, 15, 11, 12, 142, time.UTC)),
				Modu: pointer.String("FSK"),
				Datr: pointer.String("50000"),
				Size: pointer.Uint(14),
				Lsnr: pointer.Float64(5.7),
			},
			nil,
			`{"lsnr":5.7,"modu":"FSK","size":14,"datr":50000,"time":"2016-01-06T15:11:12.000000142Z"}`,
		},
	}

	for _, test := range tests {
		Desc(t, "Marshal medatadata: %v", pointer.DumpPStruct(test.Metadata, false))
		raw, err := json.Marshal(test.Metadata)

		if err != test.WantError {
			Ko(t, "Expected error to be %v but got %v", test.WantError, err)
			continue
		}

		str := string(raw)
		if str != test.WantJSON {
			Ko(t, "Marshaled data don't match expectation.\nWant: %s\nGot:  %s", test.WantJSON, str)
			continue
		}
		Ok(t)
	}
}
<commit_msg>[broker] Write test for metadata.UnmarshalJSON()<commit_after>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package components

import (
	"encoding/json"
	"github.com/thethingsnetwork/core/utils/pointer"
	. "github.com/thethingsnetwork/core/utils/testing"
	"reflect"
	"testing"
	"time"
)

var commonTests = []struct {
	Metadata  Metadata
	WantError error
	JSON      string
}{
	{ // Basic attributes, uint, string and float64
		Metadata{Chan: pointer.Uint(2), Codr: pointer.String("4/6"), Freq: pointer.Float64(864.125)},
		nil,
		`{"chan":2,"codr":"4/6","freq":864.125}`,
	},

	{ // Basic attributes #2, int and bool
		Metadata{Imme: pointer.Bool(true), Rssi: pointer.Int(-54)},
		nil,
		`{"imme":true,"rssi":-54}`,
	},

	{ // Datr attr, FSK type
		Metadata{Datr: pointer.String("50000"), Modu: pointer.String("FSK")},
		nil,
		`{"modu":"FSK","datr":50000}`,
	},

	{ // Datr attr, lora modulation
		Metadata{Datr: pointer.String("SF7BW125"), Modu: pointer.String("LORA")},
		nil,
		`{"modu":"LORA","datr":"SF7BW125"}`,
	},

	{ // Time attr
		Metadata{Time: pointer.Time(time.Date(2016, 1, 6, 15, 11, 12, 142, time.UTC))},
		nil,
		`{"time":"2016-01-06T15:11:12.000000142Z"}`,
	},

	{ // Mixed
		Metadata{
			Time: pointer.Time(time.Date(2016, 1, 6, 15, 11, 12, 142, time.UTC)),
			Modu: pointer.String("FSK"),
			Datr: pointer.String("50000"),
			Size: pointer.Uint(14),
			Lsnr: pointer.Float64(5.7),
		},
		nil,
		`{"lsnr":5.7,"modu":"FSK","size":14,"datr":50000,"time":"2016-01-06T15:11:12.000000142Z"}`,
	},
}

var unmarshalTests = []struct {
	Metadata  Metadata
	WantError error
	JSON      string
}{
	{ // Local time
		Metadata{Time: pointer.Time(time.Date(2016, 1, 6, 15, 11, 12, 0, time.UTC))},
		nil,
		`{"time":"2016-01-06 15:11:12 GMT"}`,
	},

	{ // RFC3339 time
		Metadata{Time: pointer.Time(time.Date(2016, 1, 6, 15, 11, 12, 142000000, time.UTC))},
		nil,
		`{"time":"2016-01-06T15:11:12.142000Z"}`,
	},
}

// The broker can handle an uplink packet
func TestMarshaljson(t *testing.T) {
	for _, test := range commonTests {
		Desc(t, "Marshal medatadata: %v", pointer.DumpPStruct(test.Metadata, false))
		raw, err := json.Marshal(test.Metadata)
		checkErrors(t, test.WantError, err)
		checkJSON(t, test.JSON, raw)
	}
}

func TestUnmarshalJSON(t *testing.T) {
	for _, test := range append(commonTests, unmarshalTests...) {
		Desc(t, "Unmarshal json: %s", test.JSON)
		metadata := Metadata{}
		err := json.Unmarshal([]byte(test.JSON), &metadata)
		checkErrors(t, test.WantError, err)
		checkMetadata(t, test.Metadata, metadata)
	}
}

// ----- Check utilities

// Check that errors match
func checkErrors(t *testing.T, want error, got error) {
	if got == want {
		Ok(t)
		return
	}
	Ko(t, "Expected error to be %v but got %v", want, got)
}

// Check that obtained json matches expected one
func checkJSON(t *testing.T, want string, got []byte) {
	str := string(got)
	if str == want {
		Ok(t)
		return
	}
	Ko(t, "Marshaled data don't match expectations.\nWant: %s\nGot:  %s", want, str)
	return
}

// Check that obtained metadata matches expected one
func checkMetadata(t *testing.T, want Metadata, got Metadata) {
	if reflect.DeepEqual(want, got) {
		Ok(t)
		return
	}
	Ko(t, "Unmarshaled json don't match expectations. \nWant: %s\nGot:  %s", pointer.DumpPStruct(want, false), pointer.DumpPStruct(got, false))
}
<|endoftext|>


<commit_before>// Copyright 2012 Gonéri Le Bouder. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import "notmuch"
import "log"
import "encoding/json"
import "os"
import "io"
import "fmt"
import "regexp"
import "net/mail"
import "path"

type Filter struct {
	Field   string
	Pattern string
	Re      *regexp.Regexp
	Tags    string
}

type Result struct {
	MessageID string
	Tags      string
	Die       bool
}

const NCPU = 4 // number of CPU cores

func getMaildirLoc() (string) {
    // honor NOTMUCH_CONFIG
    home := os.Getenv("NOTMUCH_CONFIG")
    if home == "" {
        home = os.Getenv("HOME")
    }

    return path.Join(home, "Maildir")
}

func saveResult(resultOut chan Result, quit chan bool) {

	//	var query *notmuch.Query
	var nmdb *notmuch.Database
	var msgIDRegexp = regexp.MustCompile("^<(.*)>$")
	var tagRegexp = regexp.MustCompile("([\\+-])(\\S+)")

	// open the database
	if db, status := notmuch.OpenDatabase(getMaildirLoc(),
		1); status == notmuch.STATUS_SUCCESS {
		nmdb = db
	} else {
		log.Fatalf("Failed to open the database: %v\n", status)
	}
	defer nmdb.Close()

	for {
		result := <-resultOut

		if result.Die {
			nmdb.Close()
			quit <- true
                        fmt.Print("")
			return
		}

		// Message-ID without the <>
		msgID := msgIDRegexp.FindStringSubmatch(result.MessageID)[1]
		filter := "id:"
		filter += msgID
		query := nmdb.CreateQuery(filter)
		msgs := query.SearchMessages()
		msg := msgs.Get()

		msg.Freeze()
		for _, v := range tagRegexp.FindAllStringSubmatch(result.Tags, -1) {
			if v[1] == "+" {
				msg.AddTag(v[2])
			} else if v[1] == "-" {
				msg.RemoveTag(v[2])
			}
		}
		msg.Thaw()

	}
}

func studyMsg(filter []Filter, filenameIn chan string, resultOut chan Result, quit chan bool) {
	for {
		filename := <-filenameIn

		if filename == "" {
			quit <- true
			return
		}
		// We can use Notmuch for this directly because Xappian will
		// fails as soon as we have 2 concurrent goroutine
		file, err := os.Open(filename) // For read access.
		if err != nil {
			log.Fatal(err)
		}
		var msg *mail.Message
		msg, err = mail.ReadMessage(file)

		if err != nil {
			log.Fatal(err)
		}

		var result Result
		result.MessageID = msg.Header.Get("Message-Id")
		for _, f := range filter {
			if f.Re.MatchString(msg.Header.Get(f.Field)) {
				result.Tags += " "
				result.Tags += f.Tags
			}

		}
		file.Close()

		resultOut <- result
	}
}

func loadFilter() (filter []Filter) {

	file, err := os.Open(fmt.Sprintf("/%s/notmuch-filter.json",getMaildirLoc())) // For read access.
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	dec := json.NewDecoder(file)
	for {
		var f Filter
		if err := dec.Decode(&f); err == io.EOF {
			break
		} else if err != nil {
			log.Fatal(err)
		}

		var err error = nil
		if f.Re, err = regexp.Compile(f.Pattern); err != nil {
			log.Printf("error: %v\n", err)
		}

		filter = append(filter, f)
	}

	return filter
}

func studyMsgs(resultOut chan Result, quit chan bool, filenames []string) {

	filter := loadFilter()

	filenameIn := make(chan string)
	for i := 0; i < NCPU+1; i++ {
		go studyMsg(filter, filenameIn, resultOut, quit)
	}
	for _, filename := range filenames {
		filenameIn <- filename
	}

	for i := 0; i < NCPU+1; i++ {
		filenameIn <- ""
	}

}

func main() {
	var query *notmuch.Query
	var nmdb *notmuch.Database

	if db, status := notmuch.OpenDatabase(getMaildirLoc(),
		notmuch.DATABASE_MODE_READ_ONLY); status == notmuch.STATUS_SUCCESS {
		nmdb = db
	} else {
		log.Fatalf("Failed to open the database: %v\n", status)
	}

	query = nmdb.CreateQuery("tag:inbox")
	if query.CountMessages() == 0 {
		fmt.Printf("Nothing to do\n")
		os.Exit(0)
	}

	println(">", query.CountMessages(), "<")
	msgs := query.SearchMessages()

	var filenames []string
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		filenames = append(filenames, msg.GetFileName())
	}
	query.Destroy()
	nmdb.Close()

	quit := make(chan bool)
	resultOut := make(chan Result)
	go saveResult(resultOut, quit)

	studyMsgs(resultOut, quit, filenames)

	var lastResult Result
	lastResult.Die = true

	resultOut <- lastResult

	for i := 0; i < NCPU+2; i++ {
		<-quit
	}

	fmt.Printf("done\n")

}
<commit_msg>expect newly imported mail to have the "new" tag<commit_after>// Copyright 2012 Gonéri Le Bouder. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import "notmuch"
import "log"
import "encoding/json"
import "os"
import "io"
import "fmt"
import "regexp"
import "net/mail"
import "path"

type Filter struct {
	Field   string
	Pattern string
	Re      *regexp.Regexp
	Tags    string
}

type Result struct {
	MessageID string
	Tags      string
	Die       bool
}

const NCPU = 4 // number of CPU cores

func getMaildirLoc() (string) {
    // honor NOTMUCH_CONFIG
    home := os.Getenv("NOTMUCH_CONFIG")
    if home == "" {
        home = os.Getenv("HOME")
    }

    return path.Join(home, "Maildir")
}

func saveResult(resultOut chan Result, quit chan bool) {

	//	var query *notmuch.Query
	var nmdb *notmuch.Database
	var msgIDRegexp = regexp.MustCompile("^<(.*)>$")
	var tagRegexp = regexp.MustCompile("([\\+-])(\\S+)")

	// open the database
	if db, status := notmuch.OpenDatabase(getMaildirLoc(),
		1); status == notmuch.STATUS_SUCCESS {
		nmdb = db
	} else {
		log.Fatalf("Failed to open the database: %v\n", status)
	}
	defer nmdb.Close()

	for {
		result := <-resultOut

		if result.Die {
			nmdb.Close()
			quit <- true
                        fmt.Print("")
			return
		}

		// Message-ID without the <>
		msgID := msgIDRegexp.FindStringSubmatch(result.MessageID)[1]
		filter := "id:"
		filter += msgID
		query := nmdb.CreateQuery(filter)
		msgs := query.SearchMessages()
		msg := msgs.Get()

		msg.Freeze()
		for _, v := range tagRegexp.FindAllStringSubmatch(result.Tags, -1) {
			if v[1] == "+" {
				msg.AddTag(v[2])
			} else if v[1] == "-" {
				msg.RemoveTag(v[2])
			}
		}
		msg.Thaw()

	}
}

func studyMsg(filter []Filter, filenameIn chan string, resultOut chan Result, quit chan bool) {
	for {
		filename := <-filenameIn

		if filename == "" {
			quit <- true
			return
		}
		// We can use Notmuch for this directly because Xappian will
		// fails as soon as we have 2 concurrent goroutine
		file, err := os.Open(filename) // For read access.
		if err != nil {
			log.Fatal(err)
		}
		var msg *mail.Message
		msg, err = mail.ReadMessage(file)

		if err != nil {
			log.Fatal(err)
		}

		var result Result
		result.MessageID = msg.Header.Get("Message-Id")
		for _, f := range filter {
			if f.Re.MatchString(msg.Header.Get(f.Field)) {
				result.Tags += " "
				result.Tags += f.Tags
			}

		}
		file.Close()

		resultOut <- result
	}
}

func loadFilter() (filter []Filter) {

	file, err := os.Open(fmt.Sprintf("/%s/notmuch-filter.json",getMaildirLoc())) // For read access.
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	dec := json.NewDecoder(file)
	for {
		var f Filter
		if err := dec.Decode(&f); err == io.EOF {
			break
		} else if err != nil {
			log.Fatal(err)
		}

		var err error = nil
		if f.Re, err = regexp.Compile(f.Pattern); err != nil {
			log.Printf("error: %v\n", err)
		}

		filter = append(filter, f)
	}

	return filter
}

func studyMsgs(resultOut chan Result, quit chan bool, filenames []string) {

	filter := loadFilter()

	filenameIn := make(chan string)
	for i := 0; i < NCPU+1; i++ {
		go studyMsg(filter, filenameIn, resultOut, quit)
	}
	for _, filename := range filenames {
		filenameIn <- filename
	}

	for i := 0; i < NCPU+1; i++ {
		filenameIn <- ""
	}

}

func main() {
	var query *notmuch.Query
	var nmdb *notmuch.Database

	if db, status := notmuch.OpenDatabase(getMaildirLoc(),
		notmuch.DATABASE_MODE_READ_ONLY); status == notmuch.STATUS_SUCCESS {
		nmdb = db
	} else {
		log.Fatalf("Failed to open the database: %v\n", status)
	}

	query = nmdb.CreateQuery("tag:new")
	if query.CountMessages() == 0 {
		fmt.Printf("Nothing to do\n")
		os.Exit(0)
	}

	println(">", query.CountMessages(), "<")
	msgs := query.SearchMessages()

	var filenames []string
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		filenames = append(filenames, msg.GetFileName())
	}
	query.Destroy()
	nmdb.Close()

	quit := make(chan bool)
	resultOut := make(chan Result)
	go saveResult(resultOut, quit)

	studyMsgs(resultOut, quit, filenames)

	var lastResult Result
	lastResult.Die = true

	resultOut <- lastResult

	for i := 0; i < NCPU+2; i++ {
		<-quit
	}

	fmt.Printf("done\n")

}
<|endoftext|>


<commit_before>package clouddriveclient

import (
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"math"
	"math/rand"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/koofr/go-httpclient"
	"github.com/koofr/go-ioutils"
)

const DefaultMaxRetries = 5

type CloudDrive struct {
	ContentClient  *httpclient.HTTPClient
	MetadataClient *httpclient.HTTPClient
	Auth           *CloudDriveAuth
	MaxRetries     int
}

func NewCloudDrive(auth *CloudDriveAuth) (d *CloudDrive, err error) {
	d = &CloudDrive{
		Auth:       auth,
		MaxRetries: DefaultMaxRetries,
	}

	endpointUrl, _ := url.Parse("https://drive.amazonaws.com/drive/v1")

	endpointClient := httpclient.New()
	endpointClient.BaseURL = endpointUrl

	endpoint := &Endpoint{}

	endpointReq := &httpclient.RequestData{
		Method:         "GET",
		Path:           "/account/endpoint",
		ExpectedStatus: []int{http.StatusOK},
		RespEncoding:   httpclient.EncodingJSON,
		RespValue:      &endpoint,
	}

	_, err = d.Request(endpointClient, endpointReq)

	if err != nil {
		return nil, err
	}

	if !endpoint.CustomerExists {
		return nil, fmt.Errorf("CloudDrive customer does not exist.")
	}

	contentUrl, _ := url.Parse(endpoint.ContentUrl)
	metadataUrl, _ := url.Parse(endpoint.MetadataUrl)

	d.ContentClient = httpclient.New()
	d.ContentClient.BaseURL = contentUrl

	d.MetadataClient = httpclient.New()
	d.MetadataClient.BaseURL = metadataUrl

	return d, nil
}

func (d *CloudDrive) Request(client *httpclient.HTTPClient, request *httpclient.RequestData) (response *http.Response, err error) {
	retries := d.MaxRetries

	canRetry := request.CanCopy()

	if !canRetry {
		retries = 1
	}

	for retry := 0; retry < retries; retry++ {
		var currentRequest *httpclient.RequestData

		if canRetry {
			_, currentRequest = request.Copy()
		} else {
			currentRequest = request
		}

		token, err := d.Auth.ValidToken()
		if err != nil {
			return nil, err
		}

		if currentRequest.Headers == nil {
			currentRequest.Headers = http.Header{}
		}

		currentRequest.Headers.Set("Authorization", "Bearer "+token)

		response, err = client.Request(currentRequest)

		doRetry := false

		if err != nil {
			if httpErr, ok := err.(httpclient.InvalidStatusError); ok {
				doRetry = httpErr.Got == 429
			}
		} else if response.StatusCode == 429 {
			doRetry = true
		}

		if doRetry {
			seconds := rand.Intn(int(math.Pow(2, float64(retry))))

			time.Sleep(time.Duration(seconds) * time.Second)

			continue
		}

		return response, err
	}

	return nil, err
}

func (d *CloudDrive) LookupRoot() (root *Node, err error) {
	params := make(url.Values)
	params.Set("filters", "isRoot:true")

	nodes := &Nodes{}

	req := &httpclient.RequestData{
		Method:         "GET",
		Path:           "/nodes",
		Params:         params,
		ExpectedStatus: []int{http.StatusOK},
		RespEncoding:   httpclient.EncodingJSON,
		RespValue:      &nodes,
	}

	_, err = d.Request(d.MetadataClient, req)

	if err != nil {
		return nil, err
	}

	if len(nodes.Nodes) == 0 {
		return nil, fmt.Errorf("Root not found.")
	}

	root = nodes.Nodes[0]

	return root, nil
}

func (d *CloudDrive) LookupNode(parentId string, name string) (node *Node, ok bool, err error) {
	nameEscaped := strings.Replace(name, "\"", "\\\\", -1)

	params := make(url.Values)
	params.Set("filters", "parents:"+parentId+" AND name:\""+nameEscaped+"\"")

	nodes := &Nodes{}

	req := &httpclient.RequestData{
		Method:         "GET",
		Path:           "/nodes",
		Params:         params,
		ExpectedStatus: []int{http.StatusOK},
		RespEncoding:   httpclient.EncodingJSON,
		RespValue:      &nodes,
	}

	_, err = d.Request(d.MetadataClient, req)

	if err != nil {
		return nil, false, err
	}

	if len(nodes.Nodes) == 0 {
		return nil, false, nil
	}

	return nodes.Nodes[0], true, nil
}

func (d *CloudDrive) NodeChildren(parentId string) (nodes []*Node, err error) {
	nextToken := ""

	nodes = []*Node{}

	for {
		params := make(url.Values)
		params.Set("filters", "parents:"+parentId)
		if nextToken != "" {
			params.Set("startToken", nextToken)
		}

		ns := &Nodes{}

		req := &httpclient.RequestData{
			Method:         "GET",
			Path:           "/nodes",
			Params:         params,
			ExpectedStatus: []int{http.StatusOK},
			RespEncoding:   httpclient.EncodingJSON,
			RespValue:      &ns,
		}

		_, err = d.Request(d.MetadataClient, req)

		if err != nil {
			return nil, err
		}

		if len(ns.Nodes) == 0 {
			break
		}

		nodes = append(nodes, ns.Nodes...)

		if ns.NextToken == "" {
			break
		}

		nextToken = ns.NextToken
	}

	return nodes, nil
}

func (d *CloudDrive) Changes(checkpoint string) (changes *Changes, err error) {
	req := &httpclient.RequestData{
		Method:         "POST",
		Path:           "/changes",
		ExpectedStatus: []int{http.StatusOK},
	}

	if checkpoint != "" {
		req.ReqEncoding = httpclient.EncodingJSON

		req.ReqValue = struct {
			Checkpoint string `json:"checkpoint"`
		}{
			Checkpoint: checkpoint,
		}
	}

	res, err := d.Request(d.MetadataClient, req)

	if err != nil {
		return nil, err
	}

	var r io.ReadCloser = res.Body

	if res.Header.Get("Content-Encoding") == "gzip" {
		r, err = gzip.NewReader(r)

		if err != nil {
			return nil, err
		}
	}

	decoder := json.NewDecoder(r)

	changes = &Changes{}

	err = decoder.Decode(changes)

	if err != nil {
		return nil, err
	}

	res.Body.Close()

	return changes, nil
}

func (d *CloudDrive) CreateFolder(parentId string, name string) (node *Node, err error) {
	create := &NodeCreate{
		Name:    name,
		Kind:    NodeKindFolder,
		Parents: []string{parentId},
	}

	node = &Node{}

	req := &httpclient.RequestData{
		Method:         "POST",
		Path:           "/nodes",
		ExpectedStatus: []int{http.StatusCreated},
		ReqEncoding:    httpclient.EncodingJSON,
		ReqValue:       create,
		RespEncoding:   httpclient.EncodingJSON,
		RespValue:      &node,
	}

	_, err = d.Request(d.MetadataClient, req)

	if err != nil {
		return nil, err
	}

	return node, nil
}

func (d *CloudDrive) DeleteNode(nodeId string) (node *Node, err error) {
	node = &Node{}

	req := &httpclient.RequestData{
		Method:         "PUT",
		Path:           "/trash/" + nodeId,
		ExpectedStatus: []int{http.StatusOK},
		RespEncoding:   httpclient.EncodingJSON,
		RespValue:      &node,
	}

	_, err = d.Request(d.MetadataClient, req)

	if err != nil {
		return nil, err
	}

	return node, nil
}

func (d *CloudDrive) RenameNode(nodeId string, newName string) (node *Node, err error) {
	rename := &NodeRename{
		Name: newName,
	}

	node = &Node{}

	req := &httpclient.RequestData{
		Method:         "PATCH",
		Path:           "/nodes/" + nodeId,
		ExpectedStatus: []int{http.StatusOK},
		ReqEncoding:    httpclient.EncodingJSON,
		ReqValue:       rename,
		RespEncoding:   httpclient.EncodingJSON,
		RespValue:      &node,
	}

	_, err = d.Request(d.MetadataClient, req)

	if err != nil {
		return nil, err
	}

	return node, nil
}

func (d *CloudDrive) MoveNode(nodeId string, fromParentId string, toParentId string) (node *Node, err error) {
	move := &NodeMove{
		FromParent: fromParentId,
		ChildId:    nodeId,
	}

	node = &Node{}

	req := &httpclient.RequestData{
		Method:         "POST",
		Path:           "/nodes/" + toParentId + "/children",
		ExpectedStatus: []int{http.StatusOK},
		ReqEncoding:    httpclient.EncodingJSON,
		ReqValue:       move,
		RespEncoding:   httpclient.EncodingJSON,
		RespValue:      &node,
	}

	_, err = d.Request(d.MetadataClient, req)

	if err != nil {
		return nil, err
	}

	return node, nil
}

func (d *CloudDrive) DownloadNode(nodeId string, span *ioutils.FileSpan) (reader io.ReadCloser, size int64, err error) {
	req := &httpclient.RequestData{
		Method:         "GET",
		Path:           "/nodes/" + nodeId + "/content",
		ExpectedStatus: []int{http.StatusOK, http.StatusPartialContent},
	}

	if span != nil {
		req.Headers = make(http.Header)
		req.Headers.Set("Range", fmt.Sprintf("bytes=%d-%d", span.Start, span.End))
	}

	if err != nil {
		return nil, 0, err
	}

	res, err := d.Request(d.ContentClient, req)

	if err != nil {
		return nil, 0, err
	}

	return res.Body, res.ContentLength, nil
}

func (d *CloudDrive) UploadNode(parentId string, name string, reader io.Reader) (node *Node, err error) {
	create := &NodeCreate{
		Name:    name,
		Kind:    NodeKindFile,
		Parents: []string{parentId},
	}

	createJson, err := json.Marshal(create)

	if err != nil {
		return nil, err
	}

	params := make(url.Values)
	params.Set("suppress", "deduplication")

	node = &Node{}

	req := &httpclient.RequestData{
		Method:         "POST",
		Path:           "/nodes",
		Params:         params,
		ExpectedStatus: []int{http.StatusCreated},
		RespEncoding:   httpclient.EncodingJSON,
		RespValue:      &node,
	}

	extra := map[string]string{
		"metadata": string(createJson),
	}

	err = req.UploadFileExtra("file", "file", reader, extra)

	if err != nil {
		return nil, err
	}

	_, err = d.Request(d.ContentClient, req)

	if err != nil {
		return nil, err
	}

	return node, nil
}

func (d *CloudDrive) OverwriteNode(nodeId string, reader io.Reader) (node *Node, err error) {
	node = &Node{}

	req := &httpclient.RequestData{
		Method:         "PUT",
		Path:           "/nodes/" + nodeId + "/content",
		ExpectedStatus: []int{http.StatusOK},
		RespEncoding:   httpclient.EncodingJSON,
		RespValue:      &node,
	}

	err = req.UploadFile("file", "file", reader)

	if err != nil {
		return nil, err
	}

	_, err = d.Request(d.ContentClient, req)

	if err != nil {
		return nil, err
	}

	return node, nil
}

func (d *CloudDrive) Quota() (quota *Quota, err error) {
	quota = &Quota{}

	req := &httpclient.RequestData{
		Method:         "GET",
		Path:           "/account/quota",
		ExpectedStatus: []int{http.StatusOK},
		RespEncoding:   httpclient.EncodingJSON,
		RespValue:      &quota,
	}

	_, err = d.Request(d.MetadataClient, req)

	if err != nil {
		return nil, err
	}

	return quota, nil
}
<commit_msg>Too many retries error<commit_after>package clouddriveclient

import (
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"math"
	"math/rand"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/koofr/go-httpclient"
	"github.com/koofr/go-ioutils"
)

const DefaultMaxRetries = 5

type CloudDrive struct {
	ContentClient  *httpclient.HTTPClient
	MetadataClient *httpclient.HTTPClient
	Auth           *CloudDriveAuth
	MaxRetries     int
}

func NewCloudDrive(auth *CloudDriveAuth) (d *CloudDrive, err error) {
	d = &CloudDrive{
		Auth:       auth,
		MaxRetries: DefaultMaxRetries,
	}

	endpointUrl, _ := url.Parse("https://drive.amazonaws.com/drive/v1")

	endpointClient := httpclient.New()
	endpointClient.BaseURL = endpointUrl

	endpoint := &Endpoint{}

	endpointReq := &httpclient.RequestData{
		Method:         "GET",
		Path:           "/account/endpoint",
		ExpectedStatus: []int{http.StatusOK},
		RespEncoding:   httpclient.EncodingJSON,
		RespValue:      &endpoint,
	}

	_, err = d.Request(endpointClient, endpointReq)

	if err != nil {
		return nil, err
	}

	if !endpoint.CustomerExists {
		return nil, fmt.Errorf("CloudDrive customer does not exist.")
	}

	contentUrl, _ := url.Parse(endpoint.ContentUrl)
	metadataUrl, _ := url.Parse(endpoint.MetadataUrl)

	d.ContentClient = httpclient.New()
	d.ContentClient.BaseURL = contentUrl

	d.MetadataClient = httpclient.New()
	d.MetadataClient.BaseURL = metadataUrl

	return d, nil
}

func (d *CloudDrive) Request(client *httpclient.HTTPClient, request *httpclient.RequestData) (response *http.Response, err error) {
	retries := d.MaxRetries

	canRetry := request.CanCopy()

	if !canRetry {
		retries = 1
	}

	for retry := 0; retry < retries; retry++ {
		var currentRequest *httpclient.RequestData

		if canRetry {
			_, currentRequest = request.Copy()
		} else {
			currentRequest = request
		}

		token, err := d.Auth.ValidToken()
		if err != nil {
			return nil, err
		}

		if currentRequest.Headers == nil {
			currentRequest.Headers = http.Header{}
		}

		currentRequest.Headers.Set("Authorization", "Bearer "+token)

		response, err = client.Request(currentRequest)

		doRetry := false

		if err != nil {
			if httpErr, ok := err.(httpclient.InvalidStatusError); ok {
				doRetry = httpErr.Got == 429
			}
		} else if response.StatusCode == 429 {
			doRetry = true
		}

		if doRetry {
			seconds := rand.Intn(int(math.Pow(2, float64(retry))))

			time.Sleep(time.Duration(seconds) * time.Second)

			continue
		}

		return response, err
	}

	return nil, fmt.Errorf("Too many retries")
}

func (d *CloudDrive) LookupRoot() (root *Node, err error) {
	params := make(url.Values)
	params.Set("filters", "isRoot:true")

	nodes := &Nodes{}

	req := &httpclient.RequestData{
		Method:         "GET",
		Path:           "/nodes",
		Params:         params,
		ExpectedStatus: []int{http.StatusOK},
		RespEncoding:   httpclient.EncodingJSON,
		RespValue:      &nodes,
	}

	_, err = d.Request(d.MetadataClient, req)

	if err != nil {
		return nil, err
	}

	if len(nodes.Nodes) == 0 {
		return nil, fmt.Errorf("Root not found.")
	}

	root = nodes.Nodes[0]

	return root, nil
}

func (d *CloudDrive) LookupNode(parentId string, name string) (node *Node, ok bool, err error) {
	nameEscaped := strings.Replace(name, "\"", "\\\\", -1)

	params := make(url.Values)
	params.Set("filters", "parents:"+parentId+" AND name:\""+nameEscaped+"\"")

	nodes := &Nodes{}

	req := &httpclient.RequestData{
		Method:         "GET",
		Path:           "/nodes",
		Params:         params,
		ExpectedStatus: []int{http.StatusOK},
		RespEncoding:   httpclient.EncodingJSON,
		RespValue:      &nodes,
	}

	_, err = d.Request(d.MetadataClient, req)

	if err != nil {
		return nil, false, err
	}

	if len(nodes.Nodes) == 0 {
		return nil, false, nil
	}

	return nodes.Nodes[0], true, nil
}

func (d *CloudDrive) NodeChildren(parentId string) (nodes []*Node, err error) {
	nextToken := ""

	nodes = []*Node{}

	for {
		params := make(url.Values)
		params.Set("filters", "parents:"+parentId)
		if nextToken != "" {
			params.Set("startToken", nextToken)
		}

		ns := &Nodes{}

		req := &httpclient.RequestData{
			Method:         "GET",
			Path:           "/nodes",
			Params:         params,
			ExpectedStatus: []int{http.StatusOK},
			RespEncoding:   httpclient.EncodingJSON,
			RespValue:      &ns,
		}

		_, err = d.Request(d.MetadataClient, req)

		if err != nil {
			return nil, err
		}

		if len(ns.Nodes) == 0 {
			break
		}

		nodes = append(nodes, ns.Nodes...)

		if ns.NextToken == "" {
			break
		}

		nextToken = ns.NextToken
	}

	return nodes, nil
}

func (d *CloudDrive) Changes(checkpoint string) (changes *Changes, err error) {
	req := &httpclient.RequestData{
		Method:         "POST",
		Path:           "/changes",
		ExpectedStatus: []int{http.StatusOK},
	}

	if checkpoint != "" {
		req.ReqEncoding = httpclient.EncodingJSON

		req.ReqValue = struct {
			Checkpoint string `json:"checkpoint"`
		}{
			Checkpoint: checkpoint,
		}
	}

	res, err := d.Request(d.MetadataClient, req)

	if err != nil {
		return nil, err
	}

	var r io.ReadCloser = res.Body

	if res.Header.Get("Content-Encoding") == "gzip" {
		r, err = gzip.NewReader(r)

		if err != nil {
			return nil, err
		}
	}

	decoder := json.NewDecoder(r)

	changes = &Changes{}

	err = decoder.Decode(changes)

	if err != nil {
		return nil, err
	}

	res.Body.Close()

	return changes, nil
}

func (d *CloudDrive) CreateFolder(parentId string, name string) (node *Node, err error) {
	create := &NodeCreate{
		Name:    name,
		Kind:    NodeKindFolder,
		Parents: []string{parentId},
	}

	node = &Node{}

	req := &httpclient.RequestData{
		Method:         "POST",
		Path:           "/nodes",
		ExpectedStatus: []int{http.StatusCreated},
		ReqEncoding:    httpclient.EncodingJSON,
		ReqValue:       create,
		RespEncoding:   httpclient.EncodingJSON,
		RespValue:      &node,
	}

	_, err = d.Request(d.MetadataClient, req)

	if err != nil {
		return nil, err
	}

	return node, nil
}

func (d *CloudDrive) DeleteNode(nodeId string) (node *Node, err error) {
	node = &Node{}

	req := &httpclient.RequestData{
		Method:         "PUT",
		Path:           "/trash/" + nodeId,
		ExpectedStatus: []int{http.StatusOK},
		RespEncoding:   httpclient.EncodingJSON,
		RespValue:      &node,
	}

	_, err = d.Request(d.MetadataClient, req)

	if err != nil {
		return nil, err
	}

	return node, nil
}

func (d *CloudDrive) RenameNode(nodeId string, newName string) (node *Node, err error) {
	rename := &NodeRename{
		Name: newName,
	}

	node = &Node{}

	req := &httpclient.RequestData{
		Method:         "PATCH",
		Path:           "/nodes/" + nodeId,
		ExpectedStatus: []int{http.StatusOK},
		ReqEncoding:    httpclient.EncodingJSON,
		ReqValue:       rename,
		RespEncoding:   httpclient.EncodingJSON,
		RespValue:      &node,
	}

	_, err = d.Request(d.MetadataClient, req)

	if err != nil {
		return nil, err
	}

	return node, nil
}

func (d *CloudDrive) MoveNode(nodeId string, fromParentId string, toParentId string) (node *Node, err error) {
	move := &NodeMove{
		FromParent: fromParentId,
		ChildId:    nodeId,
	}

	node = &Node{}

	req := &httpclient.RequestData{
		Method:         "POST",
		Path:           "/nodes/" + toParentId + "/children",
		ExpectedStatus: []int{http.StatusOK},
		ReqEncoding:    httpclient.EncodingJSON,
		ReqValue:       move,
		RespEncoding:   httpclient.EncodingJSON,
		RespValue:      &node,
	}

	_, err = d.Request(d.MetadataClient, req)

	if err != nil {
		return nil, err
	}

	return node, nil
}

func (d *CloudDrive) DownloadNode(nodeId string, span *ioutils.FileSpan) (reader io.ReadCloser, size int64, err error) {
	req := &httpclient.RequestData{
		Method:         "GET",
		Path:           "/nodes/" + nodeId + "/content",
		ExpectedStatus: []int{http.StatusOK, http.StatusPartialContent},
	}

	if span != nil {
		req.Headers = make(http.Header)
		req.Headers.Set("Range", fmt.Sprintf("bytes=%d-%d", span.Start, span.End))
	}

	if err != nil {
		return nil, 0, err
	}

	res, err := d.Request(d.ContentClient, req)

	if err != nil {
		return nil, 0, err
	}

	return res.Body, res.ContentLength, nil
}

func (d *CloudDrive) UploadNode(parentId string, name string, reader io.Reader) (node *Node, err error) {
	create := &NodeCreate{
		Name:    name,
		Kind:    NodeKindFile,
		Parents: []string{parentId},
	}

	createJson, err := json.Marshal(create)

	if err != nil {
		return nil, err
	}

	params := make(url.Values)
	params.Set("suppress", "deduplication")

	node = &Node{}

	req := &httpclient.RequestData{
		Method:         "POST",
		Path:           "/nodes",
		Params:         params,
		ExpectedStatus: []int{http.StatusCreated},
		RespEncoding:   httpclient.EncodingJSON,
		RespValue:      &node,
	}

	extra := map[string]string{
		"metadata": string(createJson),
	}

	err = req.UploadFileExtra("file", "file", reader, extra)

	if err != nil {
		return nil, err
	}

	_, err = d.Request(d.ContentClient, req)

	if err != nil {
		return nil, err
	}

	return node, nil
}

func (d *CloudDrive) OverwriteNode(nodeId string, reader io.Reader) (node *Node, err error) {
	node = &Node{}

	req := &httpclient.RequestData{
		Method:         "PUT",
		Path:           "/nodes/" + nodeId + "/content",
		ExpectedStatus: []int{http.StatusOK},
		RespEncoding:   httpclient.EncodingJSON,
		RespValue:      &node,
	}

	err = req.UploadFile("file", "file", reader)

	if err != nil {
		return nil, err
	}

	_, err = d.Request(d.ContentClient, req)

	if err != nil {
		return nil, err
	}

	return node, nil
}

func (d *CloudDrive) Quota() (quota *Quota, err error) {
	quota = &Quota{}

	req := &httpclient.RequestData{
		Method:         "GET",
		Path:           "/account/quota",
		ExpectedStatus: []int{http.StatusOK},
		RespEncoding:   httpclient.EncodingJSON,
		RespValue:      &quota,
	}

	_, err = d.Request(d.MetadataClient, req)

	if err != nil {
		return nil, err
	}

	return quota, nil
}
<|endoftext|>


<commit_before>package presilo

import (
	"fmt"
	"regexp"
	"strings"
)

/*
  Generates valid CSharp code for a given schema.

	C# code contains DataContractJsonSerializer annotations,
	which are required for proper serialization/deserialization of fields.
	Unfortunately this isn't available before .NET 4.5, so any generated code
	will need to be compiled with .NET 4.5+
*/
func GenerateCSharp(schema *ObjectSchema, module string, tabstyle string) string {

	var buffer *BufferedFormatString

	buffer = NewBufferedFormatString(tabstyle)

	generateCSharpImports(schema, buffer)
	buffer.Print("\n")
	generateCSharpNamespace(schema, buffer, module)
	buffer.Print("\n")
	generateCSharpTypeDeclaration(schema, buffer)
	buffer.Print("\n")
	generateCSharpConstructor(schema, buffer)
	buffer.Print("\n")
	generateCSharpFunctions(schema, buffer)
	buffer.AddIndentation(-1)
	buffer.Print("\n}")
	buffer.AddIndentation(-1)
	buffer.Print("\n}")

	return buffer.String()
}

func ValidateCSharpModule(module string) bool {

	pattern := "^[a-zA-Z_]+[0-9a-zA-Z_]*(\\.[a-zA-Z_]+[0-9a-zA-Z_]*)*$"
	matched, err := regexp.MatchString(pattern, module)
	return err == nil && matched
}

func generateCSharpImports(schema *ObjectSchema, buffer *BufferedFormatString) {

	buffer.Print("using System;")
	buffer.Print("\n using System.Runtime.Serialization;")

	// import regex if we need it
	if containsRegexpMatch(schema) {
		buffer.Print("\nusing System.Text.RegularExpressions;")
	}

	buffer.Print("\n")
}

func generateCSharpNamespace(schema *ObjectSchema, buffer *BufferedFormatString, module string) {

	buffer.Printf("namespace %s\n{", module)
	buffer.AddIndentation(1)
}

func generateCSharpTypeDeclaration(schema *ObjectSchema, buffer *BufferedFormatString) {

	var subschema TypeSchema
	var propertyName string

	buffer.Print("[DataContract]")
	buffer.Printf("\npublic class %s\n{", ToCamelCase(schema.Title))
	buffer.AddIndentation(1)

	for _, propertyName = range schema.GetOrderedPropertyNames() {

		subschema = schema.Properties[propertyName]

		buffer.Print("\n[DataMember]")
		buffer.Printf("\nprotected %s %s;", GenerateCSharpTypeForSchema(subschema), ToJavaCase(propertyName))
	}
}

func generateCSharpConstructor(schema *ObjectSchema, buffer *BufferedFormatString) {

	var subschema TypeSchema
	var declarations, setters []string
	var propertyName string
	var toWrite string

	buffer.Printf("\npublic %s(", ToCamelCase(schema.Title))

	for _, propertyName = range schema.RequiredProperties {

		subschema = schema.Properties[propertyName]
		propertyName = ToJavaCase(propertyName)

		toWrite = fmt.Sprintf("%s %s", GenerateCSharpTypeForSchema(subschema), propertyName)
		declarations = append(declarations, toWrite)

		toWrite = fmt.Sprintf("\nset%s(%s);", ToCamelCase(propertyName), propertyName)
		setters = append(setters, toWrite)
	}

	buffer.Print(strings.Join(declarations, ","))
	buffer.Print(")\n{")
	buffer.AddIndentation(1)

	for _, setter := range setters {
		buffer.Print(setter)
	}

	buffer.AddIndentation(-1)
	buffer.Print("\n}\n")
}

func generateCSharpFunctions(schema *ObjectSchema, buffer *BufferedFormatString) {

	var subschema TypeSchema
	var propertyName, properName, camelName, typeName string

	for _, propertyName = range schema.GetOrderedPropertyNames() {

		subschema = schema.Properties[propertyName]

		properName = ToJavaCase(propertyName)
		camelName = ToStrictCamelCase(propertyName)
		typeName = GenerateCSharpTypeForSchema(subschema)

		// getter
		buffer.Printf("\npublic %s get%s()\n{", typeName, camelName)
		buffer.AddIndentation(1)

		buffer.Printf("\nreturn this.%s;", properName)

		buffer.AddIndentation(-1)
		buffer.Print("\n}")

		// setter
		buffer.Printf("\npublic void set%s(%s value)\n{", camelName, typeName)
		buffer.AddIndentation(1)

		switch subschema.GetSchemaType() {
		case SCHEMATYPE_STRING:
			generateCSharpStringSetter(subschema.(*StringSchema), buffer)
		case SCHEMATYPE_INTEGER:
			fallthrough
		case SCHEMATYPE_NUMBER:
			generateCSharpNumericSetter(subschema.(NumericSchemaType), buffer)
		case SCHEMATYPE_OBJECT:
			generateCSharpObjectSetter(subschema.(*ObjectSchema), buffer)
		case SCHEMATYPE_ARRAY:
			generateCSharpArraySetter(subschema.(*ArraySchema), buffer)
		}

		buffer.Printf("\n%s = value;", properName)
		buffer.AddIndentation(-1)
		buffer.Print("\n}\n")
	}
}

func generateCSharpStringSetter(schema *StringSchema, buffer *BufferedFormatString) {

	if !schema.Nullable {
		generateCSharpNullCheck(buffer)
	}

	if schema.MinLength != nil {
		generateCSharpRangeCheck(*schema.MinLength, "value.Length", "was shorter than allowable minimum", "%d", false, "<", "", buffer)
	}

	if schema.MaxLength != nil {
		generateCSharpRangeCheck(*schema.MaxLength, "value.Length", "was longer than allowable maximum", "%d", false, ">", "", buffer)
	}

	if schema.MinByteLength != nil {
		generateCSharpRangeCheck(*schema.MinByteLength, "value.Length * sizeof(Char)", "had fewer bytes than allowable minimum", "%d", false, "<", "", buffer)
	}

	if schema.MaxByteLength != nil {
		generateCSharpRangeCheck(*schema.MaxByteLength, "value.Length * sizeof(Char)", "had more bytes than allowable minimum", "%d", false, ">", "", buffer)
	}

	if schema.Pattern != nil {

		buffer.Printf("\nRegex regex = new Regex(\"%s\");", sanitizeQuotedString(*schema.Pattern))
		buffer.Printf("\nif(!regex.IsMatch(value))\n{")
		buffer.AddIndentation(1)

		buffer.Printf("\nthrow new Exception(\"Value '\"+value+\"' did not match pattern '%s'\");", *schema.Pattern)

		buffer.AddIndentation(-1)
		buffer.Print("\n}")
	}
}

func generateCSharpNumericSetter(schema NumericSchemaType, buffer *BufferedFormatString) {

	if schema.HasMinimum() {
		generateCSharpRangeCheck(schema.GetMinimum(), "value", "is under the allowable minimum", schema.GetConstraintFormat(), schema.IsExclusiveMinimum(), "<=", "<", buffer)
	}

	if schema.HasMaximum() {
		generateCSharpRangeCheck(schema.GetMaximum(), "value", "is over the allowable maximum", schema.GetConstraintFormat(), schema.IsExclusiveMaximum(), ">=", ">", buffer)
	}

	if schema.HasEnum() {
		generateCSharpEnumCheck(schema, buffer, schema.GetEnum(), "", "")
	}

	if schema.HasMultiple() {

		buffer.Printf("\nif(value %% %f != 0)\n{", schema.GetMultiple())
		buffer.AddIndentation(1)

		buffer.Printf("\nthrow new Exception(\"Property '\"+value+\"' was not a multiple of %s\");", schema.GetMultiple())

		buffer.AddIndentation(-1)
		buffer.Print("\n}\n")
	}
}

func generateCSharpObjectSetter(schema *ObjectSchema, buffer *BufferedFormatString) {

	if !schema.Nullable {
		generateCSharpNullCheck(buffer)
	}
}

func generateCSharpArraySetter(schema *ArraySchema, buffer *BufferedFormatString) {

	if !schema.Nullable {
		generateCSharpNullCheck(buffer)
	}

	if schema.MinItems != nil {
		generateCSharpRangeCheck(*schema.MinItems, "value.Length", "does not have enough items", "%d", false, "<", "", buffer)
	}

	if schema.MaxItems != nil {
		generateCSharpRangeCheck(*schema.MaxItems, "value.Length", "does not have enough items", "%d", false, ">", "", buffer)
	}
}

func generateCSharpNullCheck(buffer *BufferedFormatString) {

	buffer.Printf("\nif(value == null)\n{")
	buffer.AddIndentation(1)

	buffer.Print("\nthrow new NullReferenceException(\"Cannot set property to null value\");")

	buffer.AddIndentation(-1)
	buffer.Print("\n}\n")
}

func generateCSharpRangeCheck(value interface{}, reference, message, format string, exclusive bool, comparator, exclusiveComparator string, buffer *BufferedFormatString) {

	var compareString string

	if exclusive {
		compareString = exclusiveComparator
	} else {
		compareString = comparator
	}

	buffer.Printf("\nif(%s %s "+format+")\n{", reference, compareString, value)
	buffer.AddIndentation(1)

	buffer.Printf("\nthrow new Exception(\"Property '\"+value+\"' %s.\");", message)
	buffer.AddIndentation(-1)
	buffer.Print("\n}\n")
}

/*
	Generates code which throws an error if the given [parameter]'s value is not contained in the given [validValues].
*/
func generateCSharpEnumCheck(schema TypeSchema, buffer *BufferedFormatString, enumValues []interface{}, prefix string, postfix string) {

	var typeName string
	var length int

	length = len(enumValues)

	if length <= 0 {
		return
	}

	// write array of valid values
	typeName = GenerateCSharpTypeForSchema(schema)
	buffer.Printf("%s[] validValues = new %s[]{%s%v%s", typeName, typeName, prefix, enumValues[0], postfix)

	for _, enumValue := range enumValues[1:length] {
		buffer.Printf(",%s%v%s", prefix, enumValue, postfix)
	}

	buffer.Print("};\n")

	// compare
	buffer.Print("\nbool isValid = false;")
	buffer.Print("\nfor(int i = 0; i < validValues.Length; i++)\n{")
	buffer.AddIndentation(1)

	buffer.Print("\nif(validValues[i] == value)\n{")
	buffer.AddIndentation(1)
	buffer.Print("\nisValid = true;\nbreak;")

	buffer.AddIndentation(-1)
	buffer.Print("\n}")
	buffer.AddIndentation(-1)
	buffer.Print("\n}")

	buffer.Print("\nif(!isValid)\n{")
	buffer.AddIndentation(1)
	buffer.Print("\nthrow new Exception(\"Given value '\"+value+\"' was not found in list of acceptable values\");")

	buffer.AddIndentation(-1)
	buffer.Print("\n}\n")
}

func GenerateCSharpTypeForSchema(subschema TypeSchema) string {

	switch subschema.GetSchemaType() {
	case SCHEMATYPE_NUMBER:
		return "double"
	case SCHEMATYPE_INTEGER:
		return "int"
	case SCHEMATYPE_ARRAY:
		return ToCamelCase(subschema.(*ArraySchema).Items.GetTitle()) + "[]"
	case SCHEMATYPE_OBJECT:
		return ToCamelCase(subschema.GetTitle())
	case SCHEMATYPE_STRING:
		return "string"
	case SCHEMATYPE_BOOLEAN:
		return "bool"
	}

	return "Object"
}
<commit_msg>Added datacontract original member name, and fixed a casing bug, in cs codgegen<commit_after>package presilo

import (
	"fmt"
	"regexp"
	"strings"
)

/*
  Generates valid CSharp code for a given schema.

	C# code contains DataContractJsonSerializer annotations,
	which are required for proper serialization/deserialization of fields.
	Unfortunately this isn't available before .NET 4.5, so any generated code
	will need to be compiled with .NET 4.5+
*/
func GenerateCSharp(schema *ObjectSchema, module string, tabstyle string) string {

	var buffer *BufferedFormatString

	buffer = NewBufferedFormatString(tabstyle)

	generateCSharpImports(schema, buffer)
	buffer.Print("\n")
	generateCSharpNamespace(schema, buffer, module)
	buffer.Print("\n")
	generateCSharpTypeDeclaration(schema, buffer)
	buffer.Print("\n")
	generateCSharpConstructor(schema, buffer)
	buffer.Print("\n")
	generateCSharpFunctions(schema, buffer)
	buffer.AddIndentation(-1)
	buffer.Print("\n}")
	buffer.AddIndentation(-1)
	buffer.Print("\n}")

	return buffer.String()
}

func ValidateCSharpModule(module string) bool {

	pattern := "^[a-zA-Z_]+[0-9a-zA-Z_]*(\\.[a-zA-Z_]+[0-9a-zA-Z_]*)*$"
	matched, err := regexp.MatchString(pattern, module)
	return err == nil && matched
}

func generateCSharpImports(schema *ObjectSchema, buffer *BufferedFormatString) {

	buffer.Print("using System;")
	buffer.Print("\n using System.Runtime.Serialization;")

	// import regex if we need it
	if containsRegexpMatch(schema) {
		buffer.Print("\nusing System.Text.RegularExpressions;")
	}

	buffer.Print("\n")
}

func generateCSharpNamespace(schema *ObjectSchema, buffer *BufferedFormatString, module string) {

	buffer.Printf("namespace %s\n{", module)
	buffer.AddIndentation(1)
}

func generateCSharpTypeDeclaration(schema *ObjectSchema, buffer *BufferedFormatString) {

	var subschema TypeSchema
	var propertyName string

	buffer.Print("[DataContract]")
	buffer.Printf("\npublic class %s\n{", ToCamelCase(schema.Title))
	buffer.AddIndentation(1)

	for _, propertyName = range schema.GetOrderedPropertyNames() {

		subschema = schema.Properties[propertyName]

		buffer.Printf("\n[DataMember(Name = \"%s\")]", propertyName)
		buffer.Printf("\nprotected %s %s;", GenerateCSharpTypeForSchema(subschema), ToJavaCase(propertyName))
	}
}

func generateCSharpConstructor(schema *ObjectSchema, buffer *BufferedFormatString) {

	var subschema TypeSchema
	var declarations, setters []string
	var propertyName string
	var toWrite string

	buffer.Printf("\npublic %s(", ToCamelCase(schema.Title))

	for _, propertyName = range schema.RequiredProperties {

		subschema = schema.Properties[propertyName]
		propertyName = ToJavaCase(propertyName)

		toWrite = fmt.Sprintf("%s %s", GenerateCSharpTypeForSchema(subschema), propertyName)
		declarations = append(declarations, toWrite)

		toWrite = fmt.Sprintf("\nset%s(%s);", ToStrictCamelCase(propertyName), propertyName)
		setters = append(setters, toWrite)
	}

	buffer.Print(strings.Join(declarations, ","))
	buffer.Print(")\n{")
	buffer.AddIndentation(1)

	for _, setter := range setters {
		buffer.Print(setter)
	}

	buffer.AddIndentation(-1)
	buffer.Print("\n}\n")
}

func generateCSharpFunctions(schema *ObjectSchema, buffer *BufferedFormatString) {

	var subschema TypeSchema
	var propertyName, properName, camelName, typeName string

	for _, propertyName = range schema.GetOrderedPropertyNames() {

		subschema = schema.Properties[propertyName]

		properName = ToJavaCase(propertyName)
		camelName = ToStrictCamelCase(propertyName)
		typeName = GenerateCSharpTypeForSchema(subschema)

		// getter
		buffer.Printf("\npublic %s get%s()\n{", typeName, camelName)
		buffer.AddIndentation(1)

		buffer.Printf("\nreturn this.%s;", properName)

		buffer.AddIndentation(-1)
		buffer.Print("\n}")

		// setter
		buffer.Printf("\npublic void set%s(%s value)\n{", camelName, typeName)
		buffer.AddIndentation(1)

		switch subschema.GetSchemaType() {
		case SCHEMATYPE_STRING:
			generateCSharpStringSetter(subschema.(*StringSchema), buffer)
		case SCHEMATYPE_INTEGER:
			fallthrough
		case SCHEMATYPE_NUMBER:
			generateCSharpNumericSetter(subschema.(NumericSchemaType), buffer)
		case SCHEMATYPE_OBJECT:
			generateCSharpObjectSetter(subschema.(*ObjectSchema), buffer)
		case SCHEMATYPE_ARRAY:
			generateCSharpArraySetter(subschema.(*ArraySchema), buffer)
		}

		buffer.Printf("\n%s = value;", properName)
		buffer.AddIndentation(-1)
		buffer.Print("\n}\n")
	}
}

func generateCSharpStringSetter(schema *StringSchema, buffer *BufferedFormatString) {

	if !schema.Nullable {
		generateCSharpNullCheck(buffer)
	}

	if schema.MinLength != nil {
		generateCSharpRangeCheck(*schema.MinLength, "value.Length", "was shorter than allowable minimum", "%d", false, "<", "", buffer)
	}

	if schema.MaxLength != nil {
		generateCSharpRangeCheck(*schema.MaxLength, "value.Length", "was longer than allowable maximum", "%d", false, ">", "", buffer)
	}

	if schema.MinByteLength != nil {
		generateCSharpRangeCheck(*schema.MinByteLength, "value.Length * sizeof(Char)", "had fewer bytes than allowable minimum", "%d", false, "<", "", buffer)
	}

	if schema.MaxByteLength != nil {
		generateCSharpRangeCheck(*schema.MaxByteLength, "value.Length * sizeof(Char)", "had more bytes than allowable minimum", "%d", false, ">", "", buffer)
	}

	if schema.Pattern != nil {

		buffer.Printf("\nRegex regex = new Regex(\"%s\");", sanitizeQuotedString(*schema.Pattern))
		buffer.Printf("\nif(!regex.IsMatch(value))\n{")
		buffer.AddIndentation(1)

		buffer.Printf("\nthrow new Exception(\"Value '\"+value+\"' did not match pattern '%s'\");", *schema.Pattern)

		buffer.AddIndentation(-1)
		buffer.Print("\n}")
	}
}

func generateCSharpNumericSetter(schema NumericSchemaType, buffer *BufferedFormatString) {

	if schema.HasMinimum() {
		generateCSharpRangeCheck(schema.GetMinimum(), "value", "is under the allowable minimum", schema.GetConstraintFormat(), schema.IsExclusiveMinimum(), "<=", "<", buffer)
	}

	if schema.HasMaximum() {
		generateCSharpRangeCheck(schema.GetMaximum(), "value", "is over the allowable maximum", schema.GetConstraintFormat(), schema.IsExclusiveMaximum(), ">=", ">", buffer)
	}

	if schema.HasEnum() {
		generateCSharpEnumCheck(schema, buffer, schema.GetEnum(), "", "")
	}

	if schema.HasMultiple() {

		buffer.Printf("\nif(value %% %f != 0)\n{", schema.GetMultiple())
		buffer.AddIndentation(1)

		buffer.Printf("\nthrow new Exception(\"Property '\"+value+\"' was not a multiple of %s\");", schema.GetMultiple())

		buffer.AddIndentation(-1)
		buffer.Print("\n}\n")
	}
}

func generateCSharpObjectSetter(schema *ObjectSchema, buffer *BufferedFormatString) {

	if !schema.Nullable {
		generateCSharpNullCheck(buffer)
	}
}

func generateCSharpArraySetter(schema *ArraySchema, buffer *BufferedFormatString) {

	if !schema.Nullable {
		generateCSharpNullCheck(buffer)
	}

	if schema.MinItems != nil {
		generateCSharpRangeCheck(*schema.MinItems, "value.Length", "does not have enough items", "%d", false, "<", "", buffer)
	}

	if schema.MaxItems != nil {
		generateCSharpRangeCheck(*schema.MaxItems, "value.Length", "does not have enough items", "%d", false, ">", "", buffer)
	}
}

func generateCSharpNullCheck(buffer *BufferedFormatString) {

	buffer.Printf("\nif(value == null)\n{")
	buffer.AddIndentation(1)

	buffer.Print("\nthrow new NullReferenceException(\"Cannot set property to null value\");")

	buffer.AddIndentation(-1)
	buffer.Print("\n}\n")
}

func generateCSharpRangeCheck(value interface{}, reference, message, format string, exclusive bool, comparator, exclusiveComparator string, buffer *BufferedFormatString) {

	var compareString string

	if exclusive {
		compareString = exclusiveComparator
	} else {
		compareString = comparator
	}

	buffer.Printf("\nif(%s %s "+format+")\n{", reference, compareString, value)
	buffer.AddIndentation(1)

	buffer.Printf("\nthrow new Exception(\"Property '\"+value+\"' %s.\");", message)
	buffer.AddIndentation(-1)
	buffer.Print("\n}\n")
}

/*
	Generates code which throws an error if the given [parameter]'s value is not contained in the given [validValues].
*/
func generateCSharpEnumCheck(schema TypeSchema, buffer *BufferedFormatString, enumValues []interface{}, prefix string, postfix string) {

	var typeName string
	var length int

	length = len(enumValues)

	if length <= 0 {
		return
	}

	// write array of valid values
	typeName = GenerateCSharpTypeForSchema(schema)
	buffer.Printf("%s[] validValues = new %s[]{%s%v%s", typeName, typeName, prefix, enumValues[0], postfix)

	for _, enumValue := range enumValues[1:length] {
		buffer.Printf(",%s%v%s", prefix, enumValue, postfix)
	}

	buffer.Print("};\n")

	// compare
	buffer.Print("\nbool isValid = false;")
	buffer.Print("\nfor(int i = 0; i < validValues.Length; i++)\n{")
	buffer.AddIndentation(1)

	buffer.Print("\nif(validValues[i] == value)\n{")
	buffer.AddIndentation(1)
	buffer.Print("\nisValid = true;\nbreak;")

	buffer.AddIndentation(-1)
	buffer.Print("\n}")
	buffer.AddIndentation(-1)
	buffer.Print("\n}")

	buffer.Print("\nif(!isValid)\n{")
	buffer.AddIndentation(1)
	buffer.Print("\nthrow new Exception(\"Given value '\"+value+\"' was not found in list of acceptable values\");")

	buffer.AddIndentation(-1)
	buffer.Print("\n}\n")
}

func GenerateCSharpTypeForSchema(subschema TypeSchema) string {

	switch subschema.GetSchemaType() {
	case SCHEMATYPE_NUMBER:
		return "double"
	case SCHEMATYPE_INTEGER:
		return "int"
	case SCHEMATYPE_ARRAY:
		return ToCamelCase(subschema.(*ArraySchema).Items.GetTitle()) + "[]"
	case SCHEMATYPE_OBJECT:
		return ToCamelCase(subschema.GetTitle())
	case SCHEMATYPE_STRING:
		return "string"
	case SCHEMATYPE_BOOLEAN:
		return "bool"
	}

	return "Object"
}
<|endoftext|>