<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ranch

import (
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/sirupsen/logrus"

	"k8s.io/test-infra/boskos/common"
)

// Ranch is the place which all of the Resource objects lives.
type Ranch struct {
	Storage       *Storage
	resourcesLock sync.RWMutex
	requestMgr    *RequestManager
	//
	now func() time.Time
}

func updateTime() time.Time {
	return time.Now()
}

// Public errors:

// ResourceNotFound will be returned if requested resource does not exist.
type ResourceNotFound struct {
	name string
}

func (r ResourceNotFound) Error() string {
	return fmt.Sprintf("no available resource %s, try again later.", r.name)
}

// ResourceTypeNotFound will be returned if requested resource type does not exist.
type ResourceTypeNotFound struct {
	rType string
}

func (r ResourceTypeNotFound) Error() string {
	return fmt.Sprintf("resource type %s does not exist", r.rType)
}

// OwnerNotMatch will be returned if request owner does not match current owner for target resource.
type OwnerNotMatch struct {
	request string
	owner   string
}

func (o OwnerNotMatch) Error() string {
	return fmt.Sprintf("owner mismatch request by %s, currently owned by %s", o.request, o.owner)
}

// StateNotMatch will be returned if requested state does not match current state for target resource.
type StateNotMatch struct {
	expect  string
	current string
}

func (s StateNotMatch) Error() string {
	return fmt.Sprintf("state mismatch - expected %v, current %v", s.expect, s.current)
}

// NewRanch creates a new Ranch object.
// In: config - path to resource file
//     storage - path to where to save/restore the state data
// Out: A Ranch object, loaded from config/storage, or error
func NewRanch(config string, s *Storage, ttl time.Duration) (*Ranch, error) {
	newRanch := &Ranch{
		Storage:    s,
		requestMgr: NewRequestManager(ttl),
		now:        time.Now,
	}
	if config != "" {
		if err := newRanch.SyncConfig(config); err != nil {
			return nil, err
		}
		logrus.Infof("Loaded Boskos configuration successfully")
	}
	return newRanch, nil
}

// acquireRequestPriorityKey is used as key for request priority cache.
type acquireRequestPriorityKey struct {
	rType, state string
}

// Acquire checks out a type of resource in certain state without an owner,
// and move the checked out resource to the end of the resource list.
// In: rtype - name of the target resource
//     state - current state of the requested resource
//     dest - destination state of the requested resource
//     owner - requester of the resource
//     requestID - request ID to get a priority in the queue
// Out: A valid Resource object on success, or
//      ResourceNotFound error if target type resource does not exist in target state.
func (r *Ranch) Acquire(rType, state, dest, owner, requestID string) (*common.Resource, error) {
	r.resourcesLock.Lock()
	defer r.resourcesLock.Unlock()

	// Finding Request Priority
	ts := acquireRequestPriorityKey{rType: rType, state: state}
	rank, new := r.requestMgr.GetRank(ts, requestID)

	resources, err := r.Storage.GetResources()
	if err != nil {
		logrus.WithError(err).Errorf("could not get resources")
		return nil, &ResourceNotFound{rType}
	}

	// For request priority we need to go over all the list until a matching rank
	matchingResoucesCount := 0
	typeCount := 0
	for idx := range resources {
		res := resources[idx]
		if rType == res.Type {
			typeCount++
			if state == res.State && res.Owner == "" {
				matchingResoucesCount++
				if matchingResoucesCount >= rank {
					res.Owner = owner
					res.State = dest
					updatedRes, err := r.Storage.UpdateResource(res)
					if err != nil {
						logrus.WithError(err).Errorf("could not update resource %s", res.Name)
						return nil, err
					}
					// Deleting this request since it has been fulfilled
					if requestID != "" {
						r.requestMgr.Delete(ts, requestID)
					}
					return &updatedRes, nil
				}
			}
		}
	}

	if new {
		// Checking if this a dynamic resource
		lifeCycle, err := r.Storage.GetDynamicResourceLifeCycle(rType)
		// Assuming error means no associated dynamic resource
		if err == nil {
			if typeCount < lifeCycle.MaxCount {
				// Adding a new resource
				res := common.NewResourceFromNewDynamicResourceLifeCycle(r.Storage.generateName(), &lifeCycle, r.now())
				if err := r.Storage.AddResource(res); err != nil {
					logrus.WithError(err).Warningf("unable to add a new resource of type %s", rType)
				}
				logrus.Infof("Added dynamic resource %s of type %s", res.Name, res.Type)
			}
		}
	}

	if typeCount > 0 {
		return nil, &ResourceNotFound{rType}
	}
	return nil, &ResourceTypeNotFound{rType}
}

// AcquireByState checks out resources of a given type without an owner,
// that matches a list of resources names.
// In: state - current state of the requested resource
//     dest - destination state of the requested resource
//     owner - requester of the resource
//     names - names of resource to acquire
// Out: A valid list of Resource object on success, or
//      ResourceNotFound error if target type resource does not exist in target state.
func (r *Ranch) AcquireByState(state, dest, owner string, names []string) ([]common.Resource, error) {
	r.resourcesLock.Lock()
	defer r.resourcesLock.Unlock()

	if names == nil {
		return nil, fmt.Errorf("must provide names of expected resources")
	}

	rNames := map[string]bool{}
	for _, t := range names {
		rNames[t] = true
	}

	allResources, err := r.Storage.GetResources()
	if err != nil {
		logrus.WithError(err).Errorf("could not get resources")
		return nil, &ResourceNotFound{state}
	}

	var resources []common.Resource

	for idx := range allResources {
		res := allResources[idx]
		if state == res.State {
			if res.Owner != "" {
				continue
			}
			if rNames[res.Name] {
				res.Owner = owner
				res.State = dest
				updatedRes, err := r.Storage.UpdateResource(res)
				if err != nil {
					logrus.WithError(err).Errorf("could not update resource %s", res.Name)
					return nil, err
				}
				resources = append(resources, updatedRes)
				delete(rNames, res.Name)
			}
		}
	}
	if len(rNames) != 0 {
		var missingResources []string
		for n := range rNames {
			missingResources = append(missingResources, n)
		}
		err := &ResourceNotFound{state}
		logrus.WithError(err).Errorf("could not find required resources %s", strings.Join(missingResources, ", "))
		return resources, err
	}
	return resources, nil
}

// Release unsets owner for target resource and move it to a new state.
// In: name - name of the target resource
//     dest - destination state of the resource
//     owner - owner of the resource
// Out: nil on success, or
//      OwnerNotMatch error if owner does not match current owner of the resource, or
//      ResourceNotFound error if target named resource does not exist.
func (r *Ranch) Release(name, dest, owner string) error {
	r.resourcesLock.Lock()
	defer r.resourcesLock.Unlock()

	res, err := r.Storage.GetResource(name)
	if err != nil {
		logrus.WithError(err).Errorf("unable to release resource %s", name)
		return &ResourceNotFound{name}
	}
	if owner != res.Owner {
		return &OwnerNotMatch{request: owner, owner: res.Owner}
	}

	res.Owner = ""
	res.State = dest

	if lf, err := r.Storage.GetDynamicResourceLifeCycle(res.Type); err == nil {
		// Assuming error means not existing as the only way to differentiate would be to list
		// all resources and find the right one which is more costly.
		if lf.LifeSpan != nil {
			expirationTime := r.now().Add(*lf.LifeSpan)
			res.ExpirationDate = &expirationTime
		}
	} else {
		res.ExpirationDate = nil
	}

	if _, err := r.Storage.UpdateResource(res); err != nil {
		logrus.WithError(err).Errorf("could not update resource %s", res.Name)
		return err
	}
	return nil
}

// Update updates the timestamp of a target resource.
// In: name  - name of the target resource
//     state - current state of the resource
//     owner - current owner of the resource
// 	   info  - information on how to use the resource
// Out: nil on success, or
//      OwnerNotMatch error if owner does not match current owner of the resource, or
//      ResourceNotFound error if target named resource does not exist, or
//      StateNotMatch error if state does not match current state of the resource.
func (r *Ranch) Update(name, owner, state string, ud *common.UserData) error {
	r.resourcesLock.Lock()
	defer r.resourcesLock.Unlock()

	res, err := r.Storage.GetResource(name)
	if err != nil {
		logrus.WithError(err).Errorf("could not find resource %s for update", name)
		return &ResourceNotFound{name}
	}
	if owner != res.Owner {
		return &OwnerNotMatch{request: owner, owner: res.Owner}
	}
	if state != res.State {
		return &StateNotMatch{res.State, state}
	}
	if res.UserData == nil {
		res.UserData = &common.UserData{}
	}
	res.UserData.Update(ud)
	if _, err := r.Storage.UpdateResource(res); err != nil {
		logrus.WithError(err).Errorf("could not update resource %s", res.Name)
		return err
	}
	return nil
}

// Reset unstucks a type of stale resource to a new state.
// In: rtype - type of the resource
//     state - current state of the resource
//     expire - duration before resource's last update
//     dest - destination state of expired resources
// Out: map of resource name - resource owner.
func (r *Ranch) Reset(rtype, state string, expire time.Duration, dest string) (map[string]string, error) {
	r.resourcesLock.Lock()
	defer r.resourcesLock.Unlock()

	ret := make(map[string]string)

	resources, err := r.Storage.GetResources()
	if err != nil {
		logrus.WithError(err).Errorf("cannot find resources")
		return nil, err
	}

	for idx := range resources {
		res := resources[idx]
		if rtype == res.Type && state == res.State && res.Owner != "" {
			if r.now().Sub(res.LastUpdate) > expire {
				ret[res.Name] = res.Owner
				res.Owner = ""
				res.State = dest
				if _, err := r.Storage.UpdateResource(res); err != nil {
					logrus.WithError(err).Errorf("could not update resource %s", res.Name)
					return ret, err
				}
			}
		}
	}
	return ret, nil
}

// SyncConfig updates resource list from a file
func (r *Ranch) SyncConfig(configPath string) error {
	config, err := common.ParseConfig(configPath)
	if err != nil {
		return err
	}
	if err := common.ValidateConfig(config); err != nil {
		return err
	}
	r.resourcesLock.Lock()
	defer r.resourcesLock.Unlock()
	if err := r.Storage.SyncResources(config); err != nil {
		return err
	}
	return nil
}

// StartRequestGC starts the GC of expired requests
func (r *Ranch) StartRequestGC(gcPeriod time.Duration) {
	r.requestMgr.StartGC(gcPeriod)
}

// Metric returns a metric object with metrics filled in
func (r *Ranch) Metric(rtype string) (common.Metric, error) {
	metric := common.Metric{
		Type:    rtype,
		Current: map[string]int{},
		Owners:  map[string]int{},
	}

	resources, err := r.Storage.GetResources()
	if err != nil {
		logrus.WithError(err).Error("cannot find resources")
		return metric, &ResourceNotFound{rtype}
	}

	for _, res := range resources {
		if res.Type != rtype {
			continue
		}

		if _, ok := metric.Current[res.State]; !ok {
			metric.Current[res.State] = 0
		}

		if _, ok := metric.Owners[res.Owner]; !ok {
			metric.Owners[res.Owner] = 0
		}

		metric.Current[res.State]++
		metric.Owners[res.Owner]++
	}

	if len(metric.Current) == 0 && len(metric.Owners) == 0 {
		return metric, &ResourceNotFound{rtype}
	}

	return metric, nil
}
<commit_msg>boskos: ranch: add more logging to acquire call<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ranch

import (
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/sirupsen/logrus"

	"k8s.io/test-infra/boskos/common"
)

// Ranch is the place which all of the Resource objects lives.
type Ranch struct {
	Storage       *Storage
	resourcesLock sync.RWMutex
	requestMgr    *RequestManager
	//
	now func() time.Time
}

func updateTime() time.Time {
	return time.Now()
}

// Public errors:

// ResourceNotFound will be returned if requested resource does not exist.
type ResourceNotFound struct {
	name string
}

func (r ResourceNotFound) Error() string {
	return fmt.Sprintf("no available resource %s, try again later.", r.name)
}

// ResourceTypeNotFound will be returned if requested resource type does not exist.
type ResourceTypeNotFound struct {
	rType string
}

func (r ResourceTypeNotFound) Error() string {
	return fmt.Sprintf("resource type %s does not exist", r.rType)
}

// OwnerNotMatch will be returned if request owner does not match current owner for target resource.
type OwnerNotMatch struct {
	request string
	owner   string
}

func (o OwnerNotMatch) Error() string {
	return fmt.Sprintf("owner mismatch request by %s, currently owned by %s", o.request, o.owner)
}

// StateNotMatch will be returned if requested state does not match current state for target resource.
type StateNotMatch struct {
	expect  string
	current string
}

func (s StateNotMatch) Error() string {
	return fmt.Sprintf("state mismatch - expected %v, current %v", s.expect, s.current)
}

// NewRanch creates a new Ranch object.
// In: config - path to resource file
//     storage - path to where to save/restore the state data
// Out: A Ranch object, loaded from config/storage, or error
func NewRanch(config string, s *Storage, ttl time.Duration) (*Ranch, error) {
	newRanch := &Ranch{
		Storage:    s,
		requestMgr: NewRequestManager(ttl),
		now:        time.Now,
	}
	if config != "" {
		if err := newRanch.SyncConfig(config); err != nil {
			return nil, err
		}
		logrus.Infof("Loaded Boskos configuration successfully")
	}
	return newRanch, nil
}

// acquireRequestPriorityKey is used as key for request priority cache.
type acquireRequestPriorityKey struct {
	rType, state string
}

// Acquire checks out a type of resource in certain state without an owner,
// and move the checked out resource to the end of the resource list.
// In: rtype - name of the target resource
//     state - current state of the requested resource
//     dest - destination state of the requested resource
//     owner - requester of the resource
//     requestID - request ID to get a priority in the queue
// Out: A valid Resource object on success, or
//      ResourceNotFound error if target type resource does not exist in target state.
func (r *Ranch) Acquire(rType, state, dest, owner, requestID string) (*common.Resource, error) {
	r.resourcesLock.Lock()
	defer r.resourcesLock.Unlock()

	logger := logrus.WithFields(logrus.Fields{
		"type":       rType,
		"state":      state,
		"dest":       dest,
		"owner":      owner,
		"identifier": requestID,
	})
	logger.Debug("Determining request priority...")
	ts := acquireRequestPriorityKey{rType: rType, state: state}
	rank, new := r.requestMgr.GetRank(ts, requestID)
	logger.WithFields(logrus.Fields{"rank": rank, "new": new}).Debug("Determined request priority.")

	resources, err := r.Storage.GetResources()
	if err != nil {
		logger.WithError(err).Errorf("could not get resources")
		return nil, &ResourceNotFound{rType}
	}
	logger.Debugf("Considering %d resources.", len(resources))

	// For request priority we need to go over all the list until a matching rank
	matchingResoucesCount := 0
	typeCount := 0
	for idx := range resources {
		res := resources[idx]
		if rType == res.Type {
			typeCount++
			if state == res.State && res.Owner == "" {
				matchingResoucesCount++
				if matchingResoucesCount >= rank {
					logger = logger.WithField("resource", res.Name)
					res.Owner = owner
					res.State = dest
					logger.Debug("Updating resource.")
					updatedRes, err := r.Storage.UpdateResource(res)
					if err != nil {
						logger.WithError(err).Errorf("could not update resource %s", res.Name)
						return nil, err
					}
					// Deleting this request since it has been fulfilled
					if requestID != "" {
						logger.Debug("Cleaning up requests.")
						r.requestMgr.Delete(ts, requestID)
					}
					logger.Debug("Successfully acquired resource.")
					return &updatedRes, nil
				}
			}
		}
	}

	if new {
		logger.Debug("Checking for associated dynamic resource type...")
		lifeCycle, err := r.Storage.GetDynamicResourceLifeCycle(rType)
		// Assuming error means no associated dynamic resource
		if err == nil {
			if typeCount < lifeCycle.MaxCount {
				logger.Debug("Adding new dynamic resources...")
				res := common.NewResourceFromNewDynamicResourceLifeCycle(r.Storage.generateName(), &lifeCycle, r.now())
				if err := r.Storage.AddResource(res); err != nil {
					logger.WithError(err).Warningf("unable to add a new resource of type %s", rType)
				}
				logger.Infof("Added dynamic resource %s of type %s", res.Name, res.Type)
			}
		}
	}

	if typeCount > 0 {
		return nil, &ResourceNotFound{rType}
	}
	return nil, &ResourceTypeNotFound{rType}
}

// AcquireByState checks out resources of a given type without an owner,
// that matches a list of resources names.
// In: state - current state of the requested resource
//     dest - destination state of the requested resource
//     owner - requester of the resource
//     names - names of resource to acquire
// Out: A valid list of Resource object on success, or
//      ResourceNotFound error if target type resource does not exist in target state.
func (r *Ranch) AcquireByState(state, dest, owner string, names []string) ([]common.Resource, error) {
	r.resourcesLock.Lock()
	defer r.resourcesLock.Unlock()

	if names == nil {
		return nil, fmt.Errorf("must provide names of expected resources")
	}

	rNames := map[string]bool{}
	for _, t := range names {
		rNames[t] = true
	}

	allResources, err := r.Storage.GetResources()
	if err != nil {
		logrus.WithError(err).Errorf("could not get resources")
		return nil, &ResourceNotFound{state}
	}

	var resources []common.Resource

	for idx := range allResources {
		res := allResources[idx]
		if state == res.State {
			if res.Owner != "" {
				continue
			}
			if rNames[res.Name] {
				res.Owner = owner
				res.State = dest
				updatedRes, err := r.Storage.UpdateResource(res)
				if err != nil {
					logrus.WithError(err).Errorf("could not update resource %s", res.Name)
					return nil, err
				}
				resources = append(resources, updatedRes)
				delete(rNames, res.Name)
			}
		}
	}
	if len(rNames) != 0 {
		var missingResources []string
		for n := range rNames {
			missingResources = append(missingResources, n)
		}
		err := &ResourceNotFound{state}
		logrus.WithError(err).Errorf("could not find required resources %s", strings.Join(missingResources, ", "))
		return resources, err
	}
	return resources, nil
}

// Release unsets owner for target resource and move it to a new state.
// In: name - name of the target resource
//     dest - destination state of the resource
//     owner - owner of the resource
// Out: nil on success, or
//      OwnerNotMatch error if owner does not match current owner of the resource, or
//      ResourceNotFound error if target named resource does not exist.
func (r *Ranch) Release(name, dest, owner string) error {
	r.resourcesLock.Lock()
	defer r.resourcesLock.Unlock()

	res, err := r.Storage.GetResource(name)
	if err != nil {
		logrus.WithError(err).Errorf("unable to release resource %s", name)
		return &ResourceNotFound{name}
	}
	if owner != res.Owner {
		return &OwnerNotMatch{request: owner, owner: res.Owner}
	}

	res.Owner = ""
	res.State = dest

	if lf, err := r.Storage.GetDynamicResourceLifeCycle(res.Type); err == nil {
		// Assuming error means not existing as the only way to differentiate would be to list
		// all resources and find the right one which is more costly.
		if lf.LifeSpan != nil {
			expirationTime := r.now().Add(*lf.LifeSpan)
			res.ExpirationDate = &expirationTime
		}
	} else {
		res.ExpirationDate = nil
	}

	if _, err := r.Storage.UpdateResource(res); err != nil {
		logrus.WithError(err).Errorf("could not update resource %s", res.Name)
		return err
	}
	return nil
}

// Update updates the timestamp of a target resource.
// In: name  - name of the target resource
//     state - current state of the resource
//     owner - current owner of the resource
// 	   info  - information on how to use the resource
// Out: nil on success, or
//      OwnerNotMatch error if owner does not match current owner of the resource, or
//      ResourceNotFound error if target named resource does not exist, or
//      StateNotMatch error if state does not match current state of the resource.
func (r *Ranch) Update(name, owner, state string, ud *common.UserData) error {
	r.resourcesLock.Lock()
	defer r.resourcesLock.Unlock()

	res, err := r.Storage.GetResource(name)
	if err != nil {
		logrus.WithError(err).Errorf("could not find resource %s for update", name)
		return &ResourceNotFound{name}
	}
	if owner != res.Owner {
		return &OwnerNotMatch{request: owner, owner: res.Owner}
	}
	if state != res.State {
		return &StateNotMatch{res.State, state}
	}
	if res.UserData == nil {
		res.UserData = &common.UserData{}
	}
	res.UserData.Update(ud)
	if _, err := r.Storage.UpdateResource(res); err != nil {
		logrus.WithError(err).Errorf("could not update resource %s", res.Name)
		return err
	}
	return nil
}

// Reset unstucks a type of stale resource to a new state.
// In: rtype - type of the resource
//     state - current state of the resource
//     expire - duration before resource's last update
//     dest - destination state of expired resources
// Out: map of resource name - resource owner.
func (r *Ranch) Reset(rtype, state string, expire time.Duration, dest string) (map[string]string, error) {
	r.resourcesLock.Lock()
	defer r.resourcesLock.Unlock()

	ret := make(map[string]string)

	resources, err := r.Storage.GetResources()
	if err != nil {
		logrus.WithError(err).Errorf("cannot find resources")
		return nil, err
	}

	for idx := range resources {
		res := resources[idx]
		if rtype == res.Type && state == res.State && res.Owner != "" {
			if r.now().Sub(res.LastUpdate) > expire {
				ret[res.Name] = res.Owner
				res.Owner = ""
				res.State = dest
				if _, err := r.Storage.UpdateResource(res); err != nil {
					logrus.WithError(err).Errorf("could not update resource %s", res.Name)
					return ret, err
				}
			}
		}
	}
	return ret, nil
}

// SyncConfig updates resource list from a file
func (r *Ranch) SyncConfig(configPath string) error {
	config, err := common.ParseConfig(configPath)
	if err != nil {
		return err
	}
	if err := common.ValidateConfig(config); err != nil {
		return err
	}
	r.resourcesLock.Lock()
	defer r.resourcesLock.Unlock()
	if err := r.Storage.SyncResources(config); err != nil {
		return err
	}
	return nil
}

// StartRequestGC starts the GC of expired requests
func (r *Ranch) StartRequestGC(gcPeriod time.Duration) {
	r.requestMgr.StartGC(gcPeriod)
}

// Metric returns a metric object with metrics filled in
func (r *Ranch) Metric(rtype string) (common.Metric, error) {
	metric := common.Metric{
		Type:    rtype,
		Current: map[string]int{},
		Owners:  map[string]int{},
	}

	resources, err := r.Storage.GetResources()
	if err != nil {
		logrus.WithError(err).Error("cannot find resources")
		return metric, &ResourceNotFound{rtype}
	}

	for _, res := range resources {
		if res.Type != rtype {
			continue
		}

		if _, ok := metric.Current[res.State]; !ok {
			metric.Current[res.State] = 0
		}

		if _, ok := metric.Owners[res.Owner]; !ok {
			metric.Owners[res.Owner] = 0
		}

		metric.Current[res.State]++
		metric.Owners[res.Owner]++
	}

	if len(metric.Current) == 0 && len(metric.Owners) == 0 {
		return metric, &ResourceNotFound{rtype}
	}

	return metric, nil
}
<|endoftext|>


<commit_before>package broker

import (
	"fmt"
	"os"
	"gopkg.in/ory-am/dockertest.v3"
	"log"
	"testing"
	"github.com/nats-io/go-nats"
	"github.com/pkg/errors"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

var nc *nats.Conn

func TestMain(m *testing.M) {
	pool, err := dockertest.NewPool("")
	if err != nil {
		log.Fatalf("Could not connect to docker: %s", err)
	}

	resource, err := pool.Run("nats", "0.9.4", nil)
	if err != nil {
		log.Fatalf("Could not start resource: %s", err)
	}

	if err = pool.Retry(func() error {
		var err error
		nc, err = nats.Connect(fmt.Sprintf("nats://localhost:%s", resource.GetPort("4222/tcp")))
		if err != nil {
			return err
		}
		if nc.Status() != nats.CONNECTED {
			return errors.New("Not connected yet")
		}
		return nil
	}); err != nil {
		log.Fatalf("Could not connect to docker: %s", err)
	}

	s := m.Run()
	pool.Purge(resource)
	os.Exit(s)
}

func TestBroker(t *testing.T) {
	b := Broker{N: nc}

	type req struct{ Foo string `json:"foo"` }
	type res struct{ Bar string `json:"bar"` }

	t.Run("case=1/description=should encode and decode messages", func(t *testing.T) {
		sub, err := nc.Subscribe("foo", func(m *nats.Msg) {
			var f req
			c, err := b.Parse(m, &f)
			require.Nil(t, err)

			b.Reply(m, c.RequestID, &res{Bar: f.Foo})
		})
		require.Nil(t, err)
		defer sub.Unsubscribe()

		var r res
		c, err := b.Request("foo", "request-id", &req{Foo: "bar"}, &r)
		require.Nil(t, err)
		assert.Equal(t, r.Bar, "bar")
		assert.Equal(t, c.RequestID, "request-id")
	})

	t.Run("case=2/description=should be able to write errors", func(t *testing.T) {
		sub, err := nc.Subscribe("foo", func(m *nats.Msg) {
			var f req
			c, err := b.Parse(m, &f)
			require.Nil(t, err)
			b.WriteErrorCode(m.Reply, c.RequestID, 404, errors.New("some error"))
		})
		require.Nil(t, err)
		defer sub.Unsubscribe()

		var r res
		c, err := b.Request("foo", "request-id", &req{Foo: "bar"}, &r)
		assert.NotNil(t, err)
		require.NotNil(t, c)
		assert.Equal(t, c.Status, 404)
	})

	t.Run("case=3/description=parse should always return a container", func(t *testing.T) {
		c, err := b.Parse(&nats.Msg{}, nil)
		require.NotNil(t, err)
		require.NotNil(t, c)

	})
}
<commit_msg>broker updates<commit_after>package broker

import (
	"fmt"
	"os"
	"gopkg.in/ory-am/dockertest.v3"
	"log"
	"testing"
	"github.com/nats-io/go-nats"
	"github.com/pkg/errors"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

var nc *nats.Conn

func TestMain(m *testing.M) {
	pool, err := dockertest.NewPool("")
	if err != nil {
		log.Fatalf("Could not connect to docker: %s", err)
	}

	resource, err := pool.Run("nats", "0.9.4", nil)
	if err != nil {
		log.Fatalf("Could not start resource: %s", err)
	}

	if err = pool.Retry(func() error {
		var err error
		nc, err = nats.Connect(fmt.Sprintf("nats://localhost:%s", resource.GetPort("4222/tcp")))
		if err != nil {
			return err
		}
		if nc.Status() != nats.CONNECTED {
			return errors.New("Not connected yet")
		}
		return nil
	}); err != nil {
		log.Fatalf("Could not connect to docker: %s", err)
	}

	s := m.Run()
	pool.Purge(resource)
	os.Exit(s)
}

func TestBroker(t *testing.T) {
	b := New(nc, "0.0.1")

	type req struct{ Foo string `json:"foo"` }
	type res struct{ Bar string `json:"bar"` }

	t.Run("case=1/description=should encode and decode messages", func(t *testing.T) {
		sub, err := nc.Subscribe("foo", func(m *nats.Msg) {
			var f req
			c, err := b.Parse(m, &f)
			require.Nil(t, err)

			b.Reply(m, c.RequestID, &res{Bar: f.Foo})
		})
		require.Nil(t, err)
		defer sub.Unsubscribe()

		var r res
		c, err := b.Request("foo", "request-id", &req{Foo: "bar"}, &r)
		require.Nil(t, err)
		assert.Equal(t, r.Bar, "bar")
		assert.Equal(t, c.RequestID, "request-id")
	})

	t.Run("case=2/description=should be able to write errors", func(t *testing.T) {
		sub, err := nc.Subscribe("foo", func(m *nats.Msg) {
			var f req
			c, err := b.Parse(m, &f)
			require.Nil(t, err)
			b.WriteErrorCode(m.Reply, c.RequestID, 404, errors.New("some error"))
		})
		require.Nil(t, err)
		defer sub.Unsubscribe()

		var r res
		c, err := b.Request("foo", "request-id", &req{Foo: "bar"}, &r)
		assert.NotNil(t, err)
		require.NotNil(t, c)
		assert.Equal(t, c.Status, 404)
	})

	t.Run("case=3/description=parse should always return a container", func(t *testing.T) {
		c, err := b.Parse(&nats.Msg{}, nil)
		require.NotNil(t, err)
		require.NotNil(t, c)

	})
}
<|endoftext|>


<commit_before>// Package templates implements template inheritance and exposes functions to render these
//
// inspired by http://elithrar.github.io/article/approximating-html-template-inheritance/
package render

import (
	"errors"
	"fmt"
	htmpl "html/template"
	"net/http"
	"net/url"
	"path/filepath"
	"strconv"
	"time"

	"github.com/cryptix/go/logging"
	"github.com/gorilla/mux"
	"github.com/oxtoacart/bpool"
)

type assetFunc func(name string) ([]byte, error)

var (
	// Reload is whether to reload templates on each request.
	Reload bool

	log = logging.Logger("http")

	// asset
	asset assetFunc

	// files
	templateFiles     []string
	baseTemplateFiles []string

	// all the templates that we parsed
	templates = map[string]*htmpl.Template{}

	// bufpool is shared between all render() calls
	bufpool = bpool.NewBufferPool(64)
)

func SetBaseTemplates(fn assetFunc, files []string) {
	asset = fn
	baseTemplateFiles = append(baseTemplateFiles, files...)
}

func AddTemplates(files []string) {
	templateFiles = append(templateFiles, files...)
}

var appRouter *mux.Router

func SetAppRouter(r *mux.Router) {
	appRouter = r
}

// Load loads and parses all templates that are in templateDir
func Load() {
	if appRouter == nil {
		logging.CheckFatal(errors.New("No appRouter set"))
	}

	if len(baseTemplateFiles) == 0 {
		baseTemplateFiles = []string{"navbar.tmpl", "base.tmpl"}
	}

	logging.CheckFatal(parseHTMLTemplates())
}

func parseHTMLTemplates() error {
	for _, file := range templateFiles {
		t := htmpl.New("")
		t.Funcs(htmpl.FuncMap{
			"urlTo": urlTo,
			"itoa":  strconv.Itoa,
		})

		err := parseFilesFromBindata(t, file)
		if err != nil {
			return fmt.Errorf("template %v: %s", file, err)
		}

		t = t.Lookup("base")
		if t == nil {
			return fmt.Errorf("base template not found in %v", file)
		}
		templates[file] = t
	}
	return nil
}

// Render takes a template name and any kind of named data
// renders the template to a buffer from the pool
// and writes that to the http response
func Render(w http.ResponseWriter, r *http.Request, name string, status int, data interface{}) error {
	tmpl, ok := templates[name]
	if !ok {
		return errors.New("Could not find template:" + name)
	}
	start := time.Now()

	buf := bufpool.Get()
	err := tmpl.ExecuteTemplate(buf, "base", data)
	if err != nil {
		return err
	}

	start = time.Now()

	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	w.WriteHeader(status)
	_, err = buf.WriteTo(w)
	bufpool.Put(buf)
	log.Infof("Rendered %q Status:%d (took %v)", name, status, time.Since(start))
	return err
}

// PlainError helps rendering user errors
func PlainError(w http.ResponseWriter, statusCode int, err error) {
	log.Errorf("PlainError(%d):%s\n", statusCode, err)
	http.Error(w, err.Error(), statusCode)
}

// copied from template.ParseFiles but dont use ioutil.ReadFile
func parseFilesFromBindata(t *htmpl.Template, file string) error {
	var err error

	files := make([]string, len(baseTemplateFiles)+1)
	files[0] = file
	copy(files[1:], baseTemplateFiles)
	log.Debugf("parseFile - %q", files)

	for _, filename := range files {
		var tmplBytes []byte
		tmplBytes, err = asset(filename)
		if err != nil {
			log.Noticef("parseFile - Error from Asset() - %v", err)
			return err
		}

		var name = filepath.Base(filename)
		// First template becomes return value if not already defined,
		// and we use that one for subsequent New calls to associate
		// all the templates together. Also, if this file has the same name
		// as t, this file becomes the contents of t, so
		//  t, err := New(name).Funcs(xxx).ParseFiles(name)
		// works. Otherwise we create a new template associated with t.
		var tmpl *htmpl.Template
		if t == nil {
			t = htmpl.New(name)
		}
		if name == t.Name() {
			tmpl = t
		} else {
			tmpl = t.New(name)
		}
		_, err = tmpl.Parse(string(tmplBytes))
		if err != nil {
			return err
		}
	}
	return nil
}

func urlTo(routeName string, ps ...interface{}) *url.URL {
	route := appRouter.Get(routeName)
	if route == nil {
		log.Warningf("no such route: %q (params: %v)", routeName, ps)
		return &url.URL{}
	}

	var params []string
	for _, p := range ps {
		switch v := p.(type) {
		case string:
			params = append(params, v)
		case int:
			params = append(params, strconv.Itoa(v))
		case int64:
			params = append(params, strconv.FormatInt(v, 10))

		default:
			log.Errorf("invalid param type %v in route %q", p, routeName)
			logging.CheckFatal(errors.New("invalid param"))
		}
	}

	u, err := route.URLPath(params...)
	if err != nil {
		log.Errorf("Route error: failed to make URL for route %q (params: %v): %s", routeName, params, err)
		return &url.URL{}
	}
	return u
}
<commit_msg>strip prefix from template names<commit_after>// Package templates implements template inheritance and exposes functions to render these
//
// inspired by http://elithrar.github.io/article/approximating-html-template-inheritance/
package render

import (
	"errors"
	"fmt"
	htmpl "html/template"
	"net/http"
	"net/url"
	"path/filepath"
	"strconv"
	"time"

	"github.com/cryptix/go/logging"
	"github.com/gorilla/mux"
	"github.com/oxtoacart/bpool"
)

type assetFunc func(name string) ([]byte, error)

var (
	// Reload is whether to reload templates on each request.
	Reload bool

	log = logging.Logger("http")

	// asset
	asset assetFunc

	// files
	templateFiles     []string
	baseTemplateFiles []string

	// all the templates that we parsed
	templates = map[string]*htmpl.Template{}

	// bufpool is shared between all render() calls
	bufpool = bpool.NewBufferPool(64)
)

func SetBaseTemplates(fn assetFunc, files []string) {
	asset = fn
	baseTemplateFiles = append(baseTemplateFiles, files...)
}

func AddTemplates(files []string) {
	templateFiles = append(templateFiles, files...)
}

var appRouter *mux.Router

func SetAppRouter(r *mux.Router) {
	appRouter = r
}

// Load loads and parses all templates that are in templateDir
func Load() {
	if appRouter == nil {
		logging.CheckFatal(errors.New("No appRouter set"))
	}

	if len(baseTemplateFiles) == 0 {
		baseTemplateFiles = []string{"navbar.tmpl", "base.tmpl"}
	}

	logging.CheckFatal(parseHTMLTemplates())
}

func parseHTMLTemplates() error {
	for _, file := range templateFiles {
		t := htmpl.New("")
		t.Funcs(htmpl.FuncMap{
			"urlTo": urlTo,
			"itoa":  strconv.Itoa,
		})

		err := parseFilesFromBindata(t, file)
		if err != nil {
			return fmt.Errorf("template %v: %s", file, err)
		}

		t = t.Lookup("base")
		if t == nil {
			return fmt.Errorf("base template not found in %v", file)
		}
		templates[strings.TrimPrefix(file, "tmpl/")] = t
	}
	return nil
}

// Render takes a template name and any kind of named data
// renders the template to a buffer from the pool
// and writes that to the http response
func Render(w http.ResponseWriter, r *http.Request, name string, status int, data interface{}) error {
	tmpl, ok := templates[name]
	if !ok {
		return errors.New("Could not find template:" + name)
	}
	start := time.Now()

	buf := bufpool.Get()
	err := tmpl.ExecuteTemplate(buf, "base", data)
	if err != nil {
		return err
	}

	start = time.Now()

	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	w.WriteHeader(status)
	_, err = buf.WriteTo(w)
	bufpool.Put(buf)
	log.Infof("Rendered %q Status:%d (took %v)", name, status, time.Since(start))
	return err
}

// PlainError helps rendering user errors
func PlainError(w http.ResponseWriter, statusCode int, err error) {
	log.Errorf("PlainError(%d):%s\n", statusCode, err)
	http.Error(w, err.Error(), statusCode)
}

// copied from template.ParseFiles but dont use ioutil.ReadFile
func parseFilesFromBindata(t *htmpl.Template, file string) error {
	var err error

	files := make([]string, len(baseTemplateFiles)+1)
	files[0] = file
	copy(files[1:], baseTemplateFiles)
	log.Debugf("parseFile - %q", files)

	for _, filename := range files {
		var tmplBytes []byte
		tmplBytes, err = asset(filename)
		if err != nil {
			log.Noticef("parseFile - Error from Asset() - %v", err)
			return err
		}

		var name = filepath.Base(filename)
		// First template becomes return value if not already defined,
		// and we use that one for subsequent New calls to associate
		// all the templates together. Also, if this file has the same name
		// as t, this file becomes the contents of t, so
		//  t, err := New(name).Funcs(xxx).ParseFiles(name)
		// works. Otherwise we create a new template associated with t.
		var tmpl *htmpl.Template
		if t == nil {
			t = htmpl.New(name)
		}
		if name == t.Name() {
			tmpl = t
		} else {
			tmpl = t.New(name)
		}
		_, err = tmpl.Parse(string(tmplBytes))
		if err != nil {
			return err
		}
	}
	return nil
}

func urlTo(routeName string, ps ...interface{}) *url.URL {
	route := appRouter.Get(routeName)
	if route == nil {
		log.Warningf("no such route: %q (params: %v)", routeName, ps)
		return &url.URL{}
	}

	var params []string
	for _, p := range ps {
		switch v := p.(type) {
		case string:
			params = append(params, v)
		case int:
			params = append(params, strconv.Itoa(v))
		case int64:
			params = append(params, strconv.FormatInt(v, 10))

		default:
			log.Errorf("invalid param type %v in route %q", p, routeName)
			logging.CheckFatal(errors.New("invalid param"))
		}
	}

	u, err := route.URLPath(params...)
	if err != nil {
		log.Errorf("Route error: failed to make URL for route %q (params: %v): %s", routeName, params, err)
		return &url.URL{}
	}
	return u
}
<|endoftext|>


<commit_before>package tests

import (
	"koding/db/mongodb/modelhelper"
	"socialapi/config"
	"socialapi/models"
	"socialapi/request"
	"socialapi/rest"
	"socialapi/workers/integration/webhook"
	"socialapi/workers/integration/webhook/api"
	"testing"

	"github.com/koding/integration/services"
	"github.com/koding/runner"
	. "github.com/smartystreets/goconvey/convey"
)

func newMiddlewareRequest(username, groupName string) *services.ServiceInput {
	return &services.ServiceInput{
		"username":  username,
		"eventName": "emailOpen",
		"message":   "testing it",
		"groupName": groupName,
	}
}

func newPushRequest(channelId int64, groupName string) *api.PushRequest {
	wr := &api.PushRequest{
		GroupName: groupName,
	}
	wr.Body = "hey"
	wr.ChannelId = channelId

	return wr
}

func newBotChannelRequest(nick, groupName string) *api.BotChannelRequest {
	return &api.BotChannelRequest{
		GroupName: groupName,
		Username:  nick,
	}
}

func TestWebhook(t *testing.T) {
	r := runner.New("test webhook")
	if err := r.Init(); err != nil {
		t.Fatalf("couldnt start bongo %s", err.Error())
	}
	defer r.Close()

	appConfig := config.MustRead(r.Conf.Path)
	modelhelper.Initialize(appConfig.Mongo)
	defer modelhelper.Close()

	channelIntegration := webhook.CreateTestChannelIntegration(t)

	webhook.CreateIterableIntegration(t)

	Convey("We should be able to successfully push message", t, func() {

		account, err := models.CreateAccountInBothDbsWithNick("sinan")
		So(err, ShouldBeNil)

		channel := models.CreateTypedGroupedChannelWithTest(account.Id, models.Channel_TYPE_TOPIC, channelIntegration.GroupName)
		_, err = channel.AddParticipant(account.Id)
		So(err, ShouldBeNil)

		err = rest.DoPushRequest(newPushRequest(channel.Id, channelIntegration.GroupName), channelIntegration.Token)
		So(err, ShouldBeNil)

		ses, err := models.FetchOrCreateSession(account.Nick, channelIntegration.GroupName)
		So(err, ShouldBeNil)
		So(ses, ShouldNotBeNil)

		resp, err := rest.GetHistory(channel.Id,
			&request.Query{
				AccountId: account.Id,
			},
			ses.ClientId,
		)

		So(err, ShouldBeNil)
		So(len(resp.MessageList), ShouldEqual, 1)
	})

	Convey("We should be able to successfully fetch bot channel of the user", t, func() {
		account, err := models.CreateAccountInBothDbsWithNick("sinan")
		So(err, ShouldBeNil)
		groupName := models.RandomGroupName()
		channel := models.CreateTypedGroupedChannelWithTest(account.Id, models.Channel_TYPE_GROUP, groupName)
		_, err = channel.AddParticipant(account.Id)
		So(err, ShouldBeNil)

		ses, err := models.FetchOrCreateSession(account.Nick, groupName)
		So(err, ShouldBeNil)
		So(ses, ShouldNotBeNil)

		channelId, err := rest.DoBotChannelRequest(ses.ClientId)

		So(err, ShouldBeNil)
		So(channelId, ShouldNotEqual, 0)
	})

	Convey("We should be able to successfully push messages via prepare endpoint", t, func() {

		account, err := models.CreateAccountInBothDbsWithNick(models.RandomName())
		So(err, ShouldBeNil)

		channel := models.CreateTypedGroupedChannelWithTest(account.Id, models.Channel_TYPE_GROUP, channelIntegration.GroupName)
		_, err = channel.AddParticipant(account.Id)

		pr := newMiddlewareRequest("xxx", channelIntegration.GroupName)
		err = rest.DoPrepareRequest(pr, channelIntegration.Token)
		So(err, ShouldNotBeNil)

		pr = newMiddlewareRequest(account.Nick, channelIntegration.GroupName)
		err = rest.DoPrepareRequest(pr, channelIntegration.Token)
		So(err, ShouldBeNil)

		ses, err := models.FetchOrCreateSession(account.Nick, channelIntegration.GroupName)
		So(err, ShouldBeNil)
		So(ses, ShouldNotBeNil)

		channelId, err := rest.DoBotChannelRequest(ses.ClientId)
		So(err, ShouldBeNil)

		resp, err := rest.GetHistory(channelId,
			&request.Query{
				AccountId: account.Id,
			},
			ses.ClientId,
		)

		So(err, ShouldBeNil)
		So(len(resp.MessageList), ShouldEqual, 1)
		So(resp.MessageList[0].Message.Body, ShouldEqual, "testing it")
	})

	Convey("We should not be able to send more than 100 requests per minute", t, func() {

		account, err := models.CreateAccountInBothDbsWithNick("sinan")
		So(err, ShouldBeNil)

		channel := models.CreateTypedGroupedChannelWithTest(account.Id, models.Channel_TYPE_TOPIC, channelIntegration.GroupName)
		_, err = channel.AddParticipant(account.Id)
		So(err, ShouldBeNil)

		for i := 0; i < 99; i++ {
			err = rest.DoPushRequest(newPushRequest(channel.Id, channelIntegration.GroupName), channelIntegration.Token)
			So(err, ShouldBeNil)
		}

		err = rest.DoPushRequest(newPushRequest(channel.Id, channelIntegration.GroupName), channelIntegration.Token)
		So(err, ShouldNotBeNil)

	})

}
<commit_msg>webhook: update integration test runner name<commit_after>package tests

import (
	"koding/db/mongodb/modelhelper"
	"socialapi/config"
	"socialapi/models"
	"socialapi/request"
	"socialapi/rest"
	"socialapi/workers/integration/webhook"
	"socialapi/workers/integration/webhook/api"
	"testing"

	"github.com/koding/integration/services"
	"github.com/koding/runner"
	. "github.com/smartystreets/goconvey/convey"
)

func newMiddlewareRequest(username, groupName string) *services.ServiceInput {
	return &services.ServiceInput{
		"username":  username,
		"eventName": "emailOpen",
		"message":   "testing it",
		"groupName": groupName,
	}
}

func newPushRequest(channelId int64, groupName string) *api.PushRequest {
	wr := &api.PushRequest{
		GroupName: groupName,
	}
	wr.Body = "hey"
	wr.ChannelId = channelId

	return wr
}

func newBotChannelRequest(nick, groupName string) *api.BotChannelRequest {
	return &api.BotChannelRequest{
		GroupName: groupName,
		Username:  nick,
	}
}

func TestWebhook(t *testing.T) {
	r := runner.New("test")
	if err := r.Init(); err != nil {
		t.Fatalf("couldnt start bongo %s", err.Error())
	}
	defer r.Close()

	appConfig := config.MustRead(r.Conf.Path)
	modelhelper.Initialize(appConfig.Mongo)
	defer modelhelper.Close()

	channelIntegration := webhook.CreateTestChannelIntegration(t)

	webhook.CreateIterableIntegration(t)

	Convey("We should be able to successfully push message", t, func() {

		account, err := models.CreateAccountInBothDbsWithNick("sinan")
		So(err, ShouldBeNil)

		channel := models.CreateTypedGroupedChannelWithTest(account.Id, models.Channel_TYPE_TOPIC, channelIntegration.GroupName)
		_, err = channel.AddParticipant(account.Id)
		So(err, ShouldBeNil)

		err = rest.DoPushRequest(newPushRequest(channel.Id, channelIntegration.GroupName), channelIntegration.Token)
		So(err, ShouldBeNil)

		ses, err := models.FetchOrCreateSession(account.Nick, channelIntegration.GroupName)
		So(err, ShouldBeNil)
		So(ses, ShouldNotBeNil)

		resp, err := rest.GetHistory(channel.Id,
			&request.Query{
				AccountId: account.Id,
			},
			ses.ClientId,
		)

		So(err, ShouldBeNil)
		So(len(resp.MessageList), ShouldEqual, 1)
	})

	Convey("We should be able to successfully fetch bot channel of the user", t, func() {
		account, err := models.CreateAccountInBothDbsWithNick("sinan")
		So(err, ShouldBeNil)
		groupName := models.RandomGroupName()
		channel := models.CreateTypedGroupedChannelWithTest(account.Id, models.Channel_TYPE_GROUP, groupName)
		_, err = channel.AddParticipant(account.Id)
		So(err, ShouldBeNil)

		ses, err := models.FetchOrCreateSession(account.Nick, groupName)
		So(err, ShouldBeNil)
		So(ses, ShouldNotBeNil)

		channelId, err := rest.DoBotChannelRequest(ses.ClientId)

		So(err, ShouldBeNil)
		So(channelId, ShouldNotEqual, 0)
	})

	Convey("We should be able to successfully push messages via prepare endpoint", t, func() {

		account, err := models.CreateAccountInBothDbsWithNick(models.RandomName())
		So(err, ShouldBeNil)

		channel := models.CreateTypedGroupedChannelWithTest(account.Id, models.Channel_TYPE_GROUP, channelIntegration.GroupName)
		_, err = channel.AddParticipant(account.Id)

		pr := newMiddlewareRequest("xxx", channelIntegration.GroupName)
		err = rest.DoPrepareRequest(pr, channelIntegration.Token)
		So(err, ShouldNotBeNil)

		pr = newMiddlewareRequest(account.Nick, channelIntegration.GroupName)
		err = rest.DoPrepareRequest(pr, channelIntegration.Token)
		So(err, ShouldBeNil)

		ses, err := models.FetchOrCreateSession(account.Nick, channelIntegration.GroupName)
		So(err, ShouldBeNil)
		So(ses, ShouldNotBeNil)

		channelId, err := rest.DoBotChannelRequest(ses.ClientId)
		So(err, ShouldBeNil)

		resp, err := rest.GetHistory(channelId,
			&request.Query{
				AccountId: account.Id,
			},
			ses.ClientId,
		)

		So(err, ShouldBeNil)
		So(len(resp.MessageList), ShouldEqual, 1)
		So(resp.MessageList[0].Message.Body, ShouldEqual, "testing it")
	})

	Convey("We should not be able to send more than 100 requests per minute", t, func() {

		account, err := models.CreateAccountInBothDbsWithNick("sinan")
		So(err, ShouldBeNil)

		channel := models.CreateTypedGroupedChannelWithTest(account.Id, models.Channel_TYPE_TOPIC, channelIntegration.GroupName)
		_, err = channel.AddParticipant(account.Id)
		So(err, ShouldBeNil)

		for i := 0; i < 99; i++ {
			err = rest.DoPushRequest(newPushRequest(channel.Id, channelIntegration.GroupName), channelIntegration.Token)
			So(err, ShouldBeNil)
		}

		err = rest.DoPushRequest(newPushRequest(channel.Id, channelIntegration.GroupName), channelIntegration.Token)
		So(err, ShouldNotBeNil)

	})

}
<|endoftext|>


<commit_before>package stkque

/*
  Unfortunately without true generics, I have to allow arbitrary Elements into
  the stack. The alternatives to this are:
  1) Provide a stack that allows only one type of Element (like int) which is
     very limiting.
  2) Hand-code stacks for every type: int, rune, byte, string, etc. Well, this
     is exactly the kind of meaningless effort that generics should solve. Also,
     I cannot predict every type that is ever going to come into existence!
*/
type Element interface{}
type Slice []Element

func New() Slice {
	return Slice{}
}

func (stk *Slice) Len() int {
	return len(*stk)
}

func (stk *Slice) Cap() int {
	return cap(*stk)
}

func (stk *Slice) Push(e ...Element) {
	*stk = append(*stk, e...)
}

type Empty Slice
func (empty Empty) Error() string {
	return "Cannot Peek() or Pop() on empty stack"
}

func (stk *Slice) Peekstk() (Element, error) {
	lastindex := len(*stk) - 1
	if lastindex < 0 {
		return nil, Empty(*stk)
	}
	return (*stk)[lastindex], nil
}

func (que *Slice) Peekque() (Element, error) {
	if len(*que) == 0 {
		return nil, Empty(*que)
	}
	return (*que)[0], nil
}

func shrink(slc Slice, fromincl, toexcl int) Slice {
	/*
	  As per Dr. Sedgewick's suggestion, we shrink dynamic arrays to half only
	  when their lens reach a quarter of their caps. This prevents thrashing (
	  assuming append doubles the cap on resizing)
	*/
	if len(slc) <= cap(slc) / 4 {
		newslc := make(Slice, cap(slc) / 2)
		copy(newslc, slc[fromincl:toexcl])
		return newslc[fromincl:toexcl]
	}
	return slc[fromincl:toexcl]
}

func (stk *Slice) Popstk() (Element, error) {
	lastindex := len(*stk) - 1
	if lastindex < 0 {
		return nil, Empty(*stk)
	}

	last := (*stk)[lastindex] // save last value; it won't be available after shrink
	*stk = shrink(*stk, 0, lastindex)
	return last, nil
}

func (que *Slice) Popque() (Element, error) {
	len := len(*que)
	if len == 0 {
		return nil, Empty(*que)
	}

	first := (*que)[0] // save first value; it won't be available after shrink
	*que = shrink(*que, 1, len)
	return first, nil
}<commit_msg>Fix typos and godoc formatting<commit_after>package stkque

/*
  Unfortunately without true generics, I have to allow arbitrary Elements into the stkque. The alternatives to this are:
  1) Provide a stkque that allows only one type of Element (like int) which is very limiting.
  2) Hand-code stkques for every type: int, rune, byte, string, etc. Well, this is exactly the kind of meaningless effort that generics should solve. Also, I cannot predict every type that is ever going to come into existence!
*/
type Element interface{}
type Slice []Element

func New() Slice {
	return Slice{}
}

func (stk *Slice) Len() int {
	return len(*stk)
}

func (stk *Slice) Cap() int {
	return cap(*stk)
}

func (stk *Slice) Push(xs ...Element) {
	*stk = append(*stk, xs...)
}

type Empty Slice
func (empty Empty) Error() string {
	return "Cannot Peek() or Pop() on empty stkque"
}

func (stk *Slice) Peekstk() (Element, error) {
	lastindex := len(*stk) - 1
	if lastindex < 0 {
		return nil, Empty(*stk)
	}
	return (*stk)[lastindex], nil
}

func (que *Slice) Peekque() (Element, error) {
	if len(*que) == 0 {
		return nil, Empty(*que)
	}
	return (*que)[0], nil
}

func shrink(slc Slice, fromincl, toexcl int) Slice {
	/*
	  As per Dr. Sedgewick's suggestion, we shrink dynamic arrays to half only
	  when their lens reach a quarter of their caps. This prevents thrashing (
	  assuming append doubles the cap on resizing)
	*/
	if len(slc) <= cap(slc) / 4 {
		newslc := make(Slice, cap(slc) / 2)
		copy(newslc, slc[fromincl:toexcl])
		return newslc[fromincl:toexcl]
	}
	return slc[fromincl:toexcl]
}

func (stk *Slice) Popstk() (Element, error) {
	lastindex := len(*stk) - 1
	if lastindex < 0 {
		return nil, Empty(*stk)
	}

	last := (*stk)[lastindex] // save last value; it won't be available after shrink
	*stk = shrink(*stk, 0, lastindex)
	return last, nil
}

func (que *Slice) Popque() (Element, error) {
	len := len(*que)
	if len == 0 {
		return nil, Empty(*que)
	}

	first := (*que)[0] // save first value; it won't be available after shrink
	*que = shrink(*que, 1, len)
	return first, nil
}<|endoftext|>


<commit_before>// +build integration

package store

import (
	"io/ioutil"
	"net/http/httptest"
	"os"
	"testing"

	"github.com/tomcz/s3backup/tools"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/johannesboyne/gofakes3"
	"github.com/johannesboyne/gofakes3/backend/s3mem"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestSplitRemotePath(t *testing.T) {
	bucket, objectKey, err := splitRemotePath("s3://bucket/object.key")
	if assert.NoError(t, err) {
		assert.Equal(t, "bucket", bucket)
		assert.Equal(t, "object.key", objectKey)
	}

	bucket, objectKey, err = splitRemotePath("s3://some-bucket/some/path/to/object.foo")
	if assert.NoError(t, err) {
		assert.Equal(t, "some-bucket", bucket)
		assert.Equal(t, "some/path/to/object.foo", objectKey)
	}

	_, _, err = splitRemotePath("http://example.com/wibble.bar")
	assert.Error(t, err)
}

func TestRoundTripUploadDownload(t *testing.T) {
	backend := s3mem.New()
	faker := gofakes3.New(backend)
	ts := httptest.NewServer(faker.Server())
	defer ts.Close()

	expected, err := tools.Random(4096)
	require.NoError(t, err, "Cannot create file contents")

	uploadFile, err := tools.CreateTempFile("upload", expected)
	require.NoError(t, err, "Cannot create file to upload")
	defer os.Remove(uploadFile)

	accessKey := "AKIAIOSFODNN7EXAMPLE"
	secretKey := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
	client, err := NewS3(accessKey, secretKey, "", "us-east-1", ts.URL)
	require.NoError(t, err, "failed to create S3 client")

	store := client.(*s3store)
	_, err = store.client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String("test-bucket")})
	require.NoError(t, err, "failed to create bucket")

	err = client.UploadFile("s3://test-bucket/test-file", uploadFile, "wibble")
	require.NoError(t, err, "failed to upload file")

	downloadFile := uploadFile + ".download"
	checksum, err := client.DownloadFile("s3://test-bucket/test-file", downloadFile)
	require.NoError(t, err, "failed to download file")
	defer os.Remove(downloadFile)

	actual, err := ioutil.ReadFile(downloadFile)
	require.NoError(t, err, "Cannot read downloaded file")

	assert.Equal(t, "wibble", checksum)
	assert.Equal(t, expected, actual, "File contents are different")
}
<commit_msg>ensure that checksum is truly optional<commit_after>// +build integration

package store

import (
	"io/ioutil"
	"net/http/httptest"
	"os"
	"testing"

	"github.com/tomcz/s3backup/tools"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/johannesboyne/gofakes3"
	"github.com/johannesboyne/gofakes3/backend/s3mem"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestSplitRemotePath(t *testing.T) {
	bucket, objectKey, err := splitRemotePath("s3://bucket/object.key")
	if assert.NoError(t, err) {
		assert.Equal(t, "bucket", bucket)
		assert.Equal(t, "object.key", objectKey)
	}

	bucket, objectKey, err = splitRemotePath("s3://some-bucket/some/path/to/object.foo")
	if assert.NoError(t, err) {
		assert.Equal(t, "some-bucket", bucket)
		assert.Equal(t, "some/path/to/object.foo", objectKey)
	}

	_, _, err = splitRemotePath("http://example.com/wibble.bar")
	assert.Error(t, err)
}

func TestRoundTripUploadDownload_withChecksum(t *testing.T) {
	backend := s3mem.New()
	faker := gofakes3.New(backend)
	ts := httptest.NewServer(faker.Server())
	defer ts.Close()

	expected, err := tools.Random(4096)
	require.NoError(t, err, "Cannot create file contents")

	uploadFile, err := tools.CreateTempFile("upload", expected)
	require.NoError(t, err, "Cannot create file to upload")
	defer os.Remove(uploadFile)

	accessKey := "AKIAIOSFODNN7EXAMPLE"
	secretKey := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
	client, err := NewS3(accessKey, secretKey, "", "us-east-1", ts.URL)
	require.NoError(t, err, "failed to create S3 client")

	store := client.(*s3store)
	_, err = store.client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String("test-bucket")})
	require.NoError(t, err, "failed to create bucket")

	err = client.UploadFile("s3://test-bucket/test-file", uploadFile, "wibble")
	require.NoError(t, err, "failed to upload file")

	downloadFile := uploadFile + ".download"
	checksum, err := client.DownloadFile("s3://test-bucket/test-file", downloadFile)
	require.NoError(t, err, "failed to download file")
	defer os.Remove(downloadFile)

	actual, err := ioutil.ReadFile(downloadFile)
	require.NoError(t, err, "Cannot read downloaded file")

	assert.Equal(t, "wibble", checksum)
	assert.Equal(t, expected, actual, "File contents are different")
}

func TestRoundTripUploadDownload_withoutChecksum(t *testing.T) {
	backend := s3mem.New()
	faker := gofakes3.New(backend)
	ts := httptest.NewServer(faker.Server())
	defer ts.Close()

	expected, err := tools.Random(4096)
	require.NoError(t, err, "Cannot create file contents")

	uploadFile, err := tools.CreateTempFile("upload", expected)
	require.NoError(t, err, "Cannot create file to upload")
	defer os.Remove(uploadFile)

	accessKey := "AKIAIOSFODNN7EXAMPLE"
	secretKey := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
	client, err := NewS3(accessKey, secretKey, "", "us-east-1", ts.URL)
	require.NoError(t, err, "failed to create S3 client")

	store := client.(*s3store)
	_, err = store.client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String("test-bucket")})
	require.NoError(t, err, "failed to create bucket")

	err = client.UploadFile("s3://test-bucket/test-file", uploadFile, "")
	require.NoError(t, err, "failed to upload file")

	downloadFile := uploadFile + ".download"
	checksum, err := client.DownloadFile("s3://test-bucket/test-file", downloadFile)
	require.NoError(t, err, "failed to download file")
	defer os.Remove(downloadFile)

	actual, err := ioutil.ReadFile(downloadFile)
	require.NoError(t, err, "Cannot read downloaded file")

	assert.Equal(t, "", checksum)
	assert.Equal(t, expected, actual, "File contents are different")
}
<|endoftext|>


<commit_before>// Copyright 2018, OpenCensus Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package testexecutionservice

import (
	"context"
	"errors"
	"github.com/census-ecosystem/opencensus-experiments/interoptest/src/testcoordinator/genproto"
	"sync"

	"google.golang.org/grpc"

	"go.opencensus.io/tag"
)

// Sender is the type that stores necessary information for making test requests, and sends
// test execution request to each test server.
type Sender struct {
	mu         sync.RWMutex
	startOnces []sync.Once

	canDialInsecure bool

	// The order of reqIds, reqNames and serverAddrs must match.
	reqIds             []int64
	reqNames           []string
	serverAddrs        []string
	registeredServices map[string][]*interop.Service
	tagsForServices    map[string][]*tag.Tag
}

var (
	errAlreadyStarted = errors.New("already started")
	errSizeNotMatch   = errors.New("sizes do not match")
)

// NewUnstartedSender just creates a new Sender.
// TODO: consider using options.
func NewUnstartedSender(
	canDialInsecure bool,
	reqIds []int64,
	reqNames []string,
	serverAddrs []string,
	registeredServices map[string][]*interop.Service,
	tagsForServices map[string][]*tag.Tag) (*Sender, error) {
	if len(reqIds) != len(reqNames) || len(reqIds) != len(serverAddrs) || len(reqIds) != len(registeredServices) {
		return nil, errSizeNotMatch
	}
	startOnces := make([]sync.Once, len(reqIds))
	for i := range reqIds {
		startOnces[i] = sync.Once{}
	}
	s := &Sender{
		canDialInsecure:    canDialInsecure,
		reqIds:             reqIds,
		reqNames:           reqNames,
		serverAddrs:        serverAddrs,
		registeredServices: registeredServices,
		tagsForServices:    tagsForServices,
	}
	return s, nil
}

// Start transforms each request id, request name and Services into a TestRequest.
// Then sends each TestRequest to the corresponding server, and returns the list of responses
// and errors and for each request.
func (s *Sender) Start() ([]*interop.TestResponse, []error) {
	var resps []*interop.TestResponse
	var errs []error
	for i, so := range s.startOnces {
		var resp *interop.TestResponse
		err := errAlreadyStarted
		so.Do(func() {
			s.mu.Lock()
			defer s.mu.Unlock()

			addr := s.serverAddrs[i]
			if cc, err := s.dialToServer(addr); err == nil {
				resp, err = s.send(cc, s.reqIds[i], s.reqNames[i])
			}
		})
		resps = append(resps, resp)
		errs = append(errs, err)
	}
	return resps, errs
}

// TODO: send HTTP TestRequest
func (s *Sender) send(cc *grpc.ClientConn, reqId int64, reqName string) (*interop.TestResponse, error) {
	defer cc.Close()
	services := s.registeredServices[reqName]
	var hops []*interop.ServiceHop
	for _, service := range services {
		hops = append(hops, &interop.ServiceHop{
			Service: service,
			Tags:    toTagsProto(s.tagsForServices[service.Name]),
		})
	}
	req := &interop.TestRequest{
		Id:          reqId,
		Name:        reqName,
		ServiceHops: hops,
	}

	testSvcClient := interop.NewTestExecutionServiceClient(cc)
	return testSvcClient.Test(context.Background(), req)
}

func (s *Sender) dialToServer(addr string) (*grpc.ClientConn, error) {
	var dialOpts []grpc.DialOption
	if s.canDialInsecure {
		dialOpts = append(dialOpts, grpc.WithInsecure())
	}
	return grpc.Dial(addr, dialOpts...)
}

func toTagsProto(tags []*tag.Tag) []*interop.Tag {
	var tagsProto []*interop.Tag
	for _, t := range tags {
		tagsProto = append(tagsProto, &interop.Tag{Key: t.Key.Name(), Value: t.Value})
	}
	return tagsProto
}
<commit_msg>interoptest/tc: send one request at a time instead of batching (#139)<commit_after>// Copyright 2018, OpenCensus Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package testexecutionservice

import (
	"context"
	"errors"
	"github.com/census-ecosystem/opencensus-experiments/interoptest/src/testcoordinator/genproto"
	"sync"

	"google.golang.org/grpc"

	"go.opencensus.io/tag"
)

// Sender is the type that stores necessary information for making test requests, and sends
// test execution request to each test server.
type Sender struct {
	mu        sync.RWMutex
	startOnce sync.Once

	canDialInsecure bool

	reqID              int64
	reqName            string
	serverAddr         string
	registeredServices map[string][]*interop.Service
	tagsForServices    map[string][]*tag.Tag
}

var (
	errAlreadyStarted = errors.New("already started")
	errSizeNotMatch   = errors.New("sizes do not match")
)

// NewUnstartedSender just creates a new Sender.
// TODO: consider using options.
func NewUnstartedSender(
	canDialInsecure bool,
	reqID int64,
	reqName string,
	serverAddr string,
	registeredServices map[string][]*interop.Service,
	tagsForServices map[string][]*tag.Tag) (*Sender, error) {
	s := &Sender{
		canDialInsecure:    canDialInsecure,
		reqID:              reqID,
		reqName:            reqName,
		serverAddr:         serverAddr,
		registeredServices: registeredServices,
		tagsForServices:    tagsForServices,
	}
	return s, nil
}

// Start transforms the request id, request name and Services into a TestRequest.
// Then sends a TestRequest to the corresponding server, and returns the response
// and error.
func (s *Sender) Start() (*interop.TestResponse, error) {
	var resp *interop.TestResponse
	err := errAlreadyStarted
	s.startOnce.Do(func() {
		s.mu.Lock()
		defer s.mu.Unlock()

		addr := s.serverAddr
		if cc, err := s.dialToServer(addr); err == nil {
			resp, err = s.send(cc, s.reqID, s.reqName)
		}
	})
	return resp, err
}

// TODO: send HTTP TestRequest
func (s *Sender) send(cc *grpc.ClientConn, reqID int64, reqName string) (*interop.TestResponse, error) {
	defer cc.Close()
	services := s.registeredServices[reqName]
	var hops []*interop.ServiceHop
	for _, service := range services {
		hops = append(hops, &interop.ServiceHop{
			Service: service,
			Tags:    toTagsProto(s.tagsForServices[service.Name]),
		})
	}
	req := &interop.TestRequest{
		Id:          reqID,
		Name:        reqName,
		ServiceHops: hops,
	}

	testSvcClient := interop.NewTestExecutionServiceClient(cc)
	return testSvcClient.Test(context.Background(), req)
}

func (s *Sender) dialToServer(addr string) (*grpc.ClientConn, error) {
	var dialOpts []grpc.DialOption
	if s.canDialInsecure {
		dialOpts = append(dialOpts, grpc.WithInsecure())
	}
	return grpc.Dial(addr, dialOpts...)
}

func toTagsProto(tags []*tag.Tag) []*interop.Tag {
	var tagsProto []*interop.Tag
	for _, t := range tags {
		tagsProto = append(tagsProto, &interop.Tag{Key: t.Key.Name(), Value: t.Value})
	}
	return tagsProto
}
<|endoftext|>


<commit_before>package http

import (
	"bytes"
	"context"
	"expvar"
	"fmt"
	"io"
	"math"
	"net"
	"net/http"
	"net/url"
	"strconv"
	"strings"

	"github.com/anacrolix/missinggo/httptoo"
	"github.com/anacrolix/torrent/bencode"
	"github.com/anacrolix/torrent/tracker/shared"
	"github.com/anacrolix/torrent/tracker/udp"
	"github.com/anacrolix/torrent/version"
)

var vars = expvar.NewMap("tracker/http")

func setAnnounceParams(_url *url.URL, ar *AnnounceRequest, opts AnnounceOpt) {
	q := url.Values{}

	q.Set("key", strconv.FormatInt(int64(ar.Key), 10))
	q.Set("info_hash", string(ar.InfoHash[:]))
	q.Set("peer_id", string(ar.PeerId[:]))
	// AFAICT, port is mandatory, and there's no implied port key.
	q.Set("port", fmt.Sprintf("%d", ar.Port))
	q.Set("uploaded", strconv.FormatInt(ar.Uploaded, 10))
	q.Set("downloaded", strconv.FormatInt(ar.Downloaded, 10))

	// The AWS S3 tracker returns "400 Bad Request: left(-1) was not in the valid range 0 -
	// 9223372036854775807" if left is out of range, or "500 Internal Server Error: Internal Server
	// Error" if omitted entirely.
	left := ar.Left
	if left < 0 {
		left = math.MaxInt64
	}
	q.Set("left", strconv.FormatInt(left, 10))

	if ar.Event != shared.None {
		q.Set("event", ar.Event.String())
	}
	// http://stackoverflow.com/questions/17418004/why-does-tracker-server-not-understand-my-request-bittorrent-protocol
	q.Set("compact", "1")
	// According to https://wiki.vuze.com/w/Message_Stream_Encryption. TODO:
	// Take EncryptionPolicy or something like it as a parameter.
	q.Set("supportcrypto", "1")
	doIp := func(versionKey string, ip net.IP) {
		if ip == nil {
			return
		}
		ipString := ip.String()
		q.Set(versionKey, ipString)
		// Let's try listing them. BEP 3 mentions having an "ip" param, and BEP 7 says we can list
		// addresses for other address-families, although it's not encouraged.
		q.Add("ip", ipString)
	}
	doIp("ipv4", opts.ClientIp4)
	doIp("ipv6", opts.ClientIp6)
	// We're operating purely on query-escaped strings, where + would have already been encoded to
	// %2B, and + has no other special meaning. See https://github.com/anacrolix/torrent/issues/534.
	qstr := strings.ReplaceAll(q.Encode(), "+", "%20")

	// Some private trackers require the original query param to be in the first position.
	if _url.RawQuery != "" {
		_url.RawQuery += "&" + qstr
	} else {
		_url.RawQuery = qstr
	}
}

type AnnounceOpt struct {
	UserAgent  string
	HostHeader string
	ClientIp4  net.IP
	ClientIp6  net.IP
}

type AnnounceRequest = udp.AnnounceRequest

func (cl Client) Announce(ctx context.Context, ar AnnounceRequest, opt AnnounceOpt) (ret AnnounceResponse, err error) {
	_url := httptoo.CopyURL(cl.url_)
	setAnnounceParams(_url, &ar, opt)
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, _url.String(), nil)
	userAgent := opt.UserAgent
	if userAgent == "" {
		userAgent = version.DefaultHttpUserAgent
	}
	if userAgent != "" {
		req.Header.Set("User-Agent", userAgent)
	}
	req.Host = opt.HostHeader
	resp, err := cl.hc.Do(req)
	if err != nil {
		return
	}
	defer resp.Body.Close()
	var buf bytes.Buffer
	io.Copy(&buf, resp.Body)
	if resp.StatusCode != 200 {
		err = fmt.Errorf("response from tracker: %s: %s", resp.Status, buf.String())
		return
	}
	var trackerResponse HttpResponse
	err = bencode.Unmarshal(buf.Bytes(), &trackerResponse)
	if _, ok := err.(bencode.ErrUnusedTrailingBytes); ok {
		err = nil
	} else if err != nil {
		err = fmt.Errorf("error decoding %q: %s", buf.Bytes(), err)
		return
	}
	if trackerResponse.FailureReason != "" {
		err = fmt.Errorf("tracker gave failure reason: %q", trackerResponse.FailureReason)
		return
	}
	vars.Add("successful http announces", 1)
	ret.Interval = trackerResponse.Interval
	ret.Leechers = trackerResponse.Incomplete
	ret.Seeders = trackerResponse.Complete
	if len(trackerResponse.Peers) != 0 {
		vars.Add("http responses with nonempty peers key", 1)
	}
	ret.Peers = trackerResponse.Peers
	if len(trackerResponse.Peers6) != 0 {
		vars.Add("http responses with nonempty peers6 key", 1)
	}
	for _, na := range trackerResponse.Peers6 {
		ret.Peers = append(ret.Peers, Peer{
			IP:   na.IP,
			Port: na.Port,
		})
	}
	return
}

type AnnounceResponse struct {
	Interval int32 // Minimum seconds the local peer should wait before next announce.
	Leechers int32
	Seeders  int32
	Peers    []Peer
}
<commit_msg>Quote http tracker error response body<commit_after>package http

import (
	"bytes"
	"context"
	"expvar"
	"fmt"
	"io"
	"math"
	"net"
	"net/http"
	"net/url"
	"strconv"
	"strings"

	"github.com/anacrolix/missinggo/httptoo"
	"github.com/anacrolix/torrent/bencode"
	"github.com/anacrolix/torrent/tracker/shared"
	"github.com/anacrolix/torrent/tracker/udp"
	"github.com/anacrolix/torrent/version"
)

var vars = expvar.NewMap("tracker/http")

func setAnnounceParams(_url *url.URL, ar *AnnounceRequest, opts AnnounceOpt) {
	q := url.Values{}

	q.Set("key", strconv.FormatInt(int64(ar.Key), 10))
	q.Set("info_hash", string(ar.InfoHash[:]))
	q.Set("peer_id", string(ar.PeerId[:]))
	// AFAICT, port is mandatory, and there's no implied port key.
	q.Set("port", fmt.Sprintf("%d", ar.Port))
	q.Set("uploaded", strconv.FormatInt(ar.Uploaded, 10))
	q.Set("downloaded", strconv.FormatInt(ar.Downloaded, 10))

	// The AWS S3 tracker returns "400 Bad Request: left(-1) was not in the valid range 0 -
	// 9223372036854775807" if left is out of range, or "500 Internal Server Error: Internal Server
	// Error" if omitted entirely.
	left := ar.Left
	if left < 0 {
		left = math.MaxInt64
	}
	q.Set("left", strconv.FormatInt(left, 10))

	if ar.Event != shared.None {
		q.Set("event", ar.Event.String())
	}
	// http://stackoverflow.com/questions/17418004/why-does-tracker-server-not-understand-my-request-bittorrent-protocol
	q.Set("compact", "1")
	// According to https://wiki.vuze.com/w/Message_Stream_Encryption. TODO:
	// Take EncryptionPolicy or something like it as a parameter.
	q.Set("supportcrypto", "1")
	doIp := func(versionKey string, ip net.IP) {
		if ip == nil {
			return
		}
		ipString := ip.String()
		q.Set(versionKey, ipString)
		// Let's try listing them. BEP 3 mentions having an "ip" param, and BEP 7 says we can list
		// addresses for other address-families, although it's not encouraged.
		q.Add("ip", ipString)
	}
	doIp("ipv4", opts.ClientIp4)
	doIp("ipv6", opts.ClientIp6)
	// We're operating purely on query-escaped strings, where + would have already been encoded to
	// %2B, and + has no other special meaning. See https://github.com/anacrolix/torrent/issues/534.
	qstr := strings.ReplaceAll(q.Encode(), "+", "%20")

	// Some private trackers require the original query param to be in the first position.
	if _url.RawQuery != "" {
		_url.RawQuery += "&" + qstr
	} else {
		_url.RawQuery = qstr
	}
}

type AnnounceOpt struct {
	UserAgent  string
	HostHeader string
	ClientIp4  net.IP
	ClientIp6  net.IP
}

type AnnounceRequest = udp.AnnounceRequest

func (cl Client) Announce(ctx context.Context, ar AnnounceRequest, opt AnnounceOpt) (ret AnnounceResponse, err error) {
	_url := httptoo.CopyURL(cl.url_)
	setAnnounceParams(_url, &ar, opt)
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, _url.String(), nil)
	userAgent := opt.UserAgent
	if userAgent == "" {
		userAgent = version.DefaultHttpUserAgent
	}
	if userAgent != "" {
		req.Header.Set("User-Agent", userAgent)
	}
	req.Host = opt.HostHeader
	resp, err := cl.hc.Do(req)
	if err != nil {
		return
	}
	defer resp.Body.Close()
	var buf bytes.Buffer
	io.Copy(&buf, resp.Body)
	if resp.StatusCode != 200 {
		err = fmt.Errorf("response from tracker: %s: %q", resp.Status, buf.Bytes())
		return
	}
	var trackerResponse HttpResponse
	err = bencode.Unmarshal(buf.Bytes(), &trackerResponse)
	if _, ok := err.(bencode.ErrUnusedTrailingBytes); ok {
		err = nil
	} else if err != nil {
		err = fmt.Errorf("error decoding %q: %s", buf.Bytes(), err)
		return
	}
	if trackerResponse.FailureReason != "" {
		err = fmt.Errorf("tracker gave failure reason: %q", trackerResponse.FailureReason)
		return
	}
	vars.Add("successful http announces", 1)
	ret.Interval = trackerResponse.Interval
	ret.Leechers = trackerResponse.Incomplete
	ret.Seeders = trackerResponse.Complete
	if len(trackerResponse.Peers) != 0 {
		vars.Add("http responses with nonempty peers key", 1)
	}
	ret.Peers = trackerResponse.Peers
	if len(trackerResponse.Peers6) != 0 {
		vars.Add("http responses with nonempty peers6 key", 1)
	}
	for _, na := range trackerResponse.Peers6 {
		ret.Peers = append(ret.Peers, Peer{
			IP:   na.IP,
			Port: na.Port,
		})
	}
	return
}

type AnnounceResponse struct {
	Interval int32 // Minimum seconds the local peer should wait before next announce.
	Leechers int32
	Seeders  int32
	Peers    []Peer
}
<|endoftext|>


<commit_before>package superast

import (
	"encoding/json"
	"go/ast"
	"go/parser"
	"go/token"
	"io"
	"io/ioutil"
	"os"
	"path"
	"testing"
)

func toJSON(t *testing.T, a *AST) []byte {
	b, err := json.MarshalIndent(a.RootBlock, "", "  ")
	if err != nil {
		t.Errorf("Could not generate JSON from AST: %s", err)
	}
	b = append(b, '\n')
	return b
}

func doTest(t *testing.T, name string, in, out io.Reader) {
	fset := token.NewFileSet()
	f, err := parser.ParseFile(fset, name+".go", in, 0)
	if err != nil {
		t.Errorf("Failed parsing source file: %s", err)
	}
	a := NewAST(fset)
	ast.Walk(a, f)
	jsonWant, err := ioutil.ReadAll(out)
	if err != nil {
		t.Errorf("Failed reading json file: %s", err)
	}
	if string(jsonWant) != string(toJSON(t, a)) {
		t.Errorf("Mismatching JSON outputs in the test '%s'", name)
	}
}

const testsDir = "tests"

func TestCases(t *testing.T) {
	entries, err := ioutil.ReadDir(testsDir)
	if err != nil {
		return
	}
	for _, e := range entries {
		if !e.IsDir() {
			continue
		}
		name := e.Name()
		in, err := os.Open(path.Join(testsDir, name, name+".go"))
		if err != nil {
			t.Errorf("Failed opening file: %s", err)
		}
		out, err := os.Open(path.Join(testsDir, name, name+".json"))
		if err != nil {
			t.Errorf("Failed opening file: %s", err)
		}
		doTest(t, name, in, out)
	}
}
<commit_msg>Add test -write flag to update json files<commit_after>package superast

import (
	"encoding/json"
	"go/ast"
	"go/parser"
	"go/token"
	"io/ioutil"
	"os"
	"path"
	"testing"
	"flag"
)

var write = flag.Bool("write", false, "Write json results")

func init() {
	flag.Parse()
}

func toJSON(t *testing.T, a *AST) []byte {
	b, err := json.MarshalIndent(a.RootBlock, "", "  ")
	if err != nil {
		t.Errorf("Could not generate JSON from AST: %s", err)
	}
	b = append(b, '\n')
	return b
}

const testsDir = "tests"

func doTest(t *testing.T, name string) {
	fset := token.NewFileSet()
	in, err := os.Open(path.Join(testsDir, name, name+".go"))
	if err != nil {
		t.Errorf("Failed opening file: %s", err)
	}
	f, err := parser.ParseFile(fset, name+".go", in, 0)
	if err != nil {
		t.Errorf("Failed parsing source file: %s", err)
	}
	a := NewAST(fset)
	ast.Walk(a, f)
	got := toJSON(t, a)
	outPath := path.Join(testsDir, name, name+".json")
	if *write {
		out, err := os.Create(outPath)
		if err != nil {
			t.Errorf("Failed opening file: %s", err)
		}
		_, err = out.Write(got)
		if err != nil {
			t.Errorf("Failed writing json file: %s", err)
		}
	} else {
		out, err := os.Open(outPath)
		if err != nil {
			t.Errorf("Failed opening file: %s", err)
		}
		want, err := ioutil.ReadAll(out)
		if err != nil {
			t.Errorf("Failed reading json file: %s", err)
		}
		if string(want) != string(got) {
			t.Errorf("Mismatching JSON outputs in the test '%s'", name)
		}
	}
}

func TestCases(t *testing.T) {
	entries, err := ioutil.ReadDir(testsDir)
	if err != nil {
		return
	}
	for _, e := range entries {
		if !e.IsDir() {
			continue
		}
		doTest(t, e.Name())
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"image/color"
	"os"
	"path"

	"github.com/MJKWoolnough/byteio"
	"github.com/MJKWoolnough/minecraft"
	"github.com/MJKWoolnough/ora"
)

func (t Transfer) generate(name string, r *byteio.StickyReader, w *byteio.StickyWriter, f *os.File, size int64) error {
	o, err := ora.Open(f, size)
	if err != nil {
		return err
	}
	mp := t.c.NewMap()
	if mp == nil {
		return errors.New("failed to create map")
	}

	done := false
	defer func() {
		if !done {
			t.c.RemoveMap(mp.ID)
		}
		go t.c.Save()
	}()

	mp.Lock()
	mp.Name = name
	mapPath := mp.Path
	mp.Server = -2
	mp.Unlock()

	ms := DefaultMapSettings()
	ms["level-type"] = minecraft.FlatGenerator
	ms["generator-settings"] = "0"
	ms["motd"] = name

	pf, err := os.Create(path.Join(mapPath, "properties.map"))
	if err != nil {
		return err
	}

	if err = ms.WriteTo(pf); err != nil {
		return err
	}
	pf.Close()

	b := o.Bounds()
	w.WriteUint8(2)
	w.WriteInt32(int32(b.Max.X) >> 4)
	w.WriteInt32(int32(b.Max.Y) >> 4)

	gNames := t.c.Generators.Names()
	var gID int16
	if len(gNames) == 0 {
		return errors.New("no generators installed")
	} else if len(gNames) == 1 {
		gID = 0
	} else {
		w.WriteUint8(1)
		w.WriteInt16(int16(len(gNames)))
		for _, gName := range gNames {
			writeString(w, gName)
		}
		if w.Err != nil {
			return w.Err
		}
		gID = r.ReadInt16()
		if gID < 0 || int(gID) >= len(gNames) {
			return errors.New("unknown generator")
		}
	}

	g := t.c.Generators.Get(gNames[gID])

	c := make(chan paint, 1024)
	m := make(chan string, 4)
	e := make(chan struct{}, 0)
	defer close(e)
	go func() {
		defer close(c)
		defer close(m)
		for {
			select {
			case message := <-m:
				w.WriteUint8(3)
				writeString(w, message)
			case p := <-c:
				w.WriteUint8(4)
				w.WriteInt32(p.X)
				w.WriteInt32(p.Y)
				r, g, b, a := p.RGBA()
				w.WriteUint8(uint8(r >> 8))
				w.WriteUint8(uint8(g >> 8))
				w.WriteUint8(uint8(b >> 8))
				w.WriteUint8(uint8(a >> 8))
			case <-e:
				return
			}
		}
	}()

	if err = g.Generate(name, mapPath, o, c, m); err != nil {
		return err
	}

	done = true
	mp.Lock()
	mp.Server = -1
	mp.Unlock()

	return nil
}

type paint struct {
	color.Color
	X, Y int32
}
<commit_msg>Added generate nil check incase generator was removed<commit_after>package main

import (
	"errors"
	"image/color"
	"os"
	"path"

	"github.com/MJKWoolnough/byteio"
	"github.com/MJKWoolnough/minecraft"
	"github.com/MJKWoolnough/ora"
)

func (t Transfer) generate(name string, r *byteio.StickyReader, w *byteio.StickyWriter, f *os.File, size int64) error {
	o, err := ora.Open(f, size)
	if err != nil {
		return err
	}
	mp := t.c.NewMap()
	if mp == nil {
		return errors.New("failed to create map")
	}

	done := false
	defer func() {
		if !done {
			t.c.RemoveMap(mp.ID)
		}
		go t.c.Save()
	}()

	mp.Lock()
	mp.Name = name
	mapPath := mp.Path
	mp.Server = -2
	mp.Unlock()

	ms := DefaultMapSettings()
	ms["level-type"] = minecraft.FlatGenerator
	ms["generator-settings"] = "0"
	ms["motd"] = name

	pf, err := os.Create(path.Join(mapPath, "properties.map"))
	if err != nil {
		return err
	}

	if err = ms.WriteTo(pf); err != nil {
		return err
	}
	pf.Close()

	b := o.Bounds()
	w.WriteUint8(2)
	w.WriteInt32(int32(b.Max.X) >> 4)
	w.WriteInt32(int32(b.Max.Y) >> 4)

	gNames := t.c.Generators.Names()
	var gID int16
	if len(gNames) == 0 {
		return errors.New("no generators installed")
	} else if len(gNames) == 1 {
		gID = 0
	} else {
		w.WriteUint8(1)
		w.WriteInt16(int16(len(gNames)))
		for _, gName := range gNames {
			writeString(w, gName)
		}
		if w.Err != nil {
			return w.Err
		}
		gID = r.ReadInt16()
		if gID < 0 || int(gID) >= len(gNames) {
			return errors.New("unknown generator")
		}
	}

	g := t.c.Generators.Get(gNames[gID])
	if g == nil {
		return errors.New("generator removed")
	}

	c := make(chan paint, 1024)
	m := make(chan string, 4)
	e := make(chan struct{}, 0)
	defer close(e)
	go func() {
		defer close(c)
		defer close(m)
		for {
			select {
			case message := <-m:
				w.WriteUint8(3)
				writeString(w, message)
			case p := <-c:
				w.WriteUint8(4)
				w.WriteInt32(p.X)
				w.WriteInt32(p.Y)
				r, g, b, a := p.RGBA()
				w.WriteUint8(uint8(r >> 8))
				w.WriteUint8(uint8(g >> 8))
				w.WriteUint8(uint8(b >> 8))
				w.WriteUint8(uint8(a >> 8))
			case <-e:
				return
			}
		}
	}()

	if err = g.Generate(name, mapPath, o, c, m); err != nil {
		return err
	}

	done = true
	mp.Lock()
	mp.Server = -1
	mp.Unlock()

	return nil
}

type paint struct {
	color.Color
	X, Y int32
}
<|endoftext|>