<commit_before>// Copyright 2014 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"sync"

	"github.com/tsuru/docker-cluster/cluster"
	"github.com/tsuru/tsuru/action"
	"github.com/tsuru/tsuru/app"
	tsuruErrors "github.com/tsuru/tsuru/errors"
	"github.com/tsuru/tsuru/log"
	"github.com/tsuru/tsuru/provision"
	"gopkg.in/mgo.v2/bson"
)

var (
	appDBMutex sync.Mutex
	logMutex   sync.Mutex
)

type progressLog struct {
	Message string
}

type appLocker struct {
	refCount map[string]int
}

func (l *appLocker) lock(appName string) bool {
	appDBMutex.Lock()
	defer appDBMutex.Unlock()
	if l.refCount == nil {
		l.refCount = make(map[string]int)
	}
	if l.refCount[appName] > 0 {
		l.refCount[appName]++
		return true
	}
	ok, err := app.AcquireApplicationLock(appName, app.InternalAppName, "container-move")
	if err != nil || !ok {
		return false
	}
	l.refCount[appName]++
	return true
}

func (l *appLocker) unlock(appName string) {
	appDBMutex.Lock()
	defer appDBMutex.Unlock()
	if l.refCount == nil {
		return
	}
	l.refCount[appName]--
	if l.refCount[appName] <= 0 {
		l.refCount[appName] = 0
		app.ReleaseApplicationLock(appName)
	}
}

var containerMovementErr = errors.New("Error moving some containers.")

func logProgress(encoder *json.Encoder, format string, params ...interface{}) {
	logMutex.Lock()
	defer logMutex.Unlock()
	encoder.Encode(progressLog{Message: fmt.Sprintf(format, params...)})
}

func handleMoveErrors(moveErrors chan error, encoder *json.Encoder) error {
	hasError := false
	for err := range moveErrors {
		errMsg := fmt.Sprintf("Error moving container: %s", err.Error())
		log.Errorf(errMsg)
		logProgress(encoder, errMsg)
		hasError = true
	}
	if hasError {
		return containerMovementErr
	}
	return nil
}

func runReplaceUnitsPipeline(w io.Writer, a provision.App, toRemoveContainers []container, toHosts ...string) ([]container, error) {
	var toHost string
	if len(toHosts) > 0 {
		toHost = toHosts[0]
	}
	if w == nil {
		w = ioutil.Discard
	}
	args := changeUnitsPipelineArgs{
		app:        a,
		toRemove:   toRemoveContainers,
		unitsToAdd: len(toRemoveContainers),
		toHost:     toHost,
		writer:     w,
	}
	pipeline := action.NewPipeline(
		&provisionAddUnitsToHost,
		&addNewRoutes,
		&removeOldRoutes,
		&provisionRemoveOldUnits,
	)
	err := pipeline.Execute(args)
	if err != nil {
		return nil, err
	}
	err = removeImage(assembleImageName(a.GetName()))
	if err != nil {
		log.Debugf("Ignored error removing old images: %s", err.Error())
	}
	return pipeline.Result().([]container), nil
}

func runCreateUnitsPipeline(w io.Writer, a provision.App, toAddCount int) ([]container, error) {
	if w == nil {
		w = ioutil.Discard
	}
	args := changeUnitsPipelineArgs{
		app:        a,
		unitsToAdd: toAddCount,
		writer:     w,
	}
	pipeline := action.NewPipeline(
		&provisionAddUnitsToHost,
		&addNewRoutes,
	)
	err := pipeline.Execute(args)
	if err != nil {
		return nil, err
	}
	return pipeline.Result().([]container), nil
}

func moveOneContainer(c container, toHost string, errors chan error, wg *sync.WaitGroup, encoder *json.Encoder, locker *appLocker) container {
	if wg != nil {
		defer wg.Done()
	}
	locked := locker.lock(c.AppName)
	if !locked {
		errors <- fmt.Errorf("Couldn't move %s, unable to lock %q.", c.ID, c.AppName)
		return container{}
	}
	defer locker.unlock(c.AppName)
	a, err := app.GetByName(c.AppName)
	if err != nil {
		errors <- &tsuruErrors.CompositeError{
			Base:    err,
			Message: fmt.Sprintf("Error getting app %q for unit %s.", c.AppName, c.ID),
		}
		return container{}
	}
	logProgress(encoder, "Moving unit %s for %q: %s -> %s...", c.ID, c.AppName, c.HostAddr, toHost)
	addedContainers, err := runReplaceUnitsPipeline(nil, a, []container{c}, toHost)
	if err != nil {
		errors <- &tsuruErrors.CompositeError{
			Base:    err,
			Message: fmt.Sprintf("Error moving unit %s.", c.ID),
		}
		return container{}
	}
	logProgress(encoder, "Finished moving unit %s for %q.", c.ID, c.AppName)
	addedUnit := addedContainers[0].asUnit(a)
	err = a.BindUnit(&addedUnit)
	if err != nil {
		errors <- &tsuruErrors.CompositeError{
			Base:    err,
			Message: fmt.Sprintf("Error binding unit %s to service instances.", c.ID),
		}
		return container{}
	}
	logProgress(encoder, "Moved unit %s -> %s for %s with bindings.", c.ID, addedUnit.Name, c.AppName)
	return addedContainers[0]
}

func moveContainer(contId string, toHost string, encoder *json.Encoder) (container, error) {
	cont, err := getContainer(contId)
	if err != nil {
		return container{}, err
	}
	wg := sync.WaitGroup{}
	wg.Add(1)
	moveErrors := make(chan error, 1)
	locker := &appLocker{}
	createdContainer := moveOneContainer(*cont, toHost, moveErrors, &wg, encoder, locker)
	close(moveErrors)
	return createdContainer, handleMoveErrors(moveErrors, encoder)
}

func moveContainers(fromHost, toHost string, encoder *json.Encoder) error {
	containers, err := listContainersByHost(fromHost)
	if err != nil {
		return err
	}
	numberContainers := len(containers)
	if numberContainers == 0 {
		logProgress(encoder, "No units to move in %s.", fromHost)
		return nil
	}
	logProgress(encoder, "Moving %d units...", numberContainers)
	locker := &appLocker{}
	moveErrors := make(chan error, numberContainers)
	wg := sync.WaitGroup{}
	wg.Add(numberContainers)
	for _, c := range containers {
		go moveOneContainer(c, toHost, moveErrors, &wg, encoder, locker)
	}
	go func() {
		wg.Wait()
		close(moveErrors)
	}()
	return handleMoveErrors(moveErrors, encoder)
}

type hostWithContainers struct {
	HostAddr   string `bson:"_id"`
	Count      int
	Containers []container
}

func minCountHost(hosts []hostWithContainers) *hostWithContainers {
	var minCountHost *hostWithContainers
	minCount := math.MaxInt32
	for i, dest := range hosts {
		if dest.Count < minCount {
			minCount = dest.Count
			minCountHost = &hosts[i]
		}
	}
	return minCountHost
}

func rebalanceContainers(encoder *json.Encoder, dryRun bool) error {
	coll := collection()
	defer coll.Close()
	fullDocQuery := bson.M{
		// Could use $$ROOT instead of repeating fields but only in Mongo 2.6+.
		"_id":      "$_id",
		"id":       "$id",
		"name":     "$name",
		"appname":  "$appname",
		"type":     "$type",
		"ip":       "$ip",
		"image":    "$image",
		"hostaddr": "$hostaddr",
		"hostport": "$hostport",
		"status":   "$status",
		"version":  "$version",
	}
	appsPipe := coll.Pipe([]bson.M{
		{"$group": bson.M{"_id": "$appname", "count": bson.M{"$sum": 1}}},
	})
	var appsInfo []struct {
		Name  string `bson:"_id"`
		Count int
	}
	err := appsPipe.All(&appsInfo)
	if err != nil {
		return err
	}
	clusterInstance := dockerCluster()
	for _, appInfo := range appsInfo {
		if appInfo.Count < 2 {
			continue
		}
		logProgress(encoder, "Rebalancing app %q (%d units)...", appInfo.Name, appInfo.Count)
		var possibleDests []cluster.Node
		if isSegregateScheduler() {
			possibleDests, err = nodesForAppName(clusterInstance, appInfo.Name)
		} else {
			possibleDests, err = clusterInstance.Nodes()
		}
		if err != nil {
			return err
		}
		maxContPerUnit := appInfo.Count / len(possibleDests)
		overflowHosts := appInfo.Count % len(possibleDests)
		pipe := coll.Pipe([]bson.M{
			{"$match": bson.M{"hostaddr": bson.M{"$ne": ""}, "appname": appInfo.Name}},
			{"$group": bson.M{
				"_id":        "$hostaddr",
				"count":      bson.M{"$sum": 1},
				"containers": bson.M{"$push": fullDocQuery}}},
		})
		var hosts []hostWithContainers
		hostsSet := make(map[string]bool)
		err = pipe.All(&hosts)
		if err != nil {
			return err
		}
		for _, host := range hosts {
			hostsSet[host.HostAddr] = true
		}
		for _, node := range possibleDests {
			hostAddr := urlToHost(node.Address)
			_, present := hostsSet[hostAddr]
			if !present {
				hosts = append(hosts, hostWithContainers{HostAddr: hostAddr})
			}
		}
		anythingDone := false
		for _, host := range hosts {
			toMoveCount := host.Count - maxContPerUnit
			if toMoveCount <= 0 {
				continue
			}
			if overflowHosts > 0 {
				overflowHosts--
				toMoveCount--
				if toMoveCount <= 0 {
					continue
				}
			}
			anythingDone = true
			logProgress(encoder, "Trying to move %d units for %q from %s...", toMoveCount, appInfo.Name, host.HostAddr)
			locker := &appLocker{}
			wg := sync.WaitGroup{}
			moveErrors := make(chan error, toMoveCount)
			for _, cont := range host.Containers {
				minDest := minCountHost(hosts)
				if minDest.Count < maxContPerUnit {
					toMoveCount--
					minDest.Count++
					if dryRun {
						logProgress(encoder, "Would move unit %s for %q: %s -> %s...", cont.ID, appInfo.Name, cont.HostAddr, minDest.HostAddr)
					} else {
						wg.Add(1)
						go moveOneContainer(cont, minDest.HostAddr, moveErrors, &wg, encoder, locker)
					}
				}
				if toMoveCount == 0 {
					break
				}
			}
			if toMoveCount > 0 {
				logProgress(encoder, "Couldn't find suitable destination for %d units for app %q", toMoveCount, appInfo.Name)
			}
			go func() {
				wg.Wait()
				close(moveErrors)
			}()
			err := handleMoveErrors(moveErrors, encoder)
			if err != nil {
				return err
			}
		}
		if anythingDone {
			logProgress(encoder, "Rebalance finished for %q", appInfo.Name)
		} else {
			logProgress(encoder, "Nothing to do for %q", appInfo.Name)
		}
	}
	return nil
}
<commit_msg>provision/docker: containers move log without interpolation<commit_after>// Copyright 2014 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"sync"

	"github.com/tsuru/docker-cluster/cluster"
	"github.com/tsuru/tsuru/action"
	"github.com/tsuru/tsuru/app"
	tsuruErrors "github.com/tsuru/tsuru/errors"
	"github.com/tsuru/tsuru/log"
	"github.com/tsuru/tsuru/provision"
	"gopkg.in/mgo.v2/bson"
)

var (
	appDBMutex sync.Mutex
	logMutex   sync.Mutex
)

type progressLog struct {
	Message string
}

type appLocker struct {
	refCount map[string]int
}

func (l *appLocker) lock(appName string) bool {
	appDBMutex.Lock()
	defer appDBMutex.Unlock()
	if l.refCount == nil {
		l.refCount = make(map[string]int)
	}
	if l.refCount[appName] > 0 {
		l.refCount[appName]++
		return true
	}
	ok, err := app.AcquireApplicationLock(appName, app.InternalAppName, "container-move")
	if err != nil || !ok {
		return false
	}
	l.refCount[appName]++
	return true
}

func (l *appLocker) unlock(appName string) {
	appDBMutex.Lock()
	defer appDBMutex.Unlock()
	if l.refCount == nil {
		return
	}
	l.refCount[appName]--
	if l.refCount[appName] <= 0 {
		l.refCount[appName] = 0
		app.ReleaseApplicationLock(appName)
	}
}

var containerMovementErr = errors.New("Error moving some containers.")

func logProgress(encoder *json.Encoder, format string, params ...interface{}) {
	logMutex.Lock()
	defer logMutex.Unlock()
	encoder.Encode(progressLog{Message: fmt.Sprintf(format, params...)})
}

func handleMoveErrors(moveErrors chan error, encoder *json.Encoder) error {
	hasError := false
	for err := range moveErrors {
		errMsg := fmt.Sprintf("Error moving container: %s", err.Error())
		log.Error(errMsg)
		logProgress(encoder, "%s", errMsg)
		hasError = true
	}
	if hasError {
		return containerMovementErr
	}
	return nil
}

func runReplaceUnitsPipeline(w io.Writer, a provision.App, toRemoveContainers []container, toHosts ...string) ([]container, error) {
	var toHost string
	if len(toHosts) > 0 {
		toHost = toHosts[0]
	}
	if w == nil {
		w = ioutil.Discard
	}
	args := changeUnitsPipelineArgs{
		app:        a,
		toRemove:   toRemoveContainers,
		unitsToAdd: len(toRemoveContainers),
		toHost:     toHost,
		writer:     w,
	}
	pipeline := action.NewPipeline(
		&provisionAddUnitsToHost,
		&addNewRoutes,
		&removeOldRoutes,
		&provisionRemoveOldUnits,
	)
	err := pipeline.Execute(args)
	if err != nil {
		return nil, err
	}
	err = removeImage(assembleImageName(a.GetName()))
	if err != nil {
		log.Debugf("Ignored error removing old images: %s", err.Error())
	}
	return pipeline.Result().([]container), nil
}

func runCreateUnitsPipeline(w io.Writer, a provision.App, toAddCount int) ([]container, error) {
	if w == nil {
		w = ioutil.Discard
	}
	args := changeUnitsPipelineArgs{
		app:        a,
		unitsToAdd: toAddCount,
		writer:     w,
	}
	pipeline := action.NewPipeline(
		&provisionAddUnitsToHost,
		&addNewRoutes,
	)
	err := pipeline.Execute(args)
	if err != nil {
		return nil, err
	}
	return pipeline.Result().([]container), nil
}

func moveOneContainer(c container, toHost string, errors chan error, wg *sync.WaitGroup, encoder *json.Encoder, locker *appLocker) container {
	if wg != nil {
		defer wg.Done()
	}
	locked := locker.lock(c.AppName)
	if !locked {
		errors <- fmt.Errorf("Couldn't move %s, unable to lock %q.", c.ID, c.AppName)
		return container{}
	}
	defer locker.unlock(c.AppName)
	a, err := app.GetByName(c.AppName)
	if err != nil {
		errors <- &tsuruErrors.CompositeError{
			Base:    err,
			Message: fmt.Sprintf("Error getting app %q for unit %s.", c.AppName, c.ID),
		}
		return container{}
	}
	logProgress(encoder, "Moving unit %s for %q: %s -> %s...", c.ID, c.AppName, c.HostAddr, toHost)
	addedContainers, err := runReplaceUnitsPipeline(nil, a, []container{c}, toHost)
	if err != nil {
		errors <- &tsuruErrors.CompositeError{
			Base:    err,
			Message: fmt.Sprintf("Error moving unit %s.", c.ID),
		}
		return container{}
	}
	logProgress(encoder, "Finished moving unit %s for %q.", c.ID, c.AppName)
	addedUnit := addedContainers[0].asUnit(a)
	err = a.BindUnit(&addedUnit)
	if err != nil {
		errors <- &tsuruErrors.CompositeError{
			Base:    err,
			Message: fmt.Sprintf("Error binding unit %s to service instances.", c.ID),
		}
		return container{}
	}
	logProgress(encoder, "Moved unit %s -> %s for %s with bindings.", c.ID, addedUnit.Name, c.AppName)
	return addedContainers[0]
}

func moveContainer(contId string, toHost string, encoder *json.Encoder) (container, error) {
	cont, err := getContainer(contId)
	if err != nil {
		return container{}, err
	}
	wg := sync.WaitGroup{}
	wg.Add(1)
	moveErrors := make(chan error, 1)
	locker := &appLocker{}
	createdContainer := moveOneContainer(*cont, toHost, moveErrors, &wg, encoder, locker)
	close(moveErrors)
	return createdContainer, handleMoveErrors(moveErrors, encoder)
}

func moveContainers(fromHost, toHost string, encoder *json.Encoder) error {
	containers, err := listContainersByHost(fromHost)
	if err != nil {
		return err
	}
	numberContainers := len(containers)
	if numberContainers == 0 {
		logProgress(encoder, "No units to move in %s.", fromHost)
		return nil
	}
	logProgress(encoder, "Moving %d units...", numberContainers)
	locker := &appLocker{}
	moveErrors := make(chan error, numberContainers)
	wg := sync.WaitGroup{}
	wg.Add(numberContainers)
	for _, c := range containers {
		go moveOneContainer(c, toHost, moveErrors, &wg, encoder, locker)
	}
	go func() {
		wg.Wait()
		close(moveErrors)
	}()
	return handleMoveErrors(moveErrors, encoder)
}

type hostWithContainers struct {
	HostAddr   string `bson:"_id"`
	Count      int
	Containers []container
}

func minCountHost(hosts []hostWithContainers) *hostWithContainers {
	var minCountHost *hostWithContainers
	minCount := math.MaxInt32
	for i, dest := range hosts {
		if dest.Count < minCount {
			minCount = dest.Count
			minCountHost = &hosts[i]
		}
	}
	return minCountHost
}

func rebalanceContainers(encoder *json.Encoder, dryRun bool) error {
	coll := collection()
	defer coll.Close()
	fullDocQuery := bson.M{
		// Could use $$ROOT instead of repeating fields but only in Mongo 2.6+.
		"_id":      "$_id",
		"id":       "$id",
		"name":     "$name",
		"appname":  "$appname",
		"type":     "$type",
		"ip":       "$ip",
		"image":    "$image",
		"hostaddr": "$hostaddr",
		"hostport": "$hostport",
		"status":   "$status",
		"version":  "$version",
	}
	appsPipe := coll.Pipe([]bson.M{
		{"$group": bson.M{"_id": "$appname", "count": bson.M{"$sum": 1}}},
	})
	var appsInfo []struct {
		Name  string `bson:"_id"`
		Count int
	}
	err := appsPipe.All(&appsInfo)
	if err != nil {
		return err
	}
	clusterInstance := dockerCluster()
	for _, appInfo := range appsInfo {
		if appInfo.Count < 2 {
			continue
		}
		logProgress(encoder, "Rebalancing app %q (%d units)...", appInfo.Name, appInfo.Count)
		var possibleDests []cluster.Node
		if isSegregateScheduler() {
			possibleDests, err = nodesForAppName(clusterInstance, appInfo.Name)
		} else {
			possibleDests, err = clusterInstance.Nodes()
		}
		if err != nil {
			return err
		}
		maxContPerUnit := appInfo.Count / len(possibleDests)
		overflowHosts := appInfo.Count % len(possibleDests)
		pipe := coll.Pipe([]bson.M{
			{"$match": bson.M{"hostaddr": bson.M{"$ne": ""}, "appname": appInfo.Name}},
			{"$group": bson.M{
				"_id":        "$hostaddr",
				"count":      bson.M{"$sum": 1},
				"containers": bson.M{"$push": fullDocQuery}}},
		})
		var hosts []hostWithContainers
		hostsSet := make(map[string]bool)
		err = pipe.All(&hosts)
		if err != nil {
			return err
		}
		for _, host := range hosts {
			hostsSet[host.HostAddr] = true
		}
		for _, node := range possibleDests {
			hostAddr := urlToHost(node.Address)
			_, present := hostsSet[hostAddr]
			if !present {
				hosts = append(hosts, hostWithContainers{HostAddr: hostAddr})
			}
		}
		anythingDone := false
		for _, host := range hosts {
			toMoveCount := host.Count - maxContPerUnit
			if toMoveCount <= 0 {
				continue
			}
			if overflowHosts > 0 {
				overflowHosts--
				toMoveCount--
				if toMoveCount <= 0 {
					continue
				}
			}
			anythingDone = true
			logProgress(encoder, "Trying to move %d units for %q from %s...", toMoveCount, appInfo.Name, host.HostAddr)
			locker := &appLocker{}
			wg := sync.WaitGroup{}
			moveErrors := make(chan error, toMoveCount)
			for _, cont := range host.Containers {
				minDest := minCountHost(hosts)
				if minDest.Count < maxContPerUnit {
					toMoveCount--
					minDest.Count++
					if dryRun {
						logProgress(encoder, "Would move unit %s for %q: %s -> %s...", cont.ID, appInfo.Name, cont.HostAddr, minDest.HostAddr)
					} else {
						wg.Add(1)
						go moveOneContainer(cont, minDest.HostAddr, moveErrors, &wg, encoder, locker)
					}
				}
				if toMoveCount == 0 {
					break
				}
			}
			if toMoveCount > 0 {
				logProgress(encoder, "Couldn't find suitable destination for %d units for app %q", toMoveCount, appInfo.Name)
			}
			go func() {
				wg.Wait()
				close(moveErrors)
			}()
			err := handleMoveErrors(moveErrors, encoder)
			if err != nil {
				return err
			}
		}
		if anythingDone {
			logProgress(encoder, "Rebalance finished for %q", appInfo.Name)
		} else {
			logProgress(encoder, "Nothing to do for %q", appInfo.Name)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package mailchimp

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

// Client manages communication with the Mailchimp API.
type Client struct {
	client  *http.Client
	BaseURL *url.URL
	DC      string
	APIKey  string
}

// NewClient returns a new Mailchimp API client.  If a nil httpClient is
// provided, http.DefaultClient will be used. The apiKey must be in the format xyz-us11.
func NewClient(apiKey string, httpClient *http.Client) (*Client, error) {
	if len(strings.Split(apiKey, "-")) != 2 {
		return nil, errors.New("Mailchimp API Key must be formatted like: xyz-zys")
	}
	dc := strings.Split(apiKey, "-")[1] // data center
	if httpClient == nil {
		httpClient = http.DefaultClient
	}
	baseURL, _ := url.Parse(fmt.Sprintf("https://%s.api.mailchimp.com/3.0", dc))
	return &Client{
		APIKey:  apiKey,
		client:  httpClient,
		DC:      dc,
		BaseURL: baseURL,
	}, nil
}

// ErrorResponse ...
type ErrorResponse struct {
	Type   string `json:"type"`
	Title  string `json:"title"`
	Status int    `json:"status"`
	Detail string `json:"detail"`
}

// Error ...
func (e ErrorResponse) Error() string {
	return fmt.Sprintf("Error %d %s (%s)", e.Status, e.Title, e.Detail)
}

// CheckResponse ...
func CheckResponse(r *http.Response) error {
	if c := r.StatusCode; 200 <= c && c <= 299 {
		return nil
	}
	errorResponse := new(ErrorResponse)
	data, err := ioutil.ReadAll(r.Body)
	if err == nil && data != nil {
		json.Unmarshal(data, errorResponse)
	}
	return errorResponse
}

// Do ...
func (c *Client) Do(method string, path string, body interface{}) (interface{}, error) {
	var buf io.ReadWriter
	if body != nil {
		buf = new(bytes.Buffer)
		err := json.NewEncoder(buf).Encode(body)
		if err != nil {
			return nil, err
		}
	}

	apiURL := fmt.Sprintf("%s%s", c.BaseURL.String(), path)

	req, err := http.NewRequest(method, apiURL, buf)
	if err != nil {
		return nil, err
	}
	req.SetBasicAuth("", c.APIKey)

	resp, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	err = CheckResponse(resp)
	if err != nil {
		return nil, err
	}

	var v interface{}
	err = json.NewDecoder(resp.Body).Decode(&v)
	if err != nil {
		return nil, err
	}
	return v, nil
}

// Subscribe ...
func (c *Client) Subscribe(email string, listID string) (interface{}, error) {
	data := &map[string]string{
		"email_address": email,
		"status":        "subscribed",
	}
	return c.Do(
		"POST",
		fmt.Sprintf("/lists/%s/members/", listID),
		data,
	)
}
<commit_msg>Refactoring.<commit_after>package mailchimp

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

// Client manages communication with the Mailchimp API.
type Client struct {
	client  *http.Client
	BaseURL *url.URL
	DC      string
	APIKey  string
}

// NewClient returns a new Mailchimp API client.  If a nil httpClient is
// provided, http.DefaultClient will be used. The apiKey must be in the format xyz-us11.
func NewClient(apiKey string, httpClient *http.Client) (*Client, error) {
	if len(strings.Split(apiKey, "-")) != 2 {
		return nil, errors.New("Mailchimp API Key must be formatted like: xyz-zys")
	}
	dc := strings.Split(apiKey, "-")[1] // data center
	if httpClient == nil {
		httpClient = http.DefaultClient
	}
	baseURL, _ := url.Parse(fmt.Sprintf("https://%s.api.mailchimp.com/3.0", dc))
	return &Client{
		APIKey:  apiKey,
		client:  httpClient,
		DC:      dc,
		BaseURL: baseURL,
	}, nil
}

// ErrorResponse ...
type ErrorResponse struct {
	Type   string `json:"type"`
	Title  string `json:"title"`
	Status int    `json:"status"`
	Detail string `json:"detail"`
}

// Error ...
func (e ErrorResponse) Error() string {
	return fmt.Sprintf("Error %d %s (%s)", e.Status, e.Title, e.Detail)
}

// Subscribe ...
func (c *Client) Subscribe(email string, listID string) (interface{}, error) {
	data := &map[string]string{
		"email_address": email,
		"status":        "subscribed",
	}
	return c.do(
		"POST",
		fmt.Sprintf("/lists/%s/members/", listID),
		data,
	)
}

func (c *Client) do(method string, path string, body interface{}) (interface{}, error) {
	var buf io.ReadWriter
	if body != nil {
		buf = new(bytes.Buffer)
		err := json.NewEncoder(buf).Encode(body)
		if err != nil {
			return nil, err
		}
	}

	apiURL := fmt.Sprintf("%s%s", c.BaseURL.String(), path)

	req, err := http.NewRequest(method, apiURL, buf)
	if err != nil {
		return nil, err
	}
	req.SetBasicAuth("", c.APIKey)

	resp, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if err := checkResponse(resp); err != nil {
		return nil, err
	}

	var v interface{}
	err = json.NewDecoder(resp.Body).Decode(&v)
	if err != nil {
		return nil, err
	}
	return v, nil
}

func checkResponse(r *http.Response) error {
	if c := r.StatusCode; 200 <= c && c <= 299 {
		return nil
	}
	errorResponse := new(ErrorResponse)
	data, err := ioutil.ReadAll(r.Body)
	if err == nil && data != nil {
		json.Unmarshal(data, errorResponse)
	}
	return errorResponse
}
<|endoftext|>


<commit_before>package main

/*
#cgo LDFLAGS: -lOpenCL

#ifdef __APPLE__
#  include "OpenCL/opencl.h"
#else
#  include "CL/cl.h"
#endif

*/
import "C"

import (
	"fmt"
	"unsafe"
)

func main() {
	// get platform count
	var platformCount C.cl_uint
	C.clGetPlatformIDs(0, nil, &platformCount)
	fmt.Println("OpenCL platforms:", platformCount)
	// get platform IDs
	platformIDs := make([]C.cl_platform_id, platformCount)
	C.clGetPlatformIDs(platformCount, &platformIDs[0], nil)

	// get device count
	var deviceIdCount C.cl_uint
	C.clGetDeviceIDs(platformIDs[0], C.CL_DEVICE_TYPE_ALL, 0, nil, &deviceIdCount)
	fmt.Println("Devices:", deviceIdCount)
	// get device IDs
	deviceIDs := make([]C.cl_device_id, deviceIdCount)
	C.clGetDeviceIDs(platformIDs[0], C.CL_DEVICE_TYPE_ALL, deviceIdCount, &deviceIDs[0], nil)

	// make context
	contextProps := []C.cl_context_properties{
		C.CL_CONTEXT_PLATFORM,
		*(*C.cl_context_properties)(unsafe.Pointer(&platformIDs[0])),
		0,
	}

	var clErr C.cl_int
	C.clCreateContext(&contextProps[0], 1, &deviceIDs[0], nil, nil, &clErr)

}
<commit_msg>Create Contexts and Buffers<commit_after>package main

/*
#cgo LDFLAGS: -lOpenCL

#ifdef __APPLE__
#  include "OpenCL/opencl.h"
#else
#  include "CL/cl.h"
#endif

*/
import "C"

import (
	"fmt"
	"unsafe"
)

type Context struct {
	cl C.cl_context
}

type Buffer C.cl_mem

func CreateContext() *Context {
	// get platform count
	var platformCount C.cl_uint
	C.clGetPlatformIDs(0, nil, &platformCount)
	fmt.Println("OpenCL platforms:", platformCount)
	// get platform IDs
	platformIDs := make([]C.cl_platform_id, platformCount)
	C.clGetPlatformIDs(platformCount, &platformIDs[0], nil)

	// get device count
	var deviceIdCount C.cl_uint
	C.clGetDeviceIDs(platformIDs[0], C.CL_DEVICE_TYPE_ALL, 0, nil,
		&deviceIdCount)
	fmt.Println("Devices:", deviceIdCount)
	// get device IDs
	deviceIDs := make([]C.cl_device_id, deviceIdCount)
	C.clGetDeviceIDs(platformIDs[0], C.CL_DEVICE_TYPE_ALL, deviceIdCount,
		&deviceIDs[0], nil)

	// make context
	contextProps := []C.cl_context_properties{
		C.CL_CONTEXT_PLATFORM,
		*(*C.cl_context_properties)(unsafe.Pointer(&platformIDs[0])),
		0,
	}

	var clErr C.cl_int
	context := C.clCreateContext(&contextProps[0], 1, &deviceIDs[0], nil, nil,
		&clErr)
	if clErr != C.CL_SUCCESS {
		fmt.Println("CL error", clErr, "!")
		return nil
	}
	return &Context{context}
}

func (c *Context) CreateBuffer(data []float32) Buffer {
	var clErr C.cl_int
	var buffer C.cl_mem
	buffer = C.clCreateBuffer(c.cl, C.CL_MEM_READ_ONLY|C.CL_MEM_COPY_HOST_PTR,
		C.size_t(int(unsafe.Sizeof(data[0]))*len(data)), unsafe.Pointer(&data[0]),
		&clErr)

	if clErr != C.CL_SUCCESS {
		fmt.Println("CL error", clErr, "!")
		return nil
	}
	return Buffer(buffer)
}

func main() {
	ctx := CreateContext()

	x := []float32{1, 2, 3, 4}
	buffer := ctx.CreateBuffer(x)

	fmt.Println(buffer)
}
<|endoftext|>


<commit_before>package admin

import (
	"github.com/qor/qor"
	"os"
	"path"
	"path/filepath"
	"regexp"

	"text/template"
)

var layouts = map[string]*template.Template{}
var templates = map[string]*template.Template{}
var tmplSuffix = regexp.MustCompile(`(\.tmpl)?$`)
var viewDirs = []string{}

func isExistingDir(pth string) bool {
	fi, err := os.Stat(pth)
	if err != nil {
		return false
	}
	return fi.Mode().IsDir()
}

func init() {
	if root := os.Getenv("WEB_ROOT"); root != "" {
		if dir := path.Join(root, "templates/qor"); isExistingDir(dir) {
			viewDirs = append(viewDirs, dir)
		}
	}

	if dir, err := filepath.Abs("templates/qor"); err == nil && isExistingDir(dir) {
		viewDirs = append(viewDirs, dir)
	}

	if dir := path.Join(os.Getenv("GOROOT"), "site/src/github.com/qor/qor/admin/templates"); isExistingDir(dir) {
		viewDirs = append(viewDirs, dir)
	}
}

func (admin *Admin) Render(str string, context *qor.Context) {
	var tmpl *template.Template

	str = tmplSuffix.ReplaceAllString(str, ".tmpl")
	pathWithResourceName := path.Join(context.ResourceName, str)

	var paths []string
	for _, value := range []string{pathWithResourceName, str} {
		if root != "" {
			paths = append(paths, filepath.Join(root, value))
		}
		if p, e := filepath.Abs(value); e == nil {
			paths = append(paths, p)
		}
		paths = append(paths, filepath.Join(goroot, "site/src/github.com/qor/qor/admin", value))
	}

	for _, value := range paths {
		_, err := os.Stat(value)
		if !os.IsNotExist(err) {
			t, _ := template.ParseFiles(value)
			templates[value] = t
			tmpl = t
			break
		}
	}
	if tmpl != nil {
		tmpl.Execute(context.Writer, context)
	}
}
<commit_msg>update admin template<commit_after>package admin

import (
	"github.com/qor/qor"
	"os"
	"path"
	"path/filepath"
	"regexp"

	"text/template"
)

var layouts = map[string]*template.Template{}
var templates = map[string]*template.Template{}
var tmplSuffix = regexp.MustCompile(`(\.tmpl)?$`)
var viewDirs = []string{}

func isExistingDir(pth string) bool {
	fi, err := os.Stat(pth)
	if err != nil {
		return false
	}
	return fi.Mode().IsDir()
}

func init() {
	if root := os.Getenv("WEB_ROOT"); root != "" {
		if dir := path.Join(root, "templates/qor"); isExistingDir(dir) {
			viewDirs = append(viewDirs, dir)
		}
	}

	if dir, err := filepath.Abs("templates/qor"); err == nil && isExistingDir(dir) {
		viewDirs = append(viewDirs, dir)
	}

	if dir := path.Join(os.Getenv("GOROOT"), "site/src/github.com/qor/qor/admin/templates"); isExistingDir(dir) {
		viewDirs = append(viewDirs, dir)
	}
}

func (admin *Admin) Render(str string, context *qor.Context) {
	var tmpl *template.Template

	cacheKey := path.Join(context.ResourceName, str)
	if t, ok := templates[cacheKey]; !ok {
		str = tmplSuffix.ReplaceAllString(str, ".tmpl")

		tmpl = template.New("template")

		// parse layout
		paths := []string{}
		for _, p := range []string{path.Join("resources", context.ResourceName), path.Join("themes", "default")} {
			for _, d := range viewDirs {
				if isExistingDir(path.Join(d, p)) {
					paths = append(paths, path.Join(d, p))
				}
			}
		}

		for _, f := range []string{"layout.tmpl", str} {
			for _, p := range paths {
				if _, err := os.Stat(path.Join(p, f)); !os.IsNotExist(err) {
					tmpl, err = tmpl.ParseFiles(path.Join(p, f))
					break
				}
			}
		}

		for _, name := range []string{"header", "footer"} {
			if tmpl.Lookup(name) == nil {
				for _, p := range paths {
					if _, err := os.Stat(path.Join(p, name+".tmpl")); !os.IsNotExist(err) {
						tmpl, err = tmpl.ParseFiles(path.Join(p, name+".tmpl"))
						break
					}
				}
			}
		}

		templates[cacheKey] = tmpl
	} else {
		tmpl = t
	}

	if tmpl != nil {
		tmpl.Execute(context.Writer, context)
	}
}
<|endoftext|>


<commit_before>package mode

import (
	"sync"
	"time"

	"github.com/elastic/beats/libbeat/common"
	"github.com/elastic/beats/libbeat/logp"
	"github.com/elastic/beats/libbeat/outputs"
)

// LoadBalancerMode balances the sending of events between multiple connections.
//
// The balancing algorithm is mostly pull-based, with multiple workers trying to pull
// some amount of work from a shared queue. Workers will try to get a new work item
// only if they have a working/active connection. Workers without active connection
// do not participate until a connection has been re-established.
// Due to the pull based nature the algorithm will load-balance events by random
// with workers having less latencies/turn-around times potentially getting more
// work items then other workers with higher latencies. Thusly the algorithm
// dynamically adapts to resource availability of server events are forwarded to.
//
// Workers not participating in the load-balancing will continuously try to reconnect
// to their configured endpoints. Once a new connection has been established,
// these workers will participate in in load-balancing again.
//
// If a connection becomes unavailable, the events are rescheduled for another
// connection to pick up. Rescheduling events is limited to a maximum number of
// send attempts. If events have not been send after maximum number of allowed
// attemps has been passed, they will be dropped.
//
// Distributing events to workers is subject to timeout. If no worker is available to
// pickup a message for sending, the message will be dropped internally.
type LoadBalancerMode struct {
	timeout      time.Duration // Send/retry timeout. Every timeout is a failed send attempt
	waitRetry    time.Duration // Duration to wait during re-connection attempts.
	maxWaitRetry time.Duration // Maximum send/retry timeout in backoff case.

	// maximum number of configured send attempts. If set to 0, publisher will
	// block until event has been successfully published.
	maxAttempts int

	// waitGroup + signaling channel for handling shutdown
	wg   sync.WaitGroup
	done chan struct{}

	// channels for forwarding work items to workers.
	// The work channel is used by publisher to insert new events
	// into the load balancer. The work channel is synchronous blocking until timeout
	// for one worker available.
	// The retries channel is used to forward failed send attempts to other workers.
	// The retries channel is buffered to mitigate possible deadlocks when all
	// workers become unresponsive.
	work    chan eventsMessage
	retries chan eventsMessage
}

type eventsMessage struct {
	attemptsLeft int
	signaler     outputs.Signaler
	events       []common.MapStr
	event        common.MapStr
}

// NewLoadBalancerMode create a new load balancer connection mode.
func NewLoadBalancerMode(
	clients []ProtocolClient,
	maxAttempts int,
	waitRetry, timeout, maxWaitRetry time.Duration,
) (*LoadBalancerMode, error) {

	// maxAttempts signals infinite retry. Convert to -1, so attempts left and
	// and infinite retry can be more easily distinguished by load balancer
	if maxAttempts == 0 {
		maxAttempts = -1
	}

	m := &LoadBalancerMode{
		timeout:      timeout,
		maxWaitRetry: maxWaitRetry,
		waitRetry:    waitRetry,
		maxAttempts:  maxAttempts,

		work:    make(chan eventsMessage),
		retries: make(chan eventsMessage, len(clients)*2),
		done:    make(chan struct{}),
	}
	m.start(clients)

	return m, nil
}

// Close stops all workers and closes all open connections. In flight events
// are signaled as failed.
func (m *LoadBalancerMode) Close() error {
	close(m.done)
	m.wg.Wait()
	return nil
}

// PublishEvents forwards events to some load balancing worker.
func (m *LoadBalancerMode) PublishEvents(
	signaler outputs.Signaler,
	opts outputs.Options,
	events []common.MapStr,
) error {
	return m.publishEventsMessage(opts,
		eventsMessage{signaler: signaler, events: events})
}

// PublishEvent forwards the event to some load balancing worker.
func (m *LoadBalancerMode) PublishEvent(
	signaler outputs.Signaler,
	opts outputs.Options,
	event common.MapStr,
) error {
	return m.publishEventsMessage(opts,
		eventsMessage{signaler: signaler, event: event})
}

func (m *LoadBalancerMode) publishEventsMessage(
	opts outputs.Options,
	msg eventsMessage,
) error {
	maxAttempts := m.maxAttempts
	if opts.Guaranteed {
		maxAttempts = -1
	}
	msg.attemptsLeft = maxAttempts

	if ok := m.forwardEvent(m.work, msg); !ok {
		dropping(msg)
	}
	return nil
}

func (m *LoadBalancerMode) start(clients []ProtocolClient) {
	var waitStart sync.WaitGroup
	worker := func(client ProtocolClient) {
		defer func() {
			if client.IsConnected() {
				_ = client.Close()
			}
			m.wg.Done()
		}()

		waitStart.Done()
		for {
			// reconnect loop
			for !client.IsConnected() {
				if err := client.Connect(m.timeout); err == nil {
					break
				}

				select {
				case <-m.done:
					return
				case <-time.After(m.waitRetry):
				}
			}

			// receive and process messages
			var msg eventsMessage
			select {
			case <-m.done:
				return
			case msg = <-m.retries: // receive message from other failed worker
			case msg = <-m.work: // receive message from publisher
			}
			m.onMessage(client, msg)
		}
	}

	for _, client := range clients {
		m.wg.Add(1)
		waitStart.Add(1)
		go worker(client)
	}
	waitStart.Wait()
}

func (m *LoadBalancerMode) onMessage(client ProtocolClient, msg eventsMessage) {
	if msg.event != nil {
		if err := client.PublishEvent(msg.event); err != nil {
			if msg.attemptsLeft > 0 {
				msg.attemptsLeft--
			}
			m.onFail(msg, err)
			return
		}
	} else {
		events := msg.events
		total := len(events)
		var backoffCount uint

		for len(events) > 0 {
			var err error

			events, err = client.PublishEvents(events)
			if err != nil {
				if msg.attemptsLeft > 0 {
					msg.attemptsLeft--
				}

				// reset attempt count if subset of messages has been processed
				if len(events) < total && msg.attemptsLeft >= 0 {
					debug("reset fails")
					msg.attemptsLeft = m.maxAttempts
				}

				if err != ErrTempBulkFailure {
					// retry non-published subset of events in batch
					msg.events = events
					m.onFail(msg, err)
					return
				}

				if m.maxAttempts > 0 && msg.attemptsLeft == 0 {
					// no more attempts left => drop
					dropping(msg)
					return
				}

				// wait before retry
				backoff := time.Duration(int64(m.waitRetry) * (1 << backoffCount))
				if backoff > m.maxWaitRetry {
					backoff = m.maxWaitRetry
				} else {
					backoffCount++
				}
				select {
				case <-m.done: // shutdown
					outputs.SignalFailed(msg.signaler, err)
					return
				case <-time.After(backoff):
				}

				// reset total count for temporary failure loop
				total = len(events)
			}
		}
	}
	outputs.SignalCompleted(msg.signaler)
}

func (m *LoadBalancerMode) onFail(msg eventsMessage, err error) {

	logp.Info("Error publishing events (retrying): %s", err)

	if ok := m.forwardEvent(m.retries, msg); !ok {
		dropping(msg)
	}
}

func (m *LoadBalancerMode) forwardEvent(
	ch chan eventsMessage,
	msg eventsMessage,
) bool {
	if msg.attemptsLeft < 0 {
		select {
		case ch <- msg:
			return true
		case <-m.done: // shutdown
			return false
		}
	} else {
		for ; msg.attemptsLeft > 0; msg.attemptsLeft-- {
			select {
			case ch <- msg:
				return true
			case <-m.done: // shutdown
				return false
			case <-time.After(m.timeout):
			}
		}
	}
	return false
}

// dropping is called when a message is dropped. It updates the
// relevant counters and sends a failed signal.
func dropping(msg eventsMessage) {
	debug("messages dropped")
	messagesDropped.Add(1)
	outputs.SignalFailed(msg.signaler, nil)
}
<commit_msg>Simplify balance mode worker loop<commit_after>package mode

import (
	"sync"
	"time"

	"github.com/elastic/beats/libbeat/common"
	"github.com/elastic/beats/libbeat/logp"
	"github.com/elastic/beats/libbeat/outputs"
)

// LoadBalancerMode balances the sending of events between multiple connections.
//
// The balancing algorithm is mostly pull-based, with multiple workers trying to pull
// some amount of work from a shared queue. Workers will try to get a new work item
// only if they have a working/active connection. Workers without active connection
// do not participate until a connection has been re-established.
// Due to the pull based nature the algorithm will load-balance events by random
// with workers having less latencies/turn-around times potentially getting more
// work items then other workers with higher latencies. Thusly the algorithm
// dynamically adapts to resource availability of server events are forwarded to.
//
// Workers not participating in the load-balancing will continuously try to reconnect
// to their configured endpoints. Once a new connection has been established,
// these workers will participate in in load-balancing again.
//
// If a connection becomes unavailable, the events are rescheduled for another
// connection to pick up. Rescheduling events is limited to a maximum number of
// send attempts. If events have not been send after maximum number of allowed
// attemps has been passed, they will be dropped.
//
// Distributing events to workers is subject to timeout. If no worker is available to
// pickup a message for sending, the message will be dropped internally.
type LoadBalancerMode struct {
	timeout      time.Duration // Send/retry timeout. Every timeout is a failed send attempt
	waitRetry    time.Duration // Duration to wait during re-connection attempts.
	maxWaitRetry time.Duration // Maximum send/retry timeout in backoff case.

	// maximum number of configured send attempts. If set to 0, publisher will
	// block until event has been successfully published.
	maxAttempts int

	// waitGroup + signaling channel for handling shutdown
	wg   sync.WaitGroup
	done chan struct{}

	// channels for forwarding work items to workers.
	// The work channel is used by publisher to insert new events
	// into the load balancer. The work channel is synchronous blocking until timeout
	// for one worker available.
	// The retries channel is used to forward failed send attempts to other workers.
	// The retries channel is buffered to mitigate possible deadlocks when all
	// workers become unresponsive.
	work    chan eventsMessage
	retries chan eventsMessage
}

type eventsMessage struct {
	attemptsLeft int
	signaler     outputs.Signaler
	events       []common.MapStr
	event        common.MapStr
}

// NewLoadBalancerMode create a new load balancer connection mode.
func NewLoadBalancerMode(
	clients []ProtocolClient,
	maxAttempts int,
	waitRetry, timeout, maxWaitRetry time.Duration,
) (*LoadBalancerMode, error) {

	// maxAttempts signals infinite retry. Convert to -1, so attempts left and
	// and infinite retry can be more easily distinguished by load balancer
	if maxAttempts == 0 {
		maxAttempts = -1
	}

	m := &LoadBalancerMode{
		timeout:      timeout,
		maxWaitRetry: maxWaitRetry,
		waitRetry:    waitRetry,
		maxAttempts:  maxAttempts,

		work:    make(chan eventsMessage),
		retries: make(chan eventsMessage, len(clients)*2),
		done:    make(chan struct{}),
	}
	m.start(clients)

	return m, nil
}

// Close stops all workers and closes all open connections. In flight events
// are signaled as failed.
func (m *LoadBalancerMode) Close() error {
	close(m.done)
	m.wg.Wait()
	return nil
}

// PublishEvents forwards events to some load balancing worker.
func (m *LoadBalancerMode) PublishEvents(
	signaler outputs.Signaler,
	opts outputs.Options,
	events []common.MapStr,
) error {
	return m.publishEventsMessage(opts,
		eventsMessage{signaler: signaler, events: events})
}

// PublishEvent forwards the event to some load balancing worker.
func (m *LoadBalancerMode) PublishEvent(
	signaler outputs.Signaler,
	opts outputs.Options,
	event common.MapStr,
) error {
	return m.publishEventsMessage(opts,
		eventsMessage{signaler: signaler, event: event})
}

func (m *LoadBalancerMode) publishEventsMessage(
	opts outputs.Options,
	msg eventsMessage,
) error {
	maxAttempts := m.maxAttempts
	if opts.Guaranteed {
		maxAttempts = -1
	}
	msg.attemptsLeft = maxAttempts

	if ok := m.forwardEvent(m.work, msg); !ok {
		dropping(msg)
	}
	return nil
}

func (m *LoadBalancerMode) start(clients []ProtocolClient) {
	var waitStart sync.WaitGroup
	worker := func(client ProtocolClient) {
		defer func() {
			if client.IsConnected() {
				_ = client.Close()
			}
			m.wg.Done()
		}()

		waitStart.Done()
		m.clientLoop(client)
	}

	for _, client := range clients {
		m.wg.Add(1)
		waitStart.Add(1)
		go worker(client)
	}
	waitStart.Wait()
}

func (m *LoadBalancerMode) clientLoop(client ProtocolClient) {
	debug("load balancer: start client loop")
	defer debug("load balancer: stop client loop")

	backoff := newBackoff(m.done, m.waitRetry, m.maxWaitRetry)

	done := false
	for !done {
		if done = m.connect(client, backoff); !done {
			done = m.sendLoop(client, backoff)
		}
		debug("close client")
		client.Close()
	}
}

func (m *LoadBalancerMode) connect(client ProtocolClient, backoff *backoff) bool {
	for {
		debug("try to (re-)connect client")
		err := client.Connect(m.timeout)
		if !backoff.WithError(err) {
			return true
		}

		if err == nil {
			return false
		}
	}
}

func (m *LoadBalancerMode) sendLoop(client ProtocolClient, backoff *backoff) bool {
	for {
		var msg eventsMessage
		select {
		case <-m.done:
			return true
		case msg = <-m.retries: // receive message from other failed worker
		case msg = <-m.work: // receive message from publisher
		}

		done, err := m.onMessage(backoff, client, msg)
		if done || err != nil {
			return done
		}
	}
}

func (m *LoadBalancerMode) onMessage(
	backoff *backoff,
	client ProtocolClient,
	msg eventsMessage,
) (bool, error) {

	done := false
	if msg.event != nil {
		err := client.PublishEvent(msg.event)
		done = !backoff.WithError(err)
		if err != nil {
			if msg.attemptsLeft > 0 {
				msg.attemptsLeft--
			}
			m.onFail(msg, err)
			return done, err
		}
	} else {
		events := msg.events
		total := len(events)

		for len(events) > 0 {
			var err error

			events, err = client.PublishEvents(events)
			done = !backoff.WithError(err)
			if done && err != nil {
				outputs.SignalFailed(msg.signaler, err)
				return done, err
			}

			if err != nil {
				if msg.attemptsLeft > 0 {
					msg.attemptsLeft--
				}

				// reset attempt count if subset of messages has been processed
				if len(events) < total && msg.attemptsLeft >= 0 {
					debug("reset fails")
					msg.attemptsLeft = m.maxAttempts
				}

				if err != ErrTempBulkFailure {
					// retry non-published subset of events in batch
					msg.events = events
					m.onFail(msg, err)
					return done, err
				}

				if m.maxAttempts > 0 && msg.attemptsLeft == 0 {
					// no more attempts left => drop
					dropping(msg)
					return done, err
				}

				// reset total count for temporary failure loop
				total = len(events)
			}
		}
	}

	outputs.SignalCompleted(msg.signaler)
	return done, nil
}

func (m *LoadBalancerMode) onFail(msg eventsMessage, err error) {

	logp.Info("Error publishing events (retrying): %s", err)

	if !m.forwardEvent(m.retries, msg) {
		dropping(msg)
	}
}

func (m *LoadBalancerMode) forwardEvent(
	ch chan eventsMessage,
	msg eventsMessage,
) bool {
	if msg.attemptsLeft < 0 {
		select {
		case ch <- msg:
			return true
		case <-m.done: // shutdown
			return false
		}
	} else {
		for ; msg.attemptsLeft > 0; msg.attemptsLeft-- {
			select {
			case ch <- msg:
				return true
			case <-m.done: // shutdown
				return false
			case <-time.After(m.timeout):
			}
		}
	}
	return false
}

// dropping is called when a message is dropped. It updates the
// relevant counters and sends a failed signal.
func dropping(msg eventsMessage) {
	debug("messages dropped")
	messagesDropped.Add(1)
	outputs.SignalFailed(msg.signaler, nil)
}
<|endoftext|>


<commit_before>package ttransport

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strconv"
	"sync"
	"testing"
	"time"

	crand "crypto/rand"
	mrand "math/rand"

	"github.com/libp2p/go-libp2p-core/mux"
	"github.com/libp2p/go-libp2p-core/peer"
	"github.com/libp2p/go-libp2p-core/transport"
	"github.com/libp2p/go-libp2p-testing/race"

	ma "github.com/multiformats/go-multiaddr"
)

// VerboseDebugging can be set to true to enable verbose debug logging in the
// stream stress tests.
var VerboseDebugging = false

var randomness []byte

var StressTestTimeout = 1 * time.Minute

func init() {
	// read 1MB of randomness
	randomness = make([]byte, 1<<20)
	if _, err := crand.Read(randomness); err != nil {
		panic(err)
	}

	if timeout := os.Getenv("TEST_STRESS_TIMEOUT_MS"); timeout != "" {
		if v, err := strconv.ParseInt(timeout, 10, 32); err == nil {
			StressTestTimeout = time.Duration(v) * time.Millisecond
		}
	}
}

type Options struct {
	ConnNum   int
	StreamNum int
	MsgNum    int
	MsgMin    int
	MsgMax    int
}

func fullClose(t *testing.T, s mux.MuxedStream) {
	if err := s.CloseWrite(); err != nil {
		t.Error(err)
		s.Reset()
		return
	}
	b, err := ioutil.ReadAll(s)
	if err != nil {
		t.Error(err)
	}
	if len(b) != 0 {
		t.Error("expected to be done reading")
	}
	if err := s.Close(); err != nil {
		t.Error(err)
	}
}

func randBuf(size int) []byte {
	n := len(randomness) - size
	if size < 1 {
		panic(fmt.Errorf("requested too large buffer (%d). max is %d", size, len(randomness)))
	}

	start := mrand.Intn(n)
	return randomness[start : start+size]
}

func debugLog(t *testing.T, s string, args ...interface{}) {
	if VerboseDebugging {
		t.Logf(s, args...)
	}
}

func echoStream(t *testing.T, s mux.MuxedStream) {
	// echo everything
	var err error
	if VerboseDebugging {
		t.Logf("accepted stream")
		_, err = io.Copy(&logWriter{t, s}, s)
		t.Log("closing stream")
	} else {
		_, err = io.Copy(s, s) // echo everything
	}
	if err != nil {
		t.Error(err)
	}
}

type logWriter struct {
	t *testing.T
	W io.Writer
}

func (lw *logWriter) Write(buf []byte) (int, error) {
	lw.t.Logf("logwriter: writing %d bytes", len(buf))
	return lw.W.Write(buf)
}

func echo(t *testing.T, c transport.CapableConn) {
	var wg sync.WaitGroup
	defer wg.Wait()
	for {
		str, err := c.AcceptStream()
		if err != nil {
			break
		}
		wg.Add(1)
		go func() {
			defer wg.Done()
			defer str.Close()
			echoStream(t, str)
		}()
	}
}

func serve(t *testing.T, l transport.Listener) {
	var wg sync.WaitGroup
	defer wg.Wait()

	for {
		c, err := l.Accept()
		if err != nil {
			return
		}

		wg.Add(1)
		debugLog(t, "accepted connection")
		go func() {
			defer wg.Done()
			defer c.Close()
			echo(t, c)
		}()
	}
}

func SubtestStress(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID, opt Options) {
	msgsize := 1 << 11

	rateLimitN := 5000 // max of 5k funcs, because -race has 8k max.
	rateLimitChan := make(chan struct{}, rateLimitN)
	for i := 0; i < rateLimitN; i++ {
		rateLimitChan <- struct{}{}
	}

	rateLimit := func(f func()) {
		<-rateLimitChan
		f()
		rateLimitChan <- struct{}{}
	}

	writeStream := func(s mux.MuxedStream, bufs chan<- []byte) {
		debugLog(t, "writeStream %p, %d MsgNum", s, opt.MsgNum)

		for i := 0; i < opt.MsgNum; i++ {
			buf := randBuf(msgsize)
			bufs <- buf
			debugLog(t, "%p writing %d bytes (message %d/%d #%x)", s, len(buf), i, opt.MsgNum, buf[:3])
			if _, err := s.Write(buf); err != nil {
				t.Errorf("s.Write(buf): %s", err)
				continue
			}
		}
	}

	readStream := func(s mux.MuxedStream, bufs <-chan []byte) {
		debugLog(t, "readStream %p, %d MsgNum", s, opt.MsgNum)

		buf2 := make([]byte, msgsize)
		i := 0
		for buf1 := range bufs {
			i++
			debugLog(t, "%p reading %d bytes (message %d/%d #%x)", s, len(buf1), i-1, opt.MsgNum, buf1[:3])

			if _, err := io.ReadFull(s, buf2); err != nil {
				t.Errorf("io.ReadFull(s, buf2): %s", err)
				debugLog(t, "%p failed to read %d bytes (message %d/%d #%x)", s, len(buf1), i-1, opt.MsgNum, buf1[:3])
				continue
			}
			if !bytes.Equal(buf1, buf2) {
				t.Errorf("buffers not equal (%x != %x)", buf1[:3], buf2[:3])
			}
		}
	}

	openStreamAndRW := func(c mux.MuxedConn) {
		debugLog(t, "openStreamAndRW %p, %d opt.MsgNum", c, opt.MsgNum)

		s, err := c.OpenStream(context.Background())
		if err != nil {
			t.Errorf("failed to create NewStream: %s", err)
			return
		}

		bufs := make(chan []byte, opt.MsgNum)
		go func() {
			writeStream(s, bufs)
			close(bufs)
		}()

		readStream(s, bufs)
		fullClose(t, s)
	}

	openConnAndRW := func() {
		debugLog(t, "openConnAndRW")

		var wg sync.WaitGroup
		defer wg.Wait()

		l, err := ta.Listen(maddr)
		if err != nil {
			t.Error(err)
			return
		}
		defer l.Close()

		wg.Add(1)
		go func() {
			defer wg.Done()
			serve(t, l)
		}()

		c, err := tb.Dial(context.Background(), l.Multiaddr(), peerA)
		if err != nil {
			t.Error(err)
			return
		}

		// serve the outgoing conn, because some muxers assume
		// that we _always_ call serve. (this is an error?)
		wg.Add(1)
		go func() {
			defer wg.Done()
			defer c.Close()
			debugLog(t, "serving connection")
			echo(t, c)
		}()

		var openWg sync.WaitGroup
		for i := 0; i < opt.StreamNum; i++ {
			openWg.Add(1)
			go rateLimit(func() {
				defer openWg.Done()
				openStreamAndRW(c)
			})
		}
		openWg.Wait()
	}

	debugLog(t, "openConnsAndRW, %d conns", opt.ConnNum)

	var wg sync.WaitGroup
	defer wg.Wait()
	for i := 0; i < opt.ConnNum; i++ {
		wg.Add(1)
		go rateLimit(func() {
			defer wg.Done()
			openConnAndRW()
		})
	}
}

func SubtestStreamOpenStress(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	l, err := ta.Listen(maddr)
	if err != nil {
		t.Fatal(err)
	}
	defer l.Close()

	count := 10000
	workers := 5

	if race.WithRace() {
		// the race detector can only deal with 8128 simultaneous goroutines, so let's make sure we don't go overboard.
		count = 1000
	}

	var (
		connA, connB transport.CapableConn
	)

	accepted := make(chan error, 1)
	go func() {
		var err error
		connA, err = l.Accept()
		accepted <- err
	}()
	connB, err = tb.Dial(context.Background(), l.Multiaddr(), peerA)
	if err != nil {
		t.Fatal(err)
	}
	err = <-accepted
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		if connA != nil {
			connA.Close()
		}
		if connB != nil {
			connB.Close()
		}
	}()

	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		for j := 0; j < workers; j++ {
			wg.Add(1)
			go func() {
				defer wg.Done()
				for i := 0; i < count; i++ {
					s, err := connA.OpenStream(context.Background())
					if err != nil {
						t.Error(err)
						return
					}
					wg.Add(1)
					go func() {
						defer wg.Done()
						fullClose(t, s)
					}()
				}
			}()
		}
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		for i := 0; i < count*workers; i++ {
			str, err := connB.AcceptStream()
			if err != nil {
				break
			}
			wg.Add(1)
			go func() {
				defer wg.Done()
				fullClose(t, str)
			}()
		}
	}()

	timeout := time.After(StressTestTimeout)
	done := make(chan struct{})

	go func() {
		wg.Wait()
		close(done)
	}()

	select {
	case <-timeout:
		t.Fatal("timed out receiving streams")
	case <-done:
	}
}

func SubtestStreamReset(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	var wg sync.WaitGroup
	defer wg.Wait()

	l, err := ta.Listen(maddr)
	if err != nil {
		t.Fatal(err)
	}
	defer l.Close()

	wg.Add(1)
	go func() {
		defer wg.Done()

		muxa, err := l.Accept()
		if err != nil {
			t.Error(err)
			return
		}
		defer muxa.Close()

		s, err := muxa.OpenStream(context.Background())
		if err != nil {
			t.Error(err)
			return
		}
		defer s.Close()

		// Some transports won't open the stream until we write. That's
		// fine.
		_, _ = s.Write([]byte("foo"))

		time.Sleep(time.Millisecond * 50)

		_, err = s.Write([]byte("bar"))
		if err == nil {
			t.Error("should have failed to write")
		}

	}()

	muxb, err := tb.Dial(context.Background(), l.Multiaddr(), peerA)
	if err != nil {
		t.Fatal(err)
	}
	defer muxb.Close()

	str, err := muxb.AcceptStream()
	if err != nil {
		t.Error(err)
		return
	}
	str.Reset()
}

func SubtestStress1Conn1Stream1Msg(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	SubtestStress(t, ta, tb, maddr, peerA, Options{
		ConnNum:   1,
		StreamNum: 1,
		MsgNum:    1,
		MsgMax:    100,
		MsgMin:    100,
	})
}

func SubtestStress1Conn1Stream100Msg(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	SubtestStress(t, ta, tb, maddr, peerA, Options{
		ConnNum:   1,
		StreamNum: 1,
		MsgNum:    100,
		MsgMax:    100,
		MsgMin:    100,
	})
}

func SubtestStress1Conn100Stream100Msg(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	SubtestStress(t, ta, tb, maddr, peerA, Options{
		ConnNum:   1,
		StreamNum: 100,
		MsgNum:    100,
		MsgMax:    100,
		MsgMin:    100,
	})
}

func SubtestStress50Conn10Stream50Msg(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	SubtestStress(t, ta, tb, maddr, peerA, Options{
		ConnNum:   50,
		StreamNum: 10,
		MsgNum:    50,
		MsgMax:    100,
		MsgMin:    100,
	})
}

func SubtestStress1Conn1000Stream10Msg(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	SubtestStress(t, ta, tb, maddr, peerA, Options{
		ConnNum:   1,
		StreamNum: 1000,
		MsgNum:    10,
		MsgMax:    100,
		MsgMin:    100,
	})
}

func SubtestStress1Conn100Stream100Msg10MB(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	SubtestStress(t, ta, tb, maddr, peerA, Options{
		ConnNum:   1,
		StreamNum: 100,
		MsgNum:    100,
		MsgMax:    10000,
		MsgMin:    1000,
	})
}
<commit_msg>fix deadlock in the transport's serve function<commit_after>package ttransport

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strconv"
	"sync"
	"testing"
	"time"

	crand "crypto/rand"
	mrand "math/rand"

	"github.com/libp2p/go-libp2p-core/mux"
	"github.com/libp2p/go-libp2p-core/peer"
	"github.com/libp2p/go-libp2p-core/transport"
	"github.com/libp2p/go-libp2p-testing/race"

	ma "github.com/multiformats/go-multiaddr"
)

// VerboseDebugging can be set to true to enable verbose debug logging in the
// stream stress tests.
var VerboseDebugging = false

var randomness []byte

var StressTestTimeout = 1 * time.Minute

func init() {
	// read 1MB of randomness
	randomness = make([]byte, 1<<20)
	if _, err := crand.Read(randomness); err != nil {
		panic(err)
	}

	if timeout := os.Getenv("TEST_STRESS_TIMEOUT_MS"); timeout != "" {
		if v, err := strconv.ParseInt(timeout, 10, 32); err == nil {
			StressTestTimeout = time.Duration(v) * time.Millisecond
		}
	}
}

type Options struct {
	ConnNum   int
	StreamNum int
	MsgNum    int
	MsgMin    int
	MsgMax    int
}

func fullClose(t *testing.T, s mux.MuxedStream) {
	if err := s.CloseWrite(); err != nil {
		t.Error(err)
		s.Reset()
		return
	}
	b, err := ioutil.ReadAll(s)
	if err != nil {
		t.Error(err)
	}
	if len(b) != 0 {
		t.Error("expected to be done reading")
	}
	if err := s.Close(); err != nil {
		t.Error(err)
	}
}

func randBuf(size int) []byte {
	n := len(randomness) - size
	if size < 1 {
		panic(fmt.Errorf("requested too large buffer (%d). max is %d", size, len(randomness)))
	}

	start := mrand.Intn(n)
	return randomness[start : start+size]
}

func debugLog(t *testing.T, s string, args ...interface{}) {
	if VerboseDebugging {
		t.Logf(s, args...)
	}
}

func echoStream(t *testing.T, s mux.MuxedStream) {
	// echo everything
	var err error
	if VerboseDebugging {
		t.Logf("accepted stream")
		_, err = io.Copy(&logWriter{t, s}, s)
		t.Log("closing stream")
	} else {
		_, err = io.Copy(s, s) // echo everything
	}
	if err != nil {
		t.Error(err)
	}
}

type logWriter struct {
	t *testing.T
	W io.Writer
}

func (lw *logWriter) Write(buf []byte) (int, error) {
	lw.t.Logf("logwriter: writing %d bytes", len(buf))
	return lw.W.Write(buf)
}

func echo(t *testing.T, c transport.CapableConn) {
	var wg sync.WaitGroup
	defer wg.Wait()
	for {
		str, err := c.AcceptStream()
		if err != nil {
			break
		}
		wg.Add(1)
		go func() {
			defer wg.Done()
			defer str.Close()
			echoStream(t, str)
		}()
	}
}

func serve(t *testing.T, l transport.Listener) {
	var wg sync.WaitGroup
	defer wg.Wait()

	for {
		c, err := l.Accept()
		if err != nil {
			return
		}
		defer c.Close()

		wg.Add(1)
		debugLog(t, "accepted connection")
		go func() {
			defer wg.Done()
			echo(t, c)
		}()
	}
}

func SubtestStress(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID, opt Options) {
	msgsize := 1 << 11

	rateLimitN := 5000 // max of 5k funcs, because -race has 8k max.
	rateLimitChan := make(chan struct{}, rateLimitN)
	for i := 0; i < rateLimitN; i++ {
		rateLimitChan <- struct{}{}
	}

	rateLimit := func(f func()) {
		<-rateLimitChan
		f()
		rateLimitChan <- struct{}{}
	}

	writeStream := func(s mux.MuxedStream, bufs chan<- []byte) {
		debugLog(t, "writeStream %p, %d MsgNum", s, opt.MsgNum)

		for i := 0; i < opt.MsgNum; i++ {
			buf := randBuf(msgsize)
			bufs <- buf
			debugLog(t, "%p writing %d bytes (message %d/%d #%x)", s, len(buf), i, opt.MsgNum, buf[:3])
			if _, err := s.Write(buf); err != nil {
				t.Errorf("s.Write(buf): %s", err)
				continue
			}
		}
	}

	readStream := func(s mux.MuxedStream, bufs <-chan []byte) {
		debugLog(t, "readStream %p, %d MsgNum", s, opt.MsgNum)

		buf2 := make([]byte, msgsize)
		i := 0
		for buf1 := range bufs {
			i++
			debugLog(t, "%p reading %d bytes (message %d/%d #%x)", s, len(buf1), i-1, opt.MsgNum, buf1[:3])

			if _, err := io.ReadFull(s, buf2); err != nil {
				t.Errorf("io.ReadFull(s, buf2): %s", err)
				debugLog(t, "%p failed to read %d bytes (message %d/%d #%x)", s, len(buf1), i-1, opt.MsgNum, buf1[:3])
				continue
			}
			if !bytes.Equal(buf1, buf2) {
				t.Errorf("buffers not equal (%x != %x)", buf1[:3], buf2[:3])
			}
		}
	}

	openStreamAndRW := func(c mux.MuxedConn) {
		debugLog(t, "openStreamAndRW %p, %d opt.MsgNum", c, opt.MsgNum)

		s, err := c.OpenStream(context.Background())
		if err != nil {
			t.Errorf("failed to create NewStream: %s", err)
			return
		}

		bufs := make(chan []byte, opt.MsgNum)
		go func() {
			writeStream(s, bufs)
			close(bufs)
		}()

		readStream(s, bufs)
		fullClose(t, s)
	}

	openConnAndRW := func() {
		debugLog(t, "openConnAndRW")

		var wg sync.WaitGroup
		defer wg.Wait()

		l, err := ta.Listen(maddr)
		if err != nil {
			t.Error(err)
			return
		}
		defer l.Close()

		wg.Add(1)
		go func() {
			defer wg.Done()
			serve(t, l)
		}()

		c, err := tb.Dial(context.Background(), l.Multiaddr(), peerA)
		if err != nil {
			t.Error(err)
			return
		}

		// serve the outgoing conn, because some muxers assume
		// that we _always_ call serve. (this is an error?)
		wg.Add(1)
		go func() {
			defer wg.Done()
			defer c.Close()
			debugLog(t, "serving connection")
			echo(t, c)
		}()

		var openWg sync.WaitGroup
		for i := 0; i < opt.StreamNum; i++ {
			openWg.Add(1)
			go rateLimit(func() {
				defer openWg.Done()
				openStreamAndRW(c)
			})
		}
		openWg.Wait()
	}

	debugLog(t, "openConnsAndRW, %d conns", opt.ConnNum)

	var wg sync.WaitGroup
	defer wg.Wait()
	for i := 0; i < opt.ConnNum; i++ {
		wg.Add(1)
		go rateLimit(func() {
			defer wg.Done()
			openConnAndRW()
		})
	}
}

func SubtestStreamOpenStress(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	l, err := ta.Listen(maddr)
	if err != nil {
		t.Fatal(err)
	}
	defer l.Close()

	count := 10000
	workers := 5

	if race.WithRace() {
		// the race detector can only deal with 8128 simultaneous goroutines, so let's make sure we don't go overboard.
		count = 1000
	}

	var (
		connA, connB transport.CapableConn
	)

	accepted := make(chan error, 1)
	go func() {
		var err error
		connA, err = l.Accept()
		accepted <- err
	}()
	connB, err = tb.Dial(context.Background(), l.Multiaddr(), peerA)
	if err != nil {
		t.Fatal(err)
	}
	err = <-accepted
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		if connA != nil {
			connA.Close()
		}
		if connB != nil {
			connB.Close()
		}
	}()

	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		for j := 0; j < workers; j++ {
			wg.Add(1)
			go func() {
				defer wg.Done()
				for i := 0; i < count; i++ {
					s, err := connA.OpenStream(context.Background())
					if err != nil {
						t.Error(err)
						return
					}
					wg.Add(1)
					go func() {
						defer wg.Done()
						fullClose(t, s)
					}()
				}
			}()
		}
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		for i := 0; i < count*workers; i++ {
			str, err := connB.AcceptStream()
			if err != nil {
				break
			}
			wg.Add(1)
			go func() {
				defer wg.Done()
				fullClose(t, str)
			}()
		}
	}()

	timeout := time.After(StressTestTimeout)
	done := make(chan struct{})

	go func() {
		wg.Wait()
		close(done)
	}()

	select {
	case <-timeout:
		t.Fatal("timed out receiving streams")
	case <-done:
	}
}

func SubtestStreamReset(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	var wg sync.WaitGroup
	defer wg.Wait()

	l, err := ta.Listen(maddr)
	if err != nil {
		t.Fatal(err)
	}
	defer l.Close()

	wg.Add(1)
	go func() {
		defer wg.Done()

		muxa, err := l.Accept()
		if err != nil {
			t.Error(err)
			return
		}
		defer muxa.Close()

		s, err := muxa.OpenStream(context.Background())
		if err != nil {
			t.Error(err)
			return
		}
		defer s.Close()

		// Some transports won't open the stream until we write. That's
		// fine.
		_, _ = s.Write([]byte("foo"))

		time.Sleep(time.Millisecond * 50)

		_, err = s.Write([]byte("bar"))
		if err == nil {
			t.Error("should have failed to write")
		}

	}()

	muxb, err := tb.Dial(context.Background(), l.Multiaddr(), peerA)
	if err != nil {
		t.Fatal(err)
	}
	defer muxb.Close()

	str, err := muxb.AcceptStream()
	if err != nil {
		t.Error(err)
		return
	}
	str.Reset()
}

func SubtestStress1Conn1Stream1Msg(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	SubtestStress(t, ta, tb, maddr, peerA, Options{
		ConnNum:   1,
		StreamNum: 1,
		MsgNum:    1,
		MsgMax:    100,
		MsgMin:    100,
	})
}

func SubtestStress1Conn1Stream100Msg(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	SubtestStress(t, ta, tb, maddr, peerA, Options{
		ConnNum:   1,
		StreamNum: 1,
		MsgNum:    100,
		MsgMax:    100,
		MsgMin:    100,
	})
}

func SubtestStress1Conn100Stream100Msg(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	SubtestStress(t, ta, tb, maddr, peerA, Options{
		ConnNum:   1,
		StreamNum: 100,
		MsgNum:    100,
		MsgMax:    100,
		MsgMin:    100,
	})
}

func SubtestStress50Conn10Stream50Msg(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	SubtestStress(t, ta, tb, maddr, peerA, Options{
		ConnNum:   50,
		StreamNum: 10,
		MsgNum:    50,
		MsgMax:    100,
		MsgMin:    100,
	})
}

func SubtestStress1Conn1000Stream10Msg(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	SubtestStress(t, ta, tb, maddr, peerA, Options{
		ConnNum:   1,
		StreamNum: 1000,
		MsgNum:    10,
		MsgMax:    100,
		MsgMin:    100,
	})
}

func SubtestStress1Conn100Stream100Msg10MB(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	SubtestStress(t, ta, tb, maddr, peerA, Options{
		ConnNum:   1,
		StreamNum: 100,
		MsgNum:    100,
		MsgMax:    10000,
		MsgMin:    1000,
	})
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/majormjr/rcon"
	"regexp"
	"github.com/Masterminds/semver"
)

type FactorioServer struct {
	Cmd            *exec.Cmd              `json:"-"`
	Savefile       string                 `json:"savefile"`
	Latency        int                    `json:"latency"`
	BindIP         string                 `json:"bindip"`
	Port           int                    `json:"port"`
	Running        bool                   `json:"running"`
	Version        string                 `json:"fac_version"`
	BaseModVersion string                 `json:"base_mod_version"`
	SemVerVersion  *semver.Version        `json:"-"`
	StdOut         io.ReadCloser          `json:"-"`
	StdErr         io.ReadCloser          `json:"-"`
	StdIn          io.WriteCloser         `json:"-"`
	Settings       map[string]interface{} `json:"-"`
	Rcon           *rcon.RemoteConsole    `json:"-"`
	LogChan        chan []string          `json:"-"`
}

func randomPort() int {
	// Returns random port to use for rcon connection
	return rand.Intn(45000-40000) + 40000
}

func initFactorio() (f *FactorioServer, err error) {
	f = new(FactorioServer)
	f.Settings = make(map[string]interface{})

	if err = os.MkdirAll(config.FactorioConfigDir, 0755); err != nil {
		return nil, fmt.Errorf("failed to create config directory: %v", err)
	}

	settingsPath := filepath.Join(config.FactorioConfigDir, config.SettingsFile)
	var settings *os.File

	if _, err := os.Stat(settingsPath); os.IsNotExist(err) {
		// copy example settings to supplied settings file, if not exists
		log.Printf("Server settings at %s not found, copying example server settings.\n", settingsPath)

		examplePath := filepath.Join(config.FactorioDir, "data", "server-settings.example.json")

		example, err := os.Open(examplePath)
		if err != nil {
			return nil, fmt.Errorf("failed to open example server settings: %v", err)
		}
		defer example.Close()

		settings, err = os.Create(settingsPath)
		if err != nil {
			return nil, fmt.Errorf("failed to create server settings file: %v", err)
		}
		defer settings.Close()

		_, err = io.Copy(settings, example)
		if err != nil {
			return nil, fmt.Errorf("failed to copy example server settings: %v", err)
		}
	} else {
		// otherwise, open file normally
		settings, err = os.Open(settingsPath)
		if err != nil {
			return nil, fmt.Errorf("failed to open server settings file: %v", err)
		}
		defer settings.Close()
	}

	// before reading reset offset
	if _, err = settings.Seek(0, 0); err != nil {
		return nil, fmt.Errorf("error while seeking in settings file: %v", err)
	}

	if err = json.NewDecoder(settings).Decode(&f.Settings); err != nil {
		return nil, fmt.Errorf("error reading %s: %v", settingsPath, err)
	}

	log.Printf("Loaded Factorio settings from %s\n", settingsPath)

	//Load factorio version
	out, err := exec.Command(config.FactorioBinary, "--version").Output()
	if err != nil {
		log.Printf("error on loading factorio version: %s", err)
		return
	}

	reg := regexp.MustCompile("Version.*?((\\d+\\.)?(\\d+\\.)?(\\*|\\d+)+)")
	found := reg.FindStringSubmatch(string(out))
	f.Version = found[1]

	//Load baseMod version
	baseModInfoFile := filepath.Join(config.FactorioDir, "data", "base", "info.json")
	bmifBa, err := ioutil.ReadFile(baseModInfoFile)
	if err != nil {
		log.Printf("couldn't open baseMods info.json: %s", err)
		return
	}
	var modInfo ModInfo
	err = json.Unmarshal(bmifBa, &modInfo)
	if err != nil {
		log.Printf("error unmarshalling baseMods info.json to a modInfo: %s", err)
		return
	}

	f.BaseModVersion = modInfo.Version
	f.SemVerVersion, err = semver.NewVersion(modInfo.Version)
	if err != nil {
		log.Fatalf("error loading semver-factorio-version: %s", err)
	}

	return
}

func (f *FactorioServer) Run() error {
	var err error

	data, err := json.MarshalIndent(f.Settings, "", "  ")
	if err != nil {
		log.Println("Failed to marshal FactorioServerSettings: ", err)
	} else {
		ioutil.WriteFile(filepath.Join(config.FactorioConfigDir, config.SettingsFile), data, 0644)
	}

	args := []string{
		"--bind", (f.BindIP),
		"--port", strconv.Itoa(f.Port),
		"--server-settings", filepath.Join(config.FactorioConfigDir, "server-settings.json"),
		"--rcon-port", strconv.Itoa(config.FactorioRconPort),
		"--rcon-password", config.FactorioRconPass,
	}

	if f.Savefile == "Load Latest" {
		args = append(args, "--start-server-load-latest")
	} else {
		args = append(args, "--start-server", filepath.Join(config.FactorioSavesDir, f.Savefile))
	}

	log.Println("Starting server with command: ", config.FactorioBinary, args)

	f.Cmd = exec.Command(config.FactorioBinary, args...)

	f.StdOut, err = f.Cmd.StdoutPipe()
	if err != nil {
		log.Printf("Error opening stdout pipe: %s", err)
		return err
	}

	f.StdIn, err = f.Cmd.StdinPipe()
	if err != nil {
		log.Printf("Error opening stdin pipe: %s", err)
		return err
	}

	f.StdErr, err = f.Cmd.StderrPipe()
	if err != nil {
		log.Printf("Error opening stderr pipe: %s", err)
		return err
	}

	go f.parseRunningCommand(f.StdOut)
	go f.parseRunningCommand(f.StdErr)

	err = f.Cmd.Start()
	if err != nil {
		log.Printf("Factorio process failed to start: %s", err)
		return err
	}
	f.Running = true

	err = f.Cmd.Wait()
	if err != nil {
		log.Printf("Factorio process exited with error: %s", err)
		f.Running = false
		return err
	}

	return nil
}

func (f *FactorioServer) parseRunningCommand(std io.ReadCloser) (err error) {
	stdScanner := bufio.NewScanner(std)
	for stdScanner.Scan() {
		log.Printf("Factorio Server: %s", stdScanner.Text())
		if err := f.writeLog(stdScanner.Text()); err != nil {
			log.Printf("Error: %s", err)
		}

		line := strings.Fields(stdScanner.Text())
		// Ensure logline slice is in bounds
		if len(line) > 0 {
			// Check if Factorio Server reports any errors if so handle it
			if line[1] == "Error" {
				err := f.checkLogError(line)
				if err != nil {
					log.Printf("Error checking Factorio Server Error: %s", err)
				}
			}
			// If rcon port opens indicated in log connect to rcon
			rconLog := "Starting RCON interface at port " + strconv.Itoa(config.FactorioRconPort)
			// check if slice index is greater than 2 to prevent panic
			if len(line) > 2 {
				// log line for opened rcon connection
				if strings.Join(line[3:], " ") == rconLog {
					log.Printf("Rcon running on Factorio Server")
					err = connectRC()
					if err != nil {
						log.Printf("Error: %s", err)
					}
				}
			}
		}
	}
	if err := stdScanner.Err(); err != nil {
		log.Printf("Error reading std buffer: %s", err)
		return err
	}
	return nil
}

func (f *FactorioServer) writeLog(logline string) error {
	logfileName := config.FactorioDir + "factorio-server-console.log"
	file, err := os.OpenFile(logfileName, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		log.Printf("Cannot open logfile for appending Factorio Server output: %s", err)
		return err
	}
	defer file.Close()

	logline = logline + "\n"

	if _, err = file.WriteString(logline); err != nil {
		log.Printf("Error appending to factorio-server-console.log: %s", err)
		return err
	}

	return nil
}

func (f *FactorioServer) checkLogError(logline []string) error {
	// TODO Handle errors generated by running Factorio Server
	log.Println(logline)

	return nil
}

func (f *FactorioServer) Stop() error {
	if runtime.GOOS == "windows" {

		// Disable our own handling of CTRL+C, so we don't close when we send it to the console.
		setCtrlHandlingIsDisabledForThisProcess(true)

		// Send CTRL+C to all processes attached to the console (ourself, and the factorio server instance)
		sendCtrlCToPid(0)
		log.Println("Sent SIGINT to Factorio process. Factorio shutting down...")

		// Somehow, the Factorio devs managed to code the game to react appropriately to CTRL+C, including
		// saving the game, but not actually exit. So, we still have to manually kill the process, and
		// for extra fun, there's no way to know when the server save has actually completed (unless we want
		// to inject filesystem logic into what should be a process-level Stop() routine), so our best option
		// is to just wait an arbitrary amount of time and hope that the save is successful in that time.
		time.Sleep(2 * time.Second)
		f.Cmd.Process.Signal(os.Kill)

		// Re-enable handling of CTRL+C after we're sure that the factrio server is shut down.
		setCtrlHandlingIsDisabledForThisProcess(false)

		f.Running = false
		return nil
	}

	err := f.Cmd.Process.Signal(os.Interrupt)
	if err != nil {
		if err.Error() == "os: process already finished" {
			f.Running = false
			return err
		}
		log.Printf("Error sending SIGINT to Factorio process: %s", err)
		return err
	}
	f.Running = false
	log.Printf("Sent SIGINT to Factorio process. Factorio shutting down...")

	err = f.Rcon.Close()
	if err != nil {
		log.Printf("Error close rcon connection: %s", err)
	}

	return nil
}

func (f *FactorioServer) Kill() error {
	if runtime.GOOS == "windows" {

		err := f.Cmd.Process.Signal(os.Kill)
		if err != nil {
			if err.Error() == "os: process already finished" {
				f.Running = false
				return err
			}
			log.Printf("Error sending SIGKILL to Factorio process: %s", err)
			return err
		}
		f.Running = false
		log.Println("Sent SIGKILL to Factorio process. Factorio forced to exit.")

		return nil
	}

	err := f.Cmd.Process.Signal(os.Kill)
	if err != nil {
		if err.Error() == "os: process already finished" {
			f.Running = false
			return err
		}
		log.Printf("Error sending SIGKILL to Factorio process: %s", err)
		return err
	}
	f.Running = false
	log.Printf("Sent SIGKILL to Factorio process. Factorio forced to exit.")

	err = f.Rcon.Close()
	if err != nil {
		log.Printf("Error close rcon connection: %s", err)
	}

	return nil
}
<commit_msg>Fix crash on line without spaces<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/majormjr/rcon"
	"regexp"
	"github.com/Masterminds/semver"
)

type FactorioServer struct {
	Cmd            *exec.Cmd              `json:"-"`
	Savefile       string                 `json:"savefile"`
	Latency        int                    `json:"latency"`
	BindIP         string                 `json:"bindip"`
	Port           int                    `json:"port"`
	Running        bool                   `json:"running"`
	Version        string                 `json:"fac_version"`
	BaseModVersion string                 `json:"base_mod_version"`
	SemVerVersion  *semver.Version        `json:"-"`
	StdOut         io.ReadCloser          `json:"-"`
	StdErr         io.ReadCloser          `json:"-"`
	StdIn          io.WriteCloser         `json:"-"`
	Settings       map[string]interface{} `json:"-"`
	Rcon           *rcon.RemoteConsole    `json:"-"`
	LogChan        chan []string          `json:"-"`
}

func randomPort() int {
	// Returns random port to use for rcon connection
	return rand.Intn(45000-40000) + 40000
}

func initFactorio() (f *FactorioServer, err error) {
	f = new(FactorioServer)
	f.Settings = make(map[string]interface{})

	if err = os.MkdirAll(config.FactorioConfigDir, 0755); err != nil {
		return nil, fmt.Errorf("failed to create config directory: %v", err)
	}

	settingsPath := filepath.Join(config.FactorioConfigDir, config.SettingsFile)
	var settings *os.File

	if _, err := os.Stat(settingsPath); os.IsNotExist(err) {
		// copy example settings to supplied settings file, if not exists
		log.Printf("Server settings at %s not found, copying example server settings.\n", settingsPath)

		examplePath := filepath.Join(config.FactorioDir, "data", "server-settings.example.json")

		example, err := os.Open(examplePath)
		if err != nil {
			return nil, fmt.Errorf("failed to open example server settings: %v", err)
		}
		defer example.Close()

		settings, err = os.Create(settingsPath)
		if err != nil {
			return nil, fmt.Errorf("failed to create server settings file: %v", err)
		}
		defer settings.Close()

		_, err = io.Copy(settings, example)
		if err != nil {
			return nil, fmt.Errorf("failed to copy example server settings: %v", err)
		}
	} else {
		// otherwise, open file normally
		settings, err = os.Open(settingsPath)
		if err != nil {
			return nil, fmt.Errorf("failed to open server settings file: %v", err)
		}
		defer settings.Close()
	}

	// before reading reset offset
	if _, err = settings.Seek(0, 0); err != nil {
		return nil, fmt.Errorf("error while seeking in settings file: %v", err)
	}

	if err = json.NewDecoder(settings).Decode(&f.Settings); err != nil {
		return nil, fmt.Errorf("error reading %s: %v", settingsPath, err)
	}

	log.Printf("Loaded Factorio settings from %s\n", settingsPath)

	//Load factorio version
	out, err := exec.Command(config.FactorioBinary, "--version").Output()
	if err != nil {
		log.Printf("error on loading factorio version: %s", err)
		return
	}

	reg := regexp.MustCompile("Version.*?((\\d+\\.)?(\\d+\\.)?(\\*|\\d+)+)")
	found := reg.FindStringSubmatch(string(out))
	f.Version = found[1]

	//Load baseMod version
	baseModInfoFile := filepath.Join(config.FactorioDir, "data", "base", "info.json")
	bmifBa, err := ioutil.ReadFile(baseModInfoFile)
	if err != nil {
		log.Printf("couldn't open baseMods info.json: %s", err)
		return
	}
	var modInfo ModInfo
	err = json.Unmarshal(bmifBa, &modInfo)
	if err != nil {
		log.Printf("error unmarshalling baseMods info.json to a modInfo: %s", err)
		return
	}

	f.BaseModVersion = modInfo.Version
	f.SemVerVersion, err = semver.NewVersion(modInfo.Version)
	if err != nil {
		log.Fatalf("error loading semver-factorio-version: %s", err)
	}

	return
}

func (f *FactorioServer) Run() error {
	var err error

	data, err := json.MarshalIndent(f.Settings, "", "  ")
	if err != nil {
		log.Println("Failed to marshal FactorioServerSettings: ", err)
	} else {
		ioutil.WriteFile(filepath.Join(config.FactorioConfigDir, config.SettingsFile), data, 0644)
	}

	args := []string{
		"--bind", (f.BindIP),
		"--port", strconv.Itoa(f.Port),
		"--server-settings", filepath.Join(config.FactorioConfigDir, "server-settings.json"),
		"--rcon-port", strconv.Itoa(config.FactorioRconPort),
		"--rcon-password", config.FactorioRconPass,
	}

	if f.Savefile == "Load Latest" {
		args = append(args, "--start-server-load-latest")
	} else {
		args = append(args, "--start-server", filepath.Join(config.FactorioSavesDir, f.Savefile))
	}

	log.Println("Starting server with command: ", config.FactorioBinary, args)

	f.Cmd = exec.Command(config.FactorioBinary, args...)

	f.StdOut, err = f.Cmd.StdoutPipe()
	if err != nil {
		log.Printf("Error opening stdout pipe: %s", err)
		return err
	}

	f.StdIn, err = f.Cmd.StdinPipe()
	if err != nil {
		log.Printf("Error opening stdin pipe: %s", err)
		return err
	}

	f.StdErr, err = f.Cmd.StderrPipe()
	if err != nil {
		log.Printf("Error opening stderr pipe: %s", err)
		return err
	}

	go f.parseRunningCommand(f.StdOut)
	go f.parseRunningCommand(f.StdErr)

	err = f.Cmd.Start()
	if err != nil {
		log.Printf("Factorio process failed to start: %s", err)
		return err
	}
	f.Running = true

	err = f.Cmd.Wait()
	if err != nil {
		log.Printf("Factorio process exited with error: %s", err)
		f.Running = false
		return err
	}

	return nil
}

func (f *FactorioServer) parseRunningCommand(std io.ReadCloser) (err error) {
	stdScanner := bufio.NewScanner(std)
	for stdScanner.Scan() {
		log.Printf("Factorio Server: %s", stdScanner.Text())
		if err := f.writeLog(stdScanner.Text()); err != nil {
			log.Printf("Error: %s", err)
		}

		line := strings.Fields(stdScanner.Text())
		// Ensure logline slice is in bounds
		if len(line) > 1 {
			// Check if Factorio Server reports any errors if so handle it
			if line[1] == "Error" {
				err := f.checkLogError(line)
				if err != nil {
					log.Printf("Error checking Factorio Server Error: %s", err)
				}
			}
			// If rcon port opens indicated in log connect to rcon
			rconLog := "Starting RCON interface at port " + strconv.Itoa(config.FactorioRconPort)
			// check if slice index is greater than 2 to prevent panic
			if len(line) > 2 {
				// log line for opened rcon connection
				if strings.Join(line[3:], " ") == rconLog {
					log.Printf("Rcon running on Factorio Server")
					err = connectRC()
					if err != nil {
						log.Printf("Error: %s", err)
					}
				}
			}
		}
	}
	if err := stdScanner.Err(); err != nil {
		log.Printf("Error reading std buffer: %s", err)
		return err
	}
	return nil
}

func (f *FactorioServer) writeLog(logline string) error {
	logfileName := config.FactorioDir + "factorio-server-console.log"
	file, err := os.OpenFile(logfileName, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		log.Printf("Cannot open logfile for appending Factorio Server output: %s", err)
		return err
	}
	defer file.Close()

	logline = logline + "\n"

	if _, err = file.WriteString(logline); err != nil {
		log.Printf("Error appending to factorio-server-console.log: %s", err)
		return err
	}

	return nil
}

func (f *FactorioServer) checkLogError(logline []string) error {
	// TODO Handle errors generated by running Factorio Server
	log.Println(logline)

	return nil
}

func (f *FactorioServer) Stop() error {
	if runtime.GOOS == "windows" {

		// Disable our own handling of CTRL+C, so we don't close when we send it to the console.
		setCtrlHandlingIsDisabledForThisProcess(true)

		// Send CTRL+C to all processes attached to the console (ourself, and the factorio server instance)
		sendCtrlCToPid(0)
		log.Println("Sent SIGINT to Factorio process. Factorio shutting down...")

		// Somehow, the Factorio devs managed to code the game to react appropriately to CTRL+C, including
		// saving the game, but not actually exit. So, we still have to manually kill the process, and
		// for extra fun, there's no way to know when the server save has actually completed (unless we want
		// to inject filesystem logic into what should be a process-level Stop() routine), so our best option
		// is to just wait an arbitrary amount of time and hope that the save is successful in that time.
		time.Sleep(2 * time.Second)
		f.Cmd.Process.Signal(os.Kill)

		// Re-enable handling of CTRL+C after we're sure that the factrio server is shut down.
		setCtrlHandlingIsDisabledForThisProcess(false)

		f.Running = false
		return nil
	}

	err := f.Cmd.Process.Signal(os.Interrupt)
	if err != nil {
		if err.Error() == "os: process already finished" {
			f.Running = false
			return err
		}
		log.Printf("Error sending SIGINT to Factorio process: %s", err)
		return err
	}
	f.Running = false
	log.Printf("Sent SIGINT to Factorio process. Factorio shutting down...")

	err = f.Rcon.Close()
	if err != nil {
		log.Printf("Error close rcon connection: %s", err)
	}

	return nil
}

func (f *FactorioServer) Kill() error {
	if runtime.GOOS == "windows" {

		err := f.Cmd.Process.Signal(os.Kill)
		if err != nil {
			if err.Error() == "os: process already finished" {
				f.Running = false
				return err
			}
			log.Printf("Error sending SIGKILL to Factorio process: %s", err)
			return err
		}
		f.Running = false
		log.Println("Sent SIGKILL to Factorio process. Factorio forced to exit.")

		return nil
	}

	err := f.Cmd.Process.Signal(os.Kill)
	if err != nil {
		if err.Error() == "os: process already finished" {
			f.Running = false
			return err
		}
		log.Printf("Error sending SIGKILL to Factorio process: %s", err)
		return err
	}
	f.Running = false
	log.Printf("Sent SIGKILL to Factorio process. Factorio forced to exit.")

	err = f.Rcon.Close()
	if err != nil {
		log.Printf("Error close rcon connection: %s", err)
	}

	return nil
}
<|endoftext|>


<commit_before>// Package transport is an interface for connection based communication
package transport

import (
	"time"
)

// Transport is an interface which is used for communication between
// services. It uses connection based socket send/recv semantics and
// has various implementations; http, grpc, quic.
type Transport interface {
	Init(...Option) error
	Options() Options
	Dial(addr string, opts ...DialOption) (Client, error)
	Listen(addr string, opts ...ListenOption) (Listener, error)
	String() string
}

type Message struct {
	Header map[string]string
	Body   []byte
}

type Socket interface {
	Recv(*Message) error
	Send(*Message) error
	Close() error
	Local() string
	Remote() string
}

type Client interface {
	Socket
}

type Listener interface {
	Addr() string
	Close() error
	Accept(func(Socket)) error
}

type Option func(*Options)

type DialOption func(*DialOptions)

type ListenOption func(*ListenOptions)

var (
	DefaultTransport Transport = newHTTPTransport()

	DefaultDialTimeout = time.Second * 5
)

func NewTransport(opts ...Option) Transport {
	return newHTTPTransport(opts...)
}
<commit_msg>update transport package comments<commit_after>// Package transport is an interface for synchronous connection based communication
package transport

import (
	"time"
)

// Transport is an interface which is used for communication between
// services. It uses connection based socket send/recv semantics and
// has various implementations; http, grpc, quic.
type Transport interface {
	Init(...Option) error
	Options() Options
	Dial(addr string, opts ...DialOption) (Client, error)
	Listen(addr string, opts ...ListenOption) (Listener, error)
	String() string
}

type Message struct {
	Header map[string]string
	Body   []byte
}

type Socket interface {
	Recv(*Message) error
	Send(*Message) error
	Close() error
	Local() string
	Remote() string
}

type Client interface {
	Socket
}

type Listener interface {
	Addr() string
	Close() error
	Accept(func(Socket)) error
}

type Option func(*Options)

type DialOption func(*DialOptions)

type ListenOption func(*ListenOptions)

var (
	DefaultTransport Transport = newHTTPTransport()

	DefaultDialTimeout = time.Second * 5
)

func NewTransport(opts ...Option) Transport {
	return newHTTPTransport(opts...)
}
<|endoftext|>