<commit_before>package s3api

import (
	"encoding/xml"
	"crypto/sha1"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/glog"
	xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
	weed_server "github.com/chrislusf/seaweedfs/weed/server"
	"io"
	"net/http"
	"net/url"
	"strconv"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"
)

const (
	maxObjectListSizeLimit = 10000 // Limit number of objects in a listObjectsResponse.
	maxUploadsList         = 10000 // Limit number of uploads in a listUploadsResponse.
	maxPartsList           = 10000 // Limit number of parts in a listPartsResponse.
	globalMaxPartID        = 100000
)

// NewMultipartUploadHandler - New multipart upload.
func (s3a *S3ApiServer) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
	bucket, object := xhttp.GetBucketAndObject(r)

	createMultipartUploadInput := &s3.CreateMultipartUploadInput{
		Bucket:   aws.String(bucket),
		Key:      objectKey(aws.String(object)),
		Metadata: make(map[string]*string),
	}

	metadata := weed_server.SaveAmzMetaData(r, nil, false)
	for k, v := range metadata {
		createMultipartUploadInput.Metadata[k] = aws.String(string(v))
	}

	contentType := r.Header.Get("Content-Type")
	if contentType != "" {
		createMultipartUploadInput.ContentType = &contentType
	}
	response, errCode := s3a.createMultipartUpload(createMultipartUploadInput)

	glog.V(2).Info("NewMultipartUploadHandler", string(s3err.EncodeXMLResponse(response)), errCode)

	if errCode != s3err.ErrNone {
		s3err.WriteErrorResponse(w, r, errCode)
		return
	}

	writeSuccessResponseXML(w, r, response)

}

// CompleteMultipartUploadHandler - Completes multipart upload.
func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
	// https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html

	bucket, object := xhttp.GetBucketAndObject(r)

	parts := &CompleteMultipartUpload{}
	if err := xmlDecoder(r.Body, parts, r.ContentLength); err != nil {
		s3err.WriteErrorResponse(w, r, s3err.ErrMalformedXML)
		return
	}

	// Get upload id.
	uploadID, _, _, _ := getObjectResources(r.URL.Query())
	err := s3a.chkUploadID(object, uploadID)
	if err != nil {
		s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload)
		return
	}

	response, errCode := s3a.completeMultipartUpload(&s3.CompleteMultipartUploadInput{
		Bucket:   aws.String(bucket),
		Key:      objectKey(aws.String(object)),
		UploadId: aws.String(uploadID),
	}, parts)

	glog.V(2).Info("CompleteMultipartUploadHandler", string(s3err.EncodeXMLResponse(response)), errCode)

	if errCode != s3err.ErrNone {
		s3err.WriteErrorResponse(w, r, errCode)
		return
	}

	writeSuccessResponseXML(w, r, response)

}

// AbortMultipartUploadHandler - Aborts multipart upload.
func (s3a *S3ApiServer) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
	bucket, object := xhttp.GetBucketAndObject(r)

	// Get upload id.
	uploadID, _, _, _ := getObjectResources(r.URL.Query())
	err := s3a.chkUploadID(object, uploadID)
	if err != nil {
		s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload)
		return
	}

	response, errCode := s3a.abortMultipartUpload(&s3.AbortMultipartUploadInput{
		Bucket:   aws.String(bucket),
		Key:      objectKey(aws.String(object)),
		UploadId: aws.String(uploadID),
	})

	if errCode != s3err.ErrNone {
		s3err.WriteErrorResponse(w, r, errCode)
		return
	}

	glog.V(2).Info("AbortMultipartUploadHandler", string(s3err.EncodeXMLResponse(response)))

	writeSuccessResponseXML(w, r, response)

}

// ListMultipartUploadsHandler - Lists multipart uploads.
func (s3a *S3ApiServer) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
	bucket, _ := xhttp.GetBucketAndObject(r)

	prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, encodingType := getBucketMultipartResources(r.URL.Query())
	if maxUploads < 0 {
		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxUploads)
		return
	}
	if keyMarker != "" {
		// Marker not common with prefix is not implemented.
		if !strings.HasPrefix(keyMarker, prefix) {
			s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
			return
		}
	}

	response, errCode := s3a.listMultipartUploads(&s3.ListMultipartUploadsInput{
		Bucket:         aws.String(bucket),
		Delimiter:      aws.String(delimiter),
		EncodingType:   aws.String(encodingType),
		KeyMarker:      aws.String(keyMarker),
		MaxUploads:     aws.Int64(int64(maxUploads)),
		Prefix:         aws.String(prefix),
		UploadIdMarker: aws.String(uploadIDMarker),
	})

	glog.V(2).Infof("ListMultipartUploadsHandler %s errCode=%d", string(s3err.EncodeXMLResponse(response)), errCode)

	if errCode != s3err.ErrNone {
		s3err.WriteErrorResponse(w, r, errCode)
		return
	}

	// TODO handle encodingType

	writeSuccessResponseXML(w, r, response)
}

// ListObjectPartsHandler - Lists object parts in a multipart upload.
func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Request) {
	bucket, object := xhttp.GetBucketAndObject(r)

	uploadID, partNumberMarker, maxParts, _ := getObjectResources(r.URL.Query())
	if partNumberMarker < 0 {
		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidPartNumberMarker)
		return
	}
	if maxParts < 0 {
		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxParts)
		return
	}

	err := s3a.chkUploadID(object, uploadID)
	if err != nil {
		s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload)
		return
	}

	response, errCode := s3a.listObjectParts(&s3.ListPartsInput{
		Bucket:           aws.String(bucket),
		Key:              objectKey(aws.String(object)),
		MaxParts:         aws.Int64(int64(maxParts)),
		PartNumberMarker: aws.Int64(int64(partNumberMarker)),
		UploadId:         aws.String(uploadID),
	})

	if errCode != s3err.ErrNone {
		s3err.WriteErrorResponse(w, r, errCode)
		return
	}

	glog.V(2).Infof("ListObjectPartsHandler %s count=%d", string(s3err.EncodeXMLResponse(response)), len(response.Part))

	writeSuccessResponseXML(w, r, response)

}

// PutObjectPartHandler - Put an object part in a multipart upload.
func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) {
	bucket, object := xhttp.GetBucketAndObject(r)

	uploadID := r.URL.Query().Get("uploadId")
	exists, err := s3a.exists(s3a.genUploadsFolder(bucket), uploadID, true)
	if !exists {
		s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload)
		return
	}

	err = s3a.chkUploadID(object, uploadID)
	if err != nil {
		s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload)
		return
	}

	partIDString := r.URL.Query().Get("partNumber")
	partID, err := strconv.Atoi(partIDString)
	if err != nil {
		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidPart)
		return
	}
	if partID > globalMaxPartID {
		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxParts)
		return
	}

	dataReader := r.Body
	if s3a.iam.isEnabled() {
		rAuthType := getRequestAuthType(r)
		var s3ErrCode s3err.ErrorCode
		switch rAuthType {
		case authTypeStreamingSigned:
			dataReader, s3ErrCode = s3a.iam.newSignV4ChunkedReader(r)
		case authTypeSignedV2, authTypePresignedV2:
			_, s3ErrCode = s3a.iam.isReqAuthenticatedV2(r)
		case authTypePresigned, authTypeSigned:
			_, s3ErrCode = s3a.iam.reqSignatureV4Verify(r)
		}
		if s3ErrCode != s3err.ErrNone {
			s3err.WriteErrorResponse(w, r, s3ErrCode)
			return
		}
	}
	defer dataReader.Close()

	glog.V(2).Infof("PutObjectPartHandler %s %s %04d", bucket, uploadID, partID)

	uploadUrl := fmt.Sprintf("http://%s%s/%s/%04d.part?collection=%s",
		s3a.option.Filer.ToHttpAddress(), s3a.genUploadsFolder(bucket), uploadID, partID, bucket)

	if partID == 1 && r.Header.Get("Content-Type") == "" {
		dataReader = mimeDetect(r, dataReader)
	}

	etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader)
	if errCode != s3err.ErrNone {
		s3err.WriteErrorResponse(w, r, errCode)
		return
	}

	setEtag(w, etag)

	writeSuccessResponseEmpty(w, r)

}

func (s3a *S3ApiServer) genUploadsFolder(bucket string) string {
	return fmt.Sprintf("%s/%s/.uploads", s3a.option.BucketsPath, bucket)
}

// Generate uploadID hash string from object
func (s3a *S3ApiServer) generateUploadID(object string) string {
	if strings.HasPrefix(object, "/") {
		object = object[1:]
	}
	h := sha1.New()
	h.Write([]byte(object))
	return fmt.Sprintf("%x", h.Sum(nil))
}

//Check object name and uploadID when processing  multipart uploading
func (s3a *S3ApiServer) chkUploadID(object string, id string) error {

	hash := s3a.generateUploadID(object)
	if hash != id {
		glog.Errorf("object %s and uploadID %s are not matched", object, id)
		return fmt.Errorf("object %s and uploadID %s are not matched", object, id)
	}
	return nil
}

// Parse bucket url queries for ?uploads
func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int, encodingType string) {
	prefix = values.Get("prefix")
	keyMarker = values.Get("key-marker")
	uploadIDMarker = values.Get("upload-id-marker")
	delimiter = values.Get("delimiter")
	if values.Get("max-uploads") != "" {
		maxUploads, _ = strconv.Atoi(values.Get("max-uploads"))
	} else {
		maxUploads = maxUploadsList
	}
	encodingType = values.Get("encoding-type")
	return
}

// Parse object url queries
func getObjectResources(values url.Values) (uploadID string, partNumberMarker, maxParts int, encodingType string) {
	uploadID = values.Get("uploadId")
	partNumberMarker, _ = strconv.Atoi(values.Get("part-number-marker"))
	if values.Get("max-parts") != "" {
		maxParts, _ = strconv.Atoi(values.Get("max-parts"))
	} else {
		maxParts = maxPartsList
	}
	encodingType = values.Get("encoding-type")
	return
}

func xmlDecoder(body io.Reader, v interface{}, size int64) error {
	var lbody io.Reader
	if size > 0 {
		lbody = io.LimitReader(body, size)
	} else {
		lbody = body
	}
	d := xml.NewDecoder(lbody)
	d.CharsetReader = func(label string, input io.Reader) (io.Reader, error) {
		return input, nil
	}
	return d.Decode(v)
}

type CompleteMultipartUpload struct {
	Parts []CompletedPart `xml:"Part"`
}
type CompletedPart struct {
	ETag       string
	PartNumber int
}
<commit_msg>rename functions and remove uncessary check<commit_after>package s3api

import (
	"encoding/xml"
	"crypto/sha1"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/glog"
	xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
	weed_server "github.com/chrislusf/seaweedfs/weed/server"
	"io"
	"net/http"
	"net/url"
	"strconv"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"
)

const (
	maxObjectListSizeLimit = 10000 // Limit number of objects in a listObjectsResponse.
	maxUploadsList         = 10000 // Limit number of uploads in a listUploadsResponse.
	maxPartsList           = 10000 // Limit number of parts in a listPartsResponse.
	globalMaxPartID        = 100000
)

// NewMultipartUploadHandler - New multipart upload.
func (s3a *S3ApiServer) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
	bucket, object := xhttp.GetBucketAndObject(r)

	createMultipartUploadInput := &s3.CreateMultipartUploadInput{
		Bucket:   aws.String(bucket),
		Key:      objectKey(aws.String(object)),
		Metadata: make(map[string]*string),
	}

	metadata := weed_server.SaveAmzMetaData(r, nil, false)
	for k, v := range metadata {
		createMultipartUploadInput.Metadata[k] = aws.String(string(v))
	}

	contentType := r.Header.Get("Content-Type")
	if contentType != "" {
		createMultipartUploadInput.ContentType = &contentType
	}
	response, errCode := s3a.createMultipartUpload(createMultipartUploadInput)

	glog.V(2).Info("NewMultipartUploadHandler", string(s3err.EncodeXMLResponse(response)), errCode)

	if errCode != s3err.ErrNone {
		s3err.WriteErrorResponse(w, r, errCode)
		return
	}

	writeSuccessResponseXML(w, r, response)

}

// CompleteMultipartUploadHandler - Completes multipart upload.
func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
	// https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html

	bucket, object := xhttp.GetBucketAndObject(r)

	parts := &CompleteMultipartUpload{}
	if err := xmlDecoder(r.Body, parts, r.ContentLength); err != nil {
		s3err.WriteErrorResponse(w, r, s3err.ErrMalformedXML)
		return
	}

	// Get upload id.
	uploadID, _, _, _ := getObjectResources(r.URL.Query())
	err := s3a.checkUploadId(object, uploadID)
	if err != nil {
		s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload)
		return
	}

	response, errCode := s3a.completeMultipartUpload(&s3.CompleteMultipartUploadInput{
		Bucket:   aws.String(bucket),
		Key:      objectKey(aws.String(object)),
		UploadId: aws.String(uploadID),
	}, parts)

	glog.V(2).Info("CompleteMultipartUploadHandler", string(s3err.EncodeXMLResponse(response)), errCode)

	if errCode != s3err.ErrNone {
		s3err.WriteErrorResponse(w, r, errCode)
		return
	}

	writeSuccessResponseXML(w, r, response)

}

// AbortMultipartUploadHandler - Aborts multipart upload.
func (s3a *S3ApiServer) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
	bucket, object := xhttp.GetBucketAndObject(r)

	// Get upload id.
	uploadID, _, _, _ := getObjectResources(r.URL.Query())
	err := s3a.checkUploadId(object, uploadID)
	if err != nil {
		s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload)
		return
	}

	response, errCode := s3a.abortMultipartUpload(&s3.AbortMultipartUploadInput{
		Bucket:   aws.String(bucket),
		Key:      objectKey(aws.String(object)),
		UploadId: aws.String(uploadID),
	})

	if errCode != s3err.ErrNone {
		s3err.WriteErrorResponse(w, r, errCode)
		return
	}

	glog.V(2).Info("AbortMultipartUploadHandler", string(s3err.EncodeXMLResponse(response)))

	writeSuccessResponseXML(w, r, response)

}

// ListMultipartUploadsHandler - Lists multipart uploads.
func (s3a *S3ApiServer) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
	bucket, _ := xhttp.GetBucketAndObject(r)

	prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, encodingType := getBucketMultipartResources(r.URL.Query())
	if maxUploads < 0 {
		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxUploads)
		return
	}
	if keyMarker != "" {
		// Marker not common with prefix is not implemented.
		if !strings.HasPrefix(keyMarker, prefix) {
			s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
			return
		}
	}

	response, errCode := s3a.listMultipartUploads(&s3.ListMultipartUploadsInput{
		Bucket:         aws.String(bucket),
		Delimiter:      aws.String(delimiter),
		EncodingType:   aws.String(encodingType),
		KeyMarker:      aws.String(keyMarker),
		MaxUploads:     aws.Int64(int64(maxUploads)),
		Prefix:         aws.String(prefix),
		UploadIdMarker: aws.String(uploadIDMarker),
	})

	glog.V(2).Infof("ListMultipartUploadsHandler %s errCode=%d", string(s3err.EncodeXMLResponse(response)), errCode)

	if errCode != s3err.ErrNone {
		s3err.WriteErrorResponse(w, r, errCode)
		return
	}

	// TODO handle encodingType

	writeSuccessResponseXML(w, r, response)
}

// ListObjectPartsHandler - Lists object parts in a multipart upload.
func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Request) {
	bucket, object := xhttp.GetBucketAndObject(r)

	uploadID, partNumberMarker, maxParts, _ := getObjectResources(r.URL.Query())
	if partNumberMarker < 0 {
		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidPartNumberMarker)
		return
	}
	if maxParts < 0 {
		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxParts)
		return
	}

	err := s3a.checkUploadId(object, uploadID)
	if err != nil {
		s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload)
		return
	}

	response, errCode := s3a.listObjectParts(&s3.ListPartsInput{
		Bucket:           aws.String(bucket),
		Key:              objectKey(aws.String(object)),
		MaxParts:         aws.Int64(int64(maxParts)),
		PartNumberMarker: aws.Int64(int64(partNumberMarker)),
		UploadId:         aws.String(uploadID),
	})

	if errCode != s3err.ErrNone {
		s3err.WriteErrorResponse(w, r, errCode)
		return
	}

	glog.V(2).Infof("ListObjectPartsHandler %s count=%d", string(s3err.EncodeXMLResponse(response)), len(response.Part))

	writeSuccessResponseXML(w, r, response)

}

// PutObjectPartHandler - Put an object part in a multipart upload.
func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) {
	bucket, object := xhttp.GetBucketAndObject(r)

	uploadID := r.URL.Query().Get("uploadId")
	err := s3a.checkUploadId(object, uploadID)
	if err != nil {
		s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload)
		return
	}

	partIDString := r.URL.Query().Get("partNumber")
	partID, err := strconv.Atoi(partIDString)
	if err != nil {
		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidPart)
		return
	}
	if partID > globalMaxPartID {
		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxParts)
		return
	}

	dataReader := r.Body
	if s3a.iam.isEnabled() {
		rAuthType := getRequestAuthType(r)
		var s3ErrCode s3err.ErrorCode
		switch rAuthType {
		case authTypeStreamingSigned:
			dataReader, s3ErrCode = s3a.iam.newSignV4ChunkedReader(r)
		case authTypeSignedV2, authTypePresignedV2:
			_, s3ErrCode = s3a.iam.isReqAuthenticatedV2(r)
		case authTypePresigned, authTypeSigned:
			_, s3ErrCode = s3a.iam.reqSignatureV4Verify(r)
		}
		if s3ErrCode != s3err.ErrNone {
			s3err.WriteErrorResponse(w, r, s3ErrCode)
			return
		}
	}
	defer dataReader.Close()

	glog.V(2).Infof("PutObjectPartHandler %s %s %04d", bucket, uploadID, partID)

	uploadUrl := fmt.Sprintf("http://%s%s/%s/%04d.part?collection=%s",
		s3a.option.Filer.ToHttpAddress(), s3a.genUploadsFolder(bucket), uploadID, partID, bucket)

	if partID == 1 && r.Header.Get("Content-Type") == "" {
		dataReader = mimeDetect(r, dataReader)
	}

	etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader)
	if errCode != s3err.ErrNone {
		s3err.WriteErrorResponse(w, r, errCode)
		return
	}

	setEtag(w, etag)

	writeSuccessResponseEmpty(w, r)

}

func (s3a *S3ApiServer) genUploadsFolder(bucket string) string {
	return fmt.Sprintf("%s/%s/.uploads", s3a.option.BucketsPath, bucket)
}

// Generate uploadID hash string from object
func (s3a *S3ApiServer) generateUploadID(object string) string {
	if strings.HasPrefix(object, "/") {
		object = object[1:]
	}
	h := sha1.New()
	h.Write([]byte(object))
	return fmt.Sprintf("%x", h.Sum(nil))
}

//Check object name and uploadID when processing  multipart uploading
func (s3a *S3ApiServer) checkUploadId(object string, id string) error {

	hash := s3a.generateUploadID(object)
	if hash != id {
		glog.Errorf("object %s and uploadID %s are not matched", object, id)
		return fmt.Errorf("object %s and uploadID %s are not matched", object, id)
	}
	return nil
}

// Parse bucket url queries for ?uploads
func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int, encodingType string) {
	prefix = values.Get("prefix")
	keyMarker = values.Get("key-marker")
	uploadIDMarker = values.Get("upload-id-marker")
	delimiter = values.Get("delimiter")
	if values.Get("max-uploads") != "" {
		maxUploads, _ = strconv.Atoi(values.Get("max-uploads"))
	} else {
		maxUploads = maxUploadsList
	}
	encodingType = values.Get("encoding-type")
	return
}

// Parse object url queries
func getObjectResources(values url.Values) (uploadID string, partNumberMarker, maxParts int, encodingType string) {
	uploadID = values.Get("uploadId")
	partNumberMarker, _ = strconv.Atoi(values.Get("part-number-marker"))
	if values.Get("max-parts") != "" {
		maxParts, _ = strconv.Atoi(values.Get("max-parts"))
	} else {
		maxParts = maxPartsList
	}
	encodingType = values.Get("encoding-type")
	return
}

func xmlDecoder(body io.Reader, v interface{}, size int64) error {
	var lbody io.Reader
	if size > 0 {
		lbody = io.LimitReader(body, size)
	} else {
		lbody = body
	}
	d := xml.NewDecoder(lbody)
	d.CharsetReader = func(label string, input io.Reader) (io.Reader, error) {
		return input, nil
	}
	return d.Decode(v)
}

type CompleteMultipartUpload struct {
	Parts []CompletedPart `xml:"Part"`
}
type CompletedPart struct {
	ETag       string
	PartNumber int
}
<|endoftext|>


<commit_before>/*

	static is a library that helps automate creating the static directory of
	files for the webapp to function. It, along with lib/build/api, is one
	half of the build process to produce a functioning webapp given a
	config.json

	All of the steps it does could be done by hand, but they are finicky and
	error prone, so this library helps take the guess work out of them.

	Build() is the primary entrypoint for this package, which composes all of
	the build steps, which are themselves exposed as public methods int his
	package, in the right order.

	All build methods in this package take a dir parameter. This is the
	directory to produce the build into. In particular, these methods will
	create a `static` subdirectory in dir and work inside of that. dir may be
	"", which will create the static sub-folder within the current directory.
	Tools like `boardgame-util serve` create a temporary directory and use
	that, so it's easy to clean up later.

	The steps of the build process, at a high level, are as follows:

	First, create the `static` sub directory, if it doesn't already exist. All
	following steps create files and directories within that static subfolder.

	Next, it copies over all of the static resources (no directories) from
	`github.com/jkomoros/boardgame/server/static`, skipping a handful of files
	that will be generated later. These files are symlinked by default, but
	can also be copied. This step is encapsulated by CopyStaticResources.

	Next, it creates a node_modules folder that contains up to date
	dependencies given the contents of
	`github.com/jkomoros/boardgame/server/static/package.json`. Checking out
	this whole directory is expensive, so this package creates a node_modules
	in a central cache, re-upping it each time this command is run, and then
	symlinks it into the static directory. This step is encapsulated by
	LinkNodeModules.

	Next, it generates a `client_config.js`, which encodes the global
	configuration for the client webapp. It calls config.Client(false) and
	saves the result to static/client-config.js, which index.html will look
	for when booting up. This step is encapsulated by CreateClientConfigJs.

	Next, it copies in the client folders (containing boardgame-render-game-
	GAMENAME.js, and optionally boardgame-render-player-info-GAMENAME.js) into
	static/game-src. It does this by locating the on-disk location of each
	gameImport given by gameImports (typically this is configMode.Games), then
	symlinking its client folder into `static/game-src/GAMENAME`. In a modules
	context, game packages that do not yet exist on disk will be downloaded
	automatically; if you are not using modules and you have not yet `go get`
	the given game imports or a containg package, it will error. This step is
	encapsulated by LinkGameClientFolders.

	Next, it generates a `static/polymer.json`, which contains fragments
	entries for each dynamic import--specifically, the `game-src/GAMENAME
	/boardgame-render-game-GAMENAME.js` and `game-src/GAMENAME/boardgame-
	render-player-info-GAMENAME.js`, if it exists. It identifes the fragments
	to include by walking through all of the game directories in `game-src`,
	meaning it relies on the output of the previous step. It then saves this
	generated file to `static/polymer.json`. This step is encapsulated by
	CreatePolymerJson.

	The static build is not mostly complete. Optionally, BuildPolymer can be
	called to run `polymer build` on the generated static dir. This step is
	encapsulated by BuildPolymer.

	Typically direct users of this package use Build(), which automatically
	runs these steps in the proper order.

	Typically you don't use this package directly, but use `boardgame-util
	build static` or `boardgame-util serve`.

*/
package static

import (
	"errors"
	"fmt"
	"github.com/jkomoros/boardgame/boardgame-util/lib/config"
	"os"
	"os/exec"
	"path/filepath"
)

var filesToExclude map[string]bool = map[string]bool{
	".gitignore":      true,
	"README.md":       true,
	polymerConfig:     true,
	nodeModulesFolder: true,
	//Don't copy over because we'll generate our own; if we copy over and
	//generate our own we'll overwrite original.
	clientConfigJsFileName: true,
	".DS_Store":            true,
}

//Server runs a static server. directory is the folder that the `static`
//folder is contained within. If no error is returned, runs until the program
//exits. Under the cover uses `polymer serve` because imports use bare module
//specifiers that must be rewritten.
func Server(directory string, port string) error {

	if err := verifyPolymer(directory); err != nil {
		return err
	}

	staticDir := filepath.Join(directory, staticSubFolder)

	cmd := exec.Command("polymer", "serve", "--port="+port)
	cmd.Dir = staticDir
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	if err := cmd.Run(); err != nil {
		return errors.New("Couldn't `polymer serve`: " + err.Error())
	}

	return nil

}

//CleanCache clears the central cache the build system uses (currently just
//node_modules). If that cache doesn't exist, is a no op.
func CleanCache() error {

	cacheDir, err := buildCachePath()
	if err != nil {
		return errors.New("Couldn't get build cache path: " + err.Error())
	}

	//os.RemoveAll is OK if the path doesn't exist
	return os.RemoveAll(cacheDir)

}

/*

Build creates a folder of static resources for a server in the given
directory. It is the primary entrypoint for this package. It has no logic of
its own but serves to call all of the build steps in the correct order.

Specificlaly, it calls: CopyStaticResources, passing copyFiles;
LinkNodeModules; CreateClientConfigJs, passing c; LinkGameClientFolders,
passing gameImports; CreatePolymerJson, passing false. If prodBuild is true,
also calls BuildPolymer.

See the package doc for more about the specific build steps and what they do.

*/
func Build(directory string, gameImports []string, c *config.Config, prodBuild bool, copyFiles bool) (assetRoot string, err error) {

	staticDir, err := staticBuildDir(directory)
	if err != nil {
		return "", err
	}

	fmt.Println("Copying base static resources")
	if err := CopyStaticResources(directory, copyFiles); err != nil {
		return "", errors.New("Couldn't copy static resources")
	}

	fmt.Println("Updating " + nodeModulesFolder + " and linking in")
	if err := LinkNodeModules(directory); err != nil {
		return "", errors.New("Couldn't link " + nodeModulesFolder + ": " + err.Error())
	}

	fmt.Println("Creating " + clientConfigJsFileName)
	if err := CreateClientConfigJs(directory, c); err != nil {
		return "", errors.New("Couldn't create " + clientConfigJsFileName + ": " + err.Error())
	}

	fmt.Println("Creating " + gameSrcSubFolder)
	if err := LinkGameClientFolders(directory, gameImports); err != nil {
		return "", errors.New("Couldn't create " + gameSrcSubFolder + ": " + err.Error())
	}

	fmt.Println("Creating " + polymerConfig)
	if err := CreatePolymerJson(directory, false); err != nil {
		return "", errors.New("Couldn't create " + polymerConfig + ": " + err.Error())
	}

	if prodBuild {
		fmt.Println("Building bundled resources with `polymer build`")
		if err := BuildPolymer(directory); err != nil {
			return "", errors.New("Couldn't build bundled resources: " + err.Error())
		}
	}

	return staticDir, nil

}

//Clean removes all of the things created in the static subfolder within
//directory.
func Clean(directory string) error {
	return os.RemoveAll(filepath.Join(directory, staticSubFolder))
}
<commit_msg>Fix minor typo in last commit. Part of #671.<commit_after>/*

	static is a library that helps automate creating the static directory of
	files for the webapp to function. It, along with lib/build/api, is one
	half of the build process to produce a functioning webapp given a
	config.json

	All of the steps it does could be done by hand, but they are finicky and
	error prone, so this library helps take the guess work out of them.

	Build() is the primary entrypoint for this package, which composes all of
	the build steps, which are themselves exposed as public methods int his
	package, in the right order.

	All build methods in this package take a dir parameter. This is the
	directory to produce the build into. In particular, these methods will
	create a `static` subdirectory in dir and work inside of that. dir may be
	"", which will create the static sub-folder within the current directory.
	Tools like `boardgame-util serve` create a temporary directory and use
	that, so it's easy to clean up later.

	The steps of the build process, at a high level, are as follows:

	First, create the `static` sub directory, if it doesn't already exist. All
	following steps create files and directories within that static subfolder.

	Next, it copies over all of the static resources (no directories) from
	`github.com/jkomoros/boardgame/server/static`, skipping a handful of files
	that will be generated later. These files are symlinked by default, but
	can also be copied. This step is encapsulated by CopyStaticResources.

	Next, it creates a node_modules folder that contains up to date
	dependencies given the contents of
	`github.com/jkomoros/boardgame/server/static/package.json`. Checking out
	this whole directory is expensive, so this package creates a node_modules
	in a central cache, re-upping it each time this command is run, and then
	symlinks it into the static directory. This step is encapsulated by
	LinkNodeModules.

	Next, it generates a `client_config.js`, which encodes the global
	configuration for the client webapp. It calls config.Client(false) and
	saves the result to static/client-config.js, which index.html will look
	for when booting up. This step is encapsulated by CreateClientConfigJs.

	Next, it copies in the client folders (containing boardgame-render-game-
	GAMENAME.js, and optionally boardgame-render-player-info-GAMENAME.js) into
	static/game-src. It does this by locating the on-disk location of each
	gameImport given by gameImports (typically this is configMode.Games), then
	symlinking its client folder into `static/game-src/GAMENAME`. In a modules
	context, game packages that do not yet exist on disk will be downloaded
	automatically; if you are not using modules and you have not yet `go get`
	the given game imports or a containg package, it will error. This step is
	encapsulated by LinkGameClientFolders.

	Next, it generates a `static/polymer.json`, which contains fragments
	entries for each dynamic import--specifically, the `game-src/GAMENAME
	/boardgame-render-game-GAMENAME.js` and `game-src/GAMENAME/boardgame-
	render-player-info-GAMENAME.js`, if it exists. It identifes the fragments
	to include by walking through all of the game directories in `game-src`,
	meaning it relies on the output of the previous step. It then saves this
	generated file to `static/polymer.json`. This step is encapsulated by
	CreatePolymerJson.

	The static build is now mostly complete. Optionally, BuildPolymer can be
	called to run `polymer build` on the generated static dir. This step is
	encapsulated by BuildPolymer.

	Typically direct users of this package use Build(), which automatically
	runs these steps in the proper order.

	Typically you don't use this package directly, but use `boardgame-util
	build static` or `boardgame-util serve`.

*/
package static

import (
	"errors"
	"fmt"
	"github.com/jkomoros/boardgame/boardgame-util/lib/config"
	"os"
	"os/exec"
	"path/filepath"
)

var filesToExclude map[string]bool = map[string]bool{
	".gitignore":      true,
	"README.md":       true,
	polymerConfig:     true,
	nodeModulesFolder: true,
	//Don't copy over because we'll generate our own; if we copy over and
	//generate our own we'll overwrite original.
	clientConfigJsFileName: true,
	".DS_Store":            true,
}

//Server runs a static server. directory is the folder that the `static`
//folder is contained within. If no error is returned, runs until the program
//exits. Under the cover uses `polymer serve` because imports use bare module
//specifiers that must be rewritten.
func Server(directory string, port string) error {

	if err := verifyPolymer(directory); err != nil {
		return err
	}

	staticDir := filepath.Join(directory, staticSubFolder)

	cmd := exec.Command("polymer", "serve", "--port="+port)
	cmd.Dir = staticDir
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	if err := cmd.Run(); err != nil {
		return errors.New("Couldn't `polymer serve`: " + err.Error())
	}

	return nil

}

//CleanCache clears the central cache the build system uses (currently just
//node_modules). If that cache doesn't exist, is a no op.
func CleanCache() error {

	cacheDir, err := buildCachePath()
	if err != nil {
		return errors.New("Couldn't get build cache path: " + err.Error())
	}

	//os.RemoveAll is OK if the path doesn't exist
	return os.RemoveAll(cacheDir)

}

/*

Build creates a folder of static resources for a server in the given
directory. It is the primary entrypoint for this package. It has no logic of
its own but serves to call all of the build steps in the correct order.

Specificlaly, it calls: CopyStaticResources, passing copyFiles;
LinkNodeModules; CreateClientConfigJs, passing c; LinkGameClientFolders,
passing gameImports; CreatePolymerJson, passing false. If prodBuild is true,
also calls BuildPolymer.

See the package doc for more about the specific build steps and what they do.

*/
func Build(directory string, gameImports []string, c *config.Config, prodBuild bool, copyFiles bool) (assetRoot string, err error) {

	staticDir, err := staticBuildDir(directory)
	if err != nil {
		return "", err
	}

	fmt.Println("Copying base static resources")
	if err := CopyStaticResources(directory, copyFiles); err != nil {
		return "", errors.New("Couldn't copy static resources")
	}

	fmt.Println("Updating " + nodeModulesFolder + " and linking in")
	if err := LinkNodeModules(directory); err != nil {
		return "", errors.New("Couldn't link " + nodeModulesFolder + ": " + err.Error())
	}

	fmt.Println("Creating " + clientConfigJsFileName)
	if err := CreateClientConfigJs(directory, c); err != nil {
		return "", errors.New("Couldn't create " + clientConfigJsFileName + ": " + err.Error())
	}

	fmt.Println("Creating " + gameSrcSubFolder)
	if err := LinkGameClientFolders(directory, gameImports); err != nil {
		return "", errors.New("Couldn't create " + gameSrcSubFolder + ": " + err.Error())
	}

	fmt.Println("Creating " + polymerConfig)
	if err := CreatePolymerJson(directory, false); err != nil {
		return "", errors.New("Couldn't create " + polymerConfig + ": " + err.Error())
	}

	if prodBuild {
		fmt.Println("Building bundled resources with `polymer build`")
		if err := BuildPolymer(directory); err != nil {
			return "", errors.New("Couldn't build bundled resources: " + err.Error())
		}
	}

	return staticDir, nil

}

//Clean removes all of the things created in the static subfolder within
//directory.
func Clean(directory string) error {
	return os.RemoveAll(filepath.Join(directory, staticSubFolder))
}
<|endoftext|>


<commit_before>package horenso

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"reflect"
	"testing"
)

func temp() string {
	f, err := ioutil.TempFile("", "")
	if err != nil {
		panic(err)
	}
	defer f.Close()
	return f.Name()
}

func parseReport(fname string) Report {
	byt, err := ioutil.ReadFile(fname)
	if err != nil {
		panic("failed to read " + fname)
	}
	r := Report{}
	json.Unmarshal(byt, &r)
	return r
}

func TestRun(t *testing.T) {
	noticeReport := temp()
	fname := temp()
	fname2 := temp()
	_, o, cmdArgs, err := parseArgs([]string{
		"--noticer",
		"go run testdata/reporter.go " + noticeReport,
		"-n", "invalid",
		"--reporter",
		"go run testdata/reporter.go " + fname,
		"-r",
		"go run testdata/reporter.go " + fname2,
		"--",
		"go", "run", "testdata/run.go",
	})
	if err != nil {
		t.Errorf("err should be nil but: %s", err)
	}
	r, err := o.run(cmdArgs)
	if err != nil {
		t.Errorf("err should be nil but: %s", err)
	}

	if *r.ExitCode != 0 {
		t.Errorf("exit code should be 0 but: %d", r.ExitCode)
	}

	expect := "1\n2\n3\n"
	if r.Output != expect {
		t.Errorf("output should be %s but: %s", expect, r.Output)
	}
	if r.Stdout != expect {
		t.Errorf("output should be %s but: %s", expect, r.Stdout)
	}
	if r.Stderr != "" {
		t.Errorf("output should be empty but: %s", r.Stderr)
	}
	if r.StartAt == nil {
		t.Errorf("StartAt shouldn't be nil")
	}
	if r.EndAt == nil {
		t.Errorf("EtartAt shouldn't be nil")
	}
	expectedHostname, _ := os.Hostname()
	if r.Hostname != expectedHostname {
		t.Errorf("Hostname should be %s but: %s", expectedHostname, r.Hostname)
	}

	rr := parseReport(fname)
	if !deepEqual(r, rr) {
		t.Errorf("something went wrong. expect: %#v, got: %#v", r, rr)
	}
	rr2 := parseReport(fname2)
	if !deepEqual(r, rr2) {
		t.Errorf("something went wrong. expect: %#v, got: %#v", r, rr2)
	}

	nr := parseReport(noticeReport)
	if *nr.Pid != *r.Pid {
		t.Errorf("something went wrong")
	}
	if nr.Output != "" {
		t.Errorf("something went wrong")
	}
	if nr.StartAt == nil {
		t.Errorf("StartAt shouldn't be nil")
	}
	if nr.EndAt != nil {
		t.Errorf("EndAt should be nil")
	}
	if nr.ExitCode != nil {
		t.Errorf("ExitCode should be nil")
	}
	if nr.Hostname != r.Hostname {
		t.Errorf("something went wrong")
	}
}

func TestRunHugeOutput(t *testing.T) {
	noticeReport := temp()
	fname := temp()
	fname2 := temp()
	_, o, cmdArgs, err := parseArgs([]string{
		"--noticer",
		"go run testdata/reporter.go " + noticeReport,
		"-n", "invalid",
		"--reporter",
		"go run testdata/reporter.go " + fname,
		"-r",
		"go run testdata/reporter.go " + fname2,
		"--",
		"go", "run", "testdata/run_hugeoutput.go",
	})
	if err != nil {
		t.Errorf("err should be nil but: %s", err)
	}
	r, err := o.run(cmdArgs)
	if err != nil {
		t.Errorf("err should be nil but: %s", err)
	}

	if *r.ExitCode != 0 {
		t.Errorf("exit code should be 0 but: %d", r.ExitCode)
	}

	expect := 64*1024 + 1
	if len(r.Output) != expect {
		t.Errorf("output should be %d bytes but: %d bytes", expect, len(r.Output))
	}
	if len(r.Stdout) != expect {
		t.Errorf("output should be %d bytes but: %d bytes", expect, len(r.Stdout))
	}
	if r.Stderr != "" {
		t.Errorf("output should be empty but: %s", r.Stderr)
	}
	if r.StartAt == nil {
		t.Errorf("StartAt shouldn't be nil")
	}
	if r.EndAt == nil {
		t.Errorf("EtartAt shouldn't be nil")
	}
	expectedHostname, _ := os.Hostname()
	if r.Hostname != expectedHostname {
		t.Errorf("Hostname should be %s but: %s", expectedHostname, r.Hostname)
	}

	rr := parseReport(fname)
	if !deepEqual(r, rr) {
		t.Errorf("something went wrong. expect: %#v, got: %#v", r, rr)
	}
	rr2 := parseReport(fname2)
	if !deepEqual(r, rr2) {
		t.Errorf("something went wrong. expect: %#v, got: %#v", r, rr2)
	}

	nr := parseReport(noticeReport)
	if *nr.Pid != *r.Pid {
		t.Errorf("something went wrong")
	}
	if nr.Output != "" {
		t.Errorf("something went wrong")
	}
	if nr.StartAt == nil {
		t.Errorf("StartAt shouldn't be nil")
	}
	if nr.EndAt != nil {
		t.Errorf("EndAt should be nil")
	}
	if nr.ExitCode != nil {
		t.Errorf("ExitCode should be nil")
	}
	if nr.Hostname != r.Hostname {
		t.Errorf("something went wrong")
	}
}

func deepEqual(r1, r2 Report) bool {
	return r1.Command == r2.Command &&
		reflect.DeepEqual(r1.CommandArgs, r2.CommandArgs) &&
		r1.Tag == r2.Tag &&
		r1.Output == r2.Output &&
		r1.Stdout == r2.Stdout &&
		r1.Stderr == r2.Stderr &&
		*r1.ExitCode == *r2.ExitCode &&
		r1.Result == r2.Result &&
		*r1.Pid == *r2.Pid &&
		r1.Hostname == r2.Hostname &&
		r1.Signaled == r2.Signaled
}
<commit_msg>use ioutil.Discard in testing for surpressing noisy output<commit_after>package horenso

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"reflect"
	"testing"
)

func temp() string {
	f, err := ioutil.TempFile("", "")
	if err != nil {
		panic(err)
	}
	defer f.Close()
	return f.Name()
}

func parseReport(fname string) Report {
	byt, err := ioutil.ReadFile(fname)
	if err != nil {
		panic("failed to read " + fname)
	}
	r := Report{}
	json.Unmarshal(byt, &r)
	return r
}

func TestRun(t *testing.T) {
	noticeReport := temp()
	fname := temp()
	fname2 := temp()
	_, ho, cmdArgs, err := parseArgs([]string{
		"--noticer",
		"go run testdata/reporter.go " + noticeReport,
		"-n", "invalid",
		"--reporter",
		"go run testdata/reporter.go " + fname,
		"-r",
		"go run testdata/reporter.go " + fname2,
		"--",
		"go", "run", "testdata/run.go",
	})
	if err != nil {
		t.Errorf("err should be nil but: %s", err)
	}
	ho.errStream = ioutil.Discard
	ho.outStream = ioutil.Discard

	r, err := ho.run(cmdArgs)
	if err != nil {
		t.Errorf("err should be nil but: %s", err)
	}

	if *r.ExitCode != 0 {
		t.Errorf("exit code should be 0 but: %d", r.ExitCode)
	}

	expect := "1\n2\n3\n"
	if r.Output != expect {
		t.Errorf("output should be %s but: %s", expect, r.Output)
	}
	if r.Stdout != expect {
		t.Errorf("output should be %s but: %s", expect, r.Stdout)
	}
	if r.Stderr != "" {
		t.Errorf("output should be empty but: %s", r.Stderr)
	}
	if r.StartAt == nil {
		t.Errorf("StartAt shouldn't be nil")
	}
	if r.EndAt == nil {
		t.Errorf("EtartAt shouldn't be nil")
	}
	expectedHostname, _ := os.Hostname()
	if r.Hostname != expectedHostname {
		t.Errorf("Hostname should be %s but: %s", expectedHostname, r.Hostname)
	}

	rr := parseReport(fname)
	if !deepEqual(r, rr) {
		t.Errorf("something went wrong. expect: %#v, got: %#v", r, rr)
	}
	rr2 := parseReport(fname2)
	if !deepEqual(r, rr2) {
		t.Errorf("something went wrong. expect: %#v, got: %#v", r, rr2)
	}

	nr := parseReport(noticeReport)
	if *nr.Pid != *r.Pid {
		t.Errorf("something went wrong")
	}
	if nr.Output != "" {
		t.Errorf("something went wrong")
	}
	if nr.StartAt == nil {
		t.Errorf("StartAt shouldn't be nil")
	}
	if nr.EndAt != nil {
		t.Errorf("EndAt should be nil")
	}
	if nr.ExitCode != nil {
		t.Errorf("ExitCode should be nil")
	}
	if nr.Hostname != r.Hostname {
		t.Errorf("something went wrong")
	}
}

func TestRunHugeOutput(t *testing.T) {
	noticeReport := temp()
	fname := temp()
	fname2 := temp()
	_, ho, cmdArgs, err := parseArgs([]string{
		"--noticer",
		"go run testdata/reporter.go " + noticeReport,
		"-n", "invalid",
		"--reporter",
		"go run testdata/reporter.go " + fname,
		"-r",
		"go run testdata/reporter.go " + fname2,
		"--",
		"go", "run", "testdata/run_hugeoutput.go",
	})
	if err != nil {
		t.Errorf("err should be nil but: %s", err)
	}
	ho.errStream = ioutil.Discard
	ho.outStream = ioutil.Discard

	r, err := ho.run(cmdArgs)
	if err != nil {
		t.Errorf("err should be nil but: %s", err)
	}

	if *r.ExitCode != 0 {
		t.Errorf("exit code should be 0 but: %d", r.ExitCode)
	}

	expect := 64*1024 + 1
	if len(r.Output) != expect {
		t.Errorf("output should be %d bytes but: %d bytes", expect, len(r.Output))
	}
	if len(r.Stdout) != expect {
		t.Errorf("output should be %d bytes but: %d bytes", expect, len(r.Stdout))
	}
	if r.Stderr != "" {
		t.Errorf("output should be empty but: %s", r.Stderr)
	}
	if r.StartAt == nil {
		t.Errorf("StartAt shouldn't be nil")
	}
	if r.EndAt == nil {
		t.Errorf("EtartAt shouldn't be nil")
	}
	expectedHostname, _ := os.Hostname()
	if r.Hostname != expectedHostname {
		t.Errorf("Hostname should be %s but: %s", expectedHostname, r.Hostname)
	}

	rr := parseReport(fname)
	if !deepEqual(r, rr) {
		t.Errorf("something went wrong. expect: %#v, got: %#v", r, rr)
	}
	rr2 := parseReport(fname2)
	if !deepEqual(r, rr2) {
		t.Errorf("something went wrong. expect: %#v, got: %#v", r, rr2)
	}

	nr := parseReport(noticeReport)
	if *nr.Pid != *r.Pid {
		t.Errorf("something went wrong")
	}
	if nr.Output != "" {
		t.Errorf("something went wrong")
	}
	if nr.StartAt == nil {
		t.Errorf("StartAt shouldn't be nil")
	}
	if nr.EndAt != nil {
		t.Errorf("EndAt should be nil")
	}
	if nr.ExitCode != nil {
		t.Errorf("ExitCode should be nil")
	}
	if nr.Hostname != r.Hostname {
		t.Errorf("something went wrong")
	}
}

func deepEqual(r1, r2 Report) bool {
	return r1.Command == r2.Command &&
		reflect.DeepEqual(r1.CommandArgs, r2.CommandArgs) &&
		r1.Tag == r2.Tag &&
		r1.Output == r2.Output &&
		r1.Stdout == r2.Stdout &&
		r1.Stderr == r2.Stderr &&
		*r1.ExitCode == *r2.ExitCode &&
		r1.Result == r2.Result &&
		*r1.Pid == *r2.Pid &&
		r1.Hostname == r2.Hostname &&
		r1.Signaled == r2.Signaled
}
<|endoftext|>


<commit_before>package execution

import (
	"fmt"
	"pfi/sensorbee/sensorbee/bql/udf"
	"pfi/sensorbee/sensorbee/core"
	"pfi/sensorbee/sensorbee/data"
)

type groupbyExecutionPlan struct {
	streamRelationStreamExecutionPlan
}

// tmpGroupData is an intermediate data structure to represent
// a set of rows that have the same values for GROUP BY columns.
type tmpGroupData struct {
	// this is the group (e.g. [1, "toy"]), where the values are
	// in order of the items in the GROUP BY clause
	group data.Array
	// for each aggregate function, we hold an array with the
	// input values.
	aggData map[string][]data.Value
	// as per our assumptions about grouping, the non-aggregation
	// data should be identical within every group
	nonAggData data.Map
}

// CanBuildGroupbyExecutionPlan checks whether the given statement
// allows to use an groupbyExecutionPlan.
func CanBuildGroupbyExecutionPlan(lp *LogicalPlan, reg udf.FunctionRegistry) bool {
	return lp.GroupingStmt
}

// NewGroupbyExecutionPlan builds a plan that follows the
// theoretical processing model. It supports only statements
// that use aggregation.
//
// After each tuple arrives,
// - compute the contents of the current window using the
//   specified window size/type,
// - perform a SELECT query on that data,
// - compute the data that need to be emitted by comparison with
//   the previous run's results.
func NewGroupbyExecutionPlan(lp *LogicalPlan, reg udf.FunctionRegistry) (PhysicalPlan, error) {
	underlying, err := newStreamRelationStreamExecutionPlan(lp, reg)
	if err != nil {
		return nil, err
	}
	return &groupbyExecutionPlan{
		*underlying,
	}, nil
}

// Process takes an input tuple and returns a slice of Map values that
// correspond to the results of the query represented by this execution
// plan. Note that the order of items in the returned slice is undefined
// and cannot be relied on.
func (ep *groupbyExecutionPlan) Process(input *core.Tuple) ([]data.Map, error) {
	return ep.process(input, ep.performQueryOnBuffer)
}

// performQueryOnBuffer computes the projections of a SELECT query on the data
// stored in `ep.filteredInputRows`. The query results (which is a set of
// data.Value, not core.Tuple) is stored in ep.curResults. The data
// that was stored in ep.curResults before this method was called is
// moved to ep.prevResults. Note that the order of values in ep.curResults
// is undefined.
//
// In case of an error the contents of ep.curResults will still be
// the same as before the call (so that the next run performs as
// if no error had happened), but the contents of ep.curResults are
// undefined.
func (ep *groupbyExecutionPlan) performQueryOnBuffer() error {
	// reuse the allocated memory
	output := ep.prevResults[0:0]
	// remember the previous results
	ep.prevResults = ep.curResults

	rollback := func() {
		// NB. ep.prevResults currently points to an slice with
		//     results from the previous run. ep.curResults points
		//     to the same slice. output points to a different slice
		//     with a different underlying array.
		//     in the next run, output will be reusing the underlying
		//     storage of the current ep.prevResults to hold results.
		//     therefore when we leave this function we must make
		//     sure that ep.prevResults and ep.curResults have
		//     different underlying arrays or ISTREAM/DSTREAM will
		//     return wrong results.
		ep.prevResults = output
	}

	// collect a list of all aggregate parameter evaluators in all
	// projections. this is necessary to avoid duplicate evaluation
	// if the same parameter is used in multiple aggregation funcs.
	allAggEvaluators := map[string]Evaluator{}
	for _, proj := range ep.projections {
		for key, agg := range proj.aggrEvals {
			allAggEvaluators[key] = agg
		}
	}

	// groups holds one item for every combination of values that
	// appear in the GROUP BY clause
	// TODO we should not consider the HashValue a 1:1 mapping to
	//      group keys, simply because it is a hash
	groups := map[data.HashValue]*tmpGroupData{}
	// we also keep a list of group keys so that we can still loop
	// over them in the order they were added
	groupKeys := []data.HashValue{}

	// findOrCreateGroup looks up the group that has the given
	// groupValues in the `groups`map. if there is no such
	// group, a new one is created and a copy of the given map
	// is used as a representative of this group's values.
	findOrCreateGroup := func(groupValues []data.Value, groupHash data.HashValue, nonGroupValues data.Map) (*tmpGroupData, error) {
		// find the correct group
		group, exists := groups[groupHash]
		// if there is no such group, create one
		if !exists {
			newGroup := &tmpGroupData{
				// the values that make up this group
				groupValues,
				// the input values of the aggregate functions
				map[string][]data.Value{},
				// a representative set of values for this group for later evaluation
				// TODO actually we don't need the whole map,
				//      just the parts common to the whole group
				nonGroupValues.Copy(),
			}
			// initialize the map with the aggregate function inputs
			for _, proj := range ep.projections {
				for key := range proj.aggrEvals {
					newGroup.aggData[key] = make([]data.Value, 0, 1)
				}
			}
			groups[groupHash] = newGroup
			group = newGroup
			groupKeys = append(groupKeys, groupHash)
		}

		// return a pointer to the (found or created) group
		return group, nil
	}

	// function to compute the grouping expressions and store the
	// input for aggregate functions in the correct group.
	evalItem := func(io *inputRowWithCachedResult) error {
		var itemGroupValues data.Array
		// if we have a cached result, use this
		if io.cache != nil {
			cachedGroupValues, err := data.AsArray(io.cache)
			if err != nil {
				return fmt.Errorf("cached data was not an array: %v", io.cache)
			}
			itemGroupValues = cachedGroupValues
		} else {
			// otherwise, compute the expressions in the GROUP BY to find
			// the correct group to append to
			itemGroupValues = make([]data.Value, len(ep.groupList))
			for i, eval := range ep.groupList {
				// ordinary "flat" expression
				value, err := eval.Eval(*io.input)
				if err != nil {
					return err
				}
				itemGroupValues[i] = value
			}
			io.cache = itemGroupValues
			io.hash = data.Hash(io.cache)
		}

		itemGroup, err := findOrCreateGroup(itemGroupValues, io.hash, *io.input)
		if err != nil {
			return err
		}

		// now compute all the input data for the aggregate functions,
		// e.g. for `SELECT count(a) + max(b/2)`, compute `a` and `b/2`
		for key, agg := range allAggEvaluators {
			value, err := agg.Eval(*io.input)
			if err != nil {
				return err
			}
			// store this value in the output map
			itemGroup.aggData[key] = append(itemGroup.aggData[key], value)
		}
		return nil
	}

	evalGroup := func(group *tmpGroupData) error {
		result := data.Map(make(map[string]data.Value, len(ep.projections)))
		// collect input for aggregate functions into an array
		// within each group
		for key := range allAggEvaluators {
			group.nonAggData[key] = data.Array(group.aggData[key])
			delete(group.aggData, key)
		}
		// evaluate HAVING condition, if there is one
		for _, proj := range ep.projections {
			if proj.alias == ":having:" {
				havingResult, err := proj.evaluator.Eval(group.nonAggData)
				if err != nil {
					return err
				}
				havingResultBool, err := data.ToBool(havingResult)
				if err != nil {
					return err
				}
				// if it evaluated to false, do not further process this group
				// (ToBool also evalutes the NULL value to false, so we don't
				// need to treat this specially)
				if !havingResultBool {
					return nil
				}
				break
			}
		}
		// now evaluate all other projections
		for _, proj := range ep.projections {
			if proj.alias == ":having:" {
				continue
			}
			// now evaluate this projection on the flattened data
			value, err := proj.evaluator.Eval(group.nonAggData)
			if err != nil {
				return err
			}
			if err := assignOutputValue(result, proj.alias, proj.aliasPath, value); err != nil {
				return err
			}
		}
		output = append(output, resultRow{row: result, hash: data.Hash(result)})
		return nil
	}

	evalNoGroup := func() error {
		// if we have an empty group list *and* a GROUP BY clause,
		// we have to return an empty result (because there are no
		// rows with "the same values"). but if the list is empty and
		// we *don't* have a GROUP BY clause, then we need to compute
		// all foldables and aggregates with an empty input
		if len(ep.groupList) > 0 {
			return nil
		}
		input := data.Map{}
		result := data.Map(make(map[string]data.Value, len(ep.projections)))
		for _, proj := range ep.projections {
			// collect input for aggregate functions
			if proj.hasAggregate {
				for key := range proj.aggrEvals {
					input[key] = data.Array{}
				}
			}
			// now evaluate this projection on the flattened data.
			// note that input has *only* the keys of the empty
			// arrays, no other columns, but we cannot have other
			// columns involved in the projection (since we know
			// that GROUP BY is empty).
			value, err := proj.evaluator.Eval(input)
			if err != nil {
				return err
			}
			if err := assignOutputValue(result, proj.alias, proj.aliasPath, value); err != nil {
				return err
			}
		}
		output = append(output, resultRow{row: result, hash: data.Hash(result)})
		return nil
	}

	// compute the output for each item in ep.filteredInputRows
	for e := ep.filteredInputRows.Front(); e != nil; e = e.Next() {
		item := e.Value.(*inputRowWithCachedResult)
		if err := evalItem(item); err != nil {
			rollback()
			return err
		}
	}

	// if we arrive here, then the input for the aggregation functions
	// is in the `group` list and we need to compute aggregation and output.
	// NB. we do not directly loop over the `groups` map to avoid random order.
	for _, groupKey := range groupKeys {
		group := groups[groupKey]
		if err := evalGroup(group); err != nil {
			rollback()
			return err
		}
	}
	if len(groups) == 0 {
		if err := evalNoGroup(); err != nil {
			rollback()
			return err
		}
	}

	ep.curResults = output
	return nil
}
<commit_msg>add handling of hash collisions for GROUP BY values<commit_after>package execution

import (
	"fmt"
	"pfi/sensorbee/sensorbee/bql/udf"
	"pfi/sensorbee/sensorbee/core"
	"pfi/sensorbee/sensorbee/data"
)

type groupbyExecutionPlan struct {
	streamRelationStreamExecutionPlan
}

// tmpGroupData is an intermediate data structure to represent
// a set of rows that have the same values for GROUP BY columns.
type tmpGroupData struct {
	// this is the group (e.g. [1, "toy"]), where the values are
	// in order of the items in the GROUP BY clause
	group data.Array
	// for each aggregate function, we hold an array with the
	// input values.
	aggData map[string][]data.Value
	// as per our assumptions about grouping, the non-aggregation
	// data should be identical within every group
	nonAggData data.Map
}

// CanBuildGroupbyExecutionPlan checks whether the given statement
// allows to use an groupbyExecutionPlan.
func CanBuildGroupbyExecutionPlan(lp *LogicalPlan, reg udf.FunctionRegistry) bool {
	return lp.GroupingStmt
}

// NewGroupbyExecutionPlan builds a plan that follows the
// theoretical processing model. It supports only statements
// that use aggregation.
//
// After each tuple arrives,
// - compute the contents of the current window using the
//   specified window size/type,
// - perform a SELECT query on that data,
// - compute the data that need to be emitted by comparison with
//   the previous run's results.
func NewGroupbyExecutionPlan(lp *LogicalPlan, reg udf.FunctionRegistry) (PhysicalPlan, error) {
	underlying, err := newStreamRelationStreamExecutionPlan(lp, reg)
	if err != nil {
		return nil, err
	}
	return &groupbyExecutionPlan{
		*underlying,
	}, nil
}

// Process takes an input tuple and returns a slice of Map values that
// correspond to the results of the query represented by this execution
// plan. Note that the order of items in the returned slice is undefined
// and cannot be relied on.
func (ep *groupbyExecutionPlan) Process(input *core.Tuple) ([]data.Map, error) {
	return ep.process(input, ep.performQueryOnBuffer)
}

// performQueryOnBuffer computes the projections of a SELECT query on the data
// stored in `ep.filteredInputRows`. The query results (which is a set of
// data.Value, not core.Tuple) is stored in ep.curResults. The data
// that was stored in ep.curResults before this method was called is
// moved to ep.prevResults. Note that the order of values in ep.curResults
// is undefined.
//
// In case of an error the contents of ep.curResults will still be
// the same as before the call (so that the next run performs as
// if no error had happened), but the contents of ep.curResults are
// undefined.
func (ep *groupbyExecutionPlan) performQueryOnBuffer() error {
	// reuse the allocated memory
	output := ep.prevResults[0:0]
	// remember the previous results
	ep.prevResults = ep.curResults

	rollback := func() {
		// NB. ep.prevResults currently points to an slice with
		//     results from the previous run. ep.curResults points
		//     to the same slice. output points to a different slice
		//     with a different underlying array.
		//     in the next run, output will be reusing the underlying
		//     storage of the current ep.prevResults to hold results.
		//     therefore when we leave this function we must make
		//     sure that ep.prevResults and ep.curResults have
		//     different underlying arrays or ISTREAM/DSTREAM will
		//     return wrong results.
		ep.prevResults = output
	}

	// collect a list of all aggregate parameter evaluators in all
	// projections. this is necessary to avoid duplicate evaluation
	// if the same parameter is used in multiple aggregation funcs.
	allAggEvaluators := map[string]Evaluator{}
	for _, proj := range ep.projections {
		for key, agg := range proj.aggrEvals {
			allAggEvaluators[key] = agg
		}
	}

	// groups holds one item for every combination of values that
	// appear in the GROUP BY clause
	groups := map[data.HashValue][]*tmpGroupData{}
	// we also keep a list of group keys so that we can still loop
	// over them in the order they were added
	groupKeys := []data.HashValue{}

	// findOrCreateGroup looks up the group that has the given
	// groupValues in the `groups`map. if there is no such
	// group, a new one is created and a copy of the given map
	// is used as a representative of this group's values.
	findOrCreateGroup := func(groupValues []data.Value, groupHash data.HashValue, nonGroupValues data.Map) (*tmpGroupData, error) {
		mkGroup := func() *tmpGroupData {
			newGroup := &tmpGroupData{
				// the values that make up this group
				groupValues,
				// the input values of the aggregate functions
				map[string][]data.Value{},
				// a representative set of values for this group for later evaluation
				// TODO actually we don't need the whole map,
				//      just the parts common to the whole group
				nonGroupValues.Copy(),
			}
			// initialize the map with the aggregate function inputs
			for _, proj := range ep.projections {
				for key := range proj.aggrEvals {
					newGroup.aggData[key] = make([]data.Value, 0, 1)
				}
			}
			return newGroup
		}

		// find the correct group
		groupCandidates, exists := groups[groupHash]
		var group *tmpGroupData
		// if there is no such group, create one
		if !exists {
			group = mkGroup()
			groups[groupHash] = []*tmpGroupData{group}
			groupKeys = append(groupKeys, groupHash)
		} else {
			// if we arrive here, there is a group with the same hash value
			// but we need to validate the data is actually the same
			for _, groupCandidate := range groupCandidates {
				if data.Equal(data.Array(groupValues), groupCandidate.group) {
					group = groupCandidate
					break
				}
			}
			// no group with the same groupValues was found, so create
			// one and append it to the list of groups with the same hash
			if group == nil {
				group = mkGroup()
				groups[groupHash] = append(groupCandidates, group)
			}
		}
		// return a pointer to the (found or created) group
		return group, nil
	}

	// function to compute the grouping expressions and store the
	// input for aggregate functions in the correct group.
	evalItem := func(io *inputRowWithCachedResult) error {
		var itemGroupValues data.Array
		// if we have a cached result, use this
		if io.cache != nil {
			cachedGroupValues, err := data.AsArray(io.cache)
			if err != nil {
				return fmt.Errorf("cached data was not an array: %v", io.cache)
			}
			itemGroupValues = cachedGroupValues
		} else {
			// otherwise, compute the expressions in the GROUP BY to find
			// the correct group to append to
			itemGroupValues = make([]data.Value, len(ep.groupList))
			for i, eval := range ep.groupList {
				// ordinary "flat" expression
				value, err := eval.Eval(*io.input)
				if err != nil {
					return err
				}
				itemGroupValues[i] = value
			}
			io.cache = itemGroupValues
			io.hash = data.Hash(io.cache)
		}

		itemGroup, err := findOrCreateGroup(itemGroupValues, io.hash, *io.input)
		if err != nil {
			return err
		}

		// now compute all the input data for the aggregate functions,
		// e.g. for `SELECT count(a) + max(b/2)`, compute `a` and `b/2`
		for key, agg := range allAggEvaluators {
			value, err := agg.Eval(*io.input)
			if err != nil {
				return err
			}
			// store this value in the output map
			itemGroup.aggData[key] = append(itemGroup.aggData[key], value)
		}
		return nil
	}

	evalGroup := func(group *tmpGroupData) error {
		result := data.Map(make(map[string]data.Value, len(ep.projections)))
		// collect input for aggregate functions into an array
		// within each group
		for key := range allAggEvaluators {
			group.nonAggData[key] = data.Array(group.aggData[key])
			delete(group.aggData, key)
		}
		// evaluate HAVING condition, if there is one
		for _, proj := range ep.projections {
			if proj.alias == ":having:" {
				havingResult, err := proj.evaluator.Eval(group.nonAggData)
				if err != nil {
					return err
				}
				havingResultBool, err := data.ToBool(havingResult)
				if err != nil {
					return err
				}
				// if it evaluated to false, do not further process this group
				// (ToBool also evalutes the NULL value to false, so we don't
				// need to treat this specially)
				if !havingResultBool {
					return nil
				}
				break
			}
		}
		// now evaluate all other projections
		for _, proj := range ep.projections {
			if proj.alias == ":having:" {
				continue
			}
			// now evaluate this projection on the flattened data
			value, err := proj.evaluator.Eval(group.nonAggData)
			if err != nil {
				return err
			}
			if err := assignOutputValue(result, proj.alias, proj.aliasPath, value); err != nil {
				return err
			}
		}
		output = append(output, resultRow{row: result, hash: data.Hash(result)})
		return nil
	}

	evalNoGroup := func() error {
		// if we have an empty group list *and* a GROUP BY clause,
		// we have to return an empty result (because there are no
		// rows with "the same values"). but if the list is empty and
		// we *don't* have a GROUP BY clause, then we need to compute
		// all foldables and aggregates with an empty input
		if len(ep.groupList) > 0 {
			return nil
		}
		input := data.Map{}
		result := data.Map(make(map[string]data.Value, len(ep.projections)))
		for _, proj := range ep.projections {
			// collect input for aggregate functions
			if proj.hasAggregate {
				for key := range proj.aggrEvals {
					input[key] = data.Array{}
				}
			}
			// now evaluate this projection on the flattened data.
			// note that input has *only* the keys of the empty
			// arrays, no other columns, but we cannot have other
			// columns involved in the projection (since we know
			// that GROUP BY is empty).
			value, err := proj.evaluator.Eval(input)
			if err != nil {
				return err
			}
			if err := assignOutputValue(result, proj.alias, proj.aliasPath, value); err != nil {
				return err
			}
		}
		output = append(output, resultRow{row: result, hash: data.Hash(result)})
		return nil
	}

	// compute the output for each item in ep.filteredInputRows
	for e := ep.filteredInputRows.Front(); e != nil; e = e.Next() {
		item := e.Value.(*inputRowWithCachedResult)
		if err := evalItem(item); err != nil {
			rollback()
			return err
		}
	}

	// if we arrive here, then the input for the aggregation functions
	// is in the `group` list and we need to compute aggregation and output.
	// NB. we do not directly loop over the `groups` map to avoid random order.
	for _, groupKey := range groupKeys {
		groupsWithSameHash := groups[groupKey]
		for _, group := range groupsWithSameHash {
			if err := evalGroup(group); err != nil {
				rollback()
				return err
			}
		}
	}
	if len(groups) == 0 {
		if err := evalNoGroup(); err != nil {
			rollback()
			return err
		}
	}

	ep.curResults = output
	return nil
}
<|endoftext|>


<commit_before>package container

import (
	"errors"
	"fmt"
	"net"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"

	clustertypes "github.com/docker/docker/daemon/cluster/provider"
	"github.com/docker/docker/reference"
	"github.com/docker/engine-api/types"
	enginecontainer "github.com/docker/engine-api/types/container"
	"github.com/docker/engine-api/types/network"
	"github.com/docker/swarmkit/agent/exec"
	"github.com/docker/swarmkit/api"
)

const (
	// Explicitly use the kernel's default setting for CPU quota of 100ms.
	// https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt
	cpuQuotaPeriod = 100 * time.Millisecond

	// systemLabelPrefix represents the reserved namespace for system labels.
	systemLabelPrefix = "com.docker.swarm"
)

// containerConfig converts task properties into docker container compatible
// components.
type containerConfig struct {
	task                *api.Task
	networksAttachments map[string]*api.NetworkAttachment
}

// newContainerConfig returns a validated container config. No methods should
// return an error if this function returns without error.
func newContainerConfig(t *api.Task) (*containerConfig, error) {
	var c containerConfig
	return &c, c.setTask(t)
}

func (c *containerConfig) setTask(t *api.Task) error {
	container := t.Spec.GetContainer()
	if container == nil {
		return exec.ErrRuntimeUnsupported
	}

	if container.Image == "" {
		return ErrImageRequired
	}

	// index the networks by name
	c.networksAttachments = make(map[string]*api.NetworkAttachment, len(t.Networks))
	for _, attachment := range t.Networks {
		c.networksAttachments[attachment.Network.Spec.Annotations.Name] = attachment
	}

	c.task = t
	return nil
}

func (c *containerConfig) endpoint() *api.Endpoint {
	return c.task.Endpoint
}

func (c *containerConfig) spec() *api.ContainerSpec {
	return c.task.Spec.GetContainer()
}

func (c *containerConfig) name() string {
	if c.task.Annotations.Name != "" {
		// if set, use the container Annotations.Name field, set in the orchestrator.
		return c.task.Annotations.Name
	}

	// fallback to service.slot.id.
	return strings.Join([]string{c.task.ServiceAnnotations.Name, fmt.Sprint(c.task.Slot), c.task.ID}, ".")
}

func (c *containerConfig) image() string {
	raw := c.spec().Image
	ref, err := reference.ParseNamed(raw)
	if err != nil {
		return raw
	}
	return reference.WithDefaultTag(ref).String()
}

func (c *containerConfig) volumes() map[string]struct{} {
	r := make(map[string]struct{})

	for _, mount := range c.spec().Mounts {
		// pick off all the volume mounts.
		if mount.Type != api.MountTypeVolume {
			continue
		}

		r[fmt.Sprintf("%s:%s", mount.Target, getMountMask(&mount))] = struct{}{}
	}

	return r
}

func (c *containerConfig) config() *enginecontainer.Config {
	config := &enginecontainer.Config{
		Labels:     c.labels(),
		User:       c.spec().User,
		Env:        c.spec().Env,
		WorkingDir: c.spec().Dir,
		Image:      c.image(),
		Volumes:    c.volumes(),
	}

	if len(c.spec().Command) > 0 {
		// If Command is provided, we replace the whole invocation with Command
		// by replacing Entrypoint and specifying Cmd. Args is ignored in this
		// case.
		config.Entrypoint = append(config.Entrypoint, c.spec().Command[0])
		config.Cmd = append(config.Cmd, c.spec().Command[1:]...)
	} else if len(c.spec().Args) > 0 {
		// In this case, we assume the image has an Entrypoint and Args
		// specifies the arguments for that entrypoint.
		config.Cmd = c.spec().Args
	}

	return config
}

func (c *containerConfig) labels() map[string]string {
	var (
		system = map[string]string{
			"task":         "", // mark as cluster task
			"task.id":      c.task.ID,
			"task.name":    fmt.Sprintf("%v.%v", c.task.ServiceAnnotations.Name, c.task.Slot),
			"node.id":      c.task.NodeID,
			"service.id":   c.task.ServiceID,
			"service.name": c.task.ServiceAnnotations.Name,
		}
		labels = make(map[string]string)
	)

	// base labels are those defined in the spec.
	for k, v := range c.spec().Labels {
		labels[k] = v
	}

	// we then apply the overrides from the task, which may be set via the
	// orchestrator.
	for k, v := range c.task.Annotations.Labels {
		labels[k] = v
	}

	// finally, we apply the system labels, which override all labels.
	for k, v := range system {
		labels[strings.Join([]string{systemLabelPrefix, k}, ".")] = v
	}

	return labels
}

func (c *containerConfig) bindMounts() []string {
	var r []string

	for _, val := range c.spec().Mounts {
		mask := getMountMask(&val)
		if val.Type == api.MountTypeBind {
			r = append(r, fmt.Sprintf("%s:%s:%s", val.Source, val.Target, mask))
		}
	}

	return r
}

func getMountMask(m *api.Mount) string {
	maskOpts := []string{"ro"}
	if m.Writable {
		maskOpts[0] = "rw"
	}

	if m.BindOptions != nil {
		switch m.BindOptions.Propagation {
		case api.MountPropagationPrivate:
			maskOpts = append(maskOpts, "private")
		case api.MountPropagationRPrivate:
			maskOpts = append(maskOpts, "rprivate")
		case api.MountPropagationShared:
			maskOpts = append(maskOpts, "shared")
		case api.MountPropagationRShared:
			maskOpts = append(maskOpts, "rshared")
		case api.MountPropagationSlave:
			maskOpts = append(maskOpts, "slave")
		case api.MountPropagationRSlave:
			maskOpts = append(maskOpts, "rslave")
		}
	}

	if m.VolumeOptions != nil {
		if !m.VolumeOptions.Populate {
			maskOpts = append(maskOpts, "nocopy")
		}
	}
	return strings.Join(maskOpts, ",")
}

func (c *containerConfig) hostConfig() *enginecontainer.HostConfig {
	return &enginecontainer.HostConfig{
		Resources: c.resources(),
		Binds:     c.bindMounts(),
	}
}

// This handles the case of volumes that are defined inside a service Mount
func (c *containerConfig) volumeCreateRequest(mount *api.Mount) *types.VolumeCreateRequest {
	var (
		driverName string
		driverOpts map[string]string
		labels     map[string]string
	)

	if mount.VolumeOptions != nil && mount.VolumeOptions.DriverConfig != nil {
		driverName = mount.VolumeOptions.DriverConfig.Name
		driverOpts = mount.VolumeOptions.DriverConfig.Options
		labels = mount.VolumeOptions.Labels
	}

	if mount.VolumeOptions != nil {
		return &types.VolumeCreateRequest{
			Name:       mount.Source,
			Driver:     driverName,
			DriverOpts: driverOpts,
			Labels:     labels,
		}
	}
	return nil
}

func (c *containerConfig) resources() enginecontainer.Resources {
	resources := enginecontainer.Resources{}

	// If no limits are specified let the engine use its defaults.
	//
	// TODO(aluzzardi): We might want to set some limits anyway otherwise
	// "unlimited" tasks will step over the reservation of other tasks.
	r := c.task.Spec.Resources
	if r == nil || r.Limits == nil {
		return resources
	}

	if r.Limits.MemoryBytes > 0 {
		resources.Memory = r.Limits.MemoryBytes
	}

	if r.Limits.NanoCPUs > 0 {
		// CPU Period must be set in microseconds.
		resources.CPUPeriod = int64(cpuQuotaPeriod / time.Microsecond)
		resources.CPUQuota = r.Limits.NanoCPUs * resources.CPUPeriod / 1e9
	}

	return resources
}

// Docker daemon supports just 1 network during container create.
func (c *containerConfig) createNetworkingConfig() *network.NetworkingConfig {
	var networks []*api.NetworkAttachment
	if c.task.Spec.GetContainer() != nil {
		networks = c.task.Networks
	}

	epConfig := make(map[string]*network.EndpointSettings)
	if len(networks) > 0 {
		epConfig[networks[0].Network.Spec.Annotations.Name] = getEndpointConfig(networks[0])
	}

	return &network.NetworkingConfig{EndpointsConfig: epConfig}
}

// TODO: Merge this function with createNetworkingConfig after daemon supports multiple networks in container create
func (c *containerConfig) connectNetworkingConfig() *network.NetworkingConfig {
	var networks []*api.NetworkAttachment
	if c.task.Spec.GetContainer() != nil {
		networks = c.task.Networks
	}

	// First network is used during container create. Other networks are used in "docker network connect"
	if len(networks) < 2 {
		return nil
	}

	epConfig := make(map[string]*network.EndpointSettings)
	for _, na := range networks[1:] {
		epConfig[na.Network.Spec.Annotations.Name] = getEndpointConfig(na)
	}
	return &network.NetworkingConfig{EndpointsConfig: epConfig}
}

func getEndpointConfig(na *api.NetworkAttachment) *network.EndpointSettings {
	var ipv4, ipv6 string
	for _, addr := range na.Addresses {
		ip, _, err := net.ParseCIDR(addr)
		if err != nil {
			continue
		}

		if ip.To4() != nil {
			ipv4 = ip.String()
			continue
		}

		if ip.To16() != nil {
			ipv6 = ip.String()
		}
	}

	return &network.EndpointSettings{
		IPAMConfig: &network.EndpointIPAMConfig{
			IPv4Address: ipv4,
			IPv6Address: ipv6,
		},
	}
}

func (c *containerConfig) virtualIP(networkID string) string {
	if c.task.Endpoint == nil {
		return ""
	}

	for _, eVip := range c.task.Endpoint.VirtualIPs {
		// We only support IPv4 VIPs for now.
		if eVip.NetworkID == networkID {
			vip, _, err := net.ParseCIDR(eVip.Addr)
			if err != nil {
				return ""
			}

			return vip.String()
		}
	}

	return ""
}

func (c *containerConfig) serviceConfig() *clustertypes.ServiceConfig {
	if len(c.task.Networks) == 0 {
		return nil
	}

	logrus.Debugf("Creating service config in agent for t = %+v", c.task)
	svcCfg := &clustertypes.ServiceConfig{
		Name:             c.task.ServiceAnnotations.Name,
		Aliases:          make(map[string][]string),
		ID:               c.task.ServiceID,
		VirtualAddresses: make(map[string]*clustertypes.VirtualAddress),
	}

	for _, na := range c.task.Networks {
		svcCfg.VirtualAddresses[na.Network.ID] = &clustertypes.VirtualAddress{
			// We support only IPv4 virtual IP for now.
			IPv4: c.virtualIP(na.Network.ID),
		}
		if len(na.Aliases) > 0 {
			svcCfg.Aliases[na.Network.ID] = na.Aliases
		}
	}

	if c.task.Endpoint != nil {
		for _, ePort := range c.task.Endpoint.Ports {
			svcCfg.ExposedPorts = append(svcCfg.ExposedPorts, &clustertypes.PortConfig{
				Name:          ePort.Name,
				Protocol:      int32(ePort.Protocol),
				TargetPort:    ePort.TargetPort,
				PublishedPort: ePort.PublishedPort,
			})
		}
	}

	return svcCfg
}

// networks returns a list of network names attached to the container. The
// returned name can be used to lookup the corresponding network create
// options.
func (c *containerConfig) networks() []string {
	var networks []string

	for name := range c.networksAttachments {
		networks = append(networks, name)
	}

	return networks
}

func (c *containerConfig) networkCreateRequest(name string) (clustertypes.NetworkCreateRequest, error) {
	na, ok := c.networksAttachments[name]
	if !ok {
		return clustertypes.NetworkCreateRequest{}, errors.New("container: unknown network referenced")
	}

	options := types.NetworkCreate{
		// ID:     na.Network.ID,
		Driver: na.Network.DriverState.Name,
		IPAM: network.IPAM{
			Driver: na.Network.IPAM.Driver.Name,
		},
		Options:        na.Network.DriverState.Options,
		CheckDuplicate: true,
	}

	for _, ic := range na.Network.IPAM.Configs {
		c := network.IPAMConfig{
			Subnet:  ic.Subnet,
			IPRange: ic.Range,
			Gateway: ic.Gateway,
		}
		options.IPAM.Config = append(options.IPAM.Config, c)
	}

	return clustertypes.NetworkCreateRequest{na.Network.ID, types.NetworkCreateRequest{Name: name, NetworkCreate: options}}, nil
}
<commit_msg>Volume mounts need to use "Binds" API field<commit_after>package container

import (
	"errors"
	"fmt"
	"net"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"

	clustertypes "github.com/docker/docker/daemon/cluster/provider"
	"github.com/docker/docker/reference"
	"github.com/docker/engine-api/types"
	enginecontainer "github.com/docker/engine-api/types/container"
	"github.com/docker/engine-api/types/network"
	"github.com/docker/swarmkit/agent/exec"
	"github.com/docker/swarmkit/api"
)

const (
	// Explicitly use the kernel's default setting for CPU quota of 100ms.
	// https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt
	cpuQuotaPeriod = 100 * time.Millisecond

	// systemLabelPrefix represents the reserved namespace for system labels.
	systemLabelPrefix = "com.docker.swarm"
)

// containerConfig converts task properties into docker container compatible
// components.
type containerConfig struct {
	task                *api.Task
	networksAttachments map[string]*api.NetworkAttachment
}

// newContainerConfig returns a validated container config. No methods should
// return an error if this function returns without error.
func newContainerConfig(t *api.Task) (*containerConfig, error) {
	var c containerConfig
	return &c, c.setTask(t)
}

func (c *containerConfig) setTask(t *api.Task) error {
	container := t.Spec.GetContainer()
	if container == nil {
		return exec.ErrRuntimeUnsupported
	}

	if container.Image == "" {
		return ErrImageRequired
	}

	// index the networks by name
	c.networksAttachments = make(map[string]*api.NetworkAttachment, len(t.Networks))
	for _, attachment := range t.Networks {
		c.networksAttachments[attachment.Network.Spec.Annotations.Name] = attachment
	}

	c.task = t
	return nil
}

func (c *containerConfig) endpoint() *api.Endpoint {
	return c.task.Endpoint
}

func (c *containerConfig) spec() *api.ContainerSpec {
	return c.task.Spec.GetContainer()
}

func (c *containerConfig) name() string {
	if c.task.Annotations.Name != "" {
		// if set, use the container Annotations.Name field, set in the orchestrator.
		return c.task.Annotations.Name
	}

	// fallback to service.slot.id.
	return strings.Join([]string{c.task.ServiceAnnotations.Name, fmt.Sprint(c.task.Slot), c.task.ID}, ".")
}

func (c *containerConfig) image() string {
	raw := c.spec().Image
	ref, err := reference.ParseNamed(raw)
	if err != nil {
		return raw
	}
	return reference.WithDefaultTag(ref).String()
}

func (c *containerConfig) volumes() map[string]struct{} {
	r := make(map[string]struct{})

	for _, m := range c.spec().Mounts {
		// pick off all the volume mounts.
		if m.Type != api.MountTypeVolume || m.Source != "" {
			continue
		}
		r[m.Target] = struct{}{}
	}

	return r
}

func (c *containerConfig) config() *enginecontainer.Config {
	config := &enginecontainer.Config{
		Labels:     c.labels(),
		User:       c.spec().User,
		Env:        c.spec().Env,
		WorkingDir: c.spec().Dir,
		Image:      c.image(),
		Volumes:    c.volumes(),
	}

	if len(c.spec().Command) > 0 {
		// If Command is provided, we replace the whole invocation with Command
		// by replacing Entrypoint and specifying Cmd. Args is ignored in this
		// case.
		config.Entrypoint = append(config.Entrypoint, c.spec().Command[0])
		config.Cmd = append(config.Cmd, c.spec().Command[1:]...)
	} else if len(c.spec().Args) > 0 {
		// In this case, we assume the image has an Entrypoint and Args
		// specifies the arguments for that entrypoint.
		config.Cmd = c.spec().Args
	}

	return config
}

func (c *containerConfig) labels() map[string]string {
	var (
		system = map[string]string{
			"task":         "", // mark as cluster task
			"task.id":      c.task.ID,
			"task.name":    fmt.Sprintf("%v.%v", c.task.ServiceAnnotations.Name, c.task.Slot),
			"node.id":      c.task.NodeID,
			"service.id":   c.task.ServiceID,
			"service.name": c.task.ServiceAnnotations.Name,
		}
		labels = make(map[string]string)
	)

	// base labels are those defined in the spec.
	for k, v := range c.spec().Labels {
		labels[k] = v
	}

	// we then apply the overrides from the task, which may be set via the
	// orchestrator.
	for k, v := range c.task.Annotations.Labels {
		labels[k] = v
	}

	// finally, we apply the system labels, which override all labels.
	for k, v := range system {
		labels[strings.Join([]string{systemLabelPrefix, k}, ".")] = v
	}

	return labels
}

func (c *containerConfig) bindMounts() []string {
	var r []string

	for _, val := range c.spec().Mounts {
		mask := getMountMask(&val)
		if val.Type == api.MountTypeBind || (val.Type == api.MountTypeVolume && val.Source != "") {
			r = append(r, fmt.Sprintf("%s:%s:%s", val.Source, val.Target, mask))
		}
	}

	return r
}

func getMountMask(m *api.Mount) string {
	maskOpts := []string{"ro"}
	if m.Writable {
		maskOpts[0] = "rw"
	}

	if m.BindOptions != nil {
		switch m.BindOptions.Propagation {
		case api.MountPropagationPrivate:
			maskOpts = append(maskOpts, "private")
		case api.MountPropagationRPrivate:
			maskOpts = append(maskOpts, "rprivate")
		case api.MountPropagationShared:
			maskOpts = append(maskOpts, "shared")
		case api.MountPropagationRShared:
			maskOpts = append(maskOpts, "rshared")
		case api.MountPropagationSlave:
			maskOpts = append(maskOpts, "slave")
		case api.MountPropagationRSlave:
			maskOpts = append(maskOpts, "rslave")
		}
	}

	if m.VolumeOptions != nil {
		if !m.VolumeOptions.Populate {
			maskOpts = append(maskOpts, "nocopy")
		}
	}
	return strings.Join(maskOpts, ",")
}

func (c *containerConfig) hostConfig() *enginecontainer.HostConfig {
	return &enginecontainer.HostConfig{
		Resources: c.resources(),
		Binds:     c.bindMounts(),
	}
}

// This handles the case of volumes that are defined inside a service Mount
func (c *containerConfig) volumeCreateRequest(mount *api.Mount) *types.VolumeCreateRequest {
	var (
		driverName string
		driverOpts map[string]string
		labels     map[string]string
	)

	if mount.VolumeOptions != nil && mount.VolumeOptions.DriverConfig != nil {
		driverName = mount.VolumeOptions.DriverConfig.Name
		driverOpts = mount.VolumeOptions.DriverConfig.Options
		labels = mount.VolumeOptions.Labels
	}

	if mount.VolumeOptions != nil {
		return &types.VolumeCreateRequest{
			Name:       mount.Source,
			Driver:     driverName,
			DriverOpts: driverOpts,
			Labels:     labels,
		}
	}
	return nil
}

func (c *containerConfig) resources() enginecontainer.Resources {
	resources := enginecontainer.Resources{}

	// If no limits are specified let the engine use its defaults.
	//
	// TODO(aluzzardi): We might want to set some limits anyway otherwise
	// "unlimited" tasks will step over the reservation of other tasks.
	r := c.task.Spec.Resources
	if r == nil || r.Limits == nil {
		return resources
	}

	if r.Limits.MemoryBytes > 0 {
		resources.Memory = r.Limits.MemoryBytes
	}

	if r.Limits.NanoCPUs > 0 {
		// CPU Period must be set in microseconds.
		resources.CPUPeriod = int64(cpuQuotaPeriod / time.Microsecond)
		resources.CPUQuota = r.Limits.NanoCPUs * resources.CPUPeriod / 1e9
	}

	return resources
}

// Docker daemon supports just 1 network during container create.
func (c *containerConfig) createNetworkingConfig() *network.NetworkingConfig {
	var networks []*api.NetworkAttachment
	if c.task.Spec.GetContainer() != nil {
		networks = c.task.Networks
	}

	epConfig := make(map[string]*network.EndpointSettings)
	if len(networks) > 0 {
		epConfig[networks[0].Network.Spec.Annotations.Name] = getEndpointConfig(networks[0])
	}

	return &network.NetworkingConfig{EndpointsConfig: epConfig}
}

// TODO: Merge this function with createNetworkingConfig after daemon supports multiple networks in container create
func (c *containerConfig) connectNetworkingConfig() *network.NetworkingConfig {
	var networks []*api.NetworkAttachment
	if c.task.Spec.GetContainer() != nil {
		networks = c.task.Networks
	}

	// First network is used during container create. Other networks are used in "docker network connect"
	if len(networks) < 2 {
		return nil
	}

	epConfig := make(map[string]*network.EndpointSettings)
	for _, na := range networks[1:] {
		epConfig[na.Network.Spec.Annotations.Name] = getEndpointConfig(na)
	}
	return &network.NetworkingConfig{EndpointsConfig: epConfig}
}

func getEndpointConfig(na *api.NetworkAttachment) *network.EndpointSettings {
	var ipv4, ipv6 string
	for _, addr := range na.Addresses {
		ip, _, err := net.ParseCIDR(addr)
		if err != nil {
			continue
		}

		if ip.To4() != nil {
			ipv4 = ip.String()
			continue
		}

		if ip.To16() != nil {
			ipv6 = ip.String()
		}
	}

	return &network.EndpointSettings{
		IPAMConfig: &network.EndpointIPAMConfig{
			IPv4Address: ipv4,
			IPv6Address: ipv6,
		},
	}
}

func (c *containerConfig) virtualIP(networkID string) string {
	if c.task.Endpoint == nil {
		return ""
	}

	for _, eVip := range c.task.Endpoint.VirtualIPs {
		// We only support IPv4 VIPs for now.
		if eVip.NetworkID == networkID {
			vip, _, err := net.ParseCIDR(eVip.Addr)
			if err != nil {
				return ""
			}

			return vip.String()
		}
	}

	return ""
}

func (c *containerConfig) serviceConfig() *clustertypes.ServiceConfig {
	if len(c.task.Networks) == 0 {
		return nil
	}

	logrus.Debugf("Creating service config in agent for t = %+v", c.task)
	svcCfg := &clustertypes.ServiceConfig{
		Name:             c.task.ServiceAnnotations.Name,
		Aliases:          make(map[string][]string),
		ID:               c.task.ServiceID,
		VirtualAddresses: make(map[string]*clustertypes.VirtualAddress),
	}

	for _, na := range c.task.Networks {
		svcCfg.VirtualAddresses[na.Network.ID] = &clustertypes.VirtualAddress{
			// We support only IPv4 virtual IP for now.
			IPv4: c.virtualIP(na.Network.ID),
		}
		if len(na.Aliases) > 0 {
			svcCfg.Aliases[na.Network.ID] = na.Aliases
		}
	}

	if c.task.Endpoint != nil {
		for _, ePort := range c.task.Endpoint.Ports {
			svcCfg.ExposedPorts = append(svcCfg.ExposedPorts, &clustertypes.PortConfig{
				Name:          ePort.Name,
				Protocol:      int32(ePort.Protocol),
				TargetPort:    ePort.TargetPort,
				PublishedPort: ePort.PublishedPort,
			})
		}
	}

	return svcCfg
}

// networks returns a list of network names attached to the container. The
// returned name can be used to lookup the corresponding network create
// options.
func (c *containerConfig) networks() []string {
	var networks []string

	for name := range c.networksAttachments {
		networks = append(networks, name)
	}

	return networks
}

func (c *containerConfig) networkCreateRequest(name string) (clustertypes.NetworkCreateRequest, error) {
	na, ok := c.networksAttachments[name]
	if !ok {
		return clustertypes.NetworkCreateRequest{}, errors.New("container: unknown network referenced")
	}

	options := types.NetworkCreate{
		// ID:     na.Network.ID,
		Driver: na.Network.DriverState.Name,
		IPAM: network.IPAM{
			Driver: na.Network.IPAM.Driver.Name,
		},
		Options:        na.Network.DriverState.Options,
		CheckDuplicate: true,
	}

	for _, ic := range na.Network.IPAM.Configs {
		c := network.IPAMConfig{
			Subnet:  ic.Subnet,
			IPRange: ic.Range,
			Gateway: ic.Gateway,
		}
		options.IPAM.Config = append(options.IPAM.Config, c)
	}

	return clustertypes.NetworkCreateRequest{na.Network.ID, types.NetworkCreateRequest{Name: name, NetworkCreate: options}}, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package audio

import (
	"fmt"
	"io"
	"sync"
	"time"

	"github.com/hajimehoshi/ebiten"
)

type mixedPlayersStream struct {
	context      *Context
	writtenBytes int
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

const (
	channelNum     = 2
	bytesPerSample = 2
	bitsPerSample  = bytesPerSample * 8
)

func (s *mixedPlayersStream) Read(b []byte) (int, error) {
	s.context.Lock()
	defer s.context.Unlock()

	bytesPerFrame := s.context.sampleRate * bytesPerSample * channelNum / ebiten.FPS
	x := s.context.frames*bytesPerFrame + len(b)
	if x <= s.writtenBytes {
		return 0, nil
	}

	l := len(b) / 4 * 4
	if len(s.context.players) == 0 {
		l := min(len(b), x-s.writtenBytes)
		copy(b, make([]byte, l))
		s.writtenBytes += l
		return l, nil
	}
	closed := []*Player{}
	ll := l
	for p := range s.context.players {
		_, err := p.readToBuffer(l)
		if err == io.EOF {
			closed = append(closed, p)
		} else if err != nil {
			return 0, err
		}
		ll = min(p.bufferLength()/4*4, ll)
	}
	b16s := [][]int16{}
	for p := range s.context.players {
		b16s = append(b16s, p.bufferToInt16(ll))
	}
	for i := 0; i < ll/2; i++ {
		x := 0
		for _, b16 := range b16s {
			x += int(b16[i])
		}
		if x > (1<<15)-1 {
			x = (1 << 15) - 1
		}
		if x < -(1 << 15) {
			x = -(1 << 15)
		}
		b[2*i] = byte(x)
		b[2*i+1] = byte(x >> 8)
	}
	for p := range s.context.players {
		p.proceed(ll)
	}
	for _, p := range closed {
		delete(s.context.players, p)
	}
	s.writtenBytes += ll
	return ll, nil
}

// TODO: Enable to specify the format like Mono8?

type Context struct {
	sampleRate  int
	stream      *mixedPlayersStream
	players     map[*Player]struct{}
	innerPlayer *player
	frames      int
	sync.Mutex
}

func NewContext(sampleRate int) *Context {
	// TODO: Panic if one context exists.
	c := &Context{
		sampleRate: sampleRate,
		players:    map[*Player]struct{}{},
	}
	c.stream = &mixedPlayersStream{
		context: c,
	}
	p, err := startPlaying(c.stream, c.sampleRate)
	if err != nil {
		panic(fmt.Sprintf("audio: NewContext error: %v", err))
	}
	c.innerPlayer = p
	return c
}

// Update proceeds the inner (logical) time of the context by 1/60 second.
// This is expected to be called in the game's updating function (sync mode)
// or an independent goroutine with timers (unsync mode).
// In sync mode, the game logical time syncs the audio logical time and
// you will find audio stops when the game stops e.g. when the window is deactivated.
// In unsync mode, the audio never stops even when the game stops.
func (c *Context) Update() {
	c.Lock()
	defer c.Unlock()
	c.frames++
}

// SampleRate returns the sample rate.
func (c *Context) SampleRate() int {
	c.Lock()
	defer c.Unlock()
	return c.sampleRate
}

type Player struct {
	context *Context
	src     io.ReadSeeker
	buf     []byte
	pos     int64
}

// NewPlayer creates a new player with the given data to the given channel.
// The given data is queued to the end of the buffer.
// This may not be played immediately when data already exists in the buffer.
//
// src's format must be linear PCM (16bits, 2 channel stereo, little endian)
// without a header (e.g. RIFF header).
func (c *Context) NewPlayer(src io.ReadSeeker) (*Player, error) {
	c.Lock()
	defer c.Unlock()
	p := &Player{
		context: c,
		src:     src,
		buf:     []byte{},
	}
	// Get the current position of the source.
	pos, err := p.src.Seek(0, 1)
	if err != nil {
		return nil, err
	}
	p.pos = pos
	return p, nil
}

func (p *Player) readToBuffer(length int) (int, error) {
	bb := make([]byte, length)
	n, err := p.src.Read(bb)
	if 0 < n {
		p.buf = append(p.buf, bb[:n]...)
	}
	return n, err
}

func (p *Player) bufferToInt16(lengthInBytes int) []int16 {
	r := make([]int16, lengthInBytes/2)
	for i := 0; i < lengthInBytes/2; i++ {
		r[i] = int16(p.buf[2*i]) | (int16(p.buf[2*i+1]) << 8)
	}
	return r
}

func (p *Player) proceed(length int) {
	p.buf = p.buf[length:]
	p.pos += int64(length)
}

func (p *Player) bufferLength() int {
	return len(p.buf)
}

func (p *Player) Play() error {
	p.context.Lock()
	defer p.context.Unlock()

	p.context.players[p] = struct{}{}
	return nil
}

func (p *Player) IsPlaying() bool {
	_, ok := p.context.players[p]
	return ok
}

func (p *Player) Rewind() error {
	return p.Seek(0)
}

func (p *Player) Seek(offset time.Duration) error {
	p.buf = []byte{}
	o := int64(offset) * int64(p.context.sampleRate) / int64(time.Second)
	pos, err := p.src.Seek(o, 0)
	if err != nil {
		return err
	}
	p.pos = pos
	return nil
}

func (p *Player) Pause() error {
	p.context.Lock()
	defer p.context.Unlock()

	delete(p.context.players, p)
	return nil
}

func (p *Player) Current() time.Duration {
	return time.Duration(p.pos) * time.Second / time.Duration(p.context.sampleRate)
}

// TODO: Volume / SetVolume?
// TODO: Panning
<commit_msg>audio: Bug fix: a buffer size passed to OpenAL must be a multiple of 4 (#187)<commit_after>// Copyright 2015 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package audio

import (
	"fmt"
	"io"
	"sync"
	"time"

	"github.com/hajimehoshi/ebiten"
)

type mixedPlayersStream struct {
	context      *Context
	writtenBytes int
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

const (
	channelNum     = 2
	bytesPerSample = 2
	bitsPerSample  = bytesPerSample * 8
)

func (s *mixedPlayersStream) Read(b []byte) (int, error) {
	s.context.Lock()
	defer s.context.Unlock()

	bytesPerFrame := s.context.sampleRate * bytesPerSample * channelNum / ebiten.FPS
	x := s.context.frames*bytesPerFrame + len(b)
	if x <= s.writtenBytes {
		return 0, nil
	}

	if len(s.context.players) == 0 {
		l := min(len(b), x-s.writtenBytes)
		l &= ^3
		copy(b, make([]byte, l))
		s.writtenBytes += l
		return l, nil
	}
	closed := []*Player{}
	l := len(b)
	for p := range s.context.players {
		_, err := p.readToBuffer(l)
		if err == io.EOF {
			closed = append(closed, p)
		} else if err != nil {
			return 0, err
		}
		l = min(p.bufferLength()/4*4, l)
	}
	l &= ^3
	b16s := [][]int16{}
	for p := range s.context.players {
		b16s = append(b16s, p.bufferToInt16(l))
	}
	for i := 0; i < l/2; i++ {
		x := 0
		for _, b16 := range b16s {
			x += int(b16[i])
		}
		if x > (1<<15)-1 {
			x = (1 << 15) - 1
		}
		if x < -(1 << 15) {
			x = -(1 << 15)
		}
		b[2*i] = byte(x)
		b[2*i+1] = byte(x >> 8)
	}
	for p := range s.context.players {
		p.proceed(l)
	}
	for _, p := range closed {
		delete(s.context.players, p)
	}
	s.writtenBytes += l
	return l, nil
}

// TODO: Enable to specify the format like Mono8?

type Context struct {
	sampleRate  int
	stream      *mixedPlayersStream
	players     map[*Player]struct{}
	innerPlayer *player
	frames      int
	sync.Mutex
}

func NewContext(sampleRate int) *Context {
	// TODO: Panic if one context exists.
	c := &Context{
		sampleRate: sampleRate,
		players:    map[*Player]struct{}{},
	}
	c.stream = &mixedPlayersStream{
		context: c,
	}
	p, err := startPlaying(c.stream, c.sampleRate)
	if err != nil {
		panic(fmt.Sprintf("audio: NewContext error: %v", err))
	}
	c.innerPlayer = p
	return c
}

// Update proceeds the inner (logical) time of the context by 1/60 second.
// This is expected to be called in the game's updating function (sync mode)
// or an independent goroutine with timers (unsync mode).
// In sync mode, the game logical time syncs the audio logical time and
// you will find audio stops when the game stops e.g. when the window is deactivated.
// In unsync mode, the audio never stops even when the game stops.
func (c *Context) Update() {
	c.Lock()
	defer c.Unlock()
	c.frames++
}

// SampleRate returns the sample rate.
func (c *Context) SampleRate() int {
	c.Lock()
	defer c.Unlock()
	return c.sampleRate
}

type Player struct {
	context *Context
	src     io.ReadSeeker
	buf     []byte
	pos     int64
}

// NewPlayer creates a new player with the given data to the given channel.
// The given data is queued to the end of the buffer.
// This may not be played immediately when data already exists in the buffer.
//
// src's format must be linear PCM (16bits, 2 channel stereo, little endian)
// without a header (e.g. RIFF header).
func (c *Context) NewPlayer(src io.ReadSeeker) (*Player, error) {
	c.Lock()
	defer c.Unlock()
	p := &Player{
		context: c,
		src:     src,
		buf:     []byte{},
	}
	// Get the current position of the source.
	pos, err := p.src.Seek(0, 1)
	if err != nil {
		return nil, err
	}
	p.pos = pos
	return p, nil
}

func (p *Player) readToBuffer(length int) (int, error) {
	bb := make([]byte, length)
	n, err := p.src.Read(bb)
	if 0 < n {
		p.buf = append(p.buf, bb[:n]...)
	}
	return n, err
}

func (p *Player) bufferToInt16(lengthInBytes int) []int16 {
	r := make([]int16, lengthInBytes/2)
	for i := 0; i < lengthInBytes/2; i++ {
		r[i] = int16(p.buf[2*i]) | (int16(p.buf[2*i+1]) << 8)
	}
	return r
}

func (p *Player) proceed(length int) {
	p.buf = p.buf[length:]
	p.pos += int64(length)
}

func (p *Player) bufferLength() int {
	return len(p.buf)
}

func (p *Player) Play() error {
	p.context.Lock()
	defer p.context.Unlock()

	p.context.players[p] = struct{}{}
	return nil
}

func (p *Player) IsPlaying() bool {
	_, ok := p.context.players[p]
	return ok
}

func (p *Player) Rewind() error {
	return p.Seek(0)
}

func (p *Player) Seek(offset time.Duration) error {
	p.buf = []byte{}
	o := int64(offset) * int64(p.context.sampleRate) / int64(time.Second)
	pos, err := p.src.Seek(o, 0)
	if err != nil {
		return err
	}
	p.pos = pos
	return nil
}

func (p *Player) Pause() error {
	p.context.Lock()
	defer p.context.Unlock()

	delete(p.context.players, p)
	return nil
}

func (p *Player) Current() time.Duration {
	return time.Duration(p.pos) * time.Second / time.Duration(p.context.sampleRate)
}

// TODO: Volume / SetVolume?
// TODO: Panning
<|endoftext|>