<commit_before>package s3

import (
	"errors"
	"io/ioutil"
	"net/url"
	"os"

	"golang.org/x/text/unicode/norm"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/livesense-inc/fanlin/lib/content"
	"github.com/livesense-inc/fanlin/lib/error"
)

var s3GetSourceFunc = getS3ImageBinary

// Test dedicated function
func setS3GetFunc(f func(region, bucket, key string, file *os.File) ([]byte, error)) {
	s3GetSourceFunc = f
}

func GetImageBinary(c *content.Content) ([]byte, error) {
	if c == nil {
		return nil, errors.New("content is nil")
	}
	s3url := c.SourcePlace
	u, err := url.Parse(s3url)
	if err != nil {
		return nil, imgproxyerr.New(imgproxyerr.WARNING, errors.New("can not parse s3 url"))
	}

	file, err := ioutil.TempFile(os.TempDir(), "s3_img")
	defer func() {
		os.Remove(file.Name())
		file.Close()
	}()
	if err != nil {
		return nil, imgproxyerr.New(imgproxyerr.ERROR, err)
	}

	bucket := u.Host

	if region, ok := c.Meta["region"].(string); ok {
		path, err := url.QueryUnescape(u.EscapedPath())
		if err != nil {
			return nil, err
		}
		if form, ok := c.Meta["norm_form"].(string); ok {
			path, err = NormalizePath(path, form)
			if err != nil {
				return nil, err
			}
		} else {
			path = norm.NFD.String(path)
		}
		return s3GetSourceFunc(region, bucket, path, file)
	}
	return nil, imgproxyerr.New(imgproxyerr.ERROR, errors.New("can not parse configure"))
}

func NormalizePath(path string, form string) (string, error) {
	switch form {
	case "nfd":
		return norm.NFD.String(path), nil
	case "nfc":
		return norm.NFC.String(path), nil
	case "nfkc":
		return norm.NFKC.String(path), nil
	case "nfkd":
		return norm.NFKD.String(path), nil
	}
	return "", imgproxyerr.New(imgproxyerr.WARNING, errors.New("invalid normalization form(" + form + ")"))
}

func getS3ImageBinary(region, bucket, key string, file *os.File) ([]byte, error) {
	downloader := s3manager.NewDownloader(session.New(&aws.Config{Region: aws.String(region)}))
	_, err := downloader.Download(file,
		&s3.GetObjectInput{
			Bucket: aws.String(bucket),
			Key:    aws.String(key),
		},
	)
	if err != nil {
		return nil, imgproxyerr.New(imgproxyerr.WARNING, err)
	}
	bin, err := ioutil.ReadFile(file.Name())
	return bin, imgproxyerr.New(imgproxyerr.ERROR, err)
}

func init() {
	content.RegisterContentType("s3", GetImageBinary)
}
<commit_msg>norm_formを何も指定しない場合、ノー変換にします<commit_after>package s3

import (
	"errors"
	"io/ioutil"
	"net/url"
	"os"

	"golang.org/x/text/unicode/norm"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/livesense-inc/fanlin/lib/content"
	"github.com/livesense-inc/fanlin/lib/error"
)

var s3GetSourceFunc = getS3ImageBinary

// Test dedicated function
func setS3GetFunc(f func(region, bucket, key string, file *os.File) ([]byte, error)) {
	s3GetSourceFunc = f
}

func GetImageBinary(c *content.Content) ([]byte, error) {
	if c == nil {
		return nil, errors.New("content is nil")
	}
	s3url := c.SourcePlace
	u, err := url.Parse(s3url)
	if err != nil {
		return nil, imgproxyerr.New(imgproxyerr.WARNING, errors.New("can not parse s3 url"))
	}

	file, err := ioutil.TempFile(os.TempDir(), "s3_img")
	defer func() {
		os.Remove(file.Name())
		file.Close()
	}()
	if err != nil {
		return nil, imgproxyerr.New(imgproxyerr.ERROR, err)
	}

	bucket := u.Host

	if region, ok := c.Meta["region"].(string); ok {
		path, err := url.QueryUnescape(u.EscapedPath())
		if err != nil {
			return nil, err
		}
		if form, ok := c.Meta["norm_form"].(string); ok {
			path, err = NormalizePath(path, form)
			if err != nil {
				return nil, err
			}
		}
		return s3GetSourceFunc(region, bucket, path, file)
	}
	return nil, imgproxyerr.New(imgproxyerr.ERROR, errors.New("can not parse configure"))
}

func NormalizePath(path string, form string) (string, error) {
	switch form {
	case "nfd":
		return norm.NFD.String(path), nil
	case "nfc":
		return norm.NFC.String(path), nil
	case "nfkc":
		return norm.NFKC.String(path), nil
	case "nfkd":
		return norm.NFKD.String(path), nil
	}
	return "", imgproxyerr.New(imgproxyerr.WARNING, errors.New("invalid normalization form(" + form + ")"))
}

func getS3ImageBinary(region, bucket, key string, file *os.File) ([]byte, error) {
	downloader := s3manager.NewDownloader(session.New(&aws.Config{Region: aws.String(region)}))
	_, err := downloader.Download(file,
		&s3.GetObjectInput{
			Bucket: aws.String(bucket),
			Key:    aws.String(key),
		},
	)
	if err != nil {
		return nil, imgproxyerr.New(imgproxyerr.WARNING, err)
	}
	bin, err := ioutil.ReadFile(file.Name())
	return bin, imgproxyerr.New(imgproxyerr.ERROR, err)
}

func init() {
	content.RegisterContentType("s3", GetImageBinary)
}
<|endoftext|>


<commit_before>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package elastigo

import (
	"bytes"
	"crypto/rand"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"strconv"
	"testing"
	"time"

	"github.com/araddon/gou"
	"github.com/bmizerany/assert"
)

//  go test -bench=".*"
//  go test -bench="Bulk"

var (
	buffers        = make([]*bytes.Buffer, 0)
	totalBytesSent int
	messageSets    int
)

func init() {
	flag.Parse()
	if testing.Verbose() {
		gou.SetupLogging("debug")
	}
}

// take two ints, compare, need to be within 5%
func closeInt(a, b int) bool {
	c := float64(a) / float64(b)
	if c >= .95 && c <= 1.05 {
		return true
	}
	return false
}

func TestBulkIndexerBasic(t *testing.T) {
	InitTests(true)
	c := NewTestConn()
	indexer := c.NewBulkIndexer(3)
	indexer.Sender = func(buf *bytes.Buffer) error {
		messageSets += 1
		totalBytesSent += buf.Len()
		buffers = append(buffers, buf)
		//		log.Printf("buffer:%s", string(buf.Bytes()))
		return indexer.Send(buf)
	}
	indexer.Start()

	date := time.Unix(1257894000, 0)
	data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0).Format(time.RFC3339)}

	err := indexer.Index("users", "user", "1", "", &date, data, true)

	waitFor(func() bool {
		return len(buffers) > 0
	}, 5)
	// part of request is url, so lets factor that in
	//totalBytesSent = totalBytesSent - len(*eshost)
	assert.T(t, len(buffers) == 1, fmt.Sprintf("Should have sent one operation but was %d", len(buffers)))
	assert.T(t, indexer.NumErrors() == 0 && err == nil, fmt.Sprintf("Should not have any errors. NumErrors: %v, err:%v", indexer.NumErrors(), err))
	expectedBytes := 160
	assert.T(t, totalBytesSent == expectedBytes, fmt.Sprintf("Should have sent %v bytes but was %v", expectedBytes, totalBytesSent))

	err = indexer.Index("users", "user", "2", "", nil, data, true)
	<-time.After(time.Millisecond * 10) // we need to wait for doc to hit send channel
	// this will test to ensure that Flush actually catches a doc
	indexer.Flush()
	totalBytesSent = totalBytesSent - len(*eshost)
	assert.T(t, err == nil, fmt.Sprintf("Should have nil error  =%v", err))
	assert.T(t, len(buffers) == 2, fmt.Sprintf("Should have another buffer ct=%d", len(buffers)))

	assert.T(t, indexer.NumErrors() == 0, fmt.Sprintf("Should not have any errors %d", indexer.NumErrors()))
	expectedBytes = 282 // with refresh
	assert.T(t, closeInt(totalBytesSent, expectedBytes), fmt.Sprintf("Should have sent %v bytes but was %v", expectedBytes, totalBytesSent))

	indexer.Stop()
}

// currently broken in drone.io
func XXXTestBulkUpdate(t *testing.T) {
	InitTests(true)
	c := NewTestConn()
	c.Port = "9200"
	indexer := c.NewBulkIndexer(3)
	indexer.Sender = func(buf *bytes.Buffer) error {
		messageSets += 1
		totalBytesSent += buf.Len()
		buffers = append(buffers, buf)
		return indexer.Send(buf)
	}
	indexer.Start()

	date := time.Unix(1257894000, 0)
	user := map[string]interface{}{
		"name": "smurfs", "age": 22, "date": date, "count": 1,
	}

	// Lets make sure the data is in the index ...
	_, err := c.Index("users", "user", "5", nil, user)

	// script and params
	data := map[string]interface{}{
		"script": "ctx._source.count += 2",
	}
	err = indexer.Update("users", "user", "5", "", &date, data, true)
	// So here's the deal. Flushing does seem to work, you just have to give the
	// channel a moment to recieve the message ...
	//	<- time.After(time.Millisecond * 20)
	//	indexer.Flush()

	waitFor(func() bool {
		return len(buffers) > 0
	}, 5)

	indexer.Stop()

	assert.T(t, indexer.NumErrors() == 0 && err == nil, fmt.Sprintf("Should not have any errors, bulkErrorCt:%v, err:%v", indexer.NumErrors(), err))

	response, err := c.Get("users", "user", "5", nil)
	assert.T(t, err == nil, fmt.Sprintf("Should not have any errors  %v", err))
	m := make(map[string]interface{})
	json.Unmarshal([]byte(*response.Source), &m)
	newCount := m["count"]
	assert.T(t, newCount.(float64) == 3,
		fmt.Sprintf("Should have update count: %#v ... %#v", m["count"], response))
}

func TestBulkSmallBatch(t *testing.T) {
	InitTests(true)
	c := NewTestConn()

	date := time.Unix(1257894000, 0)
	data := map[string]interface{}{"name": "smurfs", "age": 22, "date": date}

	// Now tests small batches
	indexer := c.NewBulkIndexer(1)
	indexer.BufferDelayMax = 100 * time.Millisecond
	indexer.BulkMaxDocs = 2
	messageSets = 0
	indexer.Sender = func(buf *bytes.Buffer) error {
		messageSets += 1
		return indexer.Send(buf)
	}
	indexer.Start()
	<-time.After(time.Millisecond * 20)

	indexer.Index("users", "user", "2", "", &date, data, true)
	indexer.Index("users", "user", "3", "", &date, data, true)
	indexer.Index("users", "user", "4", "", &date, data, true)
	<-time.After(time.Millisecond * 200)
	//	indexer.Flush()
	indexer.Stop()
	assert.T(t, messageSets == 2, fmt.Sprintf("Should have sent 2 message sets %d", messageSets))

}

func XXXTestBulkErrors(t *testing.T) {
	// lets set a bad port, and hope we get a conn refused error?
	c := NewTestConn()
	c.Port = "27845"
	defer func() {
		c.Port = "9200"
	}()
	indexer := c.NewBulkIndexerErrors(10, 1)
	indexer.Start()
	errorCt := 0
	go func() {
		for i := 0; i < 20; i++ {
			date := time.Unix(1257894000, 0)
			data := map[string]interface{}{"name": "smurfs", "age": 22, "date": date}
			indexer.Index("users", "user", strconv.Itoa(i), "", &date, data, true)
		}
	}()
	var errBuf *ErrorBuffer
	for errBuf = range indexer.ErrorChannel {
		errorCt++
		break
	}
	if errBuf.Buf.Len() > 0 {
		gou.Debug(errBuf.Err)
	}
	assert.T(t, errorCt > 0, fmt.Sprintf("ErrorCt should be > 0 %d", errorCt))
	indexer.Stop()
}

/*
BenchmarkSend	18:33:00 bulk_test.go:131: Sent 1 messages in 0 sets totaling 0 bytes
18:33:00 bulk_test.go:131: Sent 100 messages in 1 sets totaling 145889 bytes
18:33:01 bulk_test.go:131: Sent 10000 messages in 100 sets totaling 14608888 bytes
18:33:05 bulk_test.go:131: Sent 20000 messages in 99 sets totaling 14462790 bytes
   20000	    234526 ns/op

*/
func BenchmarkSend(b *testing.B) {
	InitTests(true)
	c := NewTestConn()
	b.StartTimer()
	totalBytes := 0
	sets := 0
	indexer := c.NewBulkIndexer(1)
	indexer.Sender = func(buf *bytes.Buffer) error {
		totalBytes += buf.Len()
		sets += 1
		//log.Println("got bulk")
		return indexer.Send(buf)
	}
	for i := 0; i < b.N; i++ {
		about := make([]byte, 1000)
		rand.Read(about)
		data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0), "about": about}
		indexer.Index("users", "user", strconv.Itoa(i), "", nil, data, true)
	}
	log.Printf("Sent %d messages in %d sets totaling %d bytes \n", b.N, sets, totalBytes)
	if indexer.NumErrors() != 0 {
		b.Fail()
	}
}

/*
TODO:  this should be faster than above

BenchmarkSendBytes	18:33:05 bulk_test.go:169: Sent 1 messages in 0 sets totaling 0 bytes
18:33:05 bulk_test.go:169: Sent 100 messages in 2 sets totaling 292299 bytes
18:33:09 bulk_test.go:169: Sent 10000 messages in 99 sets totaling 14473800 bytes
   10000	    373529 ns/op

*/
func BenchmarkSendBytes(b *testing.B) {
	InitTests(true)
	c := NewTestConn()
	about := make([]byte, 1000)
	rand.Read(about)
	data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0), "about": about}
	body, _ := json.Marshal(data)
	b.StartTimer()
	totalBytes := 0
	sets := 0
	indexer := c.NewBulkIndexer(1)
	indexer.Sender = func(buf *bytes.Buffer) error {
		totalBytes += buf.Len()
		sets += 1
		return indexer.Send(buf)
	}
	for i := 0; i < b.N; i++ {
		indexer.Index("users", "user", strconv.Itoa(i), "", nil, body, true)
	}
	log.Printf("Sent %d messages in %d sets totaling %d bytes \n", b.N, sets, totalBytes)
	if indexer.NumErrors() != 0 {
		b.Fail()
	}
}
<commit_msg>Don't use globals which are shared among tests.<commit_after>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package elastigo

import (
	"bytes"
	"crypto/rand"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"strconv"
	"testing"
	"time"

	"github.com/araddon/gou"
	"github.com/bmizerany/assert"
)

//  go test -bench=".*"
//  go test -bench="Bulk"

func init() {
	flag.Parse()
	if testing.Verbose() {
		gou.SetupLogging("debug")
	}
}

// take two ints, compare, need to be within 5%
func closeInt(a, b int) bool {
	c := float64(a) / float64(b)
	if c >= .95 && c <= 1.05 {
		return true
	}
	return false
}

func TestBulkIndexerBasic(t *testing.T) {
	testIndex := "users"
	var (
		buffers        = make([]*bytes.Buffer, 0)
		totalBytesSent int
		messageSets    int
	)

	InitTests(true)
	c := NewTestConn()
	indexer := c.NewBulkIndexer(3)
	indexer.Sender = func(buf *bytes.Buffer) error {
		messageSets += 1
		totalBytesSent += buf.Len()
		buffers = append(buffers, buf)
		// log.Printf("buffer:%s", string(buf.Bytes()))
		return indexer.Send(buf)
	}
	indexer.Start()

	date := time.Unix(1257894000, 0)
	data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0).Format(time.RFC3339)}

	err := indexer.Index(testIndex, "user", "1", "", &date, data, true)

	waitFor(func() bool {
		return len(buffers) > 0
	}, 5)
	// part of request is url, so lets factor that in
	//totalBytesSent = totalBytesSent - len(*eshost)
	assert.T(t, len(buffers) == 1, fmt.Sprintf("Should have sent one operation but was %d", len(buffers)))
	assert.T(t, indexer.NumErrors() == 0 && err == nil, fmt.Sprintf("Should not have any errors. NumErrors: %v, err:%v", indexer.NumErrors(), err))
	expectedBytes := 160
	assert.T(t, totalBytesSent == expectedBytes, fmt.Sprintf("Should have sent %v bytes but was %v", expectedBytes, totalBytesSent))

	err = indexer.Index(testIndex, "user", "2", "", nil, data, true)
	<-time.After(time.Millisecond * 10) // we need to wait for doc to hit send channel
	// this will test to ensure that Flush actually catches a doc
	indexer.Flush()
	totalBytesSent = totalBytesSent - len(*eshost)
	assert.T(t, err == nil, fmt.Sprintf("Should have nil error  =%v", err))
	assert.T(t, len(buffers) == 2, fmt.Sprintf("Should have another buffer ct=%d", len(buffers)))

	assert.T(t, indexer.NumErrors() == 0, fmt.Sprintf("Should not have any errors %d", indexer.NumErrors()))
	expectedBytes = 282 // with refresh
	assert.T(t, closeInt(totalBytesSent, expectedBytes), fmt.Sprintf("Should have sent %v bytes but was %v", expectedBytes, totalBytesSent))

	indexer.Stop()
}

// currently broken in drone.io
func XXXTestBulkUpdate(t *testing.T) {
	var (
		buffers        = make([]*bytes.Buffer, 0)
		totalBytesSent int
		messageSets    int
	)

	InitTests(true)
	c := NewTestConn()
	c.Port = "9200"
	indexer := c.NewBulkIndexer(3)
	indexer.Sender = func(buf *bytes.Buffer) error {
		messageSets += 1
		totalBytesSent += buf.Len()
		buffers = append(buffers, buf)
		return indexer.Send(buf)
	}
	indexer.Start()

	date := time.Unix(1257894000, 0)
	user := map[string]interface{}{
		"name": "smurfs", "age": 22, "date": date, "count": 1,
	}

	// Lets make sure the data is in the index ...
	_, err := c.Index("users", "user", "5", nil, user)

	// script and params
	data := map[string]interface{}{
		"script": "ctx._source.count += 2",
	}
	err = indexer.Update("users", "user", "5", "", &date, data, true)
	// So here's the deal. Flushing does seem to work, you just have to give the
	// channel a moment to recieve the message ...
	//	<- time.After(time.Millisecond * 20)
	//	indexer.Flush()

	waitFor(func() bool {
		return len(buffers) > 0
	}, 5)

	indexer.Stop()

	assert.T(t, indexer.NumErrors() == 0 && err == nil, fmt.Sprintf("Should not have any errors, bulkErrorCt:%v, err:%v", indexer.NumErrors(), err))

	response, err := c.Get("users", "user", "5", nil)
	assert.T(t, err == nil, fmt.Sprintf("Should not have any errors  %v", err))
	m := make(map[string]interface{})
	json.Unmarshal([]byte(*response.Source), &m)
	newCount := m["count"]
	assert.T(t, newCount.(float64) == 3,
		fmt.Sprintf("Should have update count: %#v ... %#v", m["count"], response))
}

func TestBulkSmallBatch(t *testing.T) {
	var (
		messageSets int
	)

	InitTests(true)
	c := NewTestConn()

	date := time.Unix(1257894000, 0)
	data := map[string]interface{}{"name": "smurfs", "age": 22, "date": date}

	// Now tests small batches
	indexer := c.NewBulkIndexer(1)
	indexer.BufferDelayMax = 100 * time.Millisecond
	indexer.BulkMaxDocs = 2
	messageSets = 0
	indexer.Sender = func(buf *bytes.Buffer) error {
		messageSets += 1
		return indexer.Send(buf)
	}
	indexer.Start()
	<-time.After(time.Millisecond * 20)

	indexer.Index("users", "user", "2", "", &date, data, true)
	indexer.Index("users", "user", "3", "", &date, data, true)
	indexer.Index("users", "user", "4", "", &date, data, true)
	<-time.After(time.Millisecond * 200)
	//	indexer.Flush()
	indexer.Stop()
	assert.T(t, messageSets == 2, fmt.Sprintf("Should have sent 2 message sets %d", messageSets))

}

func XXXTestBulkErrors(t *testing.T) {
	// lets set a bad port, and hope we get a conn refused error?
	c := NewTestConn()
	c.Port = "27845"
	defer func() {
		c.Port = "9200"
	}()
	indexer := c.NewBulkIndexerErrors(10, 1)
	indexer.Start()
	errorCt := 0
	go func() {
		for i := 0; i < 20; i++ {
			date := time.Unix(1257894000, 0)
			data := map[string]interface{}{"name": "smurfs", "age": 22, "date": date}
			indexer.Index("users", "user", strconv.Itoa(i), "", &date, data, true)
		}
	}()
	var errBuf *ErrorBuffer
	for errBuf = range indexer.ErrorChannel {
		errorCt++
		break
	}
	if errBuf.Buf.Len() > 0 {
		gou.Debug(errBuf.Err)
	}
	assert.T(t, errorCt > 0, fmt.Sprintf("ErrorCt should be > 0 %d", errorCt))
	indexer.Stop()
}

/*
BenchmarkSend	18:33:00 bulk_test.go:131: Sent 1 messages in 0 sets totaling 0 bytes
18:33:00 bulk_test.go:131: Sent 100 messages in 1 sets totaling 145889 bytes
18:33:01 bulk_test.go:131: Sent 10000 messages in 100 sets totaling 14608888 bytes
18:33:05 bulk_test.go:131: Sent 20000 messages in 99 sets totaling 14462790 bytes
   20000	    234526 ns/op

*/
func BenchmarkSend(b *testing.B) {
	InitTests(true)
	c := NewTestConn()
	b.StartTimer()
	totalBytes := 0
	sets := 0
	indexer := c.NewBulkIndexer(1)
	indexer.Sender = func(buf *bytes.Buffer) error {
		totalBytes += buf.Len()
		sets += 1
		//log.Println("got bulk")
		return indexer.Send(buf)
	}
	for i := 0; i < b.N; i++ {
		about := make([]byte, 1000)
		rand.Read(about)
		data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0), "about": about}
		indexer.Index("users", "user", strconv.Itoa(i), "", nil, data, true)
	}
	log.Printf("Sent %d messages in %d sets totaling %d bytes \n", b.N, sets, totalBytes)
	if indexer.NumErrors() != 0 {
		b.Fail()
	}
}

/*
TODO:  this should be faster than above

BenchmarkSendBytes	18:33:05 bulk_test.go:169: Sent 1 messages in 0 sets totaling 0 bytes
18:33:05 bulk_test.go:169: Sent 100 messages in 2 sets totaling 292299 bytes
18:33:09 bulk_test.go:169: Sent 10000 messages in 99 sets totaling 14473800 bytes
   10000	    373529 ns/op

*/
func BenchmarkSendBytes(b *testing.B) {
	InitTests(true)
	c := NewTestConn()
	about := make([]byte, 1000)
	rand.Read(about)
	data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0), "about": about}
	body, _ := json.Marshal(data)
	b.StartTimer()
	totalBytes := 0
	sets := 0
	indexer := c.NewBulkIndexer(1)
	indexer.Sender = func(buf *bytes.Buffer) error {
		totalBytes += buf.Len()
		sets += 1
		return indexer.Send(buf)
	}
	for i := 0; i < b.N; i++ {
		indexer.Index("users", "user", strconv.Itoa(i), "", nil, body, true)
	}
	log.Printf("Sent %d messages in %d sets totaling %d bytes \n", b.N, sets, totalBytes)
	if indexer.NumErrors() != 0 {
		b.Fail()
	}
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"io/ioutil"
	"log"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/lambda"
	"github.com/mitchellh/go-homedir"

	"errors"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsLambdaFunction() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsLambdaFunctionCreate,
		Read:   resourceAwsLambdaFunctionRead,
		Update: resourceAwsLambdaFunctionUpdate,
		Delete: resourceAwsLambdaFunctionDelete,

		Schema: map[string]*schema.Schema{
			"filename": &schema.Schema{
				Type:          schema.TypeString,
				Optional:      true,
				ConflictsWith: []string{"s3_bucket", "s3_key", "s3_object_version"},
			},
			"s3_bucket": &schema.Schema{
				Type:          schema.TypeString,
				Optional:      true,
				ConflictsWith: []string{"filename"},
			},
			"s3_key": &schema.Schema{
				Type:          schema.TypeString,
				Optional:      true,
				ConflictsWith: []string{"filename"},
			},
			"s3_object_version": &schema.Schema{
				Type:          schema.TypeString,
				Optional:      true,
				ConflictsWith: []string{"filename"},
			},
			"description": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},
			"function_name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"handler": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"memory_size": &schema.Schema{
				Type:     schema.TypeInt,
				Optional: true,
				Default:  128,
			},
			"role": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"runtime": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				Default:  "nodejs",
			},
			"timeout": &schema.Schema{
				Type:     schema.TypeInt,
				Optional: true,
				Default:  3,
			},
			"vpc_config": &schema.Schema{
				Type:     schema.TypeList,
				Optional: true,
				ForceNew: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"subnet_ids": &schema.Schema{
							Type:     schema.TypeSet,
							Required: true,
							ForceNew: true,
							Elem:     &schema.Schema{Type: schema.TypeString},
							Set:      schema.HashString,
						},
						"security_group_ids": &schema.Schema{
							Type:     schema.TypeSet,
							Required: true,
							ForceNew: true,
							Elem:     &schema.Schema{Type: schema.TypeString},
							Set:      schema.HashString,
						},
					},
				},
			},
			"arn": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"last_modified": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"source_code_hash": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},
		},
	}
}

// resourceAwsLambdaFunction maps to:
// CreateFunction in the API / SDK
func resourceAwsLambdaFunctionCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).lambdaconn

	functionName := d.Get("function_name").(string)
	iamRole := d.Get("role").(string)

	log.Printf("[DEBUG] Creating Lambda Function %s with role %s", functionName, iamRole)

	var functionCode *lambda.FunctionCode
	if v, ok := d.GetOk("filename"); ok {
		file, err := loadFileContent(v.(string))
		if err != nil {
			return fmt.Errorf("Unable to load %q: %s", v.(string), err)
		}
		functionCode = &lambda.FunctionCode{
			ZipFile: file,
		}
	} else {
		s3Bucket, bucketOk := d.GetOk("s3_bucket")
		s3Key, keyOk := d.GetOk("s3_key")
		s3ObjectVersion, versionOk := d.GetOk("s3_object_version")
		if !bucketOk || !keyOk {
			return errors.New("s3_bucket and s3_key must all be set while using S3 code source")
		}
		functionCode = &lambda.FunctionCode{
			S3Bucket: aws.String(s3Bucket.(string)),
			S3Key:    aws.String(s3Key.(string)),
		}
		if versionOk {
			functionCode.S3ObjectVersion = aws.String(s3ObjectVersion.(string))
		}
	}

	params := &lambda.CreateFunctionInput{
		Code:         functionCode,
		Description:  aws.String(d.Get("description").(string)),
		FunctionName: aws.String(functionName),
		Handler:      aws.String(d.Get("handler").(string)),
		MemorySize:   aws.Int64(int64(d.Get("memory_size").(int))),
		Role:         aws.String(iamRole),
		Runtime:      aws.String(d.Get("runtime").(string)),
		Timeout:      aws.Int64(int64(d.Get("timeout").(int))),
	}

	if v, ok := d.GetOk("vpc_config"); ok {
		config, err := validateVPCConfig(v)
		if err != nil {
			return err
		}

		var subnetIds []*string
		for _, id := range config["subnet_ids"].(*schema.Set).List() {
			subnetIds = append(subnetIds, aws.String(id.(string)))
		}

		var securityGroupIds []*string
		for _, id := range config["security_group_ids"].(*schema.Set).List() {
			securityGroupIds = append(securityGroupIds, aws.String(id.(string)))
		}

		params.VpcConfig = &lambda.VpcConfig{
			SubnetIds:        subnetIds,
			SecurityGroupIds: securityGroupIds,
		}
	}

	// IAM profiles can take ~10 seconds to propagate in AWS:
	// http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#launch-instance-with-role-console
	// Error creating Lambda function: InvalidParameterValueException: The role defined for the task cannot be assumed by Lambda.
	err := resource.Retry(1*time.Minute, func() *resource.RetryError {
		_, err := conn.CreateFunction(params)
		if err != nil {
			log.Printf("[ERROR] Received %q, retrying CreateFunction", err)
			if awserr, ok := err.(awserr.Error); ok {
				if awserr.Code() == "InvalidParameterValueException" {
					log.Printf("[DEBUG] InvalidParameterValueException creating Lambda Function: %s", awserr)
					return resource.RetryableError(awserr)
				}
			}
			log.Printf("[DEBUG] Error creating Lambda Function: %s", err)
			return resource.NonRetryableError(err)
		}
		return nil
	})
	if err != nil {
		return fmt.Errorf("Error creating Lambda function: %s", err)
	}

	d.SetId(d.Get("function_name").(string))

	return resourceAwsLambdaFunctionRead(d, meta)
}

// resourceAwsLambdaFunctionRead maps to:
// GetFunction in the API / SDK
func resourceAwsLambdaFunctionRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).lambdaconn

	log.Printf("[DEBUG] Fetching Lambda Function: %s", d.Id())

	params := &lambda.GetFunctionInput{
		FunctionName: aws.String(d.Get("function_name").(string)),
	}

	getFunctionOutput, err := conn.GetFunction(params)
	if err != nil {
		return err
	}

	// getFunctionOutput.Code.Location is a pre-signed URL pointing at the zip
	// file that we uploaded when we created the resource. You can use it to
	// download the code from AWS. The other part is
	// getFunctionOutput.Configuration which holds metadata.

	function := getFunctionOutput.Configuration
	// TODO error checking / handling on the Set() calls.
	d.Set("arn", function.FunctionArn)
	d.Set("description", function.Description)
	d.Set("handler", function.Handler)
	d.Set("memory_size", function.MemorySize)
	d.Set("last_modified", function.LastModified)
	d.Set("role", function.Role)
	d.Set("runtime", function.Runtime)
	d.Set("timeout", function.Timeout)
	if config := flattenLambdaVpcConfigResponse(function.VpcConfig); len(config) > 0 {
		d.Set("vpc_config", config)
	}
	d.Set("source_code_hash", function.CodeSha256)

	return nil
}

// resourceAwsLambdaFunction maps to:
// DeleteFunction in the API / SDK
func resourceAwsLambdaFunctionDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).lambdaconn

	log.Printf("[INFO] Deleting Lambda Function: %s", d.Id())

	params := &lambda.DeleteFunctionInput{
		FunctionName: aws.String(d.Get("function_name").(string)),
	}

	_, err := conn.DeleteFunction(params)
	if err != nil {
		return fmt.Errorf("Error deleting Lambda Function: %s", err)
	}

	d.SetId("")

	return nil
}

// resourceAwsLambdaFunctionUpdate maps to:
// UpdateFunctionCode in the API / SDK
func resourceAwsLambdaFunctionUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).lambdaconn

	d.Partial(true)

	codeReq := &lambda.UpdateFunctionCodeInput{
		FunctionName: aws.String(d.Id()),
	}

	codeUpdate := false
	if v, ok := d.GetOk("filename"); ok && d.HasChange("source_code_hash") {
		file, err := loadFileContent(v.(string))
		if err != nil {
			return fmt.Errorf("Unable to load %q: %s", v.(string), err)
		}
		codeReq.ZipFile = file
		codeUpdate = true
	}
	if d.HasChange("s3_bucket") || d.HasChange("s3_key") || d.HasChange("s3_object_version") {
		codeReq.S3Bucket = aws.String(d.Get("s3_bucket").(string))
		codeReq.S3Key = aws.String(d.Get("s3_key").(string))
		codeReq.S3ObjectVersion = aws.String(d.Get("s3_object_version").(string))
		codeUpdate = true
	}

	log.Printf("[DEBUG] Send Update Lambda Function Code request: %#v", codeReq)
	if codeUpdate {
		_, err := conn.UpdateFunctionCode(codeReq)
		if err != nil {
			return fmt.Errorf("Error modifying Lambda Function Code %s: %s", d.Id(), err)
		}

		d.SetPartial("filename")
		d.SetPartial("source_code_hash")
		d.SetPartial("s3_bucket")
		d.SetPartial("s3_key")
		d.SetPartial("s3_object_version")
	}

	configReq := &lambda.UpdateFunctionConfigurationInput{
		FunctionName: aws.String(d.Id()),
	}

	configUpdate := false
	if d.HasChange("description") {
		configReq.Description = aws.String(d.Get("description").(string))
		configUpdate = true
	}
	if d.HasChange("handler") {
		configReq.Handler = aws.String(d.Get("handler").(string))
		configUpdate = true
	}
	if d.HasChange("memory_size") {
		configReq.MemorySize = aws.Int64(int64(d.Get("memory_size").(int)))
		configUpdate = true
	}
	if d.HasChange("role") {
		configReq.Role = aws.String(d.Get("role").(string))
		configUpdate = true
	}
	if d.HasChange("timeout") {
		configReq.Timeout = aws.Int64(int64(d.Get("timeout").(int)))
		configUpdate = true
	}

	log.Printf("[DEBUG] Send Update Lambda Function Configuration request: %#v", configReq)
	if configUpdate {
		_, err := conn.UpdateFunctionConfiguration(configReq)
		if err != nil {
			return fmt.Errorf("Error modifying Lambda Function Configuration %s: %s", d.Id(), err)
		}
		d.SetPartial("description")
		d.SetPartial("handler")
		d.SetPartial("memory_size")
		d.SetPartial("role")
		d.SetPartial("timeout")
	}
	d.Partial(false)

	return resourceAwsLambdaFunctionRead(d, meta)
}

// loadFileContent returns contents of a file in a given path
func loadFileContent(v string) ([]byte, error) {
	filename, err := homedir.Expand(v)
	if err != nil {
		return nil, err
	}
	fileContent, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	return fileContent, nil
}

func validateVPCConfig(v interface{}) (map[string]interface{}, error) {
	configs := v.([]interface{})
	if len(configs) > 1 {
		return nil, errors.New("Only a single vpc_config block is expected")
	}

	config, ok := configs[0].(map[string]interface{})

	if !ok {
		return nil, errors.New("vpc_config is <nil>")
	}

	if config["subnet_ids"].(*schema.Set).Len() == 0 {
		return nil, errors.New("vpc_config.subnet_ids cannot be empty")
	}

	if config["security_group_ids"].(*schema.Set).Len() == 0 {
		return nil, errors.New("vpc_config.security_group_ids cannot be empty")
	}

	return config, nil
}
<commit_msg>Correctly handle missing lambda function<commit_after>package aws

import (
	"fmt"
	"io/ioutil"
	"log"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/lambda"
	"github.com/mitchellh/go-homedir"

	"errors"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsLambdaFunction() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsLambdaFunctionCreate,
		Read:   resourceAwsLambdaFunctionRead,
		Update: resourceAwsLambdaFunctionUpdate,
		Delete: resourceAwsLambdaFunctionDelete,

		Schema: map[string]*schema.Schema{
			"filename": &schema.Schema{
				Type:          schema.TypeString,
				Optional:      true,
				ConflictsWith: []string{"s3_bucket", "s3_key", "s3_object_version"},
			},
			"s3_bucket": &schema.Schema{
				Type:          schema.TypeString,
				Optional:      true,
				ConflictsWith: []string{"filename"},
			},
			"s3_key": &schema.Schema{
				Type:          schema.TypeString,
				Optional:      true,
				ConflictsWith: []string{"filename"},
			},
			"s3_object_version": &schema.Schema{
				Type:          schema.TypeString,
				Optional:      true,
				ConflictsWith: []string{"filename"},
			},
			"description": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},
			"function_name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"handler": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"memory_size": &schema.Schema{
				Type:     schema.TypeInt,
				Optional: true,
				Default:  128,
			},
			"role": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"runtime": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				Default:  "nodejs",
			},
			"timeout": &schema.Schema{
				Type:     schema.TypeInt,
				Optional: true,
				Default:  3,
			},
			"vpc_config": &schema.Schema{
				Type:     schema.TypeList,
				Optional: true,
				ForceNew: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"subnet_ids": &schema.Schema{
							Type:     schema.TypeSet,
							Required: true,
							ForceNew: true,
							Elem:     &schema.Schema{Type: schema.TypeString},
							Set:      schema.HashString,
						},
						"security_group_ids": &schema.Schema{
							Type:     schema.TypeSet,
							Required: true,
							ForceNew: true,
							Elem:     &schema.Schema{Type: schema.TypeString},
							Set:      schema.HashString,
						},
					},
				},
			},
			"arn": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"last_modified": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"source_code_hash": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},
		},
	}
}

// resourceAwsLambdaFunction maps to:
// CreateFunction in the API / SDK
func resourceAwsLambdaFunctionCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).lambdaconn

	functionName := d.Get("function_name").(string)
	iamRole := d.Get("role").(string)

	log.Printf("[DEBUG] Creating Lambda Function %s with role %s", functionName, iamRole)

	var functionCode *lambda.FunctionCode
	if v, ok := d.GetOk("filename"); ok {
		file, err := loadFileContent(v.(string))
		if err != nil {
			return fmt.Errorf("Unable to load %q: %s", v.(string), err)
		}
		functionCode = &lambda.FunctionCode{
			ZipFile: file,
		}
	} else {
		s3Bucket, bucketOk := d.GetOk("s3_bucket")
		s3Key, keyOk := d.GetOk("s3_key")
		s3ObjectVersion, versionOk := d.GetOk("s3_object_version")
		if !bucketOk || !keyOk {
			return errors.New("s3_bucket and s3_key must all be set while using S3 code source")
		}
		functionCode = &lambda.FunctionCode{
			S3Bucket: aws.String(s3Bucket.(string)),
			S3Key:    aws.String(s3Key.(string)),
		}
		if versionOk {
			functionCode.S3ObjectVersion = aws.String(s3ObjectVersion.(string))
		}
	}

	params := &lambda.CreateFunctionInput{
		Code:         functionCode,
		Description:  aws.String(d.Get("description").(string)),
		FunctionName: aws.String(functionName),
		Handler:      aws.String(d.Get("handler").(string)),
		MemorySize:   aws.Int64(int64(d.Get("memory_size").(int))),
		Role:         aws.String(iamRole),
		Runtime:      aws.String(d.Get("runtime").(string)),
		Timeout:      aws.Int64(int64(d.Get("timeout").(int))),
	}

	if v, ok := d.GetOk("vpc_config"); ok {
		config, err := validateVPCConfig(v)
		if err != nil {
			return err
		}

		var subnetIds []*string
		for _, id := range config["subnet_ids"].(*schema.Set).List() {
			subnetIds = append(subnetIds, aws.String(id.(string)))
		}

		var securityGroupIds []*string
		for _, id := range config["security_group_ids"].(*schema.Set).List() {
			securityGroupIds = append(securityGroupIds, aws.String(id.(string)))
		}

		params.VpcConfig = &lambda.VpcConfig{
			SubnetIds:        subnetIds,
			SecurityGroupIds: securityGroupIds,
		}
	}

	// IAM profiles can take ~10 seconds to propagate in AWS:
	// http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#launch-instance-with-role-console
	// Error creating Lambda function: InvalidParameterValueException: The role defined for the task cannot be assumed by Lambda.
	err := resource.Retry(1*time.Minute, func() *resource.RetryError {
		_, err := conn.CreateFunction(params)
		if err != nil {
			log.Printf("[ERROR] Received %q, retrying CreateFunction", err)
			if awserr, ok := err.(awserr.Error); ok {
				if awserr.Code() == "InvalidParameterValueException" {
					log.Printf("[DEBUG] InvalidParameterValueException creating Lambda Function: %s", awserr)
					return resource.RetryableError(awserr)
				}
			}
			log.Printf("[DEBUG] Error creating Lambda Function: %s", err)
			return resource.NonRetryableError(err)
		}
		return nil
	})
	if err != nil {
		return fmt.Errorf("Error creating Lambda function: %s", err)
	}

	d.SetId(d.Get("function_name").(string))

	return resourceAwsLambdaFunctionRead(d, meta)
}

// resourceAwsLambdaFunctionRead maps to:
// GetFunction in the API / SDK
func resourceAwsLambdaFunctionRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).lambdaconn

	log.Printf("[DEBUG] Fetching Lambda Function: %s", d.Id())

	params := &lambda.GetFunctionInput{
		FunctionName: aws.String(d.Get("function_name").(string)),
	}

	getFunctionOutput, err := conn.GetFunction(params)
	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "ResourceNotFoundException" {
			d.SetId("")
			return nil
		}
		return err
	}

	// getFunctionOutput.Code.Location is a pre-signed URL pointing at the zip
	// file that we uploaded when we created the resource. You can use it to
	// download the code from AWS. The other part is
	// getFunctionOutput.Configuration which holds metadata.

	function := getFunctionOutput.Configuration
	// TODO error checking / handling on the Set() calls.
	d.Set("arn", function.FunctionArn)
	d.Set("description", function.Description)
	d.Set("handler", function.Handler)
	d.Set("memory_size", function.MemorySize)
	d.Set("last_modified", function.LastModified)
	d.Set("role", function.Role)
	d.Set("runtime", function.Runtime)
	d.Set("timeout", function.Timeout)
	if config := flattenLambdaVpcConfigResponse(function.VpcConfig); len(config) > 0 {
		d.Set("vpc_config", config)
	}
	d.Set("source_code_hash", function.CodeSha256)

	return nil
}

// resourceAwsLambdaFunction maps to:
// DeleteFunction in the API / SDK
func resourceAwsLambdaFunctionDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).lambdaconn

	log.Printf("[INFO] Deleting Lambda Function: %s", d.Id())

	params := &lambda.DeleteFunctionInput{
		FunctionName: aws.String(d.Get("function_name").(string)),
	}

	_, err := conn.DeleteFunction(params)
	if err != nil {
		return fmt.Errorf("Error deleting Lambda Function: %s", err)
	}

	d.SetId("")

	return nil
}

// resourceAwsLambdaFunctionUpdate maps to:
// UpdateFunctionCode in the API / SDK
func resourceAwsLambdaFunctionUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).lambdaconn

	d.Partial(true)

	codeReq := &lambda.UpdateFunctionCodeInput{
		FunctionName: aws.String(d.Id()),
	}

	codeUpdate := false
	if v, ok := d.GetOk("filename"); ok && d.HasChange("source_code_hash") {
		file, err := loadFileContent(v.(string))
		if err != nil {
			return fmt.Errorf("Unable to load %q: %s", v.(string), err)
		}
		codeReq.ZipFile = file
		codeUpdate = true
	}
	if d.HasChange("s3_bucket") || d.HasChange("s3_key") || d.HasChange("s3_object_version") {
		codeReq.S3Bucket = aws.String(d.Get("s3_bucket").(string))
		codeReq.S3Key = aws.String(d.Get("s3_key").(string))
		codeReq.S3ObjectVersion = aws.String(d.Get("s3_object_version").(string))
		codeUpdate = true
	}

	log.Printf("[DEBUG] Send Update Lambda Function Code request: %#v", codeReq)
	if codeUpdate {
		_, err := conn.UpdateFunctionCode(codeReq)
		if err != nil {
			return fmt.Errorf("Error modifying Lambda Function Code %s: %s", d.Id(), err)
		}

		d.SetPartial("filename")
		d.SetPartial("source_code_hash")
		d.SetPartial("s3_bucket")
		d.SetPartial("s3_key")
		d.SetPartial("s3_object_version")
	}

	configReq := &lambda.UpdateFunctionConfigurationInput{
		FunctionName: aws.String(d.Id()),
	}

	configUpdate := false
	if d.HasChange("description") {
		configReq.Description = aws.String(d.Get("description").(string))
		configUpdate = true
	}
	if d.HasChange("handler") {
		configReq.Handler = aws.String(d.Get("handler").(string))
		configUpdate = true
	}
	if d.HasChange("memory_size") {
		configReq.MemorySize = aws.Int64(int64(d.Get("memory_size").(int)))
		configUpdate = true
	}
	if d.HasChange("role") {
		configReq.Role = aws.String(d.Get("role").(string))
		configUpdate = true
	}
	if d.HasChange("timeout") {
		configReq.Timeout = aws.Int64(int64(d.Get("timeout").(int)))
		configUpdate = true
	}

	log.Printf("[DEBUG] Send Update Lambda Function Configuration request: %#v", configReq)
	if configUpdate {
		_, err := conn.UpdateFunctionConfiguration(configReq)
		if err != nil {
			return fmt.Errorf("Error modifying Lambda Function Configuration %s: %s", d.Id(), err)
		}
		d.SetPartial("description")
		d.SetPartial("handler")
		d.SetPartial("memory_size")
		d.SetPartial("role")
		d.SetPartial("timeout")
	}
	d.Partial(false)

	return resourceAwsLambdaFunctionRead(d, meta)
}

// loadFileContent returns contents of a file in a given path
func loadFileContent(v string) ([]byte, error) {
	filename, err := homedir.Expand(v)
	if err != nil {
		return nil, err
	}
	fileContent, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	return fileContent, nil
}

func validateVPCConfig(v interface{}) (map[string]interface{}, error) {
	configs := v.([]interface{})
	if len(configs) > 1 {
		return nil, errors.New("Only a single vpc_config block is expected")
	}

	config, ok := configs[0].(map[string]interface{})

	if !ok {
		return nil, errors.New("vpc_config is <nil>")
	}

	if config["subnet_ids"].(*schema.Set).Len() == 0 {
		return nil, errors.New("vpc_config.subnet_ids cannot be empty")
	}

	if config["security_group_ids"].(*schema.Set).Len() == 0 {
		return nil, errors.New("vpc_config.security_group_ids cannot be empty")
	}

	return config, nil
}
<|endoftext|>


<commit_before>// package defcache wraps our index with performance metrics, a mutex for threadsafe access
// and synchronisation with a definition store
package defcache

import (
	"github.com/grafana/grafana/pkg/log"
	"github.com/raintank/met"
	"github.com/raintank/raintank-metric/metric_tank/idx"
	"github.com/raintank/raintank-metric/metricdef"
	"github.com/raintank/raintank-metric/schema"
	"sync"
	"time"
)

var (
	metricsToEsOK           met.Count
	metricsToEsFail         met.Count
	esPutDuration           met.Timer // note that due to our use of bulk indexer, most values will be very fast with the occasional "outlier" which triggers a flush
	idxPruneDuration        met.Timer
	idxGetDuration          met.Timer
	idxListDuration         met.Timer
	idxMatchLiteralDuration met.Timer
	idxMatchPrefixDuration  met.Timer
	idxMatchTrigramDuration met.Timer
)

// design notes:
// MT pulls in all definitions when it starts up.
// those "old" ones + whatever it sees as inputs from the metrics queue
// is enough for it to always know the complete current state
// nothing should update ES "behind its back", so we never need to pull
// from ES other then at startup.
// but other MT instances may update ES while we are down, so ES is a good
// place to pull from, until the performance is demonstrably too slow.
// there are some vectors here for race conditions but we can work those out
// later, perhaps when tacking the multiple-intervals work

type DefCache struct {
	sync.RWMutex
	idx.Idx
	defsStore metricdef.Defs
}

func New(defsStore metricdef.Defs, stats met.Backend) *DefCache {
	metricsToEsOK = stats.NewCount("metrics_to_es.ok")
	metricsToEsFail = stats.NewCount("metrics_to_es.fail")
	esPutDuration = stats.NewTimer("es_put_duration", 0)
	idxPruneDuration = stats.NewTimer("idx.prune_duration", 0)
	idxGetDuration = stats.NewTimer("idx.get_duration", 0)
	idxListDuration = stats.NewTimer("idx.list_duration", 0)
	idxMatchLiteralDuration = stats.NewTimer("idx.match_literal_duration", 0)
	idxMatchPrefixDuration = stats.NewTimer("idx.match_prefix_duration", 0)
	idxMatchTrigramDuration = stats.NewTimer("idx.match_trigram_duration", 0)

	d := &DefCache{
		sync.RWMutex{},
		*idx.New(),
		defsStore,
	}
	go d.Prune()
	d.Backfill()
	d.defsStore.SetAsyncResultCallback(d.AsyncResultCallback)
	return d
}

func (dc *DefCache) Prune() {
	t := time.Tick(3 * time.Minute)
	for range t {
		// there's some fragments that occur in a whole lot of metrics
		// for example 'litmus'
		// this only retains the trigram postlists in the index if <20%
		// of the metrics contain them.  this keeps memory usage down
		// and makes queries faster
		pre := time.Now()
		dc.Lock()
		dc.Idx.Prune(0.20)
		dc.Unlock()
		idxPruneDuration.Value(time.Now().Sub(pre))
	}
}

// backfill definitions from ES
// in theory, there is a race between defs from ES and from nsq
// in practice, it doesn't matter: you're only supposed to query MT
// after a while, after which the defs surely have stabilized.
func (dc *DefCache) Backfill() {
	total := 0
	add := func(met []*schema.MetricDefinition) {
		if len(met) > 0 {
			total += len(met)
			dc.Lock()
			// def.Id must be unique. if we process the same Id twice, bad things will happen.
			for _, def := range met {
				dc.Idx.Add(*def) // gets id auto assigned from 0 and onwards
			}
			dc.Unlock()
		}
	}
	met, scroll_id, err := dc.defsStore.GetMetrics("")
	if err != nil {
		log.Error(3, "Could not backfill from ES: %s", err)
		return
	}
	add(met)
	for scroll_id != "" {
		met, scroll_id, err = dc.defsStore.GetMetrics(scroll_id)
		if err != nil {
			log.Error(3, "Could not backfill from ES: %s", err)
			return
		}
		add(met)
	}
	log.Debug("backfilled %d metric definitions", total)
}

// Adds the metric to the defcache.
// after this function returns, it is safe to modify the data pointed to
func (dc *DefCache) Add(metric *schema.MetricData) {
	dc.Lock()
	mdef := dc.GetById(idx.MetricID(metric.Id))
	if mdef != nil {
		//If the time diff between this datapoint and the lastUpdate
		// time of the metricDef is greater than 6hours, update the metricDef.
		if mdef.LastUpdate < metric.Time-21600 {
			mdef = schema.MetricDefinitionFromMetricData(metric)
			dc.Update(*mdef)
			dc.Unlock()
			dc.addToES(mdef)
		} else {
			dc.Unlock()
		}
	} else {
		dc.Unlock()
		mdef = schema.MetricDefinitionFromMetricData(metric)
		// now that we have the mdef, let's make sure we only add this once concurrently.
		// because addToES is pretty expensive and we should only call AddRef once.
		dc.Lock()
		def := dc.GetById(idx.MetricID(metric.Id))
		if def != nil {
			// someone beat us to it. nothing left to do
			dc.Unlock()
			return
		}
		dc.Idx.Add(*mdef)
		dc.Unlock()
		dc.addToES(mdef)
	}
}

func (dc *DefCache) addToES(mdef *schema.MetricDefinition) {
	pre := time.Now()
	err := dc.defsStore.IndexMetric(mdef)
	// NOTE: indexing to ES is done asyncrounously using the bulkAPI.
	// so an error here is just an error adding the document to the
	// bulkAPI buffer.
	if err != nil {
		log.Error(3, "couldn't index to ES %s: %s", mdef.Id, err)
		metricsToEsFail.Inc(1)
	} else {
		metricsToEsOK.Inc(1)
	}
	esPutDuration.Value(time.Now().Sub(pre))
}

// make defcache aware of asynchronous index calls of metric definitions to a defstore (ES) succeeding for a particular def or not.
// if ok, nothing to do
// if not ok, we pretend it was updated 5~5.5 hours ago, so that we'll retry in half an hour to an hour if/when a new one comes in
// so yes, there is a small chance we won't get to that if no new data comes in, which is something to address later.
func (dc *DefCache) AsyncResultCallback(id string, ok bool) {
	if ok {
		return
	}
	dc.Lock()
	mdef := dc.GetById(idx.MetricID(id))
	if mdef == nil {
		dc.Unlock()
		log.Error(3, "got async callback with ES result %t for %q, however it does not exist in the internal index", ok, id)
		return
	}
	// normally we have to do dc.Update(*mdef) but in this case we can just modify the data pointed to, e.g. in the index.
	// we pretend the mdef was last updated some random time between 5h ago and 5h30min ago
	mdef.LastUpdate = time.Now().Unix() - 5*3600 - rand.Intn(30*60)
	dc.Unlock()
}

// Get gets a metricdef by metric id
// note: the defcache is clearly not a perfect all-knowning entity, it just knows the last interval of metrics seen since program start
// and we assume we can use that interval through history.
// TODO: no support for interval changes, ...
// note: do *not* modify the pointed-to data, as it will affect the data in the index!
func (dc *DefCache) Get(id string) *schema.MetricDefinition {
	pre := time.Now()
	dc.RLock()
	def := dc.GetById(idx.MetricID(id))
	dc.RUnlock()
	idxGetDuration.Value(time.Now().Sub(pre))
	return def
}

// Find returns the results of a pattern match.
// note: do *not* modify the pointed-to data, as it will affect the data in the index!
func (dc *DefCache) Find(org int, query string) ([]idx.Glob, []*schema.MetricDefinition) {
	pre := time.Now()
	dc.RLock()
	mt, globs, defs := dc.Match(org, query)
	dc.RUnlock()
	switch mt {
	case idx.MatchLiteral:
		idxMatchLiteralDuration.Value(time.Now().Sub(pre))
	case idx.MatchPrefix:
		idxMatchPrefixDuration.Value(time.Now().Sub(pre))
	case idx.MatchTrigram:
		idxMatchTrigramDuration.Value(time.Now().Sub(pre))
	}
	return globs, defs
}

// List provides all metricdefs based on the provided org. if it is:
// -1, then all metricdefs for all orgs are returned
// any other org, then all defs for that org, as well as -1 are returned
// note: do *not* modify the pointed-to data, as it will affect the data in the index!
func (dc *DefCache) List(org int) []*schema.MetricDefinition {
	pre := time.Now()
	dc.RLock()
	defs := dc.Idx.List(org)
	dc.RUnlock()
	idxListDuration.Value(time.Now().Sub(pre))
	return defs
}
<commit_msg>my bad<commit_after>// package defcache wraps our index with performance metrics, a mutex for threadsafe access
// and synchronisation with a definition store
package defcache

import (
	"math/rand"
	"sync"
	"time"

	"github.com/grafana/grafana/pkg/log"
	"github.com/raintank/met"
	"github.com/raintank/raintank-metric/metric_tank/idx"
	"github.com/raintank/raintank-metric/metricdef"
	"github.com/raintank/raintank-metric/schema"
)

var (
	metricsToEsOK           met.Count
	metricsToEsFail         met.Count
	esPutDuration           met.Timer // note that due to our use of bulk indexer, most values will be very fast with the occasional "outlier" which triggers a flush
	idxPruneDuration        met.Timer
	idxGetDuration          met.Timer
	idxListDuration         met.Timer
	idxMatchLiteralDuration met.Timer
	idxMatchPrefixDuration  met.Timer
	idxMatchTrigramDuration met.Timer
)

// design notes:
// MT pulls in all definitions when it starts up.
// those "old" ones + whatever it sees as inputs from the metrics queue
// is enough for it to always know the complete current state
// nothing should update ES "behind its back", so we never need to pull
// from ES other then at startup.
// but other MT instances may update ES while we are down, so ES is a good
// place to pull from, until the performance is demonstrably too slow.
// there are some vectors here for race conditions but we can work those out
// later, perhaps when tacking the multiple-intervals work

type DefCache struct {
	sync.RWMutex
	idx.Idx
	defsStore metricdef.Defs
}

func New(defsStore metricdef.Defs, stats met.Backend) *DefCache {
	metricsToEsOK = stats.NewCount("metrics_to_es.ok")
	metricsToEsFail = stats.NewCount("metrics_to_es.fail")
	esPutDuration = stats.NewTimer("es_put_duration", 0)
	idxPruneDuration = stats.NewTimer("idx.prune_duration", 0)
	idxGetDuration = stats.NewTimer("idx.get_duration", 0)
	idxListDuration = stats.NewTimer("idx.list_duration", 0)
	idxMatchLiteralDuration = stats.NewTimer("idx.match_literal_duration", 0)
	idxMatchPrefixDuration = stats.NewTimer("idx.match_prefix_duration", 0)
	idxMatchTrigramDuration = stats.NewTimer("idx.match_trigram_duration", 0)

	d := &DefCache{
		sync.RWMutex{},
		*idx.New(),
		defsStore,
	}
	go d.Prune()
	d.Backfill()
	d.defsStore.SetAsyncResultCallback(d.AsyncResultCallback)
	return d
}

func (dc *DefCache) Prune() {
	t := time.Tick(3 * time.Minute)
	for range t {
		// there's some fragments that occur in a whole lot of metrics
		// for example 'litmus'
		// this only retains the trigram postlists in the index if <20%
		// of the metrics contain them.  this keeps memory usage down
		// and makes queries faster
		pre := time.Now()
		dc.Lock()
		dc.Idx.Prune(0.20)
		dc.Unlock()
		idxPruneDuration.Value(time.Now().Sub(pre))
	}
}

// backfill definitions from ES
// in theory, there is a race between defs from ES and from nsq
// in practice, it doesn't matter: you're only supposed to query MT
// after a while, after which the defs surely have stabilized.
func (dc *DefCache) Backfill() {
	total := 0
	add := func(met []*schema.MetricDefinition) {
		if len(met) > 0 {
			total += len(met)
			dc.Lock()
			// def.Id must be unique. if we process the same Id twice, bad things will happen.
			for _, def := range met {
				dc.Idx.Add(*def) // gets id auto assigned from 0 and onwards
			}
			dc.Unlock()
		}
	}
	met, scroll_id, err := dc.defsStore.GetMetrics("")
	if err != nil {
		log.Error(3, "Could not backfill from ES: %s", err)
		return
	}
	add(met)
	for scroll_id != "" {
		met, scroll_id, err = dc.defsStore.GetMetrics(scroll_id)
		if err != nil {
			log.Error(3, "Could not backfill from ES: %s", err)
			return
		}
		add(met)
	}
	log.Debug("backfilled %d metric definitions", total)
}

// Adds the metric to the defcache.
// after this function returns, it is safe to modify the data pointed to
func (dc *DefCache) Add(metric *schema.MetricData) {
	dc.Lock()
	mdef := dc.GetById(idx.MetricID(metric.Id))
	if mdef != nil {
		//If the time diff between this datapoint and the lastUpdate
		// time of the metricDef is greater than 6hours, update the metricDef.
		if mdef.LastUpdate < metric.Time-21600 {
			mdef = schema.MetricDefinitionFromMetricData(metric)
			dc.Update(*mdef)
			dc.Unlock()
			dc.addToES(mdef)
		} else {
			dc.Unlock()
		}
	} else {
		dc.Unlock()
		mdef = schema.MetricDefinitionFromMetricData(metric)
		// now that we have the mdef, let's make sure we only add this once concurrently.
		// because addToES is pretty expensive and we should only call AddRef once.
		dc.Lock()
		def := dc.GetById(idx.MetricID(metric.Id))
		if def != nil {
			// someone beat us to it. nothing left to do
			dc.Unlock()
			return
		}
		dc.Idx.Add(*mdef)
		dc.Unlock()
		dc.addToES(mdef)
	}
}

func (dc *DefCache) addToES(mdef *schema.MetricDefinition) {
	pre := time.Now()
	err := dc.defsStore.IndexMetric(mdef)
	// NOTE: indexing to ES is done asyncrounously using the bulkAPI.
	// so an error here is just an error adding the document to the
	// bulkAPI buffer.
	if err != nil {
		log.Error(3, "couldn't index to ES %s: %s", mdef.Id, err)
		metricsToEsFail.Inc(1)
	} else {
		metricsToEsOK.Inc(1)
	}
	esPutDuration.Value(time.Now().Sub(pre))
}

// make defcache aware of asynchronous index calls of metric definitions to a defstore (ES) succeeding for a particular def or not.
// if ok, nothing to do
// if not ok, we pretend it was updated 5~5.5 hours ago, so that we'll retry in half an hour to an hour if/when a new one comes in
// so yes, there is a small chance we won't get to that if no new data comes in, which is something to address later.
func (dc *DefCache) AsyncResultCallback(id string, ok bool) {
	if ok {
		return
	}
	dc.Lock()
	mdef := dc.GetById(idx.MetricID(id))
	if mdef == nil {
		dc.Unlock()
		log.Error(3, "got async callback with ES result %t for %q, however it does not exist in the internal index", ok, id)
		return
	}
	// normally we have to do dc.Update(*mdef) but in this case we can just modify the data pointed to, e.g. in the index.
	// we pretend the mdef was last updated some random time between 5h ago and 5h30min ago
	mdef.LastUpdate = time.Now().Unix() - 5*3600 - rand.Intn(30*60)
	dc.Unlock()
}

// Get gets a metricdef by metric id
// note: the defcache is clearly not a perfect all-knowning entity, it just knows the last interval of metrics seen since program start
// and we assume we can use that interval through history.
// TODO: no support for interval changes, ...
// note: do *not* modify the pointed-to data, as it will affect the data in the index!
func (dc *DefCache) Get(id string) *schema.MetricDefinition {
	pre := time.Now()
	dc.RLock()
	def := dc.GetById(idx.MetricID(id))
	dc.RUnlock()
	idxGetDuration.Value(time.Now().Sub(pre))
	return def
}

// Find returns the results of a pattern match.
// note: do *not* modify the pointed-to data, as it will affect the data in the index!
func (dc *DefCache) Find(org int, query string) ([]idx.Glob, []*schema.MetricDefinition) {
	pre := time.Now()
	dc.RLock()
	mt, globs, defs := dc.Match(org, query)
	dc.RUnlock()
	switch mt {
	case idx.MatchLiteral:
		idxMatchLiteralDuration.Value(time.Now().Sub(pre))
	case idx.MatchPrefix:
		idxMatchPrefixDuration.Value(time.Now().Sub(pre))
	case idx.MatchTrigram:
		idxMatchTrigramDuration.Value(time.Now().Sub(pre))
	}
	return globs, defs
}

// List provides all metricdefs based on the provided org. if it is:
// -1, then all metricdefs for all orgs are returned
// any other org, then all defs for that org, as well as -1 are returned
// note: do *not* modify the pointed-to data, as it will affect the data in the index!
func (dc *DefCache) List(org int) []*schema.MetricDefinition {
	pre := time.Now()
	dc.RLock()
	defs := dc.Idx.List(org)
	dc.RUnlock()
	idxListDuration.Value(time.Now().Sub(pre))
	return defs
}
<|endoftext|>


<commit_before>// A email service implementation that just logs the mails.
package log

import (
	log "github.com/Sirupsen/logrus"

	kit "github.com/theduke/go-appkit"
)

type Service struct {
	debug         bool
	deps          kit.Dependencies
	defaultSender kit.EmailRecipient
}

// Ensure Service implements email.Service.
var _ kit.EmailService = (*Service)(nil)

func New(deps kit.Dependencies, defaultSender kit.EmailRecipient) *Service {
	return &Service{
		deps:          deps,
		defaultSender: defaultSender,
	}
}

func (s *Service) Debug() bool {
	return s.debug
}

func (s *Service) SetDebug(x bool) {
	s.debug = x
}

func (s *Service) Dependencies() kit.Dependencies {
	return s.deps
}

func (s *Service) SetDependencies(x kit.Dependencies) {
	s.deps = x
}

func (s *Service) SetDefaultFrom(r kit.EmailRecipient) {
	s.defaultSender = r
}

func (s Service) Send(e kit.Email) kit.Error {
	err, errs := s.SendMultiple(e)
	if err != nil {
		return err
	}
	return errs[0]
}

func (s Service) SendMultiple(emails ...kit.Email) (kit.Error, []kit.Error) {
	for _, e := range emails {
		from := e.GetFrom().GetEmail()
		if from == "" {
			from = s.defaultSender.GetEmail()
		}

		var recipients []string

		for _, recp := range e.GetTo() {
			recipients = append(recipients, recp.GetEmail())
		}
		for _, recp := range e.GetCc() {
			recipients = append(recipients, recp.GetEmail())
		}
		for _, recp := range e.GetBcc() {
			recipients = append(recipients, recp.GetEmail())
		}

		s.deps.Logger().WithFields(log.Fields{
			"action":  "send_email",
			"from":    from,
			"to":      recipients,
			"subject": e.GetSubject(),
		}).Debugf("Sending mail from %v to %v - subject %v", from, recipients, e.GetSubject())
	}

	return nil, make([]kit.Error, len(emails))
}
<commit_msg>email log backend now logs full message<commit_after>// A email service implementation that just logs the mails.
package log

import (
	"fmt"

	log "github.com/Sirupsen/logrus"

	kit "github.com/theduke/go-appkit"
)

type Service struct {
	debug         bool
	deps          kit.Dependencies
	defaultSender kit.EmailRecipient
}

// Ensure Service implements email.Service.
var _ kit.EmailService = (*Service)(nil)

func New(deps kit.Dependencies, defaultSender kit.EmailRecipient) *Service {
	return &Service{
		deps:          deps,
		defaultSender: defaultSender,
	}
}

func (s *Service) Debug() bool {
	return s.debug
}

func (s *Service) SetDebug(x bool) {
	s.debug = x
}

func (s *Service) Dependencies() kit.Dependencies {
	return s.deps
}

func (s *Service) SetDependencies(x kit.Dependencies) {
	s.deps = x
}

func (s *Service) SetDefaultFrom(r kit.EmailRecipient) {
	s.defaultSender = r
}

func (s Service) Send(e kit.Email) kit.Error {
	err, errs := s.SendMultiple(e)
	if err != nil {
		return err
	}
	return errs[0]
}

func (s Service) SendMultiple(emails ...kit.Email) (kit.Error, []kit.Error) {
	for _, e := range emails {
		from := e.GetFrom().GetEmail()
		if from == "" {
			from = s.defaultSender.GetEmail()
		}

		var recipients []string

		for _, recp := range e.GetTo() {
			recipients = append(recipients, recp.GetEmail())
		}
		for _, recp := range e.GetCc() {
			recipients = append(recipients, recp.GetEmail())
		}
		for _, recp := range e.GetBcc() {
			recipients = append(recipients, recp.GetEmail())
		}

		msg := fmt.Sprintf("Sending mail from %v to %v - subject %v", from, recipients, e.GetSubject())
		for _, part := range e.GetBodyParts() {
			msg += "\n####################\n"
			msg += string(part.GetContent())
		}
		msg += "\n####################\n\n"

		s.deps.Logger().WithFields(log.Fields{
			"action":  "send_email",
			"from":    from,
			"to":      recipients,
			"subject": e.GetSubject(),
		}).Debug(msg)
	}

	return nil, make([]kit.Error, len(emails))
}
<|endoftext|>


<commit_before>package libkbfs

import (
	"sync"

	"golang.org/x/net/context"
)

// favReq represents a request to access the logged-in user's
// favorites list.  A single request can do one or more of the
// following: refresh the current cached list, add a favorite, remove
// a favorite, and get all the favorites.  When the request is done,
// the resulting error (or nil) is sent over the done channel.  The
// given ctx is used for all network operations.
type favReq struct {
	// Request types
	refresh bool
	toAdd   []Favorite
	toDel   []Favorite
	favs    chan<- []Favorite

	// Closed when the request is done.
	done chan struct{}
	// Set before done is closed
	err error

	// Context
	ctx context.Context
}

// Favorites manages a user's favorite list.
type Favorites struct {
	config Config

	// Channels for interacting with the favorites cache
	reqChan chan *favReq

	// cache tracks the favorites for this user, that we know about.
	// It may not be consistent with the server's view of the user's
	// favorites list, if other devices have modified the list since
	// the last refresh.
	cache map[Favorite]bool

	inFlightLock sync.Mutex
	inFlightAdds map[Favorite]*favReq
}

func newFavoritesWithChan(config Config, reqChan chan *favReq) *Favorites {
	f := &Favorites{
		config:       config,
		reqChan:      reqChan,
		inFlightAdds: make(map[Favorite]*favReq),
	}
	go f.loop()
	return f
}

// NewFavorites constructs a new Favorites instance.
func NewFavorites(config Config) *Favorites {
	return newFavoritesWithChan(config, make(chan *favReq, 100))
}

func (f *Favorites) handleReq(req *favReq) (err error) {
	defer func() {
		f.inFlightLock.Lock()
		defer f.inFlightLock.Unlock()
		req.err = err
		close(req.done)
		for _, fav := range req.toAdd {
			delete(f.inFlightAdds, fav)
		}
	}()

	kbpki := f.config.KBPKI()
	// Fetch a new list if:
	//  * The user asked us to refresh
	//  * We haven't fetched it before
	//  * The user wants the list of favorites.  TODO: use the cached list
	//    once we have proper invalidation from the server.
	if req.refresh || f.cache == nil || req.favs != nil {
		folders, err := kbpki.FavoriteList(req.ctx)
		if err != nil {
			return err
		}

		f.cache = make(map[Favorite]bool)
		for _, folder := range folders {
			f.cache[*NewFavoriteFromFolder(folder)] = true
		}
		username, _, err := f.config.KBPKI().GetCurrentUserInfo(req.ctx)
		if err == nil {
			// Add favorites for the current user, that cannot be deleted.
			f.cache[Favorite{string(username), true}] = true
			f.cache[Favorite{string(username), false}] = true
		}
	}

	for _, fav := range req.toAdd {
		if f.cache[fav] {
			continue
		}
		err := kbpki.FavoriteAdd(req.ctx, fav.toKBFolder())
		if err != nil {
			f.config.MakeLogger("").CDebugf(req.ctx,
				"Failure adding favorite %v: %v", fav, err)
			return err
		}
		f.cache[fav] = true
	}

	for _, fav := range req.toDel {
		// Since our cache isn't necessarily up-to-date, always delete
		// the favorite.
		folder := fav.toKBFolder()
		err := kbpki.FavoriteDelete(req.ctx, folder)
		if err != nil {
			return err
		}
		if !folder.Private {
			// Public folders may be stored under a different name,
			// pending CORE-2695.  TODO: remove me!
			folder.Name = folder.Name + ReaderSep + "public"
			err := kbpki.FavoriteDelete(req.ctx, folder)
			if err != nil {
				return err
			}
		}
		delete(f.cache, fav)
	}

	if req.favs != nil {
		favorites := make([]Favorite, 0, len(f.cache))
		for fav := range f.cache {
			favorites = append(favorites, fav)
		}
		req.favs <- favorites
	}

	return nil
}

func (f *Favorites) loop() {
	for req := range f.reqChan {
		f.handleReq(req)
	}
}

// Shutdown shuts down this Favorites instance.
func (f *Favorites) Shutdown() {
	close(f.reqChan)
}

func (f *Favorites) waitOnReq(ctx context.Context,
	req *favReq) (retry bool, err error) {
	select {
	case <-ctx.Done():
		return false, ctx.Err()
	case <-req.done:
		err = req.err
		// If the request was canceled due to a context timeout that
		// wasn't our own, try it again.
		if err == context.Canceled || err == context.DeadlineExceeded {
			select {
			case <-ctx.Done():
				return false, err
			default:
				return true, nil
			}
		}
		return false, err
	}
}

func (f *Favorites) sendReq(ctx context.Context, req *favReq) error {
	select {
	case f.reqChan <- req:
	case <-ctx.Done():
		return ctx.Err()
	}
	// With a direct sendReq call, we'll never have a shared request,
	// so no need to check the retry status.
	_, err := f.waitOnReq(ctx, req)
	return err
}

func (f *Favorites) startOrJoinAddReq(
	ctx context.Context, fav Favorite) (req *favReq, doSend bool) {
	f.inFlightLock.Lock()
	defer f.inFlightLock.Unlock()
	req, ok := f.inFlightAdds[fav]
	if !ok {
		req = &favReq{
			ctx:   ctx,
			toAdd: []Favorite{fav},
			done:  make(chan struct{}),
		}
		f.inFlightAdds[fav] = req
		doSend = true
	}
	return req, doSend
}

// Add adds a favorite to your favorites list.
func (f *Favorites) Add(ctx context.Context, fav Favorite) error {
	doAdd := true
	var err error
	// Retry until we get an error that wasn't related to someone
	// else's context being canceled.
	for doAdd {
		req, doSend := f.startOrJoinAddReq(ctx, fav)
		if doSend {
			return f.sendReq(ctx, req)
		}
		doAdd, err = f.waitOnReq(ctx, req)
	}
	return err
}

// AddAsync initiates a request to add this favorite to your favorites
// list, if one is not already in flight, but it doesn't wait for the
// result.  (It could block while kicking off the request, if lots of
// different favorite operations are in flight.)
func (f *Favorites) AddAsync(ctx context.Context, fav Favorite) {
	req, doSend := f.startOrJoinAddReq(ctx, fav)
	if doSend {
		select {
		case f.reqChan <- req:
		case <-ctx.Done():
			return
		}
	}
}

// Delete deletes a favorite from the favorites list.  It is
// idempotent.
func (f *Favorites) Delete(ctx context.Context, fav Favorite) error {
	return f.sendReq(ctx, &favReq{
		ctx:   ctx,
		toDel: []Favorite{fav},
		done:  make(chan struct{}),
	})
}

// RefreshCache refreshes the cached list of favorites.
func (f *Favorites) RefreshCache(ctx context.Context) {
	// This request is non-blocking, so use a throw-away done channel
	// and context.
	req := &favReq{
		refresh: true,
		done:    make(chan struct{}),
		ctx:     context.Background(),
	}
	select {
	case f.reqChan <- req:
	case <-ctx.Done():
		return
	}
}

// Get returns the logged-in users list of favorites. It
// doesn't use the cache.
func (f *Favorites) Get(ctx context.Context) ([]Favorite, error) {
	favChan := make(chan []Favorite, 1)
	req := &favReq{
		ctx:  ctx,
		favs: favChan,
		done: make(chan struct{}),
	}
	err := f.sendReq(ctx, req)
	if err != nil {
		return nil, err
	}
	return <-favChan, nil
}
<commit_msg>favorites: don't rely on given context when adding asynchronously<commit_after>package libkbfs

import (
	"sync"

	"golang.org/x/net/context"
)

// favReq represents a request to access the logged-in user's
// favorites list.  A single request can do one or more of the
// following: refresh the current cached list, add a favorite, remove
// a favorite, and get all the favorites.  When the request is done,
// the resulting error (or nil) is sent over the done channel.  The
// given ctx is used for all network operations.
type favReq struct {
	// Request types
	refresh bool
	toAdd   []Favorite
	toDel   []Favorite
	favs    chan<- []Favorite

	// Closed when the request is done.
	done chan struct{}
	// Set before done is closed
	err error

	// Context
	ctx context.Context
}

// Favorites manages a user's favorite list.
type Favorites struct {
	config Config

	// Channels for interacting with the favorites cache
	reqChan chan *favReq

	// cache tracks the favorites for this user, that we know about.
	// It may not be consistent with the server's view of the user's
	// favorites list, if other devices have modified the list since
	// the last refresh.
	cache map[Favorite]bool

	inFlightLock sync.Mutex
	inFlightAdds map[Favorite]*favReq
}

func newFavoritesWithChan(config Config, reqChan chan *favReq) *Favorites {
	f := &Favorites{
		config:       config,
		reqChan:      reqChan,
		inFlightAdds: make(map[Favorite]*favReq),
	}
	go f.loop()
	return f
}

// NewFavorites constructs a new Favorites instance.
func NewFavorites(config Config) *Favorites {
	return newFavoritesWithChan(config, make(chan *favReq, 100))
}

func (f *Favorites) handleReq(req *favReq) (err error) {
	defer func() {
		f.inFlightLock.Lock()
		defer f.inFlightLock.Unlock()
		req.err = err
		close(req.done)
		for _, fav := range req.toAdd {
			delete(f.inFlightAdds, fav)
		}
	}()

	kbpki := f.config.KBPKI()
	// Fetch a new list if:
	//  * The user asked us to refresh
	//  * We haven't fetched it before
	//  * The user wants the list of favorites.  TODO: use the cached list
	//    once we have proper invalidation from the server.
	if req.refresh || f.cache == nil || req.favs != nil {
		folders, err := kbpki.FavoriteList(req.ctx)
		if err != nil {
			return err
		}

		f.cache = make(map[Favorite]bool)
		for _, folder := range folders {
			f.cache[*NewFavoriteFromFolder(folder)] = true
		}
		username, _, err := f.config.KBPKI().GetCurrentUserInfo(req.ctx)
		if err == nil {
			// Add favorites for the current user, that cannot be deleted.
			f.cache[Favorite{string(username), true}] = true
			f.cache[Favorite{string(username), false}] = true
		}
	}

	for _, fav := range req.toAdd {
		if f.cache[fav] {
			continue
		}
		err := kbpki.FavoriteAdd(req.ctx, fav.toKBFolder())
		if err != nil {
			f.config.MakeLogger("").CDebugf(req.ctx,
				"Failure adding favorite %v: %v", fav, err)
			return err
		}
		f.cache[fav] = true
	}

	for _, fav := range req.toDel {
		// Since our cache isn't necessarily up-to-date, always delete
		// the favorite.
		folder := fav.toKBFolder()
		err := kbpki.FavoriteDelete(req.ctx, folder)
		if err != nil {
			return err
		}
		if !folder.Private {
			// Public folders may be stored under a different name,
			// pending CORE-2695.  TODO: remove me!
			folder.Name = folder.Name + ReaderSep + "public"
			err := kbpki.FavoriteDelete(req.ctx, folder)
			if err != nil {
				return err
			}
		}
		delete(f.cache, fav)
	}

	if req.favs != nil {
		favorites := make([]Favorite, 0, len(f.cache))
		for fav := range f.cache {
			favorites = append(favorites, fav)
		}
		req.favs <- favorites
	}

	return nil
}

func (f *Favorites) loop() {
	for req := range f.reqChan {
		f.handleReq(req)
	}
}

// Shutdown shuts down this Favorites instance.
func (f *Favorites) Shutdown() {
	close(f.reqChan)
}

func (f *Favorites) waitOnReq(ctx context.Context,
	req *favReq) (retry bool, err error) {
	select {
	case <-ctx.Done():
		return false, ctx.Err()
	case <-req.done:
		err = req.err
		// If the request was canceled due to a context timeout that
		// wasn't our own, try it again.
		if err == context.Canceled || err == context.DeadlineExceeded {
			select {
			case <-ctx.Done():
				return false, err
			default:
				return true, nil
			}
		}
		return false, err
	}
}

func (f *Favorites) sendReq(ctx context.Context, req *favReq) error {
	select {
	case f.reqChan <- req:
	case <-ctx.Done():
		return ctx.Err()
	}
	// With a direct sendReq call, we'll never have a shared request,
	// so no need to check the retry status.
	_, err := f.waitOnReq(ctx, req)
	return err
}

func (f *Favorites) startOrJoinAddReq(
	ctx context.Context, fav Favorite) (req *favReq, doSend bool) {
	f.inFlightLock.Lock()
	defer f.inFlightLock.Unlock()
	req, ok := f.inFlightAdds[fav]
	if !ok {
		req = &favReq{
			ctx:   ctx,
			toAdd: []Favorite{fav},
			done:  make(chan struct{}),
		}
		f.inFlightAdds[fav] = req
		doSend = true
	}
	return req, doSend
}

// Add adds a favorite to your favorites list.
func (f *Favorites) Add(ctx context.Context, fav Favorite) error {
	doAdd := true
	var err error
	// Retry until we get an error that wasn't related to someone
	// else's context being canceled.
	for doAdd {
		req, doSend := f.startOrJoinAddReq(ctx, fav)
		if doSend {
			return f.sendReq(ctx, req)
		}
		doAdd, err = f.waitOnReq(ctx, req)
	}
	return err
}

// AddAsync initiates a request to add this favorite to your favorites
// list, if one is not already in flight, but it doesn't wait for the
// result.  (It could block while kicking off the request, if lots of
// different favorite operations are in flight.)
func (f *Favorites) AddAsync(ctx context.Context, fav Favorite) {
	// Use a fresh context, since we want the request to succeed even
	// if the original context is canceled.
	req, doSend := f.startOrJoinAddReq(context.Background(), fav)
	if doSend {
		select {
		case f.reqChan <- req:
		case <-ctx.Done():
			return
		}
	}
}

// Delete deletes a favorite from the favorites list.  It is
// idempotent.
func (f *Favorites) Delete(ctx context.Context, fav Favorite) error {
	return f.sendReq(ctx, &favReq{
		ctx:   ctx,
		toDel: []Favorite{fav},
		done:  make(chan struct{}),
	})
}

// RefreshCache refreshes the cached list of favorites.
func (f *Favorites) RefreshCache(ctx context.Context) {
	// This request is non-blocking, so use a throw-away done channel
	// and context.
	req := &favReq{
		refresh: true,
		done:    make(chan struct{}),
		ctx:     context.Background(),
	}
	select {
	case f.reqChan <- req:
	case <-ctx.Done():
		return
	}
}

// Get returns the logged-in users list of favorites. It
// doesn't use the cache.
func (f *Favorites) Get(ctx context.Context) ([]Favorite, error) {
	favChan := make(chan []Favorite, 1)
	req := &favReq{
		ctx:  ctx,
		favs: favChan,
		done: make(chan struct{}),
	}
	err := f.sendReq(ctx, req)
	if err != nil {
		return nil, err
	}
	return <-favChan, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"io"
	"os"
	"testing"
	"time"

	"v.io/v23"
	"v.io/v23/context"
	"v.io/v23/options"

	mtlib "v.io/x/ref/services/mounttable/lib"
	"v.io/x/ref/test"
	"v.io/x/ref/test/modules"
)

//go:generate v23 test generate

func rootMT(stdin io.Reader, stdout, stderr io.Writer, env map[string]string, args ...string) error {
	ctx, shutdown := v23.Init()
	defer shutdown()

	lspec := v23.GetListenSpec(ctx)
	server, err := v23.NewServer(ctx, options.ServesMountTable(true))
	if err != nil {
		return fmt.Errorf("root failed: %v", err)
	}
	mt, err := mtlib.NewMountTableDispatcher("")
	if err != nil {
		return fmt.Errorf("mounttable.NewMountTableDispatcher failed: %s", err)
	}
	eps, err := server.Listen(lspec)
	if err != nil {
		return fmt.Errorf("server.Listen failed: %s", err)
	}
	if err := server.ServeDispatcher("", mt); err != nil {
		return fmt.Errorf("root failed: %s", err)
	}
	fmt.Fprintf(stdout, "PID=%d\n", os.Getpid())
	for _, ep := range eps {
		fmt.Fprintf(stdout, "MT_NAME=%s\n", ep.Name())
	}
	modules.WaitForEOF(stdin)
	return nil
}

// Starts a mounttable.  Returns the name and a stop function.
func startMountTable(t *testing.T, ctx *context.T) (string, func()) {
	sh, err := modules.NewShell(ctx, nil, testing.Verbose(), t)
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}

	rootMT, err := sh.Start("rootMT", nil, "--v23.tcp.address=127.0.0.1:0")
	if err != nil {
		t.Fatalf("failed to start root mount table: %s", err)
	}
	rootMT.ExpectVar("PID")
	rootName := rootMT.ExpectVar("MT_NAME")

	return rootName, func() {
		if err := sh.Cleanup(nil, nil); err != nil {
			t.Fatalf("failed to cleanup shell: %s", rootMT.Error())
		}
	}
}

// Asserts that the channel contains members with expected names and no others.
func AssertMembersWithNames(channel *channel, expectedNames []string) error {
	members, err := channel.getMembers()
	if err != nil {
		return fmt.Errorf("channel.getMembers() failed: %v", err)
	}
	if actualLen, expectedLen := len(members), len(expectedNames); actualLen != expectedLen {
		return fmt.Errorf("Wrong number of members.  Expected %v, actual %v.", expectedLen, actualLen)
	}
	for _, expectedName := range expectedNames {
		found := false
		for _, member := range members {
			if member.Name == expectedName {
				found = true
				break
			}
		}
		if !found {
			return fmt.Errorf("Expected member with name %v, but did not find one.", expectedName)
		}
	}
	return nil
}

func TestMembers(t *testing.T) {
	ctx, shutdown := test.InitForTest()
	defer shutdown()

	mounttable, stopMountTable := startMountTable(t, ctx)
	defer stopMountTable()

	proxy := ""

	path := "path/to/channel"

	// Create a new channel.
	channel, err := newChannel(ctx, mounttable, proxy, path)
	if err != nil {
		t.Fatal("newChannel(%v, %v, %v) failed: %v", ctx, mounttable, proxy, path)
	}

	// New channel should be empty.
	if err := AssertMembersWithNames(channel, []string{}); err != nil {
		t.Error(err)
	}

	// Join the channel.
	if err := channel.join(); err != nil {
		t.Fatalf("channel.join() failed: %v", err)
	}

	// Channel should contain only current user.
	if err := AssertMembersWithNames(channel, []string{channel.UserName()}); err != nil {
		t.Error(err)
	}

	// Create and join the channel a second time.
	channel2, err := newChannel(ctx, mounttable, proxy, path)
	if err != nil {
		t.Fatal("newChannel(%v, %v, %v) failed: %v", ctx, mounttable, proxy, path)
	}
	if err := channel2.join(); err != nil {
		t.Fatalf("channel2.join() failed: %v", err)
	}

	// Channel should contain both users.
	if err := AssertMembersWithNames(channel, []string{channel.UserName(), channel2.UserName()}); err != nil {
		t.Error(err)
	}

	// Leave first instance of channel.
	if err := channel.leave(); err != nil {
		t.Fatalf("channel.leave() failed: %v", err)
	}

	// Channel should contain only second user.
	if err := AssertMembersWithNames(channel, []string{channel2.UserName()}); err != nil {
		t.Error(err)
	}

	// Leave second instance of channel.
	if err := channel2.leave(); err != nil {
		t.Fatalf("channel2.leave() failed: %v", err)
	}

	// Channel should be empty.
	if err := AssertMembersWithNames(channel, []string{}); err != nil {
		t.Error(err)
	}
}

func TestBroadcastMessage(t *testing.T) {
	ctx, shutdown := test.InitForTest()
	defer shutdown()

	mounttable, stopMountTable := startMountTable(t, ctx)
	defer stopMountTable()

	proxy := ""

	path := "path/to/channel"

	channel, err := newChannel(ctx, mounttable, proxy, path)
	if err != nil {
		t.Fatalf("newChannel(%v, %v, %v) failed: %v", ctx, mounttable, proxy, path, err)
	}

	defer channel.leave()

	if err := channel.join(); err != nil {
		t.Fatalf("channel.join() failed: %v", err)
	}

	message := "Hello Vanadium world!"

	go func() {
		// Call getMembers(), which will set channel.members, used by
		// channel.broadcastMessage().
		if _, err := channel.getMembers(); err != nil {
			t.Fatalf("channel.getMembers() failed: %v", err)
		}
		if err := channel.broadcastMessage(message); err != nil {
			t.Fatalf("channel.broadcastMessage(%v) failed: %v", message, err)
		}
	}()

	select {
	case <-time.After(1 * time.Second):
		t.Errorf("Timeout waiting for message to be received.")
	case m := <-channel.messages:
		if m.Text != message {
			t.Errorf("Expected message text to be %v but got %v", message, m.Text)
		}
		if got, want := m.SenderName, channel.UserName(); got != want {
			t.Errorf("Got m.SenderName = %v, want %v", got, want)
		}
	}
}
<commit_msg>chat: Move services/mounttable/lib to mounttablelib<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"io"
	"os"
	"testing"
	"time"

	"v.io/v23"
	"v.io/v23/context"
	"v.io/v23/options"

	"v.io/x/ref/services/mounttable/mounttablelib"
	"v.io/x/ref/test"
	"v.io/x/ref/test/modules"
)

//go:generate v23 test generate

func rootMT(stdin io.Reader, stdout, stderr io.Writer, env map[string]string, args ...string) error {
	ctx, shutdown := v23.Init()
	defer shutdown()

	lspec := v23.GetListenSpec(ctx)
	server, err := v23.NewServer(ctx, options.ServesMountTable(true))
	if err != nil {
		return fmt.Errorf("root failed: %v", err)
	}
	mt, err := mounttablelib.NewMountTableDispatcher("")
	if err != nil {
		return fmt.Errorf("mounttable.NewMountTableDispatcher failed: %s", err)
	}
	eps, err := server.Listen(lspec)
	if err != nil {
		return fmt.Errorf("server.Listen failed: %s", err)
	}
	if err := server.ServeDispatcher("", mt); err != nil {
		return fmt.Errorf("root failed: %s", err)
	}
	fmt.Fprintf(stdout, "PID=%d\n", os.Getpid())
	for _, ep := range eps {
		fmt.Fprintf(stdout, "MT_NAME=%s\n", ep.Name())
	}
	modules.WaitForEOF(stdin)
	return nil
}

// Starts a mounttable.  Returns the name and a stop function.
func startMountTable(t *testing.T, ctx *context.T) (string, func()) {
	sh, err := modules.NewShell(ctx, nil, testing.Verbose(), t)
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}

	rootMT, err := sh.Start("rootMT", nil, "--v23.tcp.address=127.0.0.1:0")
	if err != nil {
		t.Fatalf("failed to start root mount table: %s", err)
	}
	rootMT.ExpectVar("PID")
	rootName := rootMT.ExpectVar("MT_NAME")

	return rootName, func() {
		if err := sh.Cleanup(nil, nil); err != nil {
			t.Fatalf("failed to cleanup shell: %s", rootMT.Error())
		}
	}
}

// Asserts that the channel contains members with expected names and no others.
func AssertMembersWithNames(channel *channel, expectedNames []string) error {
	members, err := channel.getMembers()
	if err != nil {
		return fmt.Errorf("channel.getMembers() failed: %v", err)
	}
	if actualLen, expectedLen := len(members), len(expectedNames); actualLen != expectedLen {
		return fmt.Errorf("Wrong number of members.  Expected %v, actual %v.", expectedLen, actualLen)
	}
	for _, expectedName := range expectedNames {
		found := false
		for _, member := range members {
			if member.Name == expectedName {
				found = true
				break
			}
		}
		if !found {
			return fmt.Errorf("Expected member with name %v, but did not find one.", expectedName)
		}
	}
	return nil
}

func TestMembers(t *testing.T) {
	ctx, shutdown := test.InitForTest()
	defer shutdown()

	mounttable, stopMountTable := startMountTable(t, ctx)
	defer stopMountTable()

	proxy := ""

	path := "path/to/channel"

	// Create a new channel.
	channel, err := newChannel(ctx, mounttable, proxy, path)
	if err != nil {
		t.Fatal("newChannel(%v, %v, %v) failed: %v", ctx, mounttable, proxy, path)
	}

	// New channel should be empty.
	if err := AssertMembersWithNames(channel, []string{}); err != nil {
		t.Error(err)
	}

	// Join the channel.
	if err := channel.join(); err != nil {
		t.Fatalf("channel.join() failed: %v", err)
	}

	// Channel should contain only current user.
	if err := AssertMembersWithNames(channel, []string{channel.UserName()}); err != nil {
		t.Error(err)
	}

	// Create and join the channel a second time.
	channel2, err := newChannel(ctx, mounttable, proxy, path)
	if err != nil {
		t.Fatal("newChannel(%v, %v, %v) failed: %v", ctx, mounttable, proxy, path)
	}
	if err := channel2.join(); err != nil {
		t.Fatalf("channel2.join() failed: %v", err)
	}

	// Channel should contain both users.
	if err := AssertMembersWithNames(channel, []string{channel.UserName(), channel2.UserName()}); err != nil {
		t.Error(err)
	}

	// Leave first instance of channel.
	if err := channel.leave(); err != nil {
		t.Fatalf("channel.leave() failed: %v", err)
	}

	// Channel should contain only second user.
	if err := AssertMembersWithNames(channel, []string{channel2.UserName()}); err != nil {
		t.Error(err)
	}

	// Leave second instance of channel.
	if err := channel2.leave(); err != nil {
		t.Fatalf("channel2.leave() failed: %v", err)
	}

	// Channel should be empty.
	if err := AssertMembersWithNames(channel, []string{}); err != nil {
		t.Error(err)
	}
}

func TestBroadcastMessage(t *testing.T) {
	ctx, shutdown := test.InitForTest()
	defer shutdown()

	mounttable, stopMountTable := startMountTable(t, ctx)
	defer stopMountTable()

	proxy := ""

	path := "path/to/channel"

	channel, err := newChannel(ctx, mounttable, proxy, path)
	if err != nil {
		t.Fatalf("newChannel(%v, %v, %v) failed: %v", ctx, mounttable, proxy, path, err)
	}

	defer channel.leave()

	if err := channel.join(); err != nil {
		t.Fatalf("channel.join() failed: %v", err)
	}

	message := "Hello Vanadium world!"

	go func() {
		// Call getMembers(), which will set channel.members, used by
		// channel.broadcastMessage().
		if _, err := channel.getMembers(); err != nil {
			t.Fatalf("channel.getMembers() failed: %v", err)
		}
		if err := channel.broadcastMessage(message); err != nil {
			t.Fatalf("channel.broadcastMessage(%v) failed: %v", message, err)
		}
	}()

	select {
	case <-time.After(1 * time.Second):
		t.Errorf("Timeout waiting for message to be received.")
	case m := <-channel.messages:
		if m.Text != message {
			t.Errorf("Expected message text to be %v but got %v", message, m.Text)
		}
		if got, want := m.SenderName, channel.UserName(); got != want {
			t.Errorf("Got m.SenderName = %v, want %v", got, want)
		}
	}
}
<|endoftext|>


<commit_before>/*
Given an array A of integers, for each integer A[i] we may choose any x with -K <= x <= K, and add x to A[i].
After this process, we have some array B.
Return the smallest possible difference between the maximum value of B and the minimum value of B.

Example 1:
Input: A = [1], K = 0
Output: 0
Explanation: B = [1]

Example 2:
Input: A = [0,10], K = 2
Output: 6
Explanation: B = [2,8]

Example 3:
Input: A = [1,3,6], K = 3
Output: 0
Explanation: B = [3,3,3] or B = [4,4,4]

*/

package main

import (
	"fmt"
	"sort"
)

func main() {
	tests := [][][]int{{{1}, {0}}, {{0, 10}, {2}}, {{1, 3, 6}, {3}}, {{2, 7, 2}, {1}}}

	for _, test := range tests {
		fmt.Printf("%d\n", smallestRangeI(test[0], test[1][0]))
	}
}

func smallestRangeI(A []int, K int) int {
	// sort slice (to find median)
	sort.Ints(A)
	// find median
	var median int
	if len(A)%2 == 0 {
		// even length array
		median = (A[len(A)/2] + A[len(A)/2-1]) / 2
	} else {
		// odd length array
		if len(A) == 1 {
			median = A[0]
		} else {
			median = A[len(A)/2]
		}
	}

	fmt.Printf("-------------------\nA: %v\nK: %d\n", A, K)
	fmt.Printf("\nMedian:%d\n", median)

	var smallest int
	var largest int

	for k, v := range A {
		fmt.Printf("\n\tIndex: %d, Value: %d\n", k, v)
		var newV int
		if v > median {
			// need to add a negative amount
			fmt.Println("\tValue greater than median, need to be reduced..")
			if v-median < K {
				// v can be brought down to median value
				newV = median
				fmt.Printf("\tValue can be brought down to median (%d)\n", newV)
			} else {
				// this is the closest we can bring v to median
				newV = v - K
				fmt.Printf("\tClosest value can be brought down to median is %d\n", newV)
			}
		} else if v < median {
			// need to add a positive amount
			fmt.Println("\tValue less than median, need to be increased..")
			if median-v < K {
				// v can be brought up to median value
				newV = median
				fmt.Printf("\tValue can be brought up to median (%d)\n", newV)
			} else {
				// closest v can be brought to median
				newV = v + K
				fmt.Printf("\tClosest value can be brought up to median is %d\n", newV)
			}
		} else {
			// v == median
			fmt.Printf("\tMedian (%d) == Value (%d), no increment/decrement required.", median, v)
			newV = v
		}

		// keep track of largest and smallest newV values
		if k == 0 {
			smallest = newV
			largest = newV
		} else {
			if newV < smallest {
				smallest = newV
			}

			if newV > largest {
				largest = newV
			}
		}

		fmt.Printf("\n\tCurrent Largest: %d\n\tCurrent Smallest: %d\n", largest, smallest)
	}

	return largest - smallest
}
<commit_msg>Update code with correct diff func<commit_after>/*
Given an array A of integers, for each integer A[i] we may choose any x with -K <= x <= K, and add x to A[i].
After this process, we have some array B.
Return the smallest possible difference between the maximum value of B and the minimum value of B.

Example 1:
Input: A = [1], K = 0
Output: 0
Explanation: B = [1]

Example 2:
Input: A = [0,10], K = 2
Output: 6
Explanation: B = [2,8]

Example 3:
Input: A = [1,3,6], K = 3
Output: 0
Explanation: B = [3,3,3] or B = [4,4,4]

*/

package main

import (
	"fmt"
	"sort"
)

func main() {
	tests := [][][]int{{{1}, {0}}, {{0, 10}, {2}}, {{1, 3, 6}, {3}}, {{2, 7, 2}, {1}}}

	for _, test := range tests {
		// fmt.Printf("%d\n", smallestRangeI(test[0], test[1][0]))
		fmt.Printf("%d\n", smallestRangeI2(test[0], test[1][0]))
	}
}

// correct
func smallestRangeI2(A []int, K int) int {
	// find the largest and smallest elements of A
	largest := 0
	smallest := 0

	for k, v := range A {
		if k == 0 {
			// initialize largest/smallest
			largest = v
			smallest = v
		} else {
			// update largest/smallest values
			if v > largest {
				largest = v
			}

			if v < smallest {
				smallest = v
			}
		}
	}

	// if the largest and smallest can be decreased/increased to the same value with K, return 0
	// else, return the closest they can be brought to
	if (smallest + K) >= (largest - K) {
		return 0
	} else {
		diff := largest - smallest

		if diff/2 > K {
			return (largest - K) - (smallest + K)
		}

		return (largest - (diff / 2)) - (smallest + (diff / 2))
	}
}

// incorrect
func smallestRangeI(A []int, K int) int {
	// sort slice (to find median)
	sort.Ints(A)
	// find median
	var median int
	if len(A)%2 == 0 {
		// even length array
		median = (A[len(A)/2] + A[len(A)/2-1]) / 2
	} else {
		// odd length array
		if len(A) == 1 {
			median = A[0]
		} else {
			median = A[len(A)/2]
		}
	}

	fmt.Printf("-------------------\nA: %v\nK: %d\n", A, K)
	fmt.Printf("\nMedian:%d\n", median)

	var smallest int
	var largest int

	for k, v := range A {
		fmt.Printf("\n\tIndex: %d, Value: %d\n", k, v)
		var newV int
		if v > median {
			// need to add a negative amount
			fmt.Println("\tValue greater than median, need to be reduced..")
			if v-median < K {
				// v can be brought down to median value
				newV = median
				fmt.Printf("\tValue can be brought down to median (%d)\n", newV)
			} else {
				// this is the closest we can bring v to median
				newV = v - K
				fmt.Printf("\tClosest value can be brought down to median is %d\n", newV)
			}
		} else if v < median {
			// need to add a positive amount
			fmt.Println("\tValue less than median, need to be increased..")
			if median-v < K {
				// v can be brought up to median value
				newV = median
				fmt.Printf("\tValue can be brought up to median (%d)\n", newV)
			} else {
				// closest v can be brought to median
				newV = v + K
				fmt.Printf("\tClosest value can be brought up to median is %d\n", newV)
			}
		} else {
			// v == median
			fmt.Printf("\tMedian (%d) == Value (%d), no increment/decrement required.", median, v)
			newV = v
		}

		// keep track of largest and smallest newV values
		if k == 0 {
			smallest = newV
			largest = newV
		} else {
			if newV < smallest {
				smallest = newV
			}

			if newV > largest {
				largest = newV
			}
		}

		fmt.Printf("\n\tCurrent Largest: %d\n\tCurrent Smallest: %d\n", largest, smallest)
	}

	return largest - smallest
}
<|endoftext|>