<commit_before>package redis_test

import (
	"fmt"
	"strconv"

	"github.com/vmihailenco/redis/v2"
)

func ExampleNewTCPClient() {
	client := redis.NewTCPClient(&redis.Options{
		Addr:     "localhost:6379",
		Password: "", // no password set
		DB:       0,  // use default DB
	})
	defer client.Close()

	ping := client.Ping()
	fmt.Println(ping.Err(), ping.Val())
	// Output: <nil> PONG
}

func ExampleNewUnixClient() {
	client := redis.NewUnixClient(&redis.Options{
		Addr: "/tmp/redis.sock",
	})
	defer client.Close()

	ping := client.Ping()
	fmt.Println(ping.Err(), ping.Val())
	// Output: <nil> PONG
}

func ExampleSet() {
	client := redis.NewTCPClient(&redis.Options{
		Addr: ":6379",
	})
	defer client.Close()

	set := client.Set("foo", "bar")
	fmt.Println(set.Err(), set.Val())

	get := client.Get("foo")
	fmt.Println(get.Err(), get.Val())

	// Output: <nil> OK
	// <nil> bar
}

func ExamplePipelined() {
	client := redis.NewTCPClient(&redis.Options{
		Addr: ":6379",
	})
	defer client.Close()

	cmds, err := client.Pipelined(func(c *redis.Pipeline) {
		c.Set("key1", "hello1")
		c.Get("key2")
	})
	fmt.Println(cmds, err)
	// Output: [SET key1 hello1: OK GET key2: (nil)] (nil)
}

func ExamplePipeline() {
	client := redis.NewTCPClient(&redis.Options{
		Addr: ":6379",
	})
	defer client.Close()

	pipeline := client.Pipeline()
	set := pipeline.Set("key1", "hello1")
	get := pipeline.Get("key2")
	cmds, err := pipeline.Exec()
	fmt.Println(cmds, err)
	fmt.Println(set)
	fmt.Println(get)
	// Output: [SET key1 hello1: OK GET key2: (nil)] (nil)
	// SET key1 hello1: OK
	// GET key2: (nil)
}

func ExampleMulti() {
	incr := func(tx *redis.Multi) ([]redis.Cmder, error) {
		get := tx.Get("key")
		if err := get.Err(); err != nil && err != redis.Nil {
			return nil, err
		}

		val, _ := strconv.ParseInt(get.Val(), 10, 64)

		return tx.Exec(func() {
			tx.Set("key", strconv.FormatInt(val+1, 10))
		})
	}

	client := redis.NewTCPClient(&redis.Options{
		Addr: ":6379",
	})
	defer client.Close()

	client.Del("key")

	tx := client.Multi()
	defer tx.Close()

	watch := tx.Watch("key")
	_ = watch.Err()

	for {
		cmds, err := incr(tx)
		if err == redis.Nil {
			// Transaction failed. Repeat.
			continue
		} else if err != nil {
			panic(err)
		}
		fmt.Println(err, cmds)
		break
	}

	// Output: <nil> [SET key 1: OK]
}

func ExamplePubSub() {
	client := redis.NewTCPClient(&redis.Options{
		Addr: ":6379",
	})
	defer client.Close()

	pubsub := client.PubSub()
	defer pubsub.Close()

	err := pubsub.Subscribe("mychannel")
	_ = err

	msg, err := pubsub.Receive()
	fmt.Println(msg, err)

	pub := client.Publish("mychannel", "hello")
	_ = pub.Err()

	msg, err = pubsub.Receive()
	fmt.Println(msg, err)

	// Output: &{subscribe mychannel 1} <nil>
	// &{mychannel hello} <nil>
}

func Get(client *redis.Client, key string) *redis.StringCmd {
	cmd := redis.NewStringCmd("GET", key)
	client.Process(cmd)
	return cmd
}

func ExampleCustomCommand() {
	client := redis.NewTCPClient(&redis.Options{
		Addr: ":6379",
	})
	defer client.Close()

	get := Get(client, "key_does_not_exist")
	fmt.Println(get.Err(), get.Val())
	// Output: (nil)
}
<commit_msg>Rename example.<commit_after>package redis_test

import (
	"fmt"
	"strconv"

	"github.com/vmihailenco/redis/v2"
)

func ExampleNewTCPClient() {
	client := redis.NewTCPClient(&redis.Options{
		Addr:     "localhost:6379",
		Password: "", // no password set
		DB:       0,  // use default DB
	})
	defer client.Close()

	ping := client.Ping()
	fmt.Println(ping.Err(), ping.Val())
	// Output: <nil> PONG
}

func ExampleNewUnixClient() {
	client := redis.NewUnixClient(&redis.Options{
		Addr: "/tmp/redis.sock",
	})
	defer client.Close()

	ping := client.Ping()
	fmt.Println(ping.Err(), ping.Val())
	// Output: <nil> PONG
}

func ExampleSet() {
	client := redis.NewTCPClient(&redis.Options{
		Addr: ":6379",
	})
	defer client.Close()

	set := client.Set("foo", "bar")
	fmt.Println(set.Err(), set.Val())

	get := client.Get("foo")
	fmt.Println(get.Err(), get.Val())

	// Output: <nil> OK
	// <nil> bar
}

func ExamplePipelinePipelined() {
	client := redis.NewTCPClient(&redis.Options{
		Addr: ":6379",
	})
	defer client.Close()

	cmds, err := client.Pipelined(func(c *redis.Pipeline) {
		c.Set("key1", "hello1")
		c.Get("key2")
	})
	fmt.Println(cmds, err)
	// Output: [SET key1 hello1: OK GET key2: (nil)] (nil)
}

func ExamplePipeline() {
	client := redis.NewTCPClient(&redis.Options{
		Addr: ":6379",
	})
	defer client.Close()

	pipeline := client.Pipeline()
	set := pipeline.Set("key1", "hello1")
	get := pipeline.Get("key2")
	cmds, err := pipeline.Exec()
	fmt.Println(cmds, err)
	fmt.Println(set)
	fmt.Println(get)
	// Output: [SET key1 hello1: OK GET key2: (nil)] (nil)
	// SET key1 hello1: OK
	// GET key2: (nil)
}

func ExampleMulti() {
	incr := func(tx *redis.Multi) ([]redis.Cmder, error) {
		get := tx.Get("key")
		if err := get.Err(); err != nil && err != redis.Nil {
			return nil, err
		}

		val, _ := strconv.ParseInt(get.Val(), 10, 64)

		return tx.Exec(func() {
			tx.Set("key", strconv.FormatInt(val+1, 10))
		})
	}

	client := redis.NewTCPClient(&redis.Options{
		Addr: ":6379",
	})
	defer client.Close()

	client.Del("key")

	tx := client.Multi()
	defer tx.Close()

	watch := tx.Watch("key")
	_ = watch.Err()

	for {
		cmds, err := incr(tx)
		if err == redis.Nil {
			// Transaction failed. Repeat.
			continue
		} else if err != nil {
			panic(err)
		}
		fmt.Println(err, cmds)
		break
	}

	// Output: <nil> [SET key 1: OK]
}

func ExamplePubSub() {
	client := redis.NewTCPClient(&redis.Options{
		Addr: ":6379",
	})
	defer client.Close()

	pubsub := client.PubSub()
	defer pubsub.Close()

	err := pubsub.Subscribe("mychannel")
	_ = err

	msg, err := pubsub.Receive()
	fmt.Println(msg, err)

	pub := client.Publish("mychannel", "hello")
	_ = pub.Err()

	msg, err = pubsub.Receive()
	fmt.Println(msg, err)

	// Output: &{subscribe mychannel 1} <nil>
	// &{mychannel hello} <nil>
}

func Get(client *redis.Client, key string) *redis.StringCmd {
	cmd := redis.NewStringCmd("GET", key)
	client.Process(cmd)
	return cmd
}

func ExampleCustomCommand() {
	client := redis.NewTCPClient(&redis.Options{
		Addr: ":6379",
	})
	defer client.Close()

	get := Get(client, "key_does_not_exist")
	fmt.Println(get.Err(), get.Val())
	// Output: (nil)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"runtime"
	"strconv"
	"time"

	"github.com/goamz/goamz/aws"
	lsdb "github.com/goamz/goamz/exp/sdb"
	ls3 "github.com/goamz/goamz/s3"
	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
)

var (
	sdb       *lsdb.SDB
	domain    *lsdb.Domain
	bucket    *ls3.Bucket
	sdbDomain string
	s3Bucket  string
	startedAt string
)

var version = "0.0.0"

func addSlash(s string) string {
	return s[0:2] + "/" + s[2:40]
}

func WriteJSON(w http.ResponseWriter, data interface{}) {
	js, err := json.Marshal(data)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Header().Set("Content-Type", "application/json")
	w.Write(js)
}

func blobHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	blob, err := bucket.Get(addSlash(vars["hash"]))
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Write(blob)
}
func statsHandler(w http.ResponseWriter, r *http.Request) {
	query, err := sdb.Select(fmt.Sprintf("SELECT COUNT(*) FROM %v", sdbDomain), false)
	if err != nil {
		log.Printf("err:%v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	i := query.Items[0]
	cnt, _ := strconv.Atoi(i.Attrs[0].Value)
	sizeCounter := domain.Item("size")
	attrs, err := sizeCounter.Attrs(nil, false)
	if err != nil {
		log.Printf("err:%v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	size, _ := strconv.Atoi(attrs.Attrs[0].Value)
	WriteJSON(w, map[string]interface{}{
		"blob_count": cnt - 1,
		"blob_size":  size,
		"version":    version,
		"started_at": startedAt,
		"s3_bucket":  s3Bucket,
		"sdb_domain": sdbDomain,
	})
}
func blobsHandler(w http.ResponseWriter, r *http.Request) {
	start := r.URL.Query().Get("start")
	if start == "" {
		start = "0"
	}
	end := r.URL.Query().Get("end")
	if end == "" {
		end = time.Now().UTC().Format(time.RFC3339Nano)
	}
	query, err := sdb.Select(fmt.Sprintf("SELECT * FROM %v where time > '%s' and time <= '%s' order by time asc", sdbDomain, start, end), false)
	if err != nil {
		log.Printf("err:%v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	res := []map[string]string{}
	for _, i := range query.Items {
		m := map[string]string{
			"hash": i.Name,
		}
		for _, attr := range i.Attrs {
			m[attr.Name] = attr.Value
		}
		res = append(res, m)
	}
	WriteJSON(w, map[string]interface{}{
		"blobs": res,
		"start": start,
		"end":   end,
	})
}

func uploadHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "POST":
		mr, err := r.MultipartReader()
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		for {
			part, err := mr.NextPart()
			if err == io.EOF {
				break
			}
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			hash := part.FormName()
			var data bytes.Buffer
			data.ReadFrom(part)
			blob := data.Bytes()
			computedHash := fmt.Sprintf("%x", sha1.Sum(blob))
			if hash != computedHash {
				http.Error(w, "hash don't match", http.StatusInternalServerError)
			}
			if err := bucket.Put(addSlash(hash), blob, "", ls3.Private, ls3.Options{}); err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			item := domain.Item(hash)
			pa := &lsdb.PutAttrs{}
			pa.Add("time", time.Now().UTC().Format(time.RFC3339Nano))
			if _, err := item.PutAttrs(pa); err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			sizeCounter := domain.Item("size")
			for {
				attrs, err := sizeCounter.Attrs(nil, true)
				if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
				}
				update := &lsdb.PutAttrs{}
				var prevSize int
				if len(attrs.Attrs) != 0 {
					prevSize, _ = strconv.Atoi(attrs.Attrs[0].Value)
					update.IfValue("size", strconv.Itoa(prevSize))
				}
				update.Add("size", strconv.Itoa(prevSize+len(blob)))
				if _, err := sizeCounter.PutAttrs(update); err == nil {
					break
				}
			}
		}
	default:
		w.WriteHeader(http.StatusMethodNotAllowed)
	}
}

func main() {
	startedAt = time.Now().Format(time.RFC3339)
	auth, err := aws.EnvAuth()
	if err != nil {
		panic(err)
	}
	sdbDomain = os.Getenv("TS4_SDB_DOMAIN")
	if sdbDomain == "" {
		panic("TS4_SDB_DOMAIN not set")
	}
	s3Bucket = os.Getenv("TS4_S3_BUCKET")
	if s3Bucket == "" {
		panic("TS4_S3_BUCKET not set")
	}
	sdb = lsdb.New(auth, aws.USEast)
	s3 := ls3.New(auth, aws.USEast)
	domain = sdb.Domain(sdbDomain)
	if _, err := domain.CreateDomain(); err != nil {
		panic(err)
	}
	bucket = s3.Bucket(s3Bucket)
	if err := bucket.PutBucket(ls3.Private); err != nil {
		panic(err)
	}
	runtime.GOMAXPROCS(runtime.NumCPU())
	r := mux.NewRouter()
	r.Handle("/api/blobs", handlers.LoggingHandler(os.Stdout, http.HandlerFunc(blobsHandler)))
	r.Handle("/api/blob/{hash}", handlers.LoggingHandler(os.Stdout, http.HandlerFunc(blobHandler)))
	r.Handle("/api/upload", handlers.LoggingHandler(os.Stdout, http.HandlerFunc(uploadHandler)))
	r.Handle("/_stats", handlers.LoggingHandler(os.Stdout, http.HandlerFunc(statsHandler)))
	http.Handle("/", r)
	log.Printf("Starting ts4 version %v; %v (%v/%v)", version, runtime.Version(), runtime.GOOS, runtime.GOARCH)
	log.Printf("Listening on port 8010")
	http.ListenAndServe(":8011", nil)
}
<commit_msg>bugfix size counter for blob already indexed<commit_after>package main

import (
	"bytes"
	"crypto/rand"
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"runtime"
	"strconv"
	"time"

	"github.com/goamz/goamz/aws"
	lsdb "github.com/goamz/goamz/exp/sdb"
	ls3 "github.com/goamz/goamz/s3"
	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
)

var (
	sdb       *lsdb.SDB
	domain    *lsdb.Domain
	bucket    *ls3.Bucket
	sdbDomain string
	s3Bucket  string
	startedAt string
)

var version = "0.0.0"

func addSlash(s string) string {
	return s[0:2] + "/" + s[2:40]
}

// newUUID generates a random UUID according to RFC 4122
func newUUID() (string, error) {
	uuid := make([]byte, 16)
	n, err := io.ReadFull(rand.Reader, uuid)
	if n != len(uuid) || err != nil {
		return "", err
	}
	// variant bits; see section 4.1.1
	uuid[8] = uuid[8]&^0xc0 | 0x80
	// version 4 (pseudo-random); see section 4.1.3
	uuid[6] = uuid[6]&^0xf0 | 0x40
	return fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:]), nil
}

func WriteJSON(w http.ResponseWriter, data interface{}) {
	js, err := json.Marshal(data)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Header().Set("Content-Type", "application/json")
	w.Write(js)
}

func blobHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	blob, err := bucket.Get(addSlash(vars["hash"]))
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Write(blob)
}
func statsHandler(w http.ResponseWriter, r *http.Request) {
	query, err := sdb.Select(fmt.Sprintf("SELECT COUNT(*) FROM %v", sdbDomain), false)
	if err != nil {
		log.Printf("err:%v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	i := query.Items[0]
	cnt, _ := strconv.Atoi(i.Attrs[0].Value)
	sizeCounter := domain.Item("size")
	attrs, err := sizeCounter.Attrs(nil, false)
	if err != nil {
		log.Printf("err:%v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	size, _ := strconv.Atoi(attrs.Attrs[0].Value)
	WriteJSON(w, map[string]interface{}{
		"blob_count": cnt - 1,
		"blob_size":  size,
		"version":    version,
		"started_at": startedAt,
		"s3_bucket":  s3Bucket,
		"sdb_domain": sdbDomain,
	})
}
func blobsHandler(w http.ResponseWriter, r *http.Request) {
	start := r.URL.Query().Get("start")
	if start == "" {
		start = "0"
	}
	end := r.URL.Query().Get("end")
	if end == "" {
		end = time.Now().UTC().Format(time.RFC3339Nano)
	}
	query, err := sdb.Select(fmt.Sprintf("SELECT * FROM %v where time > '%s' and time <= '%s' order by time asc", sdbDomain, start, end), false)
	if err != nil {
		log.Printf("err:%v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	res := []map[string]string{}
	for _, i := range query.Items {
		m := map[string]string{
			"hash": i.Name,
		}
		for _, attr := range i.Attrs {
			m[attr.Name] = attr.Value
		}
		res = append(res, m)
	}
	WriteJSON(w, map[string]interface{}{
		"blobs": res,
		"start": start,
		"end":   end,
	})
}

func uploadHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "POST":
		mr, err := r.MultipartReader()
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		for {
			part, err := mr.NextPart()
			if err == io.EOF {
				break
			}
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			hash := part.FormName()
			var data bytes.Buffer
			data.ReadFrom(part)
			blob := data.Bytes()
			computedHash := fmt.Sprintf("%x", sha1.Sum(blob))
			if hash != computedHash {
				http.Error(w, "hash don't match", http.StatusInternalServerError)
			}
			if err := bucket.Put(addSlash(hash), blob, "", ls3.Private, ls3.Options{}); err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			item := domain.Item(hash)
			pa := &lsdb.PutAttrs{}
			itemAttrs, err := item.Attrs(nil, false)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			if len(itemAttrs.Attrs) != 0 {
				// the blob is already indexed
				return
			}
			pa.Add("time", time.Now().UTC().Format(time.RFC3339Nano))
			if _, err := item.PutAttrs(pa); err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			sizeCounter := domain.Item("size")
			for {
				attrs, err := sizeCounter.Attrs(nil, true)
				if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
				}
				update := &lsdb.PutAttrs{}
				var prevSize int
				if len(attrs.Attrs) != 0 {
					prevSize, _ = strconv.Atoi(attrs.Attrs[0].Value)
					update.IfValue("size", strconv.Itoa(prevSize))
				}
				update.Add("size", strconv.Itoa(prevSize+len(blob)))
				if _, err := sizeCounter.PutAttrs(update); err == nil {
					break
				}
			}
		}
	default:
		w.WriteHeader(http.StatusMethodNotAllowed)
	}
}

func main() {
	startedAt = time.Now().Format(time.RFC3339)
	auth, err := aws.EnvAuth()
	if err != nil {
		panic(err)
	}
	sdbDomain = os.Getenv("TS4_SDB_DOMAIN")
	if sdbDomain == "" {
		panic("TS4_SDB_DOMAIN not set")
	}
	s3Bucket = os.Getenv("TS4_S3_BUCKET")
	if s3Bucket == "" {
		panic("TS4_S3_BUCKET not set")
	}
	sdb = lsdb.New(auth, aws.USEast)
	s3 := ls3.New(auth, aws.USEast)
	domain = sdb.Domain(sdbDomain)
	if _, err := domain.CreateDomain(); err != nil {
		panic(err)
	}
	bucket = s3.Bucket(s3Bucket)
	if err := bucket.PutBucket(ls3.Private); err != nil {
		panic(err)
	}
	runtime.GOMAXPROCS(runtime.NumCPU())
	r := mux.NewRouter()
	r.Handle("/api/blobs", handlers.LoggingHandler(os.Stdout, http.HandlerFunc(blobsHandler)))
	r.Handle("/api/blob/{hash}", handlers.LoggingHandler(os.Stdout, http.HandlerFunc(blobHandler)))
	r.Handle("/api/upload", handlers.LoggingHandler(os.Stdout, http.HandlerFunc(uploadHandler)))
	r.Handle("/_stats", handlers.LoggingHandler(os.Stdout, http.HandlerFunc(statsHandler)))
	http.Handle("/", r)
	log.Printf("Starting ts4 version %v; %v (%v/%v)", version, runtime.Version(), runtime.GOOS, runtime.GOARCH)
	log.Printf("Listening on port 8010")
	http.ListenAndServe(":8010", nil)
}
<|endoftext|>


<commit_before>/*
Open Source Initiative OSI - The MIT License (MIT):Licensing

The MIT License (MIT)
Copyright (c) 2013 Ralph Caraveo (deckarep@gmail.com)

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

// Package mapset implements a simple and generic set collection.
// Items stored within it are unordered and unique
// It supports typical set operations: membership testing, intersection, union, difference and symmetric difference

package mapset

import (
	"fmt"
	"strings"
)

// The primary type that represents a set
type Set map[interface{}]struct{}

// Creates and returns a reference to an empty set.
func NewSet() Set {
	return make(Set)
}

// Creates and returns a reference to a set from an existing slice
func NewSetFromSlice(s []interface{}) Set {
	a := NewSet()
	for _, item := range s {
		a.Add(item)
	}
	return a
}

// Adds an item to the current set if it doesn't already exist in the set.
func (set Set) Add(i interface{}) bool {
	_, found := set[i]
	set[i] = struct{}{}
	return !found //False if it existed already
}

// Determines if a given item is already in the set.
func (set Set) Contains(i interface{}) bool {
	_, found := set[i]
	return found
}

// Determines if the given items are all in the set
func (set Set) ContainsAll(i ...interface{}) bool {
	allSet := NewSetFromSlice(i)

	if allSet.IsSubset(set) {

		return true
	}

	return false
}

// Determines if every item in the other set is in this set.
func (set Set) IsSubset(other Set) bool {
	for key, _ := range set {
		if !other.Contains(key) {
			return false
		}
	}
	return true
}

// Determines if every item of this set is in the other set.
func (set Set) IsSuperset(other Set) bool {
	return other.IsSubset(set)
}

// Returns a new set with all items in both sets.
func (set Set) Union(other Set) Set {
	unionedSet := NewSet()

	for key, _ := range set {
		unionedSet.Add(key)
	}
	for key, _ := range other {
		unionedSet.Add(key)
	}
	return unionedSet
}

// Returns a new set with items that exist only in both sets.
func (set Set) Intersect(other Set) Set {
	intersectedSet := NewSet()
	var smallerSet Set = nil
	var largerSet Set = nil

	//figure out the smaller of the two sets and loop on that one as an optimization.
	if set.Size() < other.Size() {
		smallerSet = set
		largerSet = other
	} else {
		smallerSet = other
		largerSet = set
	}

	for key, _ := range smallerSet {
		if largerSet.Contains(key) {
			intersectedSet.Add(key)
		}
	}
	return intersectedSet
}

// Returns a new set with items in the current set but not in the other set
func (set Set) Difference(other Set) Set {
	differencedSet := NewSet()

	for key, _ := range set {
		if !other.Contains(key) {
			differencedSet.Add(key)
		}
	}

	return differencedSet
}

// Returns a new set with items in the current set or the other set but not in both.
func (set Set) SymmetricDifference(other Set) Set {
	aDiff := set.Difference(other)
	bDiff := other.Difference(set)

	symDifferencedSet := aDiff.Union(bDiff)

	return symDifferencedSet
}

// Clears the entire set to be the empty set.
func (set *Set) Clear() {
	*set = make(map[interface{}]struct{})
}

// Allows the removal of a single item in the set.
func (set Set) Remove(i interface{}) {
	delete(set, i)
}

// Size returns the how many items are currently in the set.
func (set Set) Size() int {
	return len(set)
}

// Equal determines if two sets are equal to each other.
// If they both are the same size and have the same items they are considered equal.
// Order of items is not relevent for sets to be equal.
func (set Set) Equal(other Set) bool {
	if set.Size() != other.Size() {
		return false
	} else {
		for elem, _ := range set {
			if !other.Contains(elem) {
				return false
			}
		}
		return true
	}
}

// Provides a convenient string representation of the current state of the set.
func (set Set) String() string {
	items := make([]string, 0, len(set))

	for key := range set {
		items = append(items, fmt.Sprintf("%v", key))
	}
	return fmt.Sprintf("Set{%s}", strings.Join(items, ", "))
}
<commit_msg>Slight aesthetic improvement<commit_after>/*
Open Source Initiative OSI - The MIT License (MIT):Licensing

The MIT License (MIT)
Copyright (c) 2013 Ralph Caraveo (deckarep@gmail.com)

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

// Package mapset implements a simple and generic set collection.
// Items stored within it are unordered and unique
// It supports typical set operations: membership testing, intersection, union, difference and symmetric difference

package mapset

import (
	"fmt"
	"strings"
)

// The primary type that represents a set
type Set map[interface{}]struct{}

// Creates and returns a reference to an empty set.
func NewSet() Set {
	return make(Set)
}

// Creates and returns a reference to a set from an existing slice
func NewSetFromSlice(s []interface{}) Set {
	a := NewSet()
	for _, item := range s {
		a.Add(item)
	}
	return a
}

// Adds an item to the current set if it doesn't already exist in the set.
func (set Set) Add(i interface{}) bool {
	_, found := set[i]
	set[i] = struct{}{}
	return !found //False if it existed already
}

// Determines if a given item is already in the set.
func (set Set) Contains(i interface{}) bool {
	_, found := set[i]
	return found
}

// Determines if the given items are all in the set
func (set Set) ContainsAll(i ...interface{}) bool {
	allSet := NewSetFromSlice(i)

	if allSet.IsSubset(set) {

		return true
	}

	return false
}

// Determines if every item in the other set is in this set.
func (set Set) IsSubset(other Set) bool {
	for key := range set {
		if !other.Contains(key) {
			return false
		}
	}
	return true
}

// Determines if every item of this set is in the other set.
func (set Set) IsSuperset(other Set) bool {
	return other.IsSubset(set)
}

// Returns a new set with all items in both sets.
func (set Set) Union(other Set) Set {
	unionedSet := NewSet()

	for key, _ := range set {
		unionedSet.Add(key)
	}
	for key, _ := range other {
		unionedSet.Add(key)
	}
	return unionedSet
}

// Returns a new set with items that exist only in both sets.
func (set Set) Intersect(other Set) Set {
	intersectedSet := NewSet()
	var smallerSet Set = nil
	var largerSet Set = nil

	//figure out the smaller of the two sets and loop on that one as an optimization.
	if set.Size() < other.Size() {
		smallerSet = set
		largerSet = other
	} else {
		smallerSet = other
		largerSet = set
	}

	for key, _ := range smallerSet {
		if largerSet.Contains(key) {
			intersectedSet.Add(key)
		}
	}
	return intersectedSet
}

// Returns a new set with items in the current set but not in the other set
func (set Set) Difference(other Set) Set {
	differencedSet := NewSet()

	for key, _ := range set {
		if !other.Contains(key) {
			differencedSet.Add(key)
		}
	}

	return differencedSet
}

// Returns a new set with items in the current set or the other set but not in both.
func (set Set) SymmetricDifference(other Set) Set {
	aDiff := set.Difference(other)
	bDiff := other.Difference(set)

	symDifferencedSet := aDiff.Union(bDiff)

	return symDifferencedSet
}

// Clears the entire set to be the empty set.
func (set *Set) Clear() {
	*set = make(map[interface{}]struct{})
}

// Allows the removal of a single item in the set.
func (set Set) Remove(i interface{}) {
	delete(set, i)
}

// Size returns the how many items are currently in the set.
func (set Set) Size() int {
	return len(set)
}

// Equal determines if two sets are equal to each other.
// If they both are the same size and have the same items they are considered equal.
// Order of items is not relevent for sets to be equal.
func (set Set) Equal(other Set) bool {
	if set.Size() != other.Size() {
		return false
	} else {
		for elem, _ := range set {
			if !other.Contains(elem) {
				return false
			}
		}
		return true
	}
}

// Provides a convenient string representation of the current state of the set.
func (set Set) String() string {
	items := make([]string, 0, len(set))

	for key := range set {
		items = append(items, fmt.Sprintf("%v", key))
	}
	return fmt.Sprintf("Set{%s}", strings.Join(items, ", "))
}
<|endoftext|>


<commit_before>package receiver

import (
	"context"
	"fmt"
	"io"
	"sync"
	"time"

	gw "github.com/cvmfs/gateway/internal/gateway"
)

// task is the common interface of all receiver tasks
type task interface {
	Reply() chan<- error
	Context() context.Context
}

// payloadTask is the input data for a payload submission task
type payloadTask struct {
	ctx        context.Context
	leasePath  string
	payload    io.Reader
	digest     string
	headerSize int
	replyChan  chan<- error
}

// Reply returns the reply channel
func (p payloadTask) Reply() chan<- error {
	return p.replyChan
}

// Context returns the context associated with the task
func (p payloadTask) Context() context.Context {
	return p.ctx
}

// commitTask is the input data for a commit task
type commitTask struct {
	ctx         context.Context
	leasePath   string
	oldRootHash string
	newRootHash string
	tag         gw.RepositoryTag
	replyChan   chan<- error
}

// Reply returns the reply channel
func (p commitTask) Reply() chan<- error {
	return p.replyChan
}

// Context returns the context associated with the task
func (p commitTask) Context() context.Context {
	return p.ctx
}

// Pool maintains a number of parallel receiver workers to service
// payload submission and commit requests. Payload submissions are done in
// parallel, using Config.NumReceivers workers, while only a single commit
// request can be treated per repository at a time.
type Pool struct {
	tasks      chan<- task
	wg         sync.WaitGroup
	workerExec string
	mock       bool
}

// StartPool the receiver pool using the specified executable and number of payload
// submission workers
func StartPool(workerExec string, numWorkers int, mock bool) (*Pool, error) {
	// Start payload submission workers
	tasks := make(chan task)

	pool := &Pool{tasks, sync.WaitGroup{}, workerExec, mock}

	for i := 0; i < numWorkers; i++ {
		pool.wg.Add(1)
		go worker(tasks, pool, i)
	}

	gw.Log("worker_pool", gw.LogInfo).
		Msg("worker pool started")

	return pool, nil
}

// Stop all the background workers
func (p *Pool) Stop() error {
	close(p.tasks)
	p.wg.Wait()
	return nil
}

// SubmitPayload to be unpacked into the repository
// TODO: implement timeout or context?
func (p *Pool) SubmitPayload(ctx context.Context, leasePath string, payload io.Reader, digest string, headerSize int) error {
	reply := make(chan error)
	p.tasks <- payloadTask{ctx, leasePath, payload, digest, headerSize, reply}
	result := <-reply
	return result
}

// CommitLease associated with the token (transaction commit)
// TODO: implement timeout or context?
func (p *Pool) CommitLease(ctx context.Context, leasePath, oldRootHash, newRootHash string, tag gw.RepositoryTag) error {
	reply := make(chan error)
	p.tasks <- commitTask{ctx, leasePath, oldRootHash, newRootHash, tag, reply}
	result := <-reply
	return result
}

func worker(tasks <-chan task, pool *Pool, workerIdx int) {
	gw.Log("worker_pool", gw.LogDebug).
		Int("worker_id", workerIdx).
		Msg("started")

	defer pool.wg.Done()
M:
	for {
		task, more := <-tasks

		if !more {
			break M
		}

		func() {
			t0 := time.Now()
			receiver, err := NewReceiver(task.Context(), pool.workerExec, pool.mock)
			if err != nil {
				task.Reply() <- err
				return
			}
			defer func() {
				if err := receiver.Quit(); err != nil {
					task.Reply() <- err
					return
				}
			}()

			var taskType string
			var result error
			switch t := task.(type) {
			case payloadTask:
				result = receiver.SubmitPayload(t.leasePath, t.payload, t.digest, t.headerSize)
				taskType = "payload"
			case commitTask:
				result = receiver.Commit(t.leasePath, t.oldRootHash, t.newRootHash, t.tag)
				taskType = "commit"
			default:
				task.Reply() <- fmt.Errorf("unknown task type")
				return
			}

			task.Reply() <- result

			gw.LogC(task.Context(), "worker_pool", gw.LogDebug).
				Int("worker_id", workerIdx).
				Dur("task_dt", time.Since(t0)).
				Msgf("%v task complete", taskType)
		}()
	}

	gw.Log("worker_pool", gw.LogDebug).
		Int("worker_id", workerIdx).
		Msg("finished")
}
<commit_msg>log error on .Quit for the receiver<commit_after>package receiver

import (
	"context"
	"fmt"
	"io"
	"sync"
	"time"

	gw "github.com/cvmfs/gateway/internal/gateway"
)

// task is the common interface of all receiver tasks
type task interface {
	Reply() chan<- error
	Context() context.Context
}

// payloadTask is the input data for a payload submission task
type payloadTask struct {
	ctx        context.Context
	leasePath  string
	payload    io.Reader
	digest     string
	headerSize int
	replyChan  chan<- error
}

// Reply returns the reply channel
func (p payloadTask) Reply() chan<- error {
	return p.replyChan
}

// Context returns the context associated with the task
func (p payloadTask) Context() context.Context {
	return p.ctx
}

// commitTask is the input data for a commit task
type commitTask struct {
	ctx         context.Context
	leasePath   string
	oldRootHash string
	newRootHash string
	tag         gw.RepositoryTag
	replyChan   chan<- error
}

// Reply returns the reply channel
func (p commitTask) Reply() chan<- error {
	return p.replyChan
}

// Context returns the context associated with the task
func (p commitTask) Context() context.Context {
	return p.ctx
}

// Pool maintains a number of parallel receiver workers to service
// payload submission and commit requests. Payload submissions are done in
// parallel, using Config.NumReceivers workers, while only a single commit
// request can be treated per repository at a time.
type Pool struct {
	tasks      chan<- task
	wg         sync.WaitGroup
	workerExec string
	mock       bool
}

// StartPool the receiver pool using the specified executable and number of payload
// submission workers
func StartPool(workerExec string, numWorkers int, mock bool) (*Pool, error) {
	// Start payload submission workers
	tasks := make(chan task)

	pool := &Pool{tasks, sync.WaitGroup{}, workerExec, mock}

	for i := 0; i < numWorkers; i++ {
		pool.wg.Add(1)
		go worker(tasks, pool, i)
	}

	gw.Log("worker_pool", gw.LogInfo).
		Msg("worker pool started")

	return pool, nil
}

// Stop all the background workers
func (p *Pool) Stop() error {
	close(p.tasks)
	p.wg.Wait()
	return nil
}

// SubmitPayload to be unpacked into the repository
// TODO: implement timeout or context?
func (p *Pool) SubmitPayload(ctx context.Context, leasePath string, payload io.Reader, digest string, headerSize int) error {
	reply := make(chan error)
	p.tasks <- payloadTask{ctx, leasePath, payload, digest, headerSize, reply}
	result := <-reply
	return result
}

// CommitLease associated with the token (transaction commit)
// TODO: implement timeout or context?
func (p *Pool) CommitLease(ctx context.Context, leasePath, oldRootHash, newRootHash string, tag gw.RepositoryTag) error {
	reply := make(chan error)
	p.tasks <- commitTask{ctx, leasePath, oldRootHash, newRootHash, tag, reply}
	result := <-reply
	return result
}

func worker(tasks <-chan task, pool *Pool, workerIdx int) {
	gw.Log("worker_pool", gw.LogDebug).
		Int("worker_id", workerIdx).
		Msg("started")

	defer pool.wg.Done()
M:
	for {
		task, more := <-tasks

		if !more {
			break M
		}

		func() {
			t0 := time.Now()
			receiver, err := NewReceiver(task.Context(), pool.workerExec, pool.mock)
			if err != nil {
				task.Reply() <- err
				return
			}
			defer func() {
				if err := receiver.Quit(); err != nil {
					gw.Log("worker_pool", gw.LogInfo).
						Int("worker_id", workerIdx).
						Msgf("got an error while quitting: %s", err)
				}
			}()

			var taskType string
			var result error
			switch t := task.(type) {
			case payloadTask:
				result = receiver.SubmitPayload(t.leasePath, t.payload, t.digest, t.headerSize)
				taskType = "payload"
			case commitTask:
				result = receiver.Commit(t.leasePath, t.oldRootHash, t.newRootHash, t.tag)
				taskType = "commit"
			default:
				task.Reply() <- fmt.Errorf("unknown task type")
				return
			}

			task.Reply() <- result

			gw.LogC(task.Context(), "worker_pool", gw.LogDebug).
				Int("worker_id", workerIdx).
				Dur("task_dt", time.Since(t0)).
				Msgf("%v task complete", taskType)
		}()
	}

	gw.Log("worker_pool", gw.LogDebug).
		Int("worker_id", workerIdx).
		Msg("finished")
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

// The run program is invoked via "go run" from src/run.bash or
// src/run.bat conditionally builds and runs the cmd/api tool.
//
// TODO(bradfitz): the "conditional" condition is always true.
// We should only do this if the user has the hg codereview extension
// enabled and verifies that the go.tools subrepo is checked out with
// a suitably recently version. In prep for the cmd/api rewrite.
package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"strings"
)

// goToolsVersion is the hg revision of the go.tools subrepo we need
// to build cmd/api.  This only needs to be updated whenever a go/types
// bug fix is needed by the cmd/api tool.
const goToolsVersion = "6698ca2900e2"

var goroot string

func main() {
	log.SetFlags(0)
	goroot = os.Getenv("GOROOT") // should be set by run.{bash,bat}
	if goroot == "" {
		log.Fatal("No $GOROOT set.")
	}
	_, err := exec.LookPath("hg")
	if err != nil {
		fmt.Println("Skipping cmd/api checks; hg not available")
		return
	}

	gopath := prepGoPath()

	cmd := exec.Command("go", "install", "--tags=api_tool", "cmd/api")
	cmd.Env = append([]string{"GOPATH=" + gopath}, filterOut(os.Environ(), "GOARCH")...)
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Fatalf("Error installing cmd/api: %v\n%s", err, out)
	}

	out, err = exec.Command("go", "tool", "api",
		"-c", file("go1", "go1.1"),
		"-next", file("next"),
		"-except", file("except")).CombinedOutput()
	if err != nil {
		log.Fatalf("Error running API checker: %v\n%s", err, out)
	}
	fmt.Print(string(out))
}

// filterOut returns a copy of the src environment without environment
// variables from remove.
// TODO: delete when issue 6201 is fixed.
func filterOut(src []string, remove ...string) (out []string) {
S:
	for _, s := range src {
		for _, r := range remove {
			if strings.HasPrefix(s, r) && strings.HasPrefix(s, r+"=") {
				continue S
			}
		}
		out = append(out, s)
	}
	return
}

// file expands s to $GOROOT/api/s.txt.
// If there are more than 1, they're comma-separated.
func file(s ...string) string {
	if len(s) > 1 {
		return file(s[0]) + "," + file(s[1:]...)
	}
	return filepath.Join(goroot, "api", s[0]+".txt")
}

// prepGoPath returns a GOPATH for the "go" tool to compile the API tool with.
// It tries to re-use a go.tools checkout from a previous run if possible,
// else it hg clones it.
func prepGoPath() string {
	const tempBase = "go.tools.TMP"

	u, err := user.Current()
	if err != nil {
		log.Fatalf("Error getting current user: %v", err)
	}

	// The GOPATH we'll return
	gopath := filepath.Join(os.TempDir(), "gopath-api-"+cleanUsername(u.Username), goToolsVersion)

	// cloneDir is where we run "hg clone".
	cloneDir := filepath.Join(gopath, "src", "code.google.com", "p")

	// The dir we clone into. We only atomically rename it to finalDir on
	// clone success.
	tmpDir := filepath.Join(cloneDir, tempBase)

	// finalDir is where the checkout will live once it's complete.
	finalDir := filepath.Join(cloneDir, "go.tools")

	if goToolsCheckoutGood(finalDir) {
		return gopath
	}
	os.RemoveAll(finalDir) // in case it's there but corrupt
	os.RemoveAll(tmpDir)   // in case of aborted hg clone before

	if err := os.MkdirAll(cloneDir, 0700); err != nil {
		log.Fatal(err)
	}
	cmd := exec.Command("hg",
		"clone", "--rev="+goToolsVersion,
		"https://code.google.com/p/go.tools",
		tempBase)
	cmd.Dir = cloneDir
	out, err := cmd.CombinedOutput()
	if err != nil {
		if _, err := http.Head("http://ip.appspot.com/"); err != nil {
			log.Printf("# Skipping API check; network appears to be unavailable")
			os.Exit(0)
		}
		log.Fatalf("Error running hg clone on go.tools: %v\n%s", err, out)
	}
	if err := os.Rename(tmpDir, finalDir); err != nil {
		log.Fatal(err)
	}
	return gopath
}

func cleanUsername(n string) string {
	b := make([]rune, len(n))
	for i, r := range n {
		if r == '\\' || r == '/' || r == ':' {
			b[i] = '_'
		} else {
			b[i] = r
		}
	}
	return string(b)
}

func goToolsCheckoutGood(dir string) bool {
	if _, err := os.Stat(dir); err != nil {
		return false
	}

	cmd := exec.Command("hg", "id", "--id")
	cmd.Dir = dir
	out, err := cmd.Output()
	if err != nil {
		return false
	}
	id := strings.TrimSpace(string(out))
	if id != goToolsVersion {
		return false
	}

	cmd = exec.Command("hg", "status")
	cmd.Dir = dir
	out, err = cmd.Output()
	if err != nil || len(out) > 0 {
		return false
	}

	return true
}
<commit_msg>cmd/api: make it work even when cgo is disabled make use of $USER or %USERNAME% to determine the current user. Fixes issue 6578.<commit_after>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

// The run program is invoked via "go run" from src/run.bash or
// src/run.bat conditionally builds and runs the cmd/api tool.
//
// TODO(bradfitz): the "conditional" condition is always true.
// We should only do this if the user has the hg codereview extension
// enabled and verifies that the go.tools subrepo is checked out with
// a suitably recently version. In prep for the cmd/api rewrite.
package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"strings"
)

// goToolsVersion is the hg revision of the go.tools subrepo we need
// to build cmd/api.  This only needs to be updated whenever a go/types
// bug fix is needed by the cmd/api tool.
const goToolsVersion = "6698ca2900e2"

var goroot string

func main() {
	log.SetFlags(0)
	goroot = os.Getenv("GOROOT") // should be set by run.{bash,bat}
	if goroot == "" {
		log.Fatal("No $GOROOT set.")
	}
	_, err := exec.LookPath("hg")
	if err != nil {
		fmt.Println("Skipping cmd/api checks; hg not available")
		return
	}

	gopath := prepGoPath()

	cmd := exec.Command("go", "install", "--tags=api_tool", "cmd/api")
	cmd.Env = append([]string{"GOPATH=" + gopath}, filterOut(os.Environ(), "GOARCH")...)
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Fatalf("Error installing cmd/api: %v\n%s", err, out)
	}

	out, err = exec.Command("go", "tool", "api",
		"-c", file("go1", "go1.1"),
		"-next", file("next"),
		"-except", file("except")).CombinedOutput()
	if err != nil {
		log.Fatalf("Error running API checker: %v\n%s", err, out)
	}
	fmt.Print(string(out))
}

// filterOut returns a copy of the src environment without environment
// variables from remove.
// TODO: delete when issue 6201 is fixed.
func filterOut(src []string, remove ...string) (out []string) {
S:
	for _, s := range src {
		for _, r := range remove {
			if strings.HasPrefix(s, r) && strings.HasPrefix(s, r+"=") {
				continue S
			}
		}
		out = append(out, s)
	}
	return
}

// file expands s to $GOROOT/api/s.txt.
// If there are more than 1, they're comma-separated.
func file(s ...string) string {
	if len(s) > 1 {
		return file(s[0]) + "," + file(s[1:]...)
	}
	return filepath.Join(goroot, "api", s[0]+".txt")
}

// prepGoPath returns a GOPATH for the "go" tool to compile the API tool with.
// It tries to re-use a go.tools checkout from a previous run if possible,
// else it hg clones it.
func prepGoPath() string {
	const tempBase = "go.tools.TMP"

	username := ""
	u, err := user.Current()
	if err == nil {
		username = u.Username
	} else {
		// Only need to handle Unix here, as Windows's os/user uses
		// native syscall and should work fine without cgo.
		username = os.Getenv("USER")
		if username == "" {
			log.Fatalf("Error getting current user: %v", err)
		}
	}

	// The GOPATH we'll return
	gopath := filepath.Join(os.TempDir(), "gopath-api-"+cleanUsername(username), goToolsVersion)

	// cloneDir is where we run "hg clone".
	cloneDir := filepath.Join(gopath, "src", "code.google.com", "p")

	// The dir we clone into. We only atomically rename it to finalDir on
	// clone success.
	tmpDir := filepath.Join(cloneDir, tempBase)

	// finalDir is where the checkout will live once it's complete.
	finalDir := filepath.Join(cloneDir, "go.tools")

	if goToolsCheckoutGood(finalDir) {
		return gopath
	}
	os.RemoveAll(finalDir) // in case it's there but corrupt
	os.RemoveAll(tmpDir)   // in case of aborted hg clone before

	if err := os.MkdirAll(cloneDir, 0700); err != nil {
		log.Fatal(err)
	}
	cmd := exec.Command("hg",
		"clone", "--rev="+goToolsVersion,
		"https://code.google.com/p/go.tools",
		tempBase)
	cmd.Dir = cloneDir
	out, err := cmd.CombinedOutput()
	if err != nil {
		if _, err := http.Head("http://ip.appspot.com/"); err != nil {
			log.Printf("# Skipping API check; network appears to be unavailable")
			os.Exit(0)
		}
		log.Fatalf("Error running hg clone on go.tools: %v\n%s", err, out)
	}
	if err := os.Rename(tmpDir, finalDir); err != nil {
		log.Fatal(err)
	}
	return gopath
}

func cleanUsername(n string) string {
	b := make([]rune, len(n))
	for i, r := range n {
		if r == '\\' || r == '/' || r == ':' {
			b[i] = '_'
		} else {
			b[i] = r
		}
	}
	return string(b)
}

func goToolsCheckoutGood(dir string) bool {
	if _, err := os.Stat(dir); err != nil {
		return false
	}

	cmd := exec.Command("hg", "id", "--id")
	cmd.Dir = dir
	out, err := cmd.Output()
	if err != nil {
		return false
	}
	id := strings.TrimSpace(string(out))
	if id != goToolsVersion {
		return false
	}

	cmd = exec.Command("hg", "status")
	cmd.Dir = dir
	out, err = cmd.Output()
	if err != nil || len(out) > 0 {
		return false
	}

	return true
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

// The run program is invoked via "go run" from src/run.bash or
// src/run.bat conditionally builds and runs the cmd/api tool.
//
// TODO(bradfitz): the "conditional" condition is always true.
// We should only do this if the user has the hg codereview extension
// enabled and verifies that the go.tools subrepo is checked out with
// a suitably recently version. In prep for the cmd/api rewrite.
package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"runtime"
	"strings"
)

// goToolsVersion is the git revision of the x/tools subrepo we need
// to build cmd/api.  This only needs to be updated whenever a go/types
// bug fix is needed by the cmd/api tool.
const goToolsVersion = "875ff2496f865e" // aka hg 6698ca2900e2

var goroot string

func main() {
	log.SetFlags(0)
	goroot = os.Getenv("GOROOT") // should be set by run.{bash,bat}
	if goroot == "" {
		log.Fatal("No $GOROOT set.")
	}
	_, err := exec.LookPath("git")
	if err != nil {
		fmt.Println("Skipping cmd/api checks; git not available")
		return
	}

	gopath := prepGoPath()

	cmd := exec.Command("go", "install", "--tags=api_tool", "cmd/api")
	cmd.Env = append(filterOut(os.Environ(), "GOARCH", "GOPATH"), "GOPATH="+gopath)
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Fatalf("Error installing cmd/api: %v\n%s", err, out)
	}

	out, err = exec.Command("go", "tool", "api",
		"-c", file("go1", "go1.1", "go1.2", "go1.3", "go1.4"),
		"-next", file("next"),
		"-except", file("except")).CombinedOutput()
	if err != nil {
		log.Fatalf("Error running API checker: %v\n%s", err, out)
	}
	fmt.Print(string(out))
}

// filterOut returns a copy of the src environment without environment
// variables from remove.
// TODO: delete when issue 6201 is fixed.
func filterOut(src []string, remove ...string) (out []string) {
S:
	for _, s := range src {
		for _, r := range remove {
			if strings.HasPrefix(s, r) && strings.HasPrefix(s, r+"=") {
				continue S
			}
		}
		out = append(out, s)
	}
	return
}

// file expands s to $GOROOT/api/s.txt.
// If there are more than 1, they're comma-separated.
func file(s ...string) string {
	if len(s) > 1 {
		return file(s[0]) + "," + file(s[1:]...)
	}
	return filepath.Join(goroot, "api", s[0]+".txt")
}

// prepGoPath returns a GOPATH for the "go" tool to compile the API tool with.
// It tries to re-use a go.tools checkout from a previous run if possible,
// else it hg clones it.
func prepGoPath() string {
	const tempBase = "go.tools.TMP"

	username := ""
	u, err := user.Current()
	if err == nil {
		username = u.Username
	} else {
		username = os.Getenv("USER")
		if username == "" {
			username = "nobody"
		}
	}

	// The GOPATH we'll return
	gopath := filepath.Join(os.TempDir(), "gopath-api-"+cleanUsername(username)+"-"+cleanUsername(strings.Fields(runtime.Version())[0]), goToolsVersion)

	// cloneDir is where we run "git clone".
	cloneDir := filepath.Join(gopath, "src", "code.google.com", "p")

	// The dir we clone into. We only atomically rename it to finalDir on
	// clone success.
	tmpDir := filepath.Join(cloneDir, tempBase)

	// finalDir is where the checkout will live once it's complete.
	finalDir := filepath.Join(cloneDir, "go.tools")

	if goToolsCheckoutGood(finalDir) {
		return gopath
	}
	os.RemoveAll(finalDir) // in case it's there but corrupt
	os.RemoveAll(tmpDir)   // in case of aborted hg clone before

	if err := os.MkdirAll(cloneDir, 0700); err != nil {
		log.Fatal(err)
	}
	cmd := exec.Command("git", "clone", "https://go.googlesource.com/tools", tempBase)
	cmd.Dir = cloneDir
	out, err := cmd.CombinedOutput()
	if err != nil {
		if _, err := http.Head("http://ip.appspot.com/"); err != nil {
			log.Printf("# Skipping API check; network appears to be unavailable")
			os.Exit(0)
		}
		log.Fatalf("Error running git clone on x/tools: %v\n%s", err, out)
	}
	cmd = exec.Command("git", "reset", "--hard", goToolsVersion)
	cmd.Dir = tmpDir
	out, err = cmd.CombinedOutput()
	if err != nil {
		log.Fatalf("Error updating x/tools in %v to %v: %v, %s", tmpDir, goToolsVersion, err, out)
	}

	if err := os.Rename(tmpDir, finalDir); err != nil {
		log.Fatal(err)
	}
	return gopath
}

func cleanUsername(n string) string {
	b := make([]rune, len(n))
	for i, r := range n {
		if r == '\\' || r == '/' || r == ':' {
			b[i] = '_'
		} else {
			b[i] = r
		}
	}
	return string(b)
}

func goToolsCheckoutGood(dir string) bool {
	if _, err := os.Stat(dir); err != nil {
		return false
	}

	cmd := exec.Command("git", "rev-parse", "HEAD")
	cmd.Dir = dir
	out, err := cmd.Output()
	if err != nil {
		return false
	}
	id := strings.TrimSpace(string(out))
	if !strings.HasPrefix(id, goToolsVersion) {
		return false
	}

	cmd = exec.Command("git", "status", "--porcelain")
	cmd.Dir = dir
	out, err = cmd.Output()
	if err != nil || strings.TrimSpace(string(out)) != "" {
		return false
	}
	return true
}
<commit_msg>cmd/api: fix race in run.go with multiple builders on a machine<commit_after>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

// The run program is invoked via "go run" from src/run.bash or
// src/run.bat conditionally builds and runs the cmd/api tool.
//
// TODO(bradfitz): the "conditional" condition is always true.
// We should only do this if the user has the hg codereview extension
// enabled and verifies that the go.tools subrepo is checked out with
// a suitably recently version. In prep for the cmd/api rewrite.
package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"runtime"
	"strings"
)

// goToolsVersion is the git revision of the x/tools subrepo we need
// to build cmd/api.  This only needs to be updated whenever a go/types
// bug fix is needed by the cmd/api tool.
const goToolsVersion = "875ff2496f865e" // aka hg 6698ca2900e2

var goroot string

func main() {
	log.SetFlags(0)
	goroot = os.Getenv("GOROOT") // should be set by run.{bash,bat}
	if goroot == "" {
		log.Fatal("No $GOROOT set.")
	}
	_, err := exec.LookPath("git")
	if err != nil {
		fmt.Println("Skipping cmd/api checks; git not available")
		return
	}

	gopath := prepGoPath()

	cmd := exec.Command("go", "install", "--tags=api_tool", "cmd/api")
	cmd.Env = append(filterOut(os.Environ(), "GOARCH", "GOPATH"), "GOPATH="+gopath)
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Fatalf("Error installing cmd/api: %v\n%s", err, out)
	}

	out, err = exec.Command("go", "tool", "api",
		"-c", file("go1", "go1.1", "go1.2", "go1.3", "go1.4"),
		"-next", file("next"),
		"-except", file("except")).CombinedOutput()
	if err != nil {
		log.Fatalf("Error running API checker: %v\n%s", err, out)
	}
	fmt.Print(string(out))
}

// filterOut returns a copy of the src environment without environment
// variables from remove.
// TODO: delete when issue 6201 is fixed.
func filterOut(src []string, remove ...string) (out []string) {
S:
	for _, s := range src {
		for _, r := range remove {
			if strings.HasPrefix(s, r) && strings.HasPrefix(s, r+"=") {
				continue S
			}
		}
		out = append(out, s)
	}
	return
}

// file expands s to $GOROOT/api/s.txt.
// If there are more than 1, they're comma-separated.
func file(s ...string) string {
	if len(s) > 1 {
		return file(s[0]) + "," + file(s[1:]...)
	}
	return filepath.Join(goroot, "api", s[0]+".txt")
}

// prepGoPath returns a GOPATH for the "go" tool to compile the API tool with.
// It tries to re-use a go.tools checkout from a previous run if possible,
// else it hg clones it.
func prepGoPath() string {
	// Use a builder-specific temp directory name, so builders running
	// two copies don't trample on each other: https://golang.org/issue/9407
	// We don't use io.TempDir or a PID or timestamp here because we do
	// want this to be stable between runs, to minimize "git clone" calls
	// in the common case.
	var tempBase = fmt.Sprintf("go.tools.TMP.%s.%s", runtime.GOOS, runtime.GOARCH)

	username := ""
	u, err := user.Current()
	if err == nil {
		username = u.Username
	} else {
		username = os.Getenv("USER")
		if username == "" {
			username = "nobody"
		}
	}

	// The GOPATH we'll return
	gopath := filepath.Join(os.TempDir(), "gopath-api-"+cleanUsername(username)+"-"+cleanUsername(strings.Fields(runtime.Version())[0]), goToolsVersion)

	// cloneDir is where we run "git clone".
	cloneDir := filepath.Join(gopath, "src", "code.google.com", "p")

	// The dir we clone into. We only atomically rename it to finalDir on
	// clone success.
	tmpDir := filepath.Join(cloneDir, tempBase)

	// finalDir is where the checkout will live once it's complete.
	finalDir := filepath.Join(cloneDir, "go.tools")

	if goToolsCheckoutGood(finalDir) {
		return gopath
	}
	os.RemoveAll(finalDir) // in case it's there but corrupt
	os.RemoveAll(tmpDir)   // in case of aborted hg clone before

	if err := os.MkdirAll(cloneDir, 0700); err != nil {
		log.Fatal(err)
	}
	cmd := exec.Command("git", "clone", "https://go.googlesource.com/tools", tempBase)
	cmd.Dir = cloneDir
	out, err := cmd.CombinedOutput()
	if err != nil {
		if _, err := http.Head("http://ip.appspot.com/"); err != nil {
			log.Printf("# Skipping API check; network appears to be unavailable")
			os.Exit(0)
		}
		log.Fatalf("Error running git clone on x/tools: %v\n%s", err, out)
	}
	cmd = exec.Command("git", "reset", "--hard", goToolsVersion)
	cmd.Dir = tmpDir
	out, err = cmd.CombinedOutput()
	if err != nil {
		log.Fatalf("Error updating x/tools in %v to %v: %v, %s", tmpDir, goToolsVersion, err, out)
	}

	if err := os.Rename(tmpDir, finalDir); err != nil {
		log.Fatal(err)
	}
	return gopath
}

func cleanUsername(n string) string {
	b := make([]rune, len(n))
	for i, r := range n {
		if r == '\\' || r == '/' || r == ':' {
			b[i] = '_'
		} else {
			b[i] = r
		}
	}
	return string(b)
}

func goToolsCheckoutGood(dir string) bool {
	if _, err := os.Stat(dir); err != nil {
		return false
	}

	cmd := exec.Command("git", "rev-parse", "HEAD")
	cmd.Dir = dir
	out, err := cmd.Output()
	if err != nil {
		return false
	}
	id := strings.TrimSpace(string(out))
	if !strings.HasPrefix(id, goToolsVersion) {
		return false
	}

	cmd = exec.Command("git", "status", "--porcelain")
	cmd.Dir = dir
	out, err = cmd.Output()
	if err != nil || strings.TrimSpace(string(out)) != "" {
		return false
	}
	return true
}
<|endoftext|>


<commit_before>// Copyright 2018 The Go Cloud Development Kit Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package runtimevar_test

import (
	"context"
	"fmt"
	"log"

	"gocloud.dev/runtimevar"
	"gocloud.dev/runtimevar/constantvar"
)

func Example_jsonVariable() {
	// DBConfig is the sample config struct we're going to parse our JSON into.
	type DBConfig struct {
		Host     string
		Port     int
		Username string
	}

	// Here's our sample JSON config.
	const jsonConfig = `{"Host": "gocloud.dev", "Port": 8080, "Username": "testuser"}`

	// We need a Decoder that decodes raw bytes into our config.
	decoder := runtimevar.NewDecoder(DBConfig{}, runtimevar.JSONDecode)

	// Next, a construct a *Variable using a constructor from one of the
	// runtimevar subpackages. This example uses constantvar.
	v := constantvar.NewBytes([]byte(jsonConfig), decoder)
	defer v.Close()

	// Call Watch to retrieve the value.
	snapshot, err := v.Watch(context.Background())
	if err != nil {
		log.Fatalf("Error in retrieving variable: %v", err)
	}
	// snapshot.Value will be of type DBConfig.
	fmt.Printf("Config: %+v\n", snapshot.Value.(DBConfig))

	// Output:
	// Config: {Host:gocloud.dev Port:8080 Username:testuser}
}

func Example_stringVariable() {
	// Construct a *Variable using a constructor from one of the
	// runtimevar subpackages. This example uses constantvar.
	// The variable value is of type string, so we use StringDecoder.
	v := constantvar.NewBytes([]byte("hello world"), runtimevar.StringDecoder)
	defer v.Close()

	// Call Watch to retrieve the value.
	snapshot, err := v.Watch(context.Background())
	if err != nil {
		log.Fatalf("Error in retrieving variable: %v", err)
	}
	// snapshot.Value will be of type string.
	fmt.Printf("%q\n", snapshot.Value.(string))

	// Output:
	// "hello world"
}
<commit_msg>runtimevar: update examples to use Latest; add one for Watch (#1505)<commit_after>// Copyright 2018 The Go Cloud Development Kit Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package runtimevar_test

import (
	"context"
	"fmt"
	"log"

	"gocloud.dev/runtimevar"
	"gocloud.dev/runtimevar/constantvar"
)

func ExampleVariable_Latest_jsonVariable() {
	// DBConfig is the sample config struct we're going to parse our JSON into.
	type DBConfig struct {
		Host     string
		Port     int
		Username string
	}

	// Here's our sample JSON config.
	const jsonConfig = `{"Host": "gocloud.dev", "Port": 8080, "Username": "testuser"}`

	// We need a Decoder that decodes raw bytes into our config.
	decoder := runtimevar.NewDecoder(DBConfig{}, runtimevar.JSONDecode)

	// Next, a construct a *Variable using a constructor from one of the
	// runtimevar subpackages. This example uses constantvar.
	v := constantvar.NewBytes([]byte(jsonConfig), decoder)
	defer v.Close()

	// Call Latest to retrieve the value.
	snapshot, err := v.Latest(context.Background())
	if err != nil {
		log.Fatalf("Error in retrieving variable: %v", err)
	}
	// snapshot.Value will be of type DBConfig.
	fmt.Printf("Config: %+v\n", snapshot.Value.(DBConfig))

	// Output:
	// Config: {Host:gocloud.dev Port:8080 Username:testuser}
}

func ExampleVariable_Latest_stringVariable() {
	// Construct a *Variable using a constructor from one of the
	// runtimevar subpackages. This example uses constantvar.
	// The variable value is of type string, so we use StringDecoder.
	v := constantvar.NewBytes([]byte("hello world"), runtimevar.StringDecoder)
	defer v.Close()

	// Call Latest to retrieve the value.
	snapshot, err := v.Latest(context.Background())
	if err != nil {
		log.Fatalf("Error in retrieving variable: %v", err)
	}
	// snapshot.Value will be of type string.
	fmt.Printf("%q\n", snapshot.Value.(string))

	// Output:
	// "hello world"
}

func ExampleVariable_Watch() {
	// Construct a *Variable using a constructor from one of the
	// runtimevar subpackages. This example uses constantvar.
	// The variable value is of type string, so we use StringDecoder.
	v := constantvar.NewBytes([]byte("hello world"), runtimevar.StringDecoder)
	defer v.Close()

	// Call Watch in a loop from a background goroutine to see all changes,
	// including errors.
	//
	// You can use this for logging, or to trigger behaviors when the
	// config changes.
	//
	// Note that Latest always returns the latest "good" config, so seeing
	// an error from Watch doesn't mean that Latest will return one.
	go func() {
		for {
			snapshot, err := v.Watch(context.Background())
			if err == runtimevar.ErrClosed {
				// v has been closed; exit.
				return
			}
			if err == nil {
				// Casting to a string here because we used StringDecoder.
				log.Printf("New config: %v", snapshot.Value.(string))
			} else {
				log.Printf("Error loading config: %v", err)
				// Even though there's been an error loading the config,
				// v.Latest will continue to return the latest "good" value.
			}
		}
	}()
}
<|endoftext|>


<commit_before><commit_msg>reflect: make map test independent of map iteration order.<commit_after><|endoftext|>


<commit_before>package rtmptee

import (
	"fmt"
	"testing"
)

func TestBufPoolReferenceCounting(t *testing.T) {
	bp := NewBufPool(16, 64)

	// At least 16 buffers must be immediately available.
	elems := make([]*BufPoolElem, 16)
	for i := 0; i < 16; i++ {
		elems[i] = <-bp.C
		elems[i].AcquireFirst()
	}

	// We can increase/decrease the reference counter as long as it stays >0.
	elems[0].Acquire(1) // => 2
	elems[0].Acquire(2) // => 4
	elems[0].Free()     // => 3
	elems[0].Free()     // => 2
	elems[0].Free()     // => 1

	// After freeing one buffer, we should be able to get a new one immediately.
	elems[0].Free() // => 0
	elems[0] = <-bp.C
	elems[0].AcquireFirst()

	// If all buffers are taken, we can wait until one becomes free.
	go func(e *BufPoolElem) {
		e.Free()
	}(elems[0])
	elems[0] = <-bp.C
	elems[0].AcquireFirst()

	// If we free all 16 buffers, we can get 16 new ones.
	for _, elem := range elems {
		elem.Free()
	}
	for i, _ := range elems {
		elems[i] = <-bp.C
		elems[i].AcquireFirst()
	}
}

func TestBufPoolSizeManagement(t *testing.T) {
	bp := NewBufPool(16, 64)
	elems := make([]*BufPoolElem, 16)
	for i := 0; i < 16; i++ {
		elems[i] = <-bp.C
		elems[i].AcquireFirst()
	}

	// All buffers should have the right size.
	for _, elem := range elems {
		buf := elem.GetBuffer()
		if len(buf) != 64 {
			t.Fatalf(fmt.Sprintf("Buffer size was %d, expected 64", len(buf)))
		}
	}

	// We should be able to change the buffer size.
	elem := elems[0]
	elem.SetSize(23)
	actualNewLen := len(elem.GetBuffer())
	if actualNewLen != 23 {
		t.Fatalf("New buffer size was %d, expected 64", actualNewLen)
	}

	// We should be able to set it to the maximum again.
	elem.SetSize(64)
	actualNewLen = len(elem.GetBuffer())
	if actualNewLen != 64 {
		t.Fatalf("New buffer size was %d, expected 64", actualNewLen)
	}

	// But setting a too high limit causes panic.
	tooHighLimitTestOk := make(chan bool)
	go func(ok chan bool) {
		defer func() {
			ok <- recover() != nil
		}()
		elem.SetSize(65)
	}(tooHighLimitTestOk)
	if !<-tooHighLimitTestOk {
		t.Fatal("Recover() returned nil")
	}

	// Recycled buffers have the original size again.
	elem.Free()
	elem = <-bp.C
	elem.AcquireFirst()
	buf := elem.GetBuffer()
	if len(buf) != 64 {
		t.Fatalf(fmt.Sprintf("Buffer size was %d, expected 64", len(buf)))
	}
}
<commit_msg>Replace unnecessary fmt.Sprintf calls<commit_after>package rtmptee

import (
	"testing"
)

func TestBufPoolReferenceCounting(t *testing.T) {
	bp := NewBufPool(16, 64)

	// At least 16 buffers must be immediately available.
	elems := make([]*BufPoolElem, 16)
	for i := 0; i < 16; i++ {
		elems[i] = <-bp.C
		elems[i].AcquireFirst()
	}

	// We can increase/decrease the reference counter as long as it stays >0.
	elems[0].Acquire(1) // => 2
	elems[0].Acquire(2) // => 4
	elems[0].Free()     // => 3
	elems[0].Free()     // => 2
	elems[0].Free()     // => 1

	// After freeing one buffer, we should be able to get a new one immediately.
	elems[0].Free() // => 0
	elems[0] = <-bp.C
	elems[0].AcquireFirst()

	// If all buffers are taken, we can wait until one becomes free.
	go func(e *BufPoolElem) {
		e.Free()
	}(elems[0])
	elems[0] = <-bp.C
	elems[0].AcquireFirst()

	// If we free all 16 buffers, we can get 16 new ones.
	for _, elem := range elems {
		elem.Free()
	}
	for i, _ := range elems {
		elems[i] = <-bp.C
		elems[i].AcquireFirst()
	}
}

func TestBufPoolSizeManagement(t *testing.T) {
	bp := NewBufPool(16, 64)
	elems := make([]*BufPoolElem, 16)
	for i := 0; i < 16; i++ {
		elems[i] = <-bp.C
		elems[i].AcquireFirst()
	}

	// All buffers should have the right size.
	for _, elem := range elems {
		buf := elem.GetBuffer()
		if len(buf) != 64 {
			t.Fatalf("Buffer size was %d, expected 64", len(buf))
		}
	}

	// We should be able to change the buffer size.
	elem := elems[0]
	elem.SetSize(23)
	actualNewLen := len(elem.GetBuffer())
	if actualNewLen != 23 {
		t.Fatalf("New buffer size was %d, expected 64", actualNewLen)
	}

	// We should be able to set it to the maximum again.
	elem.SetSize(64)
	actualNewLen = len(elem.GetBuffer())
	if actualNewLen != 64 {
		t.Fatalf("New buffer size was %d, expected 64", actualNewLen)
	}

	// But setting a too high limit causes panic.
	tooHighLimitTestOk := make(chan bool)
	go func(ok chan bool) {
		defer func() {
			ok <- recover() != nil
		}()
		elem.SetSize(65)
	}(tooHighLimitTestOk)
	if !<-tooHighLimitTestOk {
		t.Fatal("Recover() returned nil")
	}

	// Recycled buffers have the original size again.
	elem.Free()
	elem = <-bp.C
	elem.AcquireFirst()
	buf := elem.GetBuffer()
	if len(buf) != 64 {
		t.Fatalf("Buffer size was %d, expected 64", len(buf))
	}
}
<|endoftext|>


<commit_before>// Copyright 2019 The grok_exporter Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fswatcher

import (
	"fmt"
	"github.com/sirupsen/logrus"
	"io"
	"os"
	"path/filepath"
	"syscall"
)

type watcher struct {
	fd int
}

type fileWithReader struct {
	file   *os.File
	reader *lineReader
}

func (w *watcher) unwatchDir(dir *Dir) error {
	// After calling eventProducerLoop.Close(), we need to call inotify_rm_watch()
	// in order to terminate the inotify loop. See eventProducerLoop.Close().
	success, err := syscall.InotifyRmWatch(w.fd, uint32(dir.wd))
	if success != 0 || err != nil {
		return fmt.Errorf("inotify_rm_watch(%q) failed: status=%v, err=%v", dir.path, success, err)
	} else {
		return nil
	}
}

func (w *watcher) Close() error {
	err := syscall.Close(w.fd)
	if err != nil {
		return fmt.Errorf("failed to close the inotify file descriptor: %v", err)
	} else {
		return nil
	}
}

func unwatchDirByEvent(t *fileTailer, event inotifyEvent) {
	watchedDirsAfter := make([]*Dir, 0, len(t.watchedDirs)-1)
	for _, existing := range t.watchedDirs {
		if existing.wd != int(event.Wd) {
			watchedDirsAfter = append(watchedDirsAfter, existing)
		}
	}
	t.watchedDirs = watchedDirsAfter
}

func (w *watcher) runFseventProducerLoop() fseventProducerLoop {
	return runInotifyLoop(w.fd)
}

func initWatcher() (fswatcher, Error) {
	fd, err := syscall.InotifyInit1(syscall.IN_CLOEXEC)
	if err != nil {
		return nil, NewError(NotSpecified, err, "inotify_init1() failed")
	}
	return &watcher{fd: fd}, nil
}

func (w *watcher) watchDir(path string) (*Dir, Error) {
	var (
		dir *Dir
		err error
		Err Error
	)
	dir, Err = newDir(path)
	if Err != nil {
		return nil, Err
	}
	dir.wd, err = syscall.InotifyAddWatch(w.fd, path, syscall.IN_MODIFY|syscall.IN_MOVED_FROM|syscall.IN_MOVED_TO|syscall.IN_DELETE|syscall.IN_CREATE)
	if err != nil {
		return nil, NewErrorf(NotSpecified, err, "%q: inotify_add_watch() failed", path)
	}
	return dir, nil
}

func newDir(path string) (*Dir, Error) {
	return &Dir{path: path}, nil
}

func (w *watcher) watchFile(_ fileMeta) Error {
	// nothing to do, because on Linux we watch the directory and don't need to watch individual files.
	return nil
}

func findDir(t *fileTailer, event inotifyEvent) *Dir {
	for _, dir := range t.watchedDirs {
		if dir.wd == int(event.Wd) {
			return dir
		}
	}
	return nil
}

func (w *watcher) processEvent(t *fileTailer, fsevent fsevent, log logrus.FieldLogger) Error {
	event, ok := fsevent.(inotifyEvent)
	if !ok {
		return NewErrorf(NotSpecified, nil, "received a file system event of unknown type %T", event)
	}
	dir := findDir(t, event)
	if dir == nil {
		return NewError(NotSpecified, nil, "watch list inconsistent: received a file system event for an unknown directory")
	}
	dirLogger := log.WithField("directory", dir.path)
	dirLogger.Debugf("received event: %v", event)
	if event.Mask&syscall.IN_IGNORED == syscall.IN_IGNORED {
		unwatchDirByEvent(t, event) // need to remove it from watchedDirs, because otherwise we close the removed dir on shutdown which causes an error
		return NewErrorf(NotSpecified, nil, "%s: directory was removed while being watched", dir.path)
	}
	if event.Mask&syscall.IN_MODIFY == syscall.IN_MODIFY {
		file, ok := t.watchedFiles[filepath.Join(dir.path, event.Name)]
		if !ok {
			return nil // unrelated file was modified
		}
		truncated, err := isTruncated(file.file)
		if err != nil {
			return NewErrorf(NotSpecified, err, "%v: seek() or stat() failed", file.file.Name())
		}
		if truncated {
			_, err = file.file.Seek(0, io.SeekStart)
			if err != nil {
				return NewErrorf(NotSpecified, err, "%v: seek() failed", file.file.Name())
			}
			file.reader.Clear()
		}
		readErr := t.readNewLines(file, dirLogger)
		if readErr != nil {
			return readErr
		}
	}
	if event.Mask&syscall.IN_MOVED_FROM == syscall.IN_MOVED_FROM || event.Mask&syscall.IN_DELETE == syscall.IN_DELETE || event.Mask&syscall.IN_CREATE == syscall.IN_CREATE || event.Mask&syscall.IN_MOVED_TO == syscall.IN_MOVED_TO {
		// There are a lot of corner cases here:
		// * a file is renamed, but still matches the pattern so we continue watching it (MOVED_FROM followed by MOVED_TO)
		// * a file is created overwriting an existing file
		// * a file is moved to the watched directory overwriting an existing file
		// Trying to figure out what happened from the events would be error prone.
		// Therefore, we don't care which of the above events we received, we just update our watched files with the current
		// state of the watched directory.
		err := t.syncFilesInDir(dir, true, dirLogger)
		if err != nil {
			return err
		}
	}
	return nil
}

func isTruncated(file *os.File) (bool, error) {
	currentPos, err := file.Seek(0, io.SeekCurrent)
	if err != nil {
		return false, err
	}
	fileInfo, err := file.Stat()
	if err != nil {
		return false, err
	}
	return currentPos > fileInfo.Size(), nil
}

func findSameFile(t *fileTailer, file os.FileInfo, _ string) (*fileWithReader, Error) {
	var (
		fileInfo os.FileInfo
		err      error
	)
	for _, watchedFile := range t.watchedFiles {
		fileInfo, err = watchedFile.file.Stat()
		if err != nil {
			return nil, NewErrorf(NotSpecified, err, "%v: stat failed", watchedFile.file.Name())
		}
		if os.SameFile(fileInfo, file) {
			return watchedFile, nil
		}
	}
	return nil, nil
}
<commit_msg>#124 improved error message for file system watcher<commit_after>// Copyright 2019 The grok_exporter Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fswatcher

import (
	"fmt"
	"github.com/sirupsen/logrus"
	"io"
	"os"
	"path/filepath"
	"strings"
	"syscall"
)

type watcher struct {
	fd int
}

type fileWithReader struct {
	file   *os.File
	reader *lineReader
}

func (w *watcher) unwatchDir(dir *Dir) error {
	// After calling eventProducerLoop.Close(), we need to call inotify_rm_watch()
	// in order to terminate the inotify loop. See eventProducerLoop.Close().
	success, err := syscall.InotifyRmWatch(w.fd, uint32(dir.wd))
	if success != 0 || err != nil {
		return fmt.Errorf("inotify_rm_watch(%q) failed: status=%v, err=%v", dir.path, success, err)
	} else {
		return nil
	}
}

func (w *watcher) Close() error {
	err := syscall.Close(w.fd)
	if err != nil {
		return fmt.Errorf("failed to close the inotify file descriptor: %v", err)
	} else {
		return nil
	}
}

func unwatchDirByEvent(t *fileTailer, event inotifyEvent) {
	watchedDirsAfter := make([]*Dir, 0, len(t.watchedDirs)-1)
	for _, existing := range t.watchedDirs {
		if existing.wd != int(event.Wd) {
			watchedDirsAfter = append(watchedDirsAfter, existing)
		}
	}
	t.watchedDirs = watchedDirsAfter
}

func (w *watcher) runFseventProducerLoop() fseventProducerLoop {
	return runInotifyLoop(w.fd)
}

func initWatcher() (fswatcher, Error) {
	fd, err := syscall.InotifyInit1(syscall.IN_CLOEXEC)
	if err != nil {
		return nil, NewError(NotSpecified, err, "inotify_init1() failed")
	}
	return &watcher{fd: fd}, nil
}

func (w *watcher) watchDir(path string) (*Dir, Error) {
	var (
		dir *Dir
		err error
		Err Error
	)
	dir, Err = newDir(path)
	if Err != nil {
		return nil, Err
	}
	dir.wd, err = syscall.InotifyAddWatch(w.fd, path, syscall.IN_MODIFY|syscall.IN_MOVED_FROM|syscall.IN_MOVED_TO|syscall.IN_DELETE|syscall.IN_CREATE)
	if err != nil {
		return nil, NewErrorf(NotSpecified, err, "%q: inotify_add_watch() failed", path)
	}
	return dir, nil
}

func newDir(path string) (*Dir, Error) {
	return &Dir{path: path}, nil
}

func (w *watcher) watchFile(_ fileMeta) Error {
	// nothing to do, because on Linux we watch the directory and don't need to watch individual files.
	return nil
}

func findDir(t *fileTailer, event inotifyEvent) (*Dir, Error) {
	for _, dir := range t.watchedDirs {
		if dir.wd == int(event.Wd) {
			return dir, nil
		}
	}
	// We should always find the directory in watchedDirs if we receive an inotifyEvent.
	// The following code should never be executed.
	// However, Github issue #124 is a bug report where we receive an inotifyEvent for an unwatched directory.
	// Create a detailed error message to help figuring out why this happens.
	var (
		errMsg strings.Builder
		first  = true
	)
	fmt.Fprintf(&errMsg, "watch list inconsistent: cannot find directory for file system event %q in the list of watched directories [", event.String())
	for _, dir := range t.watchedDirs {
		if !first {
			fmt.Fprintf(&errMsg, ", ")
		}
		fmt.Fprintf(&errMsg, "%q", dir.Path())
		first = false
	}
	fmt.Fprintf(&errMsg, "]")
	return nil, NewError(NotSpecified, nil, errMsg.String())
}

func (w *watcher) processEvent(t *fileTailer, fsevent fsevent, log logrus.FieldLogger) Error {
	event, ok := fsevent.(inotifyEvent)
	if !ok {
		return NewErrorf(NotSpecified, nil, "received a file system event of unknown type %T", event)
	}
	dir, Err := findDir(t, event)
	if Err != nil {
		return Err
	}
	dirLogger := log.WithField("directory", dir.path)
	dirLogger.Debugf("received event: %v", event)
	if event.Mask&syscall.IN_IGNORED == syscall.IN_IGNORED {
		unwatchDirByEvent(t, event) // need to remove it from watchedDirs, because otherwise we close the removed dir on shutdown which causes an error
		return NewErrorf(NotSpecified, nil, "%s: directory was removed while being watched", dir.path)
	}
	if event.Mask&syscall.IN_MODIFY == syscall.IN_MODIFY {
		file, ok := t.watchedFiles[filepath.Join(dir.path, event.Name)]
		if !ok {
			return nil // unrelated file was modified
		}
		truncated, err := isTruncated(file.file)
		if err != nil {
			return NewErrorf(NotSpecified, err, "%v: seek() or stat() failed", file.file.Name())
		}
		if truncated {
			_, err = file.file.Seek(0, io.SeekStart)
			if err != nil {
				return NewErrorf(NotSpecified, err, "%v: seek() failed", file.file.Name())
			}
			file.reader.Clear()
		}
		readErr := t.readNewLines(file, dirLogger)
		if readErr != nil {
			return readErr
		}
	}
	if event.Mask&syscall.IN_MOVED_FROM == syscall.IN_MOVED_FROM || event.Mask&syscall.IN_DELETE == syscall.IN_DELETE || event.Mask&syscall.IN_CREATE == syscall.IN_CREATE || event.Mask&syscall.IN_MOVED_TO == syscall.IN_MOVED_TO {
		// There are a lot of corner cases here:
		// * a file is renamed, but still matches the pattern so we continue watching it (MOVED_FROM followed by MOVED_TO)
		// * a file is created overwriting an existing file
		// * a file is moved to the watched directory overwriting an existing file
		// Trying to figure out what happened from the events would be error prone.
		// Therefore, we don't care which of the above events we received, we just update our watched files with the current
		// state of the watched directory.
		err := t.syncFilesInDir(dir, true, dirLogger)
		if err != nil {
			return err
		}
	}
	return nil
}

func isTruncated(file *os.File) (bool, error) {
	currentPos, err := file.Seek(0, io.SeekCurrent)
	if err != nil {
		return false, err
	}
	fileInfo, err := file.Stat()
	if err != nil {
		return false, err
	}
	return currentPos > fileInfo.Size(), nil
}

func findSameFile(t *fileTailer, file os.FileInfo, _ string) (*fileWithReader, Error) {
	var (
		fileInfo os.FileInfo
		err      error
	)
	for _, watchedFile := range t.watchedFiles {
		fileInfo, err = watchedFile.file.Stat()
		if err != nil {
			return nil, NewErrorf(NotSpecified, err, "%v: stat failed", watchedFile.file.Name())
		}
		if os.SameFile(fileInfo, file) {
			return watchedFile, nil
		}
	}
	return nil, nil
}
<|endoftext|>


<commit_before><commit_msg>Verify post-install jobs succeess during post-upgrade tests (#1976)<commit_after><|endoftext|>