<commit_before>package mqtt

import (
	"bytes"
	proto "github.com/huin/mqtt"
	"github.com/jeffallen/mqtt"
	"golang.org/x/net/context"
	"net"
)

type Config struct {
	Host string
	Port string
	User string
	Pass string
	CID  string
}

type MQTT struct {
	Config
	Messages chan Publish
	cl       *mqtt.ClientConn
}

type Publish struct {
	Topic string
	ID    uint16
	Msg   string
}

func New(cfg Config) *MQTT {
	return &MQTT{
		Config:   cfg,
		Messages: make(chan Publish),
	}
}

func (m *MQTT) Connect(ctx context.Context) error {
	conn, err := net.Dial("tcp", m.Host+":"+m.Port)

	if err != nil {
		panic(err)
	}

	m.cl = mqtt.NewClientConn(conn)
	m.cl.ClientId = m.CID

	err = m.cl.Connect(m.User, m.Pass)

	if err != nil {
		return err
	}

	go m.listen(ctx)

	return nil
}

func (m *MQTT) Subscribe(topic string) {
	if topic == "" {
		return
	}

	sub := []proto.TopicQos{
		{
			Topic: topic,
			Qos:   proto.QosAtLeastOnce,
		},
	}

	m.cl.Subscribe(sub)
}

func (m *MQTT) listen(ctx context.Context) {
Listen:
	for {
		select {
		case <-ctx.Done():
			break Listen
		case p := <-m.cl.Incoming:
			m.receive(p)
		}
	}
}

func (m *MQTT) receive(p *proto.Publish) {
	buf := new(bytes.Buffer)

	if err := p.Payload.WritePayload(buf); err != nil {
		panic(err)
	}

	pb := Publish{
		Topic: p.TopicName,
		ID:    p.MessageId,
		Msg:   buf.String(),
	}

	m.Messages <- pb
}
<commit_msg>Return error instead of panicing<commit_after>package mqtt

import (
	"bytes"
	proto "github.com/huin/mqtt"
	"github.com/jeffallen/mqtt"
	"golang.org/x/net/context"
	"net"
)

type Config struct {
	Host string
	Port string
	User string
	Pass string
	CID  string
}

type MQTT struct {
	Config
	Messages chan Publish
	cl       *mqtt.ClientConn
}

type Publish struct {
	Topic string
	ID    uint16
	Msg   string
}

func New(cfg Config) *MQTT {
	return &MQTT{
		Config:   cfg,
		Messages: make(chan Publish),
	}
}

func (m *MQTT) Connect(ctx context.Context) error {
	conn, err := net.Dial("tcp", m.Host+":"+m.Port)

	if err != nil {
		return err
	}

	m.cl = mqtt.NewClientConn(conn)
	m.cl.ClientId = m.CID

	err = m.cl.Connect(m.User, m.Pass)

	if err != nil {
		return err
	}

	go m.listen(ctx)

	return nil
}

func (m *MQTT) Subscribe(topic string) {
	if topic == "" {
		return
	}

	sub := []proto.TopicQos{
		{
			Topic: topic,
			Qos:   proto.QosAtLeastOnce,
		},
	}

	m.cl.Subscribe(sub)
}

func (m *MQTT) listen(ctx context.Context) {
Listen:
	for {
		select {
		case <-ctx.Done():
			break Listen
		case p := <-m.cl.Incoming:
			m.receive(p)
		}
	}
}

func (m *MQTT) receive(p *proto.Publish) {
	buf := new(bytes.Buffer)

	if err := p.Payload.WritePayload(buf); err != nil {
		panic(err)
	}

	pb := Publish{
		Topic: p.TopicName,
		ID:    p.MessageId,
		Msg:   buf.String(),
	}

	m.Messages <- pb
}
<|endoftext|>


<commit_before>// Copyright (c) 2013 Théo Crevon
//
// See the file LICENSE for copying permission.

/*
Package reflections provides high level abstractions above the
reflect library.

Reflect library is very low-level and as can be quite complex when it comes to do simple things like accessing a structure field value, a field tag...

The purpose of reflections package is to make developers life easier when it comes to introspect structures at runtime.
It's API is freely inspired from python language (getattr, setattr, hasattr...) and provides a simplified access to structure fields and tags.
*/
package reflections

import (
    "fmt"
    "errors"
    "unsafe"
    "reflect"
)

// GetField returns the value of the provided obj field. obj param
// has to be a struct type.
func GetField(obj interface{}, name string) (interface{}, error) {
    if !isStruct(obj) {
        return nil, errors.New("Cannot use GetField on a non-struct interface")
    }

    val := reflect.ValueOf(obj)
    value := val.FieldByName(name)

    if !value.IsValid() {
        errMsg := fmt.Sprintf("No such field: %s in obj", name)
        return nil, errors.New(errMsg)
    }

    return value.Interface(), nil
}

// SetField sets the provided obj field with provided value. obj param has
// to be a pointer to a struct, otherwise it will soundly fail. Provided
// value type should match with the struct field you're trying to set.
func SetField(obj interface{}, name string, value interface{}) error {
    // Fetch the field reflect.Value
    structValue := reflect.ValueOf(obj).Elem()
    structFieldValue := structValue.FieldByName(name)

    if !structFieldValue.IsValid() {
        errMsg := fmt.Sprintf("No such field: %s in obj", name)
        return errors.New(errMsg)
    }

    // If obj field value is not settable an error is thrown
    if !structFieldValue.CanSet() {
        errMsg := fmt.Sprintf("Cannot set %s field value", name)
        return errors.New(errMsg)
    }

    invalidTypeError := errors.New("Provided value type didn't match obj field type")

    switch value.(type) {
    case bool:
        if structFieldValue.Type().Kind() != reflect.Bool {
            return invalidTypeError
        }
        structFieldValue.SetBool(value.(bool))
    case int:
        if structFieldValue.Type().Kind() != reflect.Int64 {
            return invalidTypeError
        }
        structFieldValue.SetInt(value.(int64))
    case uint64:
        if structFieldValue.Type().Kind() != reflect.Uint64 {
            return invalidTypeError
        }
        structFieldValue.SetUint(value.(uint64))
    case float64:
        if structFieldValue.Type().Kind() != reflect.Float64 {
            return invalidTypeError
        }
        structFieldValue.SetFloat(value.(float64))
    case complex128:
        if structFieldValue.Type().Kind() != reflect.Complex128 {
            return invalidTypeError
        }
        structFieldValue.SetComplex(value.(complex128))
    case string:
        if structFieldValue.Type().Kind() != reflect.String {
            return invalidTypeError
        }
        structFieldValue.SetString(value.(string))
    case []byte:
        if structFieldValue.Type().Kind() != reflect.Slice {
            return invalidTypeError
        }
        structFieldValue.SetBytes(value.([]byte))
    case unsafe.Pointer:
        if structFieldValue.Type().Kind() != reflect.Ptr {
            return invalidTypeError
        }
        structFieldValue.SetPointer(value.(unsafe.Pointer))
    default:
        return errors.New("Unknow field type")
    }

    return nil
}

// HasField checks if the provided field name is part of a struct.
func HasField(obj interface{}, name string) (bool, error) {
    if !isStruct(obj) {
        return false, errors.New("Cannot use HasField on a non-struct interface")
    }

    structValue := reflect.TypeOf(obj)
    structField, ok := structValue.FieldByName(name)
    if !ok || !isExportableField(&structField) {
        return false, nil
    }

    return true, nil
}

// Fields returns the struct fields names list
func Fields(obj interface{}) ([]string, error) {
    if !isStruct(obj) {
        return nil, errors.New("Cannot use Fields on a non-struct interface")
    }

    structType := reflect.TypeOf(obj)
    fieldsCount := structType.NumField()

    var fields []string
    for i := 0; i < fieldsCount; i++ {
        field := structType.Field(i)
        if isExportableField(&field) {
            fields = append(fields, field.Name)
        }
    }

    return fields, nil
}

// Items returns the field - value struct pairs as a map
func Items(obj interface{}) (map[string]interface{}, error) {
    if !isStruct(obj) {
        return nil, errors.New("Cannot use Items on a non-struct interface")
    }

    structType := reflect.TypeOf(obj)
    structValue := reflect.ValueOf(obj)
    fieldsCount := structType.NumField()

    items := make(map[string]interface{})

    for i := 0; i < fieldsCount; i++ {
        field := structType.Field(i)
        fieldValue := structValue.Field(i)

        // Make sure only exportable and addressable fields are
        // returned by Items
        if isExportableField(&field) {
            items[field.Name] = fieldValue.Interface()
        }
    }

    return items, nil
}

// Tags lists the struct tag fields
func Tags(obj interface{}, key string) (map[string]string, error) {
    if !isStruct(obj) {
        return nil, errors.New("Cannot use Tags on a non-struct interface")
    }

    structType := reflect.TypeOf(obj)
    fieldsCount := structType.NumField()

    tags := make(map[string]string)

    for i := 0; i < fieldsCount; i++ {
        structField := structType.Field(i)

        if isExportableField(&structField) {
            tags[structField.Name] = structField.Tag.Get(key)
        }
    }

    return tags, nil
}

func isExportableField(field *reflect.StructField) bool {
    // golang variables must start with a letter,
    // so checking if first letter is within [a-z]
    // is sufficient here
    if field.Name[0] >= 97 && field.Name[0] <= 122 {
        return false
    }
    return true
}

func isStruct(obj interface{}) bool {
    return reflect.TypeOf(obj).Kind() == reflect.Struct
}

func isPointer(obj interface{}) bool {
    return reflect.TypeOf(obj).Kind() == reflect.Ptr
}
<commit_msg>Use fmt.Errorf<commit_after>// Copyright (c) 2013 Théo Crevon
//
// See the file LICENSE for copying permission.

/*
Package reflections provides high level abstractions above the
reflect library.

Reflect library is very low-level and as can be quite complex when it comes to do simple things like accessing a structure field value, a field tag...

The purpose of reflections package is to make developers life easier when it comes to introspect structures at runtime.
It's API is freely inspired from python language (getattr, setattr, hasattr...) and provides a simplified access to structure fields and tags.
*/
package reflections

import (
    "fmt"
    "errors"
    "unsafe"
    "reflect"
)

// GetField returns the value of the provided obj field. obj param
// has to be a struct type.
func GetField(obj interface{}, name string) (interface{}, error) {
    if !isStruct(obj) {
        return nil, errors.New("Cannot use GetField on a non-struct interface")
    }

    val := reflect.ValueOf(obj)
    value := val.FieldByName(name)

    if !value.IsValid() {
        return nil, fmt.Errorf("No such field: %s in obj", name)
    }

    return value.Interface(), nil
}

// SetField sets the provided obj field with provided value. obj param has
// to be a pointer to a struct, otherwise it will soundly fail. Provided
// value type should match with the struct field you're trying to set.
func SetField(obj interface{}, name string, value interface{}) error {
    // Fetch the field reflect.Value
    structValue := reflect.ValueOf(obj).Elem()
    structFieldValue := structValue.FieldByName(name)
    structFieldKind = structValue.Kind()

    if !structFieldValue.IsValid() {
        return fmt.Errorf("No such field: %s in obj", name)
    }

    // If obj field value is not settable an error is thrown
    if !structFieldValue.CanSet() {
        return fmt.Errorf("Cannot set %s field value", name)
    }

    invalidTypeError := errors.New("Provided value type didn't match obj field type")

    switch value.(type) {
    case bool:
        if structFieldValue.Type().Kind() != reflect.Bool {
            return invalidTypeError
        }
        structFieldValue.SetBool(value.(bool))
    case int:
        if structFieldValue.Type().Kind() != reflect.Int64 {
            return invalidTypeError
        }
        structFieldValue.SetInt(value.(int64))
    case uint64:
        if structFieldValue.Type().Kind() != reflect.Uint64 {
            return invalidTypeError
        }
        structFieldValue.SetUint(value.(uint64))
    case float64:
        if structFieldValue.Type().Kind() != reflect.Float64 {
            return invalidTypeError
        }
        structFieldValue.SetFloat(value.(float64))
    case complex128:
        if structFieldValue.Type().Kind() != reflect.Complex128 {
            return invalidTypeError
        }
        structFieldValue.SetComplex(value.(complex128))
    case string:
        if structFieldValue.Type().Kind() != reflect.String {
            return invalidTypeError
        }
        structFieldValue.SetString(value.(string))
    case []byte:
        if structFieldValue.Type().Kind() != reflect.Slice {
            return invalidTypeError
        }
        structFieldValue.SetBytes(value.([]byte))
    case unsafe.Pointer:
        if structFieldValue.Type().Kind() != reflect.Ptr {
            return invalidTypeError
        }
        structFieldValue.SetPointer(value.(unsafe.Pointer))
    default:
        return errors.New("Unknow field type")
    }

    return nil
}

// HasField checks if the provided field name is part of a struct.
func HasField(obj interface{}, name string) (bool, error) {
    if !isStruct(obj) {
        return false, errors.New("Cannot use HasField on a non-struct interface")
    }

    structValue := reflect.TypeOf(obj)
    structField, ok := structValue.FieldByName(name)
    if !ok || !isExportableField(&structField) {
        return false, nil
    }

    return true, nil
}

// Fields returns the struct fields names list
func Fields(obj interface{}) ([]string, error) {
    if !isStruct(obj) {
        return nil, errors.New("Cannot use Fields on a non-struct interface")
    }

    structType := reflect.TypeOf(obj)
    fieldsCount := structType.NumField()

    var fields []string
    for i := 0; i < fieldsCount; i++ {
        field := structType.Field(i)
        if isExportableField(&field) {
            fields = append(fields, field.Name)
        }
    }

    return fields, nil
}

// Items returns the field - value struct pairs as a map
func Items(obj interface{}) (map[string]interface{}, error) {
    if !isStruct(obj) {
        return nil, errors.New("Cannot use Items on a non-struct interface")
    }

    structType := reflect.TypeOf(obj)
    structValue := reflect.ValueOf(obj)
    fieldsCount := structType.NumField()

    items := make(map[string]interface{})

    for i := 0; i < fieldsCount; i++ {
        field := structType.Field(i)
        fieldValue := structValue.Field(i)

        // Make sure only exportable and addressable fields are
        // returned by Items
        if isExportableField(&field) {
            items[field.Name] = fieldValue.Interface()
        }
    }

    return items, nil
}

// Tags lists the struct tag fields
func Tags(obj interface{}, key string) (map[string]string, error) {
    if !isStruct(obj) {
        return nil, errors.New("Cannot use Tags on a non-struct interface")
    }

    structType := reflect.TypeOf(obj)
    fieldsCount := structType.NumField()

    tags := make(map[string]string)

    for i := 0; i < fieldsCount; i++ {
        structField := structType.Field(i)

        if isExportableField(&structField) {
            tags[structField.Name] = structField.Tag.Get(key)
        }
    }

    return tags, nil
}

func isExportableField(field *reflect.StructField) bool {
    // golang variables must start with a letter,
    // so checking if first letter is within [a-z]
    // is sufficient here
    if field.Name[0] >= 97 && field.Name[0] <= 122 {
        return false
    }
    return true
}

func isStruct(obj interface{}) bool {
    return reflect.TypeOf(obj).Kind() == reflect.Struct
}

func isPointer(obj interface{}) bool {
    return reflect.TypeOf(obj).Kind() == reflect.Ptr
}
<|endoftext|>


<commit_before>package cardinal

import (
	"sync"
	"time"

	"github.com/jasonmoo/bloom"
	"github.com/jasonmoo/bloom/scalable"
)

type (
	Cardinal struct {
		sync.Mutex

		buf []*Filter

		duration       time.Duration
		chunk_duration time.Duration
		last_t         time.Time
		i              int

		filter *Filter
	}

	Filter struct {
		bloom.Bloom
		uniques uint64
		total   uint64
	}
)

const (
	CHUNKS     = 10
	CHUNK_SIZE = 4096
)

func New(duration time.Duration) *Cardinal {

	buf := make([]*Filter, CHUNKS)

	// initialize with modest size to ensure
	for i, _ := range buf {
		buf[i] = &Filter{scalable.New(CHUNK_SIZE), 0, 0}
	}

	return &Cardinal{
		buf:            buf,
		filter:         buf[0],
		duration:       duration,
		chunk_duration: duration / CHUNKS,
	}

}

func (c *Cardinal) Add(token []byte) {

	c.Lock()

	t := time.Now().Truncate(c.chunk_duration)

	if !t.Equal(c.last_t) {
		c.last_t = t
		c.i++
		next_i := c.i % len(c.buf)
		// always create a new filter with the size of the previous
		// as the estimated number of items to minimize collisions
		c.buf[next_i] = &Filter{scalable.New(min(CHUNK_SIZE, c.filter.Count())), 0, 0}
		c.filter = c.buf[next_i]
	}

	// check all filters before incrementing
	if !c.check(token) {
		c.filter.Add(token)
		c.filter.uniques++
	}

	c.filter.total++

	c.Unlock()

}

func (c *Cardinal) Check(token []byte) (r bool) {
	c.Lock()
	r = c.check(token)
	c.Unlock()
	return
}

func (c *Cardinal) Cardinality() (r float64) {
	c.Lock()
	r = c.cardinality()
	c.Unlock()
	return
}

func (c *Cardinal) Count() (r uint64) {
	c.Lock()
	r = c.count()
	c.Unlock()
	return
}

func (c *Cardinal) Uniques() (r uint64) {
	c.Lock()
	r = c.uniques()
	c.Unlock()
	return
}

func (c *Cardinal) Duration() time.Duration {
	return c.duration
}

func (c *Cardinal) Reset() {
	c.Lock()
	for _, filter := range c.buf {
		filter.reset()
	}
	c.Unlock()
}

func (c *Cardinal) check(token []byte) bool {

	for _, filter := range c.buf {
		if filter.Check(token) {
			return true
		}
	}

	return false

}

func (c *Cardinal) cardinality() float64 {

	var uniques, total uint64

	for _, filter := range c.buf {
		uniques, total = uniques+filter.uniques, total+filter.total
	}

	return float64(uniques) / float64(total)

}

func (c *Cardinal) count() (total uint64) {

	for _, filter := range c.buf {
		total += filter.total
	}

	return

}

func (c *Cardinal) uniques() (uniques uint64) {

	for _, filter := range c.buf {
		uniques += filter.uniques
	}

	return

}

func (f *Filter) reset() {
	f.Bloom.Reset()
	f.uniques = 0
	f.total = 0
}

func min(a, b uint) uint {
	if a < b {
		return b
	}
	return a
}
<commit_msg>reverting to upstream bloom repo<commit_after>package cardinal

import (
	"sync"
	"time"

	"github.com/dataence/bloom"
	"github.com/dataence/bloom/scalable"
)

type (
	Cardinal struct {
		sync.Mutex

		buf []*Filter

		duration       time.Duration
		chunk_duration time.Duration
		last_t         time.Time
		i              int

		filter *Filter
	}

	Filter struct {
		bloom.Bloom
		uniques uint64
		total   uint64
	}
)

const (
	CHUNKS     = 10
	CHUNK_SIZE = 4096
)

func New(duration time.Duration) *Cardinal {

	buf := make([]*Filter, CHUNKS)

	// initialize with modest size to ensure
	for i, _ := range buf {
		buf[i] = &Filter{scalable.New(CHUNK_SIZE), 0, 0}
	}

	return &Cardinal{
		buf:            buf,
		filter:         buf[0],
		duration:       duration,
		chunk_duration: duration / CHUNKS,
	}

}

func (c *Cardinal) Add(token []byte) {

	c.Lock()

	t := time.Now().Truncate(c.chunk_duration)

	if !t.Equal(c.last_t) {
		c.last_t = t
		c.i++
		next_i := c.i % len(c.buf)
		// always create a new filter with the size of the previous
		// as the estimated number of items to minimize collisions
		c.buf[next_i] = &Filter{scalable.New(min(CHUNK_SIZE, c.filter.Count())), 0, 0}
		c.filter = c.buf[next_i]
	}

	// check all filters before incrementing
	if !c.check(token) {
		c.filter.Add(token)
		c.filter.uniques++
	}

	c.filter.total++

	c.Unlock()

}

func (c *Cardinal) Check(token []byte) (r bool) {
	c.Lock()
	r = c.check(token)
	c.Unlock()
	return
}

func (c *Cardinal) Cardinality() (r float64) {
	c.Lock()
	r = c.cardinality()
	c.Unlock()
	return
}

func (c *Cardinal) Count() (r uint64) {
	c.Lock()
	r = c.count()
	c.Unlock()
	return
}

func (c *Cardinal) Uniques() (r uint64) {
	c.Lock()
	r = c.uniques()
	c.Unlock()
	return
}

func (c *Cardinal) Duration() time.Duration {
	return c.duration
}

func (c *Cardinal) Reset() {
	c.Lock()
	for _, filter := range c.buf {
		filter.reset()
	}
	c.Unlock()
}

func (c *Cardinal) check(token []byte) bool {

	for _, filter := range c.buf {
		if filter.Check(token) {
			return true
		}
	}

	return false

}

func (c *Cardinal) cardinality() float64 {

	var uniques, total uint64

	for _, filter := range c.buf {
		uniques, total = uniques+filter.uniques, total+filter.total
	}

	return float64(uniques) / float64(total)

}

func (c *Cardinal) count() (total uint64) {

	for _, filter := range c.buf {
		total += filter.total
	}

	return

}

func (c *Cardinal) uniques() (uniques uint64) {

	for _, filter := range c.buf {
		uniques += filter.uniques
	}

	return

}

func (f *Filter) reset() {
	f.Bloom.Reset()
	f.uniques = 0
	f.total = 0
}

func min(a, b uint) uint {
	if a < b {
		return b
	}
	return a
}
<|endoftext|>


<commit_before>package blinkstick

import (
	"fmt"
	"image/color"

	"github.com/boombuler/hid"
)

// Version of Blinkstick
// One Line for this, used by release.sh script
// Keep "const Version on one line"
const Version = "0.0.13"

// VendorID blinkstick
const VendorID = 0x20a0

// ProductID blinkstick
const ProductID = 0x41e5

// USBDevice ...
type USBDevice struct {
	DeviceInfo *hid.DeviceInfo
	Device     *hid.Device
}

// Blinkstick represents a blinkstick device
type Blinkstick interface {
	List() []Blinkstick
	SetColor(color.Color) error
	GetDeviceInfo() *hid.DeviceInfo
	ListFilter(hid *hid.DeviceInfo) (bool, Blinkstick)
}

// SetColor set color
func (usbDevice *USBDevice) setColor(index byte, c color.Color) error {
	if usbDevice.Device == nil {
		if err := usbDevice.Open(); err != nil {
			return err
		}
	}
	r, g, b, _ := c.RGBA()
	d := *usbDevice.Device
	return d.WriteFeature([]byte{0x05, 0x00, index, byte(r >> 8), byte(g >> 8), byte(b >> 8)})
}

// Open open a device
func (usbDevice *USBDevice) Open() error {
	device, err := usbDevice.DeviceInfo.Open()
	if err != nil {
		return fmt.Errorf("Error while opening device: %s", err)
	}
	usbDevice.Device = &device
	return nil
}

// ListFilter is used to filter device on List
type ListFilter func(*hid.DeviceInfo) (bool, Blinkstick)

// List gets all blinkstick device
func List(opts ...ListFilter) []Blinkstick {
	out := []Blinkstick{}

	if len(opts) == 0 {
		opts = append(opts, Nano{}.ListFilter)
	}

	for di := range hid.Devices() {
		if di.VendorId == VendorID && di.ProductId == ProductID {
			for _, o := range opts {
				if toKeep, blinkstick := o(di); toKeep {
					out = append(out, blinkstick)
				}
			}

		}
	}
	return out
}
<commit_msg>[auto] bump version to v0.0.14<commit_after>package blinkstick

import (
	"fmt"
	"image/color"

	"github.com/boombuler/hid"
)

// Version of Blinkstick
// One Line for this, used by release.sh script
// Keep "const Version on one line"
const Version = "0.0.14"

// VendorID blinkstick
const VendorID = 0x20a0

// ProductID blinkstick
const ProductID = 0x41e5

// USBDevice ...
type USBDevice struct {
	DeviceInfo *hid.DeviceInfo
	Device     *hid.Device
}

// Blinkstick represents a blinkstick device
type Blinkstick interface {
	List() []Blinkstick
	SetColor(color.Color) error
	GetDeviceInfo() *hid.DeviceInfo
	ListFilter(hid *hid.DeviceInfo) (bool, Blinkstick)
}

// SetColor set color
func (usbDevice *USBDevice) setColor(index byte, c color.Color) error {
	if usbDevice.Device == nil {
		if err := usbDevice.Open(); err != nil {
			return err
		}
	}
	r, g, b, _ := c.RGBA()
	d := *usbDevice.Device
	return d.WriteFeature([]byte{0x05, 0x00, index, byte(r >> 8), byte(g >> 8), byte(b >> 8)})
}

// Open open a device
func (usbDevice *USBDevice) Open() error {
	device, err := usbDevice.DeviceInfo.Open()
	if err != nil {
		return fmt.Errorf("Error while opening device: %s", err)
	}
	usbDevice.Device = &device
	return nil
}

// ListFilter is used to filter device on List
type ListFilter func(*hid.DeviceInfo) (bool, Blinkstick)

// List gets all blinkstick device
func List(opts ...ListFilter) []Blinkstick {
	out := []Blinkstick{}

	if len(opts) == 0 {
		opts = append(opts, Nano{}.ListFilter)
	}

	for di := range hid.Devices() {
		if di.VendorId == VendorID && di.ProductId == ProductID {
			for _, o := range opts {
				if toKeep, blinkstick := o(di); toKeep {
					out = append(out, blinkstick)
				}
			}

		}
	}
	return out
}
<|endoftext|>


<commit_before>package routes

import (
	"fmt"
	"net/http"
	"reflect"
	"unsafe"

	"github.com/jeremija/peer-calls/src/server-go/iceauth"
	"github.com/jeremija/peer-calls/src/server-go/wrtc/signals"
	"github.com/jeremija/peer-calls/src/server-go/wrtc/tracks"
	"github.com/jeremija/peer-calls/src/server-go/ws/wsmessage"
	"github.com/jeremija/peer-calls/src/server-go/wshandler"
	"github.com/pion/webrtc/v2"
)

const localPeerID = "__SERVER__"

type TracksManager interface {
	Add(room string, clientID string, peerConnection tracks.PeerConnection, signaller tracks.Signaller)
}

const serverIsInitiator = true

func NewPeerToServerRoomHandler(
	wss *wshandler.WSS,
	iceServers []iceauth.ICEServer,
	tracksManager TracksManager,
) http.Handler {

	webrtcICEServers := []webrtc.ICEServer{}
	for _, iceServer := range iceServers {
		var c webrtc.ICECredentialType
		if iceServer.Username != "" && iceServer.Credential != "" {
			c = webrtc.ICECredentialTypePassword
		}
		webrtcICEServers = append(webrtcICEServers, webrtc.ICEServer{
			URLs:           iceServer.URLs,
			CredentialType: c,
			Username:       iceServer.Username,
			Credential:     iceServer.Credential,
		})
	}

	webrtcConfig := webrtc.Configuration{
		ICEServers: webrtcICEServers,
	}

	fn := func(w http.ResponseWriter, r *http.Request) {
		settingEngine := webrtc.SettingEngine{}
		// settingEngine.SetTrickle(true)
		api := webrtc.NewAPI(
			webrtc.WithMediaEngine(webrtc.MediaEngine{}),
			webrtc.WithSettingEngine(settingEngine),
		)

		// Hack to be able to update dynamic codec payload IDs with every new sdp
		// renegotiation of passive (non-server initiated) peer connections.
		field := reflect.ValueOf(api).Elem().FieldByName("mediaEngine")
		unsafeField := reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem()

		mediaEngine, ok := unsafeField.Interface().(*webrtc.MediaEngine)
		if !ok {
			log.Printf("Error in hack to obtain mediaEngine")
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		peerConnection, err := api.NewPeerConnection(webrtcConfig)

		if err != nil {
			log.Printf("Error creating peer connection: %s", err)
			w.WriteHeader(http.StatusInternalServerError)
			return
		}

		cleanup := func() {
			// TODO maybe cleanup is not necessary as we can still keep peer
			// connections after websocket conn closes
		}

		var signaller *signals.Signaller

		peerConnection.OnICEGatheringStateChange(func(state webrtc.ICEGathererState) {
			log.Printf("ICE gathering state changed: %s", state)
		})

		handleMessage := func(event wshandler.RoomEvent) {
			msg := event.Message
			adapter := event.Adapter
			room := event.Room
			clientID := event.ClientID

			initiator := localPeerID
			if !serverIsInitiator {
				initiator = clientID
			}

			var responseEventName string
			var err error

			switch msg.Type {
			case "ready":
				log.Printf("Initiator for clientID: %s is: %s", clientID, initiator)

				responseEventName = "users"
				err = adapter.Broadcast(
					wsmessage.NewMessage(responseEventName, room, map[string]interface{}{
						"initiator": initiator,
						// "initiator": clientID,
						"users": []User{{UserID: localPeerID, ClientID: localPeerID}},
					}),
				)

				if initiator == localPeerID {
					// need to do this to connect with simple peer
					// only when we are the initiator
					_, err = peerConnection.CreateDataChannel("test", nil)
					if err != nil {
						log.Printf("Error creating data channel")
						// TODO abort connection
					}
				}

				// TODO use this to get all client IDs and request all tracks of all users
				// adapter.Clients()
				if signaller == nil {
					signaller, err = signals.NewSignaller(
						initiator == localPeerID,
						peerConnection,
						mediaEngine,
						localPeerID,
						clientID,
						func(signal interface{}) {
							err := adapter.Emit(clientID, wsmessage.NewMessage("signal", room, signal))
							if err != nil {
								log.Printf("Error sending local signal to remote clientID: %s: %s", clientID, err)
								// TODO abort connection
							}
						},
					)
					if err != nil {
						err = fmt.Errorf("Error initializing signaller: %s", err)
						break
					}
					tracksManager.Add(room, clientID, peerConnection, signaller)
				}
			case "signal":
				payload, _ := msg.Payload.(map[string]interface{})
				if signaller == nil {
					err = fmt.Errorf("Ignoring signal because signaller is not initialized")
				} else {
					err = signaller.Signal(payload)
				}
			}

			if err != nil {
				log.Printf("Error handling event (event: %s, room: %s, source: %s): %s", msg.Type, room, clientID, err)
			}
		}

		wss.HandleRoomWithCleanup(w, r, handleMessage, cleanup)
	}
	return http.HandlerFunc(fn)
}
<commit_msg>Dp not broadcast when any user joins when using SFU<commit_after>package routes

import (
	"fmt"
	"net/http"
	"reflect"
	"unsafe"

	"github.com/jeremija/peer-calls/src/server-go/iceauth"
	"github.com/jeremija/peer-calls/src/server-go/wrtc/signals"
	"github.com/jeremija/peer-calls/src/server-go/wrtc/tracks"
	"github.com/jeremija/peer-calls/src/server-go/ws/wsmessage"
	"github.com/jeremija/peer-calls/src/server-go/wshandler"
	"github.com/pion/webrtc/v2"
)

const localPeerID = "__SERVER__"

type TracksManager interface {
	Add(room string, clientID string, peerConnection tracks.PeerConnection, signaller tracks.Signaller)
}

const serverIsInitiator = true

func NewPeerToServerRoomHandler(
	wss *wshandler.WSS,
	iceServers []iceauth.ICEServer,
	tracksManager TracksManager,
) http.Handler {

	webrtcICEServers := []webrtc.ICEServer{}
	for _, iceServer := range iceServers {
		var c webrtc.ICECredentialType
		if iceServer.Username != "" && iceServer.Credential != "" {
			c = webrtc.ICECredentialTypePassword
		}
		webrtcICEServers = append(webrtcICEServers, webrtc.ICEServer{
			URLs:           iceServer.URLs,
			CredentialType: c,
			Username:       iceServer.Username,
			Credential:     iceServer.Credential,
		})
	}

	webrtcConfig := webrtc.Configuration{
		ICEServers: webrtcICEServers,
	}

	fn := func(w http.ResponseWriter, r *http.Request) {
		settingEngine := webrtc.SettingEngine{}
		// settingEngine.SetTrickle(true)
		api := webrtc.NewAPI(
			webrtc.WithMediaEngine(webrtc.MediaEngine{}),
			webrtc.WithSettingEngine(settingEngine),
		)

		// Hack to be able to update dynamic codec payload IDs with every new sdp
		// renegotiation of passive (non-server initiated) peer connections.
		field := reflect.ValueOf(api).Elem().FieldByName("mediaEngine")
		unsafeField := reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem()

		mediaEngine, ok := unsafeField.Interface().(*webrtc.MediaEngine)
		if !ok {
			log.Printf("Error in hack to obtain mediaEngine")
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		peerConnection, err := api.NewPeerConnection(webrtcConfig)

		if err != nil {
			log.Printf("Error creating peer connection: %s", err)
			w.WriteHeader(http.StatusInternalServerError)
			return
		}

		cleanup := func() {
			// TODO maybe cleanup is not necessary as we can still keep peer
			// connections after websocket conn closes
		}

		var signaller *signals.Signaller

		peerConnection.OnICEGatheringStateChange(func(state webrtc.ICEGathererState) {
			log.Printf("ICE gathering state changed: %s", state)
		})

		handleMessage := func(event wshandler.RoomEvent) {
			msg := event.Message
			adapter := event.Adapter
			room := event.Room
			clientID := event.ClientID

			initiator := localPeerID
			if !serverIsInitiator {
				initiator = clientID
			}

			var responseEventName string
			var err error

			switch msg.Type {
			case "ready":
				log.Printf("Initiator for clientID: %s is: %s", clientID, initiator)

				responseEventName = "users"
				err = adapter.Emit(
					clientID,
					wsmessage.NewMessage(responseEventName, room, map[string]interface{}{
						"initiator": initiator,
						// "initiator": clientID,
						"users": []User{{UserID: localPeerID, ClientID: localPeerID}},
					}),
				)

				if initiator == localPeerID {
					// need to do this to connect with simple peer
					// only when we are the initiator
					_, err = peerConnection.CreateDataChannel("test", nil)
					if err != nil {
						log.Printf("Error creating data channel")
						// TODO abort connection
					}
				}

				// TODO use this to get all client IDs and request all tracks of all users
				// adapter.Clients()
				if signaller == nil {
					signaller, err = signals.NewSignaller(
						initiator == localPeerID,
						peerConnection,
						mediaEngine,
						localPeerID,
						clientID,
						func(signal interface{}) {
							err := adapter.Emit(clientID, wsmessage.NewMessage("signal", room, signal))
							if err != nil {
								log.Printf("Error sending local signal to remote clientID: %s: %s", clientID, err)
								// TODO abort connection
							}
						},
					)
					if err != nil {
						err = fmt.Errorf("Error initializing signaller: %s", err)
						break
					}
					tracksManager.Add(room, clientID, peerConnection, signaller)
				}
			case "signal":
				payload, _ := msg.Payload.(map[string]interface{})
				if signaller == nil {
					err = fmt.Errorf("Ignoring signal because signaller is not initialized")
				} else {
					err = signaller.Signal(payload)
				}
			}

			if err != nil {
				log.Printf("Error handling event (event: %s, room: %s, source: %s): %s", msg.Type, room, clientID, err)
			}
		}

		wss.HandleRoomWithCleanup(w, r, handleMessage, cleanup)
	}
	return http.HandlerFunc(fn)
}
<|endoftext|>


<commit_before>package server

import (
	"bytes"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"

	"github.com/pachyderm/pachyderm/src/client"
	pfsclient "github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/require"
)

func TestTensorFlow(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping integration tests in short mode")
	}

	cwd, err := os.Getwd()
	require.NoError(t, err)
	exampleDir := filepath.Join(cwd, "../../examples/tensor_flow")
	cmd := exec.Command("make", "all")
	cmd.Dir = exampleDir
	raw, err := cmd.CombinedOutput()
	require.NoError(t, err)

	cmd = exec.Command("pachctl", "list-commit", "GoT_scripts")
	cmd.Dir = exampleDir
	raw, err = cmd.CombinedOutput()
	lines := strings.Split(string(raw), "\n")
	require.Equal(t, 3, len(lines))

	getSecondField := func(line string) string {
		tokens := strings.Split(line, " ")
		seenField := 0
		for _, token := range tokens {
			if token != "" {
				seenField += 1
				if seenField == 2 {
					return token
				}
			}
		}
		return ""
	}

	// Make sure the second field is `ID`
	// Example stdout we're parsing:
	//
	// BRANCH                             ID                                 PARENT              STARTED             FINISHED            SIZE
	// c1001a97c0cc4bea825ee50cc613e039   5fc5a07edd094432acf474662ad02854   <none>              About an hour ago   About an hour ago   2.625 MiB

	require.Equal(t, "ID", getSecondField(lines[0]))
	inputCommitID := getSecondField(lines[1])
	require.NotEqual(t, "", inputCommitID)

	// Wait until the GoT_generate job has finished
	c := getPachClient(t)
	commitInfos, err := c.FlushCommit([]*pfsclient.Commit{client.NewCommit("GoT_scripts", inputCommitID)}, nil)
	require.NoError(t, err)
	require.Equal(t, 2, len(commitInfos))

	repos := []interface{}{"GoT_train", "GoT_generate"}
	var generateCommitID string
	for _, commitInfo := range commitInfos {
		require.EqualOneOf(t, repos, commitInfo.Commit.Repo.Name)
		if commitInfo.Commit.Repo.Name == "GoT_generate" {
			generateCommitID = commitInfo.Commit.ID
		}
	}

	// Make sure the final output is non zero
	var buffer bytes.Buffer
	require.NoError(t, c.GetFile("GoT_generate", generateCommitID, "new_script.txt", 0, 0, "", false, nil, &buffer))
	if buffer.Len() < 100 {
		t.Fatalf("Output GoT script is too small (has len=%v)", buffer.Len())
	}
	require.NoError(t, c.DeleteAll())
}
<commit_msg>Use client instead of CLI for this part of the test<commit_after>package server

import (
	"bytes"
	"os"
	"os/exec"
	"path/filepath"
	"testing"

	"github.com/pachyderm/pachyderm/src/client"
	pfsclient "github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/require"
)

func TestTensorFlow(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping integration tests in short mode")
	}

	c := getPachClient(t)

	cwd, err := os.Getwd()
	require.NoError(t, err)
	exampleDir := filepath.Join(cwd, "../../examples/tensor_flow")
	cmd := exec.Command("make", "all")
	cmd.Dir = exampleDir
	_, err = cmd.CombinedOutput()
	require.NoError(t, err)

	commitInfos, err := c.ListCommit([]string{"GoT_scripts"}, nil, client.CommitTypeRead, false, client.CommitStatusAll, nil)
	require.NoError(t, err)
	require.Equal(t, 1, len(commitInfos))
	inputCommitID := commitInfos[0].Commit.ID

	// Wait until the GoT_generate job has finished
	commitInfos, err = c.FlushCommit([]*pfsclient.Commit{client.NewCommit("GoT_scripts", inputCommitID)}, nil)
	require.NoError(t, err)
	require.Equal(t, 2, len(commitInfos))

	repos := []interface{}{"GoT_train", "GoT_generate"}
	var generateCommitID string
	for _, commitInfo := range commitInfos {
		require.EqualOneOf(t, repos, commitInfo.Commit.Repo.Name)
		if commitInfo.Commit.Repo.Name == "GoT_generate" {
			generateCommitID = commitInfo.Commit.ID
		}
	}

	// Make sure the final output is non zero
	var buffer bytes.Buffer
	require.NoError(t, c.GetFile("GoT_generate", generateCommitID, "new_script.txt", 0, 0, "", false, nil, &buffer))
	if buffer.Len() < 100 {
		t.Fatalf("Output GoT script is too small (has len=%v)", buffer.Len())
	}
	require.NoError(t, c.DeleteAll())
}
<|endoftext|>


<commit_before>// Package sync provides utility functions similar to `git pull/push` for PFS
package sync

import (
	"io"
	"os"
	"path/filepath"
	"sync"
	"syscall"

	pachclient "github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/limit"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/server/pkg/hashtree"
	"github.com/pachyderm/pachyderm/src/server/pkg/obj"

	"golang.org/x/sync/errgroup"
)

// Puller as a struct for managing a Pull operation.
type Puller struct {
	sync.Mutex
	// errCh contains an error from the pipe goros
	errCh chan error
	// pipes is a set containing all pipes that are currently blocking
	pipes map[string]bool
	// cleaned signals if the cleanup goroutine has been started
	cleaned bool
	// wg is used to wait for all goroutines associated with this Puller
	// to complete.
	wg sync.WaitGroup
}

// NewPuller creates a new Puller struct.
func NewPuller() *Puller {
	return &Puller{
		errCh: make(chan error, 1),
		pipes: make(map[string]bool),
	}
}

func (p *Puller) makePipe(client *pachclient.APIClient, path string, repo, commit, file string) error {
	if err := syscall.Mkfifo(path, 0666); err != nil {
		return err
	}
	func() {
		p.Lock()
		defer p.Unlock()
		p.pipes[path] = true
	}()
	// This goro will block until the user's code opens the
	// fifo.  That means we need to "abandon" this goro so that
	// the function can return and the caller can execute the
	// user's code. Waiting for this goro to return would
	// produce a deadlock. This goro will exit (if it hasn't already)
	// when CleanUp is called.
	p.wg.Add(1)
	go func() {
		defer p.wg.Done()
		if err := func() (retErr error) {
			f, err := os.OpenFile(path, os.O_WRONLY, os.ModeNamedPipe)
			if err != nil {
				return err
			}
			defer func() {
				if err := f.Close(); err != nil && retErr == nil {
					retErr = err
				}
			}()
			// If the CleanUp routine has already run, then there's
			// no point in downloading and sending the file, so we
			// exit early.
			if func() bool {
				p.Lock()
				defer p.Unlock()
				delete(p.pipes, path)
				return p.cleaned
			}() {
				return nil
			}
			return client.GetFile(repo, commit, file, 0, 0, f)
		}(); err != nil {
			select {
			case p.errCh <- err:
			default:
			}
		}
	}()
	return nil
}

func (p *Puller) makeFile(client *pachclient.APIClient, path, repo, commit, file string) (retErr error) {
	f, err := os.Create(path)
	if err != nil {
		return err
	}
	defer func() {
		if err := f.Close(); err != nil && retErr == nil {
			retErr = err
		}
	}()
	return client.GetFile(repo, commit, file, 0, 0, f)
}

// Pull clones an entire repo at a certain commit.
// root is the local path you want to clone to.
// fileInfo is the file/dir we are puuling.
// pipes causes the function to create named pipes in place of files, thus
// lazily downloading the data as it's needed.
func (p *Puller) Pull(client *pachclient.APIClient, root string, repo, commit, file string, pipes bool, concurrency int) error {
	limiter := limit.New(concurrency)
	var eg errgroup.Group
	if err := client.Walk(repo, commit, file, func(fileInfo *pfs.FileInfo) error {
		if fileInfo.FileType != pfs.FileType_FILE {
			return nil
		}
		basepath, err := filepath.Rel(file, fileInfo.File.Path)
		if err != nil {
			return err
		}
		path := filepath.Join(root, basepath)
		if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
			return err
		}
		if pipes {
			return p.makePipe(client, path, repo, commit, fileInfo.File.Path)
		}
		eg.Go(func() (retErr error) {
			limiter.Acquire()
			defer limiter.Release()
			return p.makeFile(client, path, repo, commit, fileInfo.File.Path)
		})
		return nil
	}); err != nil {
		return err
	}
	return eg.Wait()
}

// PullDiff is like Pull except that it materializes a Diff of the content
// rather than a the actual content. If newOnly is true then only new files
// will be downloaded and they will be downloaded under root. Otherwise new and
// old files will be downloaded under root/new and root/old respectively.
func (p *Puller) PullDiff(client *pachclient.APIClient, root string, newRepo, newCommit, newFile, oldRepo, oldCommit, oldFile string, newOnly bool, pipes bool, concurrency int) error {
	limiter := limit.New(concurrency)
	var eg errgroup.Group
	newFiles, oldFiles, err := client.DiffFile(newRepo, newCommit, newFile, oldRepo, oldCommit, oldFile)
	if err != nil {
		return err
	}
	for _, newFile := range newFiles {
		path := filepath.Join(root, newFile.File.Path)
		if newOnly {
			path = filepath.Join(root, "new", newFile.File.Path)
		}
		if pipes {
			if err := p.makePipe(client, path, newFile.File.Commit.Repo.Name, newFile.File.Commit.ID, newFile.File.Path); err != nil {
				return err
			}
		} else {
			newFile := newFile
			eg.Go(func() error {
				limiter.Acquire()
				defer limiter.Release()
				return p.makeFile(client, path, newFile.File.Commit.Repo.Name, newFile.File.Commit.ID, newFile.File.Path)
			})
		}
	}
	if !newOnly {
		for _, oldFile := range oldFiles {
			path := filepath.Join(root, "old", oldFile.File.Path)
			if pipes {
				if err := p.makePipe(client, path, oldFile.File.Commit.Repo.Name, oldFile.File.Commit.ID, oldFile.File.Path); err != nil {
					return err
				}
			} else {
				oldFile := oldFile
				eg.Go(func() error {
					limiter.Acquire()
					defer limiter.Release()
					return p.makeFile(client, path, oldFile.File.Commit.Repo.Name, oldFile.File.Commit.ID, oldFile.File.Path)
				})
			}
		}
	}
	return nil
}

func (p *Puller) PullTree(client *pachclient.APIClient, root string, tree hashtree.HashTree, pipes bool, concurrency int) error {
	limiter := limit.New(concurrency)
	var eg errgroup.Group
	if err := tree.Walk(func(path string, node *hashtree.NodeProto) error {
		if node.FileNode != nil {
			path := filepath.Join(root, path)
			var hashes []string
			for _, object := range node.FileNode.Objects {
				hashes = append(hashes, object.Hash)
			}
			if pipes {
				if err := syscall.Mkfifo(path, 0666); err != nil {
					return err
				}
				func() {
					p.Lock()
					defer p.Unlock()
					p.pipes[path] = true
				}()
				// This goro will block until the user's code opens the
				// fifo.  That means we need to "abandon" this goro so that
				// the function can return and the caller can execute the
				// user's code. Waiting for this goro to return would
				// produce a deadlock. This goro will exit (if it hasn't already)
				// when CleanUp is called.
				p.wg.Add(1)
				go func() {
					defer p.wg.Done()
					if err := func() (retErr error) {
						f, err := os.OpenFile(path, os.O_WRONLY, os.ModeNamedPipe)
						if err != nil {
							return err
						}
						defer func() {
							if err := f.Close(); err != nil && retErr == nil {
								retErr = err
							}
						}()
						// If the CleanUp routine has already run, then there's
						// no point in downloading and sending the file, so we
						// exit early.
						if func() bool {
							p.Lock()
							defer p.Unlock()
							delete(p.pipes, path)
							return p.cleaned
						}() {
							return nil
						}
						return client.GetObjects(hashes, 0, 0, f)
					}(); err != nil {
						select {
						case p.errCh <- err:
						default:
						}
					}
				}()
			} else {
				limiter.Acquire()
				defer limiter.Release()
				eg.Go(func() (retErr error) {
					f, err := os.Create(path)
					if err != nil {
						return err
					}
					defer func() {
						if err := f.Close(); err != nil && retErr == nil {
							retErr = err
						}
					}()
					return client.GetObjects(hashes, 0, 0, f)
				})
			}
		}
		return nil
	}); err != nil {
		return err
	}
	return eg.Wait()
}

// CleanUp cleans up blocked syscalls for pipes that were never opened. It also
// returns any errors that might have been encountered while trying to read
// data for the pipes. CleanUp should be called after all code that might
// access pipes has completed running, it should not be called concurrently.
func (p *Puller) CleanUp() error {
	var result error
	select {
	case result = <-p.errCh:
	default:
	}

	// Open all the pipes to unblock the goros
	var pipes []io.Closer
	func() {
		p.Lock()
		defer p.Unlock()
		p.cleaned = true
		for path := range p.pipes {
			f, err := os.OpenFile(path, syscall.O_NONBLOCK+os.O_RDONLY, os.ModeNamedPipe)
			if err != nil && result == nil {
				result = err
			}
			pipes = append(pipes, f)
		}
		p.pipes = make(map[string]bool)
	}()

	// Wait for all goros to exit
	p.wg.Wait()

	// Close the pipes
	for _, pipe := range pipes {
		if err := pipe.Close(); err != nil && result == nil {
			result = err
		}
	}
	return result
}

// Push puts files under root into an open commit.
func Push(client *pachclient.APIClient, root string, commit *pfs.Commit, overwrite bool) error {
	var g errgroup.Group
	if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		g.Go(func() (retErr error) {
			if path == root || info.IsDir() {
				return nil
			}

			f, err := os.Open(path)
			if err != nil {
				return err
			}
			defer func() {
				if err := f.Close(); err != nil && retErr == nil {
					retErr = err
				}
			}()

			relPath, err := filepath.Rel(root, path)
			if err != nil {
				return err
			}

			if overwrite {
				if err := client.DeleteFile(commit.Repo.Name, commit.ID, relPath); err != nil {
					return err
				}
			}

			_, err = client.PutFile(commit.Repo.Name, commit.ID, relPath, f)
			return err
		})
		return nil
	}); err != nil {
		return err
	}

	return g.Wait()
}

// PushObj pushes data from commit to an object store.
func PushObj(pachClient pachclient.APIClient, commit *pfs.Commit, objClient obj.Client, root string) error {
	var eg errgroup.Group
	sem := make(chan struct{}, 200)
	if err := pachClient.Walk(commit.Repo.Name, commit.ID, "", func(fileInfo *pfs.FileInfo) error {
		if fileInfo.FileType != pfs.FileType_FILE {
			return nil
		}
		eg.Go(func() (retErr error) {
			sem <- struct{}{}
			defer func() { <-sem }()
			w, err := objClient.Writer(filepath.Join(root, fileInfo.File.Path))
			if err != nil {
				return err
			}
			defer func() {
				if err := w.Close(); err != nil && retErr == nil {
					retErr = err
				}
			}()
			return pachClient.GetFile(commit.Repo.Name, commit.ID, fileInfo.File.Path, 0, 0, w)
		})
		return nil
	}); err != nil {
		return err
	}
	return eg.Wait()
}
<commit_msg>Refactor sync pkg to dedupe some code.<commit_after>// Package sync provides utility functions similar to `git pull/push` for PFS
package sync

import (
	"io"
	"os"
	"path/filepath"
	"sync"
	"syscall"

	pachclient "github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/limit"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/server/pkg/hashtree"
	"github.com/pachyderm/pachyderm/src/server/pkg/obj"

	"golang.org/x/sync/errgroup"
)

// Puller as a struct for managing a Pull operation.
type Puller struct {
	sync.Mutex
	// errCh contains an error from the pipe goros
	errCh chan error
	// pipes is a set containing all pipes that are currently blocking
	pipes map[string]bool
	// cleaned signals if the cleanup goroutine has been started
	cleaned bool
	// wg is used to wait for all goroutines associated with this Puller
	// to complete.
	wg sync.WaitGroup
}

// NewPuller creates a new Puller struct.
func NewPuller() *Puller {
	return &Puller{
		errCh: make(chan error, 1),
		pipes: make(map[string]bool),
	}
}

func (p *Puller) makePipe(path string, f func(io.Writer) error) error {
	if err := syscall.Mkfifo(path, 0666); err != nil {
		return err
	}
	func() {
		p.Lock()
		defer p.Unlock()
		p.pipes[path] = true
	}()
	// This goro will block until the user's code opens the
	// fifo.  That means we need to "abandon" this goro so that
	// the function can return and the caller can execute the
	// user's code. Waiting for this goro to return would
	// produce a deadlock. This goro will exit (if it hasn't already)
	// when CleanUp is called.
	p.wg.Add(1)
	go func() {
		defer p.wg.Done()
		if err := func() (retErr error) {
			file, err := os.OpenFile(path, os.O_WRONLY, os.ModeNamedPipe)
			if err != nil {
				return err
			}
			defer func() {
				if err := file.Close(); err != nil && retErr == nil {
					retErr = err
				}
			}()
			// If the CleanUp routine has already run, then there's
			// no point in downloading and sending the file, so we
			// exit early.
			if func() bool {
				p.Lock()
				defer p.Unlock()
				delete(p.pipes, path)
				return p.cleaned
			}() {
				return nil
			}
			return f(file)
		}(); err != nil {
			select {
			case p.errCh <- err:
			default:
			}
		}
	}()
	return nil
}

func (p *Puller) makeFile(path string, f func(io.Writer) error) (retErr error) {
	file, err := os.Create(path)
	if err != nil {
		return err
	}
	defer func() {
		if err := file.Close(); err != nil && retErr == nil {
			retErr = err
		}
	}()
	return f(file)
}

// Pull clones an entire repo at a certain commit.
// root is the local path you want to clone to.
// fileInfo is the file/dir we are puuling.
// pipes causes the function to create named pipes in place of files, thus
// lazily downloading the data as it's needed.
func (p *Puller) Pull(client *pachclient.APIClient, root string, repo, commit, file string, pipes bool, concurrency int) error {
	limiter := limit.New(concurrency)
	var eg errgroup.Group
	if err := client.Walk(repo, commit, file, func(fileInfo *pfs.FileInfo) error {
		if fileInfo.FileType != pfs.FileType_FILE {
			return nil
		}
		basepath, err := filepath.Rel(file, fileInfo.File.Path)
		if err != nil {
			return err
		}
		path := filepath.Join(root, basepath)
		if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
			return err
		}
		if pipes {
			return p.makePipe(path, func(w io.Writer) error {
				return client.GetFile(repo, commit, fileInfo.File.Path, 0, 0, w)
			})
		}
		eg.Go(func() (retErr error) {
			limiter.Acquire()
			defer limiter.Release()
			return p.makeFile(path, func(w io.Writer) error {
				return client.GetFile(repo, commit, fileInfo.File.Path, 0, 0, w)
			})
		})
		return nil
	}); err != nil {
		return err
	}
	return eg.Wait()
}

// PullDiff is like Pull except that it materializes a Diff of the content
// rather than a the actual content. If newOnly is true then only new files
// will be downloaded and they will be downloaded under root. Otherwise new and
// old files will be downloaded under root/new and root/old respectively.
func (p *Puller) PullDiff(client *pachclient.APIClient, root string, newRepo, newCommit, newFile, oldRepo, oldCommit, oldFile string, newOnly bool, pipes bool, concurrency int) error {
	limiter := limit.New(concurrency)
	var eg errgroup.Group
	newFiles, oldFiles, err := client.DiffFile(newRepo, newCommit, newFile, oldRepo, oldCommit, oldFile)
	if err != nil {
		return err
	}
	for _, newFile := range newFiles {
		path := filepath.Join(root, newFile.File.Path)
		if newOnly {
			path = filepath.Join(root, "new", newFile.File.Path)
		}
		if pipes {
			if err := p.makePipe(path, func(w io.Writer) error {
				return client.GetFile(newFile.File.Commit.Repo.Name, newFile.File.Commit.ID, newFile.File.Path, 0, 0, w)
			}); err != nil {
				return err
			}
		} else {
			newFile := newFile
			limiter.Acquire()
			eg.Go(func() error {
				defer limiter.Release()
				return p.makeFile(path, func(w io.Writer) error {
					return client.GetFile(newFile.File.Commit.Repo.Name, newFile.File.Commit.ID, newFile.File.Path, 0, 0, w)
				})
			})
		}
	}
	if !newOnly {
		for _, oldFile := range oldFiles {
			path := filepath.Join(root, "old", oldFile.File.Path)
			if pipes {
				if err := p.makePipe(path, func(w io.Writer) error {
					return client.GetFile(oldFile.File.Commit.Repo.Name, oldFile.File.Commit.ID, oldFile.File.Path, 0, 0, w)
				}); err != nil {
					return err
				}
			} else {
				oldFile := oldFile
				limiter.Acquire()
				eg.Go(func() error {
					defer limiter.Release()
					return p.makeFile(path, func(w io.Writer) error {
						return client.GetFile(oldFile.File.Commit.Repo.Name, oldFile.File.Commit.ID, oldFile.File.Path, 0, 0, w)
					})
				})
			}
		}
	}
	return nil
}

func (p *Puller) PullTree(client *pachclient.APIClient, root string, tree hashtree.HashTree, pipes bool, concurrency int) error {
	limiter := limit.New(concurrency)
	var eg errgroup.Group
	if err := tree.Walk(func(path string, node *hashtree.NodeProto) error {
		if node.FileNode != nil {
			path := filepath.Join(root, path)
			var hashes []string
			for _, object := range node.FileNode.Objects {
				hashes = append(hashes, object.Hash)
			}
			if pipes {
				return p.makePipe(path, func(w io.Writer) error {
					return client.GetObjects(hashes, 0, 0, w)
				})
			} else {
				limiter.Acquire()
				eg.Go(func() (retErr error) {
					defer limiter.Release()
					return p.makeFile(path, func(w io.Writer) error {
						return client.GetObjects(hashes, 0, 0, w)
					})
				})
			}
		}
		return nil
	}); err != nil {
		return err
	}
	return eg.Wait()
}

// CleanUp cleans up blocked syscalls for pipes that were never opened. It also
// returns any errors that might have been encountered while trying to read
// data for the pipes. CleanUp should be called after all code that might
// access pipes has completed running, it should not be called concurrently.
func (p *Puller) CleanUp() error {
	var result error
	select {
	case result = <-p.errCh:
	default:
	}

	// Open all the pipes to unblock the goros
	var pipes []io.Closer
	func() {
		p.Lock()
		defer p.Unlock()
		p.cleaned = true
		for path := range p.pipes {
			f, err := os.OpenFile(path, syscall.O_NONBLOCK+os.O_RDONLY, os.ModeNamedPipe)
			if err != nil && result == nil {
				result = err
			}
			pipes = append(pipes, f)
		}
		p.pipes = make(map[string]bool)
	}()

	// Wait for all goros to exit
	p.wg.Wait()

	// Close the pipes
	for _, pipe := range pipes {
		if err := pipe.Close(); err != nil && result == nil {
			result = err
		}
	}
	return result
}

// Push puts files under root into an open commit.
func Push(client *pachclient.APIClient, root string, commit *pfs.Commit, overwrite bool) error {
	var g errgroup.Group
	if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		g.Go(func() (retErr error) {
			if path == root || info.IsDir() {
				return nil
			}

			f, err := os.Open(path)
			if err != nil {
				return err
			}
			defer func() {
				if err := f.Close(); err != nil && retErr == nil {
					retErr = err
				}
			}()

			relPath, err := filepath.Rel(root, path)
			if err != nil {
				return err
			}

			if overwrite {
				if err := client.DeleteFile(commit.Repo.Name, commit.ID, relPath); err != nil {
					return err
				}
			}

			_, err = client.PutFile(commit.Repo.Name, commit.ID, relPath, f)
			return err
		})
		return nil
	}); err != nil {
		return err
	}

	return g.Wait()
}

// PushObj pushes data from commit to an object store.
func PushObj(pachClient pachclient.APIClient, commit *pfs.Commit, objClient obj.Client, root string) error {
	var eg errgroup.Group
	sem := make(chan struct{}, 200)
	if err := pachClient.Walk(commit.Repo.Name, commit.ID, "", func(fileInfo *pfs.FileInfo) error {
		if fileInfo.FileType != pfs.FileType_FILE {
			return nil
		}
		eg.Go(func() (retErr error) {
			sem <- struct{}{}
			defer func() { <-sem }()
			w, err := objClient.Writer(filepath.Join(root, fileInfo.File.Path))
			if err != nil {
				return err
			}
			defer func() {
				if err := w.Close(); err != nil && retErr == nil {
					retErr = err
				}
			}()
			return pachClient.GetFile(commit.Repo.Name, commit.ID, fileInfo.File.Path, 0, 0, w)
		})
		return nil
	}); err != nil {
		return err
	}
	return eg.Wait()
}
<|endoftext|>


<commit_before>package cluster

import (
	"context"
	"sync"
	"time"

	log "gopkg.in/inconshreveable/log15.v2"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/endpoints"
	"github.com/lxc/lxd/lxd/revert"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/logger"
)

// eventHubMinHosts is the minimum number of members that must have the event-hub role to trigger switching into
// event-hub mode (where cluster members will only connect to event-hub members rather than all members when
// operating in the normal full-mesh mode).
const eventHubMinHosts = 2

// EventMode indicates the event distribution mode.
type EventMode string

// EventModeFullMesh is when every cluster member connects to every other cluster member to pull events.
const EventModeFullMesh EventMode = "full-mesh"

// EventModeHubServer is when the cluster is operating in event-hub mode and this server is designated as a hub
// server, meaning that it will only connect to the other event-hub members and not other members.
const EventModeHubServer EventMode = "hub-server"

// EventModeHubClient is when the cluster is operating in event-hub mode and this member is designated as a hub
// client, meaning that it is expected to connect to the event-hub members.
const EventModeHubClient EventMode = "hub-client"

var listeners = map[string]*lxd.EventListener{}
var listenersNotify = map[chan struct{}][]string{}
var listenersLock sync.Mutex
var listenersUpdateLock sync.Mutex

// ServerEventMode returns the event distribution mode that this local server is operating in.
func ServerEventMode() EventMode {
	listenersLock.Lock()
	defer listenersLock.Unlock()

	return eventMode
}

// EventListenerWait waits for there to be listener connected to the specified address, or one of the event hubs
// if operating in event hub mode.
func EventListenerWait(ctx context.Context, address string) error {
	// Check if there is already a listener.
	listenersLock.Lock()
	listener, found := listeners[address]
	if found && listener.IsActive() {
		listenersLock.Unlock()
		return nil
	}

	listenAddresses := []string{address}

	// If not setup a notification for when the desired address or any of the event hubs connect.
	connected := make(chan struct{})
	listenersNotify[connected] = listenAddresses
	listenersLock.Unlock()

	defer func() {
		listenersLock.Lock()
		delete(listenersNotify, connected)
		listenersLock.Unlock()
	}()

	// Wait for the connected channel to be closed (indicating a new listener has been connected), and return.
	select {
	case <-connected:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}

// EventsUpdateListeners refreshes the cluster event listener connections.
func EventsUpdateListeners(endpoints *endpoints.Endpoints, cluster *db.Cluster, serverCert func() *shared.CertInfo, members map[int64]APIHeartbeatMember, f func(int64, api.Event)) {
	listenersUpdateLock.Lock()
	defer listenersUpdateLock.Unlock()

	// If no heartbeat members provided, populate from global database.
	if members == nil {
		var dbMembers []db.NodeInfo
		var offlineThreshold time.Duration

		err := cluster.Transaction(func(tx *db.ClusterTx) error {
			var err error

			dbMembers, err = tx.GetNodes()
			if err != nil {
				return err
			}

			offlineThreshold, err = tx.GetNodeOfflineThreshold()
			if err != nil {
				return err
			}

			return nil
		})
		if err != nil {
			logger.Warn("Failed to get current cluster members", log.Ctx{"err": err})
			return
		}

		members = make(map[int64]APIHeartbeatMember, len(dbMembers))
		for _, dbMember := range dbMembers {
			members[dbMember.ID] = APIHeartbeatMember{
				ID:            dbMember.ID,
				Name:          dbMember.Name,
				Address:       dbMember.Address,
				LastHeartbeat: dbMember.Heartbeat,
				Online:        !dbMember.IsOffline(offlineThreshold),
				Roles:         dbMember.Roles,
			}
		}
	}

	networkAddress := endpoints.NetworkAddress()

	keepListeners := make(map[string]struct{})
	wg := sync.WaitGroup{}
	for _, member := range members {
		// Don't bother trying to connect to ourselves or offline members.
		if member.Address == networkAddress || !member.Online {
			continue
		}

		listenersLock.Lock()
		listener, ok := listeners[member.Address]

		// If the member already has a listener associated to it, check that the listener is still active.
		// If it is, just move on to next member, but if not then we'll try to connect again.
		if ok {
			if listener.IsActive() {
				keepListeners[member.Address] = struct{}{} // Add to current listeners list.
				listenersLock.Unlock()
				continue
			}

			// Disconnect and delete listener, but don't delete any listenersNotify entry as there
			// might be something waiting for a future connection.
			listener.Disconnect()
			delete(listeners, member.Address)
			logger.Info("Removed inactive member event listener client", log.Ctx{"local": networkAddress, "remote": member.Address})
		}
		listenersLock.Unlock()

		keepListeners[member.Address] = struct{}{} // Add to current listeners list.

		// Connect to remote concurrently and add to active listeners if successful.
		wg.Add(1)
		go func(m APIHeartbeatMember) {
			defer wg.Done()
			listener, err := eventsConnect(m.Address, endpoints.NetworkCert(), serverCert())
			if err != nil {
				logger.Warn("Failed adding member event listener client", log.Ctx{"local": networkAddress, "remote": m.Address, "err": err})
				return
			}

			listener.AddHandler(nil, func(event api.Event) { f(m.ID, event) })

			listenersLock.Lock()
			listeners[m.Address] = listener

			// Indicate to any notifiers waiting for this member's address that it is connected.
			for connected, notifyAddresses := range listenersNotify {
				if shared.StringInSlice(m.Address, notifyAddresses) {
					close(connected)
					delete(listenersNotify, connected)
				}
			}

			logger.Info("Added member event listener client", log.Ctx{"local": networkAddress, "remote": m.Address})
			listenersLock.Unlock()
		}(member)
	}

	wg.Wait()

	// Disconnect and delete any out of date listeners and their notifiers.
	listenersLock.Lock()
	for address, listener := range listeners {
		if _, found := keepListeners[address]; !found {
			listener.Disconnect()
			delete(listeners, address)
			logger.Info("Removed old member event listener client", log.Ctx{"local": networkAddress, "remote": address})
		}
	}
	listenersLock.Unlock()
}

// Establish a client connection to get events from the given node.
func eventsConnect(address string, networkCert *shared.CertInfo, serverCert *shared.CertInfo) (*lxd.EventListener, error) {
	client, err := Connect(address, networkCert, serverCert, nil, true)
	if err != nil {
		return nil, err
	}

	revert := revert.New()
	revert.Add(func() {
		client.Disconnect()
	})

	listener, err := client.GetEventsAllProjects()
	if err != nil {
		return nil, err
	}

	revert.Success()
	return listener, nil
}
<commit_msg>lxd/cluster/events: Adds RoleInSlice function<commit_after>package cluster

import (
	"context"
	"sync"
	"time"

	log "gopkg.in/inconshreveable/log15.v2"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/endpoints"
	"github.com/lxc/lxd/lxd/revert"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/logger"
)

// eventHubMinHosts is the minimum number of members that must have the event-hub role to trigger switching into
// event-hub mode (where cluster members will only connect to event-hub members rather than all members when
// operating in the normal full-mesh mode).
const eventHubMinHosts = 2

// EventMode indicates the event distribution mode.
type EventMode string

// EventModeFullMesh is when every cluster member connects to every other cluster member to pull events.
const EventModeFullMesh EventMode = "full-mesh"

// EventModeHubServer is when the cluster is operating in event-hub mode and this server is designated as a hub
// server, meaning that it will only connect to the other event-hub members and not other members.
const EventModeHubServer EventMode = "hub-server"

// EventModeHubClient is when the cluster is operating in event-hub mode and this member is designated as a hub
// client, meaning that it is expected to connect to the event-hub members.
const EventModeHubClient EventMode = "hub-client"

var listeners = map[string]*lxd.EventListener{}
var listenersNotify = map[chan struct{}][]string{}
var listenersLock sync.Mutex
var listenersUpdateLock sync.Mutex

// ServerEventMode returns the event distribution mode that this local server is operating in.
func ServerEventMode() EventMode {
	listenersLock.Lock()
	defer listenersLock.Unlock()

	return eventMode
}

// RoleInSlice returns whether or not the rule is within the roles list.
func RoleInSlice(role db.ClusterRole, roles []db.ClusterRole) bool {
	for _, r := range roles {
		if r == role {
			return true
		}
	}

	return false
}

// EventListenerWait waits for there to be listener connected to the specified address, or one of the event hubs
// if operating in event hub mode.
func EventListenerWait(ctx context.Context, address string) error {
	// Check if there is already a listener.
	listenersLock.Lock()
	listener, found := listeners[address]
	if found && listener.IsActive() {
		listenersLock.Unlock()
		return nil
	}

	listenAddresses := []string{address}

	// If not setup a notification for when the desired address or any of the event hubs connect.
	connected := make(chan struct{})
	listenersNotify[connected] = listenAddresses
	listenersLock.Unlock()

	defer func() {
		listenersLock.Lock()
		delete(listenersNotify, connected)
		listenersLock.Unlock()
	}()

	// Wait for the connected channel to be closed (indicating a new listener has been connected), and return.
	select {
	case <-connected:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}

// EventsUpdateListeners refreshes the cluster event listener connections.
func EventsUpdateListeners(endpoints *endpoints.Endpoints, cluster *db.Cluster, serverCert func() *shared.CertInfo, members map[int64]APIHeartbeatMember, f func(int64, api.Event)) {
	listenersUpdateLock.Lock()
	defer listenersUpdateLock.Unlock()

	// If no heartbeat members provided, populate from global database.
	if members == nil {
		var dbMembers []db.NodeInfo
		var offlineThreshold time.Duration

		err := cluster.Transaction(func(tx *db.ClusterTx) error {
			var err error

			dbMembers, err = tx.GetNodes()
			if err != nil {
				return err
			}

			offlineThreshold, err = tx.GetNodeOfflineThreshold()
			if err != nil {
				return err
			}

			return nil
		})
		if err != nil {
			logger.Warn("Failed to get current cluster members", log.Ctx{"err": err})
			return
		}

		members = make(map[int64]APIHeartbeatMember, len(dbMembers))
		for _, dbMember := range dbMembers {
			members[dbMember.ID] = APIHeartbeatMember{
				ID:            dbMember.ID,
				Name:          dbMember.Name,
				Address:       dbMember.Address,
				LastHeartbeat: dbMember.Heartbeat,
				Online:        !dbMember.IsOffline(offlineThreshold),
				Roles:         dbMember.Roles,
			}
		}
	}

	networkAddress := endpoints.NetworkAddress()

	keepListeners := make(map[string]struct{})
	wg := sync.WaitGroup{}
	for _, member := range members {
		// Don't bother trying to connect to ourselves or offline members.
		if member.Address == networkAddress || !member.Online {
			continue
		}

		listenersLock.Lock()
		listener, ok := listeners[member.Address]

		// If the member already has a listener associated to it, check that the listener is still active.
		// If it is, just move on to next member, but if not then we'll try to connect again.
		if ok {
			if listener.IsActive() {
				keepListeners[member.Address] = struct{}{} // Add to current listeners list.
				listenersLock.Unlock()
				continue
			}

			// Disconnect and delete listener, but don't delete any listenersNotify entry as there
			// might be something waiting for a future connection.
			listener.Disconnect()
			delete(listeners, member.Address)
			logger.Info("Removed inactive member event listener client", log.Ctx{"local": networkAddress, "remote": member.Address})
		}
		listenersLock.Unlock()

		keepListeners[member.Address] = struct{}{} // Add to current listeners list.

		// Connect to remote concurrently and add to active listeners if successful.
		wg.Add(1)
		go func(m APIHeartbeatMember) {
			defer wg.Done()
			listener, err := eventsConnect(m.Address, endpoints.NetworkCert(), serverCert())
			if err != nil {
				logger.Warn("Failed adding member event listener client", log.Ctx{"local": networkAddress, "remote": m.Address, "err": err})
				return
			}

			listener.AddHandler(nil, func(event api.Event) { f(m.ID, event) })

			listenersLock.Lock()
			listeners[m.Address] = listener

			// Indicate to any notifiers waiting for this member's address that it is connected.
			for connected, notifyAddresses := range listenersNotify {
				if shared.StringInSlice(m.Address, notifyAddresses) {
					close(connected)
					delete(listenersNotify, connected)
				}
			}

			logger.Info("Added member event listener client", log.Ctx{"local": networkAddress, "remote": m.Address})
			listenersLock.Unlock()
		}(member)
	}

	wg.Wait()

	// Disconnect and delete any out of date listeners and their notifiers.
	listenersLock.Lock()
	for address, listener := range listeners {
		if _, found := keepListeners[address]; !found {
			listener.Disconnect()
			delete(listeners, address)
			logger.Info("Removed old member event listener client", log.Ctx{"local": networkAddress, "remote": address})
		}
	}
	listenersLock.Unlock()
}

// Establish a client connection to get events from the given node.
func eventsConnect(address string, networkCert *shared.CertInfo, serverCert *shared.CertInfo) (*lxd.EventListener, error) {
	client, err := Connect(address, networkCert, serverCert, nil, true)
	if err != nil {
		return nil, err
	}

	revert := revert.New()
	revert.Add(func() {
		client.Disconnect()
	})

	listener, err := client.GetEventsAllProjects()
	if err != nil {
		return nil, err
	}

	revert.Success()
	return listener, nil
}
<|endoftext|>


<commit_before>// +build !linux !cgo agent

package state

// State here is just an empty shim to statisfy dependencies.
type State struct {
}
<commit_msg>lxd/state/notlinux: Adds Events field to non-linux/agent State struct<commit_after>// +build !linux !cgo agent

package state

import (
	"github.com/lxc/lxd/lxd/events"
)

// State here is just an empty shim to statisfy dependencies.
type State struct {
	Events *events.Server
}
<|endoftext|>


<commit_before>package packet

import (
	"fmt"
	"os"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
	"github.com/packethost/packngo"
)

func TestAccPacketVolume_Basic(t *testing.T) {
	var volume packngo.Volume

	project_id := os.Getenv("PACKET_PROJECT_ID")
	facility := os.Getenv("PACKET_FACILITY")

	resource.Test(t, resource.TestCase{
		PreCheck:     testAccPacketVolumePreCheck(t),
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckPacketVolumeDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: fmt.Sprintf(testAccCheckPacketVolumeConfig_basic, project_id, facility),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckPacketVolumeExists("packet_volume.foobar", &volume),
					resource.TestCheckResourceAttr(
						"packet_volume.foobar", "project_id", project_id),
					resource.TestCheckResourceAttr(
						"packet_volume.foobar", "plan", "storage_1"),
					resource.TestCheckResourceAttr(
						"packet_volume.foobar", "billing_cycle", "hourly"),
					resource.TestCheckResourceAttr(
						"packet_volume.foobar", "size", "100"),
				),
			},
		},
	})
}

func testAccCheckPacketVolumeDestroy(s *terraform.State) error {
	client := testAccProvider.Meta().(*packngo.Client)

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "packet_volume" {
			continue
		}
		if _, _, err := client.Volumes.Get(rs.Primary.ID); err == nil {
			return fmt.Errorf("Volume still exists")
		}
	}

	return nil
}

func testAccCheckPacketVolumeExists(n string, volume *packngo.Volume) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}
		if rs.Primary.ID == "" {
			return fmt.Errorf("No Record ID is set")
		}

		client := testAccProvider.Meta().(*packngo.Client)

		foundVolume, _, err := client.Volumes.Get(rs.Primary.ID)
		if err != nil {
			return err
		}
		if foundVolume.ID != rs.Primary.ID {
			return fmt.Errorf("Record not found: %v - %v", rs.Primary.ID, foundVolume)
		}

		*volume = *foundVolume

		return nil
	}
}

func testAccPacketVolumePreCheck(t *testing.T) func() {
	return func() {
		testAccPreCheck(t)
		if os.Getenv("PACKET_PROJECT_ID") == "" {
			t.Fatal("PACKET_PROJECT_ID must be set")
		}
		if os.Getenv("PACKET_FACILITY") == "" {
			t.Fatal("PACKET_FACILITY must be set")
		}
	}
}

const testAccCheckPacketVolumeConfig_basic = `
resource "packet_volume" "foobar" {
    plan = "storage_1"
    billing_cycle = "hourly"
    size = 100
    project_id = "%s"
    facility = "%s"
    snapshot_policies = { snapshot_frequency = "1day", snapshot_count = 7 }
}`
<commit_msg>provider/packet: Restructure the Packet Volume test to no longer rely on environment variables<commit_after>package packet

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
	"github.com/packethost/packngo"
)

func TestAccPacketVolume_Basic(t *testing.T) {
	var volume packngo.Volume

	rs := acctest.RandString(10)

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckPacketVolumeDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: fmt.Sprintf(testAccCheckPacketVolumeConfig_basic, rs),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckPacketVolumeExists("packet_volume.foobar", &volume),
					resource.TestCheckResourceAttr(
						"packet_volume.foobar", "plan", "storage_1"),
					resource.TestCheckResourceAttr(
						"packet_volume.foobar", "billing_cycle", "hourly"),
					resource.TestCheckResourceAttr(
						"packet_volume.foobar", "size", "100"),
				),
			},
		},
	})
}

func testAccCheckPacketVolumeDestroy(s *terraform.State) error {
	client := testAccProvider.Meta().(*packngo.Client)

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "packet_volume" {
			continue
		}
		if _, _, err := client.Volumes.Get(rs.Primary.ID); err == nil {
			return fmt.Errorf("Volume still exists")
		}
	}

	return nil
}

func testAccCheckPacketVolumeExists(n string, volume *packngo.Volume) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}
		if rs.Primary.ID == "" {
			return fmt.Errorf("No Record ID is set")
		}

		client := testAccProvider.Meta().(*packngo.Client)

		foundVolume, _, err := client.Volumes.Get(rs.Primary.ID)
		if err != nil {
			return err
		}
		if foundVolume.ID != rs.Primary.ID {
			return fmt.Errorf("Record not found: %v - %v", rs.Primary.ID, foundVolume)
		}

		*volume = *foundVolume

		return nil
	}
}

const testAccCheckPacketVolumeConfig_basic = `
resource "packet_project" "foobar" {
    name = "%s"
}

resource "packet_volume" "foobar" {
    plan = "storage_1"
    billing_cycle = "hourly"
    size = 100
    project_id = "${packet_project.foobar.id}"
    facility = "ewr1"
    snapshot_policies = { snapshot_frequency = "1day", snapshot_count = 7 }
}`
<|endoftext|>