<commit_before>// Copyright 2016 Afshin Darian. All rights reserved.
// Use of this source code is governed by The MIT License
// that can be found in the LICENSE file.

// Package sleuth provides master-less peer-to-peer autodiscovery and RPC
// between HTTP services that reside on the same network. It works with minimal
// configuration and provides a mechanism to join a local network both as a
// client that offers no services and as any service that speaks HTTP. Its
// primary use case is for microservices on the same network that make calls to
// one another.
package sleuth

import (
	"fmt"
	"net/http"

	"github.com/ursiform/logger"
	"github.com/zeromq/gyre"
)

var (
	Debug = false
	group = "SLEUTH-v0"
	port  = 5670
	recv  = "RECV"
	repl  = "REPL"
)

type connection struct {
	adapter string
	name    string
	node    string
	port    int
	server  bool
	version string
}

func announce(done chan *Client, conn *connection, out *logger.Logger) {
	node, err := newNode(out, conn)
	if err != nil {
		done <- nil
		return
	}
	client := newClient(node, out)
	done <- client
	for {
		event := <-node.Events()
		switch event.Type() {
		case gyre.EventEnter:
			client.add(event)
		case gyre.EventExit, gyre.EventLeave:
			client.remove(event)
		case gyre.EventWhisper:
			client.dispatch(event)
		}
	}
}

func failure(out *logger.Logger, err error, code int) error {
	out.Error("sleuth: %s (%d)", err.Error(), code)
	return err
}

func newNode(out *logger.Logger, conn *connection) (*gyre.Gyre, error) {
	node, err := gyre.New()
	if err != nil {
		return nil, failure(out, err, ErrorInitialize)
	}
	if err := node.SetPort(conn.port); err != nil {
		return nil, failure(out, err, ErrorSetPort)
	}
	if len(conn.adapter) > 0 {
		if err := node.SetInterface(conn.adapter); err != nil {
			return nil, failure(out, err, ErrorInterface)
		}
	}
	if Debug {
		if err := node.SetVerbose(); err != nil {
			return nil, failure(out, err, ErrorSetVerbose)
		}
	}
	// If announcing a service, add service headers.
	if conn.server {
		if err := node.SetHeader("group", group); err != nil {
			return nil, failure(out, err, ErrorGroupHeader)
		}
		if err := node.SetHeader("node", node.UUID()); err != nil {
			return nil, failure(out, err, ErrorNodeHeader)
		}
		if err := node.SetHeader("type", conn.name); err != nil {
			return nil, failure(out, err, ErrorServiceHeader)
		}
		if err := node.SetHeader("version", conn.version); err != nil {
			return nil, failure(out, err, ErrorVersionHeader)
		}
	}
	if err := node.Start(); err != nil {
		return nil, failure(out, err, ErrorStart)
	}
	if err := node.Join(group); err != nil {
		node.Stop()
		return nil, failure(out, err, ErrorJoin)
	}
	var role string
	if conn.server {
		role = conn.name
	} else {
		role = "client-only"
	}
	out.Listen("sleuth: [%s:%d][%s %s]", group, conn.port, role, node.Name())
	return node, nil
}

// New is the entry point to the sleuth package. It returns a reference to a
// Client object that has joined the local network. If the handler argument is
// not nil, the Client also answers requests from other peers.
func New(handler http.Handler, configFile string) (*Client, error) {
	var file string
	if len(configFile) > 0 {
		file = configFile
	} else {
		file = ConfigFile
	}
	config := loadConfig(file)
	conn := new(connection)
	// Use the same log level as the instantiator of the client.
	out, err := logger.New(config.LogLevel)
	if err != nil {
		return nil, err
	}
	if handler == nil {
		out.Init("sleuth: New - handler is nil, client-only mode")
	} else {
		conn.name = config.Service.Name
		if len(conn.name) == 0 {
			err := fmt.Errorf("sleuth: New - %s not defined in %s",
				"service.name", ConfigFile)
			return nil, failure(out, err, ErrorServiceUndefined)
		}
	}
	conn.server = handler != nil
	conn.adapter = config.Sleuth.Interface
	if len(conn.adapter) == 0 {
		out.Warn("sleuth: New - sleuth.interface not defined in %s", ConfigFile)
	}
	conn.port = config.Sleuth.Port
	if conn.port == 0 {
		conn.port = port
	}
	conn.version = config.Service.Version
	if len(conn.version) == 0 {
		conn.version = "unknown"
	}
	done := make(chan *Client, 1)
	go announce(done, conn, out)
	client := <-done
	if client == nil {
		return nil, fmt.Errorf("sleuth: New - unable to announce")
	}
	client.log = out
	client.handler = handler
	return client, nil
}
<commit_msg>clean up instantiation<commit_after>// Copyright 2016 Afshin Darian. All rights reserved.
// Use of this source code is governed by The MIT License
// that can be found in the LICENSE file.

// Package sleuth provides master-less peer-to-peer autodiscovery and RPC
// between HTTP services that reside on the same network. It works with minimal
// configuration and provides a mechanism to join a local network both as a
// client that offers no services and as any service that speaks HTTP. Its
// primary use case is for microservices on the same network that make calls to
// one another.
package sleuth

import (
	"fmt"
	"net/http"

	"github.com/ursiform/logger"
	"github.com/zeromq/gyre"
)

var (
	Debug = false
	group = "SLEUTH-v0"
	port  = 5670
	recv  = "RECV"
	repl  = "REPL"
)

type connection struct {
	adapter string
	handler http.Handler
	name    string
	node    string
	port    int
	server  bool
	version string
}

type instance struct {
	client *Client
	err    error
}

func announce(conn *connection, out *logger.Logger, result chan *instance) {
	node, err := newNode(out, conn)
	if err != nil {
		result <- &instance{client: nil, err: err}
		return
	}
	client := newClient(node, out)
	client.handler = conn.handler
	result <- &instance{client: client, err: nil}
	for {
		event := <-node.Events()
		switch event.Type() {
		case gyre.EventEnter:
			client.add(event)
		case gyre.EventExit, gyre.EventLeave:
			client.remove(event)
		case gyre.EventWhisper:
			client.dispatch(event)
		}
	}
}

func failure(out *logger.Logger, err error, code int) error {
	out.Error("sleuth: %s (%d)", err.Error(), code)
	return err
}

func newNode(out *logger.Logger, conn *connection) (*gyre.Gyre, error) {
	node, err := gyre.New()
	if err != nil {
		return nil, failure(out, err, ErrorInitialize)
	}
	if err := node.SetPort(conn.port); err != nil {
		return nil, failure(out, err, ErrorSetPort)
	}
	if len(conn.adapter) > 0 {
		if err := node.SetInterface(conn.adapter); err != nil {
			return nil, failure(out, err, ErrorInterface)
		}
	}
	if Debug {
		if err := node.SetVerbose(); err != nil {
			return nil, failure(out, err, ErrorSetVerbose)
		}
	}
	// If announcing a service, add service headers.
	if conn.server {
		if err := node.SetHeader("group", group); err != nil {
			return nil, failure(out, err, ErrorGroupHeader)
		}
		if err := node.SetHeader("node", node.UUID()); err != nil {
			return nil, failure(out, err, ErrorNodeHeader)
		}
		if err := node.SetHeader("type", conn.name); err != nil {
			return nil, failure(out, err, ErrorServiceHeader)
		}
		if err := node.SetHeader("version", conn.version); err != nil {
			return nil, failure(out, err, ErrorVersionHeader)
		}
	}
	if err := node.Start(); err != nil {
		return nil, failure(out, err, ErrorStart)
	}
	if err := node.Join(group); err != nil {
		node.Stop()
		return nil, failure(out, err, ErrorJoin)
	}
	var role string
	if conn.server {
		role = conn.name
	} else {
		role = "client-only"
	}
	out.Listen("sleuth: [%s:%d][%s %s]", group, conn.port, role, node.Name())
	return node, nil
}

// New is the entry point to the sleuth package. It returns a reference to a
// Client object that has joined the local network. If the handler argument is
// not nil, the Client also answers requests from other peers.
func New(handler http.Handler, configFile string) (*Client, error) {
	var file string
	if len(configFile) > 0 {
		file = configFile
	} else {
		file = ConfigFile
	}
	config := loadConfig(file)
	conn := new(connection)
	// Use the same log level as the instantiator of the client.
	out, err := logger.New(config.LogLevel)
	if err != nil {
		return nil, err
	}
	if handler == nil {
		out.Init("sleuth: New - handler is nil, client-only mode")
	} else {
		conn.handler = handler
		conn.name = config.Service.Name
		if len(conn.name) == 0 {
			err := fmt.Errorf("sleuth: New - %s not defined in %s",
				"service.name", ConfigFile)
			return nil, failure(out, err, ErrorServiceUndefined)
		}
	}
	conn.server = handler != nil
	conn.adapter = config.Sleuth.Interface
	if len(conn.adapter) == 0 {
		out.Warn("sleuth: New - sleuth.interface not defined in %s", ConfigFile)
	}
	conn.port = config.Sleuth.Port
	if conn.port == 0 {
		conn.port = port
	}
	conn.version = config.Service.Version
	if len(conn.version) == 0 {
		conn.version = "unknown"
	}
	done := make(chan *instance, 1)
	go announce(conn, out, done)
	result := <-done
	return result.client, result.err
}
<|endoftext|>


<commit_before>package model

import (
	"fmt"
	"log"
	"sort"
	"sync"
)

// Collection of FieldInfo(s) (accessible by number of by name)
type FieldInfos struct {
	HasFreq      bool
	HasProx      bool
	HasPayloads  bool
	HasOffsets   bool
	HasVectors   bool
	HasNorms     bool
	HasDocValues bool

	byNumber map[int32]FieldInfo
	byName   map[string]FieldInfo
	Values   []FieldInfo // sorted by ID
}

func NewFieldInfos(infos []FieldInfo) FieldInfos {
	self := FieldInfos{byNumber: make(map[int32]FieldInfo), byName: make(map[string]FieldInfo)}

	numbers := make([]int32, 0)
	for _, info := range infos {
		if prev, ok := self.byNumber[info.Number]; ok {
			panic(fmt.Sprintf("duplicate field numbers: %v and %v have: %v", prev.Name, info.Name, info.Number))
		}
		self.byNumber[info.Number] = info
		numbers = append(numbers, info.Number)
		if prev, ok := self.byName[info.Name]; ok {
			panic(fmt.Sprintf("duplicate field names: %v and %v have: %v", prev.Number, info.Number, info.Name))
		}
		self.byName[info.Name] = info

		self.HasVectors = self.HasVectors || info.storeTermVector
		self.HasProx = self.HasProx || info.indexed && info.indexOptions >= INDEX_OPT_DOCS_AND_FREQS_AND_POSITIONS
		self.HasFreq = self.HasFreq || info.indexed && info.indexOptions != INDEX_OPT_DOCS_ONLY
		self.HasOffsets = self.HasOffsets || info.indexed && info.indexOptions >= INDEX_OPT_DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS
		self.HasNorms = self.HasNorms || info.normType != 0
		self.HasDocValues = self.HasDocValues || info.docValueType != 0
		self.HasPayloads = self.HasPayloads || info.storePayloads
	}

	sort.Sort(Int32Slice(numbers))
	self.Values = make([]FieldInfo, len(infos))
	for i, v := range numbers {
		self.Values[int32(i)] = self.byNumber[v]
	}

	return self
}

/* Returns the number of fields */
func (infos FieldInfos) Size() int {
	assert(len(infos.byNumber) == len(infos.byName))
	return len(infos.byNumber)
}

/* Return the FieldInfo object referenced by the field name */
func (infos FieldInfos) FieldInfoByName(fieldName string) FieldInfo {
	return infos.byName[fieldName]
}

/* Return the FieldInfo object referenced by the fieldNumber. */
func (infos FieldInfos) FieldInfoByNumber(fieldNumber int) FieldInfo {
	assert(fieldNumber >= 0)
	return infos.byNumber[int32(fieldNumber)]
}

func (fis FieldInfos) String() string {
	return fmt.Sprintf(`
hasFreq = %v
hasProx = %v
hasPayloads = %v
hasOffsets = %v
hasVectors = %v
hasNorms = %v
hasDocValues = %v
%v`, fis.HasFreq, fis.HasProx, fis.HasPayloads, fis.HasOffsets,
		fis.HasVectors, fis.HasNorms, fis.HasDocValues, fis.Values)
}

type FieldNumbers struct {
	sync.Locker
	numberToName map[int]string
	nameToNumber map[string]int
	// We use this to enforce that a given field never changes DV type,
	// even across segments / IndexWriter sessions:
	docValuesType map[string]DocValuesType
	// TODO: we should similarly catch an attempt to turn norms back on
	// after they were already ommitted; today we silently discard the
	// norm but this is badly trappy
	lowestUnassignedFieldNumber int
}

func NewFieldNumbers() *FieldNumbers {
	return &FieldNumbers{
		Locker:        &sync.Mutex{},
		nameToNumber:  make(map[string]int),
		numberToName:  make(map[int]string),
		docValuesType: make(map[string]DocValuesType),
	}
}

func (fn *FieldNumbers) AddOrGet(info FieldInfo) int {
	return fn.addOrGet(info.Name, int(info.Number), info.docValueType)
}

/*
Returns the global field number for the given field name. If the name
does not exist yet it tries to add it with the given preferred field
number assigned if possible otherwise the first unassigned field
number is used as the field number.
*/
func (fn *FieldNumbers) addOrGet(name string, preferredNumber int, dv DocValuesType) int {
	fn.Lock()
	defer fn.Unlock()

	if dv != 0 {
		currentDv, ok := fn.docValuesType[name]
		if !ok || currentDv == 0 {
			fn.docValuesType[name] = dv
		} else if currentDv != dv {
			log.Panicf("cannot change DocValues type from %v to %v for field '%v'", currentDv, dv, name)
		}
	}
	number, ok := fn.nameToNumber[name]
	if !ok {
		_, ok = fn.numberToName[preferredNumber]
		if preferredNumber != -1 && !ok {
			// cool - we can use this number globally
			number = preferredNumber
		} else {
			// find a new FieldNumber
			for _, ok = fn.numberToName[fn.lowestUnassignedFieldNumber]; ok; {
				// might not be up to date - lets do the work once needed
				fn.lowestUnassignedFieldNumber++
			}
			number = fn.lowestUnassignedFieldNumber
		}

		fn.numberToName[number] = name
		fn.nameToNumber[name] = number
	}
	return number
}

type FieldInfosBuilder struct {
	byName             map[string]FieldInfo
	globalFieldNumbers *FieldNumbers
}

func NewFieldInfosBuilder(globalFieldNumbers *FieldNumbers) *FieldInfosBuilder {
	assert(globalFieldNumbers != nil)
	return &FieldInfosBuilder{
		byName:             make(map[string]FieldInfo),
		globalFieldNumbers: globalFieldNumbers,
	}
}

func assert(ok bool) {
	assert2(ok, "assert fail")
}

func assert2(ok bool, msg string, args ...interface{}) {
	if !ok {
		panic(fmt.Sprintf(msg, args...))
	}
}

/*
NOTE: this method does not carry over termVector booleans nor
docValuesType; the indexer chain  (TermVectorsConsumerPerField,
DocFieldProcessor) must set these fields when they succeed in
consuming the document
*/
func (b *FieldInfosBuilder) AddOrUpdate(name string, fieldType IndexableFieldType) FieldInfo {
	// TODO: really, indexer shouldn't even call this method (it's only
	// called from DocFieldProcessor); rather, each component in the
	// chain should update what it "owns". E.g., fieldType.indexOptions()
	// should be updated by maybe FreqProxTermsWriterPerField:
	return b.addOrUpdateInternal(name, -1, fieldType.Indexed(), false,
		fieldType.OmitNorms(), false,
		fieldType.IndexOptions(), fieldType.DocValueType(), DocValuesType(0))
}

func (b *FieldInfosBuilder) addOrUpdateInternal(name string,
	preferredFieldNumber int, isIndexed bool, storeTermVector bool,
	omitNorms bool, storePayloads bool, indexOptions IndexOptions,
	docValues DocValuesType, normType DocValuesType) FieldInfo {
	if fi, ok := b.byName[name]; ok {
		panic("not implemented yet")
		return fi
	} else {
		panic("not implemented yte")
	}
}

func (b *FieldInfosBuilder) Finish() FieldInfos {
	var infos []FieldInfo
	for _, v := range b.byName {
		infos = append(infos, v)
	}
	return NewFieldInfos(infos)
}
<commit_msg>implement FieldInfosBuilder.addOrUpdateInternal()<commit_after>package model

import (
	"fmt"
	"log"
	"sort"
	"sync"
)

// Collection of FieldInfo(s) (accessible by number of by name)
type FieldInfos struct {
	HasFreq      bool
	HasProx      bool
	HasPayloads  bool
	HasOffsets   bool
	HasVectors   bool
	HasNorms     bool
	HasDocValues bool

	byNumber map[int32]FieldInfo
	byName   map[string]FieldInfo
	Values   []FieldInfo // sorted by ID
}

func NewFieldInfos(infos []FieldInfo) FieldInfos {
	self := FieldInfos{byNumber: make(map[int32]FieldInfo), byName: make(map[string]FieldInfo)}

	numbers := make([]int32, 0)
	for _, info := range infos {
		if prev, ok := self.byNumber[info.Number]; ok {
			panic(fmt.Sprintf("duplicate field numbers: %v and %v have: %v", prev.Name, info.Name, info.Number))
		}
		self.byNumber[info.Number] = info
		numbers = append(numbers, info.Number)
		if prev, ok := self.byName[info.Name]; ok {
			panic(fmt.Sprintf("duplicate field names: %v and %v have: %v", prev.Number, info.Number, info.Name))
		}
		self.byName[info.Name] = info

		self.HasVectors = self.HasVectors || info.storeTermVector
		self.HasProx = self.HasProx || info.indexed && info.indexOptions >= INDEX_OPT_DOCS_AND_FREQS_AND_POSITIONS
		self.HasFreq = self.HasFreq || info.indexed && info.indexOptions != INDEX_OPT_DOCS_ONLY
		self.HasOffsets = self.HasOffsets || info.indexed && info.indexOptions >= INDEX_OPT_DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS
		self.HasNorms = self.HasNorms || info.normType != 0
		self.HasDocValues = self.HasDocValues || info.docValueType != 0
		self.HasPayloads = self.HasPayloads || info.storePayloads
	}

	sort.Sort(Int32Slice(numbers))
	self.Values = make([]FieldInfo, len(infos))
	for i, v := range numbers {
		self.Values[int32(i)] = self.byNumber[v]
	}

	return self
}

/* Returns the number of fields */
func (infos FieldInfos) Size() int {
	assert(len(infos.byNumber) == len(infos.byName))
	return len(infos.byNumber)
}

/* Return the FieldInfo object referenced by the field name */
func (infos FieldInfos) FieldInfoByName(fieldName string) FieldInfo {
	return infos.byName[fieldName]
}

/* Return the FieldInfo object referenced by the fieldNumber. */
func (infos FieldInfos) FieldInfoByNumber(fieldNumber int) FieldInfo {
	assert(fieldNumber >= 0)
	return infos.byNumber[int32(fieldNumber)]
}

func (fis FieldInfos) String() string {
	return fmt.Sprintf(`
hasFreq = %v
hasProx = %v
hasPayloads = %v
hasOffsets = %v
hasVectors = %v
hasNorms = %v
hasDocValues = %v
%v`, fis.HasFreq, fis.HasProx, fis.HasPayloads, fis.HasOffsets,
		fis.HasVectors, fis.HasNorms, fis.HasDocValues, fis.Values)
}

type FieldNumbers struct {
	sync.Locker
	numberToName map[int]string
	nameToNumber map[string]int
	// We use this to enforce that a given field never changes DV type,
	// even across segments / IndexWriter sessions:
	docValuesType map[string]DocValuesType
	// TODO: we should similarly catch an attempt to turn norms back on
	// after they were already ommitted; today we silently discard the
	// norm but this is badly trappy
	lowestUnassignedFieldNumber int
}

func NewFieldNumbers() *FieldNumbers {
	return &FieldNumbers{
		Locker:        &sync.Mutex{},
		nameToNumber:  make(map[string]int),
		numberToName:  make(map[int]string),
		docValuesType: make(map[string]DocValuesType),
	}
}

func (fn *FieldNumbers) AddOrGet(info FieldInfo) int {
	return fn.addOrGet(info.Name, int(info.Number), info.docValueType)
}

/*
Returns the global field number for the given field name. If the name
does not exist yet it tries to add it with the given preferred field
number assigned if possible otherwise the first unassigned field
number is used as the field number.
*/
func (fn *FieldNumbers) addOrGet(name string, preferredNumber int, dv DocValuesType) int {
	fn.Lock()
	defer fn.Unlock()

	if dv != 0 {
		currentDv, ok := fn.docValuesType[name]
		if !ok || currentDv == 0 {
			fn.docValuesType[name] = dv
		} else if currentDv != dv {
			log.Panicf("cannot change DocValues type from %v to %v for field '%v'", currentDv, dv, name)
		}
	}
	number, ok := fn.nameToNumber[name]
	if !ok {
		_, ok = fn.numberToName[preferredNumber]
		if preferredNumber != -1 && !ok {
			// cool - we can use this number globally
			number = preferredNumber
		} else {
			// find a new FieldNumber
			for _, ok = fn.numberToName[fn.lowestUnassignedFieldNumber]; ok; {
				// might not be up to date - lets do the work once needed
				fn.lowestUnassignedFieldNumber++
			}
			number = fn.lowestUnassignedFieldNumber
		}

		fn.numberToName[number] = name
		fn.nameToNumber[name] = number
	}
	return number
}

type FieldInfosBuilder struct {
	byName             map[string]FieldInfo
	globalFieldNumbers *FieldNumbers
}

func NewFieldInfosBuilder(globalFieldNumbers *FieldNumbers) *FieldInfosBuilder {
	assert(globalFieldNumbers != nil)
	return &FieldInfosBuilder{
		byName:             make(map[string]FieldInfo),
		globalFieldNumbers: globalFieldNumbers,
	}
}

func assert(ok bool) {
	assert2(ok, "assert fail")
}

func assert2(ok bool, msg string, args ...interface{}) {
	if !ok {
		panic(fmt.Sprintf(msg, args...))
	}
}

/*
NOTE: this method does not carry over termVector booleans nor
docValuesType; the indexer chain  (TermVectorsConsumerPerField,
DocFieldProcessor) must set these fields when they succeed in
consuming the document
*/
func (b *FieldInfosBuilder) AddOrUpdate(name string, fieldType IndexableFieldType) FieldInfo {
	// TODO: really, indexer shouldn't even call this method (it's only
	// called from DocFieldProcessor); rather, each component in the
	// chain should update what it "owns". E.g., fieldType.indexOptions()
	// should be updated by maybe FreqProxTermsWriterPerField:
	return b.addOrUpdateInternal(name, -1, fieldType.Indexed(), false,
		fieldType.OmitNorms(), false,
		fieldType.IndexOptions(), fieldType.DocValueType(), DocValuesType(0))
}

func (b *FieldInfosBuilder) addOrUpdateInternal(name string,
	preferredFieldNumber int, isIndexed bool, storeTermVector bool,
	omitNorms bool, storePayloads bool, indexOptions IndexOptions,
	docValues DocValuesType, normType DocValuesType) FieldInfo {
	if fi, ok := b.byName[name]; ok {
		panic("not implemented yet")
		return fi
	} else {
		// This field wasn't yet added to this in-RAM segment's
		// FieldInfos, so now we get a global number for this field. If
		// the field was seen before then we'll get the same name and
		// number, else we'll allocate a new one:
		fieldNumber := int32(b.globalFieldNumbers.addOrGet(name, preferredFieldNumber, docValues))
		fi = NewFieldInfo(name, isIndexed, fieldNumber, storeTermVector,
			omitNorms, storePayloads, indexOptions, docValues, normType, nil)
		b.byName[fi.Name] = fi
		return fi
	}
}

func (b *FieldInfosBuilder) Finish() FieldInfos {
	var infos []FieldInfo
	for _, v := range b.byName {
		infos = append(infos, v)
	}
	return NewFieldInfos(infos)
}
<|endoftext|>


<commit_before>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package api

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"net"
	"strings"
	"time"

	"github.com/TheThingsNetwork/ttn/utils/backoff"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
)

// KeepAlive indicates the keep-alive time for the Dialer
var KeepAlive = 10 * time.Second

// MaxRetries indicates how often clients should retry dialing a component
var MaxRetries = 100

// Timeout for connections
var Timeout = 2 * time.Second

// DialOptions to use in TTN gRPC
var DialOptions = []grpc.DialOption{
	WithKeepAliveDialer(),
	grpc.WithBlock(),
	grpc.FailOnNonTempDialError(true),
	grpc.WithTimeout(Timeout),
}

func dial(address string, tlsConfig *tls.Config, fallback bool) (conn *grpc.ClientConn, err error) {
	ctx := GetLogger().WithField("Address", address)
	retries := 0
	retriesLeft := MaxRetries
	opts := DialOptions
	if tlsConfig != nil {
		tlsConfig.ServerName = strings.SplitN(address, ":", 2)[0] // trim the port
		opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)))
	} else {
		opts = append(opts, grpc.WithInsecure())
	}
	for retriesLeft > 0 {
		conn, err = grpc.Dial(
			address,
			opts...,
		)
		if err == nil {
			ctx.Debug("Connected")
			return
		}

		switch err := err.(type) {
		case *net.OpError:
			// Dial problem
			if err.Op == "dial" {
				ctx.WithError(err).Debug("Could not connect, reconnecting...")
			}
		case x509.CertificateInvalidError,
			x509.ConstraintViolationError,
			x509.HostnameError,
			x509.InsecureAlgorithmError,
			x509.SystemRootsError,
			x509.UnhandledCriticalExtension,
			x509.UnknownAuthorityError:
			// Non-temporary error while connecting to a TLS-enabled server
			return nil, err
		case tls.RecordHeaderError:
			if fallback {
				ctx.WithError(err).Warn("Could not connect with TLS, reconnecting without it...")
				return dial(address, nil, fallback)
			}
			return nil, err
		default:
			GetLogger().WithField("ErrType", fmt.Sprintf("%T", err)).WithError(err).Error("Unhandled dial error [please create issue on Github]")
			return nil, err
		}

		// Backoff
		time.Sleep(backoff.Backoff(retries))
		retries++
		retriesLeft--
	}
	return
}

// RootCAs to use in API connections
var RootCAs *x509.CertPool

func init() {
	RootCAs, _ = x509.SystemCertPool()
}

// Dial an address
func Dial(address string) (*grpc.ClientConn, error) {
	tlsConfig := &tls.Config{RootCAs: RootCAs}
	return dial(address, tlsConfig, true)
}

// DialWithCert dials the address using the given TLS cert
func DialWithCert(address string, cert string) (*grpc.ClientConn, error) {
	rootCAs := x509.NewCertPool()
	ok := rootCAs.AppendCertsFromPEM([]byte(cert))
	if !ok {
		panic("failed to parse root certificate")
	}
	tlsConfig := &tls.Config{RootCAs: rootCAs}
	return dial(address, tlsConfig, false)
}

// WithKeepAliveDialer creates a dialer with the configured KeepAlive time
func WithKeepAliveDialer() grpc.DialOption {
	return grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
		d := net.Dialer{Timeout: timeout, KeepAlive: KeepAlive}
		return d.Dial("tcp", addr)
	})
}
<commit_msg>Move gRPC reconnect logic to Dialer<commit_after>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package api

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"net"
	"strings"
	"time"

	"github.com/TheThingsNetwork/ttn/utils/backoff"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
)

// KeepAlive indicates the keep-alive time for the Dialer
var KeepAlive = 10 * time.Second

// MaxRetries indicates how often clients should retry dialing a component
var MaxRetries = 100

// Timeout for connections
var Timeout = 2 * time.Second

// DialOptions to use in TTN gRPC
var DialOptions = []grpc.DialOption{
	WithTTNDialer(),
	grpc.WithBlock(),
	grpc.FailOnNonTempDialError(true),
	grpc.WithTimeout(Timeout),
}

func dial(address string, tlsConfig *tls.Config, fallback bool) (conn *grpc.ClientConn, err error) {
	ctx := GetLogger().WithField("Address", address)
	opts := DialOptions
	if tlsConfig != nil {
		tlsConfig.ServerName = strings.SplitN(address, ":", 2)[0] // trim the port
		opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)))
	} else {
		opts = append(opts, grpc.WithInsecure())
	}
	conn, err = grpc.Dial(
		address,
		opts...,
	)
	if err == nil {
		return
	}

	switch err := err.(type) {
	case x509.CertificateInvalidError,
		x509.ConstraintViolationError,
		x509.HostnameError,
		x509.InsecureAlgorithmError,
		x509.SystemRootsError,
		x509.UnhandledCriticalExtension,
		x509.UnknownAuthorityError:
		// Non-temporary error while connecting to a TLS-enabled server
		return nil, err
	case tls.RecordHeaderError:
		if fallback {
			ctx.WithError(err).Warn("Could not connect to gRPC server with TLS, reconnecting without it...")
			return dial(address, nil, fallback)
		}
		return nil, err
	}

	GetLogger().WithField("ErrType", fmt.Sprintf("%T", err)).WithError(err).Error("Unhandled dial error [please create issue on Github]")
	return nil, err
}

// RootCAs to use in API connections
var RootCAs *x509.CertPool

func init() {
	RootCAs, _ = x509.SystemCertPool()
}

// Dial an address
func Dial(address string) (*grpc.ClientConn, error) {
	tlsConfig := &tls.Config{RootCAs: RootCAs}
	return dial(address, tlsConfig, true)
}

// DialWithCert dials the address using the given TLS cert
func DialWithCert(address string, cert string) (*grpc.ClientConn, error) {
	rootCAs := x509.NewCertPool()
	ok := rootCAs.AppendCertsFromPEM([]byte(cert))
	if !ok {
		panic("failed to parse root certificate")
	}
	tlsConfig := &tls.Config{RootCAs: rootCAs}
	return dial(address, tlsConfig, false)
}

// WithTTNDialer creates a dialer for TTN
func WithTTNDialer() grpc.DialOption {
	return grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
		ctx := GetLogger().WithField("Address", addr)
		d := net.Dialer{Timeout: timeout, KeepAlive: KeepAlive}
		var retries int
		for {
			conn, err := d.Dial("tcp", addr)
			if err == nil {
				ctx.Debug("Connected to gRPC server")
				return conn, nil
			}
			if err, ok := err.(*net.OpError); ok && err.Op == "dial" && retries <= MaxRetries {
				ctx.WithError(err).Debug("Could not connect to gRPC server, reconnecting...")
				time.Sleep(backoff.Backoff(retries))
				retries++
				continue
			}
			return nil, err
		}
	})
}
<|endoftext|>


<commit_before>package job

import (
	"context"
	"github.com/pkg/errors"
	"github.com/problame/go-streamrpc"
	"github.com/zrepl/zrepl/config"
	"github.com/zrepl/zrepl/daemon/connecter"
	"github.com/zrepl/zrepl/daemon/filters"
	"github.com/zrepl/zrepl/daemon/logging"
	"github.com/zrepl/zrepl/endpoint"
	"github.com/zrepl/zrepl/replication"
	"sync"
	"github.com/zrepl/zrepl/daemon/pruner"
	"github.com/zrepl/zrepl/pruning"
)

type Push struct {
	name      string
	connecter streamrpc.Connecter
	fsfilter  endpoint.FSFilter

	keepRulesSender []pruning.KeepRule
	keepRulesReceiver []pruning.KeepRule

	mtx         sync.Mutex
	replication *replication.Replication
}

func PushFromConfig(g config.Global, in *config.PushJob) (j *Push, err error) {

	j = &Push{}
	j.name = in.Name

	j.connecter, err = connecter.FromConfig(g, in.Replication.Connect)

	if j.fsfilter, err = filters.DatasetMapFilterFromConfig(in.Replication.Filesystems); err != nil {
		return nil, errors.Wrap(err, "cannnot build filesystem filter")
	}

	return j, nil
}

func (j *Push) Name() string { return j.name }

func (j *Push) Status() interface{} {
	return nil // FIXME
}

func (j *Push) Run(ctx context.Context) {
	log := GetLogger(ctx)

	defer log.Info("job exiting")

	log.Debug("wait for wakeups")

	invocationCount := 0
outer:
	for {
		select {
		case <-ctx.Done():
			log.WithError(ctx.Err()).Info("context")
			break outer
		case <-WaitWakeup(ctx):
			invocationCount++
			invLog := log.WithField("invocation", invocationCount)
			j.do(WithLogger(ctx, invLog))
		}
	}
}

func (j *Push) do(ctx context.Context) {

	log := GetLogger(ctx)

	client, err := streamrpc.NewClient(j.connecter, &streamrpc.ClientConfig{STREAMRPC_CONFIG})
	if err != nil {
		log.WithError(err).Error("cannot create streamrpc client")
	}
	defer client.Close()

	sender := endpoint.NewSender(j.fsfilter, filters.NewAnyFSVFilter())
	receiver := endpoint.NewRemote(client)

	j.mtx.Lock()
	rep := replication.NewReplication()
	j.mtx.Unlock()

	ctx = logging.WithSubsystemLoggers(ctx, log)
	rep.Drive(ctx, sender, receiver)

	// Prune sender
	senderPruner := pruner.NewPruner(sender, receiver, j.keepRulesSender)
	senderPruner.Prune(ctx)

	// Prune receiver
	receiverPruner := pruner.NewPruner(receiver, receiver, j.keepRulesReceiver)
	receiverPruner.Prune(ctx)

}

<commit_msg>finish pruning implementation in push job<commit_after>package job

import (
	"context"
	"github.com/pkg/errors"
	"github.com/problame/go-streamrpc"
	"github.com/zrepl/zrepl/config"
	"github.com/zrepl/zrepl/daemon/connecter"
	"github.com/zrepl/zrepl/daemon/filters"
	"github.com/zrepl/zrepl/daemon/logging"
	"github.com/zrepl/zrepl/daemon/pruner"
	"github.com/zrepl/zrepl/endpoint"
	"github.com/zrepl/zrepl/pruning"
	"github.com/zrepl/zrepl/replication"
	"sync"
	"time"
)

type Push struct {
	name      string
	connecter streamrpc.Connecter
	fsfilter  endpoint.FSFilter

	keepRulesSender   []pruning.KeepRule
	keepRulesReceiver []pruning.KeepRule

	mtx         sync.Mutex
	replication *replication.Replication
}

func PushFromConfig(g config.Global, in *config.PushJob) (j *Push, err error) {

	j = &Push{}
	j.name = in.Name

	j.connecter, err = connecter.FromConfig(g, in.Replication.Connect)

	if j.fsfilter, err = filters.DatasetMapFilterFromConfig(in.Replication.Filesystems); err != nil {
		return nil, errors.Wrap(err, "cannnot build filesystem filter")
	}

	j.keepRulesReceiver, err = pruning.RulesFromConfig(in.Pruning.KeepReceiver)
	if err != nil {
		return nil, errors.Wrap(err, "cannot build receiver pruning rules")
	}

	j.keepRulesSender, err = pruning.RulesFromConfig(in.Pruning.KeepSender)
	if err != nil {
		return nil, errors.Wrap(err, "cannot build sender pruning rules")
	}

	return j, nil
}

func (j *Push) Name() string { return j.name }

func (j *Push) Status() interface{} {
	return nil // FIXME
}

func (j *Push) Run(ctx context.Context) {
	log := GetLogger(ctx)

	defer log.Info("job exiting")

	log.Debug("wait for wakeups")

	invocationCount := 0
outer:
	for {
		select {
		case <-ctx.Done():
			log.WithError(ctx.Err()).Info("context")
			break outer
		case <-WaitWakeup(ctx):
			invocationCount++
			invLog := log.WithField("invocation", invocationCount)
			j.do(WithLogger(ctx, invLog))
		}
	}
}

func (j *Push) do(ctx context.Context) {

	log := GetLogger(ctx)

	client, err := streamrpc.NewClient(j.connecter, &streamrpc.ClientConfig{STREAMRPC_CONFIG})
	if err != nil {
		log.WithError(err).Error("cannot create streamrpc client")
	}
	defer client.Close()

	sender := endpoint.NewSender(j.fsfilter, filters.NewAnyFSVFilter())
	receiver := endpoint.NewRemote(client)

	j.mtx.Lock()
	rep := replication.NewReplication()
	j.mtx.Unlock()

	ctx = logging.WithSubsystemLoggers(ctx, log)
	rep.Drive(ctx, sender, receiver)

	// Prune sender
	senderPruner := pruner.NewPruner(10*time.Second, sender, sender, j.keepRulesSender) // FIXME constant
	senderPruner.Prune(pruner.WithLogger(ctx, pruner.GetLogger(ctx).WithField("prune_side", "sender")))

	// Prune receiver
	receiverPruner := pruner.NewPruner(10*time.Second, receiver, sender, j.keepRulesReceiver) // FIXME constant
	receiverPruner.Prune(pruner.WithLogger(ctx, pruner.GetLogger(ctx).WithField("prune_side", "receiver")))

}
<|endoftext|>


<commit_before>// Copyright 2014 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package api

import (
	"encoding/json"
	"net/http"

	"github.com/ajg/form"
	"github.com/tsuru/tsuru/auth"
	"github.com/tsuru/tsuru/errors"
	"github.com/tsuru/tsuru/event"
	"github.com/tsuru/tsuru/iaas"
	"github.com/tsuru/tsuru/permission"
	"gopkg.in/mgo.v2"
)

// title: machine list
// path: /iaas/machines
// method: GET
// produce: application/json
// responses:
//   200: OK
//   401: Unauthorized
func machinesList(w http.ResponseWriter, r *http.Request, token auth.Token) error {
	machines, err := iaas.ListMachines()
	if err != nil {
		return err
	}
	contexts := permission.ContextsForPermission(token, permission.PermMachineRead)
	allowedIaaS := map[string]struct{}{}
	for _, c := range contexts {
		if c.CtxType == permission.CtxGlobal {
			allowedIaaS = nil
			break
		}
		if c.CtxType == permission.CtxIaaS {
			allowedIaaS[c.Value] = struct{}{}
		}
	}
	for i := 0; allowedIaaS != nil && i < len(machines); i++ {
		if _, ok := allowedIaaS[machines[i].Iaas]; !ok {
			machines = append(machines[:i], machines[i+1:]...)
			i--
		}
	}
	w.Header().Add("Content-Type", "application/json")
	return json.NewEncoder(w).Encode(machines)
}

// title: machine destroy
// path: /iaas/machines/{machine_id}
// method: DELETE
// responses:
//   200: OK
//   400: Invalid data
//   401: Unauthorized
//   404: Not found
func machineDestroy(w http.ResponseWriter, r *http.Request, token auth.Token) (err error) {
	r.ParseForm()
	machineID := r.URL.Query().Get(":machine_id")
	if machineID == "" {
		return &errors.HTTP{Code: http.StatusBadRequest, Message: "machine id is required"}
	}
	m, err := iaas.FindMachineById(machineID)
	if err != nil {
		if err == mgo.ErrNotFound {
			return &errors.HTTP{Code: http.StatusNotFound, Message: "machine not found"}
		}
		return err
	}
	iaasCtx := permission.Context(permission.CtxIaaS, m.Iaas)
	allowed := permission.Check(token, permission.PermMachineDelete, iaasCtx)
	if !allowed {
		return permission.ErrUnauthorized
	}
	evt, err := event.New(&event.Opts{
		Target:     event.Target{Type: event.TargetTypeIaas, Value: m.Iaas},
		Kind:       permission.PermMachineDelete,
		Owner:      token,
		CustomData: event.FormToCustomData(r.Form),
		Allowed:    event.Allowed(permission.PermMachineReadEvents, iaasCtx),
	})
	if err != nil {
		return err
	}
	defer func() { evt.Done(err) }()
	return m.Destroy()
}

// title: machine template list
// path: /iaas/templates
// method: GET
// produce: application/json
// responses:
//   200: OK
//   401: Unauthorized
func templatesList(w http.ResponseWriter, r *http.Request, token auth.Token) error {
	templates, err := iaas.ListTemplates()
	if err != nil {
		return err
	}
	contexts := permission.ContextsForPermission(token, permission.PermMachineTemplateRead)
	allowedIaaS := map[string]struct{}{}
	for _, c := range contexts {
		if c.CtxType == permission.CtxGlobal {
			allowedIaaS = nil
			break
		}
		if c.CtxType == permission.CtxIaaS {
			allowedIaaS[c.Value] = struct{}{}
		}
	}
	for i := 0; allowedIaaS != nil && i < len(templates); i++ {
		if _, ok := allowedIaaS[templates[i].IaaSName]; !ok {
			templates = append(templates[:i], templates[i+1:]...)
			i--
		}
	}
	w.Header().Add("Content-Type", "application/json")
	return json.NewEncoder(w).Encode(templates)
}

// title: template create
// path: /iaas/templates
// method: POST
// consume: application/x-www-form-urlencoded
// responses:
//   201: Template created
//   400: Invalid data
//   401: Unauthorized
func templateCreate(w http.ResponseWriter, r *http.Request, token auth.Token) (err error) {
	err = r.ParseForm()
	if err != nil {
		return &errors.HTTP{Code: http.StatusBadRequest, Message: err.Error()}
	}
	var paramTemplate iaas.Template
	dec := form.NewDecoder(nil)
	dec.IgnoreUnknownKeys(true)
	err = dec.DecodeValues(&paramTemplate, r.Form)
	if err != nil {
		return &errors.HTTP{Code: http.StatusBadRequest, Message: err.Error()}
	}
	iaasCtx := permission.Context(permission.CtxIaaS, paramTemplate.IaaSName)
	allowed := permission.Check(token, permission.PermMachineTemplateCreate, iaasCtx)
	if !allowed {
		return permission.ErrUnauthorized
	}
	evt, err := event.New(&event.Opts{
		Target:     event.Target{Type: event.TargetTypeIaas, Value: paramTemplate.IaaSName},
		Kind:       permission.PermMachineTemplateCreate,
		Owner:      token,
		CustomData: event.FormToCustomData(r.Form),
		Allowed:    event.Allowed(permission.PermMachineReadEvents, iaasCtx),
	})
	if err != nil {
		return err
	}
	defer func() { evt.Done(err) }()
	err = paramTemplate.Save()
	if err != nil {
		return err
	}
	w.WriteHeader(http.StatusCreated)
	return nil
}

// title: template destroy
// path: /iaas/templates/{template_name}
// method: DELETE
// responses:
//   200: OK
//   401: Unauthorized
//   404: Not found
func templateDestroy(w http.ResponseWriter, r *http.Request, token auth.Token) (err error) {
	r.ParseForm()
	templateName := r.URL.Query().Get(":template_name")
	t, err := iaas.FindTemplate(templateName)
	if err != nil {
		if err == mgo.ErrNotFound {
			return &errors.HTTP{Code: http.StatusNotFound, Message: "template not found"}
		}
		return err
	}
	iaasCtx := permission.Context(permission.CtxIaaS, t.IaaSName)
	allowed := permission.Check(token, permission.PermMachineTemplateDelete, iaasCtx)
	if !allowed {
		return permission.ErrUnauthorized
	}
	evt, err := event.New(&event.Opts{
		Target:     event.Target{Type: event.TargetTypeIaas, Value: t.IaaSName},
		Kind:       permission.PermMachineTemplateDelete,
		Owner:      token,
		CustomData: event.FormToCustomData(r.Form),
		Allowed:    event.Allowed(permission.PermMachineReadEvents, iaasCtx),
	})
	if err != nil {
		return err
	}
	defer func() { evt.Done(err) }()
	return iaas.DestroyTemplate(templateName)
}

// title: template update
// path: /iaas/templates/{template_name}
// method: PUT
// consume: application/x-www-form-urlencoded
// responses:
//   200: OK
//   400: Invalid data
//   401: Unauthorized
//   404: Not found
func templateUpdate(w http.ResponseWriter, r *http.Request, token auth.Token) (err error) {
	err = r.ParseForm()
	if err != nil {
		return &errors.HTTP{Code: http.StatusBadRequest, Message: err.Error()}
	}
	var paramTemplate iaas.Template
	dec := form.NewDecoder(nil)
	dec.IgnoreUnknownKeys(true)
	err = dec.DecodeValues(&paramTemplate, r.Form)
	if err != nil {
		return &errors.HTTP{Code: http.StatusBadRequest, Message: err.Error()}
	}
	templateName := r.URL.Query().Get(":template_name")
	iaasName := r.Form.Get("IaaSName")
	dbTpl, err := iaas.FindTemplate(templateName)
	if err != nil {
		if err == mgo.ErrNotFound {
			return &errors.HTTP{Code: http.StatusNotFound, Message: "template not found"}
		}
		return err
	}
	if (dbTpl.IaaSName != iaasName) && (iaasName != "") {
		dbTpl.IaaSName = iaasName
	}
	iaasCtx := permission.Context(permission.CtxIaaS, dbTpl.IaaSName)
	allowed := permission.Check(token, permission.PermMachineTemplateUpdate, iaasCtx)
	if !allowed {
		return permission.ErrUnauthorized
	}
	evt, err := event.New(&event.Opts{
		Target:     event.Target{Type: event.TargetTypeIaas, Value: dbTpl.IaaSName},
		Kind:       permission.PermMachineTemplateUpdate,
		Owner:      token,
		CustomData: event.FormToCustomData(r.Form),
		Allowed:    event.Allowed(permission.PermMachineReadEvents, iaasCtx),
	})
	if err != nil {
		return err
	}
	defer func() { evt.Done(err) }()
	return dbTpl.Update(&paramTemplate)
}
<commit_msg>iaas: update IaasName<commit_after>// Copyright 2014 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package api

import (
	"encoding/json"
	"net/http"

	"github.com/ajg/form"
	"github.com/tsuru/tsuru/auth"
	"github.com/tsuru/tsuru/errors"
	"github.com/tsuru/tsuru/event"
	"github.com/tsuru/tsuru/iaas"
	"github.com/tsuru/tsuru/permission"
	"gopkg.in/mgo.v2"
)

// title: machine list
// path: /iaas/machines
// method: GET
// produce: application/json
// responses:
//   200: OK
//   401: Unauthorized
func machinesList(w http.ResponseWriter, r *http.Request, token auth.Token) error {
	machines, err := iaas.ListMachines()
	if err != nil {
		return err
	}
	contexts := permission.ContextsForPermission(token, permission.PermMachineRead)
	allowedIaaS := map[string]struct{}{}
	for _, c := range contexts {
		if c.CtxType == permission.CtxGlobal {
			allowedIaaS = nil
			break
		}
		if c.CtxType == permission.CtxIaaS {
			allowedIaaS[c.Value] = struct{}{}
		}
	}
	for i := 0; allowedIaaS != nil && i < len(machines); i++ {
		if _, ok := allowedIaaS[machines[i].Iaas]; !ok {
			machines = append(machines[:i], machines[i+1:]...)
			i--
		}
	}
	w.Header().Add("Content-Type", "application/json")
	return json.NewEncoder(w).Encode(machines)
}

// title: machine destroy
// path: /iaas/machines/{machine_id}
// method: DELETE
// responses:
//   200: OK
//   400: Invalid data
//   401: Unauthorized
//   404: Not found
func machineDestroy(w http.ResponseWriter, r *http.Request, token auth.Token) (err error) {
	r.ParseForm()
	machineID := r.URL.Query().Get(":machine_id")
	if machineID == "" {
		return &errors.HTTP{Code: http.StatusBadRequest, Message: "machine id is required"}
	}
	m, err := iaas.FindMachineById(machineID)
	if err != nil {
		if err == mgo.ErrNotFound {
			return &errors.HTTP{Code: http.StatusNotFound, Message: "machine not found"}
		}
		return err
	}
	iaasCtx := permission.Context(permission.CtxIaaS, m.Iaas)
	allowed := permission.Check(token, permission.PermMachineDelete, iaasCtx)
	if !allowed {
		return permission.ErrUnauthorized
	}
	evt, err := event.New(&event.Opts{
		Target:     event.Target{Type: event.TargetTypeIaas, Value: m.Iaas},
		Kind:       permission.PermMachineDelete,
		Owner:      token,
		CustomData: event.FormToCustomData(r.Form),
		Allowed:    event.Allowed(permission.PermMachineReadEvents, iaasCtx),
	})
	if err != nil {
		return err
	}
	defer func() { evt.Done(err) }()
	return m.Destroy()
}

// title: machine template list
// path: /iaas/templates
// method: GET
// produce: application/json
// responses:
//   200: OK
//   401: Unauthorized
func templatesList(w http.ResponseWriter, r *http.Request, token auth.Token) error {
	templates, err := iaas.ListTemplates()
	if err != nil {
		return err
	}
	contexts := permission.ContextsForPermission(token, permission.PermMachineTemplateRead)
	allowedIaaS := map[string]struct{}{}
	for _, c := range contexts {
		if c.CtxType == permission.CtxGlobal {
			allowedIaaS = nil
			break
		}
		if c.CtxType == permission.CtxIaaS {
			allowedIaaS[c.Value] = struct{}{}
		}
	}
	for i := 0; allowedIaaS != nil && i < len(templates); i++ {
		if _, ok := allowedIaaS[templates[i].IaaSName]; !ok {
			templates = append(templates[:i], templates[i+1:]...)
			i--
		}
	}
	w.Header().Add("Content-Type", "application/json")
	return json.NewEncoder(w).Encode(templates)
}

// title: template create
// path: /iaas/templates
// method: POST
// consume: application/x-www-form-urlencoded
// responses:
//   201: Template created
//   400: Invalid data
//   401: Unauthorized
func templateCreate(w http.ResponseWriter, r *http.Request, token auth.Token) (err error) {
	err = r.ParseForm()
	if err != nil {
		return &errors.HTTP{Code: http.StatusBadRequest, Message: err.Error()}
	}
	var paramTemplate iaas.Template
	dec := form.NewDecoder(nil)
	dec.IgnoreUnknownKeys(true)
	err = dec.DecodeValues(&paramTemplate, r.Form)
	if err != nil {
		return &errors.HTTP{Code: http.StatusBadRequest, Message: err.Error()}
	}
	iaasCtx := permission.Context(permission.CtxIaaS, paramTemplate.IaaSName)
	allowed := permission.Check(token, permission.PermMachineTemplateCreate, iaasCtx)
	if !allowed {
		return permission.ErrUnauthorized
	}
	evt, err := event.New(&event.Opts{
		Target:     event.Target{Type: event.TargetTypeIaas, Value: paramTemplate.IaaSName},
		Kind:       permission.PermMachineTemplateCreate,
		Owner:      token,
		CustomData: event.FormToCustomData(r.Form),
		Allowed:    event.Allowed(permission.PermMachineReadEvents, iaasCtx),
	})
	if err != nil {
		return err
	}
	defer func() { evt.Done(err) }()
	err = paramTemplate.Save()
	if err != nil {
		return err
	}
	w.WriteHeader(http.StatusCreated)
	return nil
}

// title: template destroy
// path: /iaas/templates/{template_name}
// method: DELETE
// responses:
//   200: OK
//   401: Unauthorized
//   404: Not found
func templateDestroy(w http.ResponseWriter, r *http.Request, token auth.Token) (err error) {
	r.ParseForm()
	templateName := r.URL.Query().Get(":template_name")
	t, err := iaas.FindTemplate(templateName)
	if err != nil {
		if err == mgo.ErrNotFound {
			return &errors.HTTP{Code: http.StatusNotFound, Message: "template not found"}
		}
		return err
	}
	iaasCtx := permission.Context(permission.CtxIaaS, t.IaaSName)
	allowed := permission.Check(token, permission.PermMachineTemplateDelete, iaasCtx)
	if !allowed {
		return permission.ErrUnauthorized
	}
	evt, err := event.New(&event.Opts{
		Target:     event.Target{Type: event.TargetTypeIaas, Value: t.IaaSName},
		Kind:       permission.PermMachineTemplateDelete,
		Owner:      token,
		CustomData: event.FormToCustomData(r.Form),
		Allowed:    event.Allowed(permission.PermMachineReadEvents, iaasCtx),
	})
	if err != nil {
		return err
	}
	defer func() { evt.Done(err) }()
	return iaas.DestroyTemplate(templateName)
}

// title: template update
// path: /iaas/templates/{template_name}
// method: PUT
// consume: application/x-www-form-urlencoded
// responses:
//   200: OK
//   400: Invalid data
//   401: Unauthorized
//   404: Not found
func templateUpdate(w http.ResponseWriter, r *http.Request, token auth.Token) (err error) {
	err = r.ParseForm()
	if err != nil {
		return &errors.HTTP{Code: http.StatusBadRequest, Message: err.Error()}
	}
	var paramTemplate iaas.Template
	dec := form.NewDecoder(nil)
	dec.IgnoreUnknownKeys(true)
	err = dec.DecodeValues(&paramTemplate, r.Form)
	if err != nil {
		return &errors.HTTP{Code: http.StatusBadRequest, Message: err.Error()}
	}
	templateName := r.URL.Query().Get(":template_name")
	dbTpl, err := iaas.FindTemplate(templateName)
	if err != nil {
		if err == mgo.ErrNotFound {
			return &errors.HTTP{Code: http.StatusNotFound, Message: "template not found"}
		}
		return err
	}
	if r.Form.Get("IaaSName") != "" {
		dbTpl.IaaSName = r.Form.Get("IaaSName")
	}
	iaasCtx := permission.Context(permission.CtxIaaS, dbTpl.IaaSName)
	allowed := permission.Check(token, permission.PermMachineTemplateUpdate, iaasCtx)
	if !allowed {
		return permission.ErrUnauthorized
	}
	evt, err := event.New(&event.Opts{
		Target:     event.Target{Type: event.TargetTypeIaas, Value: dbTpl.IaaSName},
		Kind:       permission.PermMachineTemplateUpdate,
		Owner:      token,
		CustomData: event.FormToCustomData(r.Form),
		Allowed:    event.Allowed(permission.PermMachineReadEvents, iaasCtx),
	})
	if err != nil {
		return err
	}
	defer func() { evt.Done(err) }()
	return dbTpl.Update(&paramTemplate)
}
<|endoftext|>


<commit_before>package aws

import (
	"regexp"

	"github.com/hashicorp/terraform/helper/validation"

	"github.com/aws/aws-sdk-go/service/athena"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsAthenaWorkgroup() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsAthenaWorkgroupCreate,
		Read:   resourceAwsAthenaWorkgroupRead,
		Update: resourceAwsAthenaWorkgroupUpdate,
		Delete: resourceAwsAthenaWorkgroupDelete,

		Schema: map[string]*schema.Schema{
			"name": {
				Type:         schema.TypeString,
				Required:     true,
				ForceNew:     true,
			},
			"description": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"configuration": {
				Type:     schema.TypeList,
				Optional: true,
				MaxItems: 1,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"bytes_scanned_cutoff_per_query": {
							Type: schema.TypeInt,
							Optional: true,
						},
						"enforce_workgroup_configuration": {
							Type: schema.TypeBool,
							Optional: true,
						},
						"publish_cloudwatch_metrics_enable": {
							Type: schema.TypeBool,
							Optional: true,
						}
						"result_configuration":{
							Type: schema.TypeList,
							Optional: true,
							MaxItems: 1,
							Elem: &schema.Resource{
								"output_location": {
									Type: schema.TypeString,
									Optional: true,
								},
								"encryption_configuration": {
									Type: schema.TypeList,
									Optional: true,
									MaxItems: 1,
									Elem: &schema.Resource{
										"encryption_option": {
											Type: schema.TypeString,
											Optional: true,
											ValidateFunc: validation.StringInSlice([]string{"SSE_S3", "SSE_KMS", "CSE_KMS"}, false)
										},
										"kms_key": {
											Type: schema.TypeString,
											Optional: true,
										},
									},
								},
							},
						},
					},
				},
			},
			"tags" : tagsSchema(),
		},
	}
}
<commit_msg>Moved to TopLevel Parameters<commit_after>package aws

import (
	"log"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/athena"
	"github.com/hashicorp/terraform/helper/validation"

	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsAthenaWorkgroup() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsAthenaWorkgroupCreate,
		Read:   resourceAwsAthenaWorkgroupRead,
		Update: resourceAwsAthenaWorkgroupUpdate,
		Delete: resourceAwsAthenaWorkgroupDelete,

		Schema: map[string]*schema.Schema{
			"name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"description": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"bytes_scanned_cutoff_per_query": {
				Type:         schema.TypeInt,
				Optional:     true,
				ValidateFunc: validation.IntAtLeast(10485760),
			},
			"enforce_workgroup_configuration": {
				Type:     schema.TypeBool,
				Optional: true,
			},
			"publish_cloudwatch_metrics_enable": {
				Type:     schema.TypeBool,
				Optional: true,
			},
			"output_location": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"encryption_option": {
				Type:     schema.TypeString,
				Required: true,
				ValidateFunc: validation.StringInSlice([]string{
					athena.EncryptionOptionCseKms,
					athena.EncryptionOptionSseKms,
					athena.EncryptionOptionSseS3,
				}, false),
			},
			"kms_key": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"tags": tagsSchema(),
		},
	}
}

func resourceAwsAthenaWorkgroupCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).athenaconn

	name := d.Get("name").(string)

	input := &athena.CreateWorkGroupInput{
		Name: aws.String(name),
	}

	if v, ok := d.GetOk("description"); ok {
		input.Description = aws.String(v.(string))
	}

	resp, err := conn.CreateWorkGroup(input)

	if err != nil {
		return err
	}

	d.SetId(name)

	return resourceAwsAthenaWorkgroupRead(d, meta)
}

func resourceAwsAthenaWorkgroupRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).athenaconn

	input := &athena.GetWorkGroupInput{
		WorkGroup: aws.String(d.Id()),
	}

	resp, err := conn.GetWorkGroup(input)

	if err != nil {
		if isAWSErr(err, athena.ErrCodeInvalidRequestException, d.Id()) {
			log.Printf("[WARN] Athena WorkGroup (%s) not found, removing from state", d.Id())
			d.SetId("")
			return nil
		}
		return err
	}

	d.Set("name", resp.WorkGroup.Name)
	d.Set("description", resp.WorkGroup.Description)
	d.Set("bytes_scanned_cutoff_per_query", resp.WorkGroup.Configuration.BytesScannedCutoffPerQuery)
	d.Set("publish_cloudwatch_metrics_enabled", resp.WorkGroup.Configuration.PublishCloudWatchMetricsEnabled)
	d.Set("enforce_workgroup_configuration", resp.WorkGroup.Configuration.EnforceWorkGroupConfiguration)
	d.Set("output_location", resp.WorkGroup.Configuration.ResultConfiguration.OutputLocation)
	d.Set("encryption_option", resp.WorkGroup.Configuration.ResultConfiguration.EncryptionConfiguration.EncryptionOption)
	d.Set("kms_key", resp.WorkGroup.Configuration.ResultConfiguration.EncryptionConfiguration.KmsKey)

	return nil
}
<|endoftext|>


<commit_before>package main

import "os"
import "log"
import "fmt"
import "strconv"
import (
	"database/sql"
	_ "github.com/lib/pq"
)
import "net/http"
import "net/url"
import "encoding/json"

var db *sql.DB //to share with our handlers

func SafeValues(v *url.Values) bool {
	log.Printf("safe %+v", v)
	return true //for now
}

func makePoint(v *url.Values) string {
	point := fmt.Sprintf("POINT(%s %s)", v.Get("lon"), v.Get("lat"))
	return point
}

func recordlocations(v *url.Values) {
	p := makePoint(v)
	txn, err := db.Begin()
	if err != nil {
		log.Fatal(err)
	}
	label := fmt.Sprintf("%s", v.Get("label"))
	acc, err := strconv.ParseFloat(v.Get("acc"), 64)
	if err != nil {
		log.Fatal(err)
	}
	_, err = txn.Exec("insert into locations ( label, acc, geom ) values ( $1, $2, ST_PointFromText( $3, 4326) )", label, acc, p)
	if err != nil {
		log.Fatal(err)
	}
	err = txn.Commit()
	if err != nil {
		log.Fatal(err)
	}
}

func LocationHandler(w http.ResponseWriter, req *http.Request) {
	log.Println("handling url", req.URL)
	if req.Method == "GET" {
		if req.URL.RawQuery != "" {
			values, err := url.ParseQuery(req.URL.RawQuery)
			if err != nil {
				log.Fatal(err)
			}
			log.Println(values)
			if SafeValues(&values) {
				p := makePoint(&values)
				go recordlocations(&values)
				log.Println("point:", p)
				q := "select name from adminareas where st_contains(adminareas.geom, st_geomfromtext( $1 , 4326))"
				rows, err := db.Query(q, p)
				if err != nil {
					log.Print("db error", err)
				}
				var l []string
				for rows.Next() {
					var name string
					rows.Scan(&name)
					l = append(l, name)
				}

				m := make(map[string][]string)
				m["names"] = l
				j, err := json.Marshal(m)
				if err != nil {
					log.Fatal(err)
				}
				h := w.Header()
				h.Add("Access-Control-Allow-Origin", "*")
				w.Write(j)
				return
			}
		} else {
			http.Error(w, "No Content", http.StatusNoContent)
			return
		}

	}
}

func main() {
	log.Println("smalld starting")
	db_connection := os.Getenv("SMALLD_DB_CONNECTION")
	url_base := os.Getenv("SMALLD_URL_BASE")
	listen_address :=os.Getenv("SMALLD_LISTEN_ADDRESS")
	options := os.Getenv("SMALLD_OPTIONS") //override command line flags
	log.Println("SMALLD_DB_CONNECTION:", db_connection)
	log.Println("SMALLD_URL_BASE:", url_base)
	log.Println("SMALLD_LISTEN_ADDRESS", listen_address)
	log.Println("SMALLD_OPTIONS:", options)
	var err error
	db, err = sql.Open("postgres", db_connection)
	err = db.Ping()
	if err != nil {
		log.Fatal(err)
	}
	log.Println("connected to database")
	http.HandleFunc("/location", LocationHandler)
	log.Println("registered LocationHandler")
	http.ListenAndServe(listen_address, nil)
}
<commit_msg>fixing issues from golint<commit_after>package main

import "os"
import "log"
import "fmt"
import "strconv"
import (
	"database/sql"
	_ "github.com/lib/pq"
)
import "net/http"
import "net/url"
import "encoding/json"

var db *sql.DB //to share with our handlers

func safeValues(v *url.Values) bool {
	log.Printf("safe %+v", v)
	return true //for now
}

func makePoint(v *url.Values) string {
	point := fmt.Sprintf("POINT(%s %s)", v.Get("lon"), v.Get("lat"))
	return point
}

func recordlocations(v *url.Values) {
	p := makePoint(v)
	txn, err := db.Begin()
	if err != nil {
		log.Fatal(err)
	}
	label := fmt.Sprintf("%s", v.Get("label"))
	acc, err := strconv.ParseFloat(v.Get("acc"), 64)
	if err != nil {
		log.Fatal(err)
	}
	_, err = txn.Exec("insert into locations ( label, acc, geom ) values ( $1, $2, ST_PointFromText( $3, 4326) )", label, acc, p)
	if err != nil {
		log.Fatal(err)
	}
	err = txn.Commit()
	if err != nil {
		log.Fatal(err)
	}
}

// LocationHandler is the main entry point for smalld
// it receives the get request parses the location data from it
// and logs the values to the location table.
func LocationHandler(w http.ResponseWriter, req *http.Request) {
	log.Println("handling url", req.URL)
	if req.Method == "GET" {
		if req.URL.RawQuery != "" {
			values, err := url.ParseQuery(req.URL.RawQuery)
			if err != nil {
				log.Fatal(err)
			}
			log.Println(values)
			if SafeValues(&values) {
				p := makePoint(&values)
				go recordlocations(&values)
				log.Println("point:", p)
				q := "select name from adminareas where st_contains(adminareas.geom, st_geomfromtext( $1 , 4326))"
				rows, err := db.Query(q, p)
				if err != nil {
					log.Print("db error", err)
				}
				var l []string
				for rows.Next() {
					var name string
					rows.Scan(&name)
					l = append(l, name)
				}

				m := make(map[string][]string)
				m["names"] = l
				j, err := json.Marshal(m)
				if err != nil {
					log.Fatal(err)
				}
				h := w.Header()
				h.Add("Access-Control-Allow-Origin", "*")
				w.Write(j)
				return
			}
		} else {
			http.Error(w, "No Content", http.StatusNoContent)
			return
		}

	}
}

func main() {
	log.Println("smalld starting")
	dbCconnection := os.Getenv("SMALLD_DB_CONNECTION")
	urlBase := os.Getenv("SMALLD_URL_BASE")
	listenAddress := os.Getenv("SMALLD_LISTEN_ADDRESS")
	options := os.Getenv("SMALLD_OPTIONS") //override command line flags
	log.Println("SMALLD_DB_CONNECTION:", dbConnection)
	log.Println("SMALLD_URL_BASE:", urlBase)
	log.Println("SMALLD_LISTEN_ADDRESS", listenAddress)
	log.Println("SMALLD_OPTIONS:", options)
	var err error
	db, err = sql.Open("postgres", dbConnection)
	err = db.Ping()
	if err != nil {
		log.Fatal(err)
	}
	log.Println("connected to database")
	http.HandleFunc("/location", LocationHandler)
	log.Println("registered LocationHandler")
	http.ListenAndServe(listenAddress, nil)
}
<|endoftext|>


<commit_before>package azurerm

import (
	"fmt"
	"net/http"

	"github.com/Azure/azure-sdk-for-go/arm/dns"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceArmDnsARecord() *schema.Resource {
	return &schema.Resource{
		Create: resourceArmDnsARecordCreateOrUpdate,
		Read:   resourceArmDnsARecordRead,
		Update: resourceArmDnsARecordCreateOrUpdate,
		Delete: resourceArmDnsARecordDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"resource_group_name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"zone_name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},

			"records": &schema.Schema{
				Type:     schema.TypeSet,
				Required: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Set:      schema.HashString,
			},

			"ttl": &schema.Schema{
				Type:     schema.TypeInt,
				Required: true,
			},

			"etag": {
				Type:     schema.TypeString,
				Computed: true,
			},

			"tags": tagsSchema(),
		},
	}
}

func resourceArmDnsARecordCreateOrUpdate(d *schema.ResourceData, meta interface{}) error {
	dnsClient := meta.(*ArmClient).dnsClient

	name := d.Get("name").(string)
	resGroup := d.Get("resource_group_name").(string)
	zoneName := d.Get("zone_name").(string)
	ttl := int64(d.Get("ttl").(int))
	eTag := d.Get("etag").(string)

	tags := d.Get("tags").(map[string]interface{})
	metadata := expandTags(tags)

	records, err := expandAzureRmDnsARecords(d)
	props := dns.RecordSetProperties{
		Metadata: metadata,
		TTL:      &ttl,
		ARecords: &records,
	}

	parameters := dns.RecordSet{
		Name:                &name,
		RecordSetProperties: &props,
	}

	//last parameter is set to empty to allow updates to records after creation
	// (per SDK, set it to '*' to prevent updates, all other values are ignored)
	resp, err := dnsClient.CreateOrUpdate(resGroup, zoneName, name, dns.A, parameters, eTag, "")
	if err != nil {
		return err
	}

	if resp.ID == nil {
		return fmt.Errorf("Cannot read DNS A Record %s (resource group %s) ID", name, resGroup)
	}

	d.SetId(*resp.ID)

	return resourceArmDnsARecordRead(d, meta)
}

func resourceArmDnsARecordRead(d *schema.ResourceData, meta interface{}) error {
	dnsClient := meta.(*ArmClient).dnsClient

	id, err := parseAzureResourceID(d.Id())
	if err != nil {
		return err
	}

	resGroup := id.ResourceGroup
	name := id.Path["A"]
	zoneName := id.Path["dnszones"]

	resp, err := dnsClient.Get(resGroup, zoneName, name, dns.A)
	if err != nil {
		return fmt.Errorf("Error reading DNS A record %s: %v", name, err)
	}
	if resp.StatusCode == http.StatusNotFound {
		d.SetId("")
		return nil
	}

	d.Set("name", name)
	d.Set("resource_group_name", resGroup)
	d.Set("zone_name", zoneName)
	d.Set("ttl", resp.TTL)
	d.Set("etag", resp.Etag)

	if err := d.Set("records", flattenAzureRmDnsARecords(resp.ARecords)); err != nil {
		return err
	}
	flattenAndSetTags(d, resp.Metadata)

	return nil
}

func resourceArmDnsARecordDelete(d *schema.ResourceData, meta interface{}) error {
	dnsClient := meta.(*ArmClient).dnsClient

	id, err := parseAzureResourceID(d.Id())
	if err != nil {
		return err
	}

	resGroup := id.ResourceGroup
	name := id.Path["A"]
	zoneName := id.Path["dnszones"]

	resp, error := dnsClient.Delete(resGroup, zoneName, name, dns.A, "")
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("Error deleting DNS A Record %s: %s", name, error)
	}

	return nil
}

func flattenAzureRmDnsARecords(records *[]dns.ARecord) []string {
	results := make([]string, 0, len(*records))

	if records != nil {
		for _, record := range *records {
			results = append(results, *record.Ipv4Address)
		}
	}

	return results
}

func expandAzureRmDnsARecords(d *schema.ResourceData) ([]dns.ARecord, error) {
	recordStrings := d.Get("records").(*schema.Set).List()
	records := make([]dns.ARecord, len(recordStrings))

	for i, v := range recordStrings {
		ipv4 := v.(string)
		records[i] = dns.ARecord{
			Ipv4Address: &ipv4,
		}
	}

	return records, nil
}
<commit_msg>respond to PR feedback<commit_after>package azurerm

import (
	"fmt"
	"net/http"

	"github.com/Azure/azure-sdk-for-go/arm/dns"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceArmDnsARecord() *schema.Resource {
	return &schema.Resource{
		Create: resourceArmDnsARecordCreateOrUpdate,
		Read:   resourceArmDnsARecordRead,
		Update: resourceArmDnsARecordCreateOrUpdate,
		Delete: resourceArmDnsARecordDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"resource_group_name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"zone_name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},

			"records": &schema.Schema{
				Type:     schema.TypeSet,
				Required: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Set:      schema.HashString,
			},

			"ttl": &schema.Schema{
				Type:     schema.TypeInt,
				Required: true,
			},

			"tags": tagsSchema(),
		},
	}
}

func resourceArmDnsARecordCreateOrUpdate(d *schema.ResourceData, meta interface{}) error {
	dnsClient := meta.(*ArmClient).dnsClient

	name := d.Get("name").(string)
	resGroup := d.Get("resource_group_name").(string)
	zoneName := d.Get("zone_name").(string)
	ttl := int64(d.Get("ttl").(int))

	tags := d.Get("tags").(map[string]interface{})
	metadata := expandTags(tags)

	records, err := expandAzureRmDnsARecords(d)
	props := dns.RecordSetProperties{
		Metadata: metadata,
		TTL:      &ttl,
		ARecords: &records,
	}

	parameters := dns.RecordSet{
		Name:                &name,
		RecordSetProperties: &props,
	}

	//last parameter is set to empty to allow updates to records after creation
	// (per SDK, set it to '*' to prevent updates, all other values are ignored)
	resp, err := dnsClient.CreateOrUpdate(resGroup, zoneName, name, dns.A, parameters, "", "")
	if err != nil {
		return err
	}

	if resp.ID == nil {
		return fmt.Errorf("Cannot read DNS A Record %s (resource group %s) ID", name, resGroup)
	}

	d.SetId(*resp.ID)

	return resourceArmDnsARecordRead(d, meta)
}

func resourceArmDnsARecordRead(d *schema.ResourceData, meta interface{}) error {
	dnsClient := meta.(*ArmClient).dnsClient

	id, err := parseAzureResourceID(d.Id())
	if err != nil {
		return err
	}

	resGroup := id.ResourceGroup
	name := id.Path["A"]
	zoneName := id.Path["dnszones"]

	resp, err := dnsClient.Get(resGroup, zoneName, name, dns.A)
	if err != nil {
		return fmt.Errorf("Error reading DNS A record %s: %v", name, err)
	}
	if resp.StatusCode == http.StatusNotFound {
		d.SetId("")
		return nil
	}

	d.Set("name", name)
	d.Set("resource_group_name", resGroup)
	d.Set("zone_name", zoneName)
	d.Set("ttl", resp.TTL)

	if err := d.Set("records", flattenAzureRmDnsARecords(resp.ARecords)); err != nil {
		return err
	}
	flattenAndSetTags(d, resp.Metadata)

	return nil
}

func resourceArmDnsARecordDelete(d *schema.ResourceData, meta interface{}) error {
	dnsClient := meta.(*ArmClient).dnsClient

	id, err := parseAzureResourceID(d.Id())
	if err != nil {
		return err
	}

	resGroup := id.ResourceGroup
	name := id.Path["A"]
	zoneName := id.Path["dnszones"]

	resp, error := dnsClient.Delete(resGroup, zoneName, name, dns.A, "")
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("Error deleting DNS A Record %s: %+v", name, error)
	}

	return nil
}

func flattenAzureRmDnsARecords(records *[]dns.ARecord) []string {
	results := make([]string, 0, len(*records))

	if records != nil {
		for _, record := range *records {
			results = append(results, *record.Ipv4Address)
		}
	}

	return results
}

func expandAzureRmDnsARecords(d *schema.ResourceData) ([]dns.ARecord, error) {
	recordStrings := d.Get("records").(*schema.Set).List()
	records := make([]dns.ARecord, len(recordStrings))

	for i, v := range recordStrings {
		ipv4 := v.(string)
		records[i] = dns.ARecord{
			Ipv4Address: &ipv4,
		}
	}

	return records, nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"github.com/stevencorona/elastic-haproxy/elb"
	"github.com/stevencorona/elastic-haproxy/haproxy"
	"github.com/stevencorona/elastic-haproxy/statsd"
	"log"
	"os"
	"os/signal"
	"syscall"
	"time"
)

var defaultConfigFile = "config/elastic.toml"
var flagConfigFile string

func main() {

	haproxy.Transform()
	os.Exit(1)

	flag.StringVar(&flagConfigFile, "configFile", defaultConfigFile, "Path to toml file")
	flag.Parse()

	conf := LoadConfig(flagConfigFile)

	server := new(haproxy.Server)

	// We use two channels— one to send actions to the server and one to recieve
	// notifications from it. Create them right now.
	actionChan := make(chan haproxy.Action)
	notificationChan := make(chan haproxy.Event)

	// Handle signals gracefully in another goroutine
	go gracefulSignals(server, notificationChan)

	// Start up the HAProxy Server
	go server.Start(notificationChan, actionChan)

	// Setup the ELB HTTP Handlers
	go elb.SetupApiHandlers()

	// Fire up statsd goroutine if statsd is enabled. This might be better off in
	// a seperate binary to monitor HAProxy.
	if conf.Statsd.Enabled {
		go statsd.SendMetrics(server)
	}

	// Event loop for handling events from the HAProxy server
	// (right now, it only sends start/stop notifications)
	for {
		<-notificationChan
		log.Println("Received a notification")
		time.Sleep(2 * time.Second)

		server.Socket = conf.Haproxy.Socket
		serverInfo := server.GetInfo()
		log.Println(serverInfo)
	}

}

func gracefulSignals(server *haproxy.Server) {
	signals := make(chan os.Signal, 1)
	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM, syscall.SIGKILL, syscall.SIGQUIT)

	for {
		s := <-signals
		log.Println("Received a signal", s)

		if s == syscall.SIGQUIT {
			log.Println("Caught SIGQUIT, Stopping HAProxy")
			server.ActionChan <- haproxy.WantsStop

			// Race condition, this exits before we stop :( It should wait!
			<-notificationChan
			os.Exit(1)
		}

		server.ActionChan <- haproxy.WantsReload
	}
}
<commit_msg>more docs<commit_after>package main

import (
	"flag"
	"github.com/stevencorona/elastic-haproxy/elb"
	"github.com/stevencorona/elastic-haproxy/haproxy"
	"github.com/stevencorona/elastic-haproxy/statsd"
	"log"
	"os"
	"os/signal"
	"syscall"
	"time"
)

var defaultConfigFile = "config/elastic.toml"
var flagConfigFile string

func main() {

	haproxy.Transform()
	os.Exit(1)

	flag.StringVar(&flagConfigFile, "configFile", defaultConfigFile, "Path to toml file")
	flag.Parse()

	conf := LoadConfig(flagConfigFile)

	server := new(haproxy.Server)

	// We use two channels— one to send actions to the server and one to recieve
	// notifications from it. Create them right now.
	actionChan := make(chan haproxy.Action)
	notificationChan := make(chan haproxy.Event)

	// Handle signals gracefully in another goroutine
	go gracefulSignals(server, notificationChan)

	// Start up the HAProxy Server
	go server.Start(notificationChan, actionChan)

	// Setup the ELB HTTP Handlers
	go elb.SetupApiHandlers()

	// Fire up statsd goroutine if statsd is enabled. This might be better off in
	// a seperate binary to monitor HAProxy.
	if conf.Statsd.Enabled {
		go statsd.SendMetrics(server)
	}

	// Event loop for handling events from the HAProxy server
	// (right now, it only sends start/stop notifications)
	for {
		<-notificationChan
		log.Println("Received a notification")
		time.Sleep(2 * time.Second)

		server.Socket = conf.Haproxy.Socket
		serverInfo := server.GetInfo()
		log.Println(serverInfo)
	}

}

func gracefulSignals(server *haproxy.Server) {
	signals := make(chan os.Signal, 1)
	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM, syscall.SIGKILL, syscall.SIGQUIT)

	for {
		s := <-signals
		log.Println("Received a signal", s)

		if s == syscall.SIGQUIT {
			log.Println("Caught SIGQUIT, Stopping HAProxy")

			// Tell server to stop and wait for a response
			server.ActionChan <- haproxy.WantsStop

			// Race condition, this exits before we stop :( It should wait!
			<-notificationChan
			os.Exit(1)
		}

		server.ActionChan <- haproxy.WantsReload
	}
}
<|endoftext|>


<commit_before>package local

import (
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"testing"
	"time"

	"github.com/ncw/rclone/fs"
	"github.com/ncw/rclone/fs/config/configmap"
	"github.com/ncw/rclone/fs/hash"
	"github.com/ncw/rclone/fstest"
	"github.com/ncw/rclone/lib/file"
	"github.com/ncw/rclone/lib/readers"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// TestMain drives the tests
func TestMain(m *testing.M) {
	fstest.TestMain(m)
}

func TestMapper(t *testing.T) {
	m := newMapper()
	assert.Equal(t, m.m, map[string]string{})
	assert.Equal(t, "potato", m.Save("potato", "potato"))
	assert.Equal(t, m.m, map[string]string{})
	assert.Equal(t, "-r'áö", m.Save("-r?'a´o¨", "-r'áö"))
	assert.Equal(t, m.m, map[string]string{
		"-r'áö": "-r?'a´o¨",
	})
	assert.Equal(t, "potato", m.Load("potato"))
	assert.Equal(t, "-r?'a´o¨", m.Load("-r'áö"))
}

// Test copy with source file that's updating
func TestUpdatingCheck(t *testing.T) {
	r := fstest.NewRun(t)
	defer r.Finalise()
	filePath := "sub dir/local test"
	r.WriteFile(filePath, "content", time.Now())

	fd, err := file.Open(path.Join(r.LocalName, filePath))
	if err != nil {
		t.Fatalf("failed opening file %q: %v", filePath, err)
	}

	fi, err := fd.Stat()
	require.NoError(t, err)
	o := &Object{size: fi.Size(), modTime: fi.ModTime(), fs: &Fs{}}
	wrappedFd := readers.NewLimitedReadCloser(fd, -1)
	hash, err := hash.NewMultiHasherTypes(hash.Supported)
	require.NoError(t, err)
	in := localOpenFile{
		o:    o,
		in:   wrappedFd,
		hash: hash,
		fd:   fd,
	}

	buf := make([]byte, 1)
	_, err = in.Read(buf)
	require.NoError(t, err)

	r.WriteFile(filePath, "content updated", time.Now())
	_, err = in.Read(buf)
	require.Errorf(t, err, "can't copy - source file is being updated")

	// turn the checking off and try again
	in.o.fs.opt.NoCheckUpdated = true

	r.WriteFile(filePath, "content updated", time.Now())
	_, err = in.Read(buf)
	require.NoError(t, err)

}

func TestSymlink(t *testing.T) {
	r := fstest.NewRun(t)
	defer r.Finalise()
	f := r.Flocal.(*Fs)
	dir := f.root

	// Write a file
	modTime1 := fstest.Time("2001-02-03T04:05:10.123123123Z")
	file1 := r.WriteFile("file.txt", "hello", modTime1)

	// Write a symlink
	modTime2 := fstest.Time("2002-02-03T04:05:10.123123123Z")
	symlinkPath := filepath.Join(dir, "symlink.txt")
	require.NoError(t, os.Symlink("file.txt", symlinkPath))
	require.NoError(t, lChtimes(symlinkPath, modTime2, modTime2))

	// Object viewed as symlink
	file2 := fstest.NewItem("symlink.txt"+linkSuffix, "file.txt", modTime2)

	// Object viewed as destination
	file2d := fstest.NewItem("symlink.txt", "hello", modTime1)

	// Check with no symlink flags
	fstest.CheckItems(t, r.Flocal, file1)
	fstest.CheckItems(t, r.Fremote)

	// Set fs into "-L" mode
	f.opt.FollowSymlinks = true
	f.opt.TranslateSymlinks = false
	f.lstat = os.Stat

	fstest.CheckItems(t, r.Flocal, file1, file2d)
	fstest.CheckItems(t, r.Fremote)

	// Set fs into "-l" mode
	f.opt.FollowSymlinks = false
	f.opt.TranslateSymlinks = true
	f.lstat = os.Lstat

	fstest.CheckListingWithPrecision(t, r.Flocal, []fstest.Item{file1, file2}, nil, fs.ModTimeNotSupported)
	if haveLChtimes {
		fstest.CheckItems(t, r.Flocal, file1, file2)
	}

	// Create a symlink
	modTime3 := fstest.Time("2002-03-03T04:05:10.123123123Z")
	file3 := r.WriteObjectTo(r.Flocal, "symlink2.txt"+linkSuffix, "file.txt", modTime3, false)
	fstest.CheckListingWithPrecision(t, r.Flocal, []fstest.Item{file1, file2, file3}, nil, fs.ModTimeNotSupported)
	if haveLChtimes {
		fstest.CheckItems(t, r.Flocal, file1, file2, file3)
	}

	// Check it got the correct contents
	symlinkPath = filepath.Join(dir, "symlink2.txt")
	fi, err := os.Lstat(symlinkPath)
	require.NoError(t, err)
	assert.False(t, fi.Mode().IsRegular())
	linkText, err := os.Readlink(symlinkPath)
	require.NoError(t, err)
	assert.Equal(t, "file.txt", linkText)

	// Check that NewObject gets the correct object
	o, err := r.Flocal.NewObject("symlink2.txt" + linkSuffix)
	require.NoError(t, err)
	assert.Equal(t, "symlink2.txt"+linkSuffix, o.Remote())
	assert.Equal(t, int64(8), o.Size())

	// Check that NewObject doesn't see the non suffixed version
	_, err = r.Flocal.NewObject("symlink2.txt")
	require.Equal(t, fs.ErrorObjectNotFound, err)

	// Check reading the object
	in, err := o.Open()
	require.NoError(t, err)
	contents, err := ioutil.ReadAll(in)
	require.NoError(t, err)
	require.Equal(t, "file.txt", string(contents))
	require.NoError(t, in.Close())

	// Check reading the object with range
	in, err = o.Open(&fs.RangeOption{Start: 2, End: 5})
	require.NoError(t, err)
	contents, err = ioutil.ReadAll(in)
	require.NoError(t, err)
	require.Equal(t, "file.txt"[2:5+1], string(contents))
	require.NoError(t, in.Close())
}

func TestSymlinkError(t *testing.T) {
	m := configmap.Simple{
		"links":      "true",
		"copy_links": "true",
	}
	_, err := NewFs("local", "/", m)
	assert.Equal(t, errLinksAndCopyLinks, err)
}
<commit_msg>local: make sure we close file handle in local tests<commit_after>package local

import (
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"testing"
	"time"

	"github.com/ncw/rclone/fs"
	"github.com/ncw/rclone/fs/config/configmap"
	"github.com/ncw/rclone/fs/hash"
	"github.com/ncw/rclone/fstest"
	"github.com/ncw/rclone/lib/file"
	"github.com/ncw/rclone/lib/readers"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// TestMain drives the tests
func TestMain(m *testing.M) {
	fstest.TestMain(m)
}

func TestMapper(t *testing.T) {
	m := newMapper()
	assert.Equal(t, m.m, map[string]string{})
	assert.Equal(t, "potato", m.Save("potato", "potato"))
	assert.Equal(t, m.m, map[string]string{})
	assert.Equal(t, "-r'áö", m.Save("-r?'a´o¨", "-r'áö"))
	assert.Equal(t, m.m, map[string]string{
		"-r'áö": "-r?'a´o¨",
	})
	assert.Equal(t, "potato", m.Load("potato"))
	assert.Equal(t, "-r?'a´o¨", m.Load("-r'áö"))
}

// Test copy with source file that's updating
func TestUpdatingCheck(t *testing.T) {
	r := fstest.NewRun(t)
	defer r.Finalise()
	filePath := "sub dir/local test"
	r.WriteFile(filePath, "content", time.Now())

	fd, err := file.Open(path.Join(r.LocalName, filePath))
	if err != nil {
		t.Fatalf("failed opening file %q: %v", filePath, err)
	}
	defer func() {
		require.NoError(t, fd.Close())
	}()

	fi, err := fd.Stat()
	require.NoError(t, err)
	o := &Object{size: fi.Size(), modTime: fi.ModTime(), fs: &Fs{}}
	wrappedFd := readers.NewLimitedReadCloser(fd, -1)
	hash, err := hash.NewMultiHasherTypes(hash.Supported)
	require.NoError(t, err)
	in := localOpenFile{
		o:    o,
		in:   wrappedFd,
		hash: hash,
		fd:   fd,
	}

	buf := make([]byte, 1)
	_, err = in.Read(buf)
	require.NoError(t, err)

	r.WriteFile(filePath, "content updated", time.Now())
	_, err = in.Read(buf)
	require.Errorf(t, err, "can't copy - source file is being updated")

	// turn the checking off and try again
	in.o.fs.opt.NoCheckUpdated = true

	r.WriteFile(filePath, "content updated", time.Now())
	_, err = in.Read(buf)
	require.NoError(t, err)

}

func TestSymlink(t *testing.T) {
	r := fstest.NewRun(t)
	defer r.Finalise()
	f := r.Flocal.(*Fs)
	dir := f.root

	// Write a file
	modTime1 := fstest.Time("2001-02-03T04:05:10.123123123Z")
	file1 := r.WriteFile("file.txt", "hello", modTime1)

	// Write a symlink
	modTime2 := fstest.Time("2002-02-03T04:05:10.123123123Z")
	symlinkPath := filepath.Join(dir, "symlink.txt")
	require.NoError(t, os.Symlink("file.txt", symlinkPath))
	require.NoError(t, lChtimes(symlinkPath, modTime2, modTime2))

	// Object viewed as symlink
	file2 := fstest.NewItem("symlink.txt"+linkSuffix, "file.txt", modTime2)

	// Object viewed as destination
	file2d := fstest.NewItem("symlink.txt", "hello", modTime1)

	// Check with no symlink flags
	fstest.CheckItems(t, r.Flocal, file1)
	fstest.CheckItems(t, r.Fremote)

	// Set fs into "-L" mode
	f.opt.FollowSymlinks = true
	f.opt.TranslateSymlinks = false
	f.lstat = os.Stat

	fstest.CheckItems(t, r.Flocal, file1, file2d)
	fstest.CheckItems(t, r.Fremote)

	// Set fs into "-l" mode
	f.opt.FollowSymlinks = false
	f.opt.TranslateSymlinks = true
	f.lstat = os.Lstat

	fstest.CheckListingWithPrecision(t, r.Flocal, []fstest.Item{file1, file2}, nil, fs.ModTimeNotSupported)
	if haveLChtimes {
		fstest.CheckItems(t, r.Flocal, file1, file2)
	}

	// Create a symlink
	modTime3 := fstest.Time("2002-03-03T04:05:10.123123123Z")
	file3 := r.WriteObjectTo(r.Flocal, "symlink2.txt"+linkSuffix, "file.txt", modTime3, false)
	fstest.CheckListingWithPrecision(t, r.Flocal, []fstest.Item{file1, file2, file3}, nil, fs.ModTimeNotSupported)
	if haveLChtimes {
		fstest.CheckItems(t, r.Flocal, file1, file2, file3)
	}

	// Check it got the correct contents
	symlinkPath = filepath.Join(dir, "symlink2.txt")
	fi, err := os.Lstat(symlinkPath)
	require.NoError(t, err)
	assert.False(t, fi.Mode().IsRegular())
	linkText, err := os.Readlink(symlinkPath)
	require.NoError(t, err)
	assert.Equal(t, "file.txt", linkText)

	// Check that NewObject gets the correct object
	o, err := r.Flocal.NewObject("symlink2.txt" + linkSuffix)
	require.NoError(t, err)
	assert.Equal(t, "symlink2.txt"+linkSuffix, o.Remote())
	assert.Equal(t, int64(8), o.Size())

	// Check that NewObject doesn't see the non suffixed version
	_, err = r.Flocal.NewObject("symlink2.txt")
	require.Equal(t, fs.ErrorObjectNotFound, err)

	// Check reading the object
	in, err := o.Open()
	require.NoError(t, err)
	contents, err := ioutil.ReadAll(in)
	require.NoError(t, err)
	require.Equal(t, "file.txt", string(contents))
	require.NoError(t, in.Close())

	// Check reading the object with range
	in, err = o.Open(&fs.RangeOption{Start: 2, End: 5})
	require.NoError(t, err)
	contents, err = ioutil.ReadAll(in)
	require.NoError(t, err)
	require.Equal(t, "file.txt"[2:5+1], string(contents))
	require.NoError(t, in.Close())
}

func TestSymlinkError(t *testing.T) {
	m := configmap.Simple{
		"links":      "true",
		"copy_links": "true",
	}
	_, err := NewFs("local", "/", m)
	assert.Equal(t, errLinksAndCopyLinks, err)
}
<|endoftext|>


<commit_before>package engine_test

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"log"
	"testing"

	"github.com/Sirupsen/logrus"
	"github.com/hashicorp/raft"
	"github.com/luizbafilho/fusis/config"
	"github.com/luizbafilho/fusis/engine"
	"github.com/luizbafilho/fusis/ipvs"
	"github.com/spf13/viper"

	_ "github.com/luizbafilho/fusis/provider/none" // to intialize
	. "gopkg.in/check.v1"
)

// Hook up gocheck into the "go test" runner.
func Test(t *testing.T) { TestingT(t) }

type EngineSuite struct {
	ipvs        *ipvs.Ipvs
	service     *ipvs.Service
	destination *ipvs.Destination
	engine      *engine.Engine
}

var _ = Suite(&EngineSuite{})

func (s *EngineSuite) SetUpSuite(c *C) {
	logrus.SetOutput(ioutil.Discard)
	s.readConfig()

	s.service = &ipvs.Service{
		Name:         "test",
		Host:         "10.0.1.1",
		Port:         80,
		Scheduler:    "lc",
		Protocol:     "tcp",
		Destinations: []ipvs.Destination{},
	}

	s.destination = &ipvs.Destination{
		Name:      "test",
		Host:      "192.168.1.1",
		Port:      80,
		Mode:      "nat",
		Weight:    1,
		ServiceId: "test",
	}
}

func (s *EngineSuite) SetUpTest(c *C) {
	eng, err := engine.New()
	c.Assert(err, IsNil)

	s.engine = eng

	go watchCommandCh(eng)
}

func (s *EngineSuite) TearDownTest(c *C) {
	s.ipvs.Flush()
}

func (s *EngineSuite) readConfig() {
	viper.SetConfigType("json")

	var sampleConfig = []byte(`
	{
		"provider":{
			"type": "none",
			"params": {
				"interface": "eth0",
				"vipRange": "192.168.0.0/28"
			}
		}
	}
	`)

	viper.ReadConfig(bytes.NewBuffer(sampleConfig))
	viper.Unmarshal(&config.Balancer)
}

func makeLog(cmd *engine.Command) *raft.Log {
	bytes, err := json.Marshal(cmd)
	if err != nil {
		log.Fatalf("err: %v", err)
	}

	return &raft.Log{
		Index: 1,
		Term:  1,
		Type:  raft.LogCommand,
		Data:  bytes,
	}
}

func watchCommandCh(engine *engine.Engine) {
	for {
		<-engine.CommandCh
	}
}

func (s *EngineSuite) addService(c *C) {
	cmd := &engine.Command{
		Op:      engine.AddServiceOp,
		Service: s.service,
	}

	resp := s.engine.Apply(makeLog(cmd))
	if resp != nil {
		c.Fatalf("resp: %v", resp)
	}
}

func (s *EngineSuite) delService(c *C) {
	cmd := &engine.Command{
		Op:      engine.DelServiceOp,
		Service: s.service,
	}

	resp := s.engine.Apply(makeLog(cmd))
	if resp != nil {
		c.Fatalf("resp: %v", resp)
	}
}

func (s *EngineSuite) addDestination(c *C) {
	cmd := &engine.Command{
		Op:          engine.AddDestinationOp,
		Service:     s.service,
		Destination: s.destination,
	}

	resp := s.engine.Apply(makeLog(cmd))
	if resp != nil {
		c.Fatalf("resp: %v", resp)
	}
}

func (s *EngineSuite) TestApplyAddService(c *C) {
	s.addService(c)

	c.Assert(s.engine.State.GetServices(), DeepEquals, &[]ipvs.Service{*s.service})
	svcs, err := s.engine.Ipvs.GetServices()
	c.Assert(err, IsNil)

	c.Assert(len(svcs), Equals, 1)
	c.Assert(svcs[0].Address.String(), DeepEquals, s.service.Host)
}

func (s *EngineSuite) TestApplyDelService(c *C) {
	s.addService(c)
	s.delService(c)

	c.Assert(s.engine.State.GetServices(), DeepEquals, &[]ipvs.Service{})
	svcs, err := s.engine.Ipvs.GetServices()
	c.Assert(err, IsNil)

	c.Assert(len(svcs), Equals, 0)
}

func (s *EngineSuite) TestApplyAddDestination(c *C) {
	s.addService(c)
	s.addDestination(c)

	dst, err := s.engine.State.GetDestination(s.destination.Name)
	c.Assert(err, IsNil)

	c.Assert(dst, DeepEquals, s.destination)
	dests, err := s.engine.Ipvs.GetDestinations(s.service.ToIpvsService())
	c.Assert(err, IsNil)

	c.Assert(len(dests), Equals, 1)
	c.Assert(dests[0].Address.String(), DeepEquals, s.destination.Host)
}

func (s *EngineSuite) TestApplyDelDestination(c *C) {
	s.addService(c)
	s.addDestination(c)

	cmd := &engine.Command{
		Op:          engine.DelDestinationOp,
		Service:     s.service,
		Destination: s.destination,
	}

	resp := s.engine.Apply(makeLog(cmd))
	if resp != nil {
		c.Fatalf("resp: %v", resp)
	}

	dests, err := s.engine.Ipvs.GetDestinations(s.service.ToIpvsService())
	c.Assert(err, IsNil)

	c.Assert(len(dests), Equals, 0)
}
<commit_msg>[engine] adding restore and persist tests to engine<commit_after>package engine_test

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"log"
	"testing"

	"github.com/Sirupsen/logrus"
	"github.com/hashicorp/raft"
	"github.com/luizbafilho/fusis/config"
	"github.com/luizbafilho/fusis/engine"
	"github.com/luizbafilho/fusis/ipvs"
	"github.com/spf13/viper"

	_ "github.com/luizbafilho/fusis/provider/none" // to intialize
	. "gopkg.in/check.v1"
)

// Hook up gocheck into the "go test" runner.
func Test(t *testing.T) { TestingT(t) }

type EngineSuite struct {
	ipvs        *ipvs.Ipvs
	service     *ipvs.Service
	destination *ipvs.Destination
	engine      *engine.Engine
}

var _ = Suite(&EngineSuite{})

func (s *EngineSuite) SetUpSuite(c *C) {
	logrus.SetOutput(ioutil.Discard)
	s.readConfig()

	s.service = &ipvs.Service{
		Name:         "test",
		Host:         "10.0.1.1",
		Port:         80,
		Scheduler:    "lc",
		Protocol:     "tcp",
		Destinations: []ipvs.Destination{},
	}

	s.destination = &ipvs.Destination{
		Name:      "test",
		Host:      "192.168.1.1",
		Port:      80,
		Mode:      "nat",
		Weight:    1,
		ServiceId: "test",
	}
}

func (s *EngineSuite) SetUpTest(c *C) {
	eng, err := engine.New()
	c.Assert(err, IsNil)

	s.engine = eng

	go watchCommandCh(eng)
}

func (s *EngineSuite) TearDownTest(c *C) {
	s.ipvs.Flush()
}

type MockSink struct {
	*bytes.Buffer
	cancel bool
}

func (m *MockSink) ID() string {
	return "Mock"
}

func (m *MockSink) Cancel() error {
	m.cancel = true
	return nil
}

func (m *MockSink) Close() error {
	return nil
}

func (s *EngineSuite) readConfig() {
	viper.SetConfigType("json")

	var sampleConfig = []byte(`
	{
		"provider":{
			"type": "none",
			"params": {
				"interface": "eth0",
				"vipRange": "192.168.0.0/28"
			}
		}
	}
	`)

	viper.ReadConfig(bytes.NewBuffer(sampleConfig))
	viper.Unmarshal(&config.Balancer)
}

func makeLog(cmd *engine.Command) *raft.Log {
	bytes, err := json.Marshal(cmd)
	if err != nil {
		log.Fatalf("err: %v", err)
	}

	return &raft.Log{
		Index: 1,
		Term:  1,
		Type:  raft.LogCommand,
		Data:  bytes,
	}
}

func watchCommandCh(engine *engine.Engine) {
	for {
		<-engine.CommandCh
	}
}

func (s *EngineSuite) addService(c *C) {
	cmd := &engine.Command{
		Op:      engine.AddServiceOp,
		Service: s.service,
	}

	resp := s.engine.Apply(makeLog(cmd))
	if resp != nil {
		c.Fatalf("resp: %v", resp)
	}
}

func (s *EngineSuite) delService(c *C) {
	cmd := &engine.Command{
		Op:      engine.DelServiceOp,
		Service: s.service,
	}

	resp := s.engine.Apply(makeLog(cmd))
	if resp != nil {
		c.Fatalf("resp: %v", resp)
	}
}

func (s *EngineSuite) addDestination(c *C) {
	cmd := &engine.Command{
		Op:          engine.AddDestinationOp,
		Service:     s.service,
		Destination: s.destination,
	}

	resp := s.engine.Apply(makeLog(cmd))
	if resp != nil {
		c.Fatalf("resp: %v", resp)
	}
}

func (s *EngineSuite) TestApplyAddService(c *C) {
	s.addService(c)

	c.Assert(s.engine.State.GetServices(), DeepEquals, &[]ipvs.Service{*s.service})
	svcs, err := s.engine.Ipvs.GetServices()
	c.Assert(err, IsNil)

	c.Assert(len(svcs), Equals, 1)
	c.Assert(svcs[0].Address.String(), DeepEquals, s.service.Host)
}

func (s *EngineSuite) TestApplyDelService(c *C) {
	s.addService(c)
	s.delService(c)

	c.Assert(s.engine.State.GetServices(), DeepEquals, &[]ipvs.Service{})
	svcs, err := s.engine.Ipvs.GetServices()
	c.Assert(err, IsNil)

	c.Assert(len(svcs), Equals, 0)
}

func (s *EngineSuite) TestApplyAddDestination(c *C) {
	s.addService(c)
	s.addDestination(c)

	dst, err := s.engine.State.GetDestination(s.destination.Name)
	c.Assert(err, IsNil)

	c.Assert(dst, DeepEquals, s.destination)
	dests, err := s.engine.Ipvs.GetDestinations(s.service.ToIpvsService())
	c.Assert(err, IsNil)

	c.Assert(len(dests), Equals, 1)
	c.Assert(dests[0].Address.String(), DeepEquals, s.destination.Host)
}

func (s *EngineSuite) TestApplyDelDestination(c *C) {
	s.addService(c)
	s.addDestination(c)

	cmd := &engine.Command{
		Op:          engine.DelDestinationOp,
		Service:     s.service,
		Destination: s.destination,
	}

	resp := s.engine.Apply(makeLog(cmd))
	if resp != nil {
		c.Fatalf("resp: %v", resp)
	}

	dests, err := s.engine.Ipvs.GetDestinations(s.service.ToIpvsService())
	c.Assert(err, IsNil)

	c.Assert(len(dests), Equals, 0)
}

func (s *EngineSuite) TestSnapshotRestore(c *C) {
	s.addService(c)
	s.addDestination(c)

	snap, err := s.engine.Snapshot()
	c.Assert(err, IsNil)
	defer snap.Release()

	buf := bytes.NewBuffer(nil)
	sink := &MockSink{buf, false}
	err = snap.Persist(sink)
	c.Assert(err, IsNil)

	s.engine.Ipvs.Flush()

	eng, err := engine.New()
	c.Assert(err, IsNil)

	err = eng.Restore(sink)
	c.Assert(err, IsNil)

	s.service.Destinations = []ipvs.Destination{*s.destination}

	c.Assert(eng.State.GetServices(), DeepEquals, &[]ipvs.Service{*s.service})

	svcs, err := s.engine.Ipvs.GetServices()
	c.Assert(err, IsNil)

	c.Assert(len(svcs), Equals, 1)
	c.Assert(svcs[0].Address.String(), DeepEquals, s.service.Host)
	c.Assert(svcs[0].Destinations[0].Address.String(), Equals, s.destination.Host)
}
<|endoftext|>