<commit_before>// Copyright (c) 2016-2017 The btcsuite developers
// Copyright (c) 2016-2017 The Lightning Network Developers
// Copyright (c) 2018-2019 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package gcs

import (
	"encoding/binary"
	"fmt"
	"math"
	"sort"
	"sync"

	"github.com/dchest/siphash"
	"github.com/decred/dcrd/chaincfg/chainhash"
	"github.com/decred/dcrd/crypto/blake256"
)

// Inspired by https://github.com/rasky/gcs

// KeySize is the size of the byte array required for key material for the
// SipHash keyed hash function.
const KeySize = 16

// uint64s implements sort.Interface for *[]uint64
type uint64s []uint64

func (s *uint64s) Len() int           { return len(*s) }
func (s *uint64s) Less(i, j int) bool { return (*s)[i] < (*s)[j] }
func (s *uint64s) Swap(i, j int)      { (*s)[i], (*s)[j] = (*s)[j], (*s)[i] }

// filter describes a versioned immutable filter that can be built from a set of
// data elements, serialized, deserialized, and queried in a thread-safe manner.
//
// It is used internally to implement the exported filter version types.
//
// See FilterV1 for more details.
type filter struct {
	version     uint16
	n           uint32
	p           uint8
	modulusNP   uint64
	filterNData []byte
	filterData  []byte // Slice into filterNData with raw filter bytes.
}

// Filter describes an immutable filter that can be built from a set of data
// elements, serialized, deserialized, and queried in a thread-safe manner. The
// serialized form is compressed as a Golomb Coded Set (GCS), but does not
// include N or P to allow the user to encode the metadata separately if
// necessary. The hash function used is SipHash, a keyed function; the key used
// in building the filter is required in order to match filter values and is
// not included in the serialized form.
type FilterV1 struct {
	filter
}

// newFilter builds a new GCS filter of the specified version with the collision
// probability of `1/(2**P)`, key `key`, and including every `[]byte` in `data`
// as a member of the set.
func newFilter(version uint16, P uint8, key [KeySize]byte, data [][]byte) (*filter, error) {
	if len(data) > math.MaxInt32 {
		str := fmt.Sprintf("unable to create filter with %d entries greater "+
			"than max allowed %d", len(data), math.MaxInt32)
		return nil, makeError(ErrNTooBig, str)
	}
	if P > 32 {
		str := fmt.Sprintf("P value of %d is greater than max allowed 32", P)
		return nil, makeError(ErrPTooBig, str)
	}

	// Create the filter object and insert metadata.
	modP := uint64(1 << P)
	modPMask := modP - 1
	f := filter{
		version:   version,
		n:         uint32(len(data)),
		p:         P,
		modulusNP: uint64(len(data)) * modP,
	}

	// Nothing to do for an empty filter.
	if len(data) == 0 {
		return &f, nil
	}

	// Allocate filter data.
	values := make([]uint64, 0, len(data))

	// Insert the hash (modulo N*P) of each data element into a slice and
	// sort the slice.
	k0 := binary.LittleEndian.Uint64(key[0:8])
	k1 := binary.LittleEndian.Uint64(key[8:16])
	for _, d := range data {
		v := siphash.Hash(k0, k1, d) % f.modulusNP
		values = append(values, v)
	}
	sort.Sort((*uint64s)(&values))

	var b bitWriter

	// Write the sorted list of values into the filter bitstream,
	// compressing it using Golomb coding.
	var value, lastValue, remainder uint64
	for _, v := range values {
		// Calculate the difference between this value and the last,
		// modulo P.
		remainder = (v - lastValue) & modPMask

		// Calculate the difference between this value and the last,
		// divided by P.
		value = (v - lastValue - remainder) >> f.p
		lastValue = v

		// Write the P multiple into the bitstream in unary; the
		// average should be around 1 (2 bits - 0b10).
		for value > 0 {
			b.writeOne()
			value--
		}
		b.writeZero()

		// Write the remainder as a big-endian integer with enough bits
		// to represent the appropriate collision probability.
		b.writeNBits(remainder, uint(f.p))
	}

	// Save the filter data internally as n + filter bytes
	ndata := make([]byte, 4+len(b.bytes))
	binary.BigEndian.PutUint32(ndata, f.n)
	copy(ndata[4:], b.bytes)
	f.filterNData = ndata
	f.filterData = ndata[4:]

	return &f, nil
}

// NewFilter builds a new version 1 GCS filter with the collision probability of
// `1/(2**P)`, key `key`, and including every `[]byte` in `data` as a member of
// the set.
func NewFilterV1(P uint8, key [KeySize]byte, data [][]byte) (*FilterV1, error) {
	filter, err := newFilter(1, P, key, data)
	if err != nil {
		return nil, err
	}
	return &FilterV1{filter: *filter}, nil
}

// FromBytesV1 deserializes a version 1 GCS filter from a known P and serialized
// filter as returned by Bytes().
func FromBytesV1(P uint8, d []byte) (*FilterV1, error) {
	// Basic sanity check.
	if P > 32 {
		str := fmt.Sprintf("P value of %d is greater than max allowed 32", P)
		return nil, makeError(ErrPTooBig, str)
	}

	var n uint32
	var filterData []byte
	if len(d) >= 4 {
		n = binary.BigEndian.Uint32(d[:4])
		filterData = d[4:]
	} else if len(d) < 4 && len(d) != 0 {
		str := "number of items serialization missing"
		return nil, makeError(ErrMisserialized, str)
	}

	f := filter{
		version:     1,
		n:           n,
		p:           P,
		modulusNP:   uint64(n) * uint64(1<<P),
		filterNData: d,
		filterData:  filterData,
	}
	return &FilterV1{filter: f}, nil
}

// Bytes returns the serialized format of the GCS filter which includes N, but
// does not include P (returned by a separate method) or the key used by
// SipHash.
func (f *filter) Bytes() []byte {
	return f.filterNData
}

// P returns the filter's collision probability as a negative power of 2 (that
// is, a collision probability of `1/2**20` is represented as 20).
func (f *filter) P() uint8 {
	return f.p
}

// N returns the size of the data set used to build the filter.
func (f *filter) N() uint32 {
	return f.n
}

// Match checks whether a []byte value is likely (within collision probability)
// to be a member of the set represented by the filter.
func (f *filter) Match(key [KeySize]byte, data []byte) bool {
	// An empty filter or empty data can't possibly match anything.
	if len(f.filterData) == 0 || len(data) == 0 {
		return false
	}

	// Create a filter bitstream.
	b := newBitReader(f.filterData)

	// Hash our search term with the same parameters as the filter.
	k0 := binary.LittleEndian.Uint64(key[0:8])
	k1 := binary.LittleEndian.Uint64(key[8:16])
	term := siphash.Hash(k0, k1, data) % f.modulusNP

	// Go through the search filter and look for the desired value.
	var lastValue uint64
	for lastValue <= term {
		// Read the difference between previous and new value from
		// bitstream.
		value, err := f.readFullUint64(&b)
		if err != nil {
			return false
		}

		// Add the previous value to it.
		value += lastValue
		if value == term {
			return true
		}

		lastValue = value
	}

	return false
}

// matchPool pools allocations for match data.
var matchPool sync.Pool

// MatchAny checks whether any []byte value is likely (within collision
// probability) to be a member of the set represented by the filter faster than
// calling Match() for each value individually.
func (f *filter) MatchAny(key [KeySize]byte, data [][]byte) bool {
	// An empty filter or empty data can't possibly match anything.
	if len(f.filterData) == 0 || len(data) == 0 {
		return false
	}

	// Create a filter bitstream.
	b := newBitReader(f.filterData)

	// Create an uncompressed filter of the search values.
	var values *[]uint64
	if v := matchPool.Get(); v != nil {
		values = v.(*[]uint64)
		*values = (*values)[:0]
	} else {
		vs := make([]uint64, 0, len(data))
		values = &vs
	}
	defer matchPool.Put(values)
	k0 := binary.LittleEndian.Uint64(key[0:8])
	k1 := binary.LittleEndian.Uint64(key[8:16])
	for _, d := range data {
		v := siphash.Hash(k0, k1, d) % f.modulusNP
		*values = append(*values, v)
	}
	sort.Sort((*uint64s)(values))

	// Zip down the filters, comparing values until we either run out of
	// values to compare in one of the filters or we reach a matching
	// value.
	searchSize := len(data)
	var searchIdx int
	var filterVal uint64
nextFilterVal:
	for i := uint32(0); i < f.n; i++ {
		// Read the next item to compare from the filter.
		delta, err := f.readFullUint64(&b)
		if err != nil {
			return false
		}
		filterVal += delta

		// Iterate through the values to search until either a match is found
		// or the search value exceeds the current filter value.
		for ; searchIdx < searchSize; searchIdx++ {
			searchVal := (*values)[searchIdx]
			if searchVal == filterVal {
				return true
			}

			// Move to the next filter item once the current search value
			// exceeds it.
			if searchVal > filterVal {
				continue nextFilterVal
			}
		}

		// Exit early when there are no more values to search for.
		break
	}

	return false
}

// readFullUint64 reads a value represented by the sum of a unary multiple of
// the filter's P modulus (`2**P`) and a big-endian P-bit remainder.
func (f *filter) readFullUint64(b *bitReader) (uint64, error) {
	v, err := b.readUnary()
	if err != nil {
		return 0, err
	}

	rem, err := b.readNBits(uint(f.p))
	if err != nil {
		return 0, err
	}

	// Add the multiple and the remainder.
	return v<<f.p + rem, nil
}

// Hash returns the BLAKE256 hash of the filter.
func (f *filter) Hash() chainhash.Hash {
	// Empty filters have a hash of all zeroes.
	if len(f.filterNData) == 0 {
		return chainhash.Hash{}
	}

	return chainhash.Hash(blake256.Sum256(f.filterNData))
}

// MakeHeaderForFilter makes a filter chain header for a filter, given the
// filter and the previous filter chain header.
func MakeHeaderForFilter(filter *FilterV1, prevHeader *chainhash.Hash) chainhash.Hash {
	// In the buffer we created above we'll compute hash || prevHash as an
	// intermediate value.
	var filterTip [2 * chainhash.HashSize]byte
	filterHash := filter.Hash()
	copy(filterTip[:], filterHash[:])
	copy(filterTip[chainhash.HashSize:], prevHeader[:])

	// The final filter hash is the blake256 of the hash computed above.
	return chainhash.Hash(blake256.Sum256(filterTip[:]))
}
<commit_msg>gcs: Group V1 filter funcs after filter defs.<commit_after>// Copyright (c) 2016-2017 The btcsuite developers
// Copyright (c) 2016-2017 The Lightning Network Developers
// Copyright (c) 2018-2019 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package gcs

import (
	"encoding/binary"
	"fmt"
	"math"
	"sort"
	"sync"

	"github.com/dchest/siphash"
	"github.com/decred/dcrd/chaincfg/chainhash"
	"github.com/decred/dcrd/crypto/blake256"
)

// Inspired by https://github.com/rasky/gcs

// KeySize is the size of the byte array required for key material for the
// SipHash keyed hash function.
const KeySize = 16

// uint64s implements sort.Interface for *[]uint64
type uint64s []uint64

func (s *uint64s) Len() int           { return len(*s) }
func (s *uint64s) Less(i, j int) bool { return (*s)[i] < (*s)[j] }
func (s *uint64s) Swap(i, j int)      { (*s)[i], (*s)[j] = (*s)[j], (*s)[i] }

// filter describes a versioned immutable filter that can be built from a set of
// data elements, serialized, deserialized, and queried in a thread-safe manner.
//
// It is used internally to implement the exported filter version types.
//
// See FilterV1 for more details.
type filter struct {
	version     uint16
	n           uint32
	p           uint8
	modulusNP   uint64
	filterNData []byte
	filterData  []byte // Slice into filterNData with raw filter bytes.
}

// newFilter builds a new GCS filter of the specified version with the collision
// probability of `1/(2**P)`, key `key`, and including every `[]byte` in `data`
// as a member of the set.
func newFilter(version uint16, P uint8, key [KeySize]byte, data [][]byte) (*filter, error) {
	if len(data) > math.MaxInt32 {
		str := fmt.Sprintf("unable to create filter with %d entries greater "+
			"than max allowed %d", len(data), math.MaxInt32)
		return nil, makeError(ErrNTooBig, str)
	}
	if P > 32 {
		str := fmt.Sprintf("P value of %d is greater than max allowed 32", P)
		return nil, makeError(ErrPTooBig, str)
	}

	// Create the filter object and insert metadata.
	modP := uint64(1 << P)
	modPMask := modP - 1
	f := filter{
		version:   version,
		n:         uint32(len(data)),
		p:         P,
		modulusNP: uint64(len(data)) * modP,
	}

	// Nothing to do for an empty filter.
	if len(data) == 0 {
		return &f, nil
	}

	// Allocate filter data.
	values := make([]uint64, 0, len(data))

	// Insert the hash (modulo N*P) of each data element into a slice and
	// sort the slice.
	k0 := binary.LittleEndian.Uint64(key[0:8])
	k1 := binary.LittleEndian.Uint64(key[8:16])
	for _, d := range data {
		v := siphash.Hash(k0, k1, d) % f.modulusNP
		values = append(values, v)
	}
	sort.Sort((*uint64s)(&values))

	var b bitWriter

	// Write the sorted list of values into the filter bitstream,
	// compressing it using Golomb coding.
	var value, lastValue, remainder uint64
	for _, v := range values {
		// Calculate the difference between this value and the last,
		// modulo P.
		remainder = (v - lastValue) & modPMask

		// Calculate the difference between this value and the last,
		// divided by P.
		value = (v - lastValue - remainder) >> f.p
		lastValue = v

		// Write the P multiple into the bitstream in unary; the
		// average should be around 1 (2 bits - 0b10).
		for value > 0 {
			b.writeOne()
			value--
		}
		b.writeZero()

		// Write the remainder as a big-endian integer with enough bits
		// to represent the appropriate collision probability.
		b.writeNBits(remainder, uint(f.p))
	}

	// Save the filter data internally as n + filter bytes
	ndata := make([]byte, 4+len(b.bytes))
	binary.BigEndian.PutUint32(ndata, f.n)
	copy(ndata[4:], b.bytes)
	f.filterNData = ndata
	f.filterData = ndata[4:]

	return &f, nil
}

// Bytes returns the serialized format of the GCS filter which includes N, but
// does not include P (returned by a separate method) or the key used by
// SipHash.
func (f *filter) Bytes() []byte {
	return f.filterNData
}

// P returns the filter's collision probability as a negative power of 2 (that
// is, a collision probability of `1/2**20` is represented as 20).
func (f *filter) P() uint8 {
	return f.p
}

// N returns the size of the data set used to build the filter.
func (f *filter) N() uint32 {
	return f.n
}

// readFullUint64 reads a value represented by the sum of a unary multiple of
// the filter's P modulus (`2**P`) and a big-endian P-bit remainder.
func (f *filter) readFullUint64(b *bitReader) (uint64, error) {
	v, err := b.readUnary()
	if err != nil {
		return 0, err
	}

	rem, err := b.readNBits(uint(f.p))
	if err != nil {
		return 0, err
	}

	// Add the multiple and the remainder.
	return v<<f.p + rem, nil
}

// Match checks whether a []byte value is likely (within collision probability)
// to be a member of the set represented by the filter.
func (f *filter) Match(key [KeySize]byte, data []byte) bool {
	// An empty filter or empty data can't possibly match anything.
	if len(f.filterData) == 0 || len(data) == 0 {
		return false
	}

	// Create a filter bitstream.
	b := newBitReader(f.filterData)

	// Hash our search term with the same parameters as the filter.
	k0 := binary.LittleEndian.Uint64(key[0:8])
	k1 := binary.LittleEndian.Uint64(key[8:16])
	term := siphash.Hash(k0, k1, data) % f.modulusNP

	// Go through the search filter and look for the desired value.
	var lastValue uint64
	for lastValue <= term {
		// Read the difference between previous and new value from
		// bitstream.
		value, err := f.readFullUint64(&b)
		if err != nil {
			return false
		}

		// Add the previous value to it.
		value += lastValue
		if value == term {
			return true
		}

		lastValue = value
	}

	return false
}

// matchPool pools allocations for match data.
var matchPool sync.Pool

// MatchAny checks whether any []byte value is likely (within collision
// probability) to be a member of the set represented by the filter faster than
// calling Match() for each value individually.
func (f *filter) MatchAny(key [KeySize]byte, data [][]byte) bool {
	// An empty filter or empty data can't possibly match anything.
	if len(f.filterData) == 0 || len(data) == 0 {
		return false
	}

	// Create a filter bitstream.
	b := newBitReader(f.filterData)

	// Create an uncompressed filter of the search values.
	var values *[]uint64
	if v := matchPool.Get(); v != nil {
		values = v.(*[]uint64)
		*values = (*values)[:0]
	} else {
		vs := make([]uint64, 0, len(data))
		values = &vs
	}
	defer matchPool.Put(values)
	k0 := binary.LittleEndian.Uint64(key[0:8])
	k1 := binary.LittleEndian.Uint64(key[8:16])
	for _, d := range data {
		v := siphash.Hash(k0, k1, d) % f.modulusNP
		*values = append(*values, v)
	}
	sort.Sort((*uint64s)(values))

	// Zip down the filters, comparing values until we either run out of
	// values to compare in one of the filters or we reach a matching
	// value.
	searchSize := len(data)
	var searchIdx int
	var filterVal uint64
nextFilterVal:
	for i := uint32(0); i < f.n; i++ {
		// Read the next item to compare from the filter.
		delta, err := f.readFullUint64(&b)
		if err != nil {
			return false
		}
		filterVal += delta

		// Iterate through the values to search until either a match is found
		// or the search value exceeds the current filter value.
		for ; searchIdx < searchSize; searchIdx++ {
			searchVal := (*values)[searchIdx]
			if searchVal == filterVal {
				return true
			}

			// Move to the next filter item once the current search value
			// exceeds it.
			if searchVal > filterVal {
				continue nextFilterVal
			}
		}

		// Exit early when there are no more values to search for.
		break
	}

	return false
}

// Hash returns the BLAKE256 hash of the filter.
func (f *filter) Hash() chainhash.Hash {
	// Empty filters have a hash of all zeroes.
	if len(f.filterNData) == 0 {
		return chainhash.Hash{}
	}

	return chainhash.Hash(blake256.Sum256(f.filterNData))
}

// Filter describes an immutable filter that can be built from a set of data
// elements, serialized, deserialized, and queried in a thread-safe manner. The
// serialized form is compressed as a Golomb Coded Set (GCS), but does not
// include N or P to allow the user to encode the metadata separately if
// necessary. The hash function used is SipHash, a keyed function; the key used
// in building the filter is required in order to match filter values and is
// not included in the serialized form.
type FilterV1 struct {
	filter
}

// NewFilter builds a new version 1 GCS filter with the collision probability of
// `1/(2**P)`, key `key`, and including every `[]byte` in `data` as a member of
// the set.
func NewFilterV1(P uint8, key [KeySize]byte, data [][]byte) (*FilterV1, error) {
	filter, err := newFilter(1, P, key, data)
	if err != nil {
		return nil, err
	}
	return &FilterV1{filter: *filter}, nil
}

// FromBytesV1 deserializes a version 1 GCS filter from a known P and serialized
// filter as returned by Bytes().
func FromBytesV1(P uint8, d []byte) (*FilterV1, error) {
	// Basic sanity check.
	if P > 32 {
		str := fmt.Sprintf("P value of %d is greater than max allowed 32", P)
		return nil, makeError(ErrPTooBig, str)
	}

	var n uint32
	var filterData []byte
	if len(d) >= 4 {
		n = binary.BigEndian.Uint32(d[:4])
		filterData = d[4:]
	} else if len(d) < 4 && len(d) != 0 {
		str := "number of items serialization missing"
		return nil, makeError(ErrMisserialized, str)
	}

	f := filter{
		version:     1,
		n:           n,
		p:           P,
		modulusNP:   uint64(n) * uint64(1<<P),
		filterNData: d,
		filterData:  filterData,
	}
	return &FilterV1{filter: f}, nil
}

// MakeHeaderForFilter makes a filter chain header for a filter, given the
// filter and the previous filter chain header.
func MakeHeaderForFilter(filter *FilterV1, prevHeader *chainhash.Hash) chainhash.Hash {
	// In the buffer we created above we'll compute hash || prevHash as an
	// intermediate value.
	var filterTip [2 * chainhash.HashSize]byte
	filterHash := filter.Hash()
	copy(filterTip[:], filterHash[:])
	copy(filterTip[chainhash.HashSize:], prevHeader[:])

	// The final filter hash is the blake256 of the hash computed above.
	return chainhash.Hash(blake256.Sum256(filterTip[:]))
}
<|endoftext|>


<commit_before>// +build linux

/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package console

import (
	"io"
	"os"
	"sync"

	"golang.org/x/sys/unix"
)

const (
	maxEvents = 128
)

// Epoller manages multiple epoll consoles using edge-triggered epoll api so we
// dont have to deal with repeated wake-up of EPOLLER or EPOLLHUP.
// For more details, see:
// - https://github.com/systemd/systemd/pull/4262
// - https://github.com/moby/moby/issues/27202
//
// Example usage of Epoller and EpollConsole can be as follow:
//
//	epoller, _ := NewEpoller()
//	epollConsole, _ := epoller.Add(console)
//	go epoller.Wait()
//	var (
//		b  bytes.Buffer
//		wg sync.WaitGroup
//	)
//	wg.Add(1)
//	go func() {
//		io.Copy(&b, epollConsole)
//		wg.Done()
//	}()
//	// perform I/O on the console
//	epollConsole.Shutdown(epoller.CloseConsole)
//	wg.Wait()
//	epollConsole.Close()
type Epoller struct {
	efd       int
	mu        sync.Mutex
	fdMapping map[int]*EpollConsole
}

// NewEpoller returns an instance of epoller with a valid epoll fd.
func NewEpoller() (*Epoller, error) {
	efd, err := unix.EpollCreate1(unix.EPOLL_CLOEXEC)
	if err != nil {
		return nil, err
	}
	return &Epoller{
		efd:       efd,
		fdMapping: make(map[int]*EpollConsole),
	}, nil
}

// Add creates an epoll console based on the provided console. The console will
// be registered with EPOLLET (i.e. using edge-triggered notification) and its
// file descriptor will be set to non-blocking mode. After this, user should use
// the return console to perform I/O.
func (e *Epoller) Add(console Console) (*EpollConsole, error) {
	sysfd := int(console.Fd())
	// Set sysfd to non-blocking mode
	if err := unix.SetNonblock(sysfd, true); err != nil {
		return nil, err
	}

	ev := unix.EpollEvent{
		Events: unix.EPOLLIN | unix.EPOLLOUT | unix.EPOLLRDHUP | unix.EPOLLET,
		Fd:     int32(sysfd),
	}
	if err := unix.EpollCtl(e.efd, unix.EPOLL_CTL_ADD, sysfd, &ev); err != nil {
		return nil, err
	}
	ef := &EpollConsole{
		Console: console,
		sysfd:   sysfd,
		readc:   sync.NewCond(&sync.Mutex{}),
		writec:  sync.NewCond(&sync.Mutex{}),
	}
	e.mu.Lock()
	e.fdMapping[sysfd] = ef
	e.mu.Unlock()
	return ef, nil
}

// Wait starts the loop to wait for its consoles' notifications and signal
// appropriate console that it can perform I/O.
func (e *Epoller) Wait() error {
	events := make([]unix.EpollEvent, maxEvents)
	for {
		n, err := unix.EpollWait(e.efd, events, -1)
		if err != nil {
			// EINTR: The call was interrupted by a signal handler before either
			// any of the requested events occurred or the timeout expired
			if err == unix.EINTR {
				continue
			}
			return err
		}
		for i := 0; i < n; i++ {
			ev := &events[i]
			// the console is ready to be read from
			if ev.Events&(unix.EPOLLIN|unix.EPOLLHUP|unix.EPOLLERR) != 0 {
				if epfile := e.getConsole(int(ev.Fd)); epfile != nil {
					epfile.signalRead()
				}
			}
			// the console is ready to be written to
			if ev.Events&(unix.EPOLLOUT|unix.EPOLLHUP|unix.EPOLLERR) != 0 {
				if epfile := e.getConsole(int(ev.Fd)); epfile != nil {
					epfile.signalWrite()
				}
			}
		}
	}
}

// CloseConsole unregisters the console's file descriptor from epoll interface
func (e *Epoller) CloseConsole(fd int) error {
	e.mu.Lock()
	defer e.mu.Unlock()
	delete(e.fdMapping, fd)
	return unix.EpollCtl(e.efd, unix.EPOLL_CTL_DEL, fd, &unix.EpollEvent{})
}

func (e *Epoller) getConsole(sysfd int) *EpollConsole {
	e.mu.Lock()
	f := e.fdMapping[sysfd]
	e.mu.Unlock()
	return f
}

// Close closes the epoll fd
func (e *Epoller) Close() error {
	return unix.Close(e.efd)
}

// EpollConsole acts like a console but registers its file descriptor with an
// epoll fd and uses epoll API to perform I/O.
type EpollConsole struct {
	Console
	readc  *sync.Cond
	writec *sync.Cond
	sysfd  int
	closed bool
}

// Read reads up to len(p) bytes into p. It returns the number of bytes read
// (0 <= n <= len(p)) and any error encountered.
//
// If the console's read returns EAGAIN or EIO, we assume that it's a
// temporary error because the other side went away and wait for the signal
// generated by epoll event to continue.
func (ec *EpollConsole) Read(p []byte) (n int, err error) {
	var read int
	ec.readc.L.Lock()
	defer ec.readc.L.Unlock()
	for {
		read, err = ec.Console.Read(p[n:])
		n += read
		if err != nil {
			var hangup bool
			if perr, ok := err.(*os.PathError); ok {
				hangup = (perr.Err == unix.EAGAIN || perr.Err == unix.EIO)
			} else {
				hangup = (err == unix.EAGAIN || err == unix.EIO)
			}
			// if the other end disappear, assume this is temporary and wait for the
			// signal to continue again. Unless we didnt read anything and the
			// console is already marked as closed then we should exit
			if hangup && !(n == 0 && len(p) > 0 && ec.closed) {
				ec.readc.Wait()
				continue
			}
		}
		break
	}
	// if we didnt read anything then return io.EOF to end gracefully
	if n == 0 && len(p) > 0 && err == nil {
		err = io.EOF
	}
	// signal for others that we finished the read
	ec.readc.Signal()
	return n, err
}

// Writes len(p) bytes from p to the console. It returns the number of bytes
// written from p (0 <= n <= len(p)) and any error encountered that caused
// the write to stop early.
//
// If writes to the console returns EAGAIN or EIO, we assume that it's a
// temporary error because the other side went away and wait for the signal
// generated by epoll event to continue.
func (ec *EpollConsole) Write(p []byte) (n int, err error) {
	var written int
	ec.writec.L.Lock()
	defer ec.writec.L.Unlock()
	for {
		written, err = ec.Console.Write(p[n:])
		n += written
		if err != nil {
			var hangup bool
			if perr, ok := err.(*os.PathError); ok {
				hangup = (perr.Err == unix.EAGAIN || perr.Err == unix.EIO)
			} else {
				hangup = (err == unix.EAGAIN || err == unix.EIO)
			}
			// if the other end disappears, assume this is temporary and wait for the
			// signal to continue again.
			if hangup {
				ec.writec.Wait()
				continue
			}
		}
		// unrecoverable error, break the loop and return the error
		break
	}
	if n < len(p) && err == nil {
		err = io.ErrShortWrite
	}
	// signal for others that we finished the write
	ec.writec.Signal()
	return n, err
}

// Shutdown closes the file descriptor and signals call waiters for this fd.
// It accepts a callback which will be called with the console's fd. The
// callback typically will be used to do further cleanup such as unregister the
// console's fd from the epoll interface.
// User should call Shutdown and wait for all I/O operation to be finished
// before closing the console.
func (ec *EpollConsole) Shutdown(close func(int) error) error {
	ec.readc.L.Lock()
	defer ec.readc.L.Unlock()
	ec.writec.L.Lock()
	defer ec.writec.L.Unlock()

	ec.readc.Broadcast()
	ec.writec.Broadcast()
	ec.closed = true
	return close(ec.sysfd)
}

// signalRead signals that the console is readable.
func (ec *EpollConsole) signalRead() {
	ec.readc.Signal()
}

// signalWrite signals that the console is writable.
func (ec *EpollConsole) signalWrite() {
	ec.writec.Signal()
}
<commit_msg>console_linux: Fix race: lock Cond before Signal.<commit_after>// +build linux

/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package console

import (
	"io"
	"os"
	"sync"

	"golang.org/x/sys/unix"
)

const (
	maxEvents = 128
)

// Epoller manages multiple epoll consoles using edge-triggered epoll api so we
// dont have to deal with repeated wake-up of EPOLLER or EPOLLHUP.
// For more details, see:
// - https://github.com/systemd/systemd/pull/4262
// - https://github.com/moby/moby/issues/27202
//
// Example usage of Epoller and EpollConsole can be as follow:
//
//	epoller, _ := NewEpoller()
//	epollConsole, _ := epoller.Add(console)
//	go epoller.Wait()
//	var (
//		b  bytes.Buffer
//		wg sync.WaitGroup
//	)
//	wg.Add(1)
//	go func() {
//		io.Copy(&b, epollConsole)
//		wg.Done()
//	}()
//	// perform I/O on the console
//	epollConsole.Shutdown(epoller.CloseConsole)
//	wg.Wait()
//	epollConsole.Close()
type Epoller struct {
	efd       int
	mu        sync.Mutex
	fdMapping map[int]*EpollConsole
}

// NewEpoller returns an instance of epoller with a valid epoll fd.
func NewEpoller() (*Epoller, error) {
	efd, err := unix.EpollCreate1(unix.EPOLL_CLOEXEC)
	if err != nil {
		return nil, err
	}
	return &Epoller{
		efd:       efd,
		fdMapping: make(map[int]*EpollConsole),
	}, nil
}

// Add creates an epoll console based on the provided console. The console will
// be registered with EPOLLET (i.e. using edge-triggered notification) and its
// file descriptor will be set to non-blocking mode. After this, user should use
// the return console to perform I/O.
func (e *Epoller) Add(console Console) (*EpollConsole, error) {
	sysfd := int(console.Fd())
	// Set sysfd to non-blocking mode
	if err := unix.SetNonblock(sysfd, true); err != nil {
		return nil, err
	}

	ev := unix.EpollEvent{
		Events: unix.EPOLLIN | unix.EPOLLOUT | unix.EPOLLRDHUP | unix.EPOLLET,
		Fd:     int32(sysfd),
	}
	if err := unix.EpollCtl(e.efd, unix.EPOLL_CTL_ADD, sysfd, &ev); err != nil {
		return nil, err
	}
	ef := &EpollConsole{
		Console: console,
		sysfd:   sysfd,
		readc:   sync.NewCond(&sync.Mutex{}),
		writec:  sync.NewCond(&sync.Mutex{}),
	}
	e.mu.Lock()
	e.fdMapping[sysfd] = ef
	e.mu.Unlock()
	return ef, nil
}

// Wait starts the loop to wait for its consoles' notifications and signal
// appropriate console that it can perform I/O.
func (e *Epoller) Wait() error {
	events := make([]unix.EpollEvent, maxEvents)
	for {
		n, err := unix.EpollWait(e.efd, events, -1)
		if err != nil {
			// EINTR: The call was interrupted by a signal handler before either
			// any of the requested events occurred or the timeout expired
			if err == unix.EINTR {
				continue
			}
			return err
		}
		for i := 0; i < n; i++ {
			ev := &events[i]
			// the console is ready to be read from
			if ev.Events&(unix.EPOLLIN|unix.EPOLLHUP|unix.EPOLLERR) != 0 {
				if epfile := e.getConsole(int(ev.Fd)); epfile != nil {
					epfile.signalRead()
				}
			}
			// the console is ready to be written to
			if ev.Events&(unix.EPOLLOUT|unix.EPOLLHUP|unix.EPOLLERR) != 0 {
				if epfile := e.getConsole(int(ev.Fd)); epfile != nil {
					epfile.signalWrite()
				}
			}
		}
	}
}

// CloseConsole unregisters the console's file descriptor from epoll interface
func (e *Epoller) CloseConsole(fd int) error {
	e.mu.Lock()
	defer e.mu.Unlock()
	delete(e.fdMapping, fd)
	return unix.EpollCtl(e.efd, unix.EPOLL_CTL_DEL, fd, &unix.EpollEvent{})
}

func (e *Epoller) getConsole(sysfd int) *EpollConsole {
	e.mu.Lock()
	f := e.fdMapping[sysfd]
	e.mu.Unlock()
	return f
}

// Close closes the epoll fd
func (e *Epoller) Close() error {
	return unix.Close(e.efd)
}

// EpollConsole acts like a console but registers its file descriptor with an
// epoll fd and uses epoll API to perform I/O.
type EpollConsole struct {
	Console
	readc  *sync.Cond
	writec *sync.Cond
	sysfd  int
	closed bool
}

// Read reads up to len(p) bytes into p. It returns the number of bytes read
// (0 <= n <= len(p)) and any error encountered.
//
// If the console's read returns EAGAIN or EIO, we assume that it's a
// temporary error because the other side went away and wait for the signal
// generated by epoll event to continue.
func (ec *EpollConsole) Read(p []byte) (n int, err error) {
	var read int
	ec.readc.L.Lock()
	defer ec.readc.L.Unlock()
	for {
		read, err = ec.Console.Read(p[n:])
		n += read
		if err != nil {
			var hangup bool
			if perr, ok := err.(*os.PathError); ok {
				hangup = (perr.Err == unix.EAGAIN || perr.Err == unix.EIO)
			} else {
				hangup = (err == unix.EAGAIN || err == unix.EIO)
			}
			// if the other end disappear, assume this is temporary and wait for the
			// signal to continue again. Unless we didnt read anything and the
			// console is already marked as closed then we should exit
			if hangup && !(n == 0 && len(p) > 0 && ec.closed) {
				ec.readc.Wait()
				continue
			}
		}
		break
	}
	// if we didnt read anything then return io.EOF to end gracefully
	if n == 0 && len(p) > 0 && err == nil {
		err = io.EOF
	}
	// signal for others that we finished the read
	ec.readc.Signal()
	return n, err
}

// Writes len(p) bytes from p to the console. It returns the number of bytes
// written from p (0 <= n <= len(p)) and any error encountered that caused
// the write to stop early.
//
// If writes to the console returns EAGAIN or EIO, we assume that it's a
// temporary error because the other side went away and wait for the signal
// generated by epoll event to continue.
func (ec *EpollConsole) Write(p []byte) (n int, err error) {
	var written int
	ec.writec.L.Lock()
	defer ec.writec.L.Unlock()
	for {
		written, err = ec.Console.Write(p[n:])
		n += written
		if err != nil {
			var hangup bool
			if perr, ok := err.(*os.PathError); ok {
				hangup = (perr.Err == unix.EAGAIN || perr.Err == unix.EIO)
			} else {
				hangup = (err == unix.EAGAIN || err == unix.EIO)
			}
			// if the other end disappears, assume this is temporary and wait for the
			// signal to continue again.
			if hangup {
				ec.writec.Wait()
				continue
			}
		}
		// unrecoverable error, break the loop and return the error
		break
	}
	if n < len(p) && err == nil {
		err = io.ErrShortWrite
	}
	// signal for others that we finished the write
	ec.writec.Signal()
	return n, err
}

// Shutdown closes the file descriptor and signals call waiters for this fd.
// It accepts a callback which will be called with the console's fd. The
// callback typically will be used to do further cleanup such as unregister the
// console's fd from the epoll interface.
// User should call Shutdown and wait for all I/O operation to be finished
// before closing the console.
func (ec *EpollConsole) Shutdown(close func(int) error) error {
	ec.readc.L.Lock()
	defer ec.readc.L.Unlock()
	ec.writec.L.Lock()
	defer ec.writec.L.Unlock()

	ec.readc.Broadcast()
	ec.writec.Broadcast()
	ec.closed = true
	return close(ec.sysfd)
}

// signalRead signals that the console is readable.
func (ec *EpollConsole) signalRead() {
	ec.readc.L.Lock()
	ec.readc.Signal()
	ec.readc.L.Unlock()
}

// signalWrite signals that the console is writable.
func (ec *EpollConsole) signalWrite() {
	ec.writec.L.Lock()
	ec.writec.Signal()
	ec.writec.L.Unlock()
}
<|endoftext|>


<commit_before>package serial

// #include <termios.h>
// #include <unistd.h>
import "C"

import (
	"os"
	"io"
	"fmt"
	"syscall"
	//"unsafe"
)

type SError struct {
	msg string
}

func (e SError) String() string {
	return e.msg
}

func OpenPort(name string, baud int) (rwc io.ReadWriteCloser, err os.Error) {
	f, err := os.OpenFile(name, os.O_RDWR|os.O_NOCTTY|os.O_NONBLOCK, 0666)
	if err != nil {
		return
	}

	fd := C.int(f.Fd())
	if C.isatty(fd) != 1 {
		f.Close()
		return nil, SError{"File is not a tty"}
	}

	var st C.struct_termios
	_, err = C.tcgetattr(fd, &st)
	if err != nil {
		f.Close()
		return nil, err
	}
	var speed C.speed_t
	switch baud {
	case 115200:
		speed = C.B115200
	case 19200:
		speed = C.B19200
	case 9600:
		speed = C.B9600
	default:
		f.Close()
		return nil, SError{fmt.Sprintf("Unknown baud rate %v", baud)}
	}

	_, err = C.cfsetispeed(&st, speed)
	if err != nil {
		f.Close()
		return nil, err
	}
	_, err = C.cfsetospeed(&st, speed)
	if err != nil {
		f.Close()
		return nil, err
	}

	// Select local mode
	st.c_cflag |= (C.CLOCAL | C.CREAD)

	// Select raw mode
	st.c_lflag &= ^C.tcflag_t(C.ICANON | C.ECHO | C.ECHOE | C.ISIG)
	st.c_oflag &= ^C.tcflag_t(C.OPOST)

	_, err = C.tcsetattr(fd, C.TCSANOW, &st)
	if err != nil {
		f.Close()
		return nil, err
	}

	//fmt.Println("Tweaking", name)
	r1, _, e := syscall.Syscall(syscall.SYS_FCNTL,
		uintptr(f.Fd()),
		uintptr(syscall.F_SETFL),
		uintptr(0))
	if e != 0 || r1 != 0 {
		s := fmt.Sprint("Clearing NONBLOCK syscall error:", e, r1)
		f.Close()
		return nil, SError{s}
	}

	/*
			r1, _, e = syscall.Syscall(syscall.SYS_IOCTL,
		                uintptr(f.Fd()),
		                uintptr(0x80045402), // IOSSIOSPEED
		                uintptr(unsafe.Pointer(&baud)));
		        if e != 0 || r1 != 0 {
		                s := fmt.Sprint("Baudrate syscall error:", e, r1)
				f.Close()
				return nil, SError{s}
			}
	*/

	return f, nil
}
<commit_msg>Add a few more posix baud rates<commit_after>package serial

// #include <termios.h>
// #include <unistd.h>
import "C"

import (
	"os"
	"io"
	"fmt"
	"syscall"
	//"unsafe"
)

type SError struct {
	msg string
}

func (e SError) String() string {
	return e.msg
}

func OpenPort(name string, baud int) (rwc io.ReadWriteCloser, err os.Error) {
	f, err := os.OpenFile(name, os.O_RDWR|os.O_NOCTTY|os.O_NONBLOCK, 0666)
	if err != nil {
		return
	}

	fd := C.int(f.Fd())
	if C.isatty(fd) != 1 {
		f.Close()
		return nil, SError{"File is not a tty"}
	}

	var st C.struct_termios
	_, err = C.tcgetattr(fd, &st)
	if err != nil {
		f.Close()
		return nil, err
	}
	var speed C.speed_t
	switch baud {
	case 115200:
		speed = C.B115200
	case 76800:
                speed = C.B76800
        case 57600:
                speed = C.B57600
        case 38400:
                speed = C.B38400
	case 19200:
		speed = C.B19200
	case 9600:
		speed = C.B9600
	default:
		f.Close()
		return nil, SError{fmt.Sprintf("Unknown baud rate %v", baud)}
	}

	_, err = C.cfsetispeed(&st, speed)
	if err != nil {
		f.Close()
		return nil, err
	}
	_, err = C.cfsetospeed(&st, speed)
	if err != nil {
		f.Close()
		return nil, err
	}

	// Select local mode
	st.c_cflag |= (C.CLOCAL | C.CREAD)

	// Select raw mode
	st.c_lflag &= ^C.tcflag_t(C.ICANON | C.ECHO | C.ECHOE | C.ISIG)
	st.c_oflag &= ^C.tcflag_t(C.OPOST)

	_, err = C.tcsetattr(fd, C.TCSANOW, &st)
	if err != nil {
		f.Close()
		return nil, err
	}

	//fmt.Println("Tweaking", name)
	r1, _, e := syscall.Syscall(syscall.SYS_FCNTL,
		uintptr(f.Fd()),
		uintptr(syscall.F_SETFL),
		uintptr(0))
	if e != 0 || r1 != 0 {
		s := fmt.Sprint("Clearing NONBLOCK syscall error:", e, r1)
		f.Close()
		return nil, SError{s}
	}

	/*
			r1, _, e = syscall.Syscall(syscall.SYS_IOCTL,
		                uintptr(f.Fd()),
		                uintptr(0x80045402), // IOSSIOSPEED
		                uintptr(unsafe.Pointer(&baud)));
		        if e != 0 || r1 != 0 {
		                s := fmt.Sprint("Baudrate syscall error:", e, r1)
				f.Close()
				return nil, SError{s}
			}
	*/

	return f, nil
}
<|endoftext|>


<commit_before>package server

import (
	"compress/bzip2"
	"crypto/sha256"
	"fmt"
	"io"
	"net/http"
	"os"
	"path"
)

const (
	assetsDirectory = "assets/"
)

func init() {
	err := os.MkdirAll(assetsDirectory, os.ModeDir|0700)
	if err != nil {
		log.Fatalf("Could not create directory for storing assets: %q", err)
	}
}

// downloadAsset grabs the contents of the body of the given URL and stores
// then into $ASSETS_DIRECTORY/$BASENAME.SHA256_SUM($URL)
func downloadAsset(uri string) (localfile string, err error) {
	basename := path.Base(uri)
	fileExt := path.Ext(basename)

	// We'll be appending 65 chars to create a local file name for the asset,
	// this 60-char limit prevents creating a file name longer than 255 chars. We
	// could allow a few more characters until 255 but 60 sounds like a sane
	// limit.
	if len(basename) > 60 {
		basename = basename[:60]
	}

	localfile = assetsDirectory + fmt.Sprintf("%s.%x", basename, sha256.Sum256([]byte(uri)))

	if !fileExists(localfile) {
		var body io.Reader
		var res *http.Response

		if res, err = http.Get(uri); err != nil {
			return "", err
		}

		if res.StatusCode != http.StatusOK {
			return "", fmt.Errorf("Expecting 200 OK, got: %s", res.Status)
		}

		if res.Body != nil {
			defer res.Body.Close()
		}

		var fp *os.File

		if fp, err = os.Create(localfile); err != nil {
			return "", err
		}

		defer fp.Close()

		if fileExt == ".bz2" {
			body = bzip2.NewReader(res.Body)
		} else {
			body = res.Body
		}

		if _, err = io.Copy(fp, body); err != nil {
			return "", err
		}

	}

	return localfile, nil
}
<commit_msg>Removing superfluous check of a non nil body.<commit_after>package server

import (
	"compress/bzip2"
	"crypto/sha256"
	"fmt"
	"io"
	"net/http"
	"os"
	"path"
)

const (
	assetsDirectory = "assets/"
)

func init() {
	err := os.MkdirAll(assetsDirectory, os.ModeDir|0700)
	if err != nil {
		log.Fatalf("Could not create directory for storing assets: %q", err)
	}
}

// downloadAsset grabs the contents of the body of the given URL and stores
// then into $ASSETS_DIRECTORY/$BASENAME.SHA256_SUM($URL)
func downloadAsset(uri string) (localfile string, err error) {
	basename := path.Base(uri)
	fileExt := path.Ext(basename)

	// We'll be appending 65 chars to create a local file name for the asset,
	// this 60-char limit prevents creating a file name longer than 255 chars. We
	// could allow a few more characters until 255 but 60 sounds like a sane
	// limit.
	if len(basename) > 60 {
		basename = basename[:60]
	}

	localfile = assetsDirectory + fmt.Sprintf("%s.%x", basename, sha256.Sum256([]byte(uri)))

	if !fileExists(localfile) {
		var body io.Reader
		var res *http.Response

		if res, err = http.Get(uri); err != nil {
			return "", err
		}

		if res.StatusCode != http.StatusOK {
			return "", fmt.Errorf("Expecting 200 OK, got: %s", res.Status)
		}

		defer res.Body.Close()

		var fp *os.File

		if fp, err = os.Create(localfile); err != nil {
			return "", err
		}

		defer fp.Close()

		if fileExt == ".bz2" {
			body = bzip2.NewReader(res.Body)
		} else {
			body = res.Body
		}

		if _, err = io.Copy(fp, body); err != nil {
			return "", err
		}

	}

	return localfile, nil
}
<|endoftext|>


<commit_before>package outputamqp

import (
	"errors"

	log "github.com/Sirupsen/logrus"
	"github.com/bitly/go-hostpool"
	"github.com/streadway/amqp"
	"github.com/tsaikd/gogstash/config"
	"github.com/tsaikd/gogstash/config/logevent"
)

// ModuleName is the name used in config file
const ModuleName = "amqp"

// OutputConfig holds the output configuration json fields and internal objects
type OutputConfig struct {
	config.OutputConfig
	URLs               []string `json:"urls"`                           // Array of AMQP connection strings formatted per the [RabbitMQ URI Spec](http://www.rabbitmq.com/uri-spec.html).
	RoutingKey         string   `json:"routing_key,omitempty"`          // The message routing key used to bind the queue to the exchange. Defaults to empty string.
	Exchange           string   `json:"exchange"`                       // AMQP exchange name
	ExchangeType       string   `json:"exchange_type"`                  // AMQP exchange type (fanout, direct, topic or headers).
	ExchangeDurable    bool     `json:"exchange_durable,omitempty"`     // Whether the exchange should be configured as a durable exchange. Defaults to false.
	ExchangeAutoDelete bool     `json:"exchange_auto_delete,omitempty"` // Whether the exchange is deleted when all queues have finished and there is no publishing. Defaults to true.
	Persistent         bool     `json:"persistent,omitempty"`           // Whether published messages should be marked as persistent or transient. Defaults to false.
	RetryCount         int      `json:"retry_count,omitempty"`          // Number of attempts for sending a message. Defaults to 3.
	hostPool           hostpool.HostPool
	amqpClients        map[string]amqpConn
	evchan             chan logevent.LogEvent
}

type amqpConn struct {
	Channel    *amqp.Channel
	Connection *amqp.Connection
}

// DefaultOutputConfig returns an OutputConfig struct with default values
func DefaultOutputConfig() OutputConfig {
	return OutputConfig{
		OutputConfig: config.OutputConfig{
			CommonConfig: config.CommonConfig{
				Type: ModuleName,
			},
		},
		RoutingKey:         "",
		ExchangeDurable:    false,
		ExchangeAutoDelete: true,
		Persistent:         false,
		RetryCount:         3,
		amqpClients:        map[string]amqpConn{},

		evchan: make(chan logevent.LogEvent),
	}
}

// InitHandler initialize the output plugin
func InitHandler(confraw *config.ConfigRaw) (retconf config.TypeOutputConfig, err error) {
	conf := DefaultOutputConfig()
	if err = config.ReflectConfig(confraw, &conf); err != nil {
		return
	}

	if err = conf.initAmqpClients(); err != nil {
		return
	}

	retconf = &conf
	return
}

func (o *OutputConfig) initAmqpClients() error {
	var hosts []string

	for _, url := range o.URLs {
		if conn, err := amqp.Dial(url); err == nil {
			if ch, err := conn.Channel(); err == nil {
				o.amqpClients[url] = amqpConn{Channel: ch, Connection: conn}
				err := ch.ExchangeDeclare(
					o.Exchange,
					o.ExchangeType,
					o.ExchangeDurable,
					o.ExchangeAutoDelete,
					false,
					false,
					nil,
				)
				if err != nil {
					return err
				}
				hosts = append(hosts, url)
			}
		}
	}

	if len(hosts) == 0 {
		return errors.New("no valid amqp server connection found")
	}

	o.hostPool = hostpool.New(hosts)
	return nil
}

// Event send the event through AMQP
func (o *OutputConfig) Event(event logevent.LogEvent) (err error) {
	raw, err := event.MarshalJSON()
	if err != nil {
		log.Errorf("event Marshal failed: %v", event)
		return
	}

	exchange := event.Format(o.Exchange)
	routingKey := event.Format(o.RoutingKey)

	for i := 0; i <= o.RetryCount; i++ {
		host := o.hostPool.Get().Host()
		err = o.amqpClients[host].Channel.Publish(
			exchange,
			routingKey,
			false,
			false,
			amqp.Publishing{
				ContentType: "application/json",
				Body:        raw,
			},
		)
		if err == nil {
			break
		}
	}

	return
}
<commit_msg>AMQP Output: handle reconnections to AMQP server.<commit_after>package outputamqp

import (
	"errors"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/bitly/go-hostpool"
	"github.com/streadway/amqp"
	"github.com/tsaikd/gogstash/config"
	"github.com/tsaikd/gogstash/config/logevent"
)

// ModuleName is the name used in config file
const ModuleName = "amqp"

// OutputConfig holds the output configuration json fields and internal objects
type OutputConfig struct {
	config.OutputConfig
	URLs               []string `json:"urls"`                           // Array of AMQP connection strings formatted per the [RabbitMQ URI Spec](http://www.rabbitmq.com/uri-spec.html).
	RoutingKey         string   `json:"routing_key,omitempty"`          // The message routing key used to bind the queue to the exchange. Defaults to empty string.
	Exchange           string   `json:"exchange"`                       // AMQP exchange name
	ExchangeType       string   `json:"exchange_type"`                  // AMQP exchange type (fanout, direct, topic or headers).
	ExchangeDurable    bool     `json:"exchange_durable,omitempty"`     // Whether the exchange should be configured as a durable exchange. Defaults to false.
	ExchangeAutoDelete bool     `json:"exchange_auto_delete,omitempty"` // Whether the exchange is deleted when all queues have finished and there is no publishing. Defaults to true.
	Persistent         bool     `json:"persistent,omitempty"`           // Whether published messages should be marked as persistent or transient. Defaults to false.
	Retries            int      `json:"retries,omitempty"`              // Number of attempts to send a message. Defaults to 3.
	ReconnectDelay     int      `json:"reconnect_delay,omitempty"`      // Delay between each attempt to reconnect to AMQP server. Defaults to 30 seconds.
	hostPool           hostpool.HostPool
	amqpClients        map[string]*amqp.Channel
	evchan             chan logevent.LogEvent
}

type amqpConn struct {
	Channel    *amqp.Channel
	Connection *amqp.Connection
}

// DefaultOutputConfig returns an OutputConfig struct with default values
func DefaultOutputConfig() OutputConfig {
	return OutputConfig{
		OutputConfig: config.OutputConfig{
			CommonConfig: config.CommonConfig{
				Type: ModuleName,
			},
		},
		RoutingKey:         "",
		ExchangeDurable:    false,
		ExchangeAutoDelete: true,
		Persistent:         false,
		Retries:            3,
		ReconnectDelay:     30,
		amqpClients:        map[string]*amqp.Channel{},

		evchan: make(chan logevent.LogEvent),
	}
}

// InitHandler initialize the output plugin
func InitHandler(confraw *config.ConfigRaw) (retconf config.TypeOutputConfig, err error) {
	conf := DefaultOutputConfig()
	if err = config.ReflectConfig(confraw, &conf); err != nil {
		return
	}

	if err = conf.initAmqpClients(); err != nil {
		return
	}

	retconf = &conf
	return
}

func (o *OutputConfig) initAmqpClients() error {
	var hosts []string

	for _, url := range o.URLs {
		if conn, err := amqp.Dial(url); err == nil {
			if ch, err := conn.Channel(); err == nil {
				o.amqpClients[url] = ch
				err := ch.ExchangeDeclare(
					o.Exchange,
					o.ExchangeType,
					o.ExchangeDurable,
					o.ExchangeAutoDelete,
					false,
					false,
					nil,
				)
				if err != nil {
					return err
				}
				hosts = append(hosts, url)
			}
		}
	}

	if len(hosts) == 0 {
		return errors.New("no valid amqp server connection found")
	}

	o.hostPool = hostpool.New(hosts)
	return nil
}

// Event send the event through AMQP
func (o *OutputConfig) Event(event logevent.LogEvent) (err error) {
	raw, err := event.MarshalJSON()
	if err != nil {
		logrus.Errorf("event Marshal failed: %v", event)
		return
	}

	exchange := event.Format(o.Exchange)
	routingKey := event.Format(o.RoutingKey)

	for i := 0; i <= o.Retries; i++ {
		hp := o.hostPool.Get()
		err = o.amqpClients[hp.Host()].Publish(
			exchange,
			routingKey,
			false,
			false,
			amqp.Publishing{
				ContentType: "application/json",
				Body:        raw,
			},
		)
		if err != nil {
			hp.Mark(err)
			if len(o.amqpClients) > 1 {
				go o.reconnect(hp)
			} else {
				o.reconnect(hp)
			}
		} else {
			break
		}
	}

	return
}

func (o *OutputConfig) reconnect(hp hostpool.HostPoolResponse) {
	for {
		time.Sleep(time.Duration(o.ReconnectDelay) * time.Second)
		logrus.Info("Reconnecting to ", hp.Host())
		if conn, err := amqp.Dial(hp.Host()); err == nil {
			if ch, err := conn.Channel(); err == nil {
				err := ch.ExchangeDeclare(
					o.Exchange,
					o.ExchangeType,
					o.ExchangeDurable,
					o.ExchangeAutoDelete,
					false,
					false,
					nil,
				)
				if err == nil {
					hp.Mark(nil)
					logrus.Info("Reconnected to ", hp.Host())
					o.amqpClients[hp.Host()] = ch
					break
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package geocodio

import (
	"errors"
	"strings"
)

// Geocode single address
// See: http://geocod.io/docs/#toc_4
func (g *Geocodio) Geocode(address string) (GeocodeResult, error) {
	if address == "" {
		return GeocodeResult{}, errors.New("address must not be empty")
	}

	results, err := g.Call("/geocode", map[string]string{"q": address})
	if err != nil {
		return GeocodeResult{}, err
	}

	return results, nil
}

// GeocodeAndReturnTimezone will geocode and include Timezone in the fields response
func (g *Geocodio) GeocodeAndReturnTimezone(address string) (GeocodeResult, error) {
	return g.GeocodeReturnFields(address, "timezone")
}

// GeocodeAndReturnCongressionalDistrict will geocode and include Congressional District in the fields response
func (g *Geocodio) GeocodeAndReturnCongressionalDistrict(address string) (GeocodeResult, error) {
	return g.GeocodeReturnFields(address, "cd")
}

// GeocodeAndReturnStateLegislativeDistricts will geocode and include State Legislative Districts in the fields response
func (g *Geocodio) GeocodeAndReturnStateLegislativeDistricts(address string) (GeocodeResult, error) {
	return g.GeocodeReturnFields(address, "stateleg")
}

// TODO: School District (school)

// GeocodeReturnFields will geocode and includes additional fields in response
/*
 	See: http://geocod.io/docs/#toc_22
	Note:
		Each field counts as an additional lookup each
*/
func (g *Geocodio) GeocodeReturnFields(address string, fields ...string) (GeocodeResult, error) {
	if address == "" {
		return GeocodeResult{}, errors.New("address can not be empty")
	}

	fieldsCommaSeparated := strings.Join(fields, ",")

	results, err := g.Call("/geocode",
		map[string]string{
			"q":      address,
			"fields": fieldsCommaSeparated,
		})
	if err != nil {
		return GeocodeResult{}, err
	}

	return results, nil
}
<commit_msg>Add Zip4 helper method<commit_after>package geocodio

import (
	"errors"
	"strings"
)

// Geocode single address
// See: http://geocod.io/docs/#toc_4
func (g *Geocodio) Geocode(address string) (GeocodeResult, error) {
	if address == "" {
		return GeocodeResult{}, errors.New("address must not be empty")
	}

	results, err := g.Call("/geocode", map[string]string{"q": address})
	if err != nil {
		return GeocodeResult{}, err
	}

	return results, nil
}

// GeocodeAndReturnTimezone will geocode and include Timezone in the fields response
func (g *Geocodio) GeocodeAndReturnTimezone(address string) (GeocodeResult, error) {
	return g.GeocodeReturnFields(address, "timezone")
}

// GeocodeAndReturnZip4 will geocode and include zip4 in the fields response
func (g *Geocodio) GeocodeAndReturnZip4(address string) (GeocodeResult, error) {
	return g.GeocodeReturnFields(address, "zip4")
}

// GeocodeAndReturnCongressionalDistrict will geocode and include Congressional District in the fields response
func (g *Geocodio) GeocodeAndReturnCongressionalDistrict(address string) (GeocodeResult, error) {
	return g.GeocodeReturnFields(address, "cd")
}

// GeocodeAndReturnStateLegislativeDistricts will geocode and include State Legislative Districts in the fields response
func (g *Geocodio) GeocodeAndReturnStateLegislativeDistricts(address string) (GeocodeResult, error) {
	return g.GeocodeReturnFields(address, "stateleg")
}

// TODO: School District (school)

// GeocodeReturnFields will geocode and includes additional fields in response
/*
 	See: http://geocod.io/docs/#toc_22
	Note:
		Each field counts as an additional lookup each
*/
func (g *Geocodio) GeocodeReturnFields(address string, fields ...string) (GeocodeResult, error) {
	if address == "" {
		return GeocodeResult{}, errors.New("address can not be empty")
	}

	fieldsCommaSeparated := strings.Join(fields, ",")

	result, err := g.Call("/geocode",
		map[string]string{
			"q":      address,
			"fields": fieldsCommaSeparated,
		})
	if err != nil {
		return GeocodeResult{}, err
	}

	return result, nil
}
<|endoftext|>


<commit_before>// Package geohash is a simple implementation of the Public Domain Geohash algorithm.
// It represents the hashes using boxes which contain the resulting hashes and can reverse the box,
// using the geohash and the precision to recreate it at.
package geohash

import (
	"strings"
)

const (
	maxlat = 90.0
	maxlon = 180.0
	minlat = -90.0
	minlon = -180.0
	base32 = "0123456789bcdefghjkmnpqrstuvwxyz"
)

var bits = []int{16, 8, 4, 2, 1}

// Box represents a coordinate box. It knows it's width and height, and can figure out it's neighbors.
// It implements the Stringer interface, printing out the geohash value.
type Box struct {
	hash           string
	pres           int
	maxLat, maxLon float64
	minLat, minLon float64
}

// Decode creates a new box from an initial hash
func Decode(hash string, pres int) *Box {

	refine := func(minInterval, maxInterval float64, cd, mask int) (float64, float64) {
		if cd&mask == 0 {
			maxInterval = (minInterval + maxInterval) / 2
		} else {
			minInterval = (minInterval + maxInterval) / 2
		}

		return minInterval, maxInterval
	}

	p := 12
	if pres > 0 {
		p = pres
	}

	isEven := true
	lat0, lat1 := minlat, maxlat
	lon0, lon1 := minlon, maxlon
	latErr, lonErr := maxlat, maxlon

	for i := 0; i < len(hash); i++ {
		c := hash[i]

		cd := strings.Index(string(base32), string(c))

		for j := 0; j < 5; j++ {
			mask := bits[j]

			if isEven {
				lonErr /= 2
				lon0, lon1 = refine(lon0, lon1, cd, mask)
			} else {
				latErr /= 2
				lat0, lat1 = refine(lat0, lat1, cd, mask)
			}
			isEven = !isEven
		}
	}

	return &Box{hash: hash, pres: p, minLat: lat0, maxLat: lat1, minLon: lon0, maxLon: lon1}
}

// Encode takes coordinates and makes a geohash box
func Encode(lat, lon float64, pres int) *Box {
	hash := ""
	precision := 12
	if pres > 0 {
		precision = pres
	}

	isEven := true
	bit := 0
	ch := 0
	lat0, lat1 := minlat, maxlat
	lon0, lon1 := minlon, maxlon

	for len(hash) < precision {
		if isEven {
			mid := (lon0 + lon1) / 2.0

			if lon > mid {
				ch |= bits[bit]
				lon0 = mid
			} else {
				lon1 = mid
			}
		} else {
			mid := (lat0 + lat1) / 2.0

			if lat > mid {
				ch |= bits[bit]
				lat0 = mid
			} else {
				lat1 = mid
			}
		}

		isEven = !isEven

		if bit < 4 {
			bit++
		} else {
			hash += string(base32[ch])
			bit = 0
			ch = 0
		}
	}

	return &Box{hash: hash, maxLat: lat1, minLat: lat0, maxLon: lon1, minLon: lon0}
}

// Lat returns the latitude of a box
func (b Box) Lat() float64 {
	return (b.minLat + b.maxLat) / 2
}

// Lon returns the longitude of a box
func (b Box) Lon() float64 {
	return (b.minLon + b.maxLon) / 2
}

// Height returns the height of the box
func (b Box) Height() float64 {
	return b.maxLat - b.minLat
}

// Width returns the width of the box
func (b Box) Width() float64 {
	return b.maxLon - b.minLon
}

// Precision returns the precision of a box
func (b Box) Precision() int {
	return b.pres
}

// Neighbors calculates the 8 neighboring boxes of a box
func (b Box) Neighbors() []*Box {

	var (
		// Directly adjecent
		up    = Encode(b.Lat()+b.Height(), b.Lon(), b.Precision())
		down  = Encode(b.Lat()-b.Height(), b.Lon(), b.Precision())
		left  = Encode(b.Lat(), b.Lon()-b.Width(), b.Precision())
		right = Encode(b.Lat(), b.Lon()+b.Width(), b.Precision())

		// Corners
		upleft    = Encode(b.Lat()+b.Height(), b.Lon()-b.Width(), b.Precision())
		downleft  = Encode(b.Lat()-b.Height(), b.Lon()-b.Width(), b.Precision())
		upright   = Encode(b.Lat()+b.Height(), b.Lon()+b.Width(), b.Precision())
		downright = Encode(b.Lat()-b.Height(), b.Lon()+b.Width(), b.Precision())
	)

	return []*Box{up, down, left, right, upleft, downleft, upright, downright}
}

// String returns the geohash of the box
func (b Box) String() string {
	return b.hash
}

// Hash returns the geohash
func (b Box) Hash() string {
	return b.hash
}<commit_msg>removed typo<commit_after>// Package geohash is a simple implementation of the Public Domain Geohash algorithm.
// It represents the hashes using boxes which contain the resulting hashes and can reverse the box,
// using the geohash and the precision to recreate it.
package geohash

import (
	"strings"
)

const (
	maxlat = 90.0
	maxlon = 180.0
	minlat = -90.0
	minlon = -180.0
	base32 = "0123456789bcdefghjkmnpqrstuvwxyz"
)

var bits = []int{16, 8, 4, 2, 1}

// Box represents a coordinate box. It knows it's width and height, and can figure out it's neighbors.
// It implements the Stringer interface, printing out the geohash value.
type Box struct {
	hash           string
	pres           int
	maxLat, maxLon float64
	minLat, minLon float64
}

// Decode creates a new box from an initial hash
func Decode(hash string, pres int) *Box {

	refine := func(minInterval, maxInterval float64, cd, mask int) (float64, float64) {
		if cd&mask == 0 {
			maxInterval = (minInterval + maxInterval) / 2
		} else {
			minInterval = (minInterval + maxInterval) / 2
		}

		return minInterval, maxInterval
	}

	p := 12
	if pres > 0 {
		p = pres
	}

	isEven := true
	lat0, lat1 := minlat, maxlat
	lon0, lon1 := minlon, maxlon
	latErr, lonErr := maxlat, maxlon

	for i := 0; i < len(hash); i++ {
		c := hash[i]

		cd := strings.Index(string(base32), string(c))

		for j := 0; j < 5; j++ {
			mask := bits[j]

			if isEven {
				lonErr /= 2
				lon0, lon1 = refine(lon0, lon1, cd, mask)
			} else {
				latErr /= 2
				lat0, lat1 = refine(lat0, lat1, cd, mask)
			}
			isEven = !isEven
		}
	}

	return &Box{hash: hash, pres: p, minLat: lat0, maxLat: lat1, minLon: lon0, maxLon: lon1}
}

// Encode takes coordinates and makes a geohash box
func Encode(lat, lon float64, pres int) *Box {
	hash := ""
	precision := 12
	if pres > 0 {
		precision = pres
	}

	isEven := true
	bit := 0
	ch := 0
	lat0, lat1 := minlat, maxlat
	lon0, lon1 := minlon, maxlon

	for len(hash) < precision {
		if isEven {
			mid := (lon0 + lon1) / 2.0

			if lon > mid {
				ch |= bits[bit]
				lon0 = mid
			} else {
				lon1 = mid
			}
		} else {
			mid := (lat0 + lat1) / 2.0

			if lat > mid {
				ch |= bits[bit]
				lat0 = mid
			} else {
				lat1 = mid
			}
		}

		isEven = !isEven

		if bit < 4 {
			bit++
		} else {
			hash += string(base32[ch])
			bit = 0
			ch = 0
		}
	}

	return &Box{hash: hash, maxLat: lat1, minLat: lat0, maxLon: lon1, minLon: lon0}
}

// Lat returns the latitude of a box
func (b Box) Lat() float64 {
	return (b.minLat + b.maxLat) / 2
}

// Lon returns the longitude of a box
func (b Box) Lon() float64 {
	return (b.minLon + b.maxLon) / 2
}

// Height returns the height of the box
func (b Box) Height() float64 {
	return b.maxLat - b.minLat
}

// Width returns the width of the box
func (b Box) Width() float64 {
	return b.maxLon - b.minLon
}

// Precision returns the precision of a box
func (b Box) Precision() int {
	return b.pres
}

// Neighbors calculates the 8 neighboring boxes of a box
func (b Box) Neighbors() []*Box {

	var (
		// Directly adjecent
		up    = Encode(b.Lat()+b.Height(), b.Lon(), b.Precision())
		down  = Encode(b.Lat()-b.Height(), b.Lon(), b.Precision())
		left  = Encode(b.Lat(), b.Lon()-b.Width(), b.Precision())
		right = Encode(b.Lat(), b.Lon()+b.Width(), b.Precision())

		// Corners
		upleft    = Encode(b.Lat()+b.Height(), b.Lon()-b.Width(), b.Precision())
		downleft  = Encode(b.Lat()-b.Height(), b.Lon()-b.Width(), b.Precision())
		upright   = Encode(b.Lat()+b.Height(), b.Lon()+b.Width(), b.Precision())
		downright = Encode(b.Lat()-b.Height(), b.Lon()+b.Width(), b.Precision())
	)

	return []*Box{up, down, left, right, upleft, downleft, upright, downright}
}

// String returns the geohash of the box
func (b Box) String() string {
	return b.hash
}

// Hash returns the geohash
func (b Box) Hash() string {
	return b.hash
}<|endoftext|>


<commit_before>package service

import (
	"database/sql"
	"log"

	"github.com/charakoba-com/auth-api/db"
	"github.com/charakoba-com/auth-api/model"
	"github.com/pkg/errors"
)

// Create User
func (v *UserService) Create(tx *sql.Tx, du *db.User) error {
	log.Printf("service.User.Create %s", du.ID)

	if err := du.Create(tx); err != nil {
		return errors.Wrap(err, `creating db.User`)
	}
	return nil
}

// Lookup User
func (v *UserService) Lookup(tx *sql.Tx, id string) (*model.User, error) {
	log.Printf("service.User.Lookup %s", id)

	var mu model.User
	if err := mu.Load(tx, id); err != nil {
		return nil, errors.Wrap(err, `loading model.User`)
	}
	log.Printf("DONE: service.User.Loopkup")
	return &mu, nil
}

// Update User
func (v *UserService) Update(tx *sql.Tx, du *db.User) error {
	log.Printf("service.User.Update %s", du.ID)
	if err := du.Update(tx); err != nil {
		return errors.Wrap(err, `updating db.User`)
	}
	return nil
}

// Delete User
func (v *UserService) Delete(tx *sql.Tx, id string) error {
	log.Printf("service.User.Delete %s", id)

	du := db.User{ID: id}
	if err := du.Delete(tx); err != nil {
		return errors.Wrap(err, `deleting db.User`)
	}
	return nil
}

// Listup User
func (v *UserService) Listup(tx *sql.Tx) (model.UserList, error) {
	log.Printf("service.User.Listup")

	var userList db.UserList
	if err := userList.Listup(tx); err != nil {
		return nil, errors.Wrap(err, `loading user list`)
	}
	l := make(model.UserList, len(userList))
	for i, user := range userList {
		if err := l[i].FromDB(&user); err != nil {
			return nil, errors.Wrap(err, `converting db.User to model.User`)
		}
	}

	return l, nil
}
<commit_msg>remove: done log<commit_after>package service

import (
	"database/sql"
	"log"

	"github.com/charakoba-com/auth-api/db"
	"github.com/charakoba-com/auth-api/model"
	"github.com/pkg/errors"
)

// Create User
func (v *UserService) Create(tx *sql.Tx, du *db.User) error {
	log.Printf("service.User.Create %s", du.ID)

	if err := du.Create(tx); err != nil {
		return errors.Wrap(err, `creating db.User`)
	}
	return nil
}

// Lookup User
func (v *UserService) Lookup(tx *sql.Tx, id string) (*model.User, error) {
	log.Printf("service.User.Lookup %s", id)

	var mu model.User
	if err := mu.Load(tx, id); err != nil {
		return nil, errors.Wrap(err, `loading model.User`)
	}
	return &mu, nil
}

// Update User
func (v *UserService) Update(tx *sql.Tx, du *db.User) error {
	log.Printf("service.User.Update %s", du.ID)
	if err := du.Update(tx); err != nil {
		return errors.Wrap(err, `updating db.User`)
	}
	return nil
}

// Delete User
func (v *UserService) Delete(tx *sql.Tx, id string) error {
	log.Printf("service.User.Delete %s", id)

	du := db.User{ID: id}
	if err := du.Delete(tx); err != nil {
		return errors.Wrap(err, `deleting db.User`)
	}
	return nil
}

// Listup User
func (v *UserService) Listup(tx *sql.Tx) (model.UserList, error) {
	log.Printf("service.User.Listup")

	var userList db.UserList
	if err := userList.Listup(tx); err != nil {
		return nil, errors.Wrap(err, `loading user list`)
	}
	l := make(model.UserList, len(userList))
	for i, user := range userList {
		if err := l[i].FromDB(&user); err != nil {
			return nil, errors.Wrap(err, `converting db.User to model.User`)
		}
	}

	return l, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/nsqio/go-nsq"
	"github.com/raintank/met/helper"
	"github.com/raintank/raintank-metric/metric_tank/defcache"
	"github.com/raintank/raintank-metric/metricdef"
	"github.com/raintank/raintank-metric/msg"
	"github.com/raintank/raintank-metric/schema"
	"sync"
	"testing"
)

// handler.HandleMessage some messages concurrently and make sure the entries in defcache are correct
// this can expose bad reuse of data arrays and such
func Test_HandleMessage(t *testing.T) {
	for i := 0; i < 100; i++ {
		test_HandleMessage(t)
	}
}

func test_HandleMessage(t *testing.T) {
	stats, _ := helper.New(false, "", "standard", "metrics_tank", "")
	clusterStatus = NewClusterStatus("default", false)
	initMetrics(stats)

	store := NewDevnullStore()
	aggmetrics := NewAggMetrics(store, 600, 10, 800, 8000, 10000, 0, make([]aggSetting, 0))
	defCache := defcache.New(metricdef.NewDefsMockConcurrent(), stats)
	handler := NewHandler(aggmetrics, defCache, nil)

	// timestamps start at 1 and go up from there. (we can't use 0, see AggMetric.Add())
	// handle 5 messages, each containing different metrics

	// i m id
	// 0 0 1
	// 0 1 2
	// 0 2 3
	// 0 3 4

	// 1 0 2*
	// 1 1 4*
	// 1 2 6
	// 1 3 8

	// 2 0 3*
	// 2 1 6*
	// 2 2 9
	// 2 3 12

	// 3 0 4*
	// 3 1 8*
	// 3 2 12*
	// 3 3 16

	// -> 9 unique id's == 9 unique metrics cause metrics only differ due to the id
	wg := &sync.WaitGroup{}
	wg.Add(4)
	type idToId struct {
		sync.Mutex
		ids map[string]int
	}
	tit := &idToId{
		ids: make(map[string]int),
	}
	for i := 0; i < 4; i++ {
		go func(i int, tit *idToId) {
			metrics := make([]*schema.MetricData, 4)
			for m := 0; m < len(metrics); m++ {
				id := (i + 1) * (m + 1)
				t.Logf("worker %d metric %d -> adding metric with id and orgid %d", i, m, id)

				metrics[m] = &schema.MetricData{
					Id:         "",
					OrgId:      id,
					Name:       fmt.Sprintf("some.id.%d", id),
					Metric:     "metric",
					Interval:   60,
					Value:      1234.567,
					Unit:       "ms",
					Time:       int64(id),
					TargetType: "gauge",
					Tags:       []string{fmt.Sprintf("%d", id)},
				}
				metrics[m].SetId()
				tit.Lock()
				tit.ids[metrics[m].Id] = id
				tit.Unlock()
			}
			id := nsq.MessageID{'1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'a', 's', 'd', 'f', 'g', 'h'}
			data, err := msg.CreateMsg(metrics, 1, msg.FormatMetricDataArrayMsgp)
			if err != nil {
				panic(err)
			}
			msg := nsq.NewMessage(id, data)
			err = handler.HandleMessage(msg)
			if err != nil {
				panic(err)
			}
			wg.Done()
		}(i, tit)
	}
	wg.Wait()
	defs := defCache.List(-1)
	if len(defs) != 9 {
		t.Fatalf("query for org -1 should result in 9 distinct metrics. not %d", len(defs))
	}
	for _, d := range defs {
		id := tit.ids[d.Id]
		if d.Name != fmt.Sprintf("some.id.%d", id) {
			t.Fatalf("incorrect name for %s : %s", d.Id, d.Name)
		}
		if d.OrgId != id {
			t.Fatalf("incorrect OrgId for %s : %d", d.Id, d.OrgId)
		}
		if d.Tags[0] != fmt.Sprintf("%d", id) {
			t.Fatalf("incorrect tags for %s : %s", d.Id, d.Tags)
		}
	}
	defs = defCache.List(2)
	if len(defs) != 1 {
		t.Fatalf("len of defs should be exactly 1. got defs with len %d: %s", len(defs), defs)
	}
	d := defs[0]
	if d.OrgId != 2 {
		t.Fatalf("incorrect metricdef returned: %s", d)
	}
}

func BenchmarkHandler_HandleMessage(b *testing.B) {
	stats, _ := helper.New(false, "", "standard", "metrics_tank", "")
	clusterStatus = NewClusterStatus("default", false)
	initMetrics(stats)

	store := NewDevnullStore()
	aggmetrics := NewAggMetrics(store, 600, 10, 800, 8000, 10000, 0, make([]aggSetting, 0))
	defCache := defcache.New(metricdef.NewDefsMock(), stats)
	handler := NewHandler(aggmetrics, defCache, nil)

	metrics := make([]*schema.MetricData, 10)
	for i := 0; i < len(metrics); i++ {
		metrics[i] = &schema.MetricData{
			Id:         "some.id.of.a.metric",
			OrgId:      500,
			Name:       "some.id",
			Metric:     "metric",
			Interval:   60,
			Value:      1234.567,
			Unit:       "ms",
			Time:       int64(i - len(metrics) + 1),
			TargetType: "gauge",
			Tags:       []string{"some_tag", "ok"},
		}
	}
	// timestamps start at 1 and go up from there. (we can't use 0, see AggMetric.Add())
	msgs := make([]*nsq.Message, b.N)
	for i := 0; i < len(msgs); i++ {
		id := nsq.MessageID{'1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'a', 's', 'd', 'f', 'g', 'h'}
		for j := 0; j < len(metrics); j++ {
			metrics[j].Time += 10
		}
		data, err := msg.CreateMsg(metrics, 1, msg.FormatMetricDataArrayMsgp)
		if err != nil {
			panic(err)
		}
		msgs[i] = nsq.NewMessage(id, data)
	}

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		err := handler.HandleMessage(msgs[i])
		if err != nil {
			panic(err)
		}
	}
}
<commit_msg>fix data race caused by re-initializing metrics<commit_after>package main

import (
	"fmt"
	"github.com/nsqio/go-nsq"
	"github.com/raintank/met"
	"github.com/raintank/met/helper"
	"github.com/raintank/raintank-metric/metric_tank/defcache"
	"github.com/raintank/raintank-metric/metricdef"
	"github.com/raintank/raintank-metric/msg"
	"github.com/raintank/raintank-metric/schema"
	"sync"
	"testing"
)

// handler.HandleMessage some messages concurrently and make sure the entries in defcache are correct
// this can expose bad reuse of data arrays and such
func Test_HandleMessage(t *testing.T) {

	stats, _ := helper.New(false, "", "standard", "metrics_tank", "")
	clusterStatus = NewClusterStatus("default", false)
	initMetrics(stats)

	for i := 0; i < 100; i++ {
		test_HandleMessage(t, stats)
	}
}

func test_HandleMessage(t *testing.T, stats met.Backend) {

	store := NewDevnullStore()
	aggmetrics := NewAggMetrics(store, 600, 10, 800, 8000, 10000, 0, make([]aggSetting, 0))
	defCache := defcache.New(metricdef.NewDefsMockConcurrent(), stats)
	handler := NewHandler(aggmetrics, defCache, nil)

	// timestamps start at 1 and go up from there. (we can't use 0, see AggMetric.Add())
	// handle 5 messages, each containing different metrics

	// i m id
	// 0 0 1
	// 0 1 2
	// 0 2 3
	// 0 3 4

	// 1 0 2*
	// 1 1 4*
	// 1 2 6
	// 1 3 8

	// 2 0 3*
	// 2 1 6*
	// 2 2 9
	// 2 3 12

	// 3 0 4*
	// 3 1 8*
	// 3 2 12*
	// 3 3 16

	// -> 9 unique id's == 9 unique metrics cause metrics only differ due to the id
	wg := &sync.WaitGroup{}
	wg.Add(4)
	type idToId struct {
		sync.Mutex
		ids map[string]int
	}
	tit := &idToId{
		ids: make(map[string]int),
	}
	for i := 0; i < 4; i++ {
		go func(i int, tit *idToId) {
			metrics := make([]*schema.MetricData, 4)
			for m := 0; m < len(metrics); m++ {
				id := (i + 1) * (m + 1)
				t.Logf("worker %d metric %d -> adding metric with id and orgid %d", i, m, id)

				metrics[m] = &schema.MetricData{
					Id:         "",
					OrgId:      id,
					Name:       fmt.Sprintf("some.id.%d", id),
					Metric:     "metric",
					Interval:   60,
					Value:      1234.567,
					Unit:       "ms",
					Time:       int64(id),
					TargetType: "gauge",
					Tags:       []string{fmt.Sprintf("%d", id)},
				}
				metrics[m].SetId()
				tit.Lock()
				tit.ids[metrics[m].Id] = id
				tit.Unlock()
			}
			id := nsq.MessageID{'1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'a', 's', 'd', 'f', 'g', 'h'}
			data, err := msg.CreateMsg(metrics, 1, msg.FormatMetricDataArrayMsgp)
			if err != nil {
				panic(err)
			}
			msg := nsq.NewMessage(id, data)
			err = handler.HandleMessage(msg)
			if err != nil {
				panic(err)
			}
			wg.Done()
		}(i, tit)
	}
	wg.Wait()
	defs := defCache.List(-1)
	if len(defs) != 9 {
		t.Fatalf("query for org -1 should result in 9 distinct metrics. not %d", len(defs))
	}
	for _, d := range defs {
		id := tit.ids[d.Id]
		if d.Name != fmt.Sprintf("some.id.%d", id) {
			t.Fatalf("incorrect name for %s : %s", d.Id, d.Name)
		}
		if d.OrgId != id {
			t.Fatalf("incorrect OrgId for %s : %d", d.Id, d.OrgId)
		}
		if d.Tags[0] != fmt.Sprintf("%d", id) {
			t.Fatalf("incorrect tags for %s : %s", d.Id, d.Tags)
		}
	}
	defs = defCache.List(2)
	if len(defs) != 1 {
		t.Fatalf("len of defs should be exactly 1. got defs with len %d: %s", len(defs), defs)
	}
	d := defs[0]
	if d.OrgId != 2 {
		t.Fatalf("incorrect metricdef returned: %s", d)
	}
}

func BenchmarkHandler_HandleMessage(b *testing.B) {
	stats, _ := helper.New(false, "", "standard", "metrics_tank", "")
	clusterStatus = NewClusterStatus("default", false)
	initMetrics(stats)

	store := NewDevnullStore()
	aggmetrics := NewAggMetrics(store, 600, 10, 800, 8000, 10000, 0, make([]aggSetting, 0))
	defCache := defcache.New(metricdef.NewDefsMock(), stats)
	handler := NewHandler(aggmetrics, defCache, nil)

	metrics := make([]*schema.MetricData, 10)
	for i := 0; i < len(metrics); i++ {
		metrics[i] = &schema.MetricData{
			Id:         "some.id.of.a.metric",
			OrgId:      500,
			Name:       "some.id",
			Metric:     "metric",
			Interval:   60,
			Value:      1234.567,
			Unit:       "ms",
			Time:       int64(i - len(metrics) + 1),
			TargetType: "gauge",
			Tags:       []string{"some_tag", "ok"},
		}
	}
	// timestamps start at 1 and go up from there. (we can't use 0, see AggMetric.Add())
	msgs := make([]*nsq.Message, b.N)
	for i := 0; i < len(msgs); i++ {
		id := nsq.MessageID{'1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'a', 's', 'd', 'f', 'g', 'h'}
		for j := 0; j < len(metrics); j++ {
			metrics[j].Time += 10
		}
		data, err := msg.CreateMsg(metrics, 1, msg.FormatMetricDataArrayMsgp)
		if err != nil {
			panic(err)
		}
		msgs[i] = nsq.NewMessage(id, data)
	}

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		err := handler.HandleMessage(msgs[i])
		if err != nil {
			panic(err)
		}
	}
}
<|endoftext|>


<commit_before>package millipede

import (
	"fmt"
	"testing"
)

func ExampleNew() {
	millipede := New(20)
	fmt.Println(millipede)
	// Output:
	//   ╚⊙ ⊙╝
	// ╚═(███)═╝
	//  ╚═(███)═╝
	//   ╚═(███)═╝
	//    ╚═(███)═╝
	//     ╚═(███)═╝
	//    ╚═(███)═╝
	//   ╚═(███)═╝
	//  ╚═(███)═╝
	// ╚═(███)═╝
	//  ╚═(███)═╝
	//   ╚═(███)═╝
	//    ╚═(███)═╝
	//     ╚═(███)═╝
	//    ╚═(███)═╝
	//   ╚═(███)═╝
	//  ╚═(███)═╝
	// ╚═(███)═╝
	//  ╚═(███)═╝
	//   ╚═(███)═╝
	//    ╚═(███)═╝
}

func ExampleMillipede() {
	millipede := &Millipede{
		Size:     20,
		Reverse:  false,
		Skin:     "default",
		Opposite: false,
		Width:    3,
		Curve:    4,
	}
	fmt.Println(millipede)
	// Output:
	//   ╚⊙ ⊙╝
	// ╚═(███)═╝
	//  ╚═(███)═╝
	//   ╚═(███)═╝
	//    ╚═(███)═╝
	//     ╚═(███)═╝
	//    ╚═(███)═╝
	//   ╚═(███)═╝
	//  ╚═(███)═╝
	// ╚═(███)═╝
	//  ╚═(███)═╝
	//   ╚═(███)═╝
	//    ╚═(███)═╝
	//     ╚═(███)═╝
	//    ╚═(███)═╝
	//   ╚═(███)═╝
	//  ╚═(███)═╝
	// ╚═(███)═╝
	//  ╚═(███)═╝
	//   ╚═(███)═╝
	//    ╚═(███)═╝
}

func ExampleMillipede_reverse() {
	millipede := &Millipede{
		Size:     20,
		Reverse:  true,
		Skin:     "default",
		Opposite: false,
		Width:    3,
		Curve:    4,
	}
	fmt.Println(millipede)
	// Output:
	//    ╔═(███)═╗
	//   ╔═(███)═╗
	//  ╔═(███)═╗
	// ╔═(███)═╗
	//  ╔═(███)═╗
	//   ╔═(███)═╗
	//    ╔═(███)═╗
	//     ╔═(███)═╗
	//    ╔═(███)═╗
	//   ╔═(███)═╗
	//  ╔═(███)═╗
	// ╔═(███)═╗
	//  ╔═(███)═╗
	//   ╔═(███)═╗
	//    ╔═(███)═╗
	//     ╔═(███)═╗
	//    ╔═(███)═╗
	//   ╔═(███)═╗
	//  ╔═(███)═╗
	// ╔═(███)═╗
	//   ╔⊙ ⊙╗
}

func ExampleMillipede_opposite() {
	millipede := &Millipede{
		Size:     20,
		Reverse:  false,
		Skin:     "default",
		Opposite: true,
		Width:    3,
		Curve:    4,
	}
	fmt.Println(millipede)
	// Output:
	//       ╚⊙ ⊙╝
	//     ╚═(███)═╝
	//    ╚═(███)═╝
	//   ╚═(███)═╝
	//  ╚═(███)═╝
	// ╚═(███)═╝
	//  ╚═(███)═╝
	//   ╚═(███)═╝
	//    ╚═(███)═╝
	//     ╚═(███)═╝
	//    ╚═(███)═╝
	//   ╚═(███)═╝
	//  ╚═(███)═╝
	// ╚═(███)═╝
	//  ╚═(███)═╝
	//   ╚═(███)═╝
	//    ╚═(███)═╝
	//     ╚═(███)═╝
	//    ╚═(███)═╝
	//   ╚═(███)═╝
	//  ╚═(███)═╝
}

func ExampleMillipede_skin() {
	millipede := &Millipede{
		Size:     20,
		Reverse:  false,
		Skin:     "bocal",
		Opposite: false,
		Width:    3,
		Curve:    4,
	}
	fmt.Println(millipede)
	// Output:
	//   ╚⊙ ⊙╝
	// ╚═(🐟🐟🐟)═╝
	//  ╚═(🐟🐟🐟)═╝
	//   ╚═(🐟🐟🐟)═╝
	//    ╚═(🐟🐟🐟)═╝
	//     ╚═(🐟🐟🐟)═╝
	//    ╚═(🐟🐟🐟)═╝
	//   ╚═(🐟🐟🐟)═╝
	//  ╚═(🐟🐟🐟)═╝
	// ╚═(🐟🐟🐟)═╝
	//  ╚═(🐟🐟🐟)═╝
	//   ╚═(🐟🐟🐟)═╝
	//    ╚═(🐟🐟🐟)═╝
	//     ╚═(🐟🐟🐟)═╝
	//    ╚═(🐟🐟🐟)═╝
	//   ╚═(🐟🐟🐟)═╝
	//  ╚═(🐟🐟🐟)═╝
	// ╚═(🐟🐟🐟)═╝
	//  ╚═(🐟🐟🐟)═╝
	//   ╚═(🐟🐟🐟)═╝
	//    ╚═(🐟🐟🐟)═╝
}

func ExampleMillipede_width() {
	millipede := &Millipede{
		Size:     20,
		Reverse:  false,
		Skin:     "default",
		Opposite: false,
		Width:    6,
		Curve:    4,
	}
	fmt.Println(millipede)
	// Output:
	//   ╚⊙    ⊙╝
	// ╚═(██████)═╝
	//  ╚═(██████)═╝
	//   ╚═(██████)═╝
	//    ╚═(██████)═╝
	//     ╚═(██████)═╝
	//    ╚═(██████)═╝
	//   ╚═(██████)═╝
	//  ╚═(██████)═╝
	// ╚═(██████)═╝
	//  ╚═(██████)═╝
	//   ╚═(██████)═╝
	//    ╚═(██████)═╝
	//     ╚═(██████)═╝
	//    ╚═(██████)═╝
	//   ╚═(██████)═╝
	//  ╚═(██████)═╝
	// ╚═(██████)═╝
	//  ╚═(██████)═╝
	//   ╚═(██████)═╝
	//    ╚═(██████)═╝
}

func ExampleMillipede_curve() {
	millipede := &Millipede{
		Size:     20,
		Reverse:  false,
		Skin:     "default",
		Opposite: false,
		Width:    3,
		Curve:    6,
	}
	fmt.Println(millipede)
	// Output:
	//   ╚⊙ ⊙╝
	// ╚═(███)═╝
	//  ╚═(███)═╝
	//   ╚═(███)═╝
	//    ╚═(███)═╝
	//     ╚═(███)═╝
	//      ╚═(███)═╝
	//       ╚═(███)═╝
	//      ╚═(███)═╝
	//     ╚═(███)═╝
	//    ╚═(███)═╝
	//   ╚═(███)═╝
	//  ╚═(███)═╝
	// ╚═(███)═╝
	//  ╚═(███)═╝
	//   ╚═(███)═╝
	//    ╚═(███)═╝
	//     ╚═(███)═╝
	//      ╚═(███)═╝
	//       ╚═(███)═╝
	//      ╚═(███)═╝
}

func ExampleNew_complex() {
	millipede := &Millipede{
		Size:     42,
		Reverse:  true,
		Skin:     "bocal",
		Opposite: true,
		Width:    6,
		Curve:    10,
	}
	fmt.Println(millipede)
	// Output:
	//          ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//           ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//          ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//         ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//        ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//       ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//      ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//     ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//    ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//   ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//  ╔═(🐟🐟🐟🐟🐟🐟)═╗
	// ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//  ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//   ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//    ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//     ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//      ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//       ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//        ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//         ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//          ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//           ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//          ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//         ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//        ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//       ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//      ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//     ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//    ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//   ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//  ╔═(🐟🐟🐟🐟🐟🐟)═╗
	// ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//  ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//   ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//    ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//     ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//      ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//       ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//        ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//         ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//          ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//           ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//             ╔⊙    ⊙╗
}

func BenchmarkNew(b *testing.B) {
	for i := 0; i < b.N; i++ {
		New(20)
	}
}

func BenchmarkRendering(b *testing.B) {
	millipede := New(20)
	for i := 0; i < b.N; i++ {
		fmt.Sprintf("%s", millipede)
	}
}
<commit_msg>More tests<commit_after>package millipede

import (
	"fmt"
	"testing"
)

func ExampleNew() {
	millipede := New(20)
	fmt.Println(millipede)
	// Output:
	//   ╚⊙ ⊙╝
	// ╚═(███)═╝
	//  ╚═(███)═╝
	//   ╚═(███)═╝
	//    ╚═(███)═╝
	//     ╚═(███)═╝
	//    ╚═(███)═╝
	//   ╚═(███)═╝
	//  ╚═(███)═╝
	// ╚═(███)═╝
	//  ╚═(███)═╝
	//   ╚═(███)═╝
	//    ╚═(███)═╝
	//     ╚═(███)═╝
	//    ╚═(███)═╝
	//   ╚═(███)═╝
	//  ╚═(███)═╝
	// ╚═(███)═╝
	//  ╚═(███)═╝
	//   ╚═(███)═╝
	//    ╚═(███)═╝
}

func ExampleMillipede() {
	millipede := &Millipede{
		Size:     20,
		Reverse:  false,
		Skin:     "default",
		Opposite: false,
		Width:    3,
		Curve:    4,
	}
	fmt.Println(millipede)
	// Output:
	//   ╚⊙ ⊙╝
	// ╚═(███)═╝
	//  ╚═(███)═╝
	//   ╚═(███)═╝
	//    ╚═(███)═╝
	//     ╚═(███)═╝
	//    ╚═(███)═╝
	//   ╚═(███)═╝
	//  ╚═(███)═╝
	// ╚═(███)═╝
	//  ╚═(███)═╝
	//   ╚═(███)═╝
	//    ╚═(███)═╝
	//     ╚═(███)═╝
	//    ╚═(███)═╝
	//   ╚═(███)═╝
	//  ╚═(███)═╝
	// ╚═(███)═╝
	//  ╚═(███)═╝
	//   ╚═(███)═╝
	//    ╚═(███)═╝
}

func ExampleMillipede_reverse() {
	millipede := &Millipede{
		Size:     20,
		Reverse:  true,
		Skin:     "default",
		Opposite: false,
		Width:    3,
		Curve:    4,
	}
	fmt.Println(millipede)
	// Output:
	//    ╔═(███)═╗
	//   ╔═(███)═╗
	//  ╔═(███)═╗
	// ╔═(███)═╗
	//  ╔═(███)═╗
	//   ╔═(███)═╗
	//    ╔═(███)═╗
	//     ╔═(███)═╗
	//    ╔═(███)═╗
	//   ╔═(███)═╗
	//  ╔═(███)═╗
	// ╔═(███)═╗
	//  ╔═(███)═╗
	//   ╔═(███)═╗
	//    ╔═(███)═╗
	//     ╔═(███)═╗
	//    ╔═(███)═╗
	//   ╔═(███)═╗
	//  ╔═(███)═╗
	// ╔═(███)═╗
	//   ╔⊙ ⊙╗
}

func ExampleMillipede_opposite() {
	millipede := &Millipede{
		Size:     20,
		Reverse:  false,
		Skin:     "default",
		Opposite: true,
		Width:    3,
		Curve:    4,
	}
	fmt.Println(millipede)
	// Output:
	//       ╚⊙ ⊙╝
	//     ╚═(███)═╝
	//    ╚═(███)═╝
	//   ╚═(███)═╝
	//  ╚═(███)═╝
	// ╚═(███)═╝
	//  ╚═(███)═╝
	//   ╚═(███)═╝
	//    ╚═(███)═╝
	//     ╚═(███)═╝
	//    ╚═(███)═╝
	//   ╚═(███)═╝
	//  ╚═(███)═╝
	// ╚═(███)═╝
	//  ╚═(███)═╝
	//   ╚═(███)═╝
	//    ╚═(███)═╝
	//     ╚═(███)═╝
	//    ╚═(███)═╝
	//   ╚═(███)═╝
	//  ╚═(███)═╝
}

func ExampleMillipede_skin() {
	millipede := &Millipede{
		Size:     20,
		Reverse:  false,
		Skin:     "bocal",
		Opposite: false,
		Width:    3,
		Curve:    4,
	}
	fmt.Println(millipede)
	// Output:
	//   ╚⊙ ⊙╝
	// ╚═(🐟🐟🐟)═╝
	//  ╚═(🐟🐟🐟)═╝
	//   ╚═(🐟🐟🐟)═╝
	//    ╚═(🐟🐟🐟)═╝
	//     ╚═(🐟🐟🐟)═╝
	//    ╚═(🐟🐟🐟)═╝
	//   ╚═(🐟🐟🐟)═╝
	//  ╚═(🐟🐟🐟)═╝
	// ╚═(🐟🐟🐟)═╝
	//  ╚═(🐟🐟🐟)═╝
	//   ╚═(🐟🐟🐟)═╝
	//    ╚═(🐟🐟🐟)═╝
	//     ╚═(🐟🐟🐟)═╝
	//    ╚═(🐟🐟🐟)═╝
	//   ╚═(🐟🐟🐟)═╝
	//  ╚═(🐟🐟🐟)═╝
	// ╚═(🐟🐟🐟)═╝
	//  ╚═(🐟🐟🐟)═╝
	//   ╚═(🐟🐟🐟)═╝
	//    ╚═(🐟🐟🐟)═╝
}

func ExampleMillipede_width() {
	millipede := &Millipede{
		Size:     20,
		Reverse:  false,
		Skin:     "default",
		Opposite: false,
		Width:    6,
		Curve:    4,
	}
	fmt.Println(millipede)
	// Output:
	//   ╚⊙    ⊙╝
	// ╚═(██████)═╝
	//  ╚═(██████)═╝
	//   ╚═(██████)═╝
	//    ╚═(██████)═╝
	//     ╚═(██████)═╝
	//    ╚═(██████)═╝
	//   ╚═(██████)═╝
	//  ╚═(██████)═╝
	// ╚═(██████)═╝
	//  ╚═(██████)═╝
	//   ╚═(██████)═╝
	//    ╚═(██████)═╝
	//     ╚═(██████)═╝
	//    ╚═(██████)═╝
	//   ╚═(██████)═╝
	//  ╚═(██████)═╝
	// ╚═(██████)═╝
	//  ╚═(██████)═╝
	//   ╚═(██████)═╝
	//    ╚═(██████)═╝
}

func ExampleMillipede_curve() {
	millipede := &Millipede{
		Size:     20,
		Reverse:  false,
		Skin:     "default",
		Opposite: false,
		Width:    3,
		Curve:    6,
	}
	fmt.Println(millipede)
	// Output:
	//   ╚⊙ ⊙╝
	// ╚═(███)═╝
	//  ╚═(███)═╝
	//   ╚═(███)═╝
	//    ╚═(███)═╝
	//     ╚═(███)═╝
	//      ╚═(███)═╝
	//       ╚═(███)═╝
	//      ╚═(███)═╝
	//     ╚═(███)═╝
	//    ╚═(███)═╝
	//   ╚═(███)═╝
	//  ╚═(███)═╝
	// ╚═(███)═╝
	//  ╚═(███)═╝
	//   ╚═(███)═╝
	//    ╚═(███)═╝
	//     ╚═(███)═╝
	//      ╚═(███)═╝
	//       ╚═(███)═╝
	//      ╚═(███)═╝
}

func TestMillipede_zalgo(t *testing.T) {
	// FIXME: find a better test
	millipede := New(20)
	millipede.Zalgo = true
	millipede.String()
}

func TestMillipede_small_width(t *testing.T) {
	millipede := New(20)
	millipede.Width = 2
	// FIXME: check if it exits
	//millipede.String()
}

func ExampleNew_complex() {
	millipede := &Millipede{
		Size:     42,
		Reverse:  true,
		Skin:     "bocal",
		Opposite: true,
		Width:    6,
		Curve:    10,
	}
	fmt.Println(millipede)
	// Output:
	//          ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//           ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//          ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//         ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//        ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//       ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//      ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//     ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//    ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//   ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//  ╔═(🐟🐟🐟🐟🐟🐟)═╗
	// ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//  ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//   ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//    ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//     ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//      ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//       ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//        ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//         ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//          ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//           ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//          ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//         ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//        ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//       ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//      ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//     ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//    ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//   ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//  ╔═(🐟🐟🐟🐟🐟🐟)═╗
	// ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//  ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//   ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//    ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//     ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//      ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//       ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//        ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//         ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//          ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//           ╔═(🐟🐟🐟🐟🐟🐟)═╗
	//             ╔⊙    ⊙╗
}

func ExampleStringToRuneSlice() {
	input := "╔═(🐟🐟🐟🐟🐟🐟)═╗"
	output := StringToRuneSlice(input)
	fmt.Println(output)
	// Output: [9556 9552 40 128031 128031 128031 128031 128031 128031 41 9552 9559]
}

func BenchmarkNew(b *testing.B) {
	for i := 0; i < b.N; i++ {
		New(20)
	}
}

func BenchmarkRendering(b *testing.B) {
	millipede := New(20)
	for i := 0; i < b.N; i++ {
		fmt.Sprintf("%s", millipede)
	}
}
<|endoftext|>


<commit_before>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package router

import (
	. "github.com/TheThingsNetwork/ttn/core"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	"github.com/TheThingsNetwork/ttn/utils/stats"
	"github.com/apex/log"
)

type component struct {
	Storage
	ctx log.Interface
}

// New constructs a new router
func New(db Storage, ctx log.Interface) Component {
	return component{Storage: db, ctx: ctx}
}

// Register implements the core.Component interface
func (r component) Register(reg Registration, an AckNacker) (err error) {
	defer ensureAckNack(an, nil, &err)
	stats.MarkMeter("router.registration.in")
	r.ctx.Debug("Handling registration")

	if err := r.Store(reg); err != nil {
		return errors.New(errors.Operational, err)
	}
	return nil
}

// HandleUp implements the core.Component interface
func (r component) HandleUp(data []byte, an AckNacker, up Adapter) (err error) {
	// Make sure we don't forget the AckNacker
	var ack Packet
	defer ensureAckNack(an, &ack, &err)

	// Get some logs / analytics
	stats.MarkMeter("router.uplink.in")
	r.ctx.Debug("Handling uplink packet")

	// Extract the given packet
	itf, err := UnmarshalPacket(data)
	if err != nil {
		stats.MarkMeter("router.uplink.invalid")
		r.ctx.Warn("Uplink Invalid")
		return errors.New(errors.Structural, err)
	}

	switch itf.(type) {
	case RPacket:
		packet := itf.(RPacket)

		// Lookup for an existing broker
		// NOTE We are still assuming only one broker associated to one device address.
		// We should find a mechanism to make sure that the broker in database is really
		// associated to the device to avoid trouble during overlaping.
		// Keeping track of the last FCnt maybe ? Having an overlap on the frame counter + the
		// device address might be less likely.
		entry, err := r.Lookup(packet.DevEUI())
		if err != nil && err.(errors.Failure).Nature != errors.Behavioural {
			r.ctx.Warn("Database lookup failed")
			return errors.New(errors.Operational, err)
		}

		var recipient Recipient
		if err == nil {
			rawRecipient := entry.Recipient
			if recipient, err = up.GetRecipient(rawRecipient); err != nil {
				r.ctx.Warn("Unable to retrieve Recipient")
				return errors.New(errors.Operational, err)
			}
		}

		// TODO -> Add Gateway Metadata to packet

		bpacket, err := NewBPacket(packet.Payload(), packet.Metadata())
		if err != nil {
			r.ctx.WithError(err).Warn("Unable to create router packet")
			return errors.New(errors.Structural, err)
		}

		response, err := up.Send(bpacket, recipient)

		if err != nil {
			stats.MarkMeter("router.uplink.bad_broker_response")
			r.ctx.WithError(err).Warn("Invalid response from Broker")
			return errors.New(errors.Operational, err)
		}

		itf, err := UnmarshalPacket(response)
		if err != nil {
			stats.MarkMeter("router.uplink.bad_broker_response")
			r.ctx.WithError(err).Warn("Invalid response from Broker")
			return errors.New(errors.Structural, err)
		}

		switch itf.(type) {
		case RPacket:
			ack = itf.(RPacket)
		case nil:
		default:
			return errors.New(errors.Implementation, "Unexpected packet type")
		}

		stats.MarkMeter("router.uplink.ok")
	case SPacket:
		return errors.New(errors.Implementation, "Stats packet not yet implemented")
	case JPacket:
		return errors.New(errors.Implementation, "Join Request not yet implemented")
	default:
		return errors.New(errors.Implementation, "Unreckognized packet type")
	}

	return nil
}

// HandleDown implements the core.Component interface
func (r component) HandleDown(data []byte, an AckNacker, up Adapter) error {
	return errors.New(errors.Implementation, "Handle down not implemented on router")
}

func ensureAckNack(an AckNacker, ack *Packet, err *error) {
	if err != nil && *err != nil {
		an.Nack()
	} else {
		var p Packet
		if ack != nil {
			p = *ack
		}
		an.Ack(p)
	}
}
<commit_msg>[test-router] Fix small issues in Register() method according to test results<commit_after>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package router

import (
	. "github.com/TheThingsNetwork/ttn/core"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	"github.com/TheThingsNetwork/ttn/utils/stats"
	"github.com/apex/log"
)

type component struct {
	Storage
	ctx log.Interface
}

// New constructs a new router
func New(db Storage, ctx log.Interface) Component {
	return component{Storage: db, ctx: ctx}
}

// Register implements the core.Component interface
func (r component) Register(reg Registration, an AckNacker) (err error) {
	defer ensureAckNack(an, nil, &err)
	stats.MarkMeter("router.registration.in")
	r.ctx.Debug("Handling registration")
	return r.Store(reg)
}

// HandleUp implements the core.Component interface
func (r component) HandleUp(data []byte, an AckNacker, up Adapter) (err error) {
	// Make sure we don't forget the AckNacker
	var ack Packet
	defer ensureAckNack(an, &ack, &err)

	// Get some logs / analytics
	stats.MarkMeter("router.uplink.in")
	r.ctx.Debug("Handling uplink packet")

	// Extract the given packet
	itf, err := UnmarshalPacket(data)
	if err != nil {
		stats.MarkMeter("router.uplink.invalid")
		r.ctx.Warn("Uplink Invalid")
		return errors.New(errors.Structural, err)
	}

	switch itf.(type) {
	case RPacket:
		packet := itf.(RPacket)

		// Lookup for an existing broker
		// NOTE We are still assuming only one broker associated to one device address.
		// We should find a mechanism to make sure that the broker in database is really
		// associated to the device to avoid trouble during overlaping.
		// Keeping track of the last FCnt maybe ? Having an overlap on the frame counter + the
		// device address might be less likely.
		entry, err := r.Lookup(packet.DevEUI())
		if err != nil && err.(errors.Failure).Nature != errors.Behavioural {
			r.ctx.Warn("Database lookup failed")
			return errors.New(errors.Operational, err)
		}

		var recipient Recipient
		if err == nil {
			rawRecipient := entry.Recipient
			if recipient, err = up.GetRecipient(rawRecipient); err != nil {
				r.ctx.Warn("Unable to retrieve Recipient")
				return errors.New(errors.Operational, err)
			}
		}

		// TODO -> Add Gateway Metadata to packet

		bpacket, err := NewBPacket(packet.Payload(), packet.Metadata())
		if err != nil {
			r.ctx.WithError(err).Warn("Unable to create router packet")
			return errors.New(errors.Structural, err)
		}

		response, err := up.Send(bpacket, recipient)

		if err != nil {
			stats.MarkMeter("router.uplink.bad_broker_response")
			r.ctx.WithError(err).Warn("Invalid response from Broker")
			return errors.New(errors.Operational, err)
		}

		itf, err := UnmarshalPacket(response)
		if err != nil {
			stats.MarkMeter("router.uplink.bad_broker_response")
			r.ctx.WithError(err).Warn("Invalid response from Broker")
			return errors.New(errors.Structural, err)
		}

		switch itf.(type) {
		case RPacket:
			ack = itf.(RPacket)
		case nil:
		default:
			return errors.New(errors.Implementation, "Unexpected packet type")
		}

		stats.MarkMeter("router.uplink.ok")
	case SPacket:
		return errors.New(errors.Implementation, "Stats packet not yet implemented")
	case JPacket:
		return errors.New(errors.Implementation, "Join Request not yet implemented")
	default:
		return errors.New(errors.Implementation, "Unreckognized packet type")
	}

	return nil
}

// HandleDown implements the core.Component interface
func (r component) HandleDown(data []byte, an AckNacker, up Adapter) error {
	return errors.New(errors.Implementation, "Handle down not implemented on router")
}

func ensureAckNack(an AckNacker, ack *Packet, err *error) {
	if err != nil && *err != nil {
		an.Nack()
	} else {
		var p Packet
		if ack != nil {
			p = *ack
		}
		an.Ack(p)
	}
}
<|endoftext|>