<commit_before>package ghalloc

import (
	"testing"
	"unsafe"
)

func TestNewSlab(t *testing.T) {
	sc := newSlabClass(512*KB, 1*MB)
	slab := newSlab(sc)

	if len(slab.memory) != 1*MB {
		t.Fatal("slab: allocated wrong memory slice")
	}

	//if len(slab.chunks) != 2 {
	//t.Fatal("slab: wrong slab memory partition")
	//}

	//if slab.chunks[0] != unsafe.Pointer(&slab.memory[0]) && slab.chunks[1] != unsafe.Pointer(uintptr(unsafe.Pointer(&slab.memory[0]))+uintptr(512*KB)) {
	//t.Fatal("slab: wrong pointers")
	//}
}

func TestAllocChunk(t *testing.T) {
	sc := newSlabClass(512*KB, 1*MB)
	slab := newSlab(sc)

	ptr := slab.allocChunk()

	if ptr == nil {
		t.Fatal("slab alloc: allocated nil")
	}

	if slab.allocated != 1 {
		t.Fatal("slab alloc: wrong allocation counter")
	}

	ptr = slab.allocChunk()

	if ptr == nil {
		t.Fatal("slab alloc: allocated nil")
	}

	if !slab.full {
		t.Fatal("slab alloc: wrong slab status. Expected full.")
	}

	ptr = slab.allocChunk()

	if ptr != nil {
		t.Fatal("slab alloc: Expected nil, got pointer")
	}
}

//func TestFindChunk(t *testing.T) {
//sc := newSlabClass(512*KB, 1*MB)
//slab := newSlab(sc)

//ptr := slab.allocChunk()
//invalidPtr := unsafe.Pointer(&sc)

//if slab.findChunk(ptr) < 0 {
//t.Fatal("slab find chunk: chunk is not found")
//}

//if slab.findChunk(invalidPtr) >= 0 {
//t.Fatal("slab find chunk: wrong chunk found")
//}
//}

func TestFreeChunk(t *testing.T) {
	var (
		sc         *slabClass = newSlabClass(512*KB, 1*MB)
		slab       *slab      = newSlab(sc)
		ptr        unsafe.Pointer
		invalidPtr unsafe.Pointer = unsafe.Pointer(&sc)
	)

	for !slab.full {
		ptr = slab.allocChunk()
	}

	slab.freeChunk(invalidPtr)

	if !slab.full {
		t.Fatal("slab free chunk: freed wrong chunk")
	}

	//slab.freeChunk(slab.chunks[0])

	//if slab.full {
	//t.Fatal("slab free chunk: chunk is not freed")
	//}

	slab.freeChunk(ptr)

	if slab.full {
		t.Fatal("slab free chunk: chunk is not freed")
	}
}
<commit_msg>Clean tests<commit_after>package ghalloc

import (
	"testing"
	"unsafe"
)

func TestNewSlab(t *testing.T) {
	sc := newSlabClass(512*KB, 1*MB)
	slab := newSlab(sc)

	if len(slab.memory) != 1*MB {
		t.Fatal("slab: allocated wrong memory slice")
	}
}

func TestAllocChunk(t *testing.T) {
	sc := newSlabClass(512*KB, 1*MB)
	slab := newSlab(sc)

	ptr := slab.allocChunk()

	if ptr == nil {
		t.Fatal("slab alloc: allocated nil")
	}

	if slab.allocated != 1 {
		t.Fatal("slab alloc: wrong allocation counter")
	}

	ptr = slab.allocChunk()

	if ptr == nil {
		t.Fatal("slab alloc: allocated nil")
	}

	if !slab.full {
		t.Fatal("slab alloc: wrong slab status. Expected full.")
	}

	ptr = slab.allocChunk()

	if ptr != nil {
		t.Fatal("slab alloc: Expected nil, got pointer")
	}
}

func TestFreeChunk(t *testing.T) {
	var (
		sc         *slabClass = newSlabClass(512*KB, 1*MB)
		slab       *slab      = newSlab(sc)
		ptr        unsafe.Pointer
		invalidPtr unsafe.Pointer = unsafe.Pointer(&sc)
	)

	for !slab.full {
		ptr = slab.allocChunk()
	}

	slab.freeChunk(invalidPtr)

	if !slab.full {
		t.Fatal("slab free chunk: freed wrong chunk")
	}

	slab.freeChunk(ptr)

	if slab.full {
		t.Fatal("slab free chunk: chunk is not freed")
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 The SurgeMQ Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"bufio"
	"fmt"
	"io"
	"sync/atomic"
	"runtime"
//"github.com/surgemq/message"
	"encoding/binary"
	"math"
	"sync"
)

var (
	bufcnt int64
	DefaultBufferSize int64

	DeviceInBufferSize int64
	DeviceOutBufferSize int64

	MasterInBufferSize int64
	MasterOutBufferSize int64
)

const (
	smallReadBlockSize = 512
	defaultReadBlockSize = 8192
	defaultWriteBlockSize = 8192
)

/**
2016.03.03 修改
bingbuffer结构体
 */
type buffer struct {
	readIndex  int64        //读序号
	writeIndex int64        //写序号
	ringBuffer []*ByteArray //环形buffer指针数组
	bufferSize int64        //初始化环形buffer指针数组大小
	mask       int64        //掩码：bufferSize-1
	done       int64        //是否完成
	rcond      *sync.Cond
	wcond      *sync.Cond
}

type ByteArray struct {
	bArray []byte
}

func (this *buffer)ReadCommit(index int64) {
	this.ringBuffer[index] = nil
}

/**
2016.03.03 添加
初始化ringbuffer
参数bufferSize：初始化环形buffer指针数组大小
 */
func newBuffer(size int64) (*buffer, error) {
	if size < 0 {
		return nil, bufio.ErrNegativeCount
	}
	if size == 0 {
		size = DefaultBufferSize
	}
	if !powerOfTwo64(size) {
		fmt.Printf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
		return nil, fmt.Errorf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
	}

	return &buffer{
		readIndex: int64(0), //读序号
		writeIndex: int64(0), //写序号
		ringBuffer: make([]*ByteArray, size), //环形buffer指针数组
		bufferSize: size, //初始化环形buffer指针数组大小
		mask:size - 1,
		rcond:         sync.NewCond(new(sync.Mutex)),
		wcond:         sync.NewCond(new(sync.Mutex)),
	}, nil
}

/**
2016.03.03 添加
获取当前读序号
 */
func (this *buffer)GetCurrentReadIndex() (int64) {
	return atomic.LoadInt64(&this.readIndex)
}
/**
2016.03.03 添加
获取当前写序号
 */
func (this *buffer)GetCurrentWriteIndex() (int64) {
	return atomic.LoadInt64(&this.writeIndex)
}

/**
2016.03.03 添加
读取ringbuffer指定的buffer指针，返回该指针并清空ringbuffer该位置存在的指针内容，以及将读序号加1
 */
func (this *buffer)ReadBuffer() ([]byte, int64, bool) {
	this.rcond.L.Lock()
	defer this.rcond.L.Unlock()
	readIndex := atomic.LoadInt64(&this.readIndex)
	writeIndex := atomic.LoadInt64(&this.writeIndex)
	switch  {
	case readIndex >= writeIndex:
		return nil, -1, false
	case writeIndex - readIndex > this.bufferSize:
		return nil, -1, false
	default:
		if readIndex == math.MaxInt64 {
			atomic.StoreInt64(&this.readIndex, int64(0))
		}else {
			atomic.AddInt64(&this.readIndex, int64(1))
		}

		index := readIndex & this.mask

		p_ := this.ringBuffer[index]
		//this.ringBuffer[index] = nil
		if p_ == nil {
			return nil, -1, false
		}
		p := p_.bArray

		if p == nil {
			return nil, -1, false
		}

		return p, index, true
	}
	return nil, -1, false
}

/**
2016.03.03 添加
写入ringbuffer指针，以及将写序号加1
 */
func (this *buffer)WriteBuffer(in []byte) (bool) {
	this.wcond.L.Lock()
	defer this.wcond.L.Unlock()
	readIndex := atomic.LoadInt64(&this.readIndex)
	writeIndex := atomic.LoadInt64(&this.writeIndex)
	switch  {
	case writeIndex - readIndex < 0:
		return false
	default:
		index := writeIndex & this.mask
		if writeIndex == math.MaxInt64 {
			atomic.StoreInt64(&this.writeIndex, int64(0))
		}else {
			atomic.AddInt64(&this.writeIndex, int64(1))
		}
		if this.ringBuffer[index] == nil {
			this.ringBuffer[index] = &ByteArray{bArray:in}

			return true
		}else {
			return false
		}
	}
}

/**
2016.03.03 修改
完成
 */
func (this *buffer) Close() error {
	atomic.StoreInt64(&this.done, 1)
	atomic.StoreInt64(&this.done, 1)

	this.wcond.L.Lock()
	this.rcond.Broadcast()
	this.wcond.L.Unlock()

	this.rcond.L.Lock()
	this.wcond.Broadcast()
	this.rcond.L.Unlock()

	return nil
}
/*

/**
2016.03.03 修改
向ringbuffer中写数据（从connection的中向ringbuffer中写）--生产者
*/
func (this *buffer) ReadFrom(r io.Reader) (int64, error) {
	defer this.Close()
	for {
		total := int64(0)
		if this.isDone() {
			return total, io.EOF
		}
		b := make([]byte, 5)
		n, err := r.Read(b[0:1])

		if n > 0 {
			total += int64(n)
			if err != nil {
				return total, err
			}
		}

		/**************************/
		cnt := 1

		// Let's read enough bytes to get the message header (msg type, remaining length)
		for {
			// If we have read 5 bytes and still not done, then there's a problem.
			if cnt > 4 {
				return 0, fmt.Errorf("sendrecv/peekMessageSize: 4th byte of remaining length has continuation bit set")
			}

			// Peek cnt bytes from the input buffer.
			_, err := r.Read(b[cnt:cnt + 1])
			if err != nil {
				return 0, err
			}

			// If we got enough bytes, then check the last byte to see if the continuation
			// bit is set. If so, increment cnt and continue peeking
			if b[cnt] >= 0x80 {
				cnt++
			} else {
				break
			}
		}

		// Get the remaining length of the message
		remlen, _ := binary.Uvarint(b[1:])

		// Total message length is remlen + 1 (msg type) + m (remlen bytes)
		len := int64(len(b))
		remlen_ := int64(remlen)
		total = remlen_ + int64(len)

		//mtype := message.MessageType(b[0] >> 4)
		/****************/
		//var msg message.Message
		//
		//msg, err = mtype.New()
		//if err != nil {
		//	return 0, err
		//}
		b_ := make([]byte, remlen_)
		_, err = r.Read(b_[0:])
		if err != nil {
			return 0, err
		}

		b = append(b, b_...)
		//n, err = msg.Decode(b)
		//if err != nil {
		//	return 0, err
		//}

		/*************************/

		for i := 0; i < 100; i++ {
			if this.isDone() {
				return total, io.EOF
			}
			if this.WriteBuffer(b) {
				break
			}
			runtime.Gosched()
		}

		return total, nil
	}
}

/**
2016.03.03 修改
 */
func (this *buffer) WriteTo(w io.Writer) (int64, error) {
	defer this.Close()
	total := int64(0)
	for {
		if this.isDone() {
			return total, io.EOF
		}
		p, index, ok := this.ReadBuffer()
		if !ok {
			runtime.Gosched()
			continue
		}

		Log.Debugc(func() string {
			return fmt.Sprintf("defer this.ReadCommit(%s)", index)
		})
		Log.Debugc(func() string {
			return fmt.Sprintf("WriteTo函数》》读取*p：" + string(p))
		})

		Log.Debugc(func() string {
			return fmt.Sprintf(" WriteTo(w io.Writer)(7)")
		})
		//
		//Log.Errorc(func() string {
		//	return fmt.Sprintf("msg::" + msg.Name())
		//})
		//
		//p := make([]byte, msg.Len())
		//_, err := msg.Encode(p)
		//if err != nil {
		//	Log.Errorc(func() string {
		//		return fmt.Sprintf("msg.Encode(p)")
		//	})
		//	return total, io.EOF
		//}
		// There's some data, let's process it first
		if len(p) > 0 {
			n, err := w.Write(p)
			total += int64(n)
			Log.Debugc(func() string {
				return fmt.Sprintf("Wrote %d bytes, totaling %d bytes", n, total)
			})

			if err != nil {
				Log.Errorc(func() string {
					return fmt.Sprintf("w.Write(p) error")
				})
				return total, err
			}
		}
		this.ReadCommit(index)
		return total, nil
	}
}

/**
2016.03.03 修改
*/
func (this *buffer) isDone() bool {
	if atomic.LoadInt64(&this.done) == 1 {
		return true
	}

	return false
}

func powerOfTwo64(n int64) bool {
	return n != 0 && (n & (n - 1)) == 0
}

func roundUpPowerOfTwo64(n int64) int64 {
	n--
	n |= n >> 1
	n |= n >> 2
	n |= n >> 4
	n |= n >> 8
	n |= n >> 16
	n |= n >> 32
	n++

	return n
}
<commit_msg>修改buffer读写限制<commit_after>// Copyright (c) 2014 The SurgeMQ Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"bufio"
	"fmt"
	"io"
	"sync/atomic"
	"runtime"
//"github.com/surgemq/message"
	"encoding/binary"
	"math"
	"sync"
)

var (
	bufcnt int64
	DefaultBufferSize int64

	DeviceInBufferSize int64
	DeviceOutBufferSize int64

	MasterInBufferSize int64
	MasterOutBufferSize int64
)

const (
	smallReadBlockSize = 512
	defaultReadBlockSize = 8192
	defaultWriteBlockSize = 8192
)

/**
2016.03.03 修改
bingbuffer结构体
 */
type buffer struct {
	readIndex  int64        //读序号
	writeIndex int64        //写序号
	ringBuffer []*ByteArray //环形buffer指针数组
	bufferSize int64        //初始化环形buffer指针数组大小
	mask       int64        //掩码：bufferSize-1
	done       int64        //是否完成
	rcond      *sync.Cond
	wcond      *sync.Cond
}

type ByteArray struct {
	bArray []byte
}

func (this *buffer)ReadCommit(index int64) {
	this.ringBuffer[index] = nil
}

/**
2016.03.03 添加
初始化ringbuffer
参数bufferSize：初始化环形buffer指针数组大小
 */
func newBuffer(size int64) (*buffer, error) {
	if size < 0 {
		return nil, bufio.ErrNegativeCount
	}
	if size == 0 {
		size = DefaultBufferSize
	}
	if !powerOfTwo64(size) {
		fmt.Printf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
		return nil, fmt.Errorf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
	}

	return &buffer{
		readIndex: int64(0), //读序号
		writeIndex: int64(0), //写序号
		ringBuffer: make([]*ByteArray, size), //环形buffer指针数组
		bufferSize: size, //初始化环形buffer指针数组大小
		mask:size - 1,
		rcond:         sync.NewCond(new(sync.Mutex)),
		wcond:         sync.NewCond(new(sync.Mutex)),
	}, nil
}

/**
2016.03.03 添加
获取当前读序号
 */
func (this *buffer)GetCurrentReadIndex() (int64) {
	return atomic.LoadInt64(&this.readIndex)
}
/**
2016.03.03 添加
获取当前写序号
 */
func (this *buffer)GetCurrentWriteIndex() (int64) {
	return atomic.LoadInt64(&this.writeIndex)
}

/**
2016.03.03 添加
读取ringbuffer指定的buffer指针，返回该指针并清空ringbuffer该位置存在的指针内容，以及将读序号加1
 */
func (this *buffer)ReadBuffer() ([]byte, int64, bool) {
	this.rcond.L.Lock()
	defer this.rcond.L.Unlock()

	for {
		readIndex := atomic.LoadInt64(&this.readIndex)
		writeIndex := atomic.LoadInt64(&this.writeIndex)
		switch  {
		case readIndex >= writeIndex:
			this.rcond.Wait()
		case writeIndex - readIndex > this.bufferSize:
			this.rcond.Wait()
		default:
			if readIndex == math.MaxInt64 {
				atomic.StoreInt64(&this.readIndex, int64(0))
			}else {
				atomic.AddInt64(&this.readIndex, int64(1))
			}

			index := readIndex & this.mask

			p_ := this.ringBuffer[index]
			//this.ringBuffer[index] = nil
			if p_ == nil {
				return nil, -1, false
			}
			p := p_.bArray

			if p == nil {
				return nil, -1, false
			}

			return p, index, true
		}
	}

}

/**
2016.03.03 添加
写入ringbuffer指针，以及将写序号加1
 */
func (this *buffer)WriteBuffer(in []byte) (bool) {
	this.wcond.L.Lock()
	defer this.wcond.L.Unlock()

	for {
		readIndex := atomic.LoadInt64(&this.readIndex)
		writeIndex := atomic.LoadInt64(&this.writeIndex)
		switch  {
		case writeIndex - readIndex < 0:
			this.wcond.Wait()
		default:
			index := writeIndex & this.mask
			if writeIndex == math.MaxInt64 {
				atomic.StoreInt64(&this.writeIndex, int64(0))
			}else {
				atomic.AddInt64(&this.writeIndex, int64(1))
			}
			if this.ringBuffer[index] == nil {
				this.ringBuffer[index] = &ByteArray{bArray:in}

				return true
			}else {
				return false
			}
		}
	}

}

/**
2016.03.03 修改
完成
 */
func (this *buffer) Close() error {
	atomic.StoreInt64(&this.done, 1)
	atomic.StoreInt64(&this.done, 1)

	this.wcond.L.Lock()
	this.rcond.Broadcast()
	this.wcond.L.Unlock()

	this.rcond.L.Lock()
	this.wcond.Broadcast()
	this.rcond.L.Unlock()

	return nil
}
/*

/**
2016.03.03 修改
向ringbuffer中写数据（从connection的中向ringbuffer中写）--生产者
*/
func (this *buffer) ReadFrom(r io.Reader) (int64, error) {
	defer this.Close()
	for {
		total := int64(0)
		if this.isDone() {
			return total, io.EOF
		}
		b := make([]byte, 5)
		n, err := r.Read(b[0:1])

		if n > 0 {
			total += int64(n)
			if err != nil {
				return total, err
			}
		}

		/**************************/
		cnt := 1

		// Let's read enough bytes to get the message header (msg type, remaining length)
		for {
			// If we have read 5 bytes and still not done, then there's a problem.
			if cnt > 4 {
				return 0, fmt.Errorf("sendrecv/peekMessageSize: 4th byte of remaining length has continuation bit set")
			}

			// Peek cnt bytes from the input buffer.
			_, err := r.Read(b[cnt:cnt + 1])
			if err != nil {
				return 0, err
			}

			// If we got enough bytes, then check the last byte to see if the continuation
			// bit is set. If so, increment cnt and continue peeking
			if b[cnt] >= 0x80 {
				cnt++
			} else {
				break
			}
		}

		// Get the remaining length of the message
		remlen, _ := binary.Uvarint(b[1:])

		// Total message length is remlen + 1 (msg type) + m (remlen bytes)
		len := int64(len(b))
		remlen_ := int64(remlen)
		total = remlen_ + int64(len)

		//mtype := message.MessageType(b[0] >> 4)
		/****************/
		//var msg message.Message
		//
		//msg, err = mtype.New()
		//if err != nil {
		//	return 0, err
		//}
		b_ := make([]byte, remlen_)
		_, err = r.Read(b_[0:])
		if err != nil {
			return 0, err
		}

		b = append(b, b_...)
		//n, err = msg.Decode(b)
		//if err != nil {
		//	return 0, err
		//}

		/*************************/

		for i := 0; i < 100; i++ {
			if this.isDone() {
				return total, io.EOF
			}
			if this.WriteBuffer(b) {
				break
			}
			runtime.Gosched()
		}

		return total, nil
	}
}

/**
2016.03.03 修改
 */
func (this *buffer) WriteTo(w io.Writer) (int64, error) {
	defer this.Close()
	total := int64(0)
	for {
		if this.isDone() {
			return total, io.EOF
		}
		p, index, ok := this.ReadBuffer()
		if !ok {
			runtime.Gosched()
			continue
		}

		Log.Debugc(func() string {
			return fmt.Sprintf("defer this.ReadCommit(%s)", index)
		})
		Log.Debugc(func() string {
			return fmt.Sprintf("WriteTo函数》》读取*p：" + string(p))
		})

		Log.Debugc(func() string {
			return fmt.Sprintf(" WriteTo(w io.Writer)(7)")
		})
		//
		//Log.Errorc(func() string {
		//	return fmt.Sprintf("msg::" + msg.Name())
		//})
		//
		//p := make([]byte, msg.Len())
		//_, err := msg.Encode(p)
		//if err != nil {
		//	Log.Errorc(func() string {
		//		return fmt.Sprintf("msg.Encode(p)")
		//	})
		//	return total, io.EOF
		//}
		// There's some data, let's process it first
		if len(p) > 0 {
			n, err := w.Write(p)
			total += int64(n)
			Log.Debugc(func() string {
				return fmt.Sprintf("Wrote %d bytes, totaling %d bytes", n, total)
			})

			if err != nil {
				Log.Errorc(func() string {
					return fmt.Sprintf("w.Write(p) error")
				})
				return total, err
			}
		}
		this.ReadCommit(index)
		return total, nil
	}
}

/**
2016.03.03 修改
*/
func (this *buffer) isDone() bool {
	if atomic.LoadInt64(&this.done) == 1 {
		return true
	}

	return false
}

func powerOfTwo64(n int64) bool {
	return n != 0 && (n & (n - 1)) == 0
}

func roundUpPowerOfTwo64(n int64) int64 {
	n--
	n |= n >> 1
	n |= n >> 2
	n |= n >> 4
	n |= n >> 8
	n |= n >> 16
	n |= n >> 32
	n++

	return n
}
<|endoftext|>


<commit_before>package irckit

import (
	"html"
	"regexp"
	"strings"
	"time"

	"github.com/nlopes/slack"
)

type SlackInfo struct {
	Token  string
	sc     *slack.Client
	rtm    *slack.RTM
	sinfo  *slack.Info
	susers map[string]slack.User
}

func (u *User) loginToSlack() (*slack.Client, error) {
	u.sc = slack.New(u.Token)
	u.rtm = u.sc.NewRTM()
	u.susers = make(map[string]slack.User)
	go u.rtm.ManageConnection()
	go u.handleSlack()
	//time.Sleep(time.Second * 2)
	u.sinfo = u.rtm.GetInfo()
	for u.sinfo == nil {
		time.Sleep(time.Millisecond * 500)
		logger.Debug("still waiting for sinfo")
		u.sinfo = u.rtm.GetInfo()
	}
	u.addSlackUsersToChannels()
	return u.sc, nil
}

func (u *User) logoutFromSlack() error {
	logger.Debug("calling logout from slack")
	err := u.rtm.Disconnect()
	if err != nil {
		logger.Debug("logoutfrom slack", err)
		return err
	}
	u.Srv.Logout(u)
	u.sc = nil
	logger.Info("logout succeeded")
	return nil
}

func (u *User) createSlackUser(slackuser *slack.User) *User {
	if slackuser == nil {
		return nil
	}
	if ghost, ok := u.Srv.HasUser(slackuser.Name); ok {
		return ghost
	}
	ghost := &User{Nick: slackuser.Name, User: slackuser.ID, Real: slackuser.RealName, Host: "host", Roles: "", channels: map[Channel]struct{}{}}
	ghost.MmGhostUser = true
	u.Srv.Add(ghost)
	return ghost
}

func (u *User) addSlackUserToChannel(user *slack.User, channel string, channelId string) {
	if user == nil {
		return
	}
	ghost := u.createSlackUser(user)
	if ghost == nil {
		logger.Warnf("Cannot join %v into %s", user, channel)
		return
	}
	logger.Debugf("adding %s to %s (%s)", ghost.Nick, channel, channelId)
	ch := u.Srv.Channel(channelId)
	logger.Debugf("channel: %#v %#v", ch.String(), ch.ID())
	ch.Join(ghost)
}

func (u *User) addSlackUsersToChannels() {
	srv := u.Srv
	throttle := time.Tick(time.Millisecond * 100)
	logger.Debug("in addUsersToChannels()")
	// add all users, also who are not on channels
	ch := srv.Channel("&users")
	users, _ := u.sc.GetUsers()
	for _, mmuser := range users {
		// do not add our own nick
		if mmuser.ID == u.sinfo.User.ID {
			continue
		}
		u.createSlackUser(&mmuser)
		u.addSlackUserToChannel(&mmuser, "&users", "&users")
		u.susers[mmuser.ID] = mmuser
	}
	ch.Join(u)

	channels := make(chan interface{}, 10)
	for i := 0; i < 10; i++ {
		go u.addSlackUserToChannelWorker(channels, throttle)
	}
	groups, _ := u.sc.GetGroups(true)
	mmchannels, _ := u.sc.GetChannels(true)
	for _, mmchannel := range mmchannels {
		if mmchannel.IsMember {
			logger.Debug("Adding channel", mmchannel)
			channels <- mmchannel
		}
	}
	for _, mmchannel := range groups {
		logger.Debug("Adding private channel", mmchannel)
		channels <- mmchannel
	}
	close(channels)
}

func (u *User) addSlackUserToChannelWorker(channels <-chan interface{}, throttle <-chan time.Time) {
	var ID, name string
	for {
		mmchannel, ok := <-channels
		if !ok {
			logger.Debug("Done adding user to channels")
			return
		}
		<-throttle
		switch mmchannel.(type) {
		case slack.Channel:
			ID = mmchannel.(slack.Channel).ID
			name = mmchannel.(slack.Channel).Name
			u.syncSlackChannel(ID, name)
		case slack.Group:
			ID = mmchannel.(slack.Group).ID
			name = mmchannel.(slack.Group).Name
			logger.Debugf("GROUP %#v", mmchannel.(slack.Group))
			u.syncSlackGroup(ID, name)

		}
		// exclude direct messages
		//var spoof func(string, string)
		//ch := u.Srv.Channel(mmchannel.ID)
		// post everything to the channel you haven't seen yet
	}
}

func (u *User) handleSlack() {
	for {
		/*
			if u.mc.WsQuit {
				logger.Debug("exiting handleWsMessage")
				return
			}
		*/
		logger.Debug("in handleSlack")
		for msg := range u.rtm.IncomingEvents {
			switch ev := msg.Data.(type) {
			case *slack.MessageEvent:
				u.handleSlackActionPost(ev)
			case *slack.DisconnectedEvent:
				logger.Debugf("disconnecting..")
				return
			}
		}
	}
	/*
			logger.Debugf("MMUser WsReceiver: %#v", message.Raw)
			// check if we have the users/channels in our cache. If not update
			u.checkWsActionMessage(message.Raw, updateChannelsThrottle)
			switch message.Raw.Event {
			case model.WEBSOCKET_EVENT_POSTED:
				u.handleWsActionPost(message.Raw)
			case model.WEBSOCKET_EVENT_POST_EDITED:
				u.handleWsActionPost(message.Raw)
			case model.WEBSOCKET_EVENT_USER_REMOVED:
				u.handleWsActionUserRemoved(message.Raw)
			case model.WEBSOCKET_EVENT_USER_ADDED:
				u.handleWsActionUserAdded(message.Raw)
			}
		}
	*/
}

func (u *User) handleSlackActionPost(rmsg *slack.MessageEvent) {
	var ch Channel
	logger.Debugf("handleSlackActionPost() receiving msg %#v", rmsg)
	if len(rmsg.Attachments) > 0 {
		// skip messages we made ourselves
		if rmsg.Attachments[0].CallbackID == "matterircd" {
			return
		}
	}

	user, err := u.rtm.GetUserInfo(rmsg.User)
	if err != nil {
		return
	}

	// create new "ghost" user
	ghost := u.createSlackUser(user)

	spoofUsername := user.ID
	if ghost != nil {
		spoofUsername = ghost.Nick
	}

	msgs := strings.Split(rmsg.Text, "\n")
	// direct message

	if ghost != nil {
		ch = u.Srv.Channel(rmsg.Channel)
		// join if not in channel
		if !ch.HasUser(ghost) {
			ch.Join(ghost)
		}
	}

	for _, m := range msgs {
		// cleanup the message
		m = u.replaceMention(m)
		m = u.replaceVariable(m)
		m = u.replaceChannel(m)
		m = u.replaceURL(m)
		m = html.UnescapeString(m)

		if m == "" {
			continue
		}
		if strings.HasPrefix(rmsg.Channel, "D") {
			u.MsgSpoofUser(spoofUsername, m)
		} else {
			ch.SpoofMessage(spoofUsername, m)
		}
	}
}

// sync IRC with mattermost channel state
func (u *User) syncSlackChannel(id string, name string) {
	srv := u.Srv
	info, err := u.sc.GetChannelInfo(id)
	if err != nil {
		logger.Info(err)
	}

	for _, user := range info.Members {
		if u.sinfo.User.ID != user {
			//slackuser, _ := u.sc.GetUserInfo(user)
			if slackuser, ok := u.susers[user]; ok {
				u.addSlackUserToChannel(&slackuser, "#"+name, id)
			}
		}
	}
	// before joining ourself
	for _, user := range info.Members {
		// join all the channels we're on on MM
		if user == u.sinfo.User.ID {
			ch := srv.Channel(id)
			// only join when we're not yet on the channel
			if !ch.HasUser(u) {
				logger.Debugf("syncSlackchannel adding myself to %s (id: %s)", name, id)
				ch.Join(u)
				//ch.Topic(u, u.mc.GetChannelHeader(id))
			}
			break
		}
	}
}

// sync IRC with mattermost channel state
func (u *User) syncSlackGroup(id string, name string) {
	srv := u.Srv
	info, err := u.sc.GetGroupInfo(id)
	if err != nil {
		logger.Info(err)
	}

	for _, user := range info.Members {
		if u.sinfo.User.ID != user {
			//slackuser, _ := u.sc.GetUserInfo(user)
			if slackuser, ok := u.susers[user]; ok {
				u.addSlackUserToChannel(&slackuser, "#"+name, id)
			}
		}
	}
	// before joining ourself
	for _, user := range info.Members {
		// join all the channels we're on on MM
		if user == u.sinfo.User.ID {
			ch := srv.Channel(id)
			// only join when we're not yet on the channel
			if !ch.HasUser(u) {
				logger.Debugf("syncSlackgroup adding myself to %s (id: %s)", name, id)
				ch.Join(u)
				//ch.Topic(u, u.mc.GetChannelHeader(id))
			}
			break
		}
	}
}

// @see https://api.slack.com/docs/message-formatting#linking_to_channels_and_users
func (u *User) replaceMention(text string) string {
	results := regexp.MustCompile(`<@([a-zA-z0-9]+)>`).FindAllStringSubmatch(text, -1)
	for _, r := range results {
		text = strings.Replace(text, "<@"+r[1]+">", "@"+u.userName(r[1]), -1)
	}
	return text
}

// @see https://api.slack.com/docs/message-formatting#linking_to_channels_and_users
func (u *User) replaceChannel(text string) string {
	results := regexp.MustCompile(`<#[a-zA-Z0-9]+\|(.+?)>`).FindAllStringSubmatch(text, -1)
	for _, r := range results {
		text = strings.Replace(text, r[0], "#"+r[1], -1)
	}
	return text
}

// @see https://api.slack.com/docs/message-formatting#variables
func (u *User) replaceVariable(text string) string {
	results := regexp.MustCompile(`<!((?:subteam\^)?[a-zA-Z0-9]+)(?:\|@?(.+?))?>`).FindAllStringSubmatch(text, -1)
	for _, r := range results {
		if r[2] != "" {
			text = strings.Replace(text, r[0], "@"+r[2], -1)
		} else {
			text = strings.Replace(text, r[0], "@"+r[1], -1)
		}
	}
	return text
}

// @see https://api.slack.com/docs/message-formatting#linking_to_urls
func (u *User) replaceURL(text string) string {
	results := regexp.MustCompile(`<(.*?)(\|.*?)?>`).FindAllStringSubmatch(text, -1)
	for _, r := range results {
		text = strings.Replace(text, r[0], r[1], -1)
	}
	return text
}

func (u *User) userName(id string) string {
	for _, us := range u.susers {
		if us.ID == id {
			if us.Profile.DisplayName != "" {
				return us.Profile.DisplayName
			}
			return us.Name
		}
	}
	return ""
}
<commit_msg>Fix possible race (slack). Closes 42wim/matterircd#154<commit_after>package irckit

import (
	"html"
	"regexp"
	"strings"
	"time"

	"github.com/nlopes/slack"
)

type SlackInfo struct {
	Token  string
	sc     *slack.Client
	rtm    *slack.RTM
	sinfo  *slack.Info
	susers map[string]slack.User
}

func (u *User) loginToSlack() (*slack.Client, error) {
	u.sc = slack.New(u.Token)
	u.rtm = u.sc.NewRTM()
	u.susers = make(map[string]slack.User)
	go u.rtm.ManageConnection()
	//time.Sleep(time.Second * 2)
	u.sinfo = u.rtm.GetInfo()
	for u.sinfo == nil {
		time.Sleep(time.Millisecond * 500)
		logger.Debug("still waiting for sinfo")
		u.sinfo = u.rtm.GetInfo()
	}
	go u.handleSlack()
	u.addSlackUsersToChannels()
	return u.sc, nil
}

func (u *User) logoutFromSlack() error {
	logger.Debug("calling logout from slack")
	err := u.rtm.Disconnect()
	if err != nil {
		logger.Debug("logoutfrom slack", err)
		return err
	}
	u.Srv.Logout(u)
	u.sc = nil
	logger.Info("logout succeeded")
	return nil
}

func (u *User) createSlackUser(slackuser *slack.User) *User {
	if slackuser == nil {
		return nil
	}
	if ghost, ok := u.Srv.HasUser(slackuser.Name); ok {
		return ghost
	}
	ghost := &User{Nick: slackuser.Name, User: slackuser.ID, Real: slackuser.RealName, Host: "host", Roles: "", channels: map[Channel]struct{}{}}
	ghost.MmGhostUser = true
	u.Srv.Add(ghost)
	return ghost
}

func (u *User) addSlackUserToChannel(user *slack.User, channel string, channelId string) {
	if user == nil {
		return
	}
	ghost := u.createSlackUser(user)
	if ghost == nil {
		logger.Warnf("Cannot join %v into %s", user, channel)
		return
	}
	logger.Debugf("adding %s to %s (%s)", ghost.Nick, channel, channelId)
	ch := u.Srv.Channel(channelId)
	logger.Debugf("channel: %#v %#v", ch.String(), ch.ID())
	ch.Join(ghost)
}

func (u *User) addSlackUsersToChannels() {
	srv := u.Srv
	throttle := time.Tick(time.Millisecond * 100)
	logger.Debug("in addUsersToChannels()")
	// add all users, also who are not on channels
	ch := srv.Channel("&users")
	users, _ := u.sc.GetUsers()
	for _, mmuser := range users {
		// do not add our own nick
		if mmuser.ID == u.sinfo.User.ID {
			continue
		}
		u.createSlackUser(&mmuser)
		u.addSlackUserToChannel(&mmuser, "&users", "&users")
		u.susers[mmuser.ID] = mmuser
	}
	ch.Join(u)

	channels := make(chan interface{}, 10)
	for i := 0; i < 10; i++ {
		go u.addSlackUserToChannelWorker(channels, throttle)
	}
	groups, _ := u.sc.GetGroups(true)
	mmchannels, _ := u.sc.GetChannels(true)
	for _, mmchannel := range mmchannels {
		if mmchannel.IsMember {
			logger.Debug("Adding channel", mmchannel)
			channels <- mmchannel
		}
	}
	for _, mmchannel := range groups {
		logger.Debug("Adding private channel", mmchannel)
		channels <- mmchannel
	}
	close(channels)
}

func (u *User) addSlackUserToChannelWorker(channels <-chan interface{}, throttle <-chan time.Time) {
	var ID, name string
	for {
		mmchannel, ok := <-channels
		if !ok {
			logger.Debug("Done adding user to channels")
			return
		}
		<-throttle
		switch mmchannel.(type) {
		case slack.Channel:
			ID = mmchannel.(slack.Channel).ID
			name = mmchannel.(slack.Channel).Name
			u.syncSlackChannel(ID, name)
		case slack.Group:
			ID = mmchannel.(slack.Group).ID
			name = mmchannel.(slack.Group).Name
			logger.Debugf("GROUP %#v", mmchannel.(slack.Group))
			u.syncSlackGroup(ID, name)

		}
		// exclude direct messages
		//var spoof func(string, string)
		//ch := u.Srv.Channel(mmchannel.ID)
		// post everything to the channel you haven't seen yet
	}
}

func (u *User) handleSlack() {
	for {
		/*
			if u.mc.WsQuit {
				logger.Debug("exiting handleWsMessage")
				return
			}
		*/
		logger.Debug("in handleSlack")
		for msg := range u.rtm.IncomingEvents {
			switch ev := msg.Data.(type) {
			case *slack.MessageEvent:
				u.handleSlackActionPost(ev)
			case *slack.DisconnectedEvent:
				logger.Debugf("disconnecting..")
				return
			}
		}
	}
	/*
			logger.Debugf("MMUser WsReceiver: %#v", message.Raw)
			// check if we have the users/channels in our cache. If not update
			u.checkWsActionMessage(message.Raw, updateChannelsThrottle)
			switch message.Raw.Event {
			case model.WEBSOCKET_EVENT_POSTED:
				u.handleWsActionPost(message.Raw)
			case model.WEBSOCKET_EVENT_POST_EDITED:
				u.handleWsActionPost(message.Raw)
			case model.WEBSOCKET_EVENT_USER_REMOVED:
				u.handleWsActionUserRemoved(message.Raw)
			case model.WEBSOCKET_EVENT_USER_ADDED:
				u.handleWsActionUserAdded(message.Raw)
			}
		}
	*/
}

func (u *User) handleSlackActionPost(rmsg *slack.MessageEvent) {
	var ch Channel
	logger.Debugf("handleSlackActionPost() receiving msg %#v", rmsg)
	if len(rmsg.Attachments) > 0 {
		// skip messages we made ourselves
		if rmsg.Attachments[0].CallbackID == "matterircd" {
			return
		}
	}

	user, err := u.rtm.GetUserInfo(rmsg.User)
	if err != nil {
		return
	}

	// create new "ghost" user
	ghost := u.createSlackUser(user)

	spoofUsername := user.ID
	if ghost != nil {
		spoofUsername = ghost.Nick
	}

	msgs := strings.Split(rmsg.Text, "\n")
	// direct message

	if ghost != nil {
		ch = u.Srv.Channel(rmsg.Channel)
		// join if not in channel
		if !ch.HasUser(ghost) {
			ch.Join(ghost)
		}
	}

	for _, m := range msgs {
		// cleanup the message
		m = u.replaceMention(m)
		m = u.replaceVariable(m)
		m = u.replaceChannel(m)
		m = u.replaceURL(m)
		m = html.UnescapeString(m)

		if m == "" {
			continue
		}
		if strings.HasPrefix(rmsg.Channel, "D") {
			u.MsgSpoofUser(spoofUsername, m)
		} else {
			ch.SpoofMessage(spoofUsername, m)
		}
	}
}

// sync IRC with mattermost channel state
func (u *User) syncSlackChannel(id string, name string) {
	srv := u.Srv
	info, err := u.sc.GetChannelInfo(id)
	if err != nil {
		logger.Info(err)
	}

	for _, user := range info.Members {
		if u.sinfo.User.ID != user {
			//slackuser, _ := u.sc.GetUserInfo(user)
			if slackuser, ok := u.susers[user]; ok {
				u.addSlackUserToChannel(&slackuser, "#"+name, id)
			}
		}
	}
	// before joining ourself
	for _, user := range info.Members {
		// join all the channels we're on on MM
		if user == u.sinfo.User.ID {
			ch := srv.Channel(id)
			// only join when we're not yet on the channel
			if !ch.HasUser(u) {
				logger.Debugf("syncSlackchannel adding myself to %s (id: %s)", name, id)
				ch.Join(u)
				//ch.Topic(u, u.mc.GetChannelHeader(id))
			}
			break
		}
	}
}

// sync IRC with mattermost channel state
func (u *User) syncSlackGroup(id string, name string) {
	srv := u.Srv
	info, err := u.sc.GetGroupInfo(id)
	if err != nil {
		logger.Info(err)
	}

	for _, user := range info.Members {
		if u.sinfo.User.ID != user {
			//slackuser, _ := u.sc.GetUserInfo(user)
			if slackuser, ok := u.susers[user]; ok {
				u.addSlackUserToChannel(&slackuser, "#"+name, id)
			}
		}
	}
	// before joining ourself
	for _, user := range info.Members {
		// join all the channels we're on on MM
		if user == u.sinfo.User.ID {
			ch := srv.Channel(id)
			// only join when we're not yet on the channel
			if !ch.HasUser(u) {
				logger.Debugf("syncSlackgroup adding myself to %s (id: %s)", name, id)
				ch.Join(u)
				//ch.Topic(u, u.mc.GetChannelHeader(id))
			}
			break
		}
	}
}

// @see https://api.slack.com/docs/message-formatting#linking_to_channels_and_users
func (u *User) replaceMention(text string) string {
	results := regexp.MustCompile(`<@([a-zA-z0-9]+)>`).FindAllStringSubmatch(text, -1)
	for _, r := range results {
		text = strings.Replace(text, "<@"+r[1]+">", "@"+u.userName(r[1]), -1)
	}
	return text
}

// @see https://api.slack.com/docs/message-formatting#linking_to_channels_and_users
func (u *User) replaceChannel(text string) string {
	results := regexp.MustCompile(`<#[a-zA-Z0-9]+\|(.+?)>`).FindAllStringSubmatch(text, -1)
	for _, r := range results {
		text = strings.Replace(text, r[0], "#"+r[1], -1)
	}
	return text
}

// @see https://api.slack.com/docs/message-formatting#variables
func (u *User) replaceVariable(text string) string {
	results := regexp.MustCompile(`<!((?:subteam\^)?[a-zA-Z0-9]+)(?:\|@?(.+?))?>`).FindAllStringSubmatch(text, -1)
	for _, r := range results {
		if r[2] != "" {
			text = strings.Replace(text, r[0], "@"+r[2], -1)
		} else {
			text = strings.Replace(text, r[0], "@"+r[1], -1)
		}
	}
	return text
}

// @see https://api.slack.com/docs/message-formatting#linking_to_urls
func (u *User) replaceURL(text string) string {
	results := regexp.MustCompile(`<(.*?)(\|.*?)?>`).FindAllStringSubmatch(text, -1)
	for _, r := range results {
		text = strings.Replace(text, r[0], r[1], -1)
	}
	return text
}

func (u *User) userName(id string) string {
	for _, us := range u.susers {
		if us.ID == id {
			if us.Profile.DisplayName != "" {
				return us.Profile.DisplayName
			}
			return us.Name
		}
	}
	return ""
}
<|endoftext|>


<commit_before>package service

import (
	"encoding/json"

	errgo "gopkg.in/errgo.v1"

	etcd "github.com/coreos/etcd/client"
)

func buildHostsFromNodes(nodes etcd.Nodes) (Hosts, error) {
	hosts := make(Hosts, len(nodes))
	for i, node := range nodes {
		host, err := buildHostFromNode(node)
		if err != nil {
			return nil, errgo.Mask(err)
		}
		hosts[i] = host
	}
	return hosts, nil
}

func buildHostFromNode(node *etcd.Node) (*Host, error) {
	host := &Host{}
	err := json.Unmarshal([]byte(node.Value), &host)
	if err != nil {
		return nil, errgo.Notef(err, "Unable ru unmarshal host")
	}
	return host, nil
}

func buildServiceFromNode(node *etcd.Node) (*Service, error) {
	service := &Service{}
	err := json.Unmarshal([]byte(node.Value), service)
	if err != nil {
		return nil, errgo.Notef(err, "Unable to unmarshal service")
	}
	return service, nil
}
<commit_msg>Fix typo<commit_after>package service

import (
	"encoding/json"

	errgo "gopkg.in/errgo.v1"

	etcd "github.com/coreos/etcd/client"
)

func buildHostsFromNodes(nodes etcd.Nodes) (Hosts, error) {
	hosts := make(Hosts, len(nodes))
	for i, node := range nodes {
		host, err := buildHostFromNode(node)
		if err != nil {
			return nil, errgo.Mask(err)
		}
		hosts[i] = host
	}
	return hosts, nil
}

func buildHostFromNode(node *etcd.Node) (*Host, error) {
	host := &Host{}
	err := json.Unmarshal([]byte(node.Value), &host)
	if err != nil {
		return nil, errgo.Notef(err, "Unable to unmarshal host")
	}
	return host, nil
}

func buildServiceFromNode(node *etcd.Node) (*Service, error) {
	service := &Service{}
	err := json.Unmarshal([]byte(node.Value), service)
	if err != nil {
		return nil, errgo.Notef(err, "Unable to unmarshal service")
	}
	return service, nil
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/rand"
	"crypto/rsa"
)

func foo1() {
	rsa.GenerateKey(rand.Reader, 1024) // BAD
}

func foo2() {
	size := 1024
	rsa.GenerateKey(rand.Reader, size) // BAD
}

func foo3() {
	foo5(1024) // BAD
}

func foo4() {
	foo5(2048) // GOOD
}

func foo5(size int) {
	rsa.GenerateKey(rand.Reader, size)
}

func foo6() {
	keyBits := 1024
	if keyBits >= 2047 {
		rsa.GenerateKey(rand.Reader, keyBits) // BAD
	}
}

func foo7() {
	keyBits := 1024
	if keyBits >= 2048 {
		rsa.GenerateKey(rand.Reader, keyBits) // GOOD
	}
}

func foo8() {
	keyBits := 1024
	switch {
	case keyBits >= 2047:
		rsa.GenerateKey(rand.Reader, keyBits) // BAD
	}
}

func foo9() {
	keyBits := 1024
	switch {
	case keyBits >= 2048:
		rsa.GenerateKey(rand.Reader, keyBits) // GOOD
	}
}

func foo10(customOptionSupplied bool, nonConstantKeyBits int) {
	keyBits := 0
	constantKeyBits := 1024
	if customOptionSupplied {
		keyBits = constantKeyBits
	} else {
		keyBits = nonConstantKeyBits
	}
	rsa.GenerateKey(rand.Reader, keyBits) // BAD
}
<commit_msg>Add a good variant of test case foo10<commit_after>package main

import (
	"crypto/rand"
	"crypto/rsa"
)

func foo1() {
	rsa.GenerateKey(rand.Reader, 1024) // BAD
}

func foo2() {
	size := 1024
	rsa.GenerateKey(rand.Reader, size) // BAD
}

func foo3() {
	foo5(1024) // BAD
}

func foo4() {
	foo5(2048) // GOOD
}

func foo5(size int) {
	rsa.GenerateKey(rand.Reader, size)
}

func foo6() {
	keyBits := 1024
	if keyBits >= 2047 {
		rsa.GenerateKey(rand.Reader, keyBits) // BAD
	}
}

func foo7() {
	keyBits := 1024
	if keyBits >= 2048 {
		rsa.GenerateKey(rand.Reader, keyBits) // GOOD
	}
}

func foo8() {
	keyBits := 1024
	switch {
	case keyBits >= 2047:
		rsa.GenerateKey(rand.Reader, keyBits) // BAD
	}
}

func foo9() {
	keyBits := 1024
	switch {
	case keyBits >= 2048:
		rsa.GenerateKey(rand.Reader, keyBits) // GOOD
	}
}

func foo10(customOptionSupplied bool, nonConstantKeyBits int) {
	keyBits := 0
	constantKeyBits := 1024
	if customOptionSupplied {
		keyBits = constantKeyBits
	} else {
		keyBits = nonConstantKeyBits
	}
	rsa.GenerateKey(rand.Reader, keyBits) // BAD
}

func foo11(customOptionSupplied bool, nonConstantKeyBits int) {
	keyBits := 0
	constantKeyBits := 1024
	if customOptionSupplied {
		keyBits = constantKeyBits
	} else {
		keyBits = nonConstantKeyBits
	}
	if keyBits >= 2048 {
		rsa.GenerateKey(rand.Reader, keyBits) // GOOD
	}
}
<|endoftext|>


<commit_before>package jwd

import (
	"fmt"
	"testing"

	"github.com/9uuso/go-jaro-winkler-distance"
	. "github.com/smartystreets/goconvey/convey"
)

const variance = 0.001

func TestSort(t *testing.T) {

	Convey("sort should return shorter word in character count first", t, func() {

		Convey("it should work with words with onlyy ascii on them", func() {

			shorter, longer := sort("longerword", "short")

			So(longer, ShouldEqual, "longerword")
			So(shorter, ShouldEqual, "short")

		})

		Convey("it should work with words which have unicode in them", func() {

			shorter, longer := sort("ääni", "sound")

			So(longer, ShouldEqual, "sound")
			So(shorter, ShouldEqual, "ääni")

		})
	})
}

func TestWindow(t *testing.T) {

	Convey("testing window with precalculated data", t, func() {

		So(window("dicksonx"), ShouldEqual, float64(3))

	})
}

func TestClosestIndex(t *testing.T) {

	Convey("it should return the closest index", t, func() {

		t1 := closestIndex([]rune("eiiiiieii"), 'e', 4)
		So(t1, ShouldEqual, 6)

		t2 := closestIndex([]rune("dwayne"), 'e', 4)
		So(t2, ShouldEqual, 5)

		t3 := closestIndex([]rune("dwayne"), 'n', 3)
		So(t3, ShouldEqual, 4)

		t4 := closestIndex([]rune("sound"), 'n', 2)
		So(t4, ShouldEqual, 3)
	})

}

func TestScore(t *testing.T) {

	Convey("testing score with precalculated data", t, func() {

		m := float64(4)
		s1 := "dixon"
		s2 := "dicksonx"
		t := float64(0)

		So(score(m, t, float64(len(s1)), float64(len(s2))), ShouldAlmostEqual, 0.767, variance)

	})
}

func TestCalculate(t *testing.T) {

	Convey("Calculate should not return 0", t, func() {

		So(Calculate("dwayne", "duane"), ShouldAlmostEqual, 0.84, variance)
		So(Calculate("", "duane"), ShouldEqual, 0)
		So(Calculate("sound", "ääni"), ShouldAlmostEqual, 0.483, variance)
		So(Calculate("äiti", "ÄÄNI"), ShouldAlmostEqual, 0.75, variance)
		So(Calculate("äiti", "ääNI"), ShouldAlmostEqual, 0.75, variance)
		So(Calculate("dixon", "dicksonx"), ShouldAlmostEqual, 0.814, variance)
		So(Calculate("dicksonx", "dixon"), ShouldAlmostEqual, 0.814, variance)
		So(Calculate("DICKSONX", "DIXON"), ShouldAlmostEqual, 0.814, variance)
		So(Calculate("martha", "marhta"), ShouldAlmostEqual, 0.961, variance)
		So(Calculate("'foo", "fizz"), ShouldAlmostEqual, 0.166, variance)
		So(Calculate("jones", "johnson"), ShouldAlmostEqual, 0.832, variance)
		So(Calculate("asdfg", "qwerty"), ShouldEqual, 0)

	})
}

func ExampleCalculate() {
	distance := jwd.Calculate("jones", "johnson")
	fmt.Println(distance)
	// Output: 0.8323809523809523
}
<commit_msg>fix tests<commit_after>package jwd

import (
	"fmt"
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

const variance = 0.001

func TestSort(t *testing.T) {

	Convey("sort should return shorter word in character count first", t, func() {

		Convey("it should work with words with onlyy ascii on them", func() {

			shorter, longer := sort("longerword", "short")

			So(longer, ShouldEqual, "longerword")
			So(shorter, ShouldEqual, "short")

		})

		Convey("it should work with words which have unicode in them", func() {

			shorter, longer := sort("ääni", "sound")

			So(longer, ShouldEqual, "sound")
			So(shorter, ShouldEqual, "ääni")

		})
	})
}

func TestWindow(t *testing.T) {

	Convey("testing window with precalculated data", t, func() {

		So(window("dicksonx"), ShouldEqual, float64(3))

	})
}

func TestClosestIndex(t *testing.T) {

	Convey("it should return the closest index", t, func() {

		t1 := closestIndex([]rune("eiiiiieii"), 'e', 4)
		So(t1, ShouldEqual, 6)

		t2 := closestIndex([]rune("dwayne"), 'e', 4)
		So(t2, ShouldEqual, 5)

		t3 := closestIndex([]rune("dwayne"), 'n', 3)
		So(t3, ShouldEqual, 4)

		t4 := closestIndex([]rune("sound"), 'n', 2)
		So(t4, ShouldEqual, 3)
	})

}

func TestScore(t *testing.T) {

	Convey("testing score with precalculated data", t, func() {

		m := float64(4)
		s1 := "dixon"
		s2 := "dicksonx"
		t := float64(0)

		So(score(m, t, float64(len(s1)), float64(len(s2))), ShouldAlmostEqual, 0.767, variance)

	})
}

func TestCalculate(t *testing.T) {

	Convey("Calculate should not return 0", t, func() {

		So(Calculate("dwayne", "duane"), ShouldAlmostEqual, 0.84, variance)
		So(Calculate("", "duane"), ShouldEqual, 0)
		So(Calculate("sound", "ääni"), ShouldAlmostEqual, 0.483, variance)
		So(Calculate("äiti", "ÄÄNI"), ShouldAlmostEqual, 0.75, variance)
		So(Calculate("äiti", "ääNI"), ShouldAlmostEqual, 0.75, variance)
		So(Calculate("dixon", "dicksonx"), ShouldAlmostEqual, 0.814, variance)
		So(Calculate("dicksonx", "dixon"), ShouldAlmostEqual, 0.814, variance)
		So(Calculate("DICKSONX", "DIXON"), ShouldAlmostEqual, 0.814, variance)
		So(Calculate("martha", "marhta"), ShouldAlmostEqual, 0.961, variance)
		So(Calculate("'foo", "fizz"), ShouldAlmostEqual, 0.166, variance)
		So(Calculate("jones", "johnson"), ShouldAlmostEqual, 0.832, variance)
		So(Calculate("asdfg", "qwerty"), ShouldEqual, 0)

	})
}

func ExampleCalculate() {
	distance := Calculate("jones", "johnson")
	fmt.Println(distance)
	// Output: 0.8323809523809523
}
<|endoftext|>


<commit_before>package veneur

import (
	"container/ring"
	"context"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"sync"
	"time"

	"github.com/DataDog/datadog-go/statsd"
	"github.com/Sirupsen/logrus"
	lightstep "github.com/lightstep/lightstep-tracer-go"
	opentracing "github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/ext"
	"github.com/stripe/veneur/ssf"
	"github.com/stripe/veneur/trace"
)

const datadogResourceKey = "resource"
const datadogNameKey = "name"

const lightStepOperationKey = "name"

const totalSpansFlushedMetricKey = "worker.spans_flushed_total"

// spanSink is a receiver of spans that handles sending those spans to some
// downstream sink. Calls to `Ingest(span)` are meant to give the sink control
// of the span, with periodic calls to flush as a signal for sinks that don't
// handle their own flushing in a separate goroutine, etc.
type spanSink interface {
	Name() string
	Ingest(ssf.SSFSpan) error
	Flush()
}

// DatadogSpanSink is a sink for sending spans to a Datadog trace agent.
type datadogSpanSink struct {
	HTTPClient   *http.Client
	buffer       *ring.Ring
	bufferSize   int
	mutex        *sync.Mutex
	stats        *statsd.Client
	commonTags   map[string]string
	traceAddress string
}

// NewDatadogSpanSink creates a new Datadog sink for trace spans.
func NewDatadogSpanSink(config *Config, stats *statsd.Client, httpClient *http.Client, commonTags map[string]string) (*datadogSpanSink, error) {
	return &datadogSpanSink{
		HTTPClient:   httpClient,
		bufferSize:   config.SsfBufferSize,
		buffer:       ring.New(config.SsfBufferSize),
		mutex:        &sync.Mutex{},
		stats:        stats,
		commonTags:   commonTags,
		traceAddress: config.DatadogTraceAPIAddress,
	}, nil
}

// Name returns the name of this sink.
func (dd *datadogSpanSink) Name() string {
	return "datadog"
}

// Ingest takes the span and adds it to the ringbuffer.
func (dd *datadogSpanSink) Ingest(span ssf.SSFSpan) error {
	dd.mutex.Lock()
	defer dd.mutex.Unlock()

	dd.buffer.Value = span
	dd.buffer = dd.buffer.Next()
	return nil
}

// Flush signals the sink to send it's spans to their destination. For this
// sync it means we'll be making an HTTP request to send them along. We assume
// it's beneficial to performance to defer these until the normal 10s flush.
func (dd *datadogSpanSink) Flush() {
	dd.mutex.Lock()

	ssfSpans := make([]ssf.SSFSpan, 0, dd.buffer.Len())

	dd.buffer.Do(func(t interface{}) {
		const tooEarly = 1497
		const tooLate = 1497629343000000

		if t != nil {
			ssfSpan, ok := t.(ssf.SSFSpan)
			if !ok {
				log.Error("Got an unknown object in tracing ring!")
				// We'll just skip this one so we don't poison pill or anything.
				return
			}

			var timeErr string
			if ssfSpan.StartTimestamp < tooEarly {
				timeErr = "type:tooEarly"
			}
			if ssfSpan.StartTimestamp > tooLate {
				timeErr = "type:tooLate"
			}
			if timeErr != "" {
				dd.stats.Incr("worker.trace.sink.timestamp_error", []string{timeErr}, 1) // TODO tag as dd?
			}

			if ssfSpan.Tags == nil {
				ssfSpan.Tags = make(map[string]string)
			}

			// Add common tags from veneur's config
			// this will overwrite tags already present on the span
			for k, v := range dd.commonTags {
				ssfSpan.Tags[k] = v
			}
			ssfSpans = append(ssfSpans, ssfSpan)
		}
	})

	// Reset the ring.
	dd.buffer = ring.New(dd.bufferSize)

	// We're done manipulating stuff, let Ingest loose again.
	dd.mutex.Unlock()

	serviceCount := make(map[string]int64)
	var finalTraces []*DatadogTraceSpan
	// Conver the SSFSpans into Datadog Spans
	for _, span := range ssfSpans {
		// -1 is a canonical way of passing in invalid info in Go
		// so we should support that too
		parentID := span.ParentId

		// check if this is the root span
		if parentID <= 0 {
			// we need parentId to be zero for json:omitempty to work
			parentID = 0
		}

		resource := span.Tags[datadogResourceKey]
		name := span.Tags[datadogNameKey]

		tags := map[string]string{}
		// Get the span's existing tags
		for k, v := range span.Tags {
			tags[k] = v
		}

		delete(tags, datadogNameKey)
		delete(tags, datadogResourceKey)

		// TODO implement additional metrics
		var metrics map[string]float64

		var errorCode int64
		if span.Error {
			errorCode = 2
		}

		ddspan := &DatadogTraceSpan{
			TraceID:  span.TraceId,
			SpanID:   span.Id,
			ParentID: parentID,
			Service:  span.Service,
			Name:     name,
			Resource: resource,
			Start:    span.StartTimestamp,
			Duration: span.EndTimestamp - span.StartTimestamp,
			// TODO don't hardcode
			Type:    "http",
			Error:   errorCode,
			Metrics: metrics,
			Meta:    tags,
		}
		serviceCount[span.Service]++
		finalTraces = append(finalTraces, ddspan)
	}

	if len(finalTraces) != 0 {
		// this endpoint is not documented to take an array... but it does
		// another curious constraint of this endpoint is that it does not
		// support "Content-Encoding: deflate"

		err := postHelper(context.TODO(), dd.HTTPClient, dd.stats, fmt.Sprintf("%s/spans", dd.traceAddress), finalTraces, "flush_traces", false)

		if err == nil {
			log.WithField("traces", len(finalTraces)).Info("Completed flushing traces to Datadog")
			dd.stats.Count(totalSpansFlushedMetricKey, int64(len(ssfSpans)), []string{"sink:datadog"}, 1)
			// TODO: Per service counters?
		} else {
			log.WithFields(logrus.Fields{
				"traces":        len(finalTraces),
				logrus.ErrorKey: err}).Warn("Error flushing traces to Datadog")
		}
		for service, count := range serviceCount {
			dd.stats.Count(totalSpansFlushedMetricKey, count, []string{"sink:datadog", fmt.Sprintf("service:%s", service)}, 1)
		}
	} else {
		log.Info("No traces to flush to Datadog, skipping.")
	}
}

// lightStepSpanSink is a sink for spans to be sent to the LightStep client.
type lightStepSpanSink struct {
	tracer       opentracing.Tracer
	stats        *statsd.Client
	commonTags   map[string]string
	mutex        *sync.Mutex
	serviceCount map[string]int64
}

// NewLightStepSpanSink creates a new instance of a LightStepSpanSink.
func NewLightStepSpanSink(config *Config, stats *statsd.Client, commonTags map[string]string) (*lightStepSpanSink, error) {

	var host *url.URL
	host, err := url.Parse(config.TraceLightstepCollectorHost)
	if err != nil {
		log.WithError(err).WithField(
			"host", config.TraceLightstepCollectorHost,
		).Error("Error parsing LightStep collector URL")
		return &lightStepSpanSink{}, err
	}

	port, err := strconv.Atoi(host.Port())
	if err != nil {
		port = lightstepDefaultPort
	} else {
		log.WithError(err).WithFields(logrus.Fields{
			"port":         port,
			"default_port": lightstepDefaultPort,
		}).Warn("Error parsing LightStep port, using default")
	}

	reconPeriod, err := time.ParseDuration(config.TraceLightstepReconnectPeriod)
	if err != nil {
		log.WithError(err).WithFields(logrus.Fields{
			"interval":         config.TraceLightstepReconnectPeriod,
			"default_interval": lightstepDefaultInterval,
		}).Warn("Failed to parse reconnect duration, using default.")
		reconPeriod = lightstepDefaultInterval
	}

	log.WithFields(logrus.Fields{
		"Host": host.Hostname(),
		"Port": port,
	}).Info("Dialing lightstep host")

	maxSpans := config.TraceLightstepMaximumSpans
	if maxSpans == 0 {
		maxSpans = config.SsfBufferSize
		log.WithField("max spans", maxSpans).Info("Using default maximum spans — ssf_buffer_size — for LightStep")
	}

	lightstepTracer := lightstep.NewTracer(lightstep.Options{
		AccessToken:     config.TraceLightstepAccessToken,
		ReconnectPeriod: reconPeriod,
		Collector: lightstep.Endpoint{
			Host:      host.Hostname(),
			Port:      port,
			Plaintext: true,
		},
		UseGRPC:          true,
		MaxBufferedSpans: maxSpans,
	})

	return &lightStepSpanSink{
		tracer:       lightstepTracer,
		stats:        stats,
		serviceCount: make(map[string]int64),
		mutex:        &sync.Mutex{},
	}, nil
}

// Name returns this sink's name.
func (ls *lightStepSpanSink) Name() string {
	return "lightstep"
}

// Ingest takes in a span and passed it along to the LS client after
// some sanity checks and improvements are made.
func (ls *lightStepSpanSink) Ingest(ssfSpan ssf.SSFSpan) error {
	ls.mutex.Lock()
	defer ls.mutex.Unlock()

	parentID := ssfSpan.ParentId
	if parentID <= 0 {
		parentID = 0
	}

	var errorCode int64
	if ssfSpan.Error {
		errorCode = 1
	}

	timestamp := time.Unix(ssfSpan.StartTimestamp/1e9, ssfSpan.StartTimestamp%1e9)
	sp := ls.tracer.StartSpan(
		ssfSpan.Tags[lightStepOperationKey],
		opentracing.StartTime(timestamp),
		lightstep.SetTraceID(uint64(ssfSpan.TraceId)),
		lightstep.SetSpanID(uint64(ssfSpan.Id)),
		lightstep.SetParentSpanID(uint64(parentID)))

	sp.SetTag(trace.ResourceKey, ssfSpan.Tags[trace.ResourceKey]) // TODO Why is this here?
	sp.SetTag(lightstep.ComponentNameKey, ssfSpan.Service)
	// TODO don't hardcode
	sp.SetTag("type", "http")
	sp.SetTag("error-code", errorCode)
	for k, v := range ssfSpan.Tags {
		sp.SetTag(k, v)
	}
	// And now set any veneur common tags
	for k, v := range ls.commonTags {
		sp.SetTag(k, v)
	}
	// TODO add metrics as tags to the span as well?

	if errorCode > 0 {
		// Note: this sets the OT-standard "error" tag, which
		// LightStep uses to flag error spans.
		ext.Error.Set(sp, true)
	}

	endTime := time.Unix(ssfSpan.EndTimestamp/1e9, ssfSpan.EndTimestamp%1e9)
	sp.FinishWithOptions(opentracing.FinishOptions{
		FinishTime: endTime,
	})

	service := ssfSpan.Service
	if service == "" {
		service = "unknown"
	}
	ls.serviceCount[service]++
	return nil
}

// Flush doesn't need to do anything to the LS tracer, so we emit metrics
// instead.
func (ls *lightStepSpanSink) Flush() {
	ls.mutex.Lock()
	defer ls.mutex.Unlock()

	totalCount := int64(0)
	for service, count := range ls.serviceCount {
		totalCount += count
		ls.stats.Count(totalSpansFlushedMetricKey, count, []string{"sink:lightstep", fmt.Sprintf("service:%s", service)}, 1)
	}
	ls.serviceCount = make(map[string]int64)
	log.WithField("total_spans", totalCount).Debug("Checkpointing flushed spans for Lightstep")
}
<commit_msg>Tighten up the mutex use<commit_after>package veneur

import (
	"container/ring"
	"context"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"sync"
	"time"

	"github.com/DataDog/datadog-go/statsd"
	"github.com/Sirupsen/logrus"
	lightstep "github.com/lightstep/lightstep-tracer-go"
	opentracing "github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/ext"
	"github.com/stripe/veneur/ssf"
	"github.com/stripe/veneur/trace"
)

const datadogResourceKey = "resource"
const datadogNameKey = "name"

const lightStepOperationKey = "name"

const totalSpansFlushedMetricKey = "worker.spans_flushed_total"

// spanSink is a receiver of spans that handles sending those spans to some
// downstream sink. Calls to `Ingest(span)` are meant to give the sink control
// of the span, with periodic calls to flush as a signal for sinks that don't
// handle their own flushing in a separate goroutine, etc.
type spanSink interface {
	Name() string
	Ingest(ssf.SSFSpan) error
	Flush()
}

// DatadogSpanSink is a sink for sending spans to a Datadog trace agent.
type datadogSpanSink struct {
	HTTPClient   *http.Client
	buffer       *ring.Ring
	bufferSize   int
	mutex        *sync.Mutex
	stats        *statsd.Client
	commonTags   map[string]string
	traceAddress string
}

// NewDatadogSpanSink creates a new Datadog sink for trace spans.
func NewDatadogSpanSink(config *Config, stats *statsd.Client, httpClient *http.Client, commonTags map[string]string) (*datadogSpanSink, error) {
	return &datadogSpanSink{
		HTTPClient:   httpClient,
		bufferSize:   config.SsfBufferSize,
		buffer:       ring.New(config.SsfBufferSize),
		mutex:        &sync.Mutex{},
		stats:        stats,
		commonTags:   commonTags,
		traceAddress: config.DatadogTraceAPIAddress,
	}, nil
}

// Name returns the name of this sink.
func (dd *datadogSpanSink) Name() string {
	return "datadog"
}

// Ingest takes the span and adds it to the ringbuffer.
func (dd *datadogSpanSink) Ingest(span ssf.SSFSpan) error {
	dd.mutex.Lock()
	defer dd.mutex.Unlock()

	dd.buffer.Value = span
	dd.buffer = dd.buffer.Next()
	return nil
}

// Flush signals the sink to send it's spans to their destination. For this
// sync it means we'll be making an HTTP request to send them along. We assume
// it's beneficial to performance to defer these until the normal 10s flush.
func (dd *datadogSpanSink) Flush() {
	dd.mutex.Lock()

	ssfSpans := make([]ssf.SSFSpan, 0, dd.buffer.Len())

	dd.buffer.Do(func(t interface{}) {
		const tooEarly = 1497
		const tooLate = 1497629343000000

		if t != nil {
			ssfSpan, ok := t.(ssf.SSFSpan)
			if !ok {
				log.Error("Got an unknown object in tracing ring!")
				// We'll just skip this one so we don't poison pill or anything.
				return
			}

			var timeErr string
			if ssfSpan.StartTimestamp < tooEarly {
				timeErr = "type:tooEarly"
			}
			if ssfSpan.StartTimestamp > tooLate {
				timeErr = "type:tooLate"
			}
			if timeErr != "" {
				dd.stats.Incr("worker.trace.sink.timestamp_error", []string{timeErr}, 1) // TODO tag as dd?
			}

			if ssfSpan.Tags == nil {
				ssfSpan.Tags = make(map[string]string)
			}

			// Add common tags from veneur's config
			// this will overwrite tags already present on the span
			for k, v := range dd.commonTags {
				ssfSpan.Tags[k] = v
			}
			ssfSpans = append(ssfSpans, ssfSpan)
		}
	})

	// Reset the ring.
	dd.buffer = ring.New(dd.bufferSize)

	// We're done manipulating stuff, let Ingest loose again.
	dd.mutex.Unlock()

	serviceCount := make(map[string]int64)
	var finalTraces []*DatadogTraceSpan
	// Conver the SSFSpans into Datadog Spans
	for _, span := range ssfSpans {
		// -1 is a canonical way of passing in invalid info in Go
		// so we should support that too
		parentID := span.ParentId

		// check if this is the root span
		if parentID <= 0 {
			// we need parentId to be zero for json:omitempty to work
			parentID = 0
		}

		resource := span.Tags[datadogResourceKey]
		name := span.Tags[datadogNameKey]

		tags := map[string]string{}
		// Get the span's existing tags
		for k, v := range span.Tags {
			tags[k] = v
		}

		delete(tags, datadogNameKey)
		delete(tags, datadogResourceKey)

		// TODO implement additional metrics
		var metrics map[string]float64

		var errorCode int64
		if span.Error {
			errorCode = 2
		}

		ddspan := &DatadogTraceSpan{
			TraceID:  span.TraceId,
			SpanID:   span.Id,
			ParentID: parentID,
			Service:  span.Service,
			Name:     name,
			Resource: resource,
			Start:    span.StartTimestamp,
			Duration: span.EndTimestamp - span.StartTimestamp,
			// TODO don't hardcode
			Type:    "http",
			Error:   errorCode,
			Metrics: metrics,
			Meta:    tags,
		}
		serviceCount[span.Service]++
		finalTraces = append(finalTraces, ddspan)
	}

	if len(finalTraces) != 0 {
		// this endpoint is not documented to take an array... but it does
		// another curious constraint of this endpoint is that it does not
		// support "Content-Encoding: deflate"

		err := postHelper(context.TODO(), dd.HTTPClient, dd.stats, fmt.Sprintf("%s/spans", dd.traceAddress), finalTraces, "flush_traces", false)

		if err == nil {
			log.WithField("traces", len(finalTraces)).Info("Completed flushing traces to Datadog")
			dd.stats.Count(totalSpansFlushedMetricKey, int64(len(ssfSpans)), []string{"sink:datadog"}, 1)
			// TODO: Per service counters?
		} else {
			log.WithFields(logrus.Fields{
				"traces":        len(finalTraces),
				logrus.ErrorKey: err}).Warn("Error flushing traces to Datadog")
		}
		for service, count := range serviceCount {
			dd.stats.Count(totalSpansFlushedMetricKey, count, []string{"sink:datadog", fmt.Sprintf("service:%s", service)}, 1)
		}
	} else {
		log.Info("No traces to flush to Datadog, skipping.")
	}
}

// lightStepSpanSink is a sink for spans to be sent to the LightStep client.
type lightStepSpanSink struct {
	tracer       opentracing.Tracer
	stats        *statsd.Client
	commonTags   map[string]string
	mutex        *sync.Mutex
	serviceCount map[string]int64
}

// NewLightStepSpanSink creates a new instance of a LightStepSpanSink.
func NewLightStepSpanSink(config *Config, stats *statsd.Client, commonTags map[string]string) (*lightStepSpanSink, error) {

	var host *url.URL
	host, err := url.Parse(config.TraceLightstepCollectorHost)
	if err != nil {
		log.WithError(err).WithField(
			"host", config.TraceLightstepCollectorHost,
		).Error("Error parsing LightStep collector URL")
		return &lightStepSpanSink{}, err
	}

	port, err := strconv.Atoi(host.Port())
	if err != nil {
		port = lightstepDefaultPort
	} else {
		log.WithError(err).WithFields(logrus.Fields{
			"port":         port,
			"default_port": lightstepDefaultPort,
		}).Warn("Error parsing LightStep port, using default")
	}

	reconPeriod, err := time.ParseDuration(config.TraceLightstepReconnectPeriod)
	if err != nil {
		log.WithError(err).WithFields(logrus.Fields{
			"interval":         config.TraceLightstepReconnectPeriod,
			"default_interval": lightstepDefaultInterval,
		}).Warn("Failed to parse reconnect duration, using default.")
		reconPeriod = lightstepDefaultInterval
	}

	log.WithFields(logrus.Fields{
		"Host": host.Hostname(),
		"Port": port,
	}).Info("Dialing lightstep host")

	maxSpans := config.TraceLightstepMaximumSpans
	if maxSpans == 0 {
		maxSpans = config.SsfBufferSize
		log.WithField("max spans", maxSpans).Info("Using default maximum spans — ssf_buffer_size — for LightStep")
	}

	lightstepTracer := lightstep.NewTracer(lightstep.Options{
		AccessToken:     config.TraceLightstepAccessToken,
		ReconnectPeriod: reconPeriod,
		Collector: lightstep.Endpoint{
			Host:      host.Hostname(),
			Port:      port,
			Plaintext: true,
		},
		UseGRPC:          true,
		MaxBufferedSpans: maxSpans,
	})

	return &lightStepSpanSink{
		tracer:       lightstepTracer,
		stats:        stats,
		serviceCount: make(map[string]int64),
		mutex:        &sync.Mutex{},
	}, nil
}

// Name returns this sink's name.
func (ls *lightStepSpanSink) Name() string {
	return "lightstep"
}

// Ingest takes in a span and passed it along to the LS client after
// some sanity checks and improvements are made.
func (ls *lightStepSpanSink) Ingest(ssfSpan ssf.SSFSpan) error {
	parentID := ssfSpan.ParentId
	if parentID <= 0 {
		parentID = 0
	}

	var errorCode int64
	if ssfSpan.Error {
		errorCode = 1
	}

	timestamp := time.Unix(ssfSpan.StartTimestamp/1e9, ssfSpan.StartTimestamp%1e9)
	sp := ls.tracer.StartSpan(
		ssfSpan.Tags[lightStepOperationKey],
		opentracing.StartTime(timestamp),
		lightstep.SetTraceID(uint64(ssfSpan.TraceId)),
		lightstep.SetSpanID(uint64(ssfSpan.Id)),
		lightstep.SetParentSpanID(uint64(parentID)))

	sp.SetTag(trace.ResourceKey, ssfSpan.Tags[trace.ResourceKey]) // TODO Why is this here?
	sp.SetTag(lightstep.ComponentNameKey, ssfSpan.Service)
	// TODO don't hardcode
	sp.SetTag("type", "http")
	sp.SetTag("error-code", errorCode)
	for k, v := range ssfSpan.Tags {
		sp.SetTag(k, v)
	}
	// And now set any veneur common tags
	for k, v := range ls.commonTags {
		sp.SetTag(k, v)
	}
	// TODO add metrics as tags to the span as well?

	if errorCode > 0 {
		// Note: this sets the OT-standard "error" tag, which
		// LightStep uses to flag error spans.
		ext.Error.Set(sp, true)
	}

	endTime := time.Unix(ssfSpan.EndTimestamp/1e9, ssfSpan.EndTimestamp%1e9)
	sp.FinishWithOptions(opentracing.FinishOptions{
		FinishTime: endTime,
	})

	service := ssfSpan.Service
	if service == "" {
		service = "unknown"
	}

	// Protect mutating the service count with a mutex
	ls.mutex.Lock()
	ls.serviceCount[service]++
	ls.mutex.Unlock()

	return nil
}

// Flush doesn't need to do anything to the LS tracer, so we emit metrics
// instead.
func (ls *lightStepSpanSink) Flush() {
	ls.mutex.Lock()
	defer ls.mutex.Unlock()

	totalCount := int64(0)
	for service, count := range ls.serviceCount {
		totalCount += count
		ls.stats.Count(totalSpansFlushedMetricKey, count, []string{"sink:lightstep", fmt.Sprintf("service:%s", service)}, 1)
	}
	ls.serviceCount = make(map[string]int64)
	log.WithField("total_spans", totalCount).Debug("Checkpointing flushed spans for Lightstep")
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"io"
	"log"
	"os"
	"os/exec"
	"os/signal"
	"strings"
	"syscall"
	"time"
)

// testExit is a channel that is used during testing only to trigger a return from the alternate
// function.
var testExit chan struct{}

// alternate runs a command with alternating parameters inserted in place of the placeholder. Each
// time a SIGUSR1 is received, a new command is run with the next parameter, and a SIGINT is sent to
// the previous command after the overlap duration has elapsed. The alternate logs are written to
// stderr, and the command logs are written to cmdStdout and cmdStderr.
func alternate(command string, placeholder string, params []string, overlap time.Duration,
	stderr, cmdStdout, cmdStderr io.Writer) {

	log.SetPrefix("alternate | ")
	log.SetOutput(stderr)
	log.SetFlags(0)

	log.Printf("Starting with command %q, placeholder %q, params = %q, overlap = %v\n",
		command, placeholder, params, overlap)

	// When a command exits, cmdExit receives the parameter this command was run with.
	cmdExit := make(chan string)

	// When the overlap duration has elapsed, overlapEnd receives an empty struct.
	overlapEnd := make(chan struct{})

	next := make(chan os.Signal, 1)
	// Buffer a fake signal to run the first command.
	next <- syscall.Signal(0)
	// Then listen to USR1 signals dispatched by the user.
	signal.Notify(next, syscall.SIGUSR1)

	m := newManager(params)

	for {
		select {
		case <-testExit:
			log.Println("The test exit channel received a value, exiting alternate")
			return

		case param := <-cmdExit:
			log.Printf("The command with parameter %q exited\n", param)
			m.unsetCmd(param)
			if !m.hasCmds() {
				log.Println("All commands have exited, exiting alternate")
				return
			}

		case <-overlapEnd:
			finishRotation(m)

		case signal := <-next:
			first := signal != syscall.SIGUSR1
			nextParam := m.nextParam()

			if !first {
				log.Printf("Received signal USR1, rotating to next parameter %q", nextParam)
			}

			if m.nextCmd() != nil {
				log.Printf("A command with parameter %q is already running, cannot run again",
					nextParam)
				break
			}

			s := strings.Replace(command, placeholder, nextParam, 1)
			nextCmd := cmd(s, cmdStdout, cmdStderr)
			m.setCmd(nextParam, nextCmd)

			if err := run(nextCmd, nextParam, cmdExit); err != nil {
				log.Printf("Failed to run the command with parameter %q, error: %v\n",
					err.Error())
				m.unsetCmd(nextParam)
				break
			}

			if first {
				// There is no previous command to interrupt.
				m.rotate()
				break
			}

			if overlap == 0 {
				finishRotation(m)
			} else {
				go func() {
					time.Sleep(overlap)
					overlapEnd <- struct{}{}
				}()
			}
		}
	}
}

func finishRotation(m *manager) {
	if m.nextCmd() == nil {
		// The next command is not running, cancel the rotation without interrupting the current
		// command.
		return
	}
	if c := m.currentCmd(); c != nil {
		if err := interruptCmd(c); err != nil {
			log.Printf("Failed to interrupt the command with parameter %q, error: %v\n",
				m.currentParam(), err)
		}
	}
	m.rotate()
}

func interruptCmd(c *exec.Cmd) error {
	if c == nil {
		return errors.New("interruptCmd error: cmd is nil")
	}
	p := c.Process
	if p == nil {
		return errors.New("interruptCmd error: cmd.Process is nil")
	}
	p.Signal(os.Interrupt)
	return nil
}

func cmd(s string, stdout, stderr io.Writer) *exec.Cmd {
	f := strings.Fields(s)
	c := exec.Command(f[0], f[1:]...)
	c.Stdout = stdout
	c.Stderr = stderr
	return c
}

func run(c *exec.Cmd, param string, exit chan string) error {
	log.Printf("Running command with parameter %q\n", param)
	if err := c.Start(); err != nil {
		return err
	}
	go func() {
		c.Wait()
		exit <- param
	}()
	return nil
}
<commit_msg>Kill all running commands before exiting text<commit_after>package main

import (
	"errors"
	"io"
	"log"
	"os"
	"os/exec"
	"os/signal"
	"strings"
	"syscall"
	"time"
)

// testExit is a channel that is used during testing only to trigger a return from the alternate
// function.
var testExit chan struct{}

// alternate runs a command with alternating parameters inserted in place of the placeholder. Each
// time a SIGUSR1 is received, a new command is run with the next parameter, and a SIGINT is sent to
// the previous command after the overlap duration has elapsed. The alternate logs are written to
// stderr, and the command logs are written to cmdStdout and cmdStderr.
func alternate(command string, placeholder string, params []string, overlap time.Duration,
	stderr, cmdStdout, cmdStderr io.Writer) {

	log.SetPrefix("alternate | ")
	log.SetOutput(stderr)
	log.SetFlags(0)

	log.Printf("Starting with command %q, placeholder %q, params = %q, overlap = %v\n",
		command, placeholder, params, overlap)

	// When a command exits, cmdExit receives the parameter this command was run with.
	cmdExit := make(chan string)

	// When the overlap duration has elapsed, overlapEnd receives an empty struct.
	overlapEnd := make(chan struct{})

	next := make(chan os.Signal, 1)
	// Buffer a fake signal to run the first command.
	next <- syscall.Signal(0)
	// Then listen to USR1 signals dispatched by the user.
	signal.Notify(next, syscall.SIGUSR1)

	m := newManager(params)

	for {
		select {
		case <-testExit:
			log.Println("The test exit channel received a value, exiting alternate")
			killAllCmds(params, m)
			return

		case param := <-cmdExit:
			log.Printf("The command with parameter %q exited\n", param)
			m.unsetCmd(param)
			if !m.hasCmds() {
				log.Println("All commands have exited, exiting alternate")
				return
			}

		case <-overlapEnd:
			finishRotation(m)

		case signal := <-next:
			first := signal != syscall.SIGUSR1
			nextParam := m.nextParam()

			if !first {
				log.Printf("Received signal USR1, rotating to next parameter %q", nextParam)
			}

			if m.nextCmd() != nil {
				log.Printf("A command with parameter %q is already running, cannot run again",
					nextParam)
				break
			}

			s := strings.Replace(command, placeholder, nextParam, 1)
			nextCmd := cmd(s, cmdStdout, cmdStderr)
			m.setCmd(nextParam, nextCmd)

			if err := run(nextCmd, nextParam, cmdExit); err != nil {
				log.Printf("Failed to run the command with parameter %q, error: %v\n",
					err.Error())
				m.unsetCmd(nextParam)
				break
			}

			if first {
				// There is no previous command to interrupt.
				m.rotate()
				break
			}

			if overlap == 0 {
				finishRotation(m)
			} else {
				go func() {
					time.Sleep(overlap)
					overlapEnd <- struct{}{}
				}()
			}
		}
	}
}

func finishRotation(m *manager) {
	if m.nextCmd() == nil {
		// The next command is not running, cancel the rotation without interrupting the current
		// command.
		return
	}
	if c := m.currentCmd(); c != nil {
		if err := interruptCmd(c); err != nil {
			log.Printf("Failed to interrupt the command with parameter %q, error: %v\n",
				m.currentParam(), err)
		}
	}
	m.rotate()
}

func interruptCmd(c *exec.Cmd) error {
	if c == nil {
		return errors.New("interruptCmd error: cmd is nil")
	}
	p := c.Process
	if p == nil {
		return errors.New("interruptCmd error: cmd.Process is nil")
	}
	p.Signal(os.Interrupt)
	return nil
}

func cmd(s string, stdout, stderr io.Writer) *exec.Cmd {
	f := strings.Fields(s)
	c := exec.Command(f[0], f[1:]...)
	c.Stdout = stdout
	c.Stderr = stderr
	return c
}

func run(c *exec.Cmd, param string, exit chan string) error {
	log.Printf("Running command with parameter %q\n", param)
	if err := c.Start(); err != nil {
		return err
	}
	go func() {
		c.Wait()
		exit <- param
	}()
	return nil
}

func killAllCmds(params []string, m *manager) {
	for _, param := range params {
		if c := m.cmd(param); c != nil {
			if p := c.Process; p != nil {
				log.Printf("Killing command with parameter %q\n", param)
				p.Signal(os.Kill)
			}
		}
	}
}
<|endoftext|>


<commit_before>package analytics

import "github.com/jehiah/go-strftime"
import "github.com/xtgo/uuid"
import "encoding/json"
import "net/http"
import "errors"
import "bytes"
import "time"
import "log"

// Version of the client.
var version = "2.0.0"

// Endpoint for the Segment API.
var Endpoint = "https://api.segment.io"

// DefaultContext of message batches.
var DefaultContext = map[string]interface{}{
	"library": map[string]interface{}{
		"name":    "analytics-go",
		"version": version,
	},
}

// Message interface.
type message interface {
	setMessageId(string)
	setTimestamp(string)
}

// Response from API.
type response struct {
	Message string `json:"message"`
	Code    string `json:"code"`
}

// Message fields common to all.
type Message struct {
	Type      string `json:"type,omitempty"`
	MessageId string `json:"messageId,omitempty"`
	Timestamp string `json:"timestamp,omitempty"`
	SentAt    string `json:"sentAt,omitempty"`
}

// Batch message.
type Batch struct {
	Context  map[string]interface{} `json:"context,omitempty"`
	Messages []interface{}          `json:"batch"`
	Message
}

// Identify message.
type Identify struct {
	Context     map[string]interface{} `json:"context,omitempty"`
	Traits      map[string]interface{} `json:"traits,omitempty"`
	AnonymousId string                 `json:"anonymousId,omitempty"`
	UserId      string                 `json:"userId,omitempty"`
	Message
}

// Group message.
type Group struct {
	Context     map[string]interface{} `json:"context,omitempty"`
	Traits      map[string]interface{} `json:"traits,omitempty"`
	AnonymousId string                 `json:"anonymousId,omitempty"`
	UserId      string                 `json:"userId,omitempty"`
	GroupId     string                 `json:"groupId"`
	Message
}

// Track message.
type Track struct {
	Context     map[string]interface{} `json:"context,omitempty"`
	Properties  map[string]interface{} `json:"properties,omitempty"`
	AnonymousId string                 `json:"anonymousId,omitempty"`
	UserId      string                 `json:"userId,omitempty"`
	Event       string                 `json:"event"`
	Message
}

// Page message.
type Page struct {
	Context     map[string]interface{} `json:"context,omitempty"`
	Traits      map[string]interface{} `json:"properties,omitempty"`
	AnonymousId string                 `json:"anonymousId,omitempty"`
	UserId      string                 `json:"userId,omitempty"`
	Category    string                 `json:"category,omitempty"`
	Name        string                 `json:"name,omitempty"`
	Message
}

// Alias message.
type Alias struct {
	PreviousId string `json:"previousId"`
	UserId     string `json:"userId"`
	Message
}

// Client which batches messages and flushes at the given Interval or
// when the Size limit is exceeded. Set Verbose to true to enable
// logging output.
type Client struct {
	Endpoint string
	Interval time.Duration
	Verbose  bool
	Size     int
	key      string
	msgs     chan interface{}
	quit     chan bool

	uid func() string
	now func() time.Time
}

// New client with write key.
func New(key string) *Client {
	c := &Client{
		msgs:     make(chan interface{}, 100),
		quit:     make(chan bool),
		Interval: 5 * time.Second,
		Endpoint: Endpoint,
		Size:     250,
		key:      key,
		now:      time.Now,
		uid:      uid,
	}

	go c.loop()

	return c
}

// Alias buffers an "alias" message.
func (c *Client) Alias(msg *Alias) error {
	if msg.UserId == "" {
		return errors.New("You must pass a 'userId'.")
	}

	if msg.PreviousId == "" {
		return errors.New("You must pass a 'previousId'.")
	}

	msg.Type = "alias"
	c.queue(msg)

	return nil
}

// Page buffers an "page" message.
func (c *Client) Page(msg *Page) error {
	if msg.UserId == "" && msg.AnonymousId == "" {
		return errors.New("You must pass either an 'anonymousId' or 'userId'.")
	}

	msg.Type = "page"
	c.queue(msg)

	return nil
}

// Group buffers an "group" message.
func (c *Client) Group(msg *Group) error {
	if msg.GroupId == "" {
		return errors.New("You must pass a 'groupId'.")
	}

	if msg.UserId == "" && msg.AnonymousId == "" {
		return errors.New("You must pass either an 'anonymousId' or 'userId'.")
	}

	msg.Type = "group"
	c.queue(msg)

	return nil
}

// Identify buffers an "identify" message.
func (c *Client) Identify(msg *Identify) error {
	if msg.UserId == "" && msg.AnonymousId == "" {
		return errors.New("You must pass either an 'anonymousId' or 'userId'.")
	}

	msg.Type = "identify"
	c.queue(msg)

	return nil
}

// Track buffers an "track" message.
func (c *Client) Track(msg *Track) error {
	if msg.Event == "" {
		return errors.New("You must pass 'event'.")
	}

	if msg.UserId == "" && msg.AnonymousId == "" {
		return errors.New("You must pass either an 'anonymousId' or 'userId'.")
	}

	msg.Type = "track"
	c.queue(msg)

	return nil
}

// Queue message.
func (c *Client) queue(msg message) {
	msg.setMessageId(c.uid())
	msg.setTimestamp(timestamp(c.now()))
	c.msgs <- msg
}

// Close and flush metrics.
func (c *Client) Close() error {
	c.quit <- true
	close(c.msgs)
	<-c.quit
	return nil
}

// Send batch request.
func (c *Client) send(msgs []interface{}) {
	if len(msgs) == 0 {
		return
	}

	batch := new(Batch)
	batch.Messages = msgs
	batch.MessageId = c.uid()
	batch.SentAt = timestamp(c.now())
	batch.Context = DefaultContext

	b, err := json.Marshal(batch)
	if err != nil {
		c.log("error marshalling msgs: %s", err)
		return
	}

	url := c.Endpoint + "/v1/batch"
	req, err := http.NewRequest("POST", url, bytes.NewReader(b))
	if err != nil {
		c.log("error creating request: %s", err)
		return
	}

	req.Header.Add("User-Agent", "analytics-go (version: "+version+")")
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("Content-Length", string(len(b)))
	req.SetBasicAuth(c.key, "")

	res, err := http.DefaultClient.Do(req)
	if err != nil {
		c.log("error sending request: %s", err)
		return
	}
	defer res.Body.Close()

	c.report(res)
}

// Report on response body.
func (c *Client) report(res *http.Response) {
	if res.StatusCode < 400 {
		c.verbose("response %s", res.Status)
		return
	}

	msg := new(response)
	err := json.NewDecoder(res.Body).Decode(msg)
	if err != nil {
		c.log("error reading response: %s", err)
		return
	}

	c.log("response %s: %s – %s", res.Status, msg.Code, msg.Message)
}

// Batch loop.
func (c *Client) loop() {
	var msgs []interface{}
	tick := time.NewTicker(c.Interval)

	for {
		select {
		case msg := <-c.msgs:
			c.verbose("buffer (%d/%d) %v", len(msgs), c.Size, msg)
			msgs = append(msgs, msg)
			if len(msgs) == c.Size {
				c.verbose("exceeded %d messages – flushing", c.Size)
				c.send(msgs)
				msgs = nil
			}
		case <-tick.C:
			if len(msgs) > 0 {
				c.verbose("interval reached - flushing %d", len(msgs))
				c.send(msgs)
				msgs = nil
			} else {
				c.verbose("interval reached – nothing to send")
			}
		case <-c.quit:
			c.verbose("exit requested – flushing %d", len(msgs))
			c.send(msgs)
			c.verbose("exit")
			c.quit <- true
			return
		}
	}
}

// Verbose log.
func (c *Client) verbose(msg string, args ...interface{}) {
	if c.Verbose {
		log.Printf("segment: "+msg, args...)
	}
}

// Unconditional log.
func (c *Client) log(msg string, args ...interface{}) {
	log.Printf("segment: "+msg, args...)
}

// Set message timestamp if one is not already set.
func (m *Message) setTimestamp(s string) {
	if m.Timestamp == "" {
		m.Timestamp = s
	}
}

// Set message id.
func (m *Message) setMessageId(s string) {
	m.MessageId = s
}

// Return formatted timestamp.
func timestamp(t time.Time) string {
	return strftime.Format("%Y-%m-%dT%H:%M:%S%z", t)
}

// Return uuid string.
func uid() string {
	return uuid.NewRandom().String()
}
<commit_msg>Make version a constant and export it<commit_after>package analytics

import "github.com/jehiah/go-strftime"
import "github.com/xtgo/uuid"
import "encoding/json"
import "net/http"
import "errors"
import "bytes"
import "time"
import "log"

// Version of the client.
const Version = "2.0.0"

// Endpoint for the Segment API.
const Endpoint = "https://api.segment.io"

// DefaultContext of message batches.
var DefaultContext = map[string]interface{}{
	"library": map[string]interface{}{
		"name":    "analytics-go",
		"version": Version,
	},
}

// Message interface.
type message interface {
	setMessageId(string)
	setTimestamp(string)
}

// Response from API.
type response struct {
	Message string `json:"message"`
	Code    string `json:"code"`
}

// Message fields common to all.
type Message struct {
	Type      string `json:"type,omitempty"`
	MessageId string `json:"messageId,omitempty"`
	Timestamp string `json:"timestamp,omitempty"`
	SentAt    string `json:"sentAt,omitempty"`
}

// Batch message.
type Batch struct {
	Context  map[string]interface{} `json:"context,omitempty"`
	Messages []interface{}          `json:"batch"`
	Message
}

// Identify message.
type Identify struct {
	Context     map[string]interface{} `json:"context,omitempty"`
	Traits      map[string]interface{} `json:"traits,omitempty"`
	AnonymousId string                 `json:"anonymousId,omitempty"`
	UserId      string                 `json:"userId,omitempty"`
	Message
}

// Group message.
type Group struct {
	Context     map[string]interface{} `json:"context,omitempty"`
	Traits      map[string]interface{} `json:"traits,omitempty"`
	AnonymousId string                 `json:"anonymousId,omitempty"`
	UserId      string                 `json:"userId,omitempty"`
	GroupId     string                 `json:"groupId"`
	Message
}

// Track message.
type Track struct {
	Context     map[string]interface{} `json:"context,omitempty"`
	Properties  map[string]interface{} `json:"properties,omitempty"`
	AnonymousId string                 `json:"anonymousId,omitempty"`
	UserId      string                 `json:"userId,omitempty"`
	Event       string                 `json:"event"`
	Message
}

// Page message.
type Page struct {
	Context     map[string]interface{} `json:"context,omitempty"`
	Traits      map[string]interface{} `json:"properties,omitempty"`
	AnonymousId string                 `json:"anonymousId,omitempty"`
	UserId      string                 `json:"userId,omitempty"`
	Category    string                 `json:"category,omitempty"`
	Name        string                 `json:"name,omitempty"`
	Message
}

// Alias message.
type Alias struct {
	PreviousId string `json:"previousId"`
	UserId     string `json:"userId"`
	Message
}

// Client which batches messages and flushes at the given Interval or
// when the Size limit is exceeded. Set Verbose to true to enable
// logging output.
type Client struct {
	Endpoint string
	Interval time.Duration
	Verbose  bool
	Size     int
	key      string
	msgs     chan interface{}
	quit     chan bool

	uid func() string
	now func() time.Time
}

// New client with write key.
func New(key string) *Client {
	c := &Client{
		msgs:     make(chan interface{}, 100),
		quit:     make(chan bool),
		Interval: 5 * time.Second,
		Endpoint: Endpoint,
		Size:     250,
		key:      key,
		now:      time.Now,
		uid:      uid,
	}

	go c.loop()

	return c
}

// Alias buffers an "alias" message.
func (c *Client) Alias(msg *Alias) error {
	if msg.UserId == "" {
		return errors.New("You must pass a 'userId'.")
	}

	if msg.PreviousId == "" {
		return errors.New("You must pass a 'previousId'.")
	}

	msg.Type = "alias"
	c.queue(msg)

	return nil
}

// Page buffers an "page" message.
func (c *Client) Page(msg *Page) error {
	if msg.UserId == "" && msg.AnonymousId == "" {
		return errors.New("You must pass either an 'anonymousId' or 'userId'.")
	}

	msg.Type = "page"
	c.queue(msg)

	return nil
}

// Group buffers an "group" message.
func (c *Client) Group(msg *Group) error {
	if msg.GroupId == "" {
		return errors.New("You must pass a 'groupId'.")
	}

	if msg.UserId == "" && msg.AnonymousId == "" {
		return errors.New("You must pass either an 'anonymousId' or 'userId'.")
	}

	msg.Type = "group"
	c.queue(msg)

	return nil
}

// Identify buffers an "identify" message.
func (c *Client) Identify(msg *Identify) error {
	if msg.UserId == "" && msg.AnonymousId == "" {
		return errors.New("You must pass either an 'anonymousId' or 'userId'.")
	}

	msg.Type = "identify"
	c.queue(msg)

	return nil
}

// Track buffers an "track" message.
func (c *Client) Track(msg *Track) error {
	if msg.Event == "" {
		return errors.New("You must pass 'event'.")
	}

	if msg.UserId == "" && msg.AnonymousId == "" {
		return errors.New("You must pass either an 'anonymousId' or 'userId'.")
	}

	msg.Type = "track"
	c.queue(msg)

	return nil
}

// Queue message.
func (c *Client) queue(msg message) {
	msg.setMessageId(c.uid())
	msg.setTimestamp(timestamp(c.now()))
	c.msgs <- msg
}

// Close and flush metrics.
func (c *Client) Close() error {
	c.quit <- true
	close(c.msgs)
	<-c.quit
	return nil
}

// Send batch request.
func (c *Client) send(msgs []interface{}) {
	if len(msgs) == 0 {
		return
	}

	batch := new(Batch)
	batch.Messages = msgs
	batch.MessageId = c.uid()
	batch.SentAt = timestamp(c.now())
	batch.Context = DefaultContext

	b, err := json.Marshal(batch)
	if err != nil {
		c.log("error marshalling msgs: %s", err)
		return
	}

	url := c.Endpoint + "/v1/batch"
	req, err := http.NewRequest("POST", url, bytes.NewReader(b))
	if err != nil {
		c.log("error creating request: %s", err)
		return
	}

	req.Header.Add("User-Agent", "analytics-go (version: "+Version+")")
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("Content-Length", string(len(b)))
	req.SetBasicAuth(c.key, "")

	res, err := http.DefaultClient.Do(req)
	if err != nil {
		c.log("error sending request: %s", err)
		return
	}
	defer res.Body.Close()

	c.report(res)
}

// Report on response body.
func (c *Client) report(res *http.Response) {
	if res.StatusCode < 400 {
		c.verbose("response %s", res.Status)
		return
	}

	msg := new(response)
	err := json.NewDecoder(res.Body).Decode(msg)
	if err != nil {
		c.log("error reading response: %s", err)
		return
	}

	c.log("response %s: %s – %s", res.Status, msg.Code, msg.Message)
}

// Batch loop.
func (c *Client) loop() {
	var msgs []interface{}
	tick := time.NewTicker(c.Interval)

	for {
		select {
		case msg := <-c.msgs:
			c.verbose("buffer (%d/%d) %v", len(msgs), c.Size, msg)
			msgs = append(msgs, msg)
			if len(msgs) == c.Size {
				c.verbose("exceeded %d messages – flushing", c.Size)
				c.send(msgs)
				msgs = nil
			}
		case <-tick.C:
			if len(msgs) > 0 {
				c.verbose("interval reached - flushing %d", len(msgs))
				c.send(msgs)
				msgs = nil
			} else {
				c.verbose("interval reached – nothing to send")
			}
		case <-c.quit:
			c.verbose("exit requested – flushing %d", len(msgs))
			c.send(msgs)
			c.verbose("exit")
			c.quit <- true
			return
		}
	}
}

// Verbose log.
func (c *Client) verbose(msg string, args ...interface{}) {
	if c.Verbose {
		log.Printf("segment: "+msg, args...)
	}
}

// Unconditional log.
func (c *Client) log(msg string, args ...interface{}) {
	log.Printf("segment: "+msg, args...)
}

// Set message timestamp if one is not already set.
func (m *Message) setTimestamp(s string) {
	if m.Timestamp == "" {
		m.Timestamp = s
	}
}

// Set message id.
func (m *Message) setMessageId(s string) {
	m.MessageId = s
}

// Return formatted timestamp.
func timestamp(t time.Time) string {
	return strftime.Format("%Y-%m-%dT%H:%M:%S%z", t)
}

// Return uuid string.
func uid() string {
	return uuid.NewRandom().String()
}
<|endoftext|>


<commit_before>// Package singletonqueue implements global singleton queues of tasks that can be accessed by their IDs.
// Package structure is loosely based on https://golang.org/pkg/container/heap/ .
package singletonqueue

import (
	"github.com/garyburd/redigo/redis"
	log "gopkg.in/Sirupsen/logrus.v0"
	"gopkg.in/vmihailenco/msgpack.v2"
	"os"
	"os/signal"
	"strings"
	"syscall"
)

// RedisPool is the reference to the pool used to draw connections from. Set this before calling any functions in this package.
var RedisPool *redis.Pool

//Interface singleton queue interface
type Interface interface {
	QueueID() string               // Unique ID
	Process(message Message) error // The function that will process messages
	New(queueID string) Interface  // When trying to reinitialize interface given a queueID
}

// Message is passed to Queue.WorkerFunc.
type Message struct {
	Payload []byte
	// TODO: put creation timestamp
}

// RedisQueueKey returns the redis key name of the list used for the queue.
func redisQueueKey(q Interface) string {
	return "sq:q:" + q.QueueID()
}

// RedisLockKey returns the redis key name used to "lock" the worker instance such that only one will run at a time.
func redisLockKey(q Interface) string {
	return "sq:lock:" + q.QueueID()
}

// Length returns the number of jobs currently in the queue.
func Length(q Interface) (int, error) {
	conn := RedisPool.Get()
	defer conn.Close()
	return redis.Int(conn.Do("LLEN", redisQueueKey(q)))
}

// Push queues a new job and runs the worker in a separate goroutine when none is running.
func Push(q Interface, payload []byte) error {
	message := Message{Payload: payload}
	marshaled, err := msgpack.Marshal(message)
	if err != nil {
		return err
	}
	conn := RedisPool.Get()
	defer conn.Close()
	_, err = conn.Do("LPUSH", redisQueueKey(q), marshaled)
	go EnsureWorkerIsRunning(q)
	return err
}

// PushIfEmpty queues a new job only when the queue is empty. Useful for cached values that are refreshed only every so often - this will ensure that there is no dogpile effect (multiple attempts to refresh cache at the same time). Note that the job may still be invoked multiple times in a small amount of time (eg if the second push happens after the worker has done processing the first one), thus the task should be idempotent.
func PushIfEmpty(q Interface, payload []byte) error {
	queueLength, err := Length(q)
	if err != nil {
		return err
	}
	if queueLength == 0 {
		Push(q, payload)
	} else {
		EnsureWorkerIsRunning(q)
	}
	return nil
}

// EnsureWorkerIsRunning runs as a worker if none runs, but returns immediately if one already runs. It is recommended to run this in a goroutine.
func EnsureWorkerIsRunning(q Interface) {
	logger := log.WithField("worker", q.QueueID())
	ret, err := redis.String(safeDo("SET", redisLockKey(q), 1, "NX"))
	if err != nil || ret != "OK" {
		logger.Debug("Unable to acquire lock, another worker is possibly running")
		return
	}

	logger.Info("Worker is starting")
	defer logger.Info("Worker has terminated")
	defer safeDo("DEL", redisLockKey(q))

	quit := make(chan struct{})
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, os.Interrupt)
	signal.Notify(sigs, syscall.SIGTERM)
	quitting := false

	go func() {
		for {
			select {
			case <-sigs:
				quitting = true
				return
			case <-quit:
				quitting = true
				return
			}
		}
	}()

	for {
		if quitting {
			return
		}
		ret, err := redis.Bytes(safeDo("RPOP", redisQueueKey(q)))
		//log.Info("ret:", ret, " err:", err)
		if err != nil && len(ret) == 0 {
			logger.Debug("No more jobs in the queue, exiting.")
			close(quit)
			return // no more jobs in the queue
		}
		var message Message
		err = msgpack.Unmarshal(ret, &message)
		if err != nil {
			logger.WithError(err).Error("Unmarshal failed, skipping this message")
			// skip this message
			continue
		}

		err = q.Process(message)
		if err != nil {
			logger.Error(err)
			logger.WithError(err).WithField("message", message).Error("Worker returned error")
			// TODO option to retry job if fail
		}
	}

}

func safeDo(commandName string, args ...interface{}) (interface{}, error) {
	conn := RedisPool.Get()
	defer conn.Close()
	return conn.Do(commandName, args...)
}

func Respawn(prefix string, queue Interface) {
	log.Info(prefix)
	ret, _ := redis.Strings(safeDo("KEYS", "sq:q:"+prefix+"*"))

	for _, key := range ret {
		queueID := strings.TrimPrefix(key, "sq:q:")
		newQueue := queue.New(queueID)
		if newQueue != nil {
			go EnsureWorkerIsRunning(newQueue)
		}
	}
}
<commit_msg>stop and close os signal<commit_after>// Package singletonqueue implements global singleton queues of tasks that can be accessed by their IDs.
// Package structure is loosely based on https://golang.org/pkg/container/heap/ .
package singletonqueue

import (
	"github.com/garyburd/redigo/redis"
	log "gopkg.in/Sirupsen/logrus.v0"
	"gopkg.in/vmihailenco/msgpack.v2"
	"os"
	"os/signal"
	"strings"
	"syscall"
)

// RedisPool is the reference to the pool used to draw connections from. Set this before calling any functions in this package.
var RedisPool *redis.Pool

//Interface singleton queue interface
type Interface interface {
	QueueID() string               // Unique ID
	Process(message Message) error // The function that will process messages
	New(queueID string) Interface  // When trying to reinitialize interface given a queueID
}

// Message is passed to Queue.WorkerFunc.
type Message struct {
	Payload []byte
	// TODO: put creation timestamp
}

// RedisQueueKey returns the redis key name of the list used for the queue.
func redisQueueKey(q Interface) string {
	return "sq:q:" + q.QueueID()
}

// RedisLockKey returns the redis key name used to "lock" the worker instance such that only one will run at a time.
func redisLockKey(q Interface) string {
	return "sq:lock:" + q.QueueID()
}

// Length returns the number of jobs currently in the queue.
func Length(q Interface) (int, error) {
	conn := RedisPool.Get()
	defer conn.Close()
	return redis.Int(conn.Do("LLEN", redisQueueKey(q)))
}

// Push queues a new job and runs the worker in a separate goroutine when none is running.
func Push(q Interface, payload []byte) error {
	message := Message{Payload: payload}
	marshaled, err := msgpack.Marshal(message)
	if err != nil {
		return err
	}
	conn := RedisPool.Get()
	defer conn.Close()
	_, err = conn.Do("LPUSH", redisQueueKey(q), marshaled)
	go EnsureWorkerIsRunning(q)
	return err
}

// PushIfEmpty queues a new job only when the queue is empty. Useful for cached values that are refreshed only every so often - this will ensure that there is no dogpile effect (multiple attempts to refresh cache at the same time). Note that the job may still be invoked multiple times in a small amount of time (eg if the second push happens after the worker has done processing the first one), thus the task should be idempotent.
func PushIfEmpty(q Interface, payload []byte) error {
	queueLength, err := Length(q)
	if err != nil {
		return err
	}
	if queueLength == 0 {
		Push(q, payload)
	} else {
		EnsureWorkerIsRunning(q)
	}
	return nil
}

// EnsureWorkerIsRunning runs as a worker if none runs, but returns immediately if one already runs. It is recommended to run this in a goroutine.
func EnsureWorkerIsRunning(q Interface) {
	logger := log.WithField("worker", q.QueueID())
	ret, err := redis.String(safeDo("SET", redisLockKey(q), 1, "NX"))
	if err != nil || ret != "OK" {
		logger.Debug("Unable to acquire lock, another worker is possibly running")
		return
	}

	logger.Info("Worker is starting")
	defer logger.Info("Worker has terminated")
	defer safeDo("DEL", redisLockKey(q))

	quit := make(chan struct{})
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, os.Interrupt)
	signal.Notify(sigs, syscall.SIGTERM)
	quitting := false

	go func() {
		for {
			select {
			case <-sigs:
				quitting = true
				return
			case <-quit:
				quitting = true
				return
			}
		}
	}()

	for {
		if quitting {
			signal.Stop(sigs)
			close(sigs)
			return
		}
		ret, err := redis.Bytes(safeDo("RPOP", redisQueueKey(q)))
		//log.Info("ret:", ret, " err:", err)
		if err != nil && len(ret) == 0 {
			logger.Debug("No more jobs in the queue, exiting.")
			close(quit)
			return // no more jobs in the queue
		}
		var message Message
		err = msgpack.Unmarshal(ret, &message)
		if err != nil {
			logger.WithError(err).Error("Unmarshal failed, skipping this message")
			// skip this message
			continue
		}

		err = q.Process(message)
		if err != nil {
			logger.Error(err)
			logger.WithError(err).WithField("message", message).Error("Worker returned error")
			// TODO option to retry job if fail
		}
	}

}

func safeDo(commandName string, args ...interface{}) (interface{}, error) {
	conn := RedisPool.Get()
	defer conn.Close()
	return conn.Do(commandName, args...)
}

func Respawn(prefix string, queue Interface) {
	log.Info(prefix)
	ret, _ := redis.Strings(safeDo("KEYS", "sq:q:"+prefix+"*"))

	for _, key := range ret {
		queueID := strings.TrimPrefix(key, "sq:q:")
		newQueue := queue.New(queueID)
		if newQueue != nil {
			go EnsureWorkerIsRunning(newQueue)
		}
	}
}
<|endoftext|>