<commit_before>// Copyright (C) 2016-2017 vLife Systems Ltd <http://vlifesystems.com>
// Licensed under an MIT licence.  Please see LICENSE.md for details.

// Package rule implements rules to be tested against a dataset
package rule

import (
	"fmt"
	"github.com/lawrencewoodman/ddataset"
	"github.com/lawrencewoodman/dexpr"
	"github.com/lawrencewoodman/dlit"
	"github.com/vlifesystems/rhkit/description"
	"github.com/vlifesystems/rhkit/internal"
	"github.com/vlifesystems/rhkit/internal/dexprfuncs"
	"sort"
	"strings"
	"sync"
)

var (
	generatorsMu sync.RWMutex
	generators   = make(map[string]generatorFunc)
)

// GenerationDescriber describes what sort of rules should be generated
type GenerationDescriber interface {
	// Fields indicates which fields should be used to generate rules
	Fields() []string
	// Arithmetic indicates whether to generate arithmetic rules
	Arithmetic() bool
}

type generatorFunc func(
	desc *description.Description,
	generationDesc GenerationDescriber,
	field string,
) []Rule

type Rule interface {
	fmt.Stringer
	IsTrue(record ddataset.Record) (bool, error)
	Fields() []string
}

type Tweaker interface {
	Tweak(desc *description.Description, stage int) []Rule
}

type Overlapper interface {
	Overlaps(o Rule) bool
}

type DPReducer interface {
	DPReduce() []Rule
}

type Valuer interface {
	Value() *dlit.Literal
}

type Complexity struct {
	Arithmetic bool
}

// Generate generates rules for rules that have registered a generator.
// complexity is used to indicate how complex rules should be and therefore
// has an impact on how many rules are generated.
func Generate(
	inputDescription *description.Description,
	generationDesc GenerationDescriber,
) ([]Rule, error) {
	err := checkRuleFieldsValid(inputDescription, generationDesc.Fields())
	if err != nil {
		return []Rule{}, err
	}
	rules := make([]Rule, 1)
	rules[0] = NewTrue()
	for field := range inputDescription.Fields {
		if internal.IsStringInSlice(field, generationDesc.Fields()) {
			for _, generator := range generators {
				newRules := generator(inputDescription, generationDesc, field)
				rules = append(rules, newRules...)
			}
		}
	}

	countEQVFRules := generateCountEQVF(inputDescription, generationDesc)
	rules = append(rules, countEQVFRules...)

	if len(generationDesc.Fields()) == 2 {
		cRules := Combine(rules)
		rules = append(rules, cRules...)
	}
	Sort(rules)
	return Uniq(rules), nil
}

func Combine(rules []Rule) []Rule {
	Sort(rules)
	combinedRules := make([]Rule, 0)
	numRules := len(rules)
	for i := 0; i < numRules-1; i++ {
		for j := i + 1; j < numRules; j++ {
			if andRule, err := NewAnd(rules[i], rules[j]); err == nil {
				combinedRules = append(combinedRules, andRule)
			}
			if orRule, err := NewOr(rules[i], rules[j]); err == nil {
				combinedRules = append(combinedRules, orRule)
			}
		}
	}
	return Uniq(combinedRules)
}

// Sort sorts the rules in place using their .String() method
func Sort(rules []Rule) {
	sort.Sort(byString(rules))
}

// Uniq returns the slices of Rules with duplicates removed
func Uniq(rules []Rule) []Rule {
	results := []Rule{}
	mResults := map[string]interface{}{}
	for _, r := range rules {
		if _, ok := mResults[r.String()]; !ok {
			mResults[r.String()] = nil
			results = append(results, r)
		}
	}
	return results
}

// ReduceDP returns decimal number reduced rules if they can be reduced.
// Adds True rule at end.
func ReduceDP(rules []Rule) []Rule {
	newRules := make([]Rule, 0)
	for _, r := range rules {
		switch x := r.(type) {
		case DPReducer:
			rules := x.DPReduce()
			newRules = append(newRules, rules...)
		}
	}
	newRules = append(newRules, NewTrue())
	return Uniq(newRules)
}

func commaJoinValues(values []*dlit.Literal) string {
	str := fmt.Sprintf("\"%s\"", values[0].String())
	for _, v := range values[1:] {
		str += fmt.Sprintf(",\"%s\"", v)
	}
	return str
}

// byString implements sort.Interface for []Rule
type byString []Rule

func (rs byString) Len() int { return len(rs) }
func (rs byString) Swap(i, j int) {
	rs[i], rs[j] = rs[j], rs[i]
}
func (rs byString) Less(i, j int) bool {
	return strings.Compare(rs[i].String(), rs[j].String()) == -1
}

// registerGenerator makes a rule generator available.
// If called twice with the same ruleName it panics.
func registerGenerator(ruleType string, generator generatorFunc) {
	generatorsMu.Lock()
	defer generatorsMu.Unlock()
	if _, dup := generators[ruleType]; dup {
		panic("registerGenerator called twice for ruleType: " + ruleType)
	}
	generators[ruleType] = generator
}

func generateTweakPoints(
	value, min, max *dlit.Literal,
	maxDP int,
	stage int,
) []*dlit.Literal {
	vars := map[string]*dlit.Literal{
		"min":   min,
		"max":   max,
		"maxDP": dlit.MustNew(maxDP),
		"stage": dlit.MustNew(stage),
		"value": value,
	}
	vars["step"] =
		dexpr.Eval(
			"roundto((max - min) / (10 * stage), maxDP)",
			dexprfuncs.CallFuncs,
			vars,
		)
	validValueExpr := dexpr.MustNew(
		"rp > min && rp != value && rp < max",
		dexprfuncs.CallFuncs,
	)
	low := dexpr.Eval("max(min, value - step)", dexprfuncs.CallFuncs, vars)
	high := dexpr.Eval("min(max, value + step)", dexprfuncs.CallFuncs, vars)
	points := internal.GeneratePoints(low, high, maxDP)
	tweakPoints := map[string]*dlit.Literal{}

	for _, p := range points {
		vars["p"] = p
		vars["rp"] = internal.RoundLit(p, maxDP)
		if ok, err := validValueExpr.EvalBool(vars); ok && err == nil {
			tweakPoints[vars["rp"].String()] = vars["rp"]
		}
	}

	return internal.MapLitNumsToSlice(tweakPoints)
}

type makeRoundRule func(*dlit.Literal) Rule

func roundRules(v *dlit.Literal, makeRule makeRoundRule) []Rule {
	rulesMap := map[string]Rule{}
	rules := []Rule{}
	for dp := 200; dp >= 0; dp-- {
		p := internal.RoundLit(v, dp)
		r := makeRule(p)
		if _, ok := rulesMap[r.String()]; !ok {
			rulesMap[r.String()] = r
			rules = append(rules, r)
		}
	}
	return rules
}

func calcNumSharedValues(fd1 *description.Field, fd2 *description.Field) int {
	numShared := 0
	for _, vd1 := range fd1.Values {
		if _, ok := fd2.Values[vd1.Value.String()]; ok {
			numShared++
		}
	}
	return numShared
}

var compareExpr *dexpr.Expr = dexpr.MustNew(
	"min1 < max2 && max1 > min2",
	dexprfuncs.CallFuncs,
)

func hasComparableNumberRange(
	fd1 *description.Field,
	fd2 *description.Field,
) bool {
	if fd1.Kind != description.Number || fd2.Kind != description.Number {
		return false
	}
	var isComparable bool
	vars := map[string]*dlit.Literal{
		"min1": fd1.Min,
		"max1": fd1.Max,
		"min2": fd2.Min,
		"max2": fd2.Max,
	}
	isComparable, err := compareExpr.EvalBool(vars)
	return err == nil && isComparable
}

func checkRuleFieldsValid(
	inputDescription *description.Description,
	ruleFields []string,
) error {
	if len(ruleFields) == 0 {
		return ErrNoRuleFieldsSpecified
	}
	fields := inputDescription.FieldNames()
	for _, ruleField := range ruleFields {
		if !internal.IsStringInSlice(ruleField, fields) {
			return InvalidRuleFieldError(ruleField)
		}
	}
	return nil
}

func countNumOnBits(mask string) int {
	return strings.Count(mask, "1")
}

func makeMask(numPlaces, i int) string {
	return fmt.Sprintf("%0*b", numPlaces, i)
}
<commit_msg>Remove redundant references to complexity<commit_after>// Copyright (C) 2016-2017 vLife Systems Ltd <http://vlifesystems.com>
// Licensed under an MIT licence.  Please see LICENSE.md for details.

// Package rule implements rules to be tested against a dataset
package rule

import (
	"fmt"
	"github.com/lawrencewoodman/ddataset"
	"github.com/lawrencewoodman/dexpr"
	"github.com/lawrencewoodman/dlit"
	"github.com/vlifesystems/rhkit/description"
	"github.com/vlifesystems/rhkit/internal"
	"github.com/vlifesystems/rhkit/internal/dexprfuncs"
	"sort"
	"strings"
	"sync"
)

var (
	generatorsMu sync.RWMutex
	generators   = make(map[string]generatorFunc)
)

// GenerationDescriber describes what sort of rules should be generated
type GenerationDescriber interface {
	// Fields indicates which fields should be used to generate rules
	Fields() []string
	// Arithmetic indicates whether to generate arithmetic rules
	Arithmetic() bool
}

type generatorFunc func(
	desc *description.Description,
	generationDesc GenerationDescriber,
	field string,
) []Rule

type Rule interface {
	fmt.Stringer
	IsTrue(record ddataset.Record) (bool, error)
	Fields() []string
}

type Tweaker interface {
	Tweak(desc *description.Description, stage int) []Rule
}

type Overlapper interface {
	Overlaps(o Rule) bool
}

type DPReducer interface {
	DPReduce() []Rule
}

type Valuer interface {
	Value() *dlit.Literal
}

// Generate generates rules for rules that have registered a generator.
func Generate(
	inputDescription *description.Description,
	generationDesc GenerationDescriber,
) ([]Rule, error) {
	err := checkRuleFieldsValid(inputDescription, generationDesc.Fields())
	if err != nil {
		return []Rule{}, err
	}
	rules := make([]Rule, 1)
	rules[0] = NewTrue()
	for field := range inputDescription.Fields {
		if internal.IsStringInSlice(field, generationDesc.Fields()) {
			for _, generator := range generators {
				newRules := generator(inputDescription, generationDesc, field)
				rules = append(rules, newRules...)
			}
		}
	}

	countEQVFRules := generateCountEQVF(inputDescription, generationDesc)
	rules = append(rules, countEQVFRules...)

	if len(generationDesc.Fields()) == 2 {
		cRules := Combine(rules)
		rules = append(rules, cRules...)
	}
	Sort(rules)
	return Uniq(rules), nil
}

func Combine(rules []Rule) []Rule {
	Sort(rules)
	combinedRules := make([]Rule, 0)
	numRules := len(rules)
	for i := 0; i < numRules-1; i++ {
		for j := i + 1; j < numRules; j++ {
			if andRule, err := NewAnd(rules[i], rules[j]); err == nil {
				combinedRules = append(combinedRules, andRule)
			}
			if orRule, err := NewOr(rules[i], rules[j]); err == nil {
				combinedRules = append(combinedRules, orRule)
			}
		}
	}
	return Uniq(combinedRules)
}

// Sort sorts the rules in place using their .String() method
func Sort(rules []Rule) {
	sort.Sort(byString(rules))
}

// Uniq returns the slices of Rules with duplicates removed
func Uniq(rules []Rule) []Rule {
	results := []Rule{}
	mResults := map[string]interface{}{}
	for _, r := range rules {
		if _, ok := mResults[r.String()]; !ok {
			mResults[r.String()] = nil
			results = append(results, r)
		}
	}
	return results
}

// ReduceDP returns decimal number reduced rules if they can be reduced.
// Adds True rule at end.
func ReduceDP(rules []Rule) []Rule {
	newRules := make([]Rule, 0)
	for _, r := range rules {
		switch x := r.(type) {
		case DPReducer:
			rules := x.DPReduce()
			newRules = append(newRules, rules...)
		}
	}
	newRules = append(newRules, NewTrue())
	return Uniq(newRules)
}

func commaJoinValues(values []*dlit.Literal) string {
	str := fmt.Sprintf("\"%s\"", values[0].String())
	for _, v := range values[1:] {
		str += fmt.Sprintf(",\"%s\"", v)
	}
	return str
}

// byString implements sort.Interface for []Rule
type byString []Rule

func (rs byString) Len() int { return len(rs) }
func (rs byString) Swap(i, j int) {
	rs[i], rs[j] = rs[j], rs[i]
}
func (rs byString) Less(i, j int) bool {
	return strings.Compare(rs[i].String(), rs[j].String()) == -1
}

// registerGenerator makes a rule generator available.
// If called twice with the same ruleName it panics.
func registerGenerator(ruleType string, generator generatorFunc) {
	generatorsMu.Lock()
	defer generatorsMu.Unlock()
	if _, dup := generators[ruleType]; dup {
		panic("registerGenerator called twice for ruleType: " + ruleType)
	}
	generators[ruleType] = generator
}

func generateTweakPoints(
	value, min, max *dlit.Literal,
	maxDP int,
	stage int,
) []*dlit.Literal {
	vars := map[string]*dlit.Literal{
		"min":   min,
		"max":   max,
		"maxDP": dlit.MustNew(maxDP),
		"stage": dlit.MustNew(stage),
		"value": value,
	}
	vars["step"] =
		dexpr.Eval(
			"roundto((max - min) / (10 * stage), maxDP)",
			dexprfuncs.CallFuncs,
			vars,
		)
	validValueExpr := dexpr.MustNew(
		"rp > min && rp != value && rp < max",
		dexprfuncs.CallFuncs,
	)
	low := dexpr.Eval("max(min, value - step)", dexprfuncs.CallFuncs, vars)
	high := dexpr.Eval("min(max, value + step)", dexprfuncs.CallFuncs, vars)
	points := internal.GeneratePoints(low, high, maxDP)
	tweakPoints := map[string]*dlit.Literal{}

	for _, p := range points {
		vars["p"] = p
		vars["rp"] = internal.RoundLit(p, maxDP)
		if ok, err := validValueExpr.EvalBool(vars); ok && err == nil {
			tweakPoints[vars["rp"].String()] = vars["rp"]
		}
	}

	return internal.MapLitNumsToSlice(tweakPoints)
}

type makeRoundRule func(*dlit.Literal) Rule

func roundRules(v *dlit.Literal, makeRule makeRoundRule) []Rule {
	rulesMap := map[string]Rule{}
	rules := []Rule{}
	for dp := 200; dp >= 0; dp-- {
		p := internal.RoundLit(v, dp)
		r := makeRule(p)
		if _, ok := rulesMap[r.String()]; !ok {
			rulesMap[r.String()] = r
			rules = append(rules, r)
		}
	}
	return rules
}

func calcNumSharedValues(fd1 *description.Field, fd2 *description.Field) int {
	numShared := 0
	for _, vd1 := range fd1.Values {
		if _, ok := fd2.Values[vd1.Value.String()]; ok {
			numShared++
		}
	}
	return numShared
}

var compareExpr *dexpr.Expr = dexpr.MustNew(
	"min1 < max2 && max1 > min2",
	dexprfuncs.CallFuncs,
)

func hasComparableNumberRange(
	fd1 *description.Field,
	fd2 *description.Field,
) bool {
	if fd1.Kind != description.Number || fd2.Kind != description.Number {
		return false
	}
	var isComparable bool
	vars := map[string]*dlit.Literal{
		"min1": fd1.Min,
		"max1": fd1.Max,
		"min2": fd2.Min,
		"max2": fd2.Max,
	}
	isComparable, err := compareExpr.EvalBool(vars)
	return err == nil && isComparable
}

func checkRuleFieldsValid(
	inputDescription *description.Description,
	ruleFields []string,
) error {
	if len(ruleFields) == 0 {
		return ErrNoRuleFieldsSpecified
	}
	fields := inputDescription.FieldNames()
	for _, ruleField := range ruleFields {
		if !internal.IsStringInSlice(ruleField, fields) {
			return InvalidRuleFieldError(ruleField)
		}
	}
	return nil
}

func countNumOnBits(mask string) int {
	return strings.Count(mask, "1")
}

func makeMask(numPlaces, i int) string {
	return fmt.Sprintf("%0*b", numPlaces, i)
}
<|endoftext|>


<commit_before>package fabric

import (
	"context"
	"errors"
	"fmt"
	"log"
	"net"
	"strconv"

	upnp "github.com/NebulousLabs/go-upnp"
	"go.uber.org/zap"
)

// NewTransportTCP returns a new TCP transport
func NewTransportTCP(host string, port int) Transport {
	return &TCP{
		host: host,
		port: port,
	}
}

// TCP transport
type TCP struct {
	host     string
	port     int
	listener net.Listener
}

// DialContext attemps to dial to the peer with the given addr
func (t *TCP) DialContext(ctx context.Context, addr Address) (
	net.Conn, error) {
	pr := addr.CurrentParams()
	tcon, err := net.Dial("tcp", pr)
	if err != nil {
		return nil, err
	}

	return tcon, nil
}

// CanDial checks if address can be dialed by this transport
func (t *TCP) CanDial(addr Address) (bool, error) {
	if addr.CurrentProtocol() != "tcp" {
		return false, nil
	}

	hostPort := addr.CurrentParams()
	_, portString, err := net.SplitHostPort(hostPort)
	if err != nil {
		return false, err
	}

	port, err := strconv.Atoi(portString)
	if err != nil || port == 0 || port > 65535 {
		return false, errors.New("Invalid port number")
	}

	return true, nil
}

// Listen handles the transports
func (t *TCP) Listen(ctx context.Context, handler func(context.Context, net.Conn) error) error {
	addr := fmt.Sprintf("%s:%d", t.host, t.port)
	listener, err := net.Listen("tcp", addr)
	if err != nil {
		return err
	}

	t.listener = listener

	err = t.startExternal()
	if err != nil {
		log.Println("Could not open upnp port: ", err)
	}

	go func() {
		for {
			// Listen for an incoming connection.
			conn, err := listener.Accept()
			if err != nil {
				Logger(ctx).Error("Could not accept TCP connection", zap.Error(err))
				continue
			}
			go t.handleListen(ctx, conn, handler)
		}
	}()

	return nil
}

func (t *TCP) handleListen(ctx context.Context, conn net.Conn, handler func(context.Context, net.Conn) error) {
	if err := handler(ctx, conn); err != nil {
		fmt.Println("Listen: Could not handle request. error:", err)
	}
}

func (t *TCP) startExternal() error {
	upr, err := upnp.Discover()
	if err != nil {
		return err
	}

	_, xpstr, err := net.SplitHostPort(t.listener.Addr().String())
	if err != nil {
		return err
	}

	extPort, err := strconv.ParseUint(xpstr, 10, 16)
	if err != nil {
		return err
	}

	err = upr.Clear(uint16(extPort))
	if err != nil {
		return err
	}

	err = upr.Forward(uint16(extPort), "fabric")
	if err != nil {
		return err
	}

	return nil
}

// Addresses returns the addresses the transport is listening to
func (t *TCP) Addresses() []string {
	port := t.listener.Addr().(*net.TCPAddr).Port
	addrs, err := GetAddresses(port)
	if err != nil {
		return []string{}
	}

	for i, addr := range addrs {
		addrs[i] = "tcp:" + addr
	}

	return addrs
}
<commit_msg>Change clear upnp not to return on error<commit_after>package fabric

import (
	"context"
	"errors"
	"fmt"
	"log"
	"net"
	"strconv"

	upnp "github.com/NebulousLabs/go-upnp"
	"go.uber.org/zap"
)

// NewTransportTCP returns a new TCP transport
func NewTransportTCP(host string, port int) Transport {
	return &TCP{
		host: host,
		port: port,
	}
}

// TCP transport
type TCP struct {
	host     string
	port     int
	listener net.Listener
}

// DialContext attemps to dial to the peer with the given addr
func (t *TCP) DialContext(ctx context.Context, addr Address) (
	net.Conn, error) {
	pr := addr.CurrentParams()
	tcon, err := net.Dial("tcp", pr)
	if err != nil {
		return nil, err
	}

	return tcon, nil
}

// CanDial checks if address can be dialed by this transport
func (t *TCP) CanDial(addr Address) (bool, error) {
	if addr.CurrentProtocol() != "tcp" {
		return false, nil
	}

	hostPort := addr.CurrentParams()
	_, portString, err := net.SplitHostPort(hostPort)
	if err != nil {
		return false, err
	}

	port, err := strconv.Atoi(portString)
	if err != nil || port == 0 || port > 65535 {
		return false, errors.New("Invalid port number")
	}

	return true, nil
}

// Listen handles the transports
func (t *TCP) Listen(ctx context.Context, handler func(context.Context, net.Conn) error) error {
	addr := fmt.Sprintf("%s:%d", t.host, t.port)
	listener, err := net.Listen("tcp", addr)
	if err != nil {
		return err
	}

	t.listener = listener

	err = t.startExternal()
	if err != nil {
		log.Println("Could not open upnp port: ", err)
	}

	go func() {
		for {
			// Listen for an incoming connection.
			conn, err := listener.Accept()
			if err != nil {
				Logger(ctx).Error("Could not accept TCP connection", zap.Error(err))
				continue
			}
			go t.handleListen(ctx, conn, handler)
		}
	}()

	return nil
}

func (t *TCP) handleListen(ctx context.Context, conn net.Conn, handler func(context.Context, net.Conn) error) {
	if err := handler(ctx, conn); err != nil {
		fmt.Println("Listen: Could not handle request. error:", err)
	}
}

func (t *TCP) startExternal() error {
	upr, err := upnp.Discover()
	if err != nil {
		return err
	}

	_, xpstr, err := net.SplitHostPort(t.listener.Addr().String())
	if err != nil {
		return err
	}

	extPort, err := strconv.ParseUint(xpstr, 10, 16)
	if err != nil {
		return err
	}

	err = upr.Clear(uint16(extPort))
	if err != nil {
		log.Println("Could not clear upnp: ", err)
	}

	err = upr.Forward(uint16(extPort), "fabric")
	if err != nil {
		return err
	}

	return nil
}

// Addresses returns the addresses the transport is listening to
func (t *TCP) Addresses() []string {
	port := t.listener.Addr().(*net.TCPAddr).Port
	addrs, err := GetAddresses(port)
	if err != nil {
		return []string{}
	}

	for i, addr := range addrs {
		addrs[i] = "tcp:" + addr
	}

	return addrs
}
<|endoftext|>


<commit_before>package main

import (
  "bytes"
  "encoding/binary"
  "errors"
  "fmt"
  "log"
  "net"
  "regexp"
  "runtime"
  "sync"
  "syscall"
  zmq "github.com/alecthomas/gozmq"
)

const (
  zmq_signal_output   = "O"
  zmq_signal_input    = "I"
  zmq_signal_shutdown = "S"
)

type TransportZmq struct {
  config  *NetworkConfig
  context *zmq.Context
  dealer  *zmq.Socket

  wait sync.WaitGroup

  bridge_chan chan []byte

  send_chan chan *ZMQMessage
  recv_chan chan interface{}
  recv_bridge_chan chan interface{}

  can_send chan int
}

type ZMQMessage struct {
  part  []byte
  final bool
}

var TransportZmq_Context *zmq.Context

func CreateTransportZmq(config *NetworkConfig) (*TransportZmq, error) {
  if TransportZmq_Context == nil {
    var err error
    TransportZmq_Context, err = zmq.NewContext()
    if err != nil {
      return nil, errors.New(fmt.Sprintf("Failed to create ZMQ context: %s", err))
    }
  }
  return &TransportZmq{config: config, context: TransportZmq_Context}, nil
}

func (t *TransportZmq) Connect() (err error) {
  hostport_re := regexp.MustCompile(`^\[?([^]]+)\]?:([0-9]+)$`)
  endpoints := 0

  // Outbound dealer socket will fair-queue load balance amongst peers
  if t.dealer, err = t.context.NewSocket(zmq.DEALER); err != nil {
    return
  }

  // Connect endpoints
  for _, hostport := range t.config.Servers {
    submatch := hostport_re.FindSubmatch([]byte(hostport))
    if submatch == nil {
      log.Printf("Invalid host:port given: %s\n", hostport)
      continue
    }

    // Lookup the server in DNS (if this is IP it will implicitly return)
    host := string(submatch[1])
    port := string(submatch[2])
    addresses, err := net.LookupHost(host)
    if err != nil {
      log.Printf("DNS lookup failure \"%s\": %s\n", host, err)
      continue
    }

    // Connect to each address
    for _, address := range addresses {
      addressport := net.JoinHostPort(address, port)

      if err = t.dealer.Connect("tcp://" + addressport); err != nil {
        log.Printf("Failed to connect to %s (%s), skipping", addressport, host)
        continue
      }

      log.Printf("Connected with %s (%s) \n", addressport, host)
      endpoints++
    }
  }

  if endpoints == 0 {
    return errors.New("Failed to connect to any of the specified endpoints.")
  }

  // Control sockets to connect bridge to poller
  bridge_in, err := t.context.NewSocket(zmq.PUSH)
  if err != nil {
    t.dealer.Close()
    return
  }
  if err = bridge_in.Bind("inproc://notify"); err != nil {
    t.dealer.Close()
    bridge_in.Close()
    return
  }

  bridge_out, err := t.context.NewSocket(zmq.PULL)
  if err != nil {
    t.dealer.Close()
    bridge_in.Close()
    return err
  }
  if err = bridge_out.Connect("inproc://notify"); err != nil {
    t.dealer.Close()
    bridge_in.Close()
    bridge_out.Close()
    return
  }

  // Signal channels
  t.bridge_chan = make(chan []byte, 1)
  t.send_chan = make(chan *ZMQMessage, 2)
  t.recv_chan = make(chan interface{}, 1)
  t.recv_bridge_chan = make(chan interface{}, 1)
  t.can_send = make(chan int, 1)

  // Waiter we use to wait for shutdown
  t.wait.Add(2)

  // Bridge between channels and ZMQ
  go t.bridge(bridge_in)

  // The poller
  go t.poller(bridge_out)

  return nil
}

func (t *TransportZmq) bridge(bridge_in *zmq.Socket) {
  var message interface{}

  // Wait on channel, passing into socket
  // This keeps the socket in a single thread, otherwise we have to lock the entire publisher
  runtime.LockOSThread()

  for {
    select {
    case notify := <- t.bridge_chan:
      bridge_in.Send(notify, 0)

      // Shutdown?
      if string(notify) == zmq_signal_shutdown {
        break
      }
    case message = <-t.recv_bridge_chan:
    case func () chan<- interface{} {
      if message != nil {
        return t.recv_chan
      }
      return nil
    }() <- message:
      // The reason we flush recv through the bridge and not directly to recv_chan is so that if
      // the poller was quick and had to cache a receive as the channel was full, it will stop
      // polling - flushing through bridge allows us to signal poller to start polling again
      // It is not the publisher's responsibility to do this, and TLS wouldn't need it
      bridge_in.Send([]byte(zmq_signal_input), 0)
      message = nil
    }
  }

  // We should linger by default to ensure shutdown is transmitted
  bridge_in.Close()
  runtime.UnlockOSThread()
  t.wait.Done()
}

func (t *TransportZmq) poller(bridge_out *zmq.Socket) {
  var pollitems []zmq.PollItem
  var send_stage *ZMQMessage
  var recv_stage [][]byte
  var recv_body bool

  // ZMQ sockets are not thread-safe, so we have to send/receive on same thread
  // Thus, we cannot use a sender/receiver thread pair like we can with TLS so we use a single threaded poller instead
  // In order to asynchronously send and receive we just poll and do necessary actions

  // When data is ready to send we'll get a channel ping, that is bridged to ZMQ so we can then send data
  // For receiving, we receive here and bridge it to the channels, then receive more once that's through
  runtime.LockOSThread()

  pollitems = make([]zmq.PollItem, 2)
  pollitems[0].Socket = bridge_out
  pollitems[0].Events = zmq.POLLIN | zmq.POLLOUT
  pollitems[1].Socket = t.dealer
  pollitems[1].Events = zmq.POLLIN | zmq.POLLOUT

PollLoop:
  for {
    // Poll for events
    if _, err := zmq.Poll(pollitems, -1); err != nil {
      // Retry on EINTR
      if err == syscall.EINTR {
        continue
      }

      // Failure
      t.recv_chan <- errors.New(fmt.Sprintf("zmq.Poll failure %s", err))
      break
    }

    // Process control channel
    if pollitems[0].REvents & zmq.POLLIN != 0 {
    RetryControl:
      msg, err := bridge_out.Recv(zmq.DONTWAIT)
      if err != nil {
        switch err {
        case syscall.EINTR:
          // Try again
          goto RetryControl
        case syscall.EAGAIN:
          // Poll lied, poll again
          continue
        }

        // Failure
        t.recv_chan <- errors.New(fmt.Sprintf("Pull zmq.Socket.Recv failure %s", err))
        break
      }

      switch string(msg) {
      case zmq_signal_output:
        // Start polling for send
        pollitems[1].Events = pollitems[1].Events | zmq.POLLOUT
      case zmq_signal_input:
        // If we staged a receive, process that
        if recv_stage != nil {
          select {
          case t.recv_bridge_chan <- recv_stage:
            recv_stage = nil

            // Start polling for receive
            pollitems[1].Events = pollitems[1].Events | zmq.POLLIN
          default:
            // Do nothing, we were asked for receive but channel is already full
          }
        } else {
          // Start polling for receive
          pollitems[1].Events = pollitems[1].Events | zmq.POLLIN
        }
      case zmq_signal_shutdown:
        // Shutdown
        break PollLoop
      }
    }

    // Process dealer send
    if pollitems[1].REvents & zmq.POLLOUT != 0 {
      sent_one := false

      // Something in the staging buffer?
      if send_stage != nil {
        var err error
      RetrySendStage:
        if send_stage.final {
          err = t.dealer.Send(send_stage.part, zmq.DONTWAIT)
        } else {
          err = t.dealer.Send(send_stage.part, zmq.DONTWAIT | zmq.SNDMORE)
        }
        if err != nil {
          switch err {
          case syscall.EINTR:
            // Try again
            goto RetrySendStage
          case syscall.EAGAIN:
            // Poll lied, poll again
            continue
          }

          // Failure
          t.recv_chan <- errors.New(fmt.Sprintf("Dealer zmq.Socket.Send failure %s", err))
          break
        }

        sent_one = true
      }

      // Send messages from channel
    LoopSend:
      for {
        select {
        case msg := <-t.send_chan:
          var err error
        RetrySend:
          if msg.final {
            err = t.dealer.Send(msg.part, zmq.DONTWAIT)
          } else {
            err = t.dealer.Send(msg.part, zmq.DONTWAIT | zmq.SNDMORE)
          }
          if err != nil {
            switch err {
            case syscall.EINTR:
              // Try again
              goto RetrySend
            case syscall.EAGAIN:
              // Poll lied, poll again after we check others
              send_stage = msg
              goto PollRecv
            }

            // Failure
            t.recv_chan <- errors.New(fmt.Sprintf("Dealer zmq.Socket.Send failure %s", err))
            break PollLoop
          }

          sent_one = true
        default:
          break LoopSend
        }
      }

      if sent_one {
        // We just sent something, check POLLOUT still active before signalling we can send more
        // TODO: Check why Events() is returning uint64 instead of PollEvents
        // TODO: This is broken and actually returns an error
        if events, _ := t.dealer.Events(); zmq.PollEvents(events) & zmq.POLLOUT != 0 {
          t.setChan(t.can_send)
        }
      } else {
        t.setChan(t.can_send)
      }

      pollitems[1].Events = pollitems[1].Events ^ zmq.POLLOUT
    }

    // Process dealer receive
  PollRecv:
    if pollitems[1].REvents & zmq.POLLIN != 0 {
    LoopRecv:
      for {
        // Bring in the messages
      RetryRecv:
        data, err := t.dealer.Recv(zmq.DONTWAIT)
        if err != nil {
          switch err {
          case syscall.EINTR:
            // Try again
            goto RetryRecv
          case syscall.EAGAIN:
            // Poll lied, poll again
            continue PollLoop
          }

          // Failure
          t.recv_chan <- errors.New(fmt.Sprintf("Dealer zmq.Socket.Recv failure %s", err))
          break PollLoop
        }

        more, err := t.dealer.RcvMore()
        if err != nil {
          // Failure
          t.recv_chan <- errors.New(fmt.Sprintf("Dealer zmq.Socket.RcvMore failure %s", err))
          break PollLoop
        }

        // Sanity check, and don't save until empty message
        if len(data) == 0 && more {
          // Message separator, start returning
          recv_body = true
          continue
        } else if more {
          // Ignore all but last message
        } else if recv_body {
          recv_body = false

          // Last message and receiving, validate it first
          if len(data) < 8 {
            log.Printf("Skipping invalid message: not enough data")
          } else {
            length := binary.BigEndian.Uint32(data[4:8])
            if length > 1048576 {
              log.Printf("Skipping invalid message: data too large (%d)", length)
            } else if length != uint32(len(data)) - 8 {
              log.Printf("Skipping invalid message: data has invalid length (%d != %d)", len(data) - 8, length)
            } else {
              message := [][]byte{data[0:4], data[8:]}

              // Bridge to channels
              select {
              case t.recv_bridge_chan <- message:
              default:
                // We filled the channel, stop polling until we pull something off of it and stage the recv
                recv_stage = message
                break LoopRecv
              }
            }
          }
        } // Hmmm, discard anything else

        pollitems[1].Events = pollitems[1].Events ^ zmq.POLLIN
      }
    }
  }

  bridge_out.Close()
  runtime.UnlockOSThread()
  t.wait.Done()
}

func (t *TransportZmq) setChan(set chan int) {
  select {
  case set <- 1:
  default:
  }
}

func (t *TransportZmq) CanSend() <-chan int {
  return t.can_send
}

func (t *TransportZmq) Write(signature string, message []byte) (err error) {
  var write_buffer *bytes.Buffer
  write_buffer = bytes.NewBuffer(make([]byte, 0, len(signature) + 4 + len(message)))

  if _, err = write_buffer.Write([]byte(signature)); err != nil {
    return
  }
  if err = binary.Write(write_buffer, binary.BigEndian, uint32(len(message))); err != nil {
    return
  }
  if len(message) != 0 {
    if _, err = write_buffer.Write(message); err != nil {
      return
    }
  }

  t.send_chan <- &ZMQMessage{part: []byte(""), final: false}
  t.send_chan <- &ZMQMessage{part: write_buffer.Bytes(), final: true}

  // Ask for send to start
  t.bridge_chan <- []byte(zmq_signal_output)
  return nil
}

func (t *TransportZmq) Read() <-chan interface{} {
  return t.recv_chan
}

func (t *TransportZmq) Disconnect() {
  // Send shutdown request
  t.bridge_chan <- []byte(zmq_signal_shutdown)
  t.wait.Wait()
  t.dealer.Close()
}
<commit_msg>Fix zmq losing an ack message randomly and blocking<commit_after>package main

import (
  "bytes"
  "encoding/binary"
  "errors"
  "fmt"
  "log"
  "net"
  "regexp"
  "runtime"
  "sync"
  "syscall"
  zmq "github.com/alecthomas/gozmq"
)

const (
  zmq_signal_output   = "O"
  zmq_signal_input    = "I"
  zmq_signal_shutdown = "S"
)

type TransportZmq struct {
  config  *NetworkConfig
  context *zmq.Context
  dealer  *zmq.Socket

  wait sync.WaitGroup

  bridge_chan chan []byte

  send_chan chan *ZMQMessage
  recv_chan chan interface{}
  recv_bridge_chan chan interface{}

  can_send chan int
}

type ZMQMessage struct {
  part  []byte
  final bool
}

var TransportZmq_Context *zmq.Context

func CreateTransportZmq(config *NetworkConfig) (*TransportZmq, error) {
  if TransportZmq_Context == nil {
    var err error
    TransportZmq_Context, err = zmq.NewContext()
    if err != nil {
      return nil, errors.New(fmt.Sprintf("Failed to create ZMQ context: %s", err))
    }
  }
  return &TransportZmq{config: config, context: TransportZmq_Context}, nil
}

func (t *TransportZmq) Connect() (err error) {
  hostport_re := regexp.MustCompile(`^\[?([^]]+)\]?:([0-9]+)$`)
  endpoints := 0

  // Outbound dealer socket will fair-queue load balance amongst peers
  if t.dealer, err = t.context.NewSocket(zmq.DEALER); err != nil {
    return
  }

  // Connect endpoints
  for _, hostport := range t.config.Servers {
    submatch := hostport_re.FindSubmatch([]byte(hostport))
    if submatch == nil {
      log.Printf("Invalid host:port given: %s\n", hostport)
      continue
    }

    // Lookup the server in DNS (if this is IP it will implicitly return)
    host := string(submatch[1])
    port := string(submatch[2])
    addresses, err := net.LookupHost(host)
    if err != nil {
      log.Printf("DNS lookup failure \"%s\": %s\n", host, err)
      continue
    }

    // Connect to each address
    for _, address := range addresses {
      addressport := net.JoinHostPort(address, port)

      if err = t.dealer.Connect("tcp://" + addressport); err != nil {
        log.Printf("Failed to connect to %s (%s), skipping", addressport, host)
        continue
      }

      log.Printf("Connected with %s (%s) \n", addressport, host)
      endpoints++
    }
  }

  if endpoints == 0 {
    return errors.New("Failed to connect to any of the specified endpoints.")
  }

  // Control sockets to connect bridge to poller
  bridge_in, err := t.context.NewSocket(zmq.PUSH)
  if err != nil {
    t.dealer.Close()
    return
  }
  if err = bridge_in.Bind("inproc://notify"); err != nil {
    t.dealer.Close()
    bridge_in.Close()
    return
  }

  bridge_out, err := t.context.NewSocket(zmq.PULL)
  if err != nil {
    t.dealer.Close()
    bridge_in.Close()
    return err
  }
  if err = bridge_out.Connect("inproc://notify"); err != nil {
    t.dealer.Close()
    bridge_in.Close()
    bridge_out.Close()
    return
  }

  // Signal channels
  t.bridge_chan = make(chan []byte, 1)
  t.send_chan = make(chan *ZMQMessage, 2)
  t.recv_chan = make(chan interface{}, 1)
  t.recv_bridge_chan = make(chan interface{}, 1)
  t.can_send = make(chan int, 1)

  // Waiter we use to wait for shutdown
  t.wait.Add(2)

  // Bridge between channels and ZMQ
  go t.bridge(bridge_in)

  // The poller
  go t.poller(bridge_out)

  return nil
}

func (t *TransportZmq) bridge(bridge_in *zmq.Socket) {
  var message interface{}

  // Wait on channel, passing into socket
  // This keeps the socket in a single thread, otherwise we have to lock the entire publisher
  runtime.LockOSThread()

  for {
    select {
    case notify := <- t.bridge_chan:
      bridge_in.Send(notify, 0)

      // Shutdown?
      if string(notify) == zmq_signal_shutdown {
        break
      }
    case message = <-t.recv_bridge_chan:
    case func () chan<- interface{} {
      if message != nil {
        return t.recv_chan
      }
      return nil
    }() <- message:
      // The reason we flush recv through the bridge and not directly to recv_chan is so that if
      // the poller was quick and had to cache a receive as the channel was full, it will stop
      // polling - flushing through bridge allows us to signal poller to start polling again
      // It is not the publisher's responsibility to do this, and TLS wouldn't need it
      bridge_in.Send([]byte(zmq_signal_input), 0)
      message = nil
    }
  }

  // We should linger by default to ensure shutdown is transmitted
  bridge_in.Close()
  runtime.UnlockOSThread()
  t.wait.Done()
}

func (t *TransportZmq) poller(bridge_out *zmq.Socket) {
  var pollitems []zmq.PollItem
  var send_stage *ZMQMessage
  var recv_stage [][]byte
  var recv_body bool

  // ZMQ sockets are not thread-safe, so we have to send/receive on same thread
  // Thus, we cannot use a sender/receiver thread pair like we can with TLS so we use a single threaded poller instead
  // In order to asynchronously send and receive we just poll and do necessary actions

  // When data is ready to send we'll get a channel ping, that is bridged to ZMQ so we can then send data
  // For receiving, we receive here and bridge it to the channels, then receive more once that's through
  runtime.LockOSThread()

  pollitems = make([]zmq.PollItem, 2)
  pollitems[0].Socket = bridge_out
  pollitems[0].Events = zmq.POLLIN | zmq.POLLOUT
  pollitems[1].Socket = t.dealer
  pollitems[1].Events = zmq.POLLIN | zmq.POLLOUT

PollLoop:
  for {
    // Poll for events
    if _, err := zmq.Poll(pollitems, -1); err != nil {
      // Retry on EINTR
      if err == syscall.EINTR {
        continue
      }

      // Failure
      t.recv_chan <- errors.New(fmt.Sprintf("zmq.Poll failure %s", err))
      break
    }

    // Process control channel
    if pollitems[0].REvents & zmq.POLLIN != 0 {
    RetryControl:
      msg, err := bridge_out.Recv(zmq.DONTWAIT)
      if err != nil {
        switch err {
        case syscall.EINTR:
          // Try again
          goto RetryControl
        case syscall.EAGAIN:
          // Poll lied, poll again
          continue
        }

        // Failure
        t.recv_chan <- errors.New(fmt.Sprintf("Pull zmq.Socket.Recv failure %s", err))
        break
      }

      switch string(msg) {
      case zmq_signal_output:
        // Start polling for send
        pollitems[1].Events = pollitems[1].Events | zmq.POLLOUT
      case zmq_signal_input:
        // If we staged a receive, process that
        if recv_stage != nil {
          select {
          case t.recv_bridge_chan <- recv_stage:
            recv_stage = nil

            // Start polling for receive
            pollitems[1].Events = pollitems[1].Events | zmq.POLLIN
          default:
            // Do nothing, we were asked for receive but channel is already full
          }
        } else {
          // Start polling for receive
          pollitems[1].Events = pollitems[1].Events | zmq.POLLIN
        }
      case zmq_signal_shutdown:
        // Shutdown
        break PollLoop
      }
    }

    // Process dealer send
    if pollitems[1].REvents & zmq.POLLOUT != 0 {
      sent_one := false

      // Something in the staging buffer?
      if send_stage != nil {
        var err error
      RetrySendStage:
        if send_stage.final {
          err = t.dealer.Send(send_stage.part, zmq.DONTWAIT)
        } else {
          err = t.dealer.Send(send_stage.part, zmq.DONTWAIT | zmq.SNDMORE)
        }
        if err != nil {
          switch err {
          case syscall.EINTR:
            // Try again
            goto RetrySendStage
          case syscall.EAGAIN:
            // Poll lied, poll again
            continue
          }

          // Failure
          t.recv_chan <- errors.New(fmt.Sprintf("Dealer zmq.Socket.Send failure %s", err))
          break
        }

        sent_one = true
      }

      // Send messages from channel
    LoopSend:
      for {
        select {
        case msg := <-t.send_chan:
          var err error
        RetrySend:
          if msg.final {
            err = t.dealer.Send(msg.part, zmq.DONTWAIT)
          } else {
            err = t.dealer.Send(msg.part, zmq.DONTWAIT | zmq.SNDMORE)
          }
          if err != nil {
            switch err {
            case syscall.EINTR:
              // Try again
              goto RetrySend
            case syscall.EAGAIN:
              // Poll lied, poll again after we check others
              send_stage = msg
              goto PollRecv
            }

            // Failure
            t.recv_chan <- errors.New(fmt.Sprintf("Dealer zmq.Socket.Send failure %s", err))
            break PollLoop
          }

          sent_one = true
        default:
          break LoopSend
        }
      }

      if sent_one {
        // We just sent something, check POLLOUT still active before signalling we can send more
        // TODO: Check why Events() is returning uint64 instead of PollEvents
        // TODO: This is broken and actually returns an error
        if events, _ := t.dealer.Events(); zmq.PollEvents(events) & zmq.POLLOUT != 0 {
          t.setChan(t.can_send)
        }
      } else {
        t.setChan(t.can_send)
      }

      pollitems[1].Events = pollitems[1].Events ^ zmq.POLLOUT
    }

    // Process dealer receive
  PollRecv:
    if pollitems[1].REvents & zmq.POLLIN != 0 {
    LoopRecv:
      for {
        // Bring in the messages
      RetryRecv:
        data, err := t.dealer.Recv(zmq.DONTWAIT)
        if err != nil {
          switch err {
          case syscall.EINTR:
            // Try again
            goto RetryRecv
          case syscall.EAGAIN:
            // Poll lied, poll again
            continue PollLoop
          }

          // Failure
          t.recv_chan <- errors.New(fmt.Sprintf("Dealer zmq.Socket.Recv failure %s", err))
          break PollLoop
        }

        more, err := t.dealer.RcvMore()
        if err != nil {
          // Failure
          t.recv_chan <- errors.New(fmt.Sprintf("Dealer zmq.Socket.RcvMore failure %s", err))
          break PollLoop
        }

        // Sanity check, and don't save until empty message
        if len(data) == 0 && more {
          // Message separator, start returning
          recv_body = true
          continue
        } else if more {
          // Ignore all but last message
        } else if recv_body {
          recv_body = false

          // Last message and receiving, validate it first
          if len(data) < 8 {
            log.Printf("Skipping invalid message: not enough data")
          } else {
            length := binary.BigEndian.Uint32(data[4:8])
            if length > 1048576 {
              log.Printf("Skipping invalid message: data too large (%d)", length)
            } else if length != uint32(len(data)) - 8 {
              log.Printf("Skipping invalid message: data has invalid length (%d != %d)", len(data) - 8, length)
            } else {
              message := [][]byte{data[0:4], data[8:]}

              // Bridge to channels
              select {
              case t.recv_bridge_chan <- message:
              default:
                // We filled the channel, stop polling until we pull something off of it and stage the recv
                recv_stage = message
                pollitems[1].Events = pollitems[1].Events ^ zmq.POLLIN
                break LoopRecv
              }
            }
          }
        } // Hmmm, discard anything else
      }
    }
  }
  // TODO: Can we tidy the nesting above? :/

  bridge_out.Close()
  runtime.UnlockOSThread()
  t.wait.Done()
}

func (t *TransportZmq) setChan(set chan int) {
  select {
  case set <- 1:
  default:
  }
}

func (t *TransportZmq) CanSend() <-chan int {
  return t.can_send
}

func (t *TransportZmq) Write(signature string, message []byte) (err error) {
  var write_buffer *bytes.Buffer
  write_buffer = bytes.NewBuffer(make([]byte, 0, len(signature) + 4 + len(message)))

  if _, err = write_buffer.Write([]byte(signature)); err != nil {
    return
  }
  if err = binary.Write(write_buffer, binary.BigEndian, uint32(len(message))); err != nil {
    return
  }
  if len(message) != 0 {
    if _, err = write_buffer.Write(message); err != nil {
      return
    }
  }

  t.send_chan <- &ZMQMessage{part: []byte(""), final: false}
  t.send_chan <- &ZMQMessage{part: write_buffer.Bytes(), final: true}

  // Ask for send to start
  t.bridge_chan <- []byte(zmq_signal_output)
  return nil
}

func (t *TransportZmq) Read() <-chan interface{} {
  return t.recv_chan
}

func (t *TransportZmq) Disconnect() {
  // Send shutdown request
  t.bridge_chan <- []byte(zmq_signal_shutdown)
  t.wait.Wait()
  t.dealer.Close()
}
<|endoftext|>


<commit_before>// Use of this source code is governed by a BSD-style license
// which can be found in the LICENSE file
// Package sliceDiff provides functions to determine the difference
// between two slices. There are tests and some benchmark utilities
// THIS IS INTENDED ONLY FOR "SMALL" SLICES. Use on slices greater than
// ~ 30k entries should not be used.
package sliceDiff

// Int64SliceDiff returns the int64 values that are not in
// both source slices
func Int64SliceDiff(sliceOne []int64, sliceTwo []int64) []int64 {
	var diff []int64
	for i := 0; i < 2; i++ {
		for _, s1 := range sliceOne {
			found := false
			for _, s2 := range sliceTwo {
				if s1 == s2 {
					found = true
					break
				}
			}
			if !found {
				diff = append(diff,s1)
			}
		}
		if i == 0 {
			sliceOne, sliceTwo = sliceTwo, sliceOne
		}
	}
	return diff
}

// Int32SliceDiff returns the int32 values that are not in
// both soruce slices
func Int32SliceDiff(sliceOne []int32, sliceTwo []int32) []int32 {
	var diff []int32
	for i := 0; i < 2; i++ {
		for _, s1 := range sliceOne {
			found := false
			for _, s2 := range sliceTwo {
				if s1 == s2 {
					found = true
					break
				}
			}
			if !found {
				diff = append(diff,s1)
			}
		}
		if i == 0 {
			sliceOne, sliceTwo = sliceTwo, sliceOne
		}
	}
	return diff
}

// StringSliceDiff returns the string values that are not in
// both source slices
func StringSliceDiff(sliceOne []string, sliceTwo []string) []string {
	var diff []string
	for i := 0; i < 2; i++ {
		for _, s1 := range sliceOne {
			found := false
			for _, s2 := range sliceTwo {
				if s1 == s2 {
					found = true
					break
				}
			}
			if !found {
				diff = append(diff,s1)
			}
		}
		if i == 0 {
			sliceOne, sliceTwo = sliceTwo, sliceOne
		}
	}
	return diff
}
<commit_msg>Fixed Package Documentation per go-lint<commit_after>// Package sliceDiff provides functions to determine the difference
// between two slices. There are tests and some benchmark utilities
// THIS IS INTENDED ONLY FOR "SMALL" SLICES. Use on slices greater than
// ~ 30k entries should not be used.
// Use of this source code is governed by a BSD-style license
// which can be found in the LICENSE file
package sliceDiff

// Int64SliceDiff returns the int64 values that are not in
// both source slices
func Int64SliceDiff(sliceOne []int64, sliceTwo []int64) []int64 {
	var diff []int64
	for i := 0; i < 2; i++ {
		for _, s1 := range sliceOne {
			found := false
			for _, s2 := range sliceTwo {
				if s1 == s2 {
					found = true
					break
				}
			}
			if !found {
				diff = append(diff,s1)
			}
		}
		if i == 0 {
			sliceOne, sliceTwo = sliceTwo, sliceOne
		}
	}
	return diff
}

// Int32SliceDiff returns the int32 values that are not in
// both soruce slices
func Int32SliceDiff(sliceOne []int32, sliceTwo []int32) []int32 {
	var diff []int32
	for i := 0; i < 2; i++ {
		for _, s1 := range sliceOne {
			found := false
			for _, s2 := range sliceTwo {
				if s1 == s2 {
					found = true
					break
				}
			}
			if !found {
				diff = append(diff,s1)
			}
		}
		if i == 0 {
			sliceOne, sliceTwo = sliceTwo, sliceOne
		}
	}
	return diff
}

// StringSliceDiff returns the string values that are not in
// both source slices
func StringSliceDiff(sliceOne []string, sliceTwo []string) []string {
	var diff []string
	for i := 0; i < 2; i++ {
		for _, s1 := range sliceOne {
			found := false
			for _, s2 := range sliceTwo {
				if s1 == s2 {
					found = true
					break
				}
			}
			if !found {
				diff = append(diff,s1)
			}
		}
		if i == 0 {
			sliceOne, sliceTwo = sliceTwo, sliceOne
		}
	}
	return diff
}
<|endoftext|>


<commit_before>package tunnel

import (
	"os"
	"sync"
	"time"

	"github.com/micro/cli"
	"github.com/micro/go-micro"
	"github.com/micro/go-micro/client"
	"github.com/micro/go-micro/config/options"
	"github.com/micro/go-micro/proxy/mucp"
	"github.com/micro/go-micro/router"
	"github.com/micro/go-micro/server"
	tun "github.com/micro/go-micro/tunnel"
	"github.com/micro/go-micro/tunnel/transport"
	"github.com/micro/go-micro/util/log"
)

var (
	// Name of the router microservice
	Name = "go.micro.tunnel"
	// Address is the tunnel microservice bind address
	Address = ":8084"
	// Tunnel is the tunnel bind address
	Tunnel = ":9096"
	// Channel is the name of the tunnel session
	Channel = "tunnel"
	// Router is the router gossip bind address
	Router = ":9093"
	// Network is the network id
	Network = "local"
)

// run runs the micro server
func run(ctx *cli.Context, srvOpts ...micro.Option) {
	// Init plugins
	for _, p := range Plugins() {
		p.Init(ctx)
	}

	if len(ctx.GlobalString("server_name")) > 0 {
		Name = ctx.GlobalString("server_name")
	}
	if len(ctx.String("address")) > 0 {
		Address = ctx.String("address")
	}
	if len(ctx.String("network_address")) > 0 {
		Network = ctx.String("network")
	}
	if len(ctx.String("tunnel_address")) > 0 {
		Tunnel = ctx.String("tunnel")
	}
	// default gateway address
	var gateway string
	if len(ctx.String("gateway_address")) > 0 {
		gateway = ctx.String("gateway")
	}

	// create a tunnel
	t := tun.NewTunnel(
		tun.Address(Tunnel),
	)

	// connect the tunnel
	if err := t.Connect(); err != nil {
		log.Logf("[tunnel] failed to connect: %s", err)
		os.Exit(1)
	}

	// listen on tunnel
	l, err := t.Listen(Channel)
	if err != nil {
		log.Logf("[tunnel] failed to listen: %s", err)
		os.Exit(1)
	}

	// TODO: go accept tunnel connections

	// Initialise service
	service := micro.NewService(
		micro.Name(Name),
		micro.Address(Address),
		micro.RegisterTTL(time.Duration(ctx.GlobalInt("register_ttl"))*time.Second),
		micro.RegisterInterval(time.Duration(ctx.GlobalInt("register_interval"))*time.Second),
	)

	// local tunnel router
	r := router.NewRouter(
		router.Address(Router),
		router.Network(Network),
		router.Registry(service.Client().Options().Registry),
		router.Gateway(gateway),
	)

	// create tunnel client with tunnel transport
	tr := transport.NewTransport(
		transport.WithTunnel(t),
	)
	c := client.NewClient(
		client.Transport(tr),
	)

	// local proxy
	localProxy := mucp.NewProxy(
		options.WithValue("proxy.router", r),
		options.WithValue("proxy.client", c),
	)

	// local server
	localSrv := server.NewServer(
		server.WithRouter(localProxy),
	)

	// init server
	service.Init(
		micro.Server(localSrv),
	)

	var wg sync.WaitGroup

	errChan := make(chan error, 1)

	wg.Add(1)
	go func() {
		defer wg.Done()
		errChan <- service.Run()
	}()

	// we block here until either service or server fails
	if err := <-errChan; err != nil {
		log.Logf("[tunnel] error running the tunnel: %v", err)
	}

	log.Log("[tunnel] attempting to stop the tunnel")

	// TODO: stop the router etc.

	wg.Wait()

	log.Logf("[tunnel] successfully stopped")

}

func Commands(options ...micro.Option) []cli.Command {
	command := cli.Command{
		Name:  "tunnel",
		Usage: "Run the micro network tunnel",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:   "tunnel_address",
				Usage:  "Set the micro tunnel address :9096",
				EnvVar: "MICRO_TUNNEL_ADDRESS",
			},
			cli.StringFlag{
				Name:   "network_address",
				Usage:  "Set the micro network address: local",
				EnvVar: "MICRO_NETWORK_ADDRESS",
			},
			cli.StringFlag{
				Name:   "gateway_address",
				Usage:  "Set the micro default gateway address :9094",
				EnvVar: "MICRO_GATEWAY_ADDRESS",
			},
		},
		Action: func(ctx *cli.Context) {
			run(ctx, options...)
		},
	}

	for _, p := range Plugins() {
		if cmds := p.Commands(); len(cmds) > 0 {
			command.Subcommands = append(command.Subcommands, cmds...)
		}

		if flags := p.Flags(); len(flags) > 0 {
			command.Flags = append(command.Flags, flags...)
		}
	}

	return []cli.Command{command}
}
<commit_msg>Added tunnel listener.<commit_after>package tunnel

import (
	"os"
	"sync"
	"time"

	"github.com/micro/cli"
	"github.com/micro/go-micro"
	"github.com/micro/go-micro/client"
	"github.com/micro/go-micro/config/options"
	"github.com/micro/go-micro/proxy/mucp"
	"github.com/micro/go-micro/router"
	"github.com/micro/go-micro/server"
	trn "github.com/micro/go-micro/transport"
	tun "github.com/micro/go-micro/tunnel"
	"github.com/micro/go-micro/tunnel/transport"
	"github.com/micro/go-micro/util/log"
)

var (
	// Name of the router microservice
	Name = "go.micro.tunnel"
	// Address is the tunnel microservice bind address
	Address = ":8084"
	// Tunnel is the tunnel bind address
	Tunnel = ":9096"
	// Channel is the name of the tunnel session
	Channel = "tunnel"
	// Router is the router gossip bind address
	Router = ":9093"
	// Network is the network id
	Network = "local"
)

func accept(l tun.Listener, exit chan struct{}) error {
	for {
		// accept a connection
		c, err := l.Accept()
		if err != nil {
			return err
		}

		select {
		case <-exit:
			return nil
		default:
			m := new(trn.Message)
			if err := c.Recv(m); err != nil {
				return err
			}
			// TODO: do something with the message
		}
	}
}

// run runs the micro server
func run(ctx *cli.Context, srvOpts ...micro.Option) {
	// Init plugins
	for _, p := range Plugins() {
		p.Init(ctx)
	}

	if len(ctx.GlobalString("server_name")) > 0 {
		Name = ctx.GlobalString("server_name")
	}
	if len(ctx.String("address")) > 0 {
		Address = ctx.String("address")
	}
	if len(ctx.String("network_address")) > 0 {
		Network = ctx.String("network")
	}
	if len(ctx.String("tunnel_address")) > 0 {
		Tunnel = ctx.String("tunnel")
	}
	// default gateway address
	var gateway string
	if len(ctx.String("gateway_address")) > 0 {
		gateway = ctx.String("gateway")
	}

	// create a tunnel
	t := tun.NewTunnel(
		tun.Address(Tunnel),
	)

	// connect the tunnel
	if err := t.Connect(); err != nil {
		log.Logf("[tunnel] failed to connect: %s", err)
		os.Exit(1)
	}

	// listen on tunnel
	l, err := t.Listen(Channel)
	if err != nil {
		log.Logf("[tunnel] failed to listen: %s", err)
		os.Exit(1)
	}

	// TODO: go accept tunnel connections

	// Initialise service
	service := micro.NewService(
		micro.Name(Name),
		micro.Address(Address),
		micro.RegisterTTL(time.Duration(ctx.GlobalInt("register_ttl"))*time.Second),
		micro.RegisterInterval(time.Duration(ctx.GlobalInt("register_interval"))*time.Second),
	)

	// local tunnel router
	r := router.NewRouter(
		router.Id(service.Server().Options().Id),
		router.Registry(service.Client().Options().Registry),
		router.Address(Router),
		router.Network(Network),
		router.Gateway(gateway),
	)

	// create tunnel client with tunnel transport
	tr := transport.NewTransport(
		transport.WithTunnel(t),
	)
	c := client.NewClient(
		client.Transport(tr),
	)

	// local proxy
	localProxy := mucp.NewProxy(
		options.WithValue("proxy.router", r),
		options.WithValue("proxy.client", c),
	)

	// local server
	localSrv := server.NewServer(
		server.WithRouter(localProxy),
	)

	// init server
	service.Init(
		micro.Server(localSrv),
	)

	var wg sync.WaitGroup

	// error channel to collect errors and bail
	errChan := make(chan error, 2)

	// exit channelt o exit tunnel listener
	exit := make(chan struct{})

	// accept new tunnel connections
	wg.Add(1)
	go func() {
		defer wg.Done()
		errChan <- accept(l, exit)
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		errChan <- service.Run()
	}()

	// we block here until either service or server fails
	if err := <-errChan; err != nil {
		log.Logf("[tunnel] error running the tunnel: %v", err)
	}

	log.Log("[tunnel] attempting to stop the tunnel")

	// close tunnel listener
	close(exit)

	if err := l.Close(); err != nil {
		log.Logf("[tunnel] error closing the tunnel: %v", err)
	}

	// stop the router
	if err := r.Stop(); err != nil {
		log.Logf("[tunnel] error stopping tunnel router:%v", err)
	}

	wg.Wait()

	log.Logf("[tunnel] successfully stopped")
}

func Commands(options ...micro.Option) []cli.Command {
	command := cli.Command{
		Name:  "tunnel",
		Usage: "Run the micro network tunnel",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:   "tunnel_address",
				Usage:  "Set the micro tunnel address :9096",
				EnvVar: "MICRO_TUNNEL_ADDRESS",
			},
			cli.StringFlag{
				Name:   "network_address",
				Usage:  "Set the micro network address: local",
				EnvVar: "MICRO_NETWORK_ADDRESS",
			},
			cli.StringFlag{
				Name:   "gateway_address",
				Usage:  "Set the micro default gateway address :9094",
				EnvVar: "MICRO_GATEWAY_ADDRESS",
			},
		},
		Action: func(ctx *cli.Context) {
			run(ctx, options...)
		},
	}

	for _, p := range Plugins() {
		if cmds := p.Commands(); len(cmds) > 0 {
			command.Subcommands = append(command.Subcommands, cmds...)
		}

		if flags := p.Flags(); len(flags) > 0 {
			command.Flags = append(command.Flags, flags...)
		}
	}

	return []cli.Command{command}
}
<|endoftext|>


<commit_before>// Package snowflake provides a very simple Twitter snowflake generator and parser.
package snowflake

import (
	"encoding/base64"
	"encoding/binary"
	"errors"
	"fmt"
	"strconv"
	"sync"
	"time"
)

var (
	// Epoch is set to the twitter snowflake epoch of Nov 04 2010 01:42:54 UTC
	// You may customize this to set a different epoch for your application.
	Epoch int64 = 1288834974657

	// Number of bits to use for Node
	// Remember, you have a total 22 bits to share between Node/Step
	NodeBits uint8 = 10

	// Number of bits to use for Step
	// Remember, you have a total 22 bits to share between Node/Step
	StepBits uint8 = 12

	nodeMax   int64 = -1 ^ (-1 << NodeBits)
	nodeMask  int64 = nodeMax << StepBits
	stepMask  int64 = -1 ^ (-1 << StepBits)
	timeShift uint8 = NodeBits + StepBits
	nodeShift uint8 = StepBits
)

const encodeBase32Map = "ybndrfg8ejkmcpqxot1uwisza345h769"

var decodeBase32Map [256]byte

const encodeBase58Map = "123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ"

var decodeBase58Map [256]byte

// A JSONSyntaxError is returned from UnmarshalJSON if an invalid ID is provided.
type JSONSyntaxError struct{ original []byte }

func (j JSONSyntaxError) Error() string {
	return fmt.Sprintf("invalid snowflake ID %q", string(j.original))
}

// Create a map for decoding Base58.  This speeds up the process tremendously.
func init() {

	for i := 0; i < len(encodeBase58Map); i++ {
		decodeBase58Map[i] = 0xFF
	}

	for i := 0; i < len(encodeBase58Map); i++ {
		decodeBase58Map[encodeBase58Map[i]] = byte(i)
	}

	for i := 0; i < len(encodeBase32Map); i++ {
		decodeBase32Map[i] = 0xFF
	}

	for i := 0; i < len(encodeBase32Map); i++ {
		decodeBase32Map[encodeBase32Map[i]] = byte(i)
	}
}

// ErrInvalidBase58 is returned by ParseBase58 when given an invalid []byte
var ErrInvalidBase58 = errors.New("invalid base58")

// ErrInvalidBase32 is returned by ParseBase32 when given an invalid []byte
var ErrInvalidBase32 = errors.New("invalid base32")

// A Node struct holds the basic information needed for a snowflake generator
// node
type Node struct {
	mu   sync.Mutex
	time int64
	node int64
	step int64
}

// An ID is a custom type used for a snowflake ID.  This is used so we can
// attach methods onto the ID.
type ID int64

// NewNode returns a new snowflake node that can be used to generate snowflake
// IDs
func NewNode(node int64) (*Node, error) {

	// re-calc in case custom NodeBits or StepBits were set
	nodeMax = -1 ^ (-1 << NodeBits)
	nodeMask = nodeMax << StepBits
	stepMask = -1 ^ (-1 << StepBits)
	timeShift = NodeBits + StepBits
	nodeShift = StepBits

	if node < 0 || node > nodeMax {
		return nil, errors.New("Node number must be between 0 and " + strconv.Itoa(nodeMax))
	}

	return &Node{
		time: 0,
		node: node,
		step: 0,
	}, nil
}

// Generate creates and returns a unique snowflake ID
func (n *Node) Generate() ID {

	n.mu.Lock()

	now := time.Now().UnixNano() / 1000000

	if n.time == now {
		n.step = (n.step + 1) & stepMask

		if n.step == 0 {
			for now <= n.time {
				now = time.Now().UnixNano() / 1000000
			}
		}
	} else {
		n.step = 0
	}

	n.time = now

	r := ID((now-Epoch)<<timeShift |
		(n.node << nodeShift) |
		(n.step),
	)

	n.mu.Unlock()
	return r
}

// Int64 returns an int64 of the snowflake ID
func (f ID) Int64() int64 {
	return int64(f)
}

// String returns a string of the snowflake ID
func (f ID) String() string {
	return strconv.FormatInt(int64(f), 10)
}

// Base2 returns a string base2 of the snowflake ID
func (f ID) Base2() string {
	return strconv.FormatInt(int64(f), 2)
}

// Base36 returns a base36 string of the snowflake ID
func (f ID) Base36() string {
	return strconv.FormatInt(int64(f), 36)
}

// Base32 uses the z-base-32 character set but encodes and decodes similar
// to base58, allowing it to create an even smaller result string.
// NOTE: There are many different base32 implementations so becareful when
// doing any interoperation interop with other packages.
func (f ID) Base32() string {

	if f < 32 {
		return string(encodeBase32Map[f])
	}

	b := make([]byte, 0, 12)
	for f >= 32 {
		b = append(b, encodeBase32Map[f%32])
		f /= 32
	}
	b = append(b, encodeBase32Map[f])

	for x, y := 0, len(b)-1; x < y; x, y = x+1, y-1 {
		b[x], b[y] = b[y], b[x]
	}

	return string(b)
}

// ParseBase32 parses a base32 []byte into a snowflake ID
// NOTE: There are many different base32 implementations so becareful when
// doing any interoperation interop with other packages.
func ParseBase32(b []byte) (ID, error) {

	var id int64

	for i := range b {
		if decodeBase32Map[b[i]] == 0xFF {
			return -1, ErrInvalidBase32
		}
		id = id*32 + int64(decodeBase32Map[b[i]])
	}

	return ID(id), nil
}

// Base58 returns a base58 string of the snowflake ID
func (f ID) Base58() string {

	if f < 58 {
		return string(encodeBase58Map[f])
	}

	b := make([]byte, 0, 11)
	for f >= 58 {
		b = append(b, encodeBase58Map[f%58])
		f /= 58
	}
	b = append(b, encodeBase58Map[f])

	for x, y := 0, len(b)-1; x < y; x, y = x+1, y-1 {
		b[x], b[y] = b[y], b[x]
	}

	return string(b)
}

// ParseBase58 parses a base58 []byte into a snowflake ID
func ParseBase58(b []byte) (ID, error) {

	var id int64

	for i := range b {
		if decodeBase58Map[b[i]] == 0xFF {
			return -1, ErrInvalidBase58
		}
		id = id*58 + int64(decodeBase58Map[b[i]])
	}

	return ID(id), nil
}

// Base64 returns a base64 string of the snowflake ID
func (f ID) Base64() string {
	return base64.StdEncoding.EncodeToString(f.Bytes())
}

// Bytes returns a byte slice of the snowflake ID
func (f ID) Bytes() []byte {
	return []byte(f.String())
}

// IntBytes returns an array of bytes of the snowflake ID, encoded as a
// big endian integer.
func (f ID) IntBytes() [8]byte {
	var b [8]byte
	binary.BigEndian.PutUint64(b[:], uint64(f))
	return b
}

// Time returns an int64 unix timestamp of the snowflake ID time
func (f ID) Time() int64 {
	return (int64(f) >> timeShift) + Epoch
}

// Node returns an int64 of the snowflake ID node number
func (f ID) Node() int64 {
	return int64(f) & nodeMask >> nodeShift
}

// Step returns an int64 of the snowflake step (or sequence) number
func (f ID) Step() int64 {
	return int64(f) & stepMask
}

// MarshalJSON returns a json byte array string of the snowflake ID.
func (f ID) MarshalJSON() ([]byte, error) {
	buff := make([]byte, 0, 22)
	buff = append(buff, '"')
	buff = strconv.AppendInt(buff, int64(f), 10)
	buff = append(buff, '"')
	return buff, nil
}

// UnmarshalJSON converts a json byte array of a snowflake ID into an ID type.
func (f *ID) UnmarshalJSON(b []byte) error {
	if len(b) < 3 || b[0] != '"' || b[len(b)-1] != '"' {
		return JSONSyntaxError{b}
	}

	i, err := strconv.ParseInt(string(b[1:len(b)-1]), 10, 64)
	if err != nil {
		return err
	}

	*f = ID(i)
	return nil
}
<commit_msg>Int64 to string.<commit_after>// Package snowflake provides a very simple Twitter snowflake generator and parser.
package snowflake

import (
	"encoding/base64"
	"encoding/binary"
	"errors"
	"fmt"
	"strconv"
	"sync"
	"time"
)

var (
	// Epoch is set to the twitter snowflake epoch of Nov 04 2010 01:42:54 UTC
	// You may customize this to set a different epoch for your application.
	Epoch int64 = 1288834974657

	// Number of bits to use for Node
	// Remember, you have a total 22 bits to share between Node/Step
	NodeBits uint8 = 10

	// Number of bits to use for Step
	// Remember, you have a total 22 bits to share between Node/Step
	StepBits uint8 = 12

	nodeMax   int64 = -1 ^ (-1 << NodeBits)
	nodeMask  int64 = nodeMax << StepBits
	stepMask  int64 = -1 ^ (-1 << StepBits)
	timeShift uint8 = NodeBits + StepBits
	nodeShift uint8 = StepBits
)

const encodeBase32Map = "ybndrfg8ejkmcpqxot1uwisza345h769"

var decodeBase32Map [256]byte

const encodeBase58Map = "123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ"

var decodeBase58Map [256]byte

// A JSONSyntaxError is returned from UnmarshalJSON if an invalid ID is provided.
type JSONSyntaxError struct{ original []byte }

func (j JSONSyntaxError) Error() string {
	return fmt.Sprintf("invalid snowflake ID %q", string(j.original))
}

// Create a map for decoding Base58.  This speeds up the process tremendously.
func init() {

	for i := 0; i < len(encodeBase58Map); i++ {
		decodeBase58Map[i] = 0xFF
	}

	for i := 0; i < len(encodeBase58Map); i++ {
		decodeBase58Map[encodeBase58Map[i]] = byte(i)
	}

	for i := 0; i < len(encodeBase32Map); i++ {
		decodeBase32Map[i] = 0xFF
	}

	for i := 0; i < len(encodeBase32Map); i++ {
		decodeBase32Map[encodeBase32Map[i]] = byte(i)
	}
}

// ErrInvalidBase58 is returned by ParseBase58 when given an invalid []byte
var ErrInvalidBase58 = errors.New("invalid base58")

// ErrInvalidBase32 is returned by ParseBase32 when given an invalid []byte
var ErrInvalidBase32 = errors.New("invalid base32")

// A Node struct holds the basic information needed for a snowflake generator
// node
type Node struct {
	mu   sync.Mutex
	time int64
	node int64
	step int64
}

// An ID is a custom type used for a snowflake ID.  This is used so we can
// attach methods onto the ID.
type ID int64

// NewNode returns a new snowflake node that can be used to generate snowflake
// IDs
func NewNode(node int64) (*Node, error) {

	// re-calc in case custom NodeBits or StepBits were set
	nodeMax = -1 ^ (-1 << NodeBits)
	nodeMask = nodeMax << StepBits
	stepMask = -1 ^ (-1 << StepBits)
	timeShift = NodeBits + StepBits
	nodeShift = StepBits

	if node < 0 || node > nodeMax {
		return nil, errors.New("Node number must be between 0 and " + strconv.FormatInt(nodeMax, 10))
	}

	return &Node{
		time: 0,
		node: node,
		step: 0,
	}, nil
}

// Generate creates and returns a unique snowflake ID
func (n *Node) Generate() ID {

	n.mu.Lock()

	now := time.Now().UnixNano() / 1000000

	if n.time == now {
		n.step = (n.step + 1) & stepMask

		if n.step == 0 {
			for now <= n.time {
				now = time.Now().UnixNano() / 1000000
			}
		}
	} else {
		n.step = 0
	}

	n.time = now

	r := ID((now-Epoch)<<timeShift |
		(n.node << nodeShift) |
		(n.step),
	)

	n.mu.Unlock()
	return r
}

// Int64 returns an int64 of the snowflake ID
func (f ID) Int64() int64 {
	return int64(f)
}

// String returns a string of the snowflake ID
func (f ID) String() string {
	return strconv.FormatInt(int64(f), 10)
}

// Base2 returns a string base2 of the snowflake ID
func (f ID) Base2() string {
	return strconv.FormatInt(int64(f), 2)
}

// Base36 returns a base36 string of the snowflake ID
func (f ID) Base36() string {
	return strconv.FormatInt(int64(f), 36)
}

// Base32 uses the z-base-32 character set but encodes and decodes similar
// to base58, allowing it to create an even smaller result string.
// NOTE: There are many different base32 implementations so becareful when
// doing any interoperation interop with other packages.
func (f ID) Base32() string {

	if f < 32 {
		return string(encodeBase32Map[f])
	}

	b := make([]byte, 0, 12)
	for f >= 32 {
		b = append(b, encodeBase32Map[f%32])
		f /= 32
	}
	b = append(b, encodeBase32Map[f])

	for x, y := 0, len(b)-1; x < y; x, y = x+1, y-1 {
		b[x], b[y] = b[y], b[x]
	}

	return string(b)
}

// ParseBase32 parses a base32 []byte into a snowflake ID
// NOTE: There are many different base32 implementations so becareful when
// doing any interoperation interop with other packages.
func ParseBase32(b []byte) (ID, error) {

	var id int64

	for i := range b {
		if decodeBase32Map[b[i]] == 0xFF {
			return -1, ErrInvalidBase32
		}
		id = id*32 + int64(decodeBase32Map[b[i]])
	}

	return ID(id), nil
}

// Base58 returns a base58 string of the snowflake ID
func (f ID) Base58() string {

	if f < 58 {
		return string(encodeBase58Map[f])
	}

	b := make([]byte, 0, 11)
	for f >= 58 {
		b = append(b, encodeBase58Map[f%58])
		f /= 58
	}
	b = append(b, encodeBase58Map[f])

	for x, y := 0, len(b)-1; x < y; x, y = x+1, y-1 {
		b[x], b[y] = b[y], b[x]
	}

	return string(b)
}

// ParseBase58 parses a base58 []byte into a snowflake ID
func ParseBase58(b []byte) (ID, error) {

	var id int64

	for i := range b {
		if decodeBase58Map[b[i]] == 0xFF {
			return -1, ErrInvalidBase58
		}
		id = id*58 + int64(decodeBase58Map[b[i]])
	}

	return ID(id), nil
}

// Base64 returns a base64 string of the snowflake ID
func (f ID) Base64() string {
	return base64.StdEncoding.EncodeToString(f.Bytes())
}

// Bytes returns a byte slice of the snowflake ID
func (f ID) Bytes() []byte {
	return []byte(f.String())
}

// IntBytes returns an array of bytes of the snowflake ID, encoded as a
// big endian integer.
func (f ID) IntBytes() [8]byte {
	var b [8]byte
	binary.BigEndian.PutUint64(b[:], uint64(f))
	return b
}

// Time returns an int64 unix timestamp of the snowflake ID time
func (f ID) Time() int64 {
	return (int64(f) >> timeShift) + Epoch
}

// Node returns an int64 of the snowflake ID node number
func (f ID) Node() int64 {
	return int64(f) & nodeMask >> nodeShift
}

// Step returns an int64 of the snowflake step (or sequence) number
func (f ID) Step() int64 {
	return int64(f) & stepMask
}

// MarshalJSON returns a json byte array string of the snowflake ID.
func (f ID) MarshalJSON() ([]byte, error) {
	buff := make([]byte, 0, 22)
	buff = append(buff, '"')
	buff = strconv.AppendInt(buff, int64(f), 10)
	buff = append(buff, '"')
	return buff, nil
}

// UnmarshalJSON converts a json byte array of a snowflake ID into an ID type.
func (f *ID) UnmarshalJSON(b []byte) error {
	if len(b) < 3 || b[0] != '"' || b[len(b)-1] != '"' {
		return JSONSyntaxError{b}
	}

	i, err := strconv.ParseInt(string(b[1:len(b)-1]), 10, 64)
	if err != nil {
		return err
	}

	*f = ID(i)
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"io"
	"log"
	"os"
	"time"
)

type event struct {
	When     string
	Interval string
	What     string
	On       string
	ExceptOn string
	Days     map[string]string
	time     time.Time
}

type scheduler struct {
	Hue      hue
	Schedule []event
	c        chan event
}

func NewSchedulerFromJSON(j io.Reader) (err error, s *scheduler) {
	s = &scheduler{}
	dec := json.NewDecoder(j)
	if err = dec.Decode(s); err == nil {
		for _, item := range s.Schedule {
			if err = s.schedule(item); err != nil {
				return err, nil
			}
		}
	}
	s.c = make(chan event, 1)
	return err, s
}

func (s *scheduler) schedule(e event) (err error) {
	var on time.Time
	var duration time.Duration
	now := time.Now()
	zone, _ := now.Zone()
	if on, err = time.Parse("2006-01-02 "+time.Kitchen+" MST", now.Format("2006-01-02 ")+e.When+" "+zone); err != nil {
		log.Println("could not parse when of '" + e.When + "' for " + e.What)
		return
	}
	if duration, err = time.ParseDuration(e.Interval); err != nil {
		log.Println("could not parse interval of '" + e.Interval + "' for " + e.What)
		return
	}

	go func() {
		log.Println("scheduled '" + e.What + "' for: " + on.String())
		wait := time.Duration((on.UnixNano() - time.Now().UnixNano()) % int64(duration))
		if wait < 0 {
			wait += duration
		}
		time.Sleep(wait)
		s.maybeRun(time.Now(), e)
		for t := range time.NewTicker(duration).C {
			s.maybeRun(t, e)
		}
	}()
	return
}

var WEEKDAYS = []string{"Monday", "Tuesday", "Wednesday", "Thursday", "Friday"}

var HOLIDAYS = map[string]string{"Christmas Day": "2012-12-25",	"New Year's Day": "2013-01-01",	"Birthday of Martin Luther King, Jr.": "2013-01-21", "Washington's Birthday": "2013-02-18", "Memorial Day": "2013-05-27", "Independence Day": "2013-07-04", "Labor Day": "2013-09-02", "Columbus Day": "2013-10-14", "Veterans Day": "2013-11-11", "Thanksgiving Day": "2013-11-28", "Christmas Day 2013": "2013-12-25"}

func (s *scheduler) maybeRun(t time.Time, e event) {
	t = t.In(time.Local)
	run := false
	if e.On == "" {
		run = true
	} else if e.On == "weekdays" {
		d := t.Weekday().String()
		for _, wd := range WEEKDAYS  {
			if d == wd {
				run = true
				break
			}
		}
	} else if e.On == "weekends" {
		d := t.Weekday().String()
		for _, wd := range []string{"Saturday", "Sunday"}  {
			if d == wd {
				run = true
				break
			}
		}
	}
	if e.ExceptOn == "" {

	} else if e.ExceptOn == "holidays" {
		s := t.Format("2006-01-02")
		for _, v := range HOLIDAYS  {
			if s == v {
				log.Println("not running due to holiday:", e)
				run = false
				break
			}
		}
	}
	if run {
		log.Println(e.What + " at " + t.String())
		e.time = t
		s.c <- e
	}
}

func (s *scheduler) run() {
	for e := range s.c {
		s.Hue.Do(e.What)
	}
}

func NewSchedulerFromJSONPath(p string) (err error, s *scheduler) {
	if j, err := os.OpenFile(p, os.O_RDONLY, 0666); err == nil {
		defer j.Close()
		err, s = NewSchedulerFromJSON(j)
	}
	return err, s
}
<commit_msg>go fmt<commit_after>package main

import (
	"encoding/json"
	"io"
	"log"
	"os"
	"time"
)

type event struct {
	When     string
	Interval string
	What     string
	On       string
	ExceptOn string
	Days     map[string]string
	time     time.Time
}

type scheduler struct {
	Hue      hue
	Schedule []event
	c        chan event
}

func NewSchedulerFromJSON(j io.Reader) (err error, s *scheduler) {
	s = &scheduler{}
	dec := json.NewDecoder(j)
	if err = dec.Decode(s); err == nil {
		for _, item := range s.Schedule {
			if err = s.schedule(item); err != nil {
				return err, nil
			}
		}
	}
	s.c = make(chan event, 1)
	return err, s
}

func (s *scheduler) schedule(e event) (err error) {
	var on time.Time
	var duration time.Duration
	now := time.Now()
	zone, _ := now.Zone()
	if on, err = time.Parse("2006-01-02 "+time.Kitchen+" MST", now.Format("2006-01-02 ")+e.When+" "+zone); err != nil {
		log.Println("could not parse when of '" + e.When + "' for " + e.What)
		return
	}
	if duration, err = time.ParseDuration(e.Interval); err != nil {
		log.Println("could not parse interval of '" + e.Interval + "' for " + e.What)
		return
	}

	go func() {
		log.Println("scheduled '" + e.What + "' for: " + on.String())
		wait := time.Duration((on.UnixNano() - time.Now().UnixNano()) % int64(duration))
		if wait < 0 {
			wait += duration
		}
		time.Sleep(wait)
		s.maybeRun(time.Now(), e)
		for t := range time.NewTicker(duration).C {
			s.maybeRun(t, e)
		}
	}()
	return
}

var WEEKDAYS = []string{"Monday", "Tuesday", "Wednesday", "Thursday", "Friday"}

var HOLIDAYS = map[string]string{"Christmas Day": "2012-12-25", "New Year's Day": "2013-01-01", "Birthday of Martin Luther King, Jr.": "2013-01-21", "Washington's Birthday": "2013-02-18", "Memorial Day": "2013-05-27", "Independence Day": "2013-07-04", "Labor Day": "2013-09-02", "Columbus Day": "2013-10-14", "Veterans Day": "2013-11-11", "Thanksgiving Day": "2013-11-28", "Christmas Day 2013": "2013-12-25"}

func (s *scheduler) maybeRun(t time.Time, e event) {
	t = t.In(time.Local)
	run := false
	if e.On == "" {
		run = true
	} else if e.On == "weekdays" {
		d := t.Weekday().String()
		for _, wd := range WEEKDAYS {
			if d == wd {
				run = true
				break
			}
		}
	} else if e.On == "weekends" {
		d := t.Weekday().String()
		for _, wd := range []string{"Saturday", "Sunday"} {
			if d == wd {
				run = true
				break
			}
		}
	}
	if e.ExceptOn == "" {

	} else if e.ExceptOn == "holidays" {
		s := t.Format("2006-01-02")
		for _, v := range HOLIDAYS {
			if s == v {
				log.Println("not running due to holiday:", e)
				run = false
				break
			}
		}
	}
	if run {
		log.Println(e.What + " at " + t.String())
		e.time = t
		s.c <- e
	}
}

func (s *scheduler) run() {
	for e := range s.c {
		s.Hue.Do(e.What)
	}
}

func NewSchedulerFromJSONPath(p string) (err error, s *scheduler) {
	if j, err := os.OpenFile(p, os.O_RDONLY, 0666); err == nil {
		defer j.Close()
		err, s = NewSchedulerFromJSON(j)
	}
	return err, s
}
<|endoftext|>


<commit_before>package speeldoos

import (
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"os"
)

type Composer struct {
	Name string
	ID   string `xml:"id,attr,omitempty"`
}

type OpusNumber struct {
	Number    string `xml:",chardata"`
	IndexName string `xml:",attr,omitempty"`
}

func (o OpusNumber) String() string {
	if o.Number == "" {
		return ""
	}
	if o.IndexName == "" {
		return fmt.Sprintf("Op. %s", o.Number)
	} else {
		return fmt.Sprintf("%s %s", o.IndexName, o.Number)
	}
}

type Title struct {
	Title    string `xml:",chardata"`
	Language string `xml:",attr,omitempty"`
}

type Work struct {
	Composer   Composer
	Title      []Title
	OpusNumber []OpusNumber
	Parts      []Part `xml:"Parts>Part,omitempty"`
	Year       int
}

type Part struct {
	Part   string `xml:",chardata"`
	Number string `xml:"number,attr,omitempty"`
}

type Performer struct {
	Name string `xml:",chardata"`
	Role string `xml:"role,attr,omitempty"`
}

type SourceFile struct {
	Filename string `xml:",chardata"`
	Disc     int    `xml:"disc,attr,omitempty"`
}

func (s SourceFile) String() string {
	return s.Filename
}

type Performance struct {
	Work        Work
	Year        int          `xml:",omitempty"`
	Performers  []Performer  `xml:"Performers>Performer"`
	SourceFiles []SourceFile `xml:"SourceFiles>File"`
}

type Carrier struct {
	XMLName      xml.Name `xml:"https://www.inurbanus.nl/NS/speeldoos/1.0 Carrier"`
	Name         string
	ID           string        `xml:",omitempty"`
	Hash         string        `xml:"hash,attr,omitempty"`
	Source       string        `xml:"source,attr,omitempty"`
	Performances []Performance `xml:"Performances>Performance"`
}

func ImportCarrier(filename string) (*Carrier, error) {
	ip, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}

	foo := &Carrier{}
	err = xml.Unmarshal(ip, foo)
	if err != nil {
		return nil, err
	}

	return foo, nil
}

func (c *Carrier) Write(filename string) error {
	op, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer op.Close()

	w := xml.NewEncoder(op)
	w.Indent("", "	")
	err = w.Encode(c)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Documentation spree<commit_after>package speeldoos

import (
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"os"
)

// A Composer represents a composer
type Composer struct {
	Name string

	// A composer's ID is the page ID for their Wikipedia page
	ID string `xml:"id,attr,omitempty"`
}

// Opus numbers, together with the composer, ideally uniquely identify a work
type OpusNumber struct {
	Number string `xml:",chardata"`

	// Some composers use a different index (e.g. 'BWV', 'KV', 'Wq', etc.)
	IndexName string `xml:",attr,omitempty"`
}

func (o OpusNumber) String() string {
	if o.Number == "" {
		return ""
	}
	if o.IndexName == "" {
		return fmt.Sprintf("Op. %s", o.Number)
	} else {
		return fmt.Sprintf("%s %s", o.IndexName, o.Number)
	}
}

type Title struct {
	Title    string `xml:",chardata"`
	Language string `xml:",attr,omitempty"`
}

// A Work represents a musical composition.
type Work struct {
	Composer Composer

	// A work may have more than one title in different languages.
	Title []Title

	// The opus number(s) for this composition. There may be more than one index
	// for identifying works by this particular composer. Or none at all. Or
	// this work may just not appear on any of them.
	OpusNumber []OpusNumber

	// The parts that comprise this work, if any.
	Parts []Part `xml:"Parts>Part,omitempty"`

	// The year this composition was completed.
	Year int
}

type Part struct {
	Part   string `xml:",chardata"`
	Number string `xml:"number,attr,omitempty"`
}

// A Performer in one particular recording of a Work.
// An 'artist', if you will.
type Performer struct {
	Name string `xml:",chardata"`

	// The type of performer. Suggested values:
	//   * "performer"
	//   * "soloist"
	//   * "orchestra"
	//   * "ensemble"
	//   * "conductor"
	Role string `xml:"role,attr,omitempty"`
}

// A path to a file comprising the recording
type SourceFile struct {
	Filename string `xml:",chardata"`

	// The disc number, if applicable.
	Disc int `xml:"disc,attr,omitempty"`
}

func (s SourceFile) String() string {
	return s.Filename
}

// One (recording of a) performance of a Work.
type Performance struct {
	Work Work

	// The year in which the performance took place
	Year int `xml:",omitempty"`

	Performers  []Performer  `xml:"Performers>Performer"`
	SourceFiles []SourceFile `xml:"SourceFiles>File"`
}

// A Carrier can represent either a physical medium or a packaged download.
// An "album," in layman's terms.
type Carrier struct {
	XMLName xml.Name `xml:"https://www.inurbanus.nl/NS/speeldoos/1.0 Carrier"`

	Name string

	// A catalog number, or some other unique identifier within the collection
	ID string `xml:",omitempty"`

	// The SHA256 hash of the corresponding .zip bundle for this carrier.
	Hash string `xml:"hash,attr,omitempty"`

	// An indication of how this carrier ended up in this library.
	// Suggested values:
	//   * "WEB" (digital download)
	//   * "CD" (ripped cd)
	Source string `xml:"source,attr,omitempty"`

	// The performances on this carrier
	Performances []Performance `xml:"Performances>Performance"`
}

// Read a serialized Carrier from a file
func ImportCarrier(filename string) (*Carrier, error) {
	ip, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}

	foo := &Carrier{}
	err = xml.Unmarshal(ip, foo)
	if err != nil {
		return nil, err
	}

	return foo, nil
}

// Serialize the carrier to disk
func (c *Carrier) Write(filename string) error {
	op, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer op.Close()

	w := xml.NewEncoder(op)
	w.Indent("", "	")
	err = w.Encode(c)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// The MIT License (MIT)
//
// Copyright (c) 2016 winlin
//
// Permission is hereby granted, free of charge, to any person obtaining a copy of
// this software and associated documentation files (the "Software"), to deal in
// the Software without restriction, including without limitation the rights to
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
// the Software, and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

// The speex decoder, to decode the encoded speex frame to PCM samples.
package speex

/*
#cgo CFLAGS: -I${SRCDIR}/../speex-lib/objs/include
#cgo LDFLAGS: ${SRCDIR}/../speex-lib/objs/lib/libspeex.a -lm
#include "speex/speex.h"

typedef struct {
	SpeexBits bits;
	const SpeexMode* mode;
	void* state;

	int frame_size;
	int sample_rate;
} speexdec_t;

int speexdec_init(speexdec_t* h, int sample_rate, int channels) {
	h->mode = 0;
	h->state = 0;
	h->frame_size = h->sample_rate = 0;

    // TODO: support stereo speex.
    if (channels != 1) {
        return -1;
    }

    int frame_size;
    const SpeexMode* mode;
    if (1) {
        int spx_mode;
        switch (sample_rate) {
            case 8000:  spx_mode = 0; break;
            case 16000: spx_mode = 1; break;
            case 32000: spx_mode = 2; break;
            default: return -1;
        }

        mode = speex_lib_get_mode(spx_mode);
        if (!mode) {
            return -1;
        }
    }
    h->mode = mode;

    void* state = speex_decoder_init(mode);
    if (!state) {
        return -1;
    }

    h->state = state;
    speex_bits_init(&h->bits);

    if (1) {
        spx_int32_t N;
        speex_decoder_ctl(state, SPEEX_GET_FRAME_SIZE, &N);
        h->frame_size = N;

        speex_decoder_ctl(state, SPEEX_GET_SAMPLING_RATE, &N);
        h->sample_rate = N;
    }

	return 0;
}

void speexdec_close(speexdec_t* h) {
	if (h->state) {
		speex_decoder_destroy(h->state);
		speex_bits_destroy(&h->bits);
	}
	h->state = 0;
}

int speexdec_decode(speexdec_t* h, char* frame, int nb_frame, char* pcm, int* pnb_pcm, int* is_done) {
	// the output pcm must equals to the frames(each is 16bits).
	if (*pnb_pcm != h->frame_size * sizeof(spx_int16_t)) {
		return -1;
	}

	if (speex_bits_remaining(&h->bits) < 5 ||
		speex_bits_peek_unsigned(&h->bits, 5) == 0xF) {
		speex_bits_read_from(&h->bits, frame, nb_frame);
	}

	spx_int16_t* output = (spx_int16_t*)pcm;
	int ret = speex_decode_int(h->state, &h->bits, output);

	// 0 for no error, -1 for end of stream, -2 corrupt stream
	if (ret <= -2) {
		return ret;
	}

	if (ret == -1) {
		*pnb_pcm = 0;
		return 0;
	}

	if (speex_bits_remaining(&h->bits) < 5 ||
		speex_bits_peek_unsigned(&h->bits, 5) == 0xF) {
		*is_done = 1;
	}

	return 0;
}

int speexdec_frame_size(speexdec_t* h) {
	return h->frame_size;
}

int speexdec_sample_rate(speexdec_t* h) {
	return h->sample_rate;
}
*/
import "C"

import (
	"fmt"
	"unsafe"
	"bytes"
)

type SpeexDecoder struct {
	m C.speexdec_t
}

func NewSpeexDecoder() *SpeexDecoder {
	return &SpeexDecoder{}
}

// @remark only support mono speex(channels must be 1).
func (v *SpeexDecoder) Init(sampleRate, channels int) (err error) {
	if channels != 1 {
		return fmt.Errorf("only support mono(1), actual is %v", channels)
	}

	r := C.speexdec_init(&v.m, C.int(sampleRate), C.int(channels))
	if int(r) != 0 {
		return fmt.Errorf("init decoder failed, err=%v", int(r))
	}

	return
}

func (v *SpeexDecoder) Close() {
	C.speexdec_close(&v.m)
}

// @return pcm is nil when EOF.
func (v *SpeexDecoder) Decode(frame []byte) (pcm []byte, err error) {
	p := (*C.char)(unsafe.Pointer(&frame[0]))
	pSize := C.int(len(frame))

	pIsDone := C.int(0)

	var result bytes.Buffer

	for {
		if pIsDone == 1 {
			break
		}
		// each sample is 16bits(2bytes),
		// so we alloc the output to frame_size*2.
		nbPcmBytes := v.FrameSize()*2

		pcmTmp := make([]byte, nbPcmBytes)
		pPcm := (*C.char)(unsafe.Pointer(&pcmTmp[0]))
		pNbPcm := C.int(nbPcmBytes)

		r := C.speexdec_decode(&v.m, p, pSize, pPcm, &pNbPcm, &pIsDone)
		if int(r) != 0 {
			return nil,fmt.Errorf("decode failed, err=%v", int(r))
		}
		if int(pNbPcm) <= 0 {
			return nil,nil
		}
		if int(pNbPcm) != nbPcmBytes {
			return nil,fmt.Errorf("invalid pcm size %v", int(pNbPcm))
		}
		result.Write(pcmTmp)
	}
	pcm = result.Bytes()

	return
}

func (v *SpeexDecoder) FrameSize() int {
	return int(C.speexdec_frame_size(&v.m))
}

func (v *SpeexDecoder) SampleRate() int {
	return int(C.speexdec_sample_rate(&v.m))
}

func (v *SpeexDecoder) Channels() int {
	return 1
}
<commit_msg>for #6084, 调整speex_decode_int函数返回值的处理方式 (#2)<commit_after>// The MIT License (MIT)
//
// Copyright (c) 2016 winlin
//
// Permission is hereby granted, free of charge, to any person obtaining a copy of
// this software and associated documentation files (the "Software"), to deal in
// the Software without restriction, including without limitation the rights to
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
// the Software, and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

// The speex decoder, to decode the encoded speex frame to PCM samples.
package speex

/*
#cgo CFLAGS: -I${SRCDIR}/../speex-lib/objs/include
#cgo LDFLAGS: ${SRCDIR}/../speex-lib/objs/lib/libspeex.a -lm
#include "speex/speex.h"

typedef struct {
	SpeexBits bits;
	const SpeexMode* mode;
	void* state;

	int frame_size;
	int sample_rate;
} speexdec_t;

int speexdec_init(speexdec_t* h, int sample_rate, int channels) {
	h->mode = 0;
	h->state = 0;
	h->frame_size = h->sample_rate = 0;

    // TODO: support stereo speex.
    if (channels != 1) {
        return -1;
    }

    int frame_size;
    const SpeexMode* mode;
    if (1) {
        int spx_mode;
        switch (sample_rate) {
            case 8000:  spx_mode = 0; break;
            case 16000: spx_mode = 1; break;
            case 32000: spx_mode = 2; break;
            default: return -1;
        }

        mode = speex_lib_get_mode(spx_mode);
        if (!mode) {
            return -1;
        }
    }
    h->mode = mode;

    void* state = speex_decoder_init(mode);
    if (!state) {
        return -1;
    }

    h->state = state;
    speex_bits_init(&h->bits);

    if (1) {
        spx_int32_t N;
        speex_decoder_ctl(state, SPEEX_GET_FRAME_SIZE, &N);
        h->frame_size = N;

        speex_decoder_ctl(state, SPEEX_GET_SAMPLING_RATE, &N);
        h->sample_rate = N;
    }

	return 0;
}

void speexdec_close(speexdec_t* h) {
	if (h->state) {
		speex_decoder_destroy(h->state);
		speex_bits_destroy(&h->bits);
	}
	h->state = 0;
}

int speexdec_decode(speexdec_t* h, char* frame, int nb_frame, char* pcm, int* pnb_pcm, int* is_done) {
	// the output pcm must equals to the frames(each is 16bits).
	if (*pnb_pcm != h->frame_size * sizeof(spx_int16_t)) {
		return -1;
	}

	if (speex_bits_remaining(&h->bits) < 5 ||
		speex_bits_peek_unsigned(&h->bits, 5) == 0xF) {
		speex_bits_read_from(&h->bits, frame, nb_frame);
	}

	spx_int16_t* output = (spx_int16_t*)pcm;
	int ret = speex_decode_int(h->state, &h->bits, output);

	// 0 for no error, -1 for end of stream, -2 corrupt stream
	if (ret <= -2) {
		return ret;
	}

	if (ret == -1) {
		*pnb_pcm = 0;
		return 0;
	}

	if (speex_bits_remaining(&h->bits) < 5 ||
		speex_bits_peek_unsigned(&h->bits, 5) == 0xF) {
		*is_done = 1;
	}

	return 0;
}

int speexdec_frame_size(speexdec_t* h) {
	return h->frame_size;
}

int speexdec_sample_rate(speexdec_t* h) {
	return h->sample_rate;
}
*/
import "C"

import (
	"fmt"
	"unsafe"
	"bytes"
)

type SpeexDecoder struct {
	m C.speexdec_t
}

func NewSpeexDecoder() *SpeexDecoder {
	return &SpeexDecoder{}
}

// @remark only support mono speex(channels must be 1).
func (v *SpeexDecoder) Init(sampleRate, channels int) (err error) {
	if channels != 1 {
		return fmt.Errorf("only support mono(1), actual is %v", channels)
	}

	r := C.speexdec_init(&v.m, C.int(sampleRate), C.int(channels))
	if int(r) != 0 {
		return fmt.Errorf("init decoder failed, err=%v", int(r))
	}

	return
}

func (v *SpeexDecoder) Close() {
	C.speexdec_close(&v.m)
}

// @return pcm is nil when EOF.
func (v *SpeexDecoder) Decode(frame []byte) (pcm []byte, err error) {
	p := (*C.char)(unsafe.Pointer(&frame[0]))
	pSize := C.int(len(frame))

	pIsDone := C.int(0)

	var result bytes.Buffer

	for {
		if pIsDone == 1 {
			break
		}
		// each sample is 16bits(2bytes),
		// so we alloc the output to frame_size*2.
		nbPcmBytes := v.FrameSize()*2

		pcmTmp := make([]byte, nbPcmBytes)
		pPcm := (*C.char)(unsafe.Pointer(&pcmTmp[0]))
		pNbPcm := C.int(nbPcmBytes)

		r := C.speexdec_decode(&v.m, p, pSize, pPcm, &pNbPcm, &pIsDone)
		if int(r) != 0 {
			return nil,fmt.Errorf("decode failed, err=%v", int(r))
		}
		// if pNbPcm is 0, which means the stream is end, need to jump out of cycle.
		if int(pNbPcm) == 0 {
			break
		}
		if int(pNbPcm) != nbPcmBytes {
			return nil,fmt.Errorf("invalid pcm size %v", int(pNbPcm))
		}
		result.Write(pcmTmp)
	}
	pcm = result.Bytes()

	return
}

func (v *SpeexDecoder) FrameSize() int {
	return int(C.speexdec_frame_size(&v.m))
}

func (v *SpeexDecoder) SampleRate() int {
	return int(C.speexdec_sample_rate(&v.m))
}

func (v *SpeexDecoder) Channels() int {
	return 1
}
<|endoftext|>