<commit_before>package main

import (
	"code.google.com/p/log4go"
	"errors"
	"flag"
	"fmt"
	"gopkg.in/mgo.v2"
	"net/url"
	"os"
	"runtime"
	"strings"
	"time"
)

var (
	logdebug  = flag.Bool("v", false, "debug")
	logfinest = flag.Bool("vv", false, "super debug")
	logStats  = flag.Bool("stats", false, "show period mongostats")
)

var (
	source_uri = flag.String("s", "", "set the source mongo uri")
	dest_uri   = flag.String("d", "", "set the destination mongo uri")
)

var (
	// you won't need these for normal use
	from = flag.String("from", "now", "begin processing the oplog at this timestamp.  accepts timestemps in the form -from=+1 (now + 1 second), -from=now, -from=12345,123 (seconds, iterations)")
	to   = flag.String("to", "+86400", "begin processing the oplog at this timestamp.  accepts timestemps in the form -to=+1 (now + 1 second), -to=now, -to=12345,123 (seconds, iterations)")
)

var (
	initial_sync      = flag.Bool("i", false, "perform an initial sync")
	forceTableScan    = flag.Bool("forceTableScan", false, "don't sort by ids in initial sync. (sorting by _id can sometimes miss documents if _id is a non ObjectId())")
	forceIndexBuild   = flag.String("forceindex", "", "force Index builds on to either forground or background.  -forceindex foreground OR -forceindex background")
	replay_oplog      = flag.Bool("o", false, "replay the oplog from -from to -to")
	oplog_name        = flag.String("oplog", "oplog.rs", "the name of the oplog to use")
	ignore_errors     = flag.Bool("f", true, "force oplog sync, even if counts don't quite match (hope they match after oplog is sync'd)")
	allDbs            = flag.Bool("allDbs", false, "copy all the databases from the source to the destination")
	ignoreSslError    = flag.Bool("ignoreSslError", false, "ignore validation of SSL certificate")
	connectionTimeout = flag.Int("connectionTimeout", 60, "connection timeout in seconds")
)

var logger log4go.Logger

const (
	BUFFER_SIZE     = 500
	GOPOOL_SIZE     = 16
	MAX_BUFFER_SIZE = 4e7
)

func main() {
	flag.Parse()
	logger = initLogger()

	runtime.GOMAXPROCS(runtime.NumCPU())

	if *source_uri == "" || *dest_uri == "" {
		Quit(1, errors.New("Provide both source and destination URIs"))
	}

	switch *forceIndexBuild {
	case "bg", "background":
		*forceIndexBuild = "bg"
	case "fg", "foreground":
		*forceIndexBuild = "fg"
	case "im", "immediate":
		*forceIndexBuild = "im"
	case "":
		*forceIndexBuild = ""
	default:
		Quit(1, errors.New("Please provide a valid forceindex, either foreground or background, or immediate"))
	}

	if *source_uri == *dest_uri {
		Quit(1, errors.New("Source and destination can't be the same"))
	}

	// establish connections to source and destination mongos
	srcURI, srcDB, err := adminifyURI(*source_uri)
	if err != nil {
		Quit(1, err)
	}

	srcTarget := NewMongoTarget(srcURI, srcDB)
	err = srcTarget.Dial()
	source := srcTarget.dst
	if err != nil {
		logger.Critical("Cannot dial %s\n, %v", srcURI.String(), err)
		Quit(1, err)
	}
	source.SetMode(mgo.Monotonic, false)

	dstURI, dstDB, err := adminifyURI(*dest_uri)
	if err != nil {
		Quit(1, err)
	}

	target := NewTarget(dstURI, dstDB)
	err = target.Dial()
	if err != nil {
		Quit(1, fmt.Errorf("Cannot dial %s\n, %v", *dest_uri, err))
	}

	from := NewTimestamp(*from)
	to := NewTimestamp(*to)

	if *logStats {
		// periodically dump some mongo stats to stdout
		mgo.SetStats(true)
		statsChan := time.Tick(2 * time.Second)
		go func(c <-chan time.Time) {
			for _ = range c {
				s := mgo.GetStats()
				logger.Info("Sockets in use: %d, sentOps: %d, receivedOps: %d, receivedDocs: %d", s.SocketsInUse, s.SentOps, s.ReceivedOps, s.ReceivedDocs)
			}
		}(statsChan)
	}

	if *replay_oplog {
		//from, err = CurrentOplogTimestamp(source)
		//if err != nil {
		//	Quit(1, errors.New("unable to get most recent oplog timestamp"))
		//}
		logger.Info("Oplog at: %s", from)
	}
	if *initial_sync {
		logger.Info("copying data from %s to %s.", *source_uri, *dest_uri)
		// copying all the databases
		if *allDbs {
			databaseNames, err := source.DatabaseNames()
			if err != nil {
				Quit(1, err)
			}
			for _, d := range databaseNames {
				if isSpecialDatabase(d) {
					continue
				}
				logger.Info("Copying db " + d)
				target.DB(d)

				if err := target.Sync(source, srcURI, d); err != nil {
					Quit(1, err)
				}
			}

		} else {
			// copying one database
			if err := target.Sync(source, srcURI, srcDB); err != nil {
				Quit(1, err)
			}
		}
		logger.Info("initial sync completed")
	}

	if *replay_oplog {
		player, err := newLogReplayer(source, target, from, to, srcDB, dstDB)
		if err != nil {
			logger.Critical("Could not initialize logReplayer: %s", err)
			Quit(1, err)
		}

		err = player.playLog()
		if err != nil {
			logger.Error(err)
		}
	}
	Quit(0, nil)
}

func initLogger() log4go.Logger {
	loglevel := log4go.INFO
	if *logdebug {
		loglevel = log4go.DEBUG
	}
	if *logfinest {
		loglevel = log4go.FINEST
	}

	return log4go.NewDefaultLogger(loglevel)
}

func Quit(code int, err error) {
	if err != nil {
		logger.Critical("Failed: " + err.Error())
	}
	time.Sleep(100 * time.Millisecond)
	logger.Close()
	os.Exit(code)
}

func adminifyURI(s string) (*url.URL, string, error) {
	uri, err := url.Parse(s)
	if err != nil {
		return uri, "", err
	}

	db := uri.Path
	if len(db) < 2 {
		return uri, "", fmt.Errorf("invalid database name")
	}
	if db[0] == '/' {
		db = db[1:]
	}

	uri.Path = "/admin"

	logger.Debug("Connecting to URL: %v", uri)

	return uri, db, nil
}

// is the database a special database
var specialDatabases = []string{"admin", "local", "test", "config"}

func isSpecialDatabase(item string) bool {
	item = strings.ToLower(item)
	for _, v := range specialDatabases {
		if v == item {
			return true
		}
	}
	return false
}
<commit_msg>make buffer and pool configurable<commit_after>package main

import (
	"code.google.com/p/log4go"
	"errors"
	"flag"
	"fmt"
	"gopkg.in/mgo.v2"
	"net/url"
	"os"
	"runtime"
	"strings"
	"time"
)

var (
	logdebug  = flag.Bool("v", false, "debug")
	logfinest = flag.Bool("vv", false, "super debug")
	logStats  = flag.Bool("stats", false, "show period mongostats")
)

var (
	source_uri = flag.String("s", "", "set the source mongo uri")
	dest_uri   = flag.String("d", "", "set the destination mongo uri")
)

var (
	// you won't need these for normal use
	from = flag.String("from", "now", "begin processing the oplog at this timestamp.  accepts timestemps in the form -from=+1 (now + 1 second), -from=now, -from=12345,123 (seconds, iterations)")
	to   = flag.String("to", "+86400", "begin processing the oplog at this timestamp.  accepts timestemps in the form -to=+1 (now + 1 second), -to=now, -to=12345,123 (seconds, iterations)")
)

var (
	initial_sync      = flag.Bool("i", false, "perform an initial sync")
	forceTableScan    = flag.Bool("forceTableScan", false, "don't sort by ids in initial sync. (sorting by _id can sometimes miss documents if _id is a non ObjectId())")
	forceIndexBuild   = flag.String("forceindex", "", "force Index builds on to either forground or background.  -forceindex foreground OR -forceindex background")
	replay_oplog      = flag.Bool("o", false, "replay the oplog from -from to -to")
	oplog_name        = flag.String("oplog", "oplog.rs", "the name of the oplog to use")
	ignore_errors     = flag.Bool("f", true, "force oplog sync, even if counts don't quite match (hope they match after oplog is sync'd)")
	allDbs            = flag.Bool("allDbs", false, "copy all the databases from the source to the destination")
	ignoreSslError    = flag.Bool("ignoreSslError", false, "ignore validation of SSL certificate")
	connectionTimeout = flag.Int("connectionTimeout", 60, "connection timeout in seconds")
	bufferSize        = flag.Int("bufferSize", 2500, "buffer size (amount of documents to write at once)")
	poolSize          = flag.Int("poolSize", 16, "pool size (amount of parallel connections for each collection)")
)

var logger log4go.Logger

var (
	BUFFER_SIZE     = *bufferSize
	GOPOOL_SIZE     = *poolSize
	MAX_BUFFER_SIZE = 4e7
)

func main() {
	flag.Parse()
	logger = initLogger()

	runtime.GOMAXPROCS(runtime.NumCPU())

	if *source_uri == "" || *dest_uri == "" {
		Quit(1, errors.New("Provide both source and destination URIs"))
	}

	switch *forceIndexBuild {
	case "bg", "background":
		*forceIndexBuild = "bg"
	case "fg", "foreground":
		*forceIndexBuild = "fg"
	case "im", "immediate":
		*forceIndexBuild = "im"
	case "":
		*forceIndexBuild = ""
	default:
		Quit(1, errors.New("Please provide a valid forceindex, either foreground or background, or immediate"))
	}

	if *source_uri == *dest_uri {
		Quit(1, errors.New("Source and destination can't be the same"))
	}

	// establish connections to source and destination mongos
	srcURI, srcDB, err := adminifyURI(*source_uri)
	if err != nil {
		Quit(1, err)
	}

	srcTarget := NewMongoTarget(srcURI, srcDB)
	err = srcTarget.Dial()
	source := srcTarget.dst
	if err != nil {
		logger.Critical("Cannot dial %s\n, %v", srcURI.String(), err)
		Quit(1, err)
	}
	source.SetMode(mgo.Monotonic, false)

	dstURI, dstDB, err := adminifyURI(*dest_uri)
	if err != nil {
		Quit(1, err)
	}

	target := NewTarget(dstURI, dstDB)
	err = target.Dial()
	if err != nil {
		Quit(1, fmt.Errorf("Cannot dial %s\n, %v", *dest_uri, err))
	}

	from := NewTimestamp(*from)
	to := NewTimestamp(*to)

	if *logStats {
		// periodically dump some mongo stats to stdout
		mgo.SetStats(true)
		statsChan := time.Tick(2 * time.Second)
		go func(c <-chan time.Time) {
			for _ = range c {
				s := mgo.GetStats()
				logger.Info("Sockets in use: %d, sentOps: %d, receivedOps: %d, receivedDocs: %d", s.SocketsInUse, s.SentOps, s.ReceivedOps, s.ReceivedDocs)
			}
		}(statsChan)
	}

	if *replay_oplog {
		//from, err = CurrentOplogTimestamp(source)
		//if err != nil {
		//	Quit(1, errors.New("unable to get most recent oplog timestamp"))
		//}
		logger.Info("Oplog at: %s", from)
	}
	if *initial_sync {
		logger.Info("copying data from %s to %s.", *source_uri, *dest_uri)
		// copying all the databases
		if *allDbs {
			databaseNames, err := source.DatabaseNames()
			if err != nil {
				Quit(1, err)
			}
			for _, d := range databaseNames {
				if isSpecialDatabase(d) {
					continue
				}
				logger.Info("Copying db " + d)
				target.DB(d)

				if err := target.Sync(source, srcURI, d); err != nil {
					Quit(1, err)
				}
			}

		} else {
			// copying one database
			if err := target.Sync(source, srcURI, srcDB); err != nil {
				Quit(1, err)
			}
		}
		logger.Info("initial sync completed")
	}

	if *replay_oplog {
		player, err := newLogReplayer(source, target, from, to, srcDB, dstDB)
		if err != nil {
			logger.Critical("Could not initialize logReplayer: %s", err)
			Quit(1, err)
		}

		err = player.playLog()
		if err != nil {
			logger.Error(err)
		}
	}
	Quit(0, nil)
}

func initLogger() log4go.Logger {
	loglevel := log4go.INFO
	if *logdebug {
		loglevel = log4go.DEBUG
	}
	if *logfinest {
		loglevel = log4go.FINEST
	}

	return log4go.NewDefaultLogger(loglevel)
}

func Quit(code int, err error) {
	if err != nil {
		logger.Critical("Failed: " + err.Error())
	}
	time.Sleep(100 * time.Millisecond)
	logger.Close()
	os.Exit(code)
}

func adminifyURI(s string) (*url.URL, string, error) {
	uri, err := url.Parse(s)
	if err != nil {
		return uri, "", err
	}

	db := uri.Path
	if len(db) < 2 {
		return uri, "", fmt.Errorf("invalid database name")
	}
	if db[0] == '/' {
		db = db[1:]
	}

	uri.Path = "/admin"

	logger.Debug("Connecting to URL: %v", uri)

	return uri, db, nil
}

// is the database a special database
var specialDatabases = []string{"admin", "local", "test", "config"}

func isSpecialDatabase(item string) bool {
	item = strings.ToLower(item)
	for _, v := range specialDatabases {
		if v == item {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>// send.go - mixnet client send
// Copyright (C) 2018  David Stainton.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package client

import (
	"encoding/binary"
	"encoding/hex"
	"errors"
	"fmt"
	"github.com/katzenpost/client/utils"
	"io"
	"time"

	cConstants "github.com/katzenpost/client/constants"
	"github.com/katzenpost/core/constants"
	"github.com/katzenpost/core/crypto/rand"
	sConstants "github.com/katzenpost/core/sphinx/constants"
)

var ErrReplyTimeout = errors.New("failure waiting for reply, timeout reached")
var ErrMessageNotSent = errors.New("failure sending message")

func (s *Session) sendNext() {
	msg, err := s.egressQueue.Peek()
	if err != nil {
		s.fatalErrCh <- errors.New("impossible failure to Peek from queue")
		return
	}
	if msg == nil {
		s.fatalErrCh <- errors.New("impossible failure, got nil message from queue")
		return
	}
	m := msg.(*Message)
	s.doSend(m)
	_, err = s.egressQueue.Pop()
	if err != nil {
		s.fatalErrCh <- errors.New("impossible failure to Pop from queue")
	}
}

func NewRescheduler(s *Session) *rescheduler {
	r := &rescheduler{s: s}
	s.log.Debugf("Creating TimerQueue")
	r.timerQ = NewTimerQueue(r)
	return r
}

type rescheduler struct {
	s *Session
	timerQ *TimerQueue
}

func (r *rescheduler) Push(i Item) error {
	// rescheduler checks whether a message was ACK'd when the timerQ fires
	// and if it has not, reschedules the message for transmission again
	m := i.(*Message)
	if _, ok := r.s.surbIDMap.Load(*m.SURBID); ok {
		// still waiting for a SURB-ACK that hasn't arrived
		r.s.surbIDMap.Delete(*m.SURBID)
		r.s.opCh <- opRetransmit{msg :m}
	}
	return nil
}

func (s *Session) doRetransmit(msg *Message) {
	msg.Retransmissions++
	msgIdStr := fmt.Sprintf("[%v]", hex.EncodeToString(msg.ID[:]))
	s.log.Debugf("doRetransmit: %d for %s", msg.Retransmissions, msgIdStr)
	s.doSend(msg)
}

func (s *Session) doSend(msg *Message) {
	surbID := [sConstants.SURBIDLength]byte{}
	_, err := io.ReadFull(rand.Reader, surbID[:])
	if err != nil {
		s.fatalErrCh <- fmt.Errorf("impossible failure, failed to generate SURB ID for message ID %x", *msg.ID)
		return
	}
	key := []byte{}
	var eta time.Duration
	msgIdStr := fmt.Sprintf("[%v]", hex.EncodeToString(msg.ID[:]))
	if msg.WithSURB {
		msg.SURBID = &surbID
		surbIdStr := fmt.Sprintf("[%v]", hex.EncodeToString(surbID[:]))
		s.log.Debugf("doSend %s with SURB ID %s", msgIdStr, surbIdStr)
		key, eta, err = s.minclient.SendCiphertext(msg.Recipient, msg.Provider, &surbID, msg.Payload)
	} else {
		s.log.Debugf("doSend %s without SURB", msgIdStr)
		err = s.minclient.SendUnreliableCiphertext(msg.Recipient, msg.Provider, msg.Payload)
	}

	// message was sent
	if err == nil {
		msg.SentAt = time.Now()
	}
	// expect a reply
	if msg.WithSURB {
		if err == nil {
			s.log.Debugf("doSend setting ReplyETA to %v", eta)
			// increase the timeout for each retransmission
			msg.ReplyETA = eta * (1 + time.Duration(msg.Retransmissions))
			msg.Key = key
			s.surbIDMap.Store(surbID, msg)
			if msg.Reliable {
				s.log.Debugf("Sending reliable message with retransmissions")
				timeSlop := eta // add a round-trip worth of delay before timing out
				msg.QueuePriority = uint64(msg.SentAt.Add(msg.ReplyETA).Add(timeSlop).UnixNano())
				s.rescheduler.timerQ.Push(msg)
			}
		}
		// write to waiting channel or close channel if message failed to send
		if msg.IsBlocking {
			sentWaitChanRaw, ok := s.sentWaitChanMap.Load(*msg.ID)
			if !ok {
				s.fatalErrCh <- fmt.Errorf("impossible failure, sentWaitChan not found for message ID %x", *msg.ID)
				return
			}
			sentWaitChan := sentWaitChanRaw.(chan *Message)
			if err == nil {
				sentWaitChan <- msg
			} else {
				close(sentWaitChan)
			}
			return
		}
	}
	s.eventCh.In() <- &MessageSentEvent{
		MessageID: msg.ID,
		Err:       err,
		SentAt:    msg.SentAt,
		ReplyETA:  msg.ReplyETA,
	}
}

func (s *Session) sendDropDecoy(loopSvc *utils.ServiceDescriptor) {
	payload := make([]byte, constants.UserForwardPayloadLength)
	id := [cConstants.MessageIDLength]byte{}
	_, err := io.ReadFull(rand.Reader, id[:])
	if err != nil {
		s.fatalErrCh <- errors.New("failure to generate message ID for drop decoy")
		return
	}
	msg := &Message{
		ID:        &id,
		Recipient: loopSvc.Name,
		Provider:  loopSvc.Provider,
		Payload:   payload[:],
		WithSURB:  false,
		IsDecoy:   true,
	}
	s.doSend(msg)
}

func (s *Session) sendLoopDecoy(loopSvc *utils.ServiceDescriptor) {
	s.log.Info("sending loop decoy")
	payload := make([]byte, constants.UserForwardPayloadLength)
	id := [cConstants.MessageIDLength]byte{}
	_, err := io.ReadFull(rand.Reader, id[:])
	if err != nil {
		s.fatalErrCh <- errors.New("failure to generate message ID for loop decoy")
		return
	}
	msg := &Message{
		ID:        &id,
		Recipient: loopSvc.Name,
		Provider:  loopSvc.Provider,
		Payload:   payload[:],
		WithSURB:  true,
		IsDecoy:   true,
	}
	defer s.incrementDecoyLoopTally()
	s.doSend(msg)
}

func (s *Session) composeMessage(recipient, provider string, message []byte, isBlocking bool) (*Message, error) {
	s.log.Debug("SendMessage")
	if len(message) > constants.UserForwardPayloadLength-4 {
		return nil, fmt.Errorf("invalid message size: %v", len(message))
	}
	payload := make([]byte, constants.UserForwardPayloadLength)
	binary.BigEndian.PutUint32(payload[:4], uint32(len(message)))
	copy(payload[4:], message)
	id := [cConstants.MessageIDLength]byte{}
	_, err := io.ReadFull(rand.Reader, id[:])
	if err != nil {
		return nil, err
	}
	var msg = Message{
		ID:         &id,
		Recipient:  recipient,
		Provider:   provider,
		Payload:    payload[:],
		WithSURB:   true,
		IsBlocking: isBlocking,
	}
	return &msg, nil
}

// SendReliableMessage asynchronously sends messages with automatic retransmissiosn.
func (s *Session) SendReliableMessage(recipient, provider string, message []byte) (*[cConstants.MessageIDLength]byte, error) {
	msg, err := s.composeMessage(recipient, provider, message, false)
	if err != nil {
		return nil, err
	}
	msg.Reliable = true
	err = s.egressQueue.Push(msg)
	if err != nil {
		return nil, err
	}
	return msg.ID, nil
}

// SendUnreliableMessage asynchronously sends message without any automatic retransmissions.
func (s *Session) SendUnreliableMessage(recipient, provider string, message []byte) (*[cConstants.MessageIDLength]byte, error) {
	msg, err := s.composeMessage(recipient, provider, message, false)
	if err != nil {
		return nil, err
	}
	err = s.egressQueue.Push(msg)
	if err != nil {
		return nil, err
	}
	return msg.ID, nil
}

func (s *Session) BlockingSendUnreliableMessage(recipient, provider string, message []byte) ([]byte, error) {
	msg, err := s.composeMessage(recipient, provider, message, true)
	if err != nil {
		return nil, err
	}
	sentWaitChan := make(chan *Message)
	s.sentWaitChanMap.Store(*msg.ID, sentWaitChan)
	defer s.sentWaitChanMap.Delete(*msg.ID)

	replyWaitChan := make(chan []byte)
	s.replyWaitChanMap.Store(*msg.ID, replyWaitChan)
	defer s.replyWaitChanMap.Delete(*msg.ID)

	err = s.egressQueue.Push(msg)
	if err != nil {
		return nil, err
	}

	// wait until sent so that we know the ReplyETA for the waiting below
	sentMessage := <-sentWaitChan

	// if the message failed to send we will receive a nil message
	if sentMessage == nil {
		return nil, ErrMessageNotSent
	}

	// wait for reply or round trip timeout
	select {
	case reply := <-replyWaitChan:
		return reply, nil
	// these timeouts are often far too aggressive
	case <-time.After(sentMessage.ReplyETA + cConstants.RoundTripTimeSlop):
		return nil, ErrReplyTimeout
	}
	// unreachable
}

// BlockingSendReliableMessage sends a message with automatic message retransmission enabled
func (s *Session) BlockingSendReliableMessage(recipient, provider string, message []byte) ([]byte, error) {
	msg, err := s.composeMessage(recipient, provider, message, true)
	if err != nil {
		return nil, err
	}
	msg.Reliable = true
	sentWaitChan := make(chan *Message)
	s.sentWaitChanMap.Store(*msg.ID, sentWaitChan)
	defer s.sentWaitChanMap.Delete(*msg.ID)

	replyWaitChan := make(chan []byte)
	s.replyWaitChanMap.Store(*msg.ID, replyWaitChan)
	defer s.replyWaitChanMap.Delete(*msg.ID)

	err = s.egressQueue.Push(msg)
	if err != nil {
		return nil, err
	}

	// wait until sent so that we know the ReplyETA for the waiting below
	sentMessage := <-sentWaitChan

	// if the message failed to send we will receive a nil message
	if sentMessage == nil {
		return nil, ErrMessageNotSent
	}

	// TODO: it would be better to have the message automatically retransmitted a configurable number of times before emitting a failure to this channel
	// wait for reply or round trip timeout
	select {
	case reply := <-replyWaitChan:
		return reply, nil
	case <-time.After(cConstants.RoundTripTimeSlop):
		return nil, ErrReplyTimeout
	}
	// unreachable
}
<commit_msg>Do not block writing sentWaitChannel on retransmit<commit_after>// send.go - mixnet client send
// Copyright (C) 2018  David Stainton.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package client

import (
	"encoding/binary"
	"encoding/hex"
	"errors"
	"fmt"
	"github.com/katzenpost/client/utils"
	"io"
	"time"

	cConstants "github.com/katzenpost/client/constants"
	"github.com/katzenpost/core/constants"
	"github.com/katzenpost/core/crypto/rand"
	sConstants "github.com/katzenpost/core/sphinx/constants"
)

var ErrReplyTimeout = errors.New("failure waiting for reply, timeout reached")
var ErrMessageNotSent = errors.New("failure sending message")

func (s *Session) sendNext() {
	msg, err := s.egressQueue.Peek()
	if err != nil {
		s.fatalErrCh <- errors.New("impossible failure to Peek from queue")
		return
	}
	if msg == nil {
		s.fatalErrCh <- errors.New("impossible failure, got nil message from queue")
		return
	}
	m := msg.(*Message)
	s.doSend(m)
	_, err = s.egressQueue.Pop()
	if err != nil {
		s.fatalErrCh <- errors.New("impossible failure to Pop from queue")
	}
}

func NewRescheduler(s *Session) *rescheduler {
	r := &rescheduler{s: s}
	s.log.Debugf("Creating TimerQueue")
	r.timerQ = NewTimerQueue(r)
	return r
}

type rescheduler struct {
	s *Session
	timerQ *TimerQueue
}

func (r *rescheduler) Push(i Item) error {
	// rescheduler checks whether a message was ACK'd when the timerQ fires
	// and if it has not, reschedules the message for transmission again
	m := i.(*Message)
	if _, ok := r.s.surbIDMap.Load(*m.SURBID); ok {
		// still waiting for a SURB-ACK that hasn't arrived
		r.s.surbIDMap.Delete(*m.SURBID)
		r.s.opCh <- opRetransmit{msg :m}
	}
	return nil
}

func (s *Session) doRetransmit(msg *Message) {
	msg.Retransmissions++
	msgIdStr := fmt.Sprintf("[%v]", hex.EncodeToString(msg.ID[:]))
	s.log.Debugf("doRetransmit: %d for %s", msg.Retransmissions, msgIdStr)
	s.doSend(msg)
}

func (s *Session) doSend(msg *Message) {
	surbID := [sConstants.SURBIDLength]byte{}
	_, err := io.ReadFull(rand.Reader, surbID[:])
	if err != nil {
		s.fatalErrCh <- fmt.Errorf("impossible failure, failed to generate SURB ID for message ID %x", *msg.ID)
		return
	}
	key := []byte{}
	var eta time.Duration
	msgIdStr := fmt.Sprintf("[%v]", hex.EncodeToString(msg.ID[:]))
	if msg.WithSURB {
		msg.SURBID = &surbID
		surbIdStr := fmt.Sprintf("[%v]", hex.EncodeToString(surbID[:]))
		s.log.Debugf("doSend %s with SURB ID %s", msgIdStr, surbIdStr)
		key, eta, err = s.minclient.SendCiphertext(msg.Recipient, msg.Provider, &surbID, msg.Payload)
	} else {
		s.log.Debugf("doSend %s without SURB", msgIdStr)
		err = s.minclient.SendUnreliableCiphertext(msg.Recipient, msg.Provider, msg.Payload)
	}

	// message was sent
	if err == nil {
		msg.SentAt = time.Now()
	}
	// expect a reply
	if msg.WithSURB {
		if err == nil {
			s.log.Debugf("doSend setting ReplyETA to %v", eta)
			// increase the timeout for each retransmission
			msg.ReplyETA = eta * (1 + time.Duration(msg.Retransmissions))
			msg.Key = key
			s.surbIDMap.Store(surbID, msg)
			if msg.Reliable {
				s.log.Debugf("Sending reliable message with retransmissions")
				timeSlop := eta // add a round-trip worth of delay before timing out
				msg.QueuePriority = uint64(msg.SentAt.Add(msg.ReplyETA).Add(timeSlop).UnixNano())
				s.rescheduler.timerQ.Push(msg)
			}
		}
		// write to waiting channel or close channel if message failed to send
		if msg.IsBlocking {
			sentWaitChanRaw, ok := s.sentWaitChanMap.Load(*msg.ID)
			if !ok {
				s.fatalErrCh <- fmt.Errorf("impossible failure, sentWaitChan not found for message ID %x", *msg.ID)
				return
			}
			sentWaitChan := sentWaitChanRaw.(chan *Message)
			if err == nil {
				// do not block writing to the receiver if this is a retransmission
				select {
				case sentWaitChan <- msg:
				default:
				}

			} else {
				close(sentWaitChan)
			}
			return
		}
	}
	s.eventCh.In() <- &MessageSentEvent{
		MessageID: msg.ID,
		Err:       err,
		SentAt:    msg.SentAt,
		ReplyETA:  msg.ReplyETA,
	}
}

func (s *Session) sendDropDecoy(loopSvc *utils.ServiceDescriptor) {
	payload := make([]byte, constants.UserForwardPayloadLength)
	id := [cConstants.MessageIDLength]byte{}
	_, err := io.ReadFull(rand.Reader, id[:])
	if err != nil {
		s.fatalErrCh <- errors.New("failure to generate message ID for drop decoy")
		return
	}
	msg := &Message{
		ID:        &id,
		Recipient: loopSvc.Name,
		Provider:  loopSvc.Provider,
		Payload:   payload[:],
		WithSURB:  false,
		IsDecoy:   true,
	}
	s.doSend(msg)
}

func (s *Session) sendLoopDecoy(loopSvc *utils.ServiceDescriptor) {
	s.log.Info("sending loop decoy")
	payload := make([]byte, constants.UserForwardPayloadLength)
	id := [cConstants.MessageIDLength]byte{}
	_, err := io.ReadFull(rand.Reader, id[:])
	if err != nil {
		s.fatalErrCh <- errors.New("failure to generate message ID for loop decoy")
		return
	}
	msg := &Message{
		ID:        &id,
		Recipient: loopSvc.Name,
		Provider:  loopSvc.Provider,
		Payload:   payload[:],
		WithSURB:  true,
		IsDecoy:   true,
	}
	defer s.incrementDecoyLoopTally()
	s.doSend(msg)
}

func (s *Session) composeMessage(recipient, provider string, message []byte, isBlocking bool) (*Message, error) {
	s.log.Debug("SendMessage")
	if len(message) > constants.UserForwardPayloadLength-4 {
		return nil, fmt.Errorf("invalid message size: %v", len(message))
	}
	payload := make([]byte, constants.UserForwardPayloadLength)
	binary.BigEndian.PutUint32(payload[:4], uint32(len(message)))
	copy(payload[4:], message)
	id := [cConstants.MessageIDLength]byte{}
	_, err := io.ReadFull(rand.Reader, id[:])
	if err != nil {
		return nil, err
	}
	var msg = Message{
		ID:         &id,
		Recipient:  recipient,
		Provider:   provider,
		Payload:    payload[:],
		WithSURB:   true,
		IsBlocking: isBlocking,
	}
	return &msg, nil
}

// SendReliableMessage asynchronously sends messages with automatic retransmissiosn.
func (s *Session) SendReliableMessage(recipient, provider string, message []byte) (*[cConstants.MessageIDLength]byte, error) {
	msg, err := s.composeMessage(recipient, provider, message, false)
	if err != nil {
		return nil, err
	}
	msg.Reliable = true
	err = s.egressQueue.Push(msg)
	if err != nil {
		return nil, err
	}
	return msg.ID, nil
}

// SendUnreliableMessage asynchronously sends message without any automatic retransmissions.
func (s *Session) SendUnreliableMessage(recipient, provider string, message []byte) (*[cConstants.MessageIDLength]byte, error) {
	msg, err := s.composeMessage(recipient, provider, message, false)
	if err != nil {
		return nil, err
	}
	err = s.egressQueue.Push(msg)
	if err != nil {
		return nil, err
	}
	return msg.ID, nil
}

func (s *Session) BlockingSendUnreliableMessage(recipient, provider string, message []byte) ([]byte, error) {
	msg, err := s.composeMessage(recipient, provider, message, true)
	if err != nil {
		return nil, err
	}
	sentWaitChan := make(chan *Message)
	s.sentWaitChanMap.Store(*msg.ID, sentWaitChan)
	defer s.sentWaitChanMap.Delete(*msg.ID)

	replyWaitChan := make(chan []byte)
	s.replyWaitChanMap.Store(*msg.ID, replyWaitChan)
	defer s.replyWaitChanMap.Delete(*msg.ID)

	err = s.egressQueue.Push(msg)
	if err != nil {
		return nil, err
	}

	// wait until sent so that we know the ReplyETA for the waiting below
	sentMessage := <-sentWaitChan

	// if the message failed to send we will receive a nil message
	if sentMessage == nil {
		return nil, ErrMessageNotSent
	}

	// wait for reply or round trip timeout
	select {
	case reply := <-replyWaitChan:
		return reply, nil
	// these timeouts are often far too aggressive
	case <-time.After(sentMessage.ReplyETA + cConstants.RoundTripTimeSlop):
		return nil, ErrReplyTimeout
	}
	// unreachable
}

// BlockingSendReliableMessage sends a message with automatic message retransmission enabled
func (s *Session) BlockingSendReliableMessage(recipient, provider string, message []byte) ([]byte, error) {
	msg, err := s.composeMessage(recipient, provider, message, true)
	if err != nil {
		return nil, err
	}
	msg.Reliable = true
	sentWaitChan := make(chan *Message)
	s.sentWaitChanMap.Store(*msg.ID, sentWaitChan)
	defer s.sentWaitChanMap.Delete(*msg.ID)

	replyWaitChan := make(chan []byte)
	s.replyWaitChanMap.Store(*msg.ID, replyWaitChan)
	defer s.replyWaitChanMap.Delete(*msg.ID)

	err = s.egressQueue.Push(msg)
	if err != nil {
		return nil, err
	}

	// wait until sent so that we know the ReplyETA for the waiting below
	sentMessage := <-sentWaitChan

	// if the message failed to send we will receive a nil message
	if sentMessage == nil {
		return nil, ErrMessageNotSent
	}

	// TODO: it would be better to have the message automatically retransmitted a configurable number of times before emitting a failure to this channel
	// wait for reply or round trip timeout
	select {
	case reply := <-replyWaitChan:
		return reply, nil
	case <-time.After(cConstants.RoundTripTimeSlop):
		return nil, ErrReplyTimeout
	}
	// unreachable
}
<|endoftext|>


<commit_before>// send.go - mixnet client send
// Copyright (C) 2018  David Stainton.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package client

import (
	"encoding/binary"
	"encoding/hex"
	"errors"
	"fmt"
	"github.com/katzenpost/client/utils"
	"io"
	"time"

	cConstants "github.com/katzenpost/client/constants"
	"github.com/katzenpost/core/constants"
	"github.com/katzenpost/core/crypto/rand"
	sConstants "github.com/katzenpost/core/sphinx/constants"
)

var ErrReplyTimeout = errors.New("failure waiting for reply, timeout reached")
var ErrMessageNotSent = errors.New("failure sending message")

func (s *Session) sendNext() {
	msg, err := s.egressQueue.Peek()
	if err != nil {
		s.fatalErrCh <- errors.New("impossible failure to Peek from queue")
		return
	}
	if msg == nil {
		s.fatalErrCh <- errors.New("impossible failure, got nil message from queue")
		return
	}
	m := msg.(*Message)
	s.doSend(m)
	_, err = s.egressQueue.Pop()
	if err != nil {
		s.fatalErrCh <- errors.New("impossible failure to Pop from queue")
	}
}

func NewRescheduler(s *Session) *rescheduler {
	r := &rescheduler{s: s}
	s.log.Debugf("Creating TimerQueue")
	r.timerQ = NewTimerQueue(r)
	return r
}

type rescheduler struct {
	s *Session
	timerQ *TimerQueue
}

func (r *rescheduler) Push(i Item) error {
	// rescheduler checks whether a message was ACK'd when the timerQ fires
	// and if it has not, reschedules the message for transmission again
	m := i.(*Message)
	if _, ok := r.s.surbIDMap.Load(*m.SURBID); ok {
		// still waiting for a SURB-ACK that hasn't arrived
		r.s.surbIDMap.Delete(m.SURBID)
		r.s.opCh <- opRetransmit{msg :m}
	}
	return nil
}

func (s *Session) doRetransmit(msg *Message) {
	s.log.Debugf("doRetransmit for %s", msg)
	msg.Retransmissions++
	s.log.Debugf("retransmissions: %d", msg.Retransmissions)
	s.doSend(msg)
}

func (s *Session) doSend(msg *Message) {
	surbID := [sConstants.SURBIDLength]byte{}
	_, err := io.ReadFull(rand.Reader, surbID[:])
	if err != nil {
		s.fatalErrCh <- fmt.Errorf("impossible failure, failed to generate SURB ID for message ID %x", *msg.ID)
		return
	}
	key := []byte{}
	var eta time.Duration
	if msg.WithSURB {
		msg.SURBID = &surbID
		idStr := fmt.Sprintf("[%v]", hex.EncodeToString(surbID[:]))
		s.log.Debugf("doSend with SURB ID %x", idStr)
		key, eta, err = s.minclient.SendCiphertext(msg.Recipient, msg.Provider, &surbID, msg.Payload)
	} else {
		s.log.Debugf("doSend without SURB")
		err = s.minclient.SendUnreliableCiphertext(msg.Recipient, msg.Provider, msg.Payload)
	}

	// message was sent
	if err == nil {
		msg.SentAt = time.Now()
	}
	// expect a reply
	if msg.WithSURB {
		if err == nil {
			s.log.Debugf("doSend setting ReplyETA to %v", eta)
			// increase the timeout for each retransmission
			msg.ReplyETA = eta * (1 + time.Duration(msg.Retransmissions))
			msg.Key = key
			s.surbIDMap.Store(surbID, msg)
			if msg.Reliable {
				s.log.Debugf("Sending reliable message with retransmissions")
				msg.QueuePriority = uint64(msg.SentAt.Add(msg.ReplyETA).Add(cConstants.RoundTripTimeSlop).UnixNano())
				s.rescheduler.timerQ.Push(msg)
			}
		}
		// write to waiting channel or close channel if message failed to send
		if msg.IsBlocking {
			sentWaitChanRaw, ok := s.sentWaitChanMap.Load(*msg.ID)
			if !ok {
				s.fatalErrCh <- fmt.Errorf("impossible failure, sentWaitChan not found for message ID %x", *msg.ID)
				return
			}
			sentWaitChan := sentWaitChanRaw.(chan *Message)
			if err == nil {
				sentWaitChan <- msg
			} else {
				close(sentWaitChan)
			}
			return
		}
	}
	s.eventCh.In() <- &MessageSentEvent{
		MessageID: msg.ID,
		Err:       err,
		SentAt:    msg.SentAt,
		ReplyETA:  msg.ReplyETA,
	}
}

func (s *Session) sendDropDecoy(loopSvc *utils.ServiceDescriptor) {
	payload := make([]byte, constants.UserForwardPayloadLength)
	id := [cConstants.MessageIDLength]byte{}
	_, err := io.ReadFull(rand.Reader, id[:])
	if err != nil {
		s.fatalErrCh <- errors.New("failure to generate message ID for drop decoy")
		return
	}
	msg := &Message{
		ID:        &id,
		Recipient: loopSvc.Name,
		Provider:  loopSvc.Provider,
		Payload:   payload[:],
		WithSURB:  false,
		IsDecoy:   true,
	}
	s.doSend(msg)
}

func (s *Session) sendLoopDecoy(loopSvc *utils.ServiceDescriptor) {
	s.log.Info("sending loop decoy")
	payload := make([]byte, constants.UserForwardPayloadLength)
	id := [cConstants.MessageIDLength]byte{}
	_, err := io.ReadFull(rand.Reader, id[:])
	if err != nil {
		s.fatalErrCh <- errors.New("failure to generate message ID for loop decoy")
		return
	}
	msg := &Message{
		ID:        &id,
		Recipient: loopSvc.Name,
		Provider:  loopSvc.Provider,
		Payload:   payload[:],
		WithSURB:  true,
		IsDecoy:   true,
	}
	defer s.incrementDecoyLoopTally()
	s.doSend(msg)
}

func (s *Session) composeMessage(recipient, provider string, message []byte, isBlocking bool) (*Message, error) {
	s.log.Debug("SendMessage")
	if len(message) > constants.UserForwardPayloadLength-4 {
		return nil, fmt.Errorf("invalid message size: %v", len(message))
	}
	payload := make([]byte, constants.UserForwardPayloadLength)
	binary.BigEndian.PutUint32(payload[:4], uint32(len(message)))
	copy(payload[4:], message)
	id := [cConstants.MessageIDLength]byte{}
	_, err := io.ReadFull(rand.Reader, id[:])
	if err != nil {
		return nil, err
	}
	var msg = Message{
		ID:         &id,
		Recipient:  recipient,
		Provider:   provider,
		Payload:    payload[:],
		WithSURB:   true,
		IsBlocking: isBlocking,
	}
	return &msg, nil
}

// SendReliableMessage asynchronously sends messages with automatic retransmissiosn.
func (s *Session) SendReliableMessage(recipient, provider string, message []byte) (*[cConstants.MessageIDLength]byte, error) {
	msg, err := s.composeMessage(recipient, provider, message, false)
	if err != nil {
		return nil, err
	}
	msg.Reliable = true
	err = s.egressQueue.Push(msg)
	if err != nil {
		return nil, err
	}
	return msg.ID, nil
}

// SendUnreliableMessage asynchronously sends message without any automatic retransmissions.
func (s *Session) SendUnreliableMessage(recipient, provider string, message []byte) (*[cConstants.MessageIDLength]byte, error) {
	msg, err := s.composeMessage(recipient, provider, message, false)
	if err != nil {
		return nil, err
	}
	err = s.egressQueue.Push(msg)
	if err != nil {
		return nil, err
	}
	return msg.ID, nil
}

func (s *Session) BlockingSendUnreliableMessage(recipient, provider string, message []byte) ([]byte, error) {
	msg, err := s.composeMessage(recipient, provider, message, true)
	if err != nil {
		return nil, err
	}
	sentWaitChan := make(chan *Message)
	s.sentWaitChanMap.Store(*msg.ID, sentWaitChan)
	defer s.sentWaitChanMap.Delete(*msg.ID)

	replyWaitChan := make(chan []byte)
	s.replyWaitChanMap.Store(*msg.ID, replyWaitChan)
	defer s.replyWaitChanMap.Delete(*msg.ID)

	err = s.egressQueue.Push(msg)
	if err != nil {
		return nil, err
	}

	// wait until sent so that we know the ReplyETA for the waiting below
	sentMessage := <-sentWaitChan

	// if the message failed to send we will receive a nil message
	if sentMessage == nil {
		return nil, ErrMessageNotSent
	}

	// wait for reply or round trip timeout
	select {
	case reply := <-replyWaitChan:
		return reply, nil
	// these timeouts are often far too aggressive
	case <-time.After(sentMessage.ReplyETA + cConstants.RoundTripTimeSlop):
		return nil, ErrReplyTimeout
	}
	// unreachable
}
<commit_msg>fix doRetransmit log format<commit_after>// send.go - mixnet client send
// Copyright (C) 2018  David Stainton.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package client

import (
	"encoding/binary"
	"encoding/hex"
	"errors"
	"fmt"
	"github.com/katzenpost/client/utils"
	"io"
	"time"

	cConstants "github.com/katzenpost/client/constants"
	"github.com/katzenpost/core/constants"
	"github.com/katzenpost/core/crypto/rand"
	sConstants "github.com/katzenpost/core/sphinx/constants"
)

var ErrReplyTimeout = errors.New("failure waiting for reply, timeout reached")
var ErrMessageNotSent = errors.New("failure sending message")

func (s *Session) sendNext() {
	msg, err := s.egressQueue.Peek()
	if err != nil {
		s.fatalErrCh <- errors.New("impossible failure to Peek from queue")
		return
	}
	if msg == nil {
		s.fatalErrCh <- errors.New("impossible failure, got nil message from queue")
		return
	}
	m := msg.(*Message)
	s.doSend(m)
	_, err = s.egressQueue.Pop()
	if err != nil {
		s.fatalErrCh <- errors.New("impossible failure to Pop from queue")
	}
}

func NewRescheduler(s *Session) *rescheduler {
	r := &rescheduler{s: s}
	s.log.Debugf("Creating TimerQueue")
	r.timerQ = NewTimerQueue(r)
	return r
}

type rescheduler struct {
	s *Session
	timerQ *TimerQueue
}

func (r *rescheduler) Push(i Item) error {
	// rescheduler checks whether a message was ACK'd when the timerQ fires
	// and if it has not, reschedules the message for transmission again
	m := i.(*Message)
	if _, ok := r.s.surbIDMap.Load(*m.SURBID); ok {
		// still waiting for a SURB-ACK that hasn't arrived
		r.s.surbIDMap.Delete(m.SURBID)
		r.s.opCh <- opRetransmit{msg :m}
	}
	return nil
}

func (s *Session) doRetransmit(msg *Message) {
	s.log.Debugf("doRetransmit for %x", *msg.ID)
	msg.Retransmissions++
	s.log.Debugf("retransmissions: %d", msg.Retransmissions)
	s.doSend(msg)
}

func (s *Session) doSend(msg *Message) {
	surbID := [sConstants.SURBIDLength]byte{}
	_, err := io.ReadFull(rand.Reader, surbID[:])
	if err != nil {
		s.fatalErrCh <- fmt.Errorf("impossible failure, failed to generate SURB ID for message ID %x", *msg.ID)
		return
	}
	key := []byte{}
	var eta time.Duration
	if msg.WithSURB {
		msg.SURBID = &surbID
		idStr := fmt.Sprintf("[%v]", hex.EncodeToString(surbID[:]))
		s.log.Debugf("doSend with SURB ID %x", idStr)
		key, eta, err = s.minclient.SendCiphertext(msg.Recipient, msg.Provider, &surbID, msg.Payload)
	} else {
		s.log.Debugf("doSend without SURB")
		err = s.minclient.SendUnreliableCiphertext(msg.Recipient, msg.Provider, msg.Payload)
	}

	// message was sent
	if err == nil {
		msg.SentAt = time.Now()
	}
	// expect a reply
	if msg.WithSURB {
		if err == nil {
			s.log.Debugf("doSend setting ReplyETA to %v", eta)
			// increase the timeout for each retransmission
			msg.ReplyETA = eta * (1 + time.Duration(msg.Retransmissions))
			msg.Key = key
			s.surbIDMap.Store(surbID, msg)
			if msg.Reliable {
				s.log.Debugf("Sending reliable message with retransmissions")
				msg.QueuePriority = uint64(msg.SentAt.Add(msg.ReplyETA).Add(cConstants.RoundTripTimeSlop).UnixNano())
				s.rescheduler.timerQ.Push(msg)
			}
		}
		// write to waiting channel or close channel if message failed to send
		if msg.IsBlocking {
			sentWaitChanRaw, ok := s.sentWaitChanMap.Load(*msg.ID)
			if !ok {
				s.fatalErrCh <- fmt.Errorf("impossible failure, sentWaitChan not found for message ID %x", *msg.ID)
				return
			}
			sentWaitChan := sentWaitChanRaw.(chan *Message)
			if err == nil {
				sentWaitChan <- msg
			} else {
				close(sentWaitChan)
			}
			return
		}
	}
	s.eventCh.In() <- &MessageSentEvent{
		MessageID: msg.ID,
		Err:       err,
		SentAt:    msg.SentAt,
		ReplyETA:  msg.ReplyETA,
	}
}

func (s *Session) sendDropDecoy(loopSvc *utils.ServiceDescriptor) {
	payload := make([]byte, constants.UserForwardPayloadLength)
	id := [cConstants.MessageIDLength]byte{}
	_, err := io.ReadFull(rand.Reader, id[:])
	if err != nil {
		s.fatalErrCh <- errors.New("failure to generate message ID for drop decoy")
		return
	}
	msg := &Message{
		ID:        &id,
		Recipient: loopSvc.Name,
		Provider:  loopSvc.Provider,
		Payload:   payload[:],
		WithSURB:  false,
		IsDecoy:   true,
	}
	s.doSend(msg)
}

func (s *Session) sendLoopDecoy(loopSvc *utils.ServiceDescriptor) {
	s.log.Info("sending loop decoy")
	payload := make([]byte, constants.UserForwardPayloadLength)
	id := [cConstants.MessageIDLength]byte{}
	_, err := io.ReadFull(rand.Reader, id[:])
	if err != nil {
		s.fatalErrCh <- errors.New("failure to generate message ID for loop decoy")
		return
	}
	msg := &Message{
		ID:        &id,
		Recipient: loopSvc.Name,
		Provider:  loopSvc.Provider,
		Payload:   payload[:],
		WithSURB:  true,
		IsDecoy:   true,
	}
	defer s.incrementDecoyLoopTally()
	s.doSend(msg)
}

func (s *Session) composeMessage(recipient, provider string, message []byte, isBlocking bool) (*Message, error) {
	s.log.Debug("SendMessage")
	if len(message) > constants.UserForwardPayloadLength-4 {
		return nil, fmt.Errorf("invalid message size: %v", len(message))
	}
	payload := make([]byte, constants.UserForwardPayloadLength)
	binary.BigEndian.PutUint32(payload[:4], uint32(len(message)))
	copy(payload[4:], message)
	id := [cConstants.MessageIDLength]byte{}
	_, err := io.ReadFull(rand.Reader, id[:])
	if err != nil {
		return nil, err
	}
	var msg = Message{
		ID:         &id,
		Recipient:  recipient,
		Provider:   provider,
		Payload:    payload[:],
		WithSURB:   true,
		IsBlocking: isBlocking,
	}
	return &msg, nil
}

// SendReliableMessage asynchronously sends messages with automatic retransmissiosn.
func (s *Session) SendReliableMessage(recipient, provider string, message []byte) (*[cConstants.MessageIDLength]byte, error) {
	msg, err := s.composeMessage(recipient, provider, message, false)
	if err != nil {
		return nil, err
	}
	msg.Reliable = true
	err = s.egressQueue.Push(msg)
	if err != nil {
		return nil, err
	}
	return msg.ID, nil
}

// SendUnreliableMessage asynchronously sends message without any automatic retransmissions.
func (s *Session) SendUnreliableMessage(recipient, provider string, message []byte) (*[cConstants.MessageIDLength]byte, error) {
	msg, err := s.composeMessage(recipient, provider, message, false)
	if err != nil {
		return nil, err
	}
	err = s.egressQueue.Push(msg)
	if err != nil {
		return nil, err
	}
	return msg.ID, nil
}

func (s *Session) BlockingSendUnreliableMessage(recipient, provider string, message []byte) ([]byte, error) {
	msg, err := s.composeMessage(recipient, provider, message, true)
	if err != nil {
		return nil, err
	}
	sentWaitChan := make(chan *Message)
	s.sentWaitChanMap.Store(*msg.ID, sentWaitChan)
	defer s.sentWaitChanMap.Delete(*msg.ID)

	replyWaitChan := make(chan []byte)
	s.replyWaitChanMap.Store(*msg.ID, replyWaitChan)
	defer s.replyWaitChanMap.Delete(*msg.ID)

	err = s.egressQueue.Push(msg)
	if err != nil {
		return nil, err
	}

	// wait until sent so that we know the ReplyETA for the waiting below
	sentMessage := <-sentWaitChan

	// if the message failed to send we will receive a nil message
	if sentMessage == nil {
		return nil, ErrMessageNotSent
	}

	// wait for reply or round trip timeout
	select {
	case reply := <-replyWaitChan:
		return reply, nil
	// these timeouts are often far too aggressive
	case <-time.After(sentMessage.ReplyETA + cConstants.RoundTripTimeSlop):
		return nil, ErrReplyTimeout
	}
	// unreachable
}
<|endoftext|>


<commit_before>//Package soda provides HTTP GET tools for SODA (Socrata Open Data API) webservices, see http://dev.socrata.com/
package soda

import (
	"encoding/csv"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"sync"
	"time"
)

//GetRequest is a wrapper/container for SODA requests.
//This is NOT safe for use by multiple goroutines as Format, Filters and Query will be overwritten.
//Create a new GetRequest in each goroutine you use or use an OffsetGetRequest
type GetRequest struct {
	apptoken   string
	endpoint   string //endpoint without format (not .json etc at the end)
	Format     string //json, csv etc
	Filters    SimpleFilters
	Query      SoSQL
	Metadata   metadata
	HTTPClient *http.Client //For clients who need a custom HTTP client
}

//NewGetRequest creates a new GET request, the endpoint must be specified without the format.
//For example https://data.ct.gov/resource/hma6-9xbg
func NewGetRequest(endpoint, apptoken string) *GetRequest {
	return &GetRequest{
		apptoken: apptoken,
		endpoint: endpoint,
		Filters:  make(SimpleFilters),
		Metadata: newMetadata(endpoint),
	}
}

//Get executes the HTTP GET request
func (r *GetRequest) Get() (*http.Response, error) {
	//If offset is used we must specify an order
	if r.Query.Offset > 0 && len(r.Query.Order) == 0 {
		return nil, errors.New("Cannot use an offset without setting the order")
	}
	return get(r, r.URLValues().Encode())
}

//GetEndpoint returns the complete SODA URL with format
func (r *GetRequest) GetEndpoint() string {
	if r.Format == "" {
		r.Format = "json"
	}
	return fmt.Sprintf("%s.%s", r.endpoint, r.Format)
}

//URLValues returns the url.Values for the GetRequest
func (r *GetRequest) URLValues() url.Values {
	uv := make(url.Values)
	for key, val := range r.Filters.URLValues() {
		uv[key] = val
	}
	for key, val := range r.Query.URLValues() {
		uv[key] = val
	}
	return uv
}

//Count gets the total number of records in the dataset
//by executing a SODA request
func (r *GetRequest) Count() (uint, error) {

	oldformat := r.Format
	oldorder := r.Query.Order
	oldselect := r.Query.Select
	defer func() {
		r.Format = oldformat
		r.Query.Order = oldorder
		r.Query.Select = oldselect
	}()

	r.Format = "json"
	r.Query.Select = []string{"count(*)"}
	r.Query.ClearOrder()

	resp, err := r.Get()
	if err != nil {
		return 0, err
	}
	defer resp.Body.Close()

	count := []struct {
		Count string
	}{}
	err = json.NewDecoder(resp.Body).Decode(&count)
	if err != nil {
		return 0, err
	}
	if len(count) == 0 {
		return 0, errors.New("Empty count response")
	}
	icount, err := strconv.Atoi(count[0].Count)
	if err != nil {
		return 0, err
	}
	return uint(icount), nil
}

//Fields returns all the fields present in the dataset (ignores select fields).
//Spaces in fieldnames are replaced by underscores.
func (r *GetRequest) Fields() ([]string, error) {

	oldformat := r.Format
	oldorder := r.Query.Order
	oldlimit := r.Query.Limit
	oldselect := r.Query.Select
	defer func() {
		r.Format = oldformat
		r.Query.Select = oldselect
		r.Query.Order = oldorder
		r.Query.Limit = oldlimit
	}()

	fields := []string{}

	r.Format = "csv"
	r.Query.Select = []string{}
	r.Query.Limit = 0
	r.Query.ClearOrder()

	resp, err := r.Get()
	if err != nil {
		return fields, err
	}
	defer resp.Body.Close()

	csvreader := csv.NewReader(resp.Body)
	record, err := csvreader.Read()
	if err != nil && err != io.EOF {
		return fields, err
	}
	for _, field := range record {
		fields = append(fields, strings.Replace(field, " ", "_", -1))
	}
	return fields, nil
}

//Modified returns when the dataset was last updated
func (r *GetRequest) Modified() (time.Time, error) {

	oldformat := r.Format
	oldorder := r.Query.Order
	oldlimit := r.Query.Limit
	oldselect := r.Query.Select
	defer func() {
		r.Format = oldformat
		r.Query.Select = oldselect
		r.Query.Order = oldorder
		r.Query.Limit = oldlimit
	}()

	r.Format = "json"
	r.Query.Select = []string{}
	r.Query.Limit = 0
	r.Query.ClearOrder()

	resp, err := r.Get()
	if err != nil {
		return time.Time{}, err
	}
	defer resp.Body.Close()

	lms := resp.Header.Get("X-Soda2-Truth-Last-Modified")
	if lms == "" {
		lms = resp.Header.Get("Last-Modified")
	}
	if lms == "" {
		return time.Time{}, errors.New("Cannot get last modified date, field not present in HTTP header")
	}

	return time.Parse(time.RFC1123, lms)
}

//SimpleFilters is the easiest way to filter columns for equality.
//Add the column to filter on a map key and the filter value as map value.
//If you include multiple filters, the filters will be combined using a boolean AND.
//See http://dev.socrata.com/docs/filtering.html
type SimpleFilters map[string]string

//URLValues returns the url.Values for the SimpleFilters
func (sf SimpleFilters) URLValues() url.Values {
	uv := make(url.Values)
	for key, val := range sf {
		uv.Add(key, val)
	}
	return uv
}

//SoSQL implements the Socrata Query Language and is used to build more complex queries.
//See http://dev.socrata.com/docs/queries.html
type SoSQL struct {
	Select []string //The set of columns to be returned. Default: All columns, equivalent to $select=*
	Where  string   //Filters the rows to be returned. Default: No filter, and returning a max of $limit values
	Order  []struct {
		Column string //Column name
		Desc   bool   //Descending. Default: false = Ascending
	} //Specifies the order of results. Default: Unspecified order, but it will be consistent across paging
	Group  string //Column to group results on, similar to SQL Grouping. Default: No grouping
	Limit  uint   //Maximum number of results to return. Default: 1000 (with a maximum of 50,000)
	Offset uint   //Offset count into the results to start at, used for paging. Default: 0
	Q      string //Performs a full text search for a value. Default: No search

}

type Direction bool

const (
	DirAsc  Direction = false
	DirDesc Direction = true
)

//AddOrder can be called for each field you want to sort the result on.
//If parameter descending is true, the column will be sorted descending, or ascending if false.
func (sq *SoSQL) AddOrder(column string, dir Direction) {
	sq.Order = append(sq.Order, struct {
		Column string
		Desc   bool
	}{column, bool(dir)})
}

//ClearOrder removes all order fields
func (sq *SoSQL) ClearOrder() {
	sq.Order = []struct {
		Column string
		Desc   bool
	}{}
}

//URLValues returns the url.Values for the SoSQL query
func (sq *SoSQL) URLValues() url.Values {
	uv := make(url.Values)
	if len(sq.Select) > 0 {
		uv.Add("$select", strings.Join(sq.Select, ","))
	}
	if len(sq.Where) > 0 {
		uv.Add("$where", sq.Where)
	}
	if len(sq.Order) > 0 {
		order := []string{}
		for _, o := range sq.Order {
			if o.Desc {
				order = append(order, o.Column+" DESC")
			} else {
				order = append(order, o.Column+" ASC")
			}
		}
		uv.Add("$order", strings.Join(order, ","))
	}
	if len(sq.Q) > 0 {
		uv.Add("$q", sq.Q)
	}
	if len(sq.Group) > 0 {
		uv.Add("$group", sq.Group)
	}
	if sq.Limit > 0 {
		uv.Add("$limit", fmt.Sprintf("%d", sq.Limit))
	}
	if sq.Offset > 0 {
		uv.Add("$offset", fmt.Sprintf("%d", sq.Offset))
	}
	return uv
}

//OffsetGetRequest is a request getter that gets all the records using the filters and limits from gr and
//is safe to use by multiple goroutines, use Next(number) to get the next number of records.
//A sync.WaitGroup is embedded for easy concurrency.
type OffsetGetRequest struct {
	gr     *GetRequest
	m      sync.Mutex
	offset uint
	count  uint
	sync.WaitGroup
}

//ErrDone is returned by OffsetGetRequest.Next when done
var ErrDone = errors.New("Done")

//Next gets the next number of records
func (o *OffsetGetRequest) Next(number uint) (*http.Response, error) {
	o.m.Lock() //lock to protect offset
	if o.IsDone() {
		o.m.Unlock()
		return nil, ErrDone
	}
	if len(o.gr.Query.Order) == 0 { //If offset is used we must specify an order
		return nil, errors.New("Cannot use an offset without setting the order")
	}
	if o.offset+number > o.count {
		number = o.count - o.offset
	}
	o.gr.Query.Offset = o.offset
	o.gr.Query.Limit = number
	rawquery := o.gr.URLValues().Encode()
	o.offset += number
	o.m.Unlock() //unlock before the request is done
	return get(o.gr, rawquery)
}

//Count returns the number of records from memory
func (o *OffsetGetRequest) Count() uint {
	return o.count
}

//IsDone returns if we have gotten all records
func (o *OffsetGetRequest) IsDone() bool {
	return o.offset >= o.count
}

//NewOffsetGetRequest creates a new OffsetGetRequest from gr
//and does a count request to determine the number of records to get
func NewOffsetGetRequest(gr *GetRequest) (*OffsetGetRequest, error) {
	count, err := gr.Count()
	if err != nil {
		return nil, err
	}
	return &OffsetGetRequest{gr: gr, offset: 0, count: count}, nil
}

//get is the function that executes the HTTP request
func get(r *GetRequest, rawquery string) (*http.Response, error) {

	client := http.DefaultClient
	if r.HTTPClient != nil {
		client = r.HTTPClient
	}

	req, err := http.NewRequest("GET", r.GetEndpoint(), nil)
	if err != nil {
		return nil, err
	}
	req.URL.RawQuery = rawquery
	req.Header.Set("X-App-Token", r.apptoken)

	//Execute
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode >= 400 {
		errMsg, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, err
		}
		return nil, fmt.Errorf("SODA error %d:\nURL: GET %s\nResponse: %s", resp.StatusCode, req.URL.String(), errMsg)
	}

	return resp, nil
}
<commit_msg>cleaned up the code to use new go coding standards<commit_after>// Package soda provides HTTP GET tools for SODA (Socrata Open Data API) webservices, see http://dev.socrata.com/
package soda

import (
	"encoding/csv"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"sync"
	"time"
)

// GetRequest is a wrapper/container for SODA requests.
// This is NOT safe for use by multiple goroutines as Format, Filters and Query will be overwritten.
// Create a new GetRequest in each goroutine you use or use an OffsetGetRequest
type GetRequest struct {
	apptoken   string
	endpoint   string //endpoint without format (not .json etc at the end)
	Format     string //json, csv etc
	Filters    SimpleFilters
	Query      SoSQL
	Metadata   metadata
	HTTPClient *http.Client //For clients who need a custom HTTP client
}

// NewGetRequest creates a new GET request, the endpoint must be specified without the format.
// For example https://data.ct.gov/resource/hma6-9xbg
func NewGetRequest(endpoint, apptoken string) *GetRequest {
	return &GetRequest{
		apptoken: apptoken,
		endpoint: endpoint,
		Filters:  make(SimpleFilters),
		Metadata: newMetadata(endpoint),
	}
}

// Get executes the HTTP GET request
func (r *GetRequest) Get() (*http.Response, error) {
	//If offset is used we must specify an order
	if r.Query.Offset > 0 && len(r.Query.Order) == 0 {
		return nil, errors.New("cannot use an offset without setting the order")
	}
	return get(r, r.URLValues().Encode())
}

// GetEndpoint returns the complete SODA URL with format
func (r *GetRequest) GetEndpoint() string {
	if r.Format == "" {
		r.Format = "json"
	}
	return fmt.Sprintf("%s.%s", r.endpoint, r.Format)
}

// URLValues returns the url.Values for the GetRequest
func (r *GetRequest) URLValues() url.Values {
	uv := make(url.Values)
	for key, val := range r.Filters.URLValues() {
		uv[key] = val
	}
	for key, val := range r.Query.URLValues() {
		uv[key] = val
	}
	return uv
}

// Count gets the total number of records in the dataset
// by executing a SODA request
func (r *GetRequest) Count() (uint, error) {

	oldformat := r.Format
	oldorder := r.Query.Order
	oldselect := r.Query.Select
	defer func() {
		r.Format = oldformat
		r.Query.Order = oldorder
		r.Query.Select = oldselect
	}()

	r.Format = "json"
	r.Query.Select = []string{"count(*)"}
	r.Query.ClearOrder()

	resp, err := r.Get()
	if err != nil {
		return 0, err
	}
	defer resp.Body.Close()

	count := make([]struct {
		Count string
	}, 0)
	err = json.NewDecoder(resp.Body).Decode(&count)
	if err != nil {
		return 0, err
	}
	if len(count) == 0 {
		return 0, errors.New("empty count response")
	}
	icount, err := strconv.Atoi(count[0].Count)
	if err != nil {
		return 0, err
	}
	return uint(icount), nil
}

// Fields returns all the fields present in the dataset (ignores select fields).
// Spaces in fieldnames are replaced by underscores.
func (r *GetRequest) Fields() ([]string, error) {

	oldformat := r.Format
	oldorder := r.Query.Order
	oldlimit := r.Query.Limit
	oldselect := r.Query.Select
	defer func() {
		r.Format = oldformat
		r.Query.Select = oldselect
		r.Query.Order = oldorder
		r.Query.Limit = oldlimit
	}()

	r.Format = "csv"
	r.Query.Select = []string{}
	r.Query.Limit = 0
	r.Query.ClearOrder()

	resp, err := r.Get()
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	csvreader := csv.NewReader(resp.Body)
	record, err := csvreader.Read()
	if err != nil && err != io.EOF {
		return nil, err
	}
	fields := make([]string, len(record))
	for i := range record {
		fields[i] = strings.Replace(record[i], " ", "_", -1)
	}
	return fields, nil
}

// Modified returns when the dataset was last updated
func (r *GetRequest) Modified() (time.Time, error) {

	oldformat := r.Format
	oldorder := r.Query.Order
	oldlimit := r.Query.Limit
	oldselect := r.Query.Select
	defer func() {
		r.Format = oldformat
		r.Query.Select = oldselect
		r.Query.Order = oldorder
		r.Query.Limit = oldlimit
	}()

	r.Format = "json"
	r.Query.Select = []string{}
	r.Query.Limit = 0
	r.Query.ClearOrder()

	resp, err := r.Get()
	if err != nil {
		return time.Time{}, err
	}
	defer resp.Body.Close()

	lms := resp.Header.Get("X-Soda2-Truth-Last-Modified")
	if lms == "" {
		lms = resp.Header.Get("Last-Modified")
	}
	if lms == "" {
		return time.Time{}, errors.New("cannot get last modified date, field not present in HTTP header")
	}

	return time.Parse(time.RFC1123, lms)
}

// SimpleFilters is the easiest way to filter columns for equality.
// Add the column to filter on a map key and the filter value as map value.
// If you include multiple filters, the filters will be combined using a boolean AND.
// See http://dev.socrata.com/docs/filtering.html
type SimpleFilters map[string]string

//URLValues returns the url.Values for the SimpleFilters
func (sf SimpleFilters) URLValues() url.Values {
	uv := make(url.Values)
	for key, val := range sf {
		uv.Add(key, val)
	}
	return uv
}

// SoSQL implements the Socrata Query Language and is used to build more complex queries.
// See http://dev.socrata.com/docs/queries.html
type SoSQL struct {
	Select []string //The set of columns to be returned. Default: All columns, equivalent to $select=*
	Where  string   //Filters the rows to be returned. Default: No filter, and returning a max of $limit values
	Order  []struct {
		Column string //Column name
		Desc   bool   //Descending. Default: false = Ascending
	} //Specifies the order of results. Default: Unspecified order, but it will be consistent across paging
	Group  string //Column to group results on, similar to SQL Grouping. Default: No grouping
	Limit  uint   //Maximum number of results to return. Default: 1000 (with a maximum of 50,000)
	Offset uint   //Offset count into the results to start at, used for paging. Default: 0
	Q      string //Performs a full text search for a value. Default: No search

}

type Direction bool

const (
	DirAsc  Direction = false
	DirDesc Direction = true
)

// AddOrder can be called for each field you want to sort the result on.
// If parameter descending is true, the column will be sorted descending, or ascending if false.
func (sq *SoSQL) AddOrder(column string, dir Direction) {
	sq.Order = append(sq.Order, struct {
		Column string
		Desc   bool
	}{column, bool(dir)})
}

// ClearOrder removes all order fields
func (sq *SoSQL) ClearOrder() {
	sq.Order = []struct {
		Column string
		Desc   bool
	}{}
}

// URLValues returns the url.Values for the SoSQL query
func (sq *SoSQL) URLValues() url.Values {
	uv := make(url.Values)
	if len(sq.Select) > 0 {
		uv.Add("$select", strings.Join(sq.Select, ","))
	}
	if len(sq.Where) > 0 {
		uv.Add("$where", sq.Where)
	}
	if len(sq.Order) > 0 {
		order := make([]string, 0)
		for _, o := range sq.Order {
			if o.Desc {
				order = append(order, o.Column+" DESC")
			} else {
				order = append(order, o.Column+" ASC")
			}
		}
		uv.Add("$order", strings.Join(order, ","))
	}
	if len(sq.Q) > 0 {
		uv.Add("$q", sq.Q)
	}
	if len(sq.Group) > 0 {
		uv.Add("$group", sq.Group)
	}
	if sq.Limit > 0 {
		uv.Add("$limit", fmt.Sprintf("%d", sq.Limit))
	}
	if sq.Offset > 0 {
		uv.Add("$offset", fmt.Sprintf("%d", sq.Offset))
	}
	return uv
}

// OffsetGetRequest is a request getter that gets all the records using the filters and limits from gr and
// is safe to use by multiple goroutines, use Next(number) to get the next number of records.
// A sync.WaitGroup is embedded for easy concurrency.
type OffsetGetRequest struct {
	gr     *GetRequest
	m      sync.Mutex
	offset uint
	count  uint
	sync.WaitGroup
}

// ErrDone is returned by OffsetGetRequest.Next when done
var ErrDone = errors.New("Done")

// Next gets the next number of records
func (o *OffsetGetRequest) Next(number uint) (*http.Response, error) {
	o.m.Lock() //lock to protect offset
	if o.IsDone() {
		o.m.Unlock()
		return nil, ErrDone
	}
	if len(o.gr.Query.Order) == 0 { //If offset is used we must specify an order
		return nil, errors.New("cannot use an offset without setting the order")
	}
	if o.offset+number > o.count {
		number = o.count - o.offset
	}
	o.gr.Query.Offset = o.offset
	o.gr.Query.Limit = number
	rawquery := o.gr.URLValues().Encode()
	o.offset += number
	o.m.Unlock() //unlock before the request is done
	return get(o.gr, rawquery)
}

// Count returns the number of records from memory
func (o *OffsetGetRequest) Count() uint {
	return o.count
}

// IsDone returns if we have gotten all records
func (o *OffsetGetRequest) IsDone() bool {
	return o.offset >= o.count
}

// NewOffsetGetRequest creates a new OffsetGetRequest from gr
// and does a count request to determine the number of records to get
func NewOffsetGetRequest(gr *GetRequest) (*OffsetGetRequest, error) {
	count, err := gr.Count()
	if err != nil {
		return nil, err
	}
	return &OffsetGetRequest{gr: gr, offset: 0, count: count}, nil
}

// get is the function that executes the HTTP request
func get(r *GetRequest, rawquery string) (*http.Response, error) {

	client := http.DefaultClient
	if r.HTTPClient != nil {
		client = r.HTTPClient
	}

	req, err := http.NewRequest("GET", r.GetEndpoint(), nil)
	if err != nil {
		return nil, err
	}
	req.URL.RawQuery = rawquery
	req.Header.Set("X-App-Token", r.apptoken)

	// Execute
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode >= 400 {
		errMsg, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, err
		}
		return nil, fmt.Errorf("SODA error %d:\nURL: GET %s\nResponse: %s", resp.StatusCode, req.URL.String(), errMsg)
	}

	return resp, nil
}
<|endoftext|>


<commit_before>package zipkintracer

import (
	"fmt"
	"time"

	otobserver "github.com/opentracing-contrib/go-observer"
	opentracing "github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/ext"
	"github.com/opentracing/opentracing-go/log"
	"github.com/openzipkin/zipkin-go"
)

// FinisherWithDuration allows to finish span with given duration
type FinisherWithDuration interface {
	FinishedWithDuration(d time.Duration)
}

type spanImpl struct {
	tracer     *tracerImpl
	zipkinSpan zipkin.Span
	startTime  time.Time
	observer   otobserver.SpanObserver
}

func (s *spanImpl) SetOperationName(operationName string) opentracing.Span {
	if s.observer != nil {
		s.observer.OnSetOperationName(operationName)
	}

	s.zipkinSpan.SetName(operationName)
	return s
}

func (s *spanImpl) SetTag(key string, value interface{}) opentracing.Span {
	if s.observer != nil {
		s.observer.OnSetTag(key, value)
	}

	if key == string(ext.SamplingPriority) {
		if v, ok := value.(uint16); ok {
			sampling := v != 0
			nc := s.zipkinSpan.Context()
			nc.Sampled = &sampling
			// TODO change sampling in span
			return s
		}
	}
	s.zipkinSpan.Tag(key, fmt.Sprint(value))
	return s
}

func (s *spanImpl) LogKV(keyValues ...interface{}) {
	fields, err := log.InterleavedKVToFields(keyValues...)
	if err != nil {
		return
	}

	for _, field := range fields {
		s.zipkinSpan.Annotate(time.Now(), field.String())
	}
}

func (s *spanImpl) LogFields(fields ...log.Field) {
	s.logFields(time.Now(), fields...)
}

func (s *spanImpl) logFields(t time.Time, fields ...log.Field) {
	for _, field := range fields {
		s.zipkinSpan.Annotate(t, field.String())
	}
}

func (s *spanImpl) LogEvent(event string) {
	// Deprecated: do nothing
}

func (s *spanImpl) LogEventWithPayload(event string, payload interface{}) {
	// Deprecated: do nothing
}

func (s *spanImpl) Log(ld opentracing.LogData) {
	// Deprecated: do nothing
}

func (s *spanImpl) Finish() {
	s.zipkinSpan.Finish()
}

func (s *spanImpl) FinishWithOptions(opts opentracing.FinishOptions) {
	if s.observer != nil {
		s.observer.OnFinish(opts)
	}

	for _, lr := range opts.LogRecords {
		s.logFields(lr.Timestamp, lr.Fields...)
	}

	if !opts.FinishTime.IsZero() {
		f, ok := s.zipkinSpan.(FinisherWithDuration)
		if !ok {
			return
		}
		f.FinishedWithDuration(opts.FinishTime.Sub(s.startTime))
		return
	}

	s.Finish()
}

func (s *spanImpl) Tracer() opentracing.Tracer {
	return s.tracer
}

func (s *spanImpl) Context() opentracing.SpanContext {
	return &spanContextImpl{zipkinContext: s.zipkinSpan.Context()}

}

func (s *spanImpl) SetBaggageItem(key, val string) opentracing.Span {
	// Do nothing for now
	return s
}

func (s *spanImpl) BaggageItem(key string) string {
	return ""
}
<commit_msg>chore: fixes the deprecated log methods.<commit_after>package zipkintracer

import (
	"fmt"
	"time"

	otobserver "github.com/opentracing-contrib/go-observer"
	opentracing "github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/ext"
	"github.com/opentracing/opentracing-go/log"
	"github.com/openzipkin/zipkin-go"
)

// FinisherWithDuration allows to finish span with given duration
type FinisherWithDuration interface {
	FinishedWithDuration(d time.Duration)
}

type spanImpl struct {
	tracer     *tracerImpl
	zipkinSpan zipkin.Span
	startTime  time.Time
	observer   otobserver.SpanObserver
}

func (s *spanImpl) SetOperationName(operationName string) opentracing.Span {
	if s.observer != nil {
		s.observer.OnSetOperationName(operationName)
	}

	s.zipkinSpan.SetName(operationName)
	return s
}

func (s *spanImpl) SetTag(key string, value interface{}) opentracing.Span {
	if s.observer != nil {
		s.observer.OnSetTag(key, value)
	}

	if key == string(ext.SamplingPriority) {
		// there are no means for now to change the sampling decision
		// but when finishedSpanHandler is in place we could change this.
		return s
	}
	s.zipkinSpan.Tag(key, fmt.Sprint(value))
	return s
}

func (s *spanImpl) LogKV(keyValues ...interface{}) {
	fields, err := log.InterleavedKVToFields(keyValues...)
	if err != nil {
		return
	}

	for _, field := range fields {
		s.zipkinSpan.Annotate(time.Now(), field.String())
	}
}

func (s *spanImpl) LogFields(fields ...log.Field) {
	s.logFields(time.Now(), fields...)
}

func (s *spanImpl) logFields(t time.Time, fields ...log.Field) {
	for _, field := range fields {
		s.zipkinSpan.Annotate(t, field.String())
	}
}

func (s *spanImpl) LogEvent(event string) {
	s.Log(opentracing.LogData{
		Event: event,
	})
}

func (s *spanImpl) LogEventWithPayload(event string, payload interface{}) {
	s.Log(opentracing.LogData{
		Event:   event,
		Payload: payload,
	})
}

func (s *spanImpl) Log(ld opentracing.LogData) {
	if ld.Timestamp.IsZero() {
		ld.Timestamp = time.Now()
	}

	for _, f := range ld.ToLogRecord().Fields {
		s.zipkinSpan.Annotate(ld.Timestamp, f.String())
	}
}

func (s *spanImpl) Finish() {
	s.zipkinSpan.Finish()
}

func (s *spanImpl) FinishWithOptions(opts opentracing.FinishOptions) {
	if s.observer != nil {
		s.observer.OnFinish(opts)
	}

	for _, lr := range opts.LogRecords {
		s.logFields(lr.Timestamp, lr.Fields...)
	}

	if !opts.FinishTime.IsZero() {
		f, ok := s.zipkinSpan.(FinisherWithDuration)
		if !ok {
			return
		}
		f.FinishedWithDuration(opts.FinishTime.Sub(s.startTime))
		return
	}

	s.Finish()
}

func (s *spanImpl) Tracer() opentracing.Tracer {
	return s.tracer
}

func (s *spanImpl) Context() opentracing.SpanContext {
	return &spanContextImpl{zipkinContext: s.zipkinSpan.Context()}

}

func (s *spanImpl) SetBaggageItem(key, val string) opentracing.Span {
	// Do nothing for now
	return s
}

func (s *spanImpl) BaggageItem(key string) string {
	return ""
}
<|endoftext|>


<commit_before>package sroc

import (
	"net/http"
)

const (
	ANY_ORIGIN  = "*"
	ALL_METHODS = "GET, POST, PUT, DELETE"
)

type CORS struct {
	AllowOrigin  string
	AllowMethods string
	AllowHeaders string
}

func Create(origin, methods, headers string) (sroc *CORS) {
	sroc = new(CORS)
	sroc.AllowOrigin = origin
	sroc.AllowMethods = methods
	sroc.AllowHeaders = headers
	return sroc
}

func (s *CORS) AllowCORS(res http.ResponseWriter) {
	res.Header().Set("Access-Control-Allow-Origin", s.AllowOrigin)
}

func (s *CORS) Preflighted(res http.ResponseWriter) {
	res.Header().Set("Access-Control-Allow-Origin", s.AllowOrigin)
	res.Header().Set("Access-Control-Allow-Methods", s.AllowMethods)
	res.Header().Set("Access-Control-Allow-Headers", s.AllowHeaders)
	res.Header().Set("Access-Control-Allow-Credentials", "false")
	res.Write(nil)
}
<commit_msg>MOD: expose headers<commit_after>package sroc

import (
	"net/http"
)

const (
	ANY_ORIGIN  = "*"
	ALL_METHODS = "GET, POST, PUT, DELETE"
)

type CORS struct {
	AllowOrigin  string
	AllowMethods string
	AllowHeaders string
}

func Create(origin, methods, headers string) (sroc *CORS) {
	sroc = new(CORS)
	sroc.AllowOrigin = origin
	sroc.AllowMethods = methods
	sroc.AllowHeaders = headers
	return sroc
}

func (s *CORS) AllowCORS(res http.ResponseWriter) {
	res.Header().Set("Access-Control-Allow-Origin", s.AllowOrigin)
	res.Header().Set("Access-Control-Expose-Headers", s.AllowHeaders)
}

func (s *CORS) Preflighted(res http.ResponseWriter) {
	res.Header().Set("Access-Control-Allow-Origin", s.AllowOrigin)
	res.Header().Set("Access-Control-Allow-Methods", s.AllowMethods)
	res.Header().Set("Access-Control-Allow-Headers", s.AllowHeaders)
	res.Header().Set("Access-Control-Allow-Credentials", "false")
	res.Write(nil)
}
<|endoftext|>


<commit_before>/*******************************************************************************
The MIT License (MIT)

Copyright (c) 2015 Hajime Nakagami

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*******************************************************************************/

package main

import (
	"net"
)

func initialize_stub() {
}

func login(clientID string, loginName string, loginPassword string) byte {
	return CONNACK_Success
}

func logout(clientID string) {
}

func getNextMessageID(clientID string) uint16 {
	return 1
}

func getClientListByTopic(topicName string) []string {
	return []string{}
}

func sendToClient(data []byte, clientID string) bool {
	return true
}

func sendToConn(data []byte, conn net.Conn) {
	conn.Write(data)
}

func subscribe(topicID string, clientID string) byte {
	return 0 // QoS
}

func unsubscribe(topicID string, clientID string) {
	return
}
<commit_msg>add Client and Topic struct<commit_after>/*******************************************************************************
The MIT License (MIT)

Copyright (c) 2015 Hajime Nakagami

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*******************************************************************************/

package main

import (
	"github.com/streamrail/concurrent-map"
	"net"
	"sync"
)

// ------------------------------ Client --------------------------------------

type Client struct {
	sync.RWMutex
}

// ------------------------------ Topic ---------------------------------------

type Topic struct {
	Name string
	m    map[string]bool
	sync.RWMutex
}

func NewTopic(name string) *Topic {
	return &Topic{
		Name: name,
		m:    make(map[string]bool),
	}
}

func (s *Topic) Add(name string) {
	s.Lock()
	defer s.Unlock()
	s.m[name] = true
}

func (s *Topic) Remove(name string) {
	s.Lock()
	defer s.Unlock()
	delete(s.m, name)
}

func (s *Topic) Clear() {
	s.Lock()
	defer s.Unlock()
	s.m = make(map[string]bool)
}

func (s *Topic) List() []string {
	s.RLock()
	defer s.RUnlock()
	list := make([]string, 0)
	for item := range s.m {
		list = append(list, item)
	}
	return list
}

// -----------------------------------------------------------------------

func initialize_stub() {
}

func login(clientID string, loginName string, loginPassword string) byte {
	return CONNACK_Success
}

func logout(clientID string) {
}

func getNextMessageID(clientID string) uint16 {
	return 1
}

func getClientListByTopic(topicName string) []string {
	return []string{}
}

func sendToClient(data []byte, clientID string) bool {
	return true
}

func sendToConn(data []byte, conn net.Conn) {
	conn.Write(data)
}

func subscribe(topicID string, clientID string) byte {
	return 0 // QoS
}

func unsubscribe(topicID string, clientID string) {
	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"path/filepath"
	"sort"
	"strings"
	"time"
)

// Directory pairs for sync post-processing of directories
type dirpair struct {
	src string
	dst string
}

// Generate a destination path based on the source directory and
// path under that directory.
func destPath(srcdir string, dstdir string, srcfile string) string {
	var (
		sdir     []string
		sfile    []string
		ddir     []string
		dst      []string
		barefile []string
	)

	// Start with source dir with all relative path elements removed.
	sdir = []string{}
	for _, v := range strings.Split(srcdir, "/") {
		if v != "." && v != ".." && v != "" {
			sdir = append(sdir, v)
		}
	}

	// Convert to string removing empty elements, etc
	sfile = []string{}
	for _, v := range strings.Split(srcfile, "/") {
		if v != "." && v != ".." && v != "" {
			sfile = append(sfile, v)
		}
	}
	ddir = []string{}
	for _, v := range strings.Split(dstdir, "/") {
		if v != "." && v != ".." && v != "" {
			ddir = append(ddir, v)
		}
	}

	// source file with the source directory part removed
	barefile = sfile[len(sdir):]

	if strings.HasSuffix(srcdir, "/") {
		// Copy files INTO directory at destination.  full destination path is
		// the destionation directory + the source file with srcdir removed.
		dst = ddir
		dst = append(dst, barefile...)
	} else {
		// Original path does not end in "/". We preserve the
		// last path element of srcdir into the destination.
		dst = ddir
		if len(sdir) > 0 {
			dst = append(dst, sdir[len(sdir)-1])
		}
		dst = append(dst, barefile...)
	}

	return strings.Join(dst, "/")
}

// Determine if we need to copy the file pointed by srcpath in srcvfs to
// the file dstpath in dstvfs.
//
// Return:
// 	 bool
// 	 error
func needToCopy(srcvfs gsyncVfs, dstvfs gsyncVfs, srcpath string, dstpath string) (bool, error) {
	// If destination doesn't exist we need to copy
	exists, err := dstvfs.FileExists(dstpath)
	if err != nil {
		return false, err
	}
	if !exists {
		return true, nil
	}

	// If destination exists, we check mtimes truncated to the nearest second
	srcMtime, err := srcvfs.Mtime(srcpath)
	if err != nil {
		return false, err
	}
	dstMtime, err := dstvfs.Mtime(dstpath)
	if err != nil {
		return false, err
	}

	srcMtime = srcMtime.Truncate(time.Second)
	dstMtime = dstMtime.Truncate(time.Second)

	if srcMtime.After(dstMtime) {
		return true, nil
	}

	return false, nil
}

// Return true if the passed path matches one of the patterns in the exclusion
// list (opt.exclude).
//
// Return:
//   bool
//   error

func excluded(pathname string) (bool, error) {
	for _, excpat := range opt.exclude {
		match, err := filepath.Match(excpat, pathname)
		if err != nil {
			return false, err
		}
		if match {
			return match, err
		}
	}
	return false, nil
}

// Copy the content of all files/directories pointed by srcpath into dstdir.
// If srcpath is a file, the file will be copied. If it is a directory, the
// entire subtree will be copied.  Dstdir must be a directory.
//
// Like rsync, a source path ending in slash means "copy the contents of this
// directory into the destination" whereas a path not ending in a slash means
// "copy this directory and its contents into the destination."
//
// Files/directories are only copied if needed (based on the modification date
// of the file on both filesystems.) This function uses the srcvfs and dstvfs
// VFS objects to perform operations on the respective filesystems.
//
// Return:
// 	 error
func sync(srcpath string, dstdir string, srcvfs gsyncVfs, dstvfs gsyncVfs) error {
	var (
		srctree  []string
		dirpairs []dirpair
	)

	// Destination must exist and be a directory
	exists, err := dstvfs.FileExists(dstdir)
	if err != nil {
		return err
	}
	if !exists {
		return fmt.Errorf("Destination \"%s\" does not exist", dstdir)
	}

	isdir, err := dstvfs.IsDir(dstdir)
	if err != nil {
		return err
	}
	if !isdir {
		return fmt.Errorf("Destination \"%s\" is not a directory/folder", dstdir)
	}

	// Special case: If the source path is not a directory, we short circuit
	// the FileTree method here and set srctree to that single file.
	isdir, err = srcvfs.IsDir(srcpath)
	if err != nil {
		return err
	}
	if isdir {
		srctree, err = srcvfs.FileTree(srcpath)
		if err != nil {
			return err
		}
	} else {
		srctree = []string{srcpath}
	}

	// Guarantee that we'll process a directory before files inside it
	sort.Strings(srctree)

	for _, src := range srctree {
		// Check for exclusions (--exclude)
		exc, err := excluded(src)
		if err != nil {
			return err
		}
		if exc {
			log.Verboseln(2, src, "excluded from copy")
			continue
		}

		dst := destPath(srcpath, dstdir, src)

		isdir, err := srcvfs.IsDir(src)
		if err != nil {
			return err
		}
		isregular, err := srcvfs.IsRegular(src)
		if err != nil {
			return err
		}

		// Start sync operation

		if isdir {
			// Create destination dir if needed
			exists, err := dstvfs.FileExists(dst)
			if err != nil {
				return err
			}
			if !exists {
				log.Verboseln(1, dst)
				if !opt.dryrun {
					err := dstvfs.Mkdir(dst)
					if err != nil {
						return err
					}
				}
			}
			// Save directory for post processing
			d := dirpair{src, dst}
			dirpairs = append(dirpairs, d)
		} else if isregular {
			copyNeeded, err := needToCopy(srcvfs, dstvfs, src, dst)
			if err != nil {
				return err
			}

			if copyNeeded {
				if !opt.dryrun {
					r, err := srcvfs.ReadFromFile(src)
					if err != nil {
						log.Printf("Warning: Skipping \"%s\": %v\n", src, err)
						continue
					}
					err = dstvfs.WriteToFile(dst, r)
					if err != nil {
						return err
					}
					// Set destination mtime == source mtime
					mtime, err := srcvfs.Mtime(src)
					if err != nil {
						return err
					}
					err = dstvfs.SetMtime(dst, mtime)
					if err != nil {
						return err
					}
				}
				log.Verboseln(1, dst)
			}
		} else {
			log.Printf("Warning: Skipping \"%s\": not a regular file or directory.\n", src)
			continue
		}
	}

	// Set the mtimes of all destination directories to the original mtimes.
	// We have to do it here (and bottom first!) because in certain filesystems,
	// updating files inside directories will also change the directory mtime.

	if !opt.dryrun {
		for ix := len(dirpairs) - 1; ix >= 0; ix-- {
			src := dirpairs[ix].src
			dst := dirpairs[ix].dst

			mtime, err := srcvfs.Mtime(src)
			if err != nil {
				return err
			}
			err = dstvfs.SetMtime(dst, mtime)
			if err != nil {
				return err
			}
		}
	}

	return nil
}
<commit_msg>Bugfix (excluded patterns).<commit_after>package main

import (
	"fmt"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"time"
)

// Directory pairs for sync post-processing of directories
type dirpair struct {
	src string
	dst string
}

// Generate a destination path based on the source directory and
// path under that directory.
func destPath(srcdir string, dstdir string, srcfile string) string {
	var (
		sdir     []string
		sfile    []string
		ddir     []string
		dst      []string
		barefile []string
	)

	// Start with source dir with all relative path elements removed.
	sdir = []string{}
	for _, v := range strings.Split(srcdir, "/") {
		if v != "." && v != ".." && v != "" {
			sdir = append(sdir, v)
		}
	}

	// Convert to string removing empty elements, etc
	sfile = []string{}
	for _, v := range strings.Split(srcfile, "/") {
		if v != "." && v != ".." && v != "" {
			sfile = append(sfile, v)
		}
	}
	ddir = []string{}
	for _, v := range strings.Split(dstdir, "/") {
		if v != "." && v != ".." && v != "" {
			ddir = append(ddir, v)
		}
	}

	// source file with the source directory part removed
	barefile = sfile[len(sdir):]

	if strings.HasSuffix(srcdir, "/") {
		// Copy files INTO directory at destination.  full destination path is
		// the destionation directory + the source file with srcdir removed.
		dst = ddir
		dst = append(dst, barefile...)
	} else {
		// Original path does not end in "/". We preserve the
		// last path element of srcdir into the destination.
		dst = ddir
		if len(sdir) > 0 {
			dst = append(dst, sdir[len(sdir)-1])
		}
		dst = append(dst, barefile...)
	}

	return strings.Join(dst, "/")
}

// Determine if we need to copy the file pointed by srcpath in srcvfs to
// the file dstpath in dstvfs.
//
// Return:
// 	 bool
// 	 error
func needToCopy(srcvfs gsyncVfs, dstvfs gsyncVfs, srcpath string, dstpath string) (bool, error) {
	// If destination doesn't exist we need to copy
	exists, err := dstvfs.FileExists(dstpath)
	if err != nil {
		return false, err
	}
	if !exists {
		return true, nil
	}

	// If destination exists, we check mtimes truncated to the nearest second
	srcMtime, err := srcvfs.Mtime(srcpath)
	if err != nil {
		return false, err
	}
	dstMtime, err := dstvfs.Mtime(dstpath)
	if err != nil {
		return false, err
	}

	srcMtime = srcMtime.Truncate(time.Second)
	dstMtime = dstMtime.Truncate(time.Second)

	if srcMtime.After(dstMtime) {
		return true, nil
	}

	return false, nil
}

// Return true if the passed path matches one of the patterns in the exclusion
// list (opt.exclude).
//
// Return:
//   bool
//   error

func excluded(pathname string) (bool, error) {
	fname := path.Base(pathname)
	for _, excpat := range opt.exclude {
		log.Verbosef(3, "attempting to match %q to pattern %q", pathname, excpat)
		match, err := filepath.Match(excpat, fname)
		if err != nil {
			return false, err
		}
		if match {
			log.Verbosef(3, "excluding %q: matched %q", pathname, excpat)
			return match, err
		}
	}
	return false, nil
}

// Copy the content of all files/directories pointed by srcpath into dstdir.
// If srcpath is a file, the file will be copied. If it is a directory, the
// entire subtree will be copied.  Dstdir must be a directory.
//
// Like rsync, a source path ending in slash means "copy the contents of this
// directory into the destination" whereas a path not ending in a slash means
// "copy this directory and its contents into the destination."
//
// Files/directories are only copied if needed (based on the modification date
// of the file on both filesystems.) This function uses the srcvfs and dstvfs
// VFS objects to perform operations on the respective filesystems.
//
// Return:
// 	 error
func sync(srcpath string, dstdir string, srcvfs gsyncVfs, dstvfs gsyncVfs) error {
	var (
		srctree  []string
		dirpairs []dirpair
	)

	// Destination must exist and be a directory
	exists, err := dstvfs.FileExists(dstdir)
	if err != nil {
		return err
	}
	if !exists {
		return fmt.Errorf("Destination \"%s\" does not exist", dstdir)
	}

	isdir, err := dstvfs.IsDir(dstdir)
	if err != nil {
		return err
	}
	if !isdir {
		return fmt.Errorf("Destination \"%s\" is not a directory/folder", dstdir)
	}

	// Special case: If the source path is not a directory, we short circuit
	// the FileTree method here and set srctree to that single file.
	isdir, err = srcvfs.IsDir(srcpath)
	if err != nil {
		return err
	}
	if isdir {
		srctree, err = srcvfs.FileTree(srcpath)
		if err != nil {
			return err
		}
	} else {
		srctree = []string{srcpath}
	}

	// Guarantee that we'll process a directory before files inside it
	sort.Strings(srctree)

	for _, src := range srctree {
		// Check for exclusions (--exclude)
		exc, err := excluded(src)
		if err != nil {
			return err
		}
		if exc {
			log.Verboseln(2, src, "excluded from copy")
			continue
		}

		dst := destPath(srcpath, dstdir, src)

		isdir, err := srcvfs.IsDir(src)
		if err != nil {
			return err
		}
		isregular, err := srcvfs.IsRegular(src)
		if err != nil {
			return err
		}

		// Start sync operation

		if isdir {
			// Create destination dir if needed
			exists, err := dstvfs.FileExists(dst)
			if err != nil {
				return err
			}
			if !exists {
				log.Verboseln(1, dst)
				if !opt.dryrun {
					err := dstvfs.Mkdir(dst)
					if err != nil {
						return err
					}
				}
			}
			// Save directory for post processing
			d := dirpair{src, dst}
			dirpairs = append(dirpairs, d)
		} else if isregular {
			copyNeeded, err := needToCopy(srcvfs, dstvfs, src, dst)
			if err != nil {
				return err
			}

			if copyNeeded {
				if !opt.dryrun {
					r, err := srcvfs.ReadFromFile(src)
					if err != nil {
						log.Printf("Warning: Skipping \"%s\": %v\n", src, err)
						continue
					}
					err = dstvfs.WriteToFile(dst, r)
					if err != nil {
						return err
					}
					// Set destination mtime == source mtime
					mtime, err := srcvfs.Mtime(src)
					if err != nil {
						return err
					}
					err = dstvfs.SetMtime(dst, mtime)
					if err != nil {
						return err
					}
				}
				log.Verboseln(1, dst)
			}
		} else {
			log.Printf("Warning: Skipping \"%s\": not a regular file or directory.\n", src)
			continue
		}
	}

	// Set the mtimes of all destination directories to the original mtimes.
	// We have to do it here (and bottom first!) because in certain filesystems,
	// updating files inside directories will also change the directory mtime.

	if !opt.dryrun {
		for ix := len(dirpairs) - 1; ix >= 0; ix-- {
			src := dirpairs[ix].src
			dst := dirpairs[ix].dst

			mtime, err := srcvfs.Mtime(src)
			if err != nil {
				return err
			}
			err = dstvfs.SetMtime(dst, mtime)
			if err != nil {
				return err
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package nio

import (
	"io"
	"sync"
)

type bufPipeReader struct {
	*bufpipe
}

func (r *bufPipeReader) CloseWithErr(err error) error {
	if err == nil {
		err = io.ErrClosedPipe
	}
	r.bufpipe.l.Lock()
	defer r.bufpipe.l.Unlock()
	if r.bufpipe.err == nil {
		r.bufpipe.err = err
		r.bufpipe.c.Signal()
	}
	return nil
}

func (r *bufPipeReader) Close() error {
	return r.CloseWithErr(nil)
}

type bufPipeWriter struct {
	*bufpipe
}

func (w *bufPipeWriter) CloseWithErr(err error) error {
	if err == nil {
		err = io.EOF
	}
	w.bufpipe.l.Lock()
	defer w.bufpipe.l.Unlock()
	if w.bufpipe.err == nil {
		w.bufpipe.err = err
		w.bufpipe.c.Signal()
	}
	return nil
}

func (w *bufPipeWriter) Close() error {
	return w.CloseWithErr(nil)
}

type bufpipe struct {
	rl  sync.Mutex
	wl  sync.Mutex
	l   sync.Mutex
	c   *sync.Cond
	b   Buffer
	err error
}

func newBufferedPipe(buf Buffer) *bufpipe {
	s := &bufpipe{
		b: buf,
	}
	s.c = sync.NewCond(&s.l)
	return s
}

func Empty(buf Buffer) bool {
	return buf.Len() == 0
}

func Gap(buf Buffer) int64 {
	return buf.Cap() - buf.Len()
}

func (r *bufpipe) Read(p []byte) (n int, err error) {
	r.rl.Lock()
	defer r.rl.Unlock()

	r.l.Lock()
	defer r.c.Signal()
	defer r.l.Unlock()

	for Empty(r.b) {
		if r.err != nil {
			return 0, r.err
		}

		r.c.Signal()
		r.c.Wait()
	}

	n, _ = r.b.Read(p)

	return n, nil
}

func (w *bufpipe) Write(p []byte) (n int, err error) {
	w.wl.Lock()
	defer w.wl.Unlock()

	w.l.Lock()
	defer w.c.Signal()
	defer w.l.Unlock()

	if w.err != nil {
		return 0, io.ErrClosedPipe
	}

	var m int

	// more data to write
	for len(p[n:]) > 0 {

		// writes too big
		for Gap(w.b) < int64(len(p[n:])) {

			// wait for space
			for Gap(w.b) == 0 {
				w.c.Signal()
				w.c.Wait()
			}

			// chunk write to fill space
			m, err = w.b.Write(p[n : int64(n)+Gap(w.b)])
			n += m
			if err != nil {
				return n, err
			}

			// wait for more space
			w.c.Signal()
			w.c.Wait()

		}

		// check if done
		if len(p[n:]) == 0 {
			return n, nil
		}

		// write
		m, err = w.b.Write(p[n:])
		n += m
		if err != nil {
			return n, err
		}
	}

	return n, err
}
<commit_msg>hiding helper funcs<commit_after>package nio

import (
	"io"
	"sync"
)

type bufPipeReader struct {
	*bufpipe
}

func (r *bufPipeReader) CloseWithErr(err error) error {
	if err == nil {
		err = io.ErrClosedPipe
	}
	r.bufpipe.l.Lock()
	defer r.bufpipe.l.Unlock()
	if r.bufpipe.err == nil {
		r.bufpipe.err = err
		r.bufpipe.c.Signal()
	}
	return nil
}

func (r *bufPipeReader) Close() error {
	return r.CloseWithErr(nil)
}

type bufPipeWriter struct {
	*bufpipe
}

func (w *bufPipeWriter) CloseWithErr(err error) error {
	if err == nil {
		err = io.EOF
	}
	w.bufpipe.l.Lock()
	defer w.bufpipe.l.Unlock()
	if w.bufpipe.err == nil {
		w.bufpipe.err = err
		w.bufpipe.c.Signal()
	}
	return nil
}

func (w *bufPipeWriter) Close() error {
	return w.CloseWithErr(nil)
}

type bufpipe struct {
	rl  sync.Mutex
	wl  sync.Mutex
	l   sync.Mutex
	c   *sync.Cond
	b   Buffer
	err error
}

func newBufferedPipe(buf Buffer) *bufpipe {
	s := &bufpipe{
		b: buf,
	}
	s.c = sync.NewCond(&s.l)
	return s
}

func empty(buf Buffer) bool {
	return buf.Len() == 0
}

func gap(buf Buffer) int64 {
	return buf.Cap() - buf.Len()
}

func (r *bufpipe) Read(p []byte) (n int, err error) {
	r.rl.Lock()
	defer r.rl.Unlock()

	r.l.Lock()
	defer r.c.Signal()
	defer r.l.Unlock()

	for empty(r.b) {
		if r.err != nil {
			return 0, r.err
		}

		r.c.Signal()
		r.c.Wait()
	}

	n, _ = r.b.Read(p)

	return n, nil
}

func (w *bufpipe) Write(p []byte) (n int, err error) {
	w.wl.Lock()
	defer w.wl.Unlock()

	w.l.Lock()
	defer w.c.Signal()
	defer w.l.Unlock()

	if w.err != nil {
		return 0, io.ErrClosedPipe
	}

	var m int

	// more data to write
	for len(p[n:]) > 0 {

		// writes too big
		for gap(w.b) < int64(len(p[n:])) {

			// wait for space
			for gap(w.b) == 0 {
				w.c.Signal()
				w.c.Wait()
			}

			// chunk write to fill space
			m, err = w.b.Write(p[n : int64(n)+gap(w.b)])
			n += m
			if err != nil {
				return n, err
			}

			// wait for more space
			w.c.Signal()
			w.c.Wait()

		}

		// check if done
		if len(p[n:]) == 0 {
			return n, nil
		}

		// write
		m, err = w.b.Write(p[n:])
		n += m
		if err != nil {
			return n, err
		}
	}

	return n, err
}
<|endoftext|>


<commit_before>package pinba

import (
	"bytes"
	"errors"
	"sort"
	"strings"
)

type Tag struct {
	Key   string
	Value string
}

type Tags []Tag

func (t Tags) Len() int           { return len(t) }
func (t Tags) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
func (t Tags) Less(i, j int) bool { return t[i].Key < t[j].Key }

// Get will return value of tag by given key or error if no such tag exists
func (tags Tags) Get(key string) (string, error) {
	for _, tag := range tags {
		if tag.Key == key {
			return tag.Value, nil
		}
	}
	return "", errors.New("no such tag")
}

// Filter will filter (surprise :) tags by given keys, and return new Tags slice
// Code a bit strange, but that way we don't need to use append and it's almost
// twice as fast (480ns/op vs. 800ns/op)
func (tags Tags) Filter(filter []string) Tags {
	// First we count how many tags will be in result
	cnt := 0
	for _, tag := range tags {
		// Always skip empty tags
		if tag.Value == "" {
			continue
		}

		for _, f := range filter {
			if f == tag.Key {
				cnt++
				break
			}
		}
	}

	// Then allocate only what we need, and loop once again to fill result
	result := make(Tags, cnt)
	cnt = 0
	for _, tag := range tags {
		// Always skip empty tags
		if tag.Value == "" {
			continue
		}

		for _, f := range filter {
			if f == tag.Key {
				result[cnt] = tag
				cnt++
				break
			}
		}
	}
	return result
}

// String will return tags as string such as "key=value key2=value2"
// Tags will be sorted by Key's
func (tags Tags) String() string {
	sort.Sort(tags)
	var buf bytes.Buffer
	for i, tag := range tags {
		if i > 0 {
			buf.WriteString(" ")
		}
		buf.WriteString(tag.Key)
		buf.WriteString("=")
		buf.WriteString(tag.Value)
	}
	return buf.String()
}

// Stringf will return tags as formated string, with given format
// {xxx} will be replaced with value of tag xxx
func (tags Tags) Stringf(format string) string {
	result := format
	for _, tag := range tags {
		result = strings.Replace(result, "{"+tag.Key+"}", tag.Value, -1)
	}
	return result
}
<commit_msg>30% faster tags filter<commit_after>package pinba

import (
	"bytes"
	"errors"
	"sort"
	"strings"
)

// How many tags we can have. OpenTSDB limits it to 8, so let it be 16 :)
const MAX_TAGS = 16

type Tag struct {
	Key   string
	Value string
}

type Tags []Tag

func (t Tags) Len() int           { return len(t) }
func (t Tags) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
func (t Tags) Less(i, j int) bool { return t[i].Key < t[j].Key }

// Get will return value of tag by given key or error if no such tag exists
func (tags Tags) Get(key string) (string, error) {
	for _, tag := range tags {
		if tag.Key == key {
			return tag.Value, nil
		}
	}
	return "", errors.New("no such tag")
}

// Filter will filter (surprise :) tags by given keys, and return new Tags slice
// Code a bit strange, but that way we don't need to use append and it's almost
// twice as fast (480ns/op vs. 800ns/op)
func (tags Tags) Filter(filter []string) Tags {
	// First we count how many tags will be in result
	cnt := 0
	indexes := make([]int, MAX_TAGS)
	for idx, tag := range tags {
		// Always skip empty tags
		if tag.Value == "" {
			continue
		}

		for _, f := range filter {
			if f == tag.Key {
				indexes[cnt] = idx
				cnt++
				break
			}
		}
	}

	// Then allocate only what we need, and loop once again to fill result
	result := make(Tags, cnt)
	for n, idx := range indexes[:cnt] {
		result[n] = tags[idx]
	}
	return result
}

// String will return tags as string such as "key=value key2=value2"
// Tags will be sorted by Key's
func (tags Tags) String() string {
	sort.Sort(tags)
	var buf bytes.Buffer
	for i, tag := range tags {
		if i > 0 {
			buf.WriteString(" ")
		}
		buf.WriteString(tag.Key)
		buf.WriteString("=")
		buf.WriteString(tag.Value)
	}
	return buf.String()
}

// Stringf will return tags as formated string, with given format
// {xxx} will be replaced with value of tag xxx
func (tags Tags) Stringf(format string) string {
	result := format
	for _, tag := range tags {
		result = strings.Replace(result, "{"+tag.Key+"}", tag.Value, -1)
	}
	return result
}
<|endoftext|>