<commit_before>// Copyright (c) 2016 Company 0, LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

// rpc contains all structures required by the ZK protocol.
//
// A ZK session has two discrete phases:
//	1. pre session phase, used to create accounts and obtain zkserver key
//	2. session phase, used for all other RPC commands
//	3. once the key exchange is complete the server shall issue a Welcome
//         command.  The welcome command also transfer additional settings such
//         as tag depth etc.
//
// In order to exchange messages with a third party two pieces of information
// are required.  Each side must know the other's long lived public identity
// and the public DH ratchet keys.
// The process, using RPC, to obtains that information is as follows:
//	1. Alice sends Bob a Rendezvous command that contains her encrypted
//	   identity. She uses a third party communication method (phone, IRC
//	   etc)	to share the rendezvous PIN code and a shared password.
//	2. Bob obtains Alice's identity by sending a RendezvousPull command
//	   using the PIN code.  After decrypting Alice's identity blob using the
//	   share password he replies with a Cache command that contains his long
//	   lived public identity and his initial public DH ratchet keys.
//	3. Alice is notified, using the normal Push RPC mechanism, when Bob has
//	   replied.  She then replies to Bob with her public DH ratchet keys.
//
// The external identity and key exchange process is outside of the scope of
// this document.
package rpc

import (
	"crypto/sha256"
	"errors"
	"strconv"

	"github.com/companyzero/zkc/ratchet"
	"github.com/companyzero/zkc/zkidentity"
)

const (
	// pre session phase
	InitialCmdIdentify      = "identify"
	InitialCmdCreateAccount = "createaccount"
	InitialCmdSession       = "session"

	// session phase
	SessionCmdWelcome = "welcome"

	// tagged server commands
	TaggedCmdRendezvous          = "rendezvous"
	TaggedCmdRendezvousReply     = "rendezvousreply"
	TaggedCmdRendezvousPull      = "rendezvouspull"
	TaggedCmdRendezvousPullReply = "rendezvouspullreply"
	TaggedCmdCache               = "cache"
	TaggedCmdPush                = "push"
	TaggedCmdAcknowledge         = "ack"
	TaggedCmdPing                = "ping"
	TaggedCmdPong                = "pong"
	TaggedCmdIdentityPush        = "identitypush"
	TaggedCmdIdentityPushReply   = "identitypushreply"
	TaggedCmdIdentityFind        = "identityfind"
	TaggedCmdIdentityFindReply   = "identityfindreply"
	TaggedCmdIdentityPull        = "identitypull"
	TaggedCmdIdentityPullReply   = "identitypullreply"

	// misc
	MessageModeNormal = 0
	MessageModeMe     = 1
)

// CreateAccount is a PRPC that is used to create a new account on the server.
// Policy dictates if this is allowed or not.
type CreateAccount struct {
	Token          string                    // auth token
	PublicIdentity zkidentity.PublicIdentity // long lived public identity
}

// sanitized errors for CreateAccountReply
var (
	ErrCreateDisallowed = errors.New("not allowed")
	ErrInternalError    = errors.New("internal error, contact administrator")
)

// Message is the generic command that flows between a server and client and
// vice versa.  Its purpose is to add a discriminator to simplify payload
// decoding.  Additionally it has a tag that the recipient shall return
// unmodified when replying.  The tag is originated by the sender and shall be
// unique provided an answer is expected.  The receiver shall not interpret or
// use the tag in any way.
type Message struct {
	Command   string // discriminator
	TimeStamp int64  // originator timestamp
	Tag       uint32 // client generated tag, shall be unique
	//followed by Payload []byte
}

// Empty is used when there is no Payload.
type Empty struct{}

const (
	ProtocolVersion = 6
)

// Welcome is written immediately following a key exchange.  This command
// purpose is to detect if the key exchange completed on the client side.  If
// the key exchange failed the server will simply disconnect.
type Welcome struct {
	Version    int   // protocol version
	ServerTime int64 // server timestamp

	// Client shall ensure it is compatible with the server requirements
	Properties []ServerProperty // server properties
}

type ServerProperty struct {
	Key      string // name of property
	Value    string // value of property
	Required bool   // if true client must handle this entry
}

const (
	// Tag Depth is a required property.  It defines maximum outstanding
	// commands.
	PropTagDepth        = "tagdepth"
	PropTagDepthDefault = "10"

	// MOTD (Message Of The Day) is an optional property.  It is a welcome
	// message that is sent from the server to the client upon first
	// contact.  The client may display this.
	PropMOTD = "motd"

	// Max Attachment Size is a required property.  It defines the maximum
	// attachment size.  Attachment size is defined as the largest size a
	// file transfer is allowed to be.
	PropMaxAttachmentSize        = "maxattachmentsize"
	PropMaxAttachmentSizeDefault = uint64(10 * 1024 * 1024)

	// Max Chunk Size is a required property.  It defines the maximum chunk
	// size.  Chunk size is defined as the largest size a CRPC is allowed
	// to be.
	PropMaxChunkSize        = "maxchunksize"
	PropMaxChunkSizeDefault = uint64(256 * 1024)

	// Max Message Size is a required property.  It defines the maximum
	// message size.  Message size is defined as the largest size a CRPC is
	// allowed to be.  This includes message overhead etc.
	PropMaxMsgSize        = "maxmsgsize"
	PropMaxMsgSizeDefault = PropMaxChunkSizeDefault + 1024

	// Server Time is a required property.  It contains the server time
	// stamp.  The client shall warn the user if the client is not time
	// synced.  Clients and proxies really shall run NTP.
	PropServerTime = "servertime"
)

var (
	// required
	DefaultPropTagDepth = ServerProperty{
		Key:      PropTagDepth,
		Value:    PropTagDepthDefault,
		Required: true,
	}
	DefaultPropMaxAttachmentSize = ServerProperty{
		Key:      PropMaxAttachmentSize,
		Value:    strconv.FormatUint(PropMaxAttachmentSizeDefault, 10),
		Required: true,
	}
	DefaultPropMaxChunkSize = ServerProperty{
		Key:      PropMaxChunkSize,
		Value:    strconv.FormatUint(PropMaxChunkSizeDefault, 10),
		Required: true,
	}
	DefaultPropMaxMsgSize = ServerProperty{
		Key:      PropMaxMsgSize,
		Value:    strconv.FormatUint(PropMaxMsgSizeDefault, 10),
		Required: true,
	}
	DefaultServerTime = ServerProperty{
		Key:      PropServerTime,
		Value:    "", // int64 unix time
		Required: true,
	}

	// optional
	DefaultPropMOTD = ServerProperty{
		Key:      PropMOTD,
		Value:    "",
		Required: false,
	}

	// All properties must exist in this array.
	SupportedServerProperties = []ServerProperty{
		// required
		DefaultPropTagDepth,
		DefaultPropMaxAttachmentSize,
		DefaultPropMaxChunkSize,
		DefaultPropMaxMsgSize,
		DefaultServerTime,

		// optional
		DefaultPropMOTD,
	}
)

// CreateAccountReply returns a sanitized error to the client indicating
// success or failure of the CreateAccountReply command.  Errors is set to ""
// on success.
type CreateAccountReply struct {
	Error string // if create account failed error contains the reason.
}

// Push is a PRPC that is used to push cached encrypted blobs to a user.  This
// command must be acknowledged by the remote side.
type Push struct {
	From     [32]byte // sender identity
	Received int64    // server received timestamp
	Payload  []byte   // encrypted payload
}

// Cache is a PRPC that is used to store message on server for later push
// delivery.  This command must be acknowledged by the remote side.
type Cache struct {
	To      [32]byte // recipient identity
	Payload []byte   // encrypted payload
}

// Ping is a PRPC that is used to determine if the server is alive.
// This command must be acknowledged by the remote side.
type Ping struct{}
type Pong struct{}

// client to client commands

// Passthrough is a command that flows from a client through the server to
// another client.  Payload contains an encrypted CRPC. These commands shall
// not be cached by the server.
type Passthrough struct {
	To      [32]byte // recipient identity
	Payload []byte   // encrypted payload
}

// Rendezvous sends a blob to the server. Blob shall be < 4096 and and
// expiration shall be < 168 (7 * 24).
type Rendezvous struct {
	Blob       []byte // data being shared
	Expiration string // hours until Rendezvous expires
}

// RendezvousReply is a reply packet for a Rendezvous command.  Token contains
// an easy to remember PIN code to identify initial Rendezvous blob.
type RendezvousReply struct {
	Token string // Rendezvous token that identifies blob
	Error string // If an error occured Error will be != ""
}

// RendezvousPull tries to download a previously uploaded blob.
type RendezvousPull struct {
	Token string // Rendezvous token that identifies blob
}

// RendezvousPullReply contains a data blob reply to a previous RendezvousPull
// command that is identified by token.
type RendezvousPullReply struct {
	Error string // set if an error occurred
	Token string // Rendezvous token that identifies blob
	Blob  []byte // data reply to previous Rendezvous
}

type IdentityPushReply struct {
	Error string
}

type IdentityPullReply struct {
	Error string
}

type IdentityFind struct {
	Nick string
}

type IdentityFindReply struct {
	Error string
	Identity zkidentity.PublicIdentity
}

// IdentityKX contains the long lived public identify and the DH ratchet keys.
// It is the second step during the IDKX exchange.
type IdentityKX struct {
	Identity zkidentity.PublicIdentity
	KX       ratchet.KeyExchange
}

// KX contains the DH ratchet keys.  It is the third step during the IDKX
// exchange.
type KX struct {
	KX ratchet.KeyExchange
}

const (
	// CRPC commands
	CRPCCmdPrivateMessage = "privmsg"
	CRPCCmdGroupInvite    = "groupinvite"
	CRPCCmdGroupJoin      = "groupjoin"
	CRPCCmdGroupPart      = "grouppart"
	CRPCCmdGroupKill      = "groupkill"
	CRPCCmdGroupKick      = "groupkick"
	CRPCCmdGroupList      = "grouplist"
	CRPCCmdGroupListGet   = "grouplistget" // no struct used
	CRPCCmdGroupMessage   = "groupmessage"
	CRPCCmdChunkNew       = "chunknew"
	CRPCCmdChunk          = "chunk"
	CRPCCmdJanitorMessage = "janitormessage"

	// compression
	CRPCCompNone = ""
	CRPCCompZLIB = "zlib"

	// janitor
	CRPCJanitorRatchetReset = "ratchetreset"
	CRPCJanitorDeleted      = "deleted"
)

// CRPC is a client RPC message.
type CRPC struct {
	Timestamp   int64  // client side timestamp
	Command     string // discriminator
	Compression string // compression used on Payload
	//followed by Payload []byte
}

// PrivateMessage is a CRPC that contains a text message.
type PrivateMessage struct {
	Text string
	Mode uint32 // 0 regular mode, 1 /me
}

// JanitorMessage is a CRPC that tells the other party some sort of
// housekeeping occurred.
type JanitorMessage struct {
	Command string
	Reason  string
}

// GroupInvite, sender is implicit to CRPC.
// XXX Note that there is no explicit way to prohibit sender being admin.
// XXX This needs some more thought.
type GroupInvite struct {
	Name        string // group name
	Token       uint64 // invite token
	Description string // group description
	Expires     int64  // unix time when this invite expires
}

// GroupJoin
type GroupJoin struct {
	Name  string // group name
	Token uint64 // invite token, implicitly identifies sender
	Error string // accept or deny Invite
}

// GroupPart, sender is implicit to CRPC
type GroupPart struct {
	Name   string // group name
	Reason string // reason to depart group
}

// GroupKill, sender is implicit to CRPC
type GroupKill struct {
	Name   string // group name
	Reason string // reason to disassemble group
}

// GroupKick, sender is implicit to CRPC
type GroupKick struct {
	Member       [zkidentity.IdentitySize]byte // kickee
	Reason       string                        // why member was kicked
	Parted       bool                          // kicked/parted
	NewGroupList GroupList                     // new GroupList
}

// GroupList, currently we detect spoofing by ensuring the origin of the
// message.  This may not be sufficient and we may have to add a signature of
// sorts.  For now roll with this assumption.
type GroupList struct {
	Name       string // group name
	Generation uint64 // incremented every time list changes
	Timestamp  int64  // unix time last generation changed

	// all participants, [0] is administrator
	// receiver must check [0] == originator
	Members [][zkidentity.IdentitySize]byte
}

// GroupMessage is a message to a group.
type GroupMessage struct {
	Name       string // group name
	Generation uint64 // Generation used
	Message    string // Actual message
	Mode       uint32 // 0 regular mode, 1 /me
}

// ChunkNew describes a chunked file transfer initiation.
type ChunkNew struct {
	Size        uint64            // total file size
	ChunkSize   uint64            // chunk size
	Filename    string            // original filename
	Description string            // user provided description
	MIME        string            // mime type
	Digest      [sha256.Size]byte // digest of file -> unique identifier
}

type Chunk struct {
	Offset  uint64            // offset in file
	Digest  [sha256.Size]byte // digest of file -> unique identifier
	Payload []byte            // chunk
}
<commit_msg>observe gofmt<commit_after>// Copyright (c) 2016 Company 0, LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

// rpc contains all structures required by the ZK protocol.
//
// A ZK session has two discrete phases:
//	1. pre session phase, used to create accounts and obtain zkserver key
//	2. session phase, used for all other RPC commands
//	3. once the key exchange is complete the server shall issue a Welcome
//         command.  The welcome command also transfer additional settings such
//         as tag depth etc.
//
// In order to exchange messages with a third party two pieces of information
// are required.  Each side must know the other's long lived public identity
// and the public DH ratchet keys.
// The process, using RPC, to obtains that information is as follows:
//	1. Alice sends Bob a Rendezvous command that contains her encrypted
//	   identity. She uses a third party communication method (phone, IRC
//	   etc)	to share the rendezvous PIN code and a shared password.
//	2. Bob obtains Alice's identity by sending a RendezvousPull command
//	   using the PIN code.  After decrypting Alice's identity blob using the
//	   share password he replies with a Cache command that contains his long
//	   lived public identity and his initial public DH ratchet keys.
//	3. Alice is notified, using the normal Push RPC mechanism, when Bob has
//	   replied.  She then replies to Bob with her public DH ratchet keys.
//
// The external identity and key exchange process is outside of the scope of
// this document.
package rpc

import (
	"crypto/sha256"
	"errors"
	"strconv"

	"github.com/companyzero/zkc/ratchet"
	"github.com/companyzero/zkc/zkidentity"
)

const (
	// pre session phase
	InitialCmdIdentify      = "identify"
	InitialCmdCreateAccount = "createaccount"
	InitialCmdSession       = "session"

	// session phase
	SessionCmdWelcome = "welcome"

	// tagged server commands
	TaggedCmdRendezvous          = "rendezvous"
	TaggedCmdRendezvousReply     = "rendezvousreply"
	TaggedCmdRendezvousPull      = "rendezvouspull"
	TaggedCmdRendezvousPullReply = "rendezvouspullreply"
	TaggedCmdCache               = "cache"
	TaggedCmdPush                = "push"
	TaggedCmdAcknowledge         = "ack"
	TaggedCmdPing                = "ping"
	TaggedCmdPong                = "pong"
	TaggedCmdIdentityPush        = "identitypush"
	TaggedCmdIdentityPushReply   = "identitypushreply"
	TaggedCmdIdentityFind        = "identityfind"
	TaggedCmdIdentityFindReply   = "identityfindreply"
	TaggedCmdIdentityPull        = "identitypull"
	TaggedCmdIdentityPullReply   = "identitypullreply"

	// misc
	MessageModeNormal = 0
	MessageModeMe     = 1
)

// CreateAccount is a PRPC that is used to create a new account on the server.
// Policy dictates if this is allowed or not.
type CreateAccount struct {
	Token          string                    // auth token
	PublicIdentity zkidentity.PublicIdentity // long lived public identity
}

// sanitized errors for CreateAccountReply
var (
	ErrCreateDisallowed = errors.New("not allowed")
	ErrInternalError    = errors.New("internal error, contact administrator")
)

// Message is the generic command that flows between a server and client and
// vice versa.  Its purpose is to add a discriminator to simplify payload
// decoding.  Additionally it has a tag that the recipient shall return
// unmodified when replying.  The tag is originated by the sender and shall be
// unique provided an answer is expected.  The receiver shall not interpret or
// use the tag in any way.
type Message struct {
	Command   string // discriminator
	TimeStamp int64  // originator timestamp
	Tag       uint32 // client generated tag, shall be unique
	//followed by Payload []byte
}

// Empty is used when there is no Payload.
type Empty struct{}

const (
	ProtocolVersion = 6
)

// Welcome is written immediately following a key exchange.  This command
// purpose is to detect if the key exchange completed on the client side.  If
// the key exchange failed the server will simply disconnect.
type Welcome struct {
	Version    int   // protocol version
	ServerTime int64 // server timestamp

	// Client shall ensure it is compatible with the server requirements
	Properties []ServerProperty // server properties
}

type ServerProperty struct {
	Key      string // name of property
	Value    string // value of property
	Required bool   // if true client must handle this entry
}

const (
	// Tag Depth is a required property.  It defines maximum outstanding
	// commands.
	PropTagDepth        = "tagdepth"
	PropTagDepthDefault = "10"

	// MOTD (Message Of The Day) is an optional property.  It is a welcome
	// message that is sent from the server to the client upon first
	// contact.  The client may display this.
	PropMOTD = "motd"

	// Max Attachment Size is a required property.  It defines the maximum
	// attachment size.  Attachment size is defined as the largest size a
	// file transfer is allowed to be.
	PropMaxAttachmentSize        = "maxattachmentsize"
	PropMaxAttachmentSizeDefault = uint64(10 * 1024 * 1024)

	// Max Chunk Size is a required property.  It defines the maximum chunk
	// size.  Chunk size is defined as the largest size a CRPC is allowed
	// to be.
	PropMaxChunkSize        = "maxchunksize"
	PropMaxChunkSizeDefault = uint64(256 * 1024)

	// Max Message Size is a required property.  It defines the maximum
	// message size.  Message size is defined as the largest size a CRPC is
	// allowed to be.  This includes message overhead etc.
	PropMaxMsgSize        = "maxmsgsize"
	PropMaxMsgSizeDefault = PropMaxChunkSizeDefault + 1024

	// Server Time is a required property.  It contains the server time
	// stamp.  The client shall warn the user if the client is not time
	// synced.  Clients and proxies really shall run NTP.
	PropServerTime = "servertime"
)

var (
	// required
	DefaultPropTagDepth = ServerProperty{
		Key:      PropTagDepth,
		Value:    PropTagDepthDefault,
		Required: true,
	}
	DefaultPropMaxAttachmentSize = ServerProperty{
		Key:      PropMaxAttachmentSize,
		Value:    strconv.FormatUint(PropMaxAttachmentSizeDefault, 10),
		Required: true,
	}
	DefaultPropMaxChunkSize = ServerProperty{
		Key:      PropMaxChunkSize,
		Value:    strconv.FormatUint(PropMaxChunkSizeDefault, 10),
		Required: true,
	}
	DefaultPropMaxMsgSize = ServerProperty{
		Key:      PropMaxMsgSize,
		Value:    strconv.FormatUint(PropMaxMsgSizeDefault, 10),
		Required: true,
	}
	DefaultServerTime = ServerProperty{
		Key:      PropServerTime,
		Value:    "", // int64 unix time
		Required: true,
	}

	// optional
	DefaultPropMOTD = ServerProperty{
		Key:      PropMOTD,
		Value:    "",
		Required: false,
	}

	// All properties must exist in this array.
	SupportedServerProperties = []ServerProperty{
		// required
		DefaultPropTagDepth,
		DefaultPropMaxAttachmentSize,
		DefaultPropMaxChunkSize,
		DefaultPropMaxMsgSize,
		DefaultServerTime,

		// optional
		DefaultPropMOTD,
	}
)

// CreateAccountReply returns a sanitized error to the client indicating
// success or failure of the CreateAccountReply command.  Errors is set to ""
// on success.
type CreateAccountReply struct {
	Error string // if create account failed error contains the reason.
}

// Push is a PRPC that is used to push cached encrypted blobs to a user.  This
// command must be acknowledged by the remote side.
type Push struct {
	From     [32]byte // sender identity
	Received int64    // server received timestamp
	Payload  []byte   // encrypted payload
}

// Cache is a PRPC that is used to store message on server for later push
// delivery.  This command must be acknowledged by the remote side.
type Cache struct {
	To      [32]byte // recipient identity
	Payload []byte   // encrypted payload
}

// Ping is a PRPC that is used to determine if the server is alive.
// This command must be acknowledged by the remote side.
type Ping struct{}
type Pong struct{}

// client to client commands

// Passthrough is a command that flows from a client through the server to
// another client.  Payload contains an encrypted CRPC. These commands shall
// not be cached by the server.
type Passthrough struct {
	To      [32]byte // recipient identity
	Payload []byte   // encrypted payload
}

// Rendezvous sends a blob to the server. Blob shall be < 4096 and and
// expiration shall be < 168 (7 * 24).
type Rendezvous struct {
	Blob       []byte // data being shared
	Expiration string // hours until Rendezvous expires
}

// RendezvousReply is a reply packet for a Rendezvous command.  Token contains
// an easy to remember PIN code to identify initial Rendezvous blob.
type RendezvousReply struct {
	Token string // Rendezvous token that identifies blob
	Error string // If an error occured Error will be != ""
}

// RendezvousPull tries to download a previously uploaded blob.
type RendezvousPull struct {
	Token string // Rendezvous token that identifies blob
}

// RendezvousPullReply contains a data blob reply to a previous RendezvousPull
// command that is identified by token.
type RendezvousPullReply struct {
	Error string // set if an error occurred
	Token string // Rendezvous token that identifies blob
	Blob  []byte // data reply to previous Rendezvous
}

type IdentityPushReply struct {
	Error string
}

type IdentityPullReply struct {
	Error string
}

type IdentityFind struct {
	Nick string
}

type IdentityFindReply struct {
	Error    string
	Identity zkidentity.PublicIdentity
}

// IdentityKX contains the long lived public identify and the DH ratchet keys.
// It is the second step during the IDKX exchange.
type IdentityKX struct {
	Identity zkidentity.PublicIdentity
	KX       ratchet.KeyExchange
}

// KX contains the DH ratchet keys.  It is the third step during the IDKX
// exchange.
type KX struct {
	KX ratchet.KeyExchange
}

const (
	// CRPC commands
	CRPCCmdPrivateMessage = "privmsg"
	CRPCCmdGroupInvite    = "groupinvite"
	CRPCCmdGroupJoin      = "groupjoin"
	CRPCCmdGroupPart      = "grouppart"
	CRPCCmdGroupKill      = "groupkill"
	CRPCCmdGroupKick      = "groupkick"
	CRPCCmdGroupList      = "grouplist"
	CRPCCmdGroupListGet   = "grouplistget" // no struct used
	CRPCCmdGroupMessage   = "groupmessage"
	CRPCCmdChunkNew       = "chunknew"
	CRPCCmdChunk          = "chunk"
	CRPCCmdJanitorMessage = "janitormessage"

	// compression
	CRPCCompNone = ""
	CRPCCompZLIB = "zlib"

	// janitor
	CRPCJanitorRatchetReset = "ratchetreset"
	CRPCJanitorDeleted      = "deleted"
)

// CRPC is a client RPC message.
type CRPC struct {
	Timestamp   int64  // client side timestamp
	Command     string // discriminator
	Compression string // compression used on Payload
	//followed by Payload []byte
}

// PrivateMessage is a CRPC that contains a text message.
type PrivateMessage struct {
	Text string
	Mode uint32 // 0 regular mode, 1 /me
}

// JanitorMessage is a CRPC that tells the other party some sort of
// housekeeping occurred.
type JanitorMessage struct {
	Command string
	Reason  string
}

// GroupInvite, sender is implicit to CRPC.
// XXX Note that there is no explicit way to prohibit sender being admin.
// XXX This needs some more thought.
type GroupInvite struct {
	Name        string // group name
	Token       uint64 // invite token
	Description string // group description
	Expires     int64  // unix time when this invite expires
}

// GroupJoin
type GroupJoin struct {
	Name  string // group name
	Token uint64 // invite token, implicitly identifies sender
	Error string // accept or deny Invite
}

// GroupPart, sender is implicit to CRPC
type GroupPart struct {
	Name   string // group name
	Reason string // reason to depart group
}

// GroupKill, sender is implicit to CRPC
type GroupKill struct {
	Name   string // group name
	Reason string // reason to disassemble group
}

// GroupKick, sender is implicit to CRPC
type GroupKick struct {
	Member       [zkidentity.IdentitySize]byte // kickee
	Reason       string                        // why member was kicked
	Parted       bool                          // kicked/parted
	NewGroupList GroupList                     // new GroupList
}

// GroupList, currently we detect spoofing by ensuring the origin of the
// message.  This may not be sufficient and we may have to add a signature of
// sorts.  For now roll with this assumption.
type GroupList struct {
	Name       string // group name
	Generation uint64 // incremented every time list changes
	Timestamp  int64  // unix time last generation changed

	// all participants, [0] is administrator
	// receiver must check [0] == originator
	Members [][zkidentity.IdentitySize]byte
}

// GroupMessage is a message to a group.
type GroupMessage struct {
	Name       string // group name
	Generation uint64 // Generation used
	Message    string // Actual message
	Mode       uint32 // 0 regular mode, 1 /me
}

// ChunkNew describes a chunked file transfer initiation.
type ChunkNew struct {
	Size        uint64            // total file size
	ChunkSize   uint64            // chunk size
	Filename    string            // original filename
	Description string            // user provided description
	MIME        string            // mime type
	Digest      [sha256.Size]byte // digest of file -> unique identifier
}

type Chunk struct {
	Offset  uint64            // offset in file
	Digest  [sha256.Size]byte // digest of file -> unique identifier
	Payload []byte            // chunk
}
<|endoftext|>


<commit_before>package rpigpio

import (
	"errors"
	"fmt"
	"os"
	"reflect"
	"sync"
	"syscall"
	"unsafe"
)

// ChannelDirection represents the direction (IN/OUT) of a channel
type ChannelDirection uint8

// ChannelState represents the state of an output channel: high/low
type ChannelState uint8

// Pull up/down/off
type Pull uint8

// Status represents the status of the RpiGpio package
type Status uint8

const (
	// Only needed if we want to mmap /dev/mem
	bcm2835Base    = 0x20000000
	gpioBaseOffset = 0x200000
	piGPIOBase     = bcm2835Base + gpioBaseOffset
	bcm2835Max     = 0x20FFFFFF
	// ^^ can be removed if we do not want to support /dev/mem
	fselOffset          = 0
	setOffset           = 7
	clearOffset         = 10
	pinLevelOffset      = 13
	pullUpDownOffset    = 37
	pullUpDownClkOffset = 38

	memLength   = 4096
	gpioPinMask = 7
)

// Enumerate avaialable channel directions
const (
	IN ChannelDirection = iota
	OUT
)

// Enumerate possible channel states
const (
	LOW ChannelState = iota
	HIGH
)

// Pull up/down/off
const (
	PULLOFF Pull = iota
	PULLDOWN
	PULLUP
)

// Status of the RpiGpio package
const (
	NEW Status = iota
	OK
	ERR
)

// RpiGpio holds all data for a RPi GPIO implementation
type RpiGpio struct {
	lock         sync.Mutex
	mem          []uint32
	mem8         []uint8
	pinToChannel [27]int
	rpi          *RpiInfo
	status       Status
}

var pinToGpioChannelRev2 = [27]int{
	-1, -1, -1, 2, -1, 3, -1, 4, 14, -1, 15, 17, 18, 27, -1,
	22, 23, -1, 24, 10, -1, 9, 25, 11, 8, -1, 7,
}

// Deteriming GPIO number
func getGpioNumber(channel int) (uint, error) {
	return 0, nil
}

// Close cleans up the RpiGpio resources
func (gpio *RpiGpio) Close() error {
	//event_cleanup_all()???  When we implement an event handler??
	gpio.lock.Lock()
	defer gpio.lock.Unlock()
	return syscall.Munmap(gpio.mem8)
}

// NewGPIO sets up a new GPIO object
func NewGPIO() (*RpiGpio, error) {
	var err error
	gpio := new(RpiGpio)
	gpio.status = NEW
	gpio.rpi = new(RpiInfo)
	gpio.rpi.GetCPUInfo()
	switch int(gpio.rpi.piRevision) {
	case 2:
		gpio.pinToChannel = pinToGpioChannelRev2
	default:
		return nil, errors.New("Unknown Raspberry Pi hardware")
	}
	// set gpio "direction"  (in/out??)
	// pinToChannel = pinToGpioChannelRev??
	err = gpio.openGPIO()
	if err != nil {
		return nil, err
	}
	gpio.status = OK
	return gpio, nil
}

// Direction configures the direction (IN/OUT) of the channel
func (gpio *RpiGpio) Direction(channel uint8, direction ChannelDirection) (err error) {
	// Check package status is OK
	// do some error checking ; verify channel and direction are valid, etc
	// call c_gpio::setup_one()
	fsel := channel / 10
	shift := (channel % 10) * 3
	switch direction {
	case IN:
		gpio.mem[fsel] = gpio.mem[fsel] &^ (gpioPinMask << shift)
	case OUT:
		gpio.mem[fsel] = (gpio.mem[fsel] &^ (gpioPinMask << shift)) | (1 << shift)
	default:
		errString := fmt.Sprintf("Unknown channel direction: %d", direction)
		err = errors.New(errString)
	}
	return
}

// Pull sets or clears the internal pull up/down resistor for a GPIO channel
func (gpio *RpiGpio) Pull(channel uint8, direction Pull) error {
	clkRegister := (channel / 32) + pullUpDownClkOffset
	shift := channel % 32

	if err := gpio.setPull(direction); err != nil {
		return err
	}

	shortWait(150)
	gpio.mem[clkRegister] = 1 << shift
	shortWait(150)
	gpio.mem[pullUpDownOffset] &^= 3
	gpio.mem[clkRegister] = 0
	return nil
}

// Read value from channel
func (gpio *RpiGpio) Read(channel uint8) ChannelState {
	pinLevelRegister := (channel / 32) + pinLevelOffset
	shift := channel % 32
	if gpio.mem[pinLevelRegister]&(1<<shift) != 0 {
		return HIGH
	}
	return LOW
}

// Write value (0/1) to channel
func (gpio *RpiGpio) Write(channel uint8, state ChannelState) error {
	reg := channel / 32
	shift := channel % 32
	gpio.lock.Lock()

	if state == HIGH {
		reg += setOffset
	} else if state == LOW {
		reg += clearOffset
	} else {
		err := fmt.Sprintf("Unknown channel state: %d", state)
		return errors.New(err)
	}
	gpio.mem[reg] = 1 << shift
	gpio.lock.Unlock()
	return nil
}

// Cleanup the channel ; reset to INPUT and pull up/down to off
func (gpio *RpiGpio) Cleanup(channel uint8) {
	// Verify channel is valid, package status is OK, etc
	// get gpio number from channel
	// call c_gpio::cleanup_one()
	//    * call event_cleanup()
	//    * set gpio_direction = -1
	//    * set gpio to INPUT and pull up/down to off
	//    * set found for error checking later on (if working on > 1 channel at a time)

}

// eventCleanup
func eventCleanup(gpio uint) {
	// event_gpio.c:403
}

func (gpio *RpiGpio) openGPIO() (err error) {
	file, err := os.OpenFile("/dev/gpiomem", os.O_RDWR|os.O_SYNC, 0666)
	if err != nil {
		fmt.Println("Error opening /dev/gpiomem: ", err)
		return
	}
	defer file.Close()
	return gpio.mmapFile(file)
}

func (gpio *RpiGpio) mmapFile(f *os.File) (err error) {
	gpio.lock.Lock()
	defer gpio.lock.Unlock()
	// Memory map GPIO registers to byte array
	gpio.mem8, err = syscall.Mmap(
		int(f.Fd()),
		0,
		memLength,
		syscall.PROT_READ|syscall.PROT_WRITE,
		syscall.MAP_SHARED)
	if err != nil {
		return
	}

	// Convert 8-bit slice to 32-bit slice
	header := *(*reflect.SliceHeader)(unsafe.Pointer(&gpio.mem8))
	header.Len /= 4
	header.Cap /= 4
	gpio.mem = *(*[]uint32)(unsafe.Pointer(&header))
	return nil
}

func (gpio *RpiGpio) setPull(d Pull) error {
	switch d {
	case PULLOFF:
		gpio.mem[pullUpDownOffset] &^= 3
	case PULLDOWN, PULLUP:
		gpio.mem[pullUpDownOffset] = (gpio.mem[pullUpDownOffset] &^ 3) | uint32(d)
	default:
		errString := fmt.Sprintf("Unknown pull direction: %d", d)
		return errors.New(errString)
	}
	return nil
}

// run cnt nop operations
func shortWait(cnt uint32)
<commit_msg>Enumerate possible gpio pin function values<commit_after>package rpigpio

import (
	"errors"
	"fmt"
	"os"
	"reflect"
	"sync"
	"syscall"
	"unsafe"
)

// ChannelFunction represents the BCM2835 function of a channel
type ChannelFunction uint8

// ChannelDirection represents the direction (IN/OUT) of a channel
type ChannelDirection ChannelFunction

// ChannelState represents the state of an output channel: high/low
type ChannelState uint8

// Pull up/down/off
type Pull uint8

// Status represents the status of the RpiGpio package
type Status uint8

const (
	// Only needed if we want to mmap /dev/mem
	bcm2835Base    = 0x20000000
	gpioBaseOffset = 0x200000
	piGPIOBase     = bcm2835Base + gpioBaseOffset
	bcm2835Max     = 0x20FFFFFF
	// ^^ can be removed if we do not want to support /dev/mem
	fselOffset          = 0
	setOffset           = 7
	clearOffset         = 10
	pinLevelOffset      = 13
	pullUpDownOffset    = 37
	pullUpDownClkOffset = 38

	memLength   = 4096
	gpioPinMask = 7
)

// Enumerate avaialable channel directions
const (
	IN  ChannelDirection = ChannelDirection(INPUT)
	OUT ChannelDirection = ChannelDirection(OUTPUT)
)

// Enumerate avaialable channel functions
const (
	INPUT  ChannelFunction = 0
	OUTPUT                 = 1
	ALT0                   = 4
	ALT1                   = 5
	ALT2                   = 6
	ALT3                   = 7
	ALT4                   = 3
	ALT5                   = 2
)

// Enumerate possible channel states
const (
	LOW ChannelState = iota
	HIGH
)

// Pull up/down/off
const (
	PULLOFF Pull = iota
	PULLDOWN
	PULLUP
)

// Status of the RpiGpio package
const (
	NEW Status = iota
	OK
	ERR
)

// RpiGpio holds all data for a RPi GPIO implementation
type RpiGpio struct {
	lock         sync.Mutex
	mem          []uint32
	mem8         []uint8
	pinToChannel [27]int
	rpi          *RpiInfo
	status       Status
}

var pinToGpioChannelRev2 = [27]int{
	-1, -1, -1, 2, -1, 3, -1, 4, 14, -1, 15, 17, 18, 27, -1,
	22, 23, -1, 24, 10, -1, 9, 25, 11, 8, -1, 7,
}

// Deteriming GPIO number
func getGpioNumber(channel int) (uint, error) {
	return 0, nil
}

// Close cleans up the RpiGpio resources
func (gpio *RpiGpio) Close() error {
	//event_cleanup_all()???  When we implement an event handler??
	gpio.lock.Lock()
	defer gpio.lock.Unlock()
	return syscall.Munmap(gpio.mem8)
}

// NewGPIO sets up a new GPIO object
func NewGPIO() (*RpiGpio, error) {
	var err error
	gpio := new(RpiGpio)
	gpio.status = NEW
	gpio.rpi = new(RpiInfo)
	gpio.rpi.GetCPUInfo()
	switch int(gpio.rpi.piRevision) {
	case 2:
		gpio.pinToChannel = pinToGpioChannelRev2
	default:
		return nil, errors.New("Unknown Raspberry Pi hardware")
	}
	// set gpio "direction"  (in/out??)
	// pinToChannel = pinToGpioChannelRev??
	err = gpio.openGPIO()
	if err != nil {
		return nil, err
	}
	gpio.status = OK
	return gpio, nil
}

// Direction configures the direction (IN/OUT) of the channel
func (gpio *RpiGpio) Direction(channel uint8, direction ChannelDirection) (err error) {
	// Check package status is OK
	// do some error checking ; verify channel and direction are valid, etc
	// call c_gpio::setup_one()
	fsel := channel / 10
	shift := (channel % 10) * 3
	switch direction {
	case IN:
		gpio.mem[fsel] = gpio.mem[fsel] &^ (gpioPinMask << shift)
	case OUT:
		gpio.mem[fsel] = (gpio.mem[fsel] &^ (gpioPinMask << shift)) | (1 << shift)
	default:
		errString := fmt.Sprintf("Unknown channel direction: %d", direction)
		err = errors.New(errString)
	}
	return
}

// Pull sets or clears the internal pull up/down resistor for a GPIO channel
func (gpio *RpiGpio) Pull(channel uint8, direction Pull) error {
	clkRegister := (channel / 32) + pullUpDownClkOffset
	shift := channel % 32

	if err := gpio.setPull(direction); err != nil {
		return err
	}

	shortWait(150)
	gpio.mem[clkRegister] = 1 << shift
	shortWait(150)
	gpio.mem[pullUpDownOffset] &^= 3
	gpio.mem[clkRegister] = 0
	return nil
}

// Read value from channel
func (gpio *RpiGpio) Read(channel uint8) ChannelState {
	pinLevelRegister := (channel / 32) + pinLevelOffset
	shift := channel % 32
	if gpio.mem[pinLevelRegister]&(1<<shift) != 0 {
		return HIGH
	}
	return LOW
}

// Write value (0/1) to channel
func (gpio *RpiGpio) Write(channel uint8, state ChannelState) error {
	reg := channel / 32
	shift := channel % 32
	gpio.lock.Lock()

	if state == HIGH {
		reg += setOffset
	} else if state == LOW {
		reg += clearOffset
	} else {
		err := fmt.Sprintf("Unknown channel state: %d", state)
		return errors.New(err)
	}
	gpio.mem[reg] = 1 << shift
	gpio.lock.Unlock()
	return nil
}

// Cleanup the channel ; reset to INPUT and pull up/down to off
func (gpio *RpiGpio) Cleanup(channel uint8) {
	// Verify channel is valid, package status is OK, etc
	// get gpio number from channel
	// call c_gpio::cleanup_one()
	//    * call event_cleanup()
	//    * set gpio_direction = -1
	//    * set gpio to INPUT and pull up/down to off
	//    * set found for error checking later on (if working on > 1 channel at a time)

}

// eventCleanup
func eventCleanup(gpio uint) {
	// event_gpio.c:403
}

func (gpio *RpiGpio) openGPIO() (err error) {
	file, err := os.OpenFile("/dev/gpiomem", os.O_RDWR|os.O_SYNC, 0666)
	if err != nil {
		fmt.Println("Error opening /dev/gpiomem: ", err)
		return
	}
	defer file.Close()
	return gpio.mmapFile(file)
}

func (gpio *RpiGpio) mmapFile(f *os.File) (err error) {
	gpio.lock.Lock()
	defer gpio.lock.Unlock()
	// Memory map GPIO registers to byte array
	gpio.mem8, err = syscall.Mmap(
		int(f.Fd()),
		0,
		memLength,
		syscall.PROT_READ|syscall.PROT_WRITE,
		syscall.MAP_SHARED)
	if err != nil {
		return
	}

	// Convert 8-bit slice to 32-bit slice
	header := *(*reflect.SliceHeader)(unsafe.Pointer(&gpio.mem8))
	header.Len /= 4
	header.Cap /= 4
	gpio.mem = *(*[]uint32)(unsafe.Pointer(&header))
	return nil
}

func (gpio *RpiGpio) setPull(d Pull) error {
	switch d {
	case PULLOFF:
		gpio.mem[pullUpDownOffset] &^= 3
	case PULLDOWN, PULLUP:
		gpio.mem[pullUpDownOffset] = (gpio.mem[pullUpDownOffset] &^ 3) | uint32(d)
	default:
		errString := fmt.Sprintf("Unknown pull direction: %d", d)
		return errors.New(errString)
	}
	return nil
}

// run cnt nop operations
func shortWait(cnt uint32)
<|endoftext|>


<commit_before>// Go client package for the okq persitent queue
//
// TO import inside your package do:
//
//	import "github.com/mediocregopher/okq-go/okq"
//
// Connecting
//
// Use New to create a Client. This Client can have knowledge of multiple okq
// endpoints, and will attempt to reconnect at random if it loses connection. In
// most cases it will only return an error if it can't connect to any of the
// endpoints at that moment.
//
//	cl := okq.New("127.0.0.1:4777", "127.0.0.1:4778")
//
// Pushing to queues
//
// All events in okq require a unique event id. This package will automatically
// generate a unique id if you use the standard Push methods.
//
//	cl.Push("super-queue", "my awesome event")
//
// You can also create your own id by using the PushEvent methods. Remember
// though that the event id *must* be unique within that queue.
//
//	cl.PushEvent("super-queue", &okq.Event{"unique id", "my awesome event"})
//
// Consuming from queues
//
// You can turn any Client into a consumer by using the Consumer methods. These
// will block as they write incoming events to a given channel, and only return
// upon an error or a manual stop.
//
//	// Example of a consumer which should never quit
//	ch := make(chan *okq.ConsumerEvent)
//	go func() {
//		for e := range ch {
//			log.Printf("event received: %v", e.Event.Contents)
//			e.Ack()
//		}
//	}()
//
//	for {
//		err := cl.Consumer(ch, nil, "queue1", "queue2")
//		log.Printf("error received from consumer: %s", err)
//	}
//
// See the doc string for the Consumer method for more details
package okq

import (
	"errors"
	"strings"
	"time"

	"code.google.com/p/go-uuid/uuid"
	"github.com/fzzy/radix/redis"
	"github.com/grooveshark/golib/agg"
)

var uuidCh = make(chan string, 1024)
func init() {
	go func() {
		for {
			uuidCh <- uuid.New()
		}
	}()
}

// Timeout to use when reading from socket
const TIMEOUT = 30 * time.Second

// Notify timeout used in the consumer
const notify_timeout = TIMEOUT - (1 * time.Second)

// If true turns on debug logging and agg support (see
// https://github.com/grooveshark/golib)
var Debug bool

// A single event which can be read from or written to an okq instance
type Event struct {
	Id       string // Unique id of this event
	Contents string // Arbitrary contents of the event
}

func replyToEvent(r *redis.Reply) (*Event, error) {
	if r.Type == redis.NilReply {
		return nil, nil
	}
	parts, err := r.List()
	if err != nil {
		return nil, err
	} else if len(parts) < 2 {
		return nil, errors.New("not enough elements in reply")
	}
	return &Event{
		Id:       parts[0],
		Contents: parts[1],
	}, nil
}

type Client struct {
	clients map[string]*redis.Client

	// Timeout to use for reads/writes to okq. This defaults to TIMEOUT, but can
	// be overwritten immediately after NewClient is called
	Timeout time.Duration
}

// Given one or more okq endpoints (all in the same pool), returns a client
// which will interact with them. Returns an error if it can't connect to any of
// the given clients
func New(addr ...string) *Client {
	c := Client{
		clients: map[string]*redis.Client{},
		Timeout: TIMEOUT,
	}

	for i := range addr {
		c.clients[addr[i]] = nil
	}

	return &c
}

func (c *Client) getConn() (string, *redis.Client, error) {
	for addr, rclient := range c.clients {
		if rclient != nil {
			return addr, rclient, nil
		}
	}

	for addr := range c.clients {
		rclient, err := redis.DialTimeout("tcp", addr, c.Timeout)
		if err == nil {
			c.clients[addr] = rclient
			return addr, rclient, nil
		}
	}

	return "", nil, errors.New("no connectable endpoints")
}

func (c *Client) cmd(cmd string, args ...interface{}) *redis.Reply {
	for i := 0; i < 3; i++ {
		addr, rclient, err := c.getConn()
		if err != nil {
			return &redis.Reply{Type: redis.ErrorReply, Err: err}
		}

		start := time.Now()
		r := rclient.Cmd(cmd, args...)
		if err := r.Err; err != nil {
			if _, ok := err.(*redis.CmdError); !ok {
				rclient.Close()
				c.clients[addr] = nil
				continue
			}
		}
		if Debug {
			agg.Agg(strings.ToUpper(cmd), time.Since(start).Seconds())
		}

		return r
	}

	return &redis.Reply{
		Type: redis.ErrorReply,
		Err:  errors.New("could not find usable endpoint"),
	}
}

// Returns the next event which will be retrieved from the queue, without
// actually removing it from the queue. Returns nil if the queue is empty
func (c *Client) PeekNext(queue string) (*Event, error) {
	return replyToEvent(c.cmd("QRPEEK", queue))
}

// Returns the event most recently added to the queue, without actually removing
// it from the queue. Returns nil if the queue is empty
func (c *Client) PeekLast(queue string) (*Event, error) {
	return replyToEvent(c.cmd("QLPEEK", queue))
}

// Pushes the given event onto the end of the queue. The event's Id must be
// unique within that queue
func (c *Client) PushEvent(queue string, event *Event) error {
	return c.cmd("QLPUSH", queue, event.Id, event.Contents).Err
}

// Pushes an event with the given contents onto the end of the queue. The
// event's Id will be an automatically generated uuid
func (c *Client) Push(queue, contents string) error {
	event := Event{Id: <-uuidCh, Contents: contents}
	return c.PushEvent(queue, &event)
}

// Pushes the given event onto the front of the queue (meaning it will be the
// next event consumed). The event's Id must be unique within that queue
func (c *Client) PushEventHigh(queue string, event *Event) error {
	return c.cmd("QRPUSH", queue, event.Id, event.Contents).Err
}

// Pushes an event with the given contents onto the end of the queue. The
// event's Id will be an automatically generated uuid
func (c *Client) PushHigh(queue, contents string) error {
	event := Event{Id: <-uuidCh, Contents: contents}
	return c.PushEventHigh(queue, &event)
}

// Returns the statuses of the given queues, or the statuses of all the known
// queues if no queues are given
func (c *Client) Status(queue ...string) ([]string, error) {
	return c.cmd("QSTATUS", queue).List()
}

// Closes all connections that this client currently has open
func (c *Client) Close() error {
	var err error
	for addr, rclient := range c.clients {
		rerr := rclient.Close()
		if err == nil && rerr != nil {
			err = rerr
		}
		c.clients[addr] = nil
	}
	return err
}

type ack struct {
	id, queue string
}

// An event as returned by a consumer client. It contains an Event, but it must
// have Ack() called on it (unless ConsumerUnsafe is used)
type ConsumerEvent struct {
	*Event
	Queue     string
	ackCh     chan *ack
	ackStopCh chan bool
}

// Acknowledges that the given ConsumerEvent has been successfully consumed. If
// this is not called by the event's timeout the event will be put back in its
// queue to be consumed again
func (we *ConsumerEvent) Ack() {
	select {
	case we.ackCh <- &ack{we.Event.Id, we.Queue}:
	case <-we.ackStopCh:
	}
}

// Turns this client into a consumer. It will register itself on the given
// queues, and push all incoming events to the given ConsumerEvent channel (ch).
// If stopCh is not nil it can be closed in order to stop the consumer.
//
// This call:
//
// * blocks until stopCh is closed or there is a connection error
//
// * assumes ch is already being read from in a separate go-routine
//
// * assumes ch is read from until it's closed
//
// * always closes ch just before it returns (as a consequence, don't share the
// same ch amongst multiple calls)
//
// * upon the closing of stopCh will wait for any straggling Ack calls before
// closing ch and returning (although there is a 30 second timeout)
//
// * returns the connection error which caused it to close, or nil if the close
// was due to stopCh
//
func (c *Client) Consumer(
	ch chan *ConsumerEvent, stopCh chan bool, queue ...string,
) error {
	return c.consumer(ch, stopCh, queue, false)
}

// Same as Consumer, but it's not necessary to call Ack on events received from
// ch. This is more unsafe as if the consumer dies while processing the event
// the event is lost forever
func (c *Client) ConsumerUnsafe(
	ch chan *ConsumerEvent, stopCh chan bool, queue ...string,
) error {
	return c.consumer(ch, stopCh, queue, true)
}

func timedCmd(rclient *redis.Client, cmd string, args ...interface{}) *redis.Reply {
	start := time.Now()
	r := rclient.Cmd(cmd, args...)
	if Debug {
		agg.Agg(strings.ToUpper(cmd), time.Since(start).Seconds())
	}
	return r
}

func (c *Client) consumer(
	ch chan *ConsumerEvent, stopCh chan bool, queues []string, noack bool,
) error {

	if len(queues) == 0 {
		return errors.New("no queues given to read from")
	}

	addr, rclient, err := c.getConn()
	if err != nil {
		return err
	}

	// If the stopCh is EVER closed we want to make sure the ch gets closed
	// also. If we're returning and stopCh isn't closed it means there was some
	// kind of connection error, and we should close the client.  It's possible
	// that stopCh was closed AND there was a connection error, in that case the
	// faulty connection will stay in c.clients, but it will be ferreted out the
	// next time it tries to get used
	defer func() {
		select {
		case <-stopCh:
		default:
			rclient.Close()
			c.clients[addr] = nil
		}
		close(ch)
	}()

	if err := timedCmd(rclient, "QREGISTER", queues).Err; err != nil {
		return err
	}

	notifyTimeout := (c.Timeout - (1 * time.Second)).Seconds()

	// The ackHandler stuff is a bit complex. See the ackHandler method for
	// descriptions of what each of these channels is for
	ackCh := make(chan *ack)
	ackStopCh := make(chan bool) // Only for clients calling Ack()
	ackTrackCh := make(chan bool)
	ackErrCh := make(chan error)

	if !noack {
		go ackHandler(addr, c.Timeout, ackCh, ackStopCh, ackTrackCh, ackErrCh)
		// Remember that defers are executed in reverse order upon returning
		defer func() {
			// Close this to let the ackHandler know that there's no more new
			// events coming in
			close(ackTrackCh)
			// Once ackErrCh is closed we know that ackHandler is returned
			<-ackErrCh
			// Close the ackCh just in case
			close(ackCh)
		}()
	}

	for {
		r := timedCmd(rclient, "QNOTIFY", notifyTimeout)
		if err := r.Err; err != nil {
			return err
		}

		// Check if we've stopped or errord before doing anything else. We may
		// loop back to the top in the step after this so it's important to do
		// this now
		select {
		case err := <-ackErrCh:
			return err
		case <-stopCh:
			return nil
		default:
		}

		if r.Type == redis.NilReply {
			continue
		}

		q, err := r.Str()
		if err != nil {
			return err
		}

		args := []string{q}
		if noack {
			args = append(args, "NOACK")
		}

		e, err := replyToEvent(timedCmd(rclient, "QRPOP", args))
		if err != nil {
			return err
		} else if e == nil {
			continue
		}

		// Let the ackHandler know to expect an ack for this event
		if !noack {
			select {
			case ackTrackCh <- true:
			case err := <-ackErrCh:
				return err
			}
		}

		select {
		case ch <- &ConsumerEvent{e, q, ackCh, ackStopCh}:
		case err := <-ackErrCh:
			return err
		}
	}
}

// ackCh      - Main channel that incoming ack messages from clients come from
// ackStopCh  - Channel which is closed when this method returns, clients use it
// 				to know that nothing is there to receive their acks
// ackTrackCh - Used to let the ackHandler know that an event has been sent out
// 				and to expect an ack for it. When closed it means that no more
// 				new events will be sent out
// ackErrCh   - Channel used to send connection errors from this connection back
// 				to the main channel. This is always closed when this method
//				returns
func ackHandler(
	addr string, timeout time.Duration, ackCh chan *ack,
	ackStopCh, ackTrackCh chan bool, ackErrCh chan error,
) {
	defer close(ackErrCh)
	defer close(ackStopCh)
	rclient, err := redis.DialTimeout("tcp", addr, timeout)
	if err != nil {
		ackErrCh <- err
		return
	}
	defer rclient.Close()

	var track int64
ackloop:
	for {
		select {
		case a, ok := <-ackCh:
			if !ok {
				return
			}
			if err := doAck(rclient, a); err != nil {
				ackErrCh <- err
				return
			}
			track--

		case _, ok := <-ackTrackCh:
			if !ok {
				break ackloop
			}
			track++
		}
	}

	// If we're here there have been no errors and ackCh is still open, but
	// ackTrackCh is closed meaning that there are no new events being sent out
	// and we have only to wait for acks from any stragglers. After 30 seconds
	// okq has reclaimed the events anyway, so no point in waiting after that
	for ;track > 0; track-- {
		select {
		case a := <-ackCh:
			if err := doAck(rclient, a); err != nil {
				// We don't bother writing to ackErrCh, only ackTrackCh matters
				// at this point, any errors from here won't bubble up anyway
				return
			}
		case <-time.After(30 * time.Second):
			return
		}
	}
}

func doAck(rclient *redis.Client, a *ack) error {
	return timedCmd(rclient, "QACK", a.queue, a.id).Err
}
<commit_msg>use radix.v2<commit_after>// Go client package for the okq persitent queue
//
// TO import inside your package do:
//
//	import "github.com/mediocregopher/okq-go/okq"
//
// Connecting
//
// Use New to create a Client. This Client can have knowledge of multiple okq
// endpoints, and will attempt to reconnect at random if it loses connection. In
// most cases it will only return an error if it can't connect to any of the
// endpoints at that moment.
//
//	cl := okq.New("127.0.0.1:4777", "127.0.0.1:4778")
//
// Pushing to queues
//
// All events in okq require a unique event id. This package will automatically
// generate a unique id if you use the standard Push methods.
//
//	cl.Push("super-queue", "my awesome event")
//
// You can also create your own id by using the PushEvent methods. Remember
// though that the event id *must* be unique within that queue.
//
//	cl.PushEvent("super-queue", &okq.Event{"unique id", "my awesome event"})
//
// Consuming from queues
//
// You can turn any Client into a consumer by using the Consumer methods. These
// will block as they write incoming events to a given channel, and only return
// upon an error or a manual stop.
//
//	// Example of a consumer which should never quit
//	ch := make(chan *okq.ConsumerEvent)
//	go func() {
//		for e := range ch {
//			log.Printf("event received: %v", e.Event.Contents)
//			e.Ack()
//		}
//	}()
//
//	for {
//		err := cl.Consumer(ch, nil, "queue1", "queue2")
//		log.Printf("error received from consumer: %s", err)
//	}
//
// See the doc string for the Consumer method for more details
package okq

import (
	"errors"
	"strings"
	"time"

	"code.google.com/p/go-uuid/uuid"
	"github.com/grooveshark/golib/agg"
	"github.com/mediocregopher/radix.v2/redis"
)

var uuidCh = make(chan string, 1024)

func init() {
	go func() {
		for {
			uuidCh <- uuid.New()
		}
	}()
}

// Timeout to use when reading from socket
const TIMEOUT = 30 * time.Second

// Notify timeout used in the consumer
const notify_timeout = TIMEOUT - (1 * time.Second)

// If true turns on debug logging and agg support (see
// https://github.com/grooveshark/golib)
var Debug bool

// A single event which can be read from or written to an okq instance
type Event struct {
	Id       string // Unique id of this event
	Contents string // Arbitrary contents of the event
}

func replyToEvent(r *redis.Resp) (*Event, error) {
	if r.IsType(redis.Nil) {
		return nil, nil
	}
	parts, err := r.List()
	if err != nil {
		return nil, err
	} else if len(parts) < 2 {
		return nil, errors.New("not enough elements in reply")
	}
	return &Event{
		Id:       parts[0],
		Contents: parts[1],
	}, nil
}

type Client struct {
	clients map[string]*redis.Client

	// Timeout to use for reads/writes to okq. This defaults to TIMEOUT, but can
	// be overwritten immediately after NewClient is called
	Timeout time.Duration
}

// Given one or more okq endpoints (all in the same pool), returns a client
// which will interact with them. Returns an error if it can't connect to any of
// the given clients
func New(addr ...string) *Client {
	c := Client{
		clients: map[string]*redis.Client{},
		Timeout: TIMEOUT,
	}

	for i := range addr {
		c.clients[addr[i]] = nil
	}

	return &c
}

func (c *Client) getConn() (string, *redis.Client, error) {
	for addr, rclient := range c.clients {
		if rclient != nil {
			return addr, rclient, nil
		}
	}

	for addr := range c.clients {
		rclient, err := redis.DialTimeout("tcp", addr, c.Timeout)
		if err == nil {
			c.clients[addr] = rclient
			return addr, rclient, nil
		}
	}

	return "", nil, errors.New("no connectable endpoints")
}

func (c *Client) cmd(cmd string, args ...interface{}) *redis.Resp {
	for i := 0; i < 3; i++ {
		addr, rclient, err := c.getConn()
		if err != nil {
			return redis.NewResp(err)
		}

		start := time.Now()
		r := rclient.Cmd(cmd, args...)
		if err := r.Err; err != nil {
			if r.IsType(redis.IOErr) {
				rclient.Close()
				c.clients[addr] = nil
				continue
			}
		}
		if Debug {
			agg.Agg(strings.ToUpper(cmd), time.Since(start).Seconds())
		}

		return r
	}

	return redis.NewResp(errors.New("could not find usable endpoint"))
}

// Returns the next event which will be retrieved from the queue, without
// actually removing it from the queue. Returns nil if the queue is empty
func (c *Client) PeekNext(queue string) (*Event, error) {
	return replyToEvent(c.cmd("QRPEEK", queue))
}

// Returns the event most recently added to the queue, without actually removing
// it from the queue. Returns nil if the queue is empty
func (c *Client) PeekLast(queue string) (*Event, error) {
	return replyToEvent(c.cmd("QLPEEK", queue))
}

// Pushes the given event onto the end of the queue. The event's Id must be
// unique within that queue
func (c *Client) PushEvent(queue string, event *Event) error {
	return c.cmd("QLPUSH", queue, event.Id, event.Contents).Err
}

// Pushes an event with the given contents onto the end of the queue. The
// event's Id will be an automatically generated uuid
func (c *Client) Push(queue, contents string) error {
	event := Event{Id: <-uuidCh, Contents: contents}
	return c.PushEvent(queue, &event)
}

// Pushes the given event onto the front of the queue (meaning it will be the
// next event consumed). The event's Id must be unique within that queue
func (c *Client) PushEventHigh(queue string, event *Event) error {
	return c.cmd("QRPUSH", queue, event.Id, event.Contents).Err
}

// Pushes an event with the given contents onto the end of the queue. The
// event's Id will be an automatically generated uuid
func (c *Client) PushHigh(queue, contents string) error {
	event := Event{Id: <-uuidCh, Contents: contents}
	return c.PushEventHigh(queue, &event)
}

// Returns the statuses of the given queues, or the statuses of all the known
// queues if no queues are given
func (c *Client) Status(queue ...string) ([]string, error) {
	return c.cmd("QSTATUS", queue).List()
}

// Closes all connections that this client currently has open
func (c *Client) Close() error {
	var err error
	for addr, rclient := range c.clients {
		rerr := rclient.Close()
		if err == nil && rerr != nil {
			err = rerr
		}
		c.clients[addr] = nil
	}
	return err
}

type ack struct {
	id, queue string
}

// An event as returned by a consumer client. It contains an Event, but it must
// have Ack() called on it (unless ConsumerUnsafe is used)
type ConsumerEvent struct {
	*Event
	Queue     string
	ackCh     chan *ack
	ackStopCh chan bool
}

// Acknowledges that the given ConsumerEvent has been successfully consumed. If
// this is not called by the event's timeout the event will be put back in its
// queue to be consumed again
func (we *ConsumerEvent) Ack() {
	select {
	case we.ackCh <- &ack{we.Event.Id, we.Queue}:
	case <-we.ackStopCh:
	}
}

// Turns this client into a consumer. It will register itself on the given
// queues, and push all incoming events to the given ConsumerEvent channel (ch).
// If stopCh is not nil it can be closed in order to stop the consumer.
//
// This call:
//
// * blocks until stopCh is closed or there is a connection error
//
// * assumes ch is already being read from in a separate go-routine
//
// * assumes ch is read from until it's closed
//
// * always closes ch just before it returns (as a consequence, don't share the
// same ch amongst multiple calls)
//
// * upon the closing of stopCh will wait for any straggling Ack calls before
// closing ch and returning (although there is a 30 second timeout)
//
// * returns the connection error which caused it to close, or nil if the close
// was due to stopCh
//
func (c *Client) Consumer(
	ch chan *ConsumerEvent, stopCh chan bool, queue ...string,
) error {
	return c.consumer(ch, stopCh, queue, false)
}

// Same as Consumer, but it's not necessary to call Ack on events received from
// ch. This is more unsafe as if the consumer dies while processing the event
// the event is lost forever
func (c *Client) ConsumerUnsafe(
	ch chan *ConsumerEvent, stopCh chan bool, queue ...string,
) error {
	return c.consumer(ch, stopCh, queue, true)
}

func timedCmd(
	rclient *redis.Client, cmd string, args ...interface{},
) *redis.Resp {
	start := time.Now()
	r := rclient.Cmd(cmd, args...)
	if Debug {
		agg.Agg(strings.ToUpper(cmd), time.Since(start).Seconds())
	}
	return r
}

func (c *Client) consumer(
	ch chan *ConsumerEvent, stopCh chan bool, queues []string, noack bool,
) error {

	if len(queues) == 0 {
		return errors.New("no queues given to read from")
	}

	addr, rclient, err := c.getConn()
	if err != nil {
		return err
	}

	// If the stopCh is EVER closed we want to make sure the ch gets closed
	// also. If we're returning and stopCh isn't closed it means there was some
	// kind of connection error, and we should close the client.  It's possible
	// that stopCh was closed AND there was a connection error, in that case the
	// faulty connection will stay in c.clients, but it will be ferreted out the
	// next time it tries to get used
	defer func() {
		select {
		case <-stopCh:
		default:
			rclient.Close()
			c.clients[addr] = nil
		}
		close(ch)
	}()

	if err := timedCmd(rclient, "QREGISTER", queues).Err; err != nil {
		return err
	}

	notifyTimeout := (c.Timeout - (1 * time.Second)).Seconds()

	// The ackHandler stuff is a bit complex. See the ackHandler method for
	// descriptions of what each of these channels is for
	ackCh := make(chan *ack)
	ackStopCh := make(chan bool) // Only for clients calling Ack()
	ackTrackCh := make(chan bool)
	ackErrCh := make(chan error)

	if !noack {
		go ackHandler(addr, c.Timeout, ackCh, ackStopCh, ackTrackCh, ackErrCh)
		// Remember that defers are executed in reverse order upon returning
		defer func() {
			// Close this to let the ackHandler know that there's no more new
			// events coming in
			close(ackTrackCh)
			// Once ackErrCh is closed we know that ackHandler is returned
			<-ackErrCh
			// Close the ackCh just in case
			close(ackCh)
		}()
	}

	for {
		r := timedCmd(rclient, "QNOTIFY", notifyTimeout)
		if err := r.Err; err != nil {
			return err
		}

		// Check if we've stopped or errord before doing anything else. We may
		// loop back to the top in the step after this so it's important to do
		// this now
		select {
		case err := <-ackErrCh:
			return err
		case <-stopCh:
			return nil
		default:
		}

		if r.IsType(redis.Nil) {
			continue
		}

		q, err := r.Str()
		if err != nil {
			return err
		}

		args := []string{q}
		if noack {
			args = append(args, "NOACK")
		}

		e, err := replyToEvent(timedCmd(rclient, "QRPOP", args))
		if err != nil {
			return err
		} else if e == nil {
			continue
		}

		// Let the ackHandler know to expect an ack for this event
		if !noack {
			select {
			case ackTrackCh <- true:
			case err := <-ackErrCh:
				return err
			}
		}

		select {
		case ch <- &ConsumerEvent{e, q, ackCh, ackStopCh}:
		case err := <-ackErrCh:
			return err
		}
	}
}

// ackCh      - Main channel that incoming ack messages from clients come from
// ackStopCh  - Channel which is closed when this method returns, clients use it
// 				to know that nothing is there to receive their acks
// ackTrackCh - Used to let the ackHandler know that an event has been sent out
// 				and to expect an ack for it. When closed it means that no more
// 				new events will be sent out
// ackErrCh   - Channel used to send connection errors from this connection back
// 				to the main channel. This is always closed when this method
//				returns
func ackHandler(
	addr string, timeout time.Duration, ackCh chan *ack,
	ackStopCh, ackTrackCh chan bool, ackErrCh chan error,
) {
	defer close(ackErrCh)
	defer close(ackStopCh)
	rclient, err := redis.DialTimeout("tcp", addr, timeout)
	if err != nil {
		ackErrCh <- err
		return
	}
	defer rclient.Close()

	var track int64
ackloop:
	for {
		select {
		case a, ok := <-ackCh:
			if !ok {
				return
			}
			if err := doAck(rclient, a); err != nil {
				ackErrCh <- err
				return
			}
			track--

		case _, ok := <-ackTrackCh:
			if !ok {
				break ackloop
			}
			track++
		}
	}

	// If we're here there have been no errors and ackCh is still open, but
	// ackTrackCh is closed meaning that there are no new events being sent out
	// and we have only to wait for acks from any stragglers. After 30 seconds
	// okq has reclaimed the events anyway, so no point in waiting after that
	for ; track > 0; track-- {
		select {
		case a := <-ackCh:
			if err := doAck(rclient, a); err != nil {
				// We don't bother writing to ackErrCh, only ackTrackCh matters
				// at this point, any errors from here won't bubble up anyway
				return
			}
		case <-time.After(30 * time.Second):
			return
		}
	}
}

func doAck(rclient *redis.Client, a *ack) error {
	return timedCmd(rclient, "QACK", a.queue, a.id).Err
}
<|endoftext|>


<commit_before>package lfs

import (
	"bytes"
	"crypto/sha256"
	"fmt"
	"io"

	"github.com/git-lfs/git-lfs/git"
)

// runCatFileBatch uses 'git cat-file --batch' to get the object contents of a
// git object, given its sha1. The contents will be decoded into a Git LFS
// pointer. Git Blob SHA1s are read from the sha1Ch channel and fed to STDIN.
// Results are parsed from STDOUT, and any eligible LFS pointers are sent to
// pointerCh. If a Git Blob is not an LFS pointer, check the lockableSet to see
// if that blob is for a locked file. Any errors are sent to errCh. An error is
// returned if the 'git cat-file' command fails to start.
func runCatFileBatch(pointerCh chan *WrappedPointer, lockableCh chan string, lockableSet *lockableNameSet, revs *StringChannelWrapper, errCh chan error) error {
	scanner, err := NewPointerScanner()
	if err != nil {
		return err
	}

	go func() {
		for r := range revs.Results {
			canScan := scanner.Scan(r)

			if err := scanner.Err(); err != nil {
				errCh <- err
			} else if p := scanner.Pointer(); p != nil {
				pointerCh <- p
			} else if b := scanner.BlobSHA(); len(b) == 40 {
				if name, ok := lockableSet.Check(b); ok {
					lockableCh <- name
				}
			}

			if !canScan {
				break
			}
		}

		if err := revs.Wait(); err != nil {
			errCh <- err
		}

		if err := scanner.Close(); err != nil {
			errCh <- err
		}

		close(pointerCh)
		close(errCh)
		close(lockableCh)
	}()

	return nil
}

type PointerScanner struct {
	scanner *git.ObjectScanner

	blobSha     string
	contentsSha string
	pointer     *WrappedPointer
	err         error
}

func NewPointerScanner() (*PointerScanner, error) {
	scanner, err := git.NewObjectScanner()
	if err != nil {
		return nil, err
	}

	return &PointerScanner{scanner: scanner}, nil
}

func (s *PointerScanner) BlobSHA() string {
	return s.blobSha
}

func (s *PointerScanner) ContentsSha() string {
	return s.contentsSha
}

func (s *PointerScanner) Pointer() *WrappedPointer {
	return s.pointer
}

func (s *PointerScanner) Err() error {
	return s.err
}

func (s *PointerScanner) Scan(sha string) bool {
	s.pointer, s.err = nil, nil
	s.blobSha, s.contentsSha = "", ""

	b, c, p, err := s.next(sha)
	s.blobSha = b
	s.contentsSha = c
	s.pointer = p

	if err != nil {
		if err != io.EOF {
			s.err = err
		}
		return false
	}

	return true
}

func (s *PointerScanner) Close() error {
	return s.scanner.Close()
}

func (s *PointerScanner) next(blob string) (string, string, *WrappedPointer, error) {
	if !s.scanner.Scan(blob) {
		if err := s.scanner.Err(); err != nil {
			return "", "", nil, err
		}
		return "", "", nil, io.EOF
	}

	blobSha := s.scanner.Sha1()
	size := s.scanner.Size()

	sha := sha256.New()

	var buf *bytes.Buffer
	var to io.Writer = sha
	if size <= blobSizeCutoff {
		buf = bytes.NewBuffer(make([]byte, 0, size))
		to = io.MultiWriter(to, buf)
	}

	read, err := io.CopyN(to, s.scanner.Contents(), int64(size))
	if err != nil {
		return blobSha, "", nil, err
	}

	if int64(size) != read {
		return blobSha, "", nil, fmt.Errorf("expected %d bytes, read %d bytes", size, read)
	}

	var pointer *WrappedPointer
	var contentsSha string

	if size <= blobSizeCutoff {
		if p, err := DecodePointer(bytes.NewReader(buf.Bytes())); err != nil {
			contentsSha = fmt.Sprintf("%x", sha.Sum(nil))
		} else {
			pointer = &WrappedPointer{
				Sha1:    blobSha,
				Pointer: p,
			}
			contentsSha = p.Oid
		}
	} else {
		contentsSha = fmt.Sprintf("%x", sha.Sum(nil))
	}

	return blobSha, contentsSha, pointer, err
}
<commit_msg>lfs: detect errors when object scanner fails<commit_after>package lfs

import (
	"bytes"
	"crypto/sha256"
	"fmt"
	"io"

	"github.com/git-lfs/git-lfs/git"
)

// runCatFileBatch uses 'git cat-file --batch' to get the object contents of a
// git object, given its sha1. The contents will be decoded into a Git LFS
// pointer. Git Blob SHA1s are read from the sha1Ch channel and fed to STDIN.
// Results are parsed from STDOUT, and any eligible LFS pointers are sent to
// pointerCh. If a Git Blob is not an LFS pointer, check the lockableSet to see
// if that blob is for a locked file. Any errors are sent to errCh. An error is
// returned if the 'git cat-file' command fails to start.
func runCatFileBatch(pointerCh chan *WrappedPointer, lockableCh chan string, lockableSet *lockableNameSet, revs *StringChannelWrapper, errCh chan error) error {
	scanner, err := NewPointerScanner()
	if err != nil {
		return err
	}

	go func() {
		canScan := true
		for r := range revs.Results {
			canScan = scanner.Scan(r)

			if err := scanner.Err(); err != nil {
				errCh <- err
			} else if p := scanner.Pointer(); p != nil {
				pointerCh <- p
			} else if b := scanner.BlobSHA(); len(b) == 40 {
				if name, ok := lockableSet.Check(b); ok {
					lockableCh <- name
				}
			}

			if !canScan {
				break
			}
		}

		if canScan {
			if err := revs.Wait(); err != nil {
				errCh <- err
			}
		}

		if err := scanner.Close(); err != nil {
			errCh <- err
		}

		close(pointerCh)
		close(errCh)
		close(lockableCh)
	}()

	return nil
}

type PointerScanner struct {
	scanner *git.ObjectScanner

	blobSha     string
	contentsSha string
	pointer     *WrappedPointer
	err         error
}

func NewPointerScanner() (*PointerScanner, error) {
	scanner, err := git.NewObjectScanner()
	if err != nil {
		return nil, err
	}

	return &PointerScanner{scanner: scanner}, nil
}

func (s *PointerScanner) BlobSHA() string {
	return s.blobSha
}

func (s *PointerScanner) ContentsSha() string {
	return s.contentsSha
}

func (s *PointerScanner) Pointer() *WrappedPointer {
	return s.pointer
}

func (s *PointerScanner) Err() error {
	return s.err
}

func (s *PointerScanner) Scan(sha string) bool {
	s.pointer, s.err = nil, nil
	s.blobSha, s.contentsSha = "", ""

	b, c, p, err := s.next(sha)
	s.blobSha = b
	s.contentsSha = c
	s.pointer = p

	if err != nil {
		if err != io.EOF {
			s.err = err
		}
		return false
	}

	return true
}

func (s *PointerScanner) Close() error {
	return s.scanner.Close()
}

func (s *PointerScanner) next(blob string) (string, string, *WrappedPointer, error) {
	if !s.scanner.Scan(blob) {
		if err := s.scanner.Err(); err != nil {
			return "", "", nil, err
		}
		return "", "", nil, io.EOF
	}

	blobSha := s.scanner.Sha1()
	size := s.scanner.Size()

	sha := sha256.New()

	var buf *bytes.Buffer
	var to io.Writer = sha
	if size <= blobSizeCutoff {
		buf = bytes.NewBuffer(make([]byte, 0, size))
		to = io.MultiWriter(to, buf)
	}

	read, err := io.CopyN(to, s.scanner.Contents(), int64(size))
	if err != nil {
		return blobSha, "", nil, err
	}

	if int64(size) != read {
		return blobSha, "", nil, fmt.Errorf("expected %d bytes, read %d bytes", size, read)
	}

	var pointer *WrappedPointer
	var contentsSha string

	if size <= blobSizeCutoff {
		if p, err := DecodePointer(bytes.NewReader(buf.Bytes())); err != nil {
			contentsSha = fmt.Sprintf("%x", sha.Sum(nil))
		} else {
			pointer = &WrappedPointer{
				Sha1:    blobSha,
				Pointer: p,
			}
			contentsSha = p.Oid
		}
	} else {
		contentsSha = fmt.Sprintf("%x", sha.Sum(nil))
	}

	return blobSha, contentsSha, pointer, err
}
<|endoftext|>


<commit_before>package rsa

/*
	#cgo LDFLAGS: -lssl -lcrypto
	#include <openssl/rsa.h>
	#include <openssl/engine.h>
	#include <openssl/pem.h>
	#include <openssl/err.h>
	#include <stdio.h>
	#include <stdlib.h>
	#include <string.h>

	char last_error_string[2048] = {0};

 	RSA* rsa_read_pem_public(char* pem){
 		FILE * fp = fopen(pem,"r");
 		if(fp == NULL){
 			snprintf(last_error_string,sizeof(last_error_string),"open \"%s\" failed",pem);
			return NULL;
 		}

 		RSA * public_key = RSA_new();

 		if (!PEM_read_RSA_PUBKEY(fp, &public_key, NULL, NULL)){
 			snprintf(last_error_string,sizeof(last_error_string),"%s",ERR_error_string(ERR_get_error(),NULL));
			RSA_free(public_key);
			return NULL;
		}
		return public_key;
 	}

 	RSA* rsa_read_pem_private(char* pem){
 		FILE * fp = fopen(pem,"r");
 		if(fp == NULL){
 			snprintf(last_error_string,sizeof(last_error_string),"open \"%s\" failed",pem);
			return NULL;
 		}
 		RSA * private_key = RSA_new();

 		if (!PEM_read_RSAPrivateKey(fp, &private_key, NULL, NULL)){
 			snprintf(last_error_string,sizeof(last_error_string),"%s",ERR_error_string(ERR_get_error(),NULL));
			RSA_free(private_key);
			return NULL;
		}
		return private_key;
 	}

	int rsa_private_encrypt(int fromSize,unsigned char *from,char** to, char* pem, int padding){
		RSA* private_key = rsa_read_pem_private(pem);
		if(!private_key){
			return -1;
		}
		*to = (char*)malloc(sizeof(char) * RSA_size(private_key));
		int n = RSA_private_encrypt(fromSize,from,(unsigned char *)*to,private_key,padding);
		if (n == -1){
 			snprintf(last_error_string,sizeof(last_error_string),"%s",ERR_error_string(ERR_get_error(),NULL));
		}
		RSA_free(private_key);
		return n;
	}

	int rsa_public_decrypt(int fromSize,unsigned char *from,char** to, char* pem, int padding){
		RSA* public_key = rsa_read_pem_public(pem);
		if(!public_key){
			return -1;
		}
		*to = (char*)malloc(sizeof(char) * RSA_size(public_key));
		int n = RSA_public_decrypt(fromSize,from,(unsigned char *)*to,public_key,padding);
		if (n == -1){
 			snprintf(last_error_string,sizeof(last_error_string),"%s",ERR_error_string(ERR_get_error(),NULL));
		}
		RSA_free(public_key);
		return n;

	}
*/
import "C"
import "unsafe"
import "fmt"

const (
	RSA_PKCS1_PADDING = C.RSA_PKCS1_PADDING
	RSA_NO_PADDING    = C.RSA_NO_PADDING
)

func PublicDecrypt(from []byte, pem string, padding int) ([]byte, error) {
	var to *C.char = nil

	if n := C.rsa_public_decrypt(C.int(len(from)),
		(*C.uchar)(unsafe.Pointer(&from[0])),
		//(*C.uchar)(unsafe.Pointer(&to[0])),
		(**C.char)(unsafe.Pointer(&to)),
		C.CString(pem),
		C.int(padding)); n < 0 {
		return nil, fmt.Errorf("%s", C.GoString(&C.last_error_string[0]))
	} else {
		m := C.GoBytes(unsafe.Pointer(to), n)
		C.free(unsafe.Pointer(to))
		return m, nil
	}
}

func PrivateEncrypt(from []byte, pem string, padding int) ([]byte, error) {
	var to *C.char = nil

	if n := C.rsa_private_encrypt(C.int(len(from)),
		(*C.uchar)(unsafe.Pointer(&from[0])),
		(**C.char)(unsafe.Pointer(&to)),
		//(*C.uchar)(unsafe.Pointer(&to[0])),
		C.CString(pem),
		C.int(padding)); n < 0 {
		return nil, fmt.Errorf("%s", C.GoString(&C.last_error_string[0]))
	} else {
		m := C.GoBytes(unsafe.Pointer(to), n)
		C.free(unsafe.Pointer(to))
		return m, nil
	}
}

func Destroy() {
	C.ERR_free_strings()
}

func init() {
	C.ERR_load_crypto_strings()
}
<commit_msg>fix init & destroy<commit_after>package rsa

/*
	#cgo LDFLAGS: -lssl -lcrypto
	#include <openssl/rsa.h>
	#include <openssl/engine.h>
	#include <openssl/pem.h>
	#include <openssl/err.h>
	#include <stdio.h>
	#include <stdlib.h>
	#include <string.h>

	char last_error_string[2048] = {0};

 	RSA* rsa_read_pem_public(char* pem){
 		FILE * fp = fopen(pem,"r");
 		if(fp == NULL){
 			snprintf(last_error_string,sizeof(last_error_string),"open \"%s\" failed",pem);
			return NULL;
 		}

 		RSA * public_key = RSA_new();

 		if (!PEM_read_RSA_PUBKEY(fp, &public_key, NULL, NULL)){
 			snprintf(last_error_string,sizeof(last_error_string),"%s",ERR_error_string(ERR_get_error(),NULL));
			RSA_free(public_key);
			return NULL;
		}
		return public_key;
 	}

 	RSA* rsa_read_pem_private(char* pem){
 		FILE * fp = fopen(pem,"r");
 		if(fp == NULL){
 			snprintf(last_error_string,sizeof(last_error_string),"open \"%s\" failed",pem);
			return NULL;
 		}
 		RSA * private_key = RSA_new();

 		if (!PEM_read_RSAPrivateKey(fp, &private_key, NULL, NULL)){
 			snprintf(last_error_string,sizeof(last_error_string),"%s",ERR_error_string(ERR_get_error(),NULL));
			RSA_free(private_key);
			return NULL;
		}
		return private_key;
 	}

	int rsa_private_encrypt(int fromSize,unsigned char *from,char** to, char* pem, int padding){
		RSA* private_key = rsa_read_pem_private(pem);
		if(!private_key){
			return -1;
		}
		*to = (char*)malloc(sizeof(char) * RSA_size(private_key));
		int n = RSA_private_encrypt(fromSize,from,(unsigned char *)*to,private_key,padding);
		if (n == -1){
 			snprintf(last_error_string,sizeof(last_error_string),"%s",ERR_error_string(ERR_get_error(),NULL));
		}
		RSA_free(private_key);
		return n;
	}

	int rsa_public_decrypt(int fromSize,unsigned char *from,char** to, char* pem, int padding){
		RSA* public_key = rsa_read_pem_public(pem);
		if(!public_key){
			return -1;
		}
		*to = (char*)malloc(sizeof(char) * RSA_size(public_key));
		int n = RSA_public_decrypt(fromSize,from,(unsigned char *)*to,public_key,padding);
		if (n == -1){
 			snprintf(last_error_string,sizeof(last_error_string),"%s",ERR_error_string(ERR_get_error(),NULL));
		}
		RSA_free(public_key);
		return n;

	}
*/
import "C"
import "unsafe"
import "fmt"

const (
	RSA_PKCS1_PADDING = C.RSA_PKCS1_PADDING
	RSA_NO_PADDING    = C.RSA_NO_PADDING
)

func PublicDecrypt(from []byte, pem string, padding int) ([]byte, error) {
	var to *C.char = nil

	if n := C.rsa_public_decrypt(C.int(len(from)),
		(*C.uchar)(unsafe.Pointer(&from[0])),
		//(*C.uchar)(unsafe.Pointer(&to[0])),
		(**C.char)(unsafe.Pointer(&to)),
		C.CString(pem),
		C.int(padding)); n < 0 {
		return nil, fmt.Errorf("%s", C.GoString(&C.last_error_string[0]))
	} else {
		m := C.GoBytes(unsafe.Pointer(to), n)
		C.free(unsafe.Pointer(to))
		return m, nil
	}
}

func PrivateEncrypt(from []byte, pem string, padding int) ([]byte, error) {
	var to *C.char = nil

	if n := C.rsa_private_encrypt(C.int(len(from)),
		(*C.uchar)(unsafe.Pointer(&from[0])),
		(**C.char)(unsafe.Pointer(&to)),
		//(*C.uchar)(unsafe.Pointer(&to[0])),
		C.CString(pem),
		C.int(padding)); n < 0 {
		return nil, fmt.Errorf("%s", C.GoString(&C.last_error_string[0]))
	} else {
		m := C.GoBytes(unsafe.Pointer(to), n)
		C.free(unsafe.Pointer(to))
		return m, nil
	}
}

func Destroy() {
	C.ERR_clear_error()
}

func init() {
	C.ERR_load_ERR_strings()
}
<|endoftext|>


<commit_before>// Copyright 2014-2015 The project AUTHORS. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package src

import (
	"bytes"
	"errors"
	"testing"
)

func TestIgnoreWhitespaces(t *testing.T) {
	buf := bytes.NewBufferString("     x")
	scan := newScanner(buf)
	if err := scan.ignoreWhitespaces(); err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if c := scan.buf[scan.pos:]; len(c) != 1 || c[0] != 'x' {
		t.Errorf("scan.ignoreWhitespace: found '%s', expected 'x'", c)
	}
}

func TestIsWhitespace(t *testing.T) {
	ws := []byte{' ', '\n', '\t', '\r'}
	for _, c := range ws {
		if !isWhitespace(c) {
			t.Errorf("'%c' should be considered as a whitespace", c)
		}
	}

	nows := []byte{'a', ',', ';', ':', '0', '\v'}
	for _, c := range nows {
		if isWhitespace(c) {
			t.Errorf("'%c' should not be considered as a whitespace", c)
		}
	}
}

func TestIsDigit(t *testing.T) {
	digits := []byte{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9'}
	for _, c := range digits {
		if !isDigit(c) {
			t.Errorf("'%c' should be considered as a digit", c)
		}
	}

	var c byte
	for c = 'a'; c <= 'z'; c++ {
		if isDigit(c) {
			t.Errorf("'%c' should not be considered as a digit", c)
		}
	}
}

func TestNextKey(t *testing.T) {
	validKeys := []string{
		`"foo":`,
		`"foo"  :`,
		`"foo": "bar"`,
	}
	for _, keyInput := range validKeys {
		buf := bytes.NewBufferString(keyInput)
		scan := newScanner(buf)
		key, err := scan.nextKey()
		if err != nil {
			t.Error(err)
		}
		if key != "foo" {
			t.Errorf("nextKey: found '%s', expected 'foo'", key)
		}
	}

	invalidKeys := map[string]error{
		`"foo"`: errors.New("expected ':', found EOF"),
		`"foo`:  errors.New("expected key, found EOF"),
		`foo`:   errors.New("expected '\"', found 'f'"),
	}
	for keyInput, expectedErr := range invalidKeys {
		buf := bytes.NewBufferString(keyInput)
		scan := newScanner(buf)
		_, err := scan.nextKey()
		if err == nil {
			t.Errorf("nextKey: '%s' is expected to return the error '%v'", keyInput, expectedErr)
			continue
		}
		if err.Error() != expectedErr.Error() {
			t.Errorf("nextKey: found error \"%v\", expected error \"%v\"", err, expectedErr)
		}
	}
}

func TestNextValue(t *testing.T) {
	// test valid values
	validKeys := map[string]scanToken{
		`[`:     scanBeginArray,
		`]`:     scanEndArray,
		`{`:     scanBeginObject,
		`}`:     scanEndObject,
		`42`:    scanInt64Lit,
		`42.42`: scanFloat64Lit,
		`"foo"`: scanStringLit,
		`null`:  scanNullVal,
		`true`:  scanBoolLit,
		`false`: scanBoolLit,
		`,`:     scanComma,
	}
	for val, expectedTok := range validKeys {
		buf := bytes.NewBufferString(val)
		scan := newScanner(buf)
		_, tok, err := scan.nextValue()
		if err != nil {
			t.Errorf("nextValue '%s': %v", string(val), err)
		}
		if expectedTok != tok {
			t.Errorf("nextValue '%s': found '%v', expected '%v'", string(val), tok, expectedTok)
		}
	}

	// test invalid values
	invalidKeys := map[string]error{
		`z`:  errors.New("illegal value 'z'"),
		``:   errors.New("expected value, found EOF"),
		`  `: errors.New("expected value, found EOF"),
	}
	for val, expectedErr := range invalidKeys {
		buf := bytes.NewBufferString(val)
		scan := newScanner(buf)
		_, tok, err := scan.nextValue()
		if err == nil {
			t.Errorf("nextValue '%s': found no error, expected \"%v\"", string(val), expectedErr)
			continue
		}
		if err.Error() != expectedErr.Error() {
			t.Errorf("nextValue '%s': found \"%v\", expected \"%v\"", string(val), err, expectedErr)
		}
		if tok != scanIllegalToken {
			t.Errorf("nextValue '%s': found '%v', expected '%v'", string(val), tok, scanIllegalToken)
		}
	}
}

func TestReadNumber(t *testing.T) {
	// test int64
	buf := bytes.NewBufferString("42}")
	scan := newScanner(buf)
	val, tok, err := scan.readNumber()
	if err != nil {
		t.Fatal("readNumber '42}': ", err)
	}
	if tok != scanInt64Lit {
		t.Errorf("readNumber '42}': found token '%v', expected '%v'", tok, scanInt64Lit)
	}
	if s := string(val); s != "42" {
		t.Errorf("readNumber '42': found '%s', expected '42'", s)
	}

	// test float 64
	buf = bytes.NewBufferString("42.24}")
	scan = newScanner(buf)
	val, tok, err = scan.readNumber()
	if err != nil {
		t.Fatal("readNumber '42.24}': ", err)
	}
	if tok != scanFloat64Lit {
		t.Errorf("readNumber '42.24}': found token '%v', expected '%v'", tok, scanFloat64Lit)
	}
	if s := string(val); s != "42.24" {
		t.Errorf("readNumber '42.24}': found '%s', expected '42.24'", s)
	}

	// test empty number
	expectedErr := errors.New("expected number, found nothing")
	buf = bytes.NewBufferString(", foo")
	scan = newScanner(buf)
	val, tok, err = scan.readNumber()
	if err == nil {
		t.Fatalf("readNumber ', foo': found no error, expected \"%v\"", expectedErr)
	}
	if tok != scanIllegalToken {
		t.Errorf("readNumber ', foo': found token '%v', expected '%v'", tok, scanIllegalToken)
	}
	if val != nil {
		t.Errorf("readNumber ', foo': found '%s', expected 'nil'", string(val))
	}

	// TODO: test other errors
}
func TestReadNull(t *testing.T) {
	// test 'null'
	buf := bytes.NewBufferString("ull")
	scan := newScanner(buf)
	tok, err := scan.readNull()
	if err != nil {
		t.Fatal("readNull 'ull': ", err)
	}
	if tok != scanNullVal {
		t.Errorf("readNull 'ull': found '%v', expected '%v'", tok, scanNullVal)
	}

	// test error
	expectedErr := errors.New("invalid null value: found 'f', expected 'u'")
	buf = bytes.NewBufferString("foo")
	scan = newScanner(buf)
	tok, err = scan.readNull()
	if err == nil {
		t.Fatalf("readNull 'foo': found no error, expected \"%v\"", expectedErr)
	}
	if err.Error() != expectedErr.Error() {
		t.Fatalf("readNull 'foo': found \"%v\", expected \"%v\"", err, expectedErr)
	}
	if tok != scanIllegalToken {
		t.Errorf("readNull 'foo': found '%v', expected '%v'", tok, scanIllegalToken)
	}
}

func TestReadBool(t *testing.T) {
	// test 'true'
	buf := bytes.NewBufferString("rue")
	scan := newScanner(buf)
	val, err := scan.readBool('t')
	if err != nil {
		t.Fatal("readBool 'rue': ", err)
	}
	if s := string(val); s != "true" {
		t.Errorf("readBool 'rue': found '%s', expected 'true'", s)
	}

	// test 'false'
	buf = bytes.NewBufferString("alse")
	scan = newScanner(buf)
	val, err = scan.readBool('f')
	if err != nil {
		t.Fatal("readBool 'alse': ", err)
	}
	if s := string(val); s != "false" {
		t.Errorf("readBool 'alse': found '%s', expected 'false'", s)
	}

	// test error
	expectedErr := errors.New("invalid boolean value: found 'o', expected 'a'")
	buf = bytes.NewBufferString("oo")
	scan = newScanner(buf)
	val, err = scan.readBool('f')
	if err == nil {
		t.Fatal("readBool 'oo': found no error, expected \"%v\"", expectedErr)
	}
	if err.Error() != expectedErr.Error() {
		t.Fatalf("readBool 'oo': found \"%v\", expected \"%v\"", err, expectedErr)
	}
	if val != nil {
		t.Errorf("readBool 'oo': found '%s', expected 'nil'", string(val))
	}
}
<commit_msg>src/scanner: fix go vet error<commit_after>// Copyright 2014-2015 The project AUTHORS. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package src

import (
	"bytes"
	"errors"
	"testing"
)

func TestIgnoreWhitespaces(t *testing.T) {
	buf := bytes.NewBufferString("     x")
	scan := newScanner(buf)
	if err := scan.ignoreWhitespaces(); err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if c := scan.buf[scan.pos:]; len(c) != 1 || c[0] != 'x' {
		t.Errorf("scan.ignoreWhitespace: found '%s', expected 'x'", c)
	}
}

func TestIsWhitespace(t *testing.T) {
	ws := []byte{' ', '\n', '\t', '\r'}
	for _, c := range ws {
		if !isWhitespace(c) {
			t.Errorf("'%c' should be considered as a whitespace", c)
		}
	}

	nows := []byte{'a', ',', ';', ':', '0', '\v'}
	for _, c := range nows {
		if isWhitespace(c) {
			t.Errorf("'%c' should not be considered as a whitespace", c)
		}
	}
}

func TestIsDigit(t *testing.T) {
	digits := []byte{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9'}
	for _, c := range digits {
		if !isDigit(c) {
			t.Errorf("'%c' should be considered as a digit", c)
		}
	}

	var c byte
	for c = 'a'; c <= 'z'; c++ {
		if isDigit(c) {
			t.Errorf("'%c' should not be considered as a digit", c)
		}
	}
}

func TestNextKey(t *testing.T) {
	validKeys := []string{
		`"foo":`,
		`"foo"  :`,
		`"foo": "bar"`,
	}
	for _, keyInput := range validKeys {
		buf := bytes.NewBufferString(keyInput)
		scan := newScanner(buf)
		key, err := scan.nextKey()
		if err != nil {
			t.Error(err)
		}
		if key != "foo" {
			t.Errorf("nextKey: found '%s', expected 'foo'", key)
		}
	}

	invalidKeys := map[string]error{
		`"foo"`: errors.New("expected ':', found EOF"),
		`"foo`:  errors.New("expected key, found EOF"),
		`foo`:   errors.New("expected '\"', found 'f'"),
	}
	for keyInput, expectedErr := range invalidKeys {
		buf := bytes.NewBufferString(keyInput)
		scan := newScanner(buf)
		_, err := scan.nextKey()
		if err == nil {
			t.Errorf("nextKey: '%s' is expected to return the error '%v'", keyInput, expectedErr)
			continue
		}
		if err.Error() != expectedErr.Error() {
			t.Errorf("nextKey: found error \"%v\", expected error \"%v\"", err, expectedErr)
		}
	}
}

func TestNextValue(t *testing.T) {
	// test valid values
	validKeys := map[string]scanToken{
		`[`:     scanBeginArray,
		`]`:     scanEndArray,
		`{`:     scanBeginObject,
		`}`:     scanEndObject,
		`42`:    scanInt64Lit,
		`42.42`: scanFloat64Lit,
		`"foo"`: scanStringLit,
		`null`:  scanNullVal,
		`true`:  scanBoolLit,
		`false`: scanBoolLit,
		`,`:     scanComma,
	}
	for val, expectedTok := range validKeys {
		buf := bytes.NewBufferString(val)
		scan := newScanner(buf)
		_, tok, err := scan.nextValue()
		if err != nil {
			t.Errorf("nextValue '%s': %v", string(val), err)
		}
		if expectedTok != tok {
			t.Errorf("nextValue '%s': found '%v', expected '%v'", string(val), tok, expectedTok)
		}
	}

	// test invalid values
	invalidKeys := map[string]error{
		`z`:  errors.New("illegal value 'z'"),
		``:   errors.New("expected value, found EOF"),
		`  `: errors.New("expected value, found EOF"),
	}
	for val, expectedErr := range invalidKeys {
		buf := bytes.NewBufferString(val)
		scan := newScanner(buf)
		_, tok, err := scan.nextValue()
		if err == nil {
			t.Errorf("nextValue '%s': found no error, expected \"%v\"", string(val), expectedErr)
			continue
		}
		if err.Error() != expectedErr.Error() {
			t.Errorf("nextValue '%s': found \"%v\", expected \"%v\"", string(val), err, expectedErr)
		}
		if tok != scanIllegalToken {
			t.Errorf("nextValue '%s': found '%v', expected '%v'", string(val), tok, scanIllegalToken)
		}
	}
}

func TestReadNumber(t *testing.T) {
	// test int64
	buf := bytes.NewBufferString("42}")
	scan := newScanner(buf)
	val, tok, err := scan.readNumber()
	if err != nil {
		t.Fatal("readNumber '42}': ", err)
	}
	if tok != scanInt64Lit {
		t.Errorf("readNumber '42}': found token '%v', expected '%v'", tok, scanInt64Lit)
	}
	if s := string(val); s != "42" {
		t.Errorf("readNumber '42': found '%s', expected '42'", s)
	}

	// test float 64
	buf = bytes.NewBufferString("42.24}")
	scan = newScanner(buf)
	val, tok, err = scan.readNumber()
	if err != nil {
		t.Fatal("readNumber '42.24}': ", err)
	}
	if tok != scanFloat64Lit {
		t.Errorf("readNumber '42.24}': found token '%v', expected '%v'", tok, scanFloat64Lit)
	}
	if s := string(val); s != "42.24" {
		t.Errorf("readNumber '42.24}': found '%s', expected '42.24'", s)
	}

	// test empty number
	expectedErr := errors.New("expected number, found nothing")
	buf = bytes.NewBufferString(", foo")
	scan = newScanner(buf)
	val, tok, err = scan.readNumber()
	if err == nil {
		t.Fatalf("readNumber ', foo': found no error, expected \"%v\"", expectedErr)
	}
	if tok != scanIllegalToken {
		t.Errorf("readNumber ', foo': found token '%v', expected '%v'", tok, scanIllegalToken)
	}
	if val != nil {
		t.Errorf("readNumber ', foo': found '%s', expected 'nil'", string(val))
	}

	// TODO: test other errors
}
func TestReadNull(t *testing.T) {
	// test 'null'
	buf := bytes.NewBufferString("ull")
	scan := newScanner(buf)
	tok, err := scan.readNull()
	if err != nil {
		t.Fatal("readNull 'ull': ", err)
	}
	if tok != scanNullVal {
		t.Errorf("readNull 'ull': found '%v', expected '%v'", tok, scanNullVal)
	}

	// test error
	expectedErr := errors.New("invalid null value: found 'f', expected 'u'")
	buf = bytes.NewBufferString("foo")
	scan = newScanner(buf)
	tok, err = scan.readNull()
	if err == nil {
		t.Fatalf("readNull 'foo': found no error, expected \"%v\"", expectedErr)
	}
	if err.Error() != expectedErr.Error() {
		t.Fatalf("readNull 'foo': found \"%v\", expected \"%v\"", err, expectedErr)
	}
	if tok != scanIllegalToken {
		t.Errorf("readNull 'foo': found '%v', expected '%v'", tok, scanIllegalToken)
	}
}

func TestReadBool(t *testing.T) {
	// test 'true'
	buf := bytes.NewBufferString("rue")
	scan := newScanner(buf)
	val, err := scan.readBool('t')
	if err != nil {
		t.Fatal("readBool 'rue': ", err)
	}
	if s := string(val); s != "true" {
		t.Errorf("readBool 'rue': found '%s', expected 'true'", s)
	}

	// test 'false'
	buf = bytes.NewBufferString("alse")
	scan = newScanner(buf)
	val, err = scan.readBool('f')
	if err != nil {
		t.Fatal("readBool 'alse': ", err)
	}
	if s := string(val); s != "false" {
		t.Errorf("readBool 'alse': found '%s', expected 'false'", s)
	}

	// test error
	expectedErr := errors.New("invalid boolean value: found 'o', expected 'a'")
	buf = bytes.NewBufferString("oo")
	scan = newScanner(buf)
	val, err = scan.readBool('f')
	if err == nil {
		t.Fatalf("readBool 'oo': found no error, expected \"%v\"", expectedErr)
	}
	if err.Error() != expectedErr.Error() {
		t.Fatalf("readBool 'oo': found \"%v\", expected \"%v\"", err, expectedErr)
	}
	if val != nil {
		t.Errorf("readBool 'oo': found '%s', expected 'nil'", string(val))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	_ "launchpad.net/tomb"
	"log"
	_ "math"
	"strconv"
	"sync"
	"time"
)

type SandboxInfo struct {
	Id         int
	Properties map[string]string
	Duration   time.Duration
	StartTime  time.Time
	Emitter    LatencyEmitter
	WaitGroup  *sync.WaitGroup
	Factory    BehaviorFactory
}

type sandbox struct {
	id            int
	props         map[string]string
	d             time.Duration
	start         time.Time
	emitter       LatencyEmitter
	wg            *sync.WaitGroup
	behavior      Behavior
	factory       BehaviorFactory
	stall         bool
	opsPerStall   int
	stall_counter int
	stalls        int
}

func NewSandbox(info *SandboxInfo) *sandbox {
	return &sandbox{
		id:      info.Id,
		props:   info.Properties,
		d:       info.Duration,
		start:   info.StartTime,
		emitter: info.Emitter,
		wg:      info.WaitGroup,
		factory: info.Factory,
	}
}

func (this *sandbox) Start() {
	go this.loop()
}

// func (this *sandbox) Stop() (err error) {
// 	this.t.Kill(nil)
// 	return this.t.Wait()
// }

func (this *sandbox) loop() {
	//defer this.t.Done()

	this.setup()

	defer this.teardown()

	for !this.expired() {
		this.update()
	}
}

func (this *sandbox) setup() {
	err := this.parseProperties()
	if err != nil {
		log.Fatal(err)
	}

	res, err := this.init()
	if err != nil {
		log.Fatal(err)
	}

	this.behavior = res
}

func (this *sandbox) init() (res Behavior, err error) {
	defer func() {
		e := recover()
		if e != nil {
			if u, ok := e.(error); ok {
				err = u
			}
		}

		return
	}()

	res = this.factory()
	err = res.Init(this.props)
	if err != nil {
		return
	}

	return
}

func (this *sandbox) expired() (ok bool) {
	return time.Since(this.start) > this.d
}

func (this *sandbox) update() {
	_ = this.work()
}

func (this *sandbox) work() (err error) {
	defer func() {
		e := recover()
		if e != nil {
			if u, ok := e.(error); ok {
				err = u
			}
		}

		return
	}()

	t0 := time.Now()
	res := this.behavior.Work(t0)
	d := time.Since(t0)
	usec := int64(d / time.Microsecond)

	// If stalling is enabled, then introduce a 1s pause
	// every N ops. Hopefully, this will fix the issue with
	// some Work operations completing far too fast for
	// the progress/stats reporting goroutines to work.
	// I'm not sure why the select{} construct isn't picking
	// the other channels...
	if this.stall {
		this.stall_counter += 1

		if this.stall_counter > this.opsPerStall {
			this.stalls += 1
			this.stall_counter = 0
			<-time.After(1 * time.Second)
		}
	}

	this.emitter.PublishResponseTime(this.id, usec, res)
	return
}

func (this *sandbox) teardown() {
	this.close()
	this.wg.Done()
}

func (this *sandbox) close() (err error) {
	defer func() {
		e := recover()
		if e != nil {
			if u, ok := e.(error); ok {
				err = u
			}
		}

		return
	}()

	this.behavior.Close()
	this.behavior = nil
	return
}

func (this *sandbox) parseProperties() (err error) {
	props := this.props

	const (
		MinOpsPerStall = 10000
	)

	if v, ok := props["internals.OpsPerStall"]; ok {
		u, err := strconv.Atoi(v)
		if err != nil || u < MinOpsPerStall {
			return errors.New("internals.OpsPerStall must be >= 10000 ops (e.g. don't use this option when response times are slow enough to measure)")
		}

		this.stall = true
		this.opsPerStall = u
	} else {
		this.stall = false
	}

	return
}
<commit_msg>fail fast when the work() function recovers from a panic<commit_after>package main

import (
	"errors"
	_ "launchpad.net/tomb"
	"log"
	_ "math"
	"strconv"
	"sync"
	"time"
)

type SandboxInfo struct {
	Id         int
	Properties map[string]string
	Duration   time.Duration
	StartTime  time.Time
	Emitter    LatencyEmitter
	WaitGroup  *sync.WaitGroup
	Factory    BehaviorFactory
}

type sandbox struct {
	id            int
	props         map[string]string
	d             time.Duration
	start         time.Time
	emitter       LatencyEmitter
	wg            *sync.WaitGroup
	behavior      Behavior
	factory       BehaviorFactory
	stall         bool
	opsPerStall   int
	stall_counter int
	stalls        int
}

func NewSandbox(info *SandboxInfo) *sandbox {
	return &sandbox{
		id:      info.Id,
		props:   info.Properties,
		d:       info.Duration,
		start:   info.StartTime,
		emitter: info.Emitter,
		wg:      info.WaitGroup,
		factory: info.Factory,
	}
}

func (this *sandbox) Start() {
	go this.loop()
}

// func (this *sandbox) Stop() (err error) {
// 	this.t.Kill(nil)
// 	return this.t.Wait()
// }

func (this *sandbox) loop() {
	//defer this.t.Done()

	this.setup()

	defer this.teardown()

	for !this.expired() {
		this.update()
	}
}

func (this *sandbox) setup() {
	err := this.parseProperties()
	if err != nil {
		log.Fatal(err)
	}

	res, err := this.init()
	if err != nil {
		log.Fatal(err)
	}

	this.behavior = res
}

func (this *sandbox) init() (res Behavior, err error) {
	defer func() {
		e := recover()
		if e != nil {
			if u, ok := e.(error); ok {
				err = u
			}
		}

		return
	}()

	res = this.factory()
	err = res.Init(this.props)
	if err != nil {
		return
	}

	return
}

func (this *sandbox) expired() (ok bool) {
	return time.Since(this.start) > this.d
}

func (this *sandbox) update() {
	err := this.work()
	if err != nil {
		log.Fatalf("behavior panic during Work: %+v", err)
	}
}

func (this *sandbox) work() (err error) {
	defer func() {
		e := recover()
		if e != nil {
			if u, ok := e.(error); ok {
				err = u
			}
		}

		return
	}()

	t0 := time.Now()
	res := this.behavior.Work(t0)
	d := time.Since(t0)
	usec := int64(d / time.Microsecond)

	// If stalling is enabled, then introduce a 1s pause
	// every N ops. Hopefully, this will fix the issue with
	// some Work operations completing far too fast for
	// the progress/stats reporting goroutines to work.
	// I'm not sure why the select{} construct isn't picking
	// the other channels...
	if this.stall {
		this.stall_counter += 1

		if this.stall_counter > this.opsPerStall {
			this.stalls += 1
			this.stall_counter = 0
			<-time.After(1 * time.Second)
		}
	}

	this.emitter.PublishResponseTime(this.id, usec, res)
	return
}

func (this *sandbox) teardown() {
	this.close()
	this.wg.Done()
}

func (this *sandbox) close() (err error) {
	defer func() {
		e := recover()
		if e != nil {
			if u, ok := e.(error); ok {
				err = u
			}
		}

		return
	}()

	this.behavior.Close()
	this.behavior = nil
	return
}

func (this *sandbox) parseProperties() (err error) {
	props := this.props

	const (
		MinOpsPerStall = 10000
	)

	if v, ok := props["internals.OpsPerStall"]; ok {
		u, err := strconv.Atoi(v)
		if err != nil || u < MinOpsPerStall {
			return errors.New("internals.OpsPerStall must be >= 10000 ops (e.g. don't use this option when response times are slow enough to measure)")
		}

		this.stall = true
		this.opsPerStall = u
	} else {
		this.stall = false
	}

	return
}
<|endoftext|>