<commit_before>package genesis

import (
	"bytes"
	"encoding/csv"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"os"
	"strconv"

	"github.com/eris-ltd/common/go/common"
	stypes "github.com/eris-ltd/eris-db/manager/eris-mint/state/types"
	ptypes "github.com/eris-ltd/eris-db/permission/types"

	"github.com/tendermint/go-crypto"
	wire "github.com/tendermint/go-wire"
)

//------------------------------------------------------------------------------------
// core functions

func GenerateKnown(chainID, accountsPathCSV, validatorsPathCSV string) ([]byte, error) {
	var genDoc *stypes.GenesisDoc
	var err error

	// TODO [eb] eliminate reading priv_val ... [zr] where?
	if accountsPathCSV == "" || validatorsPathCSV == "" {
		return []byte{}, fmt.Errorf("both accounts.csv and validators.csv is required")

	}

	pubkeys, amts, names, perms, setbits, err := parseCsv(validatorsPathCSV)
	if err != nil {
		return nil, err
	}

	pubkeysA, amtsA, namesA, permsA, setbitsA, err := parseCsv(accountsPathCSV)
	if err != nil {
		return nil, err
	}

	genDoc = newGenDoc(chainID, len(pubkeys), len(pubkeysA))
	for i, pk := range pubkeys {
		genDocAddValidator(genDoc, pk, amts[i], names[i], perms[i], setbits[i], i)
	}
	for i, pk := range pubkeysA {
		genDocAddAccount(genDoc, pk, amtsA[i], namesA[i], permsA[i], setbitsA[i], i)
	}

	buf, buf2, n := new(bytes.Buffer), new(bytes.Buffer), new(int)
	wire.WriteJSON(genDoc, buf, n, &err)
	if err != nil {
		return nil, err
	}
	if err := json.Indent(buf2, buf.Bytes(), "", "\t"); err != nil {
		return nil, err
	}
	genesisBytes := buf2.Bytes()

	return genesisBytes, nil
}

//-----------------------------------------------------------------------------
// gendoc convenience functions

func newGenDoc(chainID string, nVal, nAcc int) *stypes.GenesisDoc {
	genDoc := stypes.GenesisDoc{
		ChainID: chainID,
		// GenesisTime: time.Now(),
	}
	genDoc.Accounts = make([]stypes.GenesisAccount, nAcc)
	genDoc.Validators = make([]stypes.GenesisValidator, nVal)
	return &genDoc
}

func genDocAddAccount(genDoc *stypes.GenesisDoc, pubKey crypto.PubKeyEd25519, amt int64, name string, perm, setbit ptypes.PermFlag, index int) {
	addr := pubKey.Address()
	acc := stypes.GenesisAccount{
		Address: addr,
		Amount:  amt,
		Name:    name,
		Permissions: &ptypes.AccountPermissions{
			Base: ptypes.BasePermissions{
				Perms:  perm,
				SetBit: setbit,
			},
		},
	}
	if index < 0 {
		genDoc.Accounts = append(genDoc.Accounts, acc)
	} else {
		genDoc.Accounts[index] = acc
	}
}

func genDocAddValidator(genDoc *stypes.GenesisDoc, pubKey crypto.PubKeyEd25519, amt int64, name string, perm, setbit ptypes.PermFlag, index int) {
	addr := pubKey.Address()
	genDoc.Validators[index] = stypes.GenesisValidator{
		PubKey: pubKey,
		Amount: amt,
		Name:   name,
		UnbondTo: []stypes.BasicAccount{
			{
				Address: addr,
				Amount:  amt,
			},
		},
	}
	// [zr] why no index < 0 like in genDocAddAccount?
}

//-----------------------------------------------------------------------------
// util functions

// convert hex strings to ed25519 pubkeys
func pubKeyStringsToPubKeys(pubkeys []string) ([]crypto.PubKeyEd25519, error) {
	pubKeys := make([]crypto.PubKeyEd25519, len(pubkeys))
	for i, k := range pubkeys {
		pubBytes, err := hex.DecodeString(k)
		if err != nil {
			return pubKeys, err
		}
		copy(pubKeys[i][:], pubBytes)
	}
	return pubKeys, nil
}

// empty is over written
func ifExistsElse(list []string, index int, defaultValue string) string {
	if len(list) > index {
		if list[index] != "" {
			return list[index]
		}
	}
	return defaultValue
}

// takes a csv in the following format: pubkey, starting balance, name, permissions, setbit
func parseCsv(filePath string) (pubKeys []crypto.PubKeyEd25519, amts []int64, names []string, perms, setbits []ptypes.PermFlag, err error) {

	csvFile, err := os.Open(filePath)
	if err != nil {
		common.Exit(fmt.Errorf("Couldn't open file: %s: %v", filePath, err))
	}
	defer csvFile.Close()

	r := csv.NewReader(csvFile)
	//r.FieldsPerRecord = # of records expected
	params, err := r.ReadAll()
	if err != nil {
		common.Exit(fmt.Errorf("Couldn't read file: %v", err))

	}

	pubkeys := make([]string, len(params))
	amtS := make([]string, len(params))
	names = make([]string, len(params))
	permsS := make([]string, len(params))
	setbitS := make([]string, len(params))
	for i, each := range params {
		pubkeys[i] = each[0]
		amtS[i] = ifExistsElse(each, 1, "1000")
		names[i] = ifExistsElse(each, 2, "")
		permsS[i] = ifExistsElse(each, 3, fmt.Sprintf("%d", ptypes.DefaultPermFlags))
		setbitS[i] = ifExistsElse(each, 4, permsS[i])
	}

	//TODO convert int to uint64, see issue #25
	perms = make([]ptypes.PermFlag, len(permsS))
	for i, perm := range permsS {
		pflag, err := strconv.Atoi(perm)
		if err != nil {
			common.Exit(fmt.Errorf("Permissions must be an integer"))
		}
		perms[i] = ptypes.PermFlag(pflag)
	}
	setbits = make([]ptypes.PermFlag, len(setbitS))
	for i, setbit := range setbitS {
		setbitsFlag, err := strconv.Atoi(setbit)
		if err != nil {
			common.Exit(fmt.Errorf("SetBits must be an integer"))
		}
		setbits[i] = ptypes.PermFlag(setbitsFlag)
	}

	// convert amts to ints
	amts = make([]int64, len(amtS))
	for i, a := range amtS {
		if amts[i], err = strconv.ParseInt(a, 10, 64); err != nil {
			err = fmt.Errorf("Invalid amount: %v", err)
			return
		}
	}

	// convert pubkey hex strings to struct
	pubKeys, err = pubKeyStringsToPubKeys(pubkeys)
	if err != nil {
		return
	}

	return pubKeys, amts, names, perms, setbits, nil
}
<commit_msg>working genesis generator; prints to stdout<commit_after>package genesis

import (
	"bytes"
	"encoding/csv"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"os"
	"strconv"

	"github.com/eris-ltd/common/go/common"
	stypes "github.com/eris-ltd/eris-db/manager/eris-mint/state/types"
	ptypes "github.com/eris-ltd/eris-db/permission/types"

	"github.com/tendermint/go-crypto"
	wire "github.com/tendermint/go-wire"
)

//------------------------------------------------------------------------------------
// core functions

func GenerateKnown(chainID, accountsPathCSV, validatorsPathCSV string) error {
	var genDoc *stypes.GenesisDoc

	// TODO [eb] eliminate reading priv_val ... [zr] where?
	if accountsPathCSV == "" || validatorsPathCSV == "" {
		return fmt.Errorf("both accounts.csv and validators.csv is required")

	}

	pubkeys, amts, names, perms, setbits, err := parseCsv(validatorsPathCSV)
	if err != nil {
		return err
	}

	pubkeysA, amtsA, namesA, permsA, setbitsA, err := parseCsv(accountsPathCSV)
	if err != nil {
		return err
	}

	genDoc = newGenDoc(chainID, len(pubkeys), len(pubkeysA))
	for i, pk := range pubkeys {
		genDocAddValidator(genDoc, pk, amts[i], names[i], perms[i], setbits[i], i)
	}
	for i, pk := range pubkeysA {
		genDocAddAccount(genDoc, pk, amtsA[i], namesA[i], permsA[i], setbitsA[i], i)
	}

	buf, buf2, n := new(bytes.Buffer), new(bytes.Buffer), new(int)
	wire.WriteJSON(genDoc, buf, n, &err)
	if err != nil {
		return err
	}
	if err := json.Indent(buf2, buf.Bytes(), "", "\t"); err != nil {
		return err
	}
	genesisString := buf2.String()
	fmt.Println(genesisString)

	return nil
}

//-----------------------------------------------------------------------------
// gendoc convenience functions

func newGenDoc(chainID string, nVal, nAcc int) *stypes.GenesisDoc {
	genDoc := stypes.GenesisDoc{
		ChainID: chainID,
		// GenesisTime: time.Now(),
	}
	genDoc.Accounts = make([]stypes.GenesisAccount, nAcc)
	genDoc.Validators = make([]stypes.GenesisValidator, nVal)
	return &genDoc
}

func genDocAddAccount(genDoc *stypes.GenesisDoc, pubKey crypto.PubKeyEd25519, amt int64, name string, perm, setbit ptypes.PermFlag, index int) {
	addr := pubKey.Address()
	acc := stypes.GenesisAccount{
		Address: addr,
		Amount:  amt,
		Name:    name,
		Permissions: &ptypes.AccountPermissions{
			Base: ptypes.BasePermissions{
				Perms:  perm,
				SetBit: setbit,
			},
		},
	}
	if index < 0 {
		genDoc.Accounts = append(genDoc.Accounts, acc)
	} else {
		genDoc.Accounts[index] = acc
	}
}

func genDocAddValidator(genDoc *stypes.GenesisDoc, pubKey crypto.PubKeyEd25519, amt int64, name string, perm, setbit ptypes.PermFlag, index int) {
	addr := pubKey.Address()
	genDoc.Validators[index] = stypes.GenesisValidator{
		PubKey: pubKey,
		Amount: amt,
		Name:   name,
		UnbondTo: []stypes.BasicAccount{
			{
				Address: addr,
				Amount:  amt,
			},
		},
	}
	// [zr] why no index < 0 like in genDocAddAccount?
}

//-----------------------------------------------------------------------------
// util functions

// convert hex strings to ed25519 pubkeys
func pubKeyStringsToPubKeys(pubkeys []string) ([]crypto.PubKeyEd25519, error) {
	pubKeys := make([]crypto.PubKeyEd25519, len(pubkeys))
	for i, k := range pubkeys {
		pubBytes, err := hex.DecodeString(k)
		if err != nil {
			return pubKeys, err
		}
		copy(pubKeys[i][:], pubBytes)
	}
	return pubKeys, nil
}

// empty is over written
func ifExistsElse(list []string, index int, defaultValue string) string {
	if len(list) > index {
		if list[index] != "" {
			return list[index]
		}
	}
	return defaultValue
}

// takes a csv in the following format: pubkey, starting balance, name, permissions, setbit
func parseCsv(filePath string) (pubKeys []crypto.PubKeyEd25519, amts []int64, names []string, perms, setbits []ptypes.PermFlag, err error) {

	csvFile, err := os.Open(filePath)
	if err != nil {
		common.Exit(fmt.Errorf("Couldn't open file: %s: %v", filePath, err))
	}
	defer csvFile.Close()

	r := csv.NewReader(csvFile)
	//r.FieldsPerRecord = # of records expected
	params, err := r.ReadAll()
	if err != nil {
		common.Exit(fmt.Errorf("Couldn't read file: %v", err))

	}

	pubkeys := make([]string, len(params))
	amtS := make([]string, len(params))
	names = make([]string, len(params))
	permsS := make([]string, len(params))
	setbitS := make([]string, len(params))
	for i, each := range params {
		pubkeys[i] = each[0]
		amtS[i] = ifExistsElse(each, 1, "1000")
		names[i] = ifExistsElse(each, 2, "")
		permsS[i] = ifExistsElse(each, 3, fmt.Sprintf("%d", ptypes.DefaultPermFlags))
		setbitS[i] = ifExistsElse(each, 4, permsS[i])
	}

	//TODO convert int to uint64, see issue #25
	perms = make([]ptypes.PermFlag, len(permsS))
	for i, perm := range permsS {
		pflag, err := strconv.Atoi(perm)
		if err != nil {
			common.Exit(fmt.Errorf("Permissions must be an integer"))
		}
		perms[i] = ptypes.PermFlag(pflag)
	}
	setbits = make([]ptypes.PermFlag, len(setbitS))
	for i, setbit := range setbitS {
		setbitsFlag, err := strconv.Atoi(setbit)
		if err != nil {
			common.Exit(fmt.Errorf("SetBits must be an integer"))
		}
		setbits[i] = ptypes.PermFlag(setbitsFlag)
	}

	// convert amts to ints
	amts = make([]int64, len(amtS))
	for i, a := range amtS {
		if amts[i], err = strconv.ParseInt(a, 10, 64); err != nil {
			err = fmt.Errorf("Invalid amount: %v", err)
			return
		}
	}

	// convert pubkey hex strings to struct
	pubKeys, err = pubKeyStringsToPubKeys(pubkeys)
	if err != nil {
		return
	}

	return pubKeys, amts, names, perms, setbits, nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gob

import (
	"bufio"
	"bytes"
	"errors"
	"io"
	"reflect"
	"sync"
)

// A Decoder manages the receipt of type and data information read from the
// remote side of a connection.
type Decoder struct {
	mutex        sync.Mutex                              // each item must be received atomically
	r            io.Reader                               // source of the data
	buf          bytes.Buffer                            // buffer for more efficient i/o from r
	wireType     map[typeId]*wireType                    // map from remote ID to local description
	decoderCache map[reflect.Type]map[typeId]**decEngine // cache of compiled engines
	ignorerCache map[typeId]**decEngine                  // ditto for ignored objects
	freeList     *decoderState                           // list of free decoderStates; avoids reallocation
	countBuf     []byte                                  // used for decoding integers while parsing messages
	tmp          []byte                                  // temporary storage for i/o; saves reallocating
	err          error
}

// NewDecoder returns a new decoder that reads from the io.Reader.
// If r does not also implement io.ByteReader, it will be wrapped in a
// bufio.Reader.
func NewDecoder(r io.Reader) *Decoder {
	dec := new(Decoder)
	// We use the ability to read bytes as a plausible surrogate for buffering.
	if _, ok := r.(io.ByteReader); !ok {
		r = bufio.NewReader(r)
	}
	dec.r = r
	dec.wireType = make(map[typeId]*wireType)
	dec.decoderCache = make(map[reflect.Type]map[typeId]**decEngine)
	dec.ignorerCache = make(map[typeId]**decEngine)
	dec.countBuf = make([]byte, 9) // counts may be uint64s (unlikely!), require 9 bytes

	return dec
}

// recvType loads the definition of a type.
func (dec *Decoder) recvType(id typeId) {
	// Have we already seen this type?  That's an error
	if id < firstUserId || dec.wireType[id] != nil {
		dec.err = errors.New("gob: duplicate type received")
		return
	}

	// Type:
	wire := new(wireType)
	dec.decodeValue(tWireType, reflect.ValueOf(wire))
	if dec.err != nil {
		return
	}
	// Remember we've seen this type.
	dec.wireType[id] = wire
}

var errBadCount = errors.New("invalid message length")

// recvMessage reads the next count-delimited item from the input. It is the converse
// of Encoder.writeMessage. It returns false on EOF or other error reading the message.
func (dec *Decoder) recvMessage() bool {
	// Read a count.
	nbytes, _, err := decodeUintReader(dec.r, dec.countBuf)
	if err != nil {
		dec.err = err
		return false
	}
	// Upper limit of 1GB, allowing room to grow a little without overflow.
	// TODO: We might want more control over this limit.
	if nbytes >= 1<<30 {
		dec.err = errBadCount
		return false
	}
	dec.readMessage(int(nbytes))
	return dec.err == nil
}

// readMessage reads the next nbytes bytes from the input.
func (dec *Decoder) readMessage(nbytes int) {
	// Allocate the buffer.
	if cap(dec.tmp) < nbytes {
		dec.tmp = make([]byte, nbytes+100) // room to grow
	}
	dec.tmp = dec.tmp[:nbytes]

	// Read the data
	_, dec.err = io.ReadFull(dec.r, dec.tmp)
	if dec.err != nil {
		if dec.err == io.EOF {
			dec.err = io.ErrUnexpectedEOF
		}
		return
	}
	dec.buf.Write(dec.tmp)
}

// toInt turns an encoded uint64 into an int, according to the marshaling rules.
func toInt(x uint64) int64 {
	i := int64(x >> 1)
	if x&1 != 0 {
		i = ^i
	}
	return i
}

func (dec *Decoder) nextInt() int64 {
	n, _, err := decodeUintReader(&dec.buf, dec.countBuf)
	if err != nil {
		dec.err = err
	}
	return toInt(n)
}

func (dec *Decoder) nextUint() uint64 {
	n, _, err := decodeUintReader(&dec.buf, dec.countBuf)
	if err != nil {
		dec.err = err
	}
	return n
}

// decodeTypeSequence parses:
// TypeSequence
//	(TypeDefinition DelimitedTypeDefinition*)?
// and returns the type id of the next value.  It returns -1 at
// EOF.  Upon return, the remainder of dec.buf is the value to be
// decoded.  If this is an interface value, it can be ignored by
// resetting that buffer.
func (dec *Decoder) decodeTypeSequence(isInterface bool) typeId {
	for dec.err == nil {
		if dec.buf.Len() == 0 {
			if !dec.recvMessage() {
				break
			}
		}
		// Receive a type id.
		id := typeId(dec.nextInt())
		if id >= 0 {
			// Value follows.
			return id
		}
		// Type definition for (-id) follows.
		dec.recvType(-id)
		// When decoding an interface, after a type there may be a
		// DelimitedValue still in the buffer.  Skip its count.
		// (Alternatively, the buffer is empty and the byte count
		// will be absorbed by recvMessage.)
		if dec.buf.Len() > 0 {
			if !isInterface {
				dec.err = errors.New("extra data in buffer")
				break
			}
			dec.nextUint()
		}
	}
	return -1
}

// Decode reads the next value from the connection and stores
// it in the data represented by the empty interface value.
// If e is nil, the value will be discarded. Otherwise,
// the value underlying e must be a pointer to the
// correct type for the next data item received.
func (dec *Decoder) Decode(e interface{}) error {
	if e == nil {
		return dec.DecodeValue(reflect.Value{})
	}
	value := reflect.ValueOf(e)
	// If e represents a value as opposed to a pointer, the answer won't
	// get back to the caller.  Make sure it's a pointer.
	if value.Type().Kind() != reflect.Ptr {
		dec.err = errors.New("gob: attempt to decode into a non-pointer")
		return dec.err
	}
	return dec.DecodeValue(value)
}

// DecodeValue reads the next value from the connection.
// If v is the zero reflect.Value (v.Kind() == Invalid), DecodeValue discards the value.
// Otherwise, it stores the value into v.  In that case, v must represent
// a non-nil pointer to data or be an assignable reflect.Value (v.CanSet())
func (dec *Decoder) DecodeValue(v reflect.Value) error {
	if v.IsValid() {
		if v.Kind() == reflect.Ptr && !v.IsNil() {
			// That's okay, we'll store through the pointer.
		} else if !v.CanSet() {
			return errors.New("gob: DecodeValue of unassignable value")
		}
	}
	// Make sure we're single-threaded through here.
	dec.mutex.Lock()
	defer dec.mutex.Unlock()

	dec.buf.Reset() // In case data lingers from previous invocation.
	dec.err = nil
	id := dec.decodeTypeSequence(false)
	if dec.err == nil {
		dec.decodeValue(id, v)
	}
	return dec.err
}

// If debug.go is compiled into the program , debugFunc prints a human-readable
// representation of the gob data read from r by calling that file's Debug function.
// Otherwise it is nil.
var debugFunc func(io.Reader)
<commit_msg>encoding/gob: reduce decoder memory Gob decoding reads a whole message into memory and then copies it into a bytes.Buffer. For large messages this wastes an entire copy of the message. In this CL, we use a staging buffer to avoid the large temporary.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gob

import (
	"bufio"
	"bytes"
	"errors"
	"io"
	"reflect"
	"sync"
)

// A Decoder manages the receipt of type and data information read from the
// remote side of a connection.
type Decoder struct {
	mutex        sync.Mutex                              // each item must be received atomically
	r            io.Reader                               // source of the data
	buf          bytes.Buffer                            // buffer for more efficient i/o from r
	wireType     map[typeId]*wireType                    // map from remote ID to local description
	decoderCache map[reflect.Type]map[typeId]**decEngine // cache of compiled engines
	ignorerCache map[typeId]**decEngine                  // ditto for ignored objects
	freeList     *decoderState                           // list of free decoderStates; avoids reallocation
	countBuf     []byte                                  // used for decoding integers while parsing messages
	tmp          []byte                                  // temporary storage for i/o; saves reallocating
	err          error
}

// NewDecoder returns a new decoder that reads from the io.Reader.
// If r does not also implement io.ByteReader, it will be wrapped in a
// bufio.Reader.
func NewDecoder(r io.Reader) *Decoder {
	dec := new(Decoder)
	// We use the ability to read bytes as a plausible surrogate for buffering.
	if _, ok := r.(io.ByteReader); !ok {
		r = bufio.NewReader(r)
	}
	dec.r = r
	dec.wireType = make(map[typeId]*wireType)
	dec.decoderCache = make(map[reflect.Type]map[typeId]**decEngine)
	dec.ignorerCache = make(map[typeId]**decEngine)
	dec.countBuf = make([]byte, 9) // counts may be uint64s (unlikely!), require 9 bytes

	return dec
}

// recvType loads the definition of a type.
func (dec *Decoder) recvType(id typeId) {
	// Have we already seen this type?  That's an error
	if id < firstUserId || dec.wireType[id] != nil {
		dec.err = errors.New("gob: duplicate type received")
		return
	}

	// Type:
	wire := new(wireType)
	dec.decodeValue(tWireType, reflect.ValueOf(wire))
	if dec.err != nil {
		return
	}
	// Remember we've seen this type.
	dec.wireType[id] = wire
}

var errBadCount = errors.New("invalid message length")

// recvMessage reads the next count-delimited item from the input. It is the converse
// of Encoder.writeMessage. It returns false on EOF or other error reading the message.
func (dec *Decoder) recvMessage() bool {
	// Read a count.
	nbytes, _, err := decodeUintReader(dec.r, dec.countBuf)
	if err != nil {
		dec.err = err
		return false
	}
	// Upper limit of 1GB, allowing room to grow a little without overflow.
	// TODO: We might want more control over this limit.
	if nbytes >= 1<<30 {
		dec.err = errBadCount
		return false
	}
	dec.readMessage(int(nbytes))
	return dec.err == nil
}

// readMessage reads the next nbytes bytes from the input.
func (dec *Decoder) readMessage(nbytes int) {
	// Allocate the dec.tmp buffer, up to 10KB.
	const maxBuf = 10 * 1024
	nTmp := nbytes
	if nTmp > maxBuf {
		nTmp = maxBuf
	}
	if cap(dec.tmp) < nTmp {
		nAlloc := nTmp + 100 // A little extra for growth.
		if nAlloc > maxBuf {
			nAlloc = maxBuf
		}
		dec.tmp = make([]byte, nAlloc)
	}
	dec.tmp = dec.tmp[:nTmp]

	// Read the data
	dec.buf.Grow(nbytes)
	for nbytes > 0 {
		if nbytes < nTmp {
			dec.tmp = dec.tmp[:nbytes]
		}
		var nRead int
		nRead, dec.err = io.ReadFull(dec.r, dec.tmp)
		if dec.err != nil {
			if dec.err == io.EOF {
				dec.err = io.ErrUnexpectedEOF
			}
			return
		}
		dec.buf.Write(dec.tmp)
		nbytes -= nRead
	}
}

// toInt turns an encoded uint64 into an int, according to the marshaling rules.
func toInt(x uint64) int64 {
	i := int64(x >> 1)
	if x&1 != 0 {
		i = ^i
	}
	return i
}

func (dec *Decoder) nextInt() int64 {
	n, _, err := decodeUintReader(&dec.buf, dec.countBuf)
	if err != nil {
		dec.err = err
	}
	return toInt(n)
}

func (dec *Decoder) nextUint() uint64 {
	n, _, err := decodeUintReader(&dec.buf, dec.countBuf)
	if err != nil {
		dec.err = err
	}
	return n
}

// decodeTypeSequence parses:
// TypeSequence
//	(TypeDefinition DelimitedTypeDefinition*)?
// and returns the type id of the next value.  It returns -1 at
// EOF.  Upon return, the remainder of dec.buf is the value to be
// decoded.  If this is an interface value, it can be ignored by
// resetting that buffer.
func (dec *Decoder) decodeTypeSequence(isInterface bool) typeId {
	for dec.err == nil {
		if dec.buf.Len() == 0 {
			if !dec.recvMessage() {
				break
			}
		}
		// Receive a type id.
		id := typeId(dec.nextInt())
		if id >= 0 {
			// Value follows.
			return id
		}
		// Type definition for (-id) follows.
		dec.recvType(-id)
		// When decoding an interface, after a type there may be a
		// DelimitedValue still in the buffer.  Skip its count.
		// (Alternatively, the buffer is empty and the byte count
		// will be absorbed by recvMessage.)
		if dec.buf.Len() > 0 {
			if !isInterface {
				dec.err = errors.New("extra data in buffer")
				break
			}
			dec.nextUint()
		}
	}
	return -1
}

// Decode reads the next value from the connection and stores
// it in the data represented by the empty interface value.
// If e is nil, the value will be discarded. Otherwise,
// the value underlying e must be a pointer to the
// correct type for the next data item received.
func (dec *Decoder) Decode(e interface{}) error {
	if e == nil {
		return dec.DecodeValue(reflect.Value{})
	}
	value := reflect.ValueOf(e)
	// If e represents a value as opposed to a pointer, the answer won't
	// get back to the caller.  Make sure it's a pointer.
	if value.Type().Kind() != reflect.Ptr {
		dec.err = errors.New("gob: attempt to decode into a non-pointer")
		return dec.err
	}
	return dec.DecodeValue(value)
}

// DecodeValue reads the next value from the connection.
// If v is the zero reflect.Value (v.Kind() == Invalid), DecodeValue discards the value.
// Otherwise, it stores the value into v.  In that case, v must represent
// a non-nil pointer to data or be an assignable reflect.Value (v.CanSet())
func (dec *Decoder) DecodeValue(v reflect.Value) error {
	if v.IsValid() {
		if v.Kind() == reflect.Ptr && !v.IsNil() {
			// That's okay, we'll store through the pointer.
		} else if !v.CanSet() {
			return errors.New("gob: DecodeValue of unassignable value")
		}
	}
	// Make sure we're single-threaded through here.
	dec.mutex.Lock()
	defer dec.mutex.Unlock()

	dec.buf.Reset() // In case data lingers from previous invocation.
	dec.err = nil
	id := dec.decodeTypeSequence(false)
	if dec.err == nil {
		dec.decodeValue(id, v)
	}
	return dec.err
}

// If debug.go is compiled into the program , debugFunc prints a human-readable
// representation of the gob data read from r by calling that file's Debug function.
// Otherwise it is nil.
var debugFunc func(io.Reader)
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"exp/norm"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"
)

func main() {
	flag.Parse()
	loadTestData()
	CharacterByCharacterTests()
	StandardTests()
	PerformanceTest()
	if errorCount == 0 {
		fmt.Println("PASS")
	}
}

const file = "NormalizationTest.txt"

var url = flag.String("url",
	"http://www.unicode.org/Public/6.0.0/ucd/"+file,
	"URL of Unicode database directory")
var localFiles = flag.Bool("local",
	false,
	"data files have been copied to the current directory; for debugging only")

var logger = log.New(os.Stderr, "", log.Lshortfile)

// This regression test runs the test set in NormalizationTest.txt
// (taken from http://www.unicode.org/Public/6.0.0/ucd/).
//
// NormalizationTest.txt has form:
// @Part0 # Specific cases
// #
// 1E0A;1E0A;0044 0307;1E0A;0044 0307; # (Ḋ; Ḋ; D◌̇; Ḋ; D◌̇; ) LATIN CAPITAL LETTER D WITH DOT ABOVE
// 1E0C;1E0C;0044 0323;1E0C;0044 0323; # (Ḍ; Ḍ; D◌̣; Ḍ; D◌̣; ) LATIN CAPITAL LETTER D WITH DOT BELOW
//
// Each test has 5 columns (c1, c2, c3, c4, c5), where
// (c1, c2, c3, c4, c5) == (c1, NFC(c1), NFD(c1), NFKC(c1), NFKD(c1))
//
// CONFORMANCE:
// 1. The following invariants must be true for all conformant implementations
//
//    NFC
//      c2 ==  NFC(c1) ==  NFC(c2) ==  NFC(c3)
//      c4 ==  NFC(c4) ==  NFC(c5)
//
//    NFD
//      c3 ==  NFD(c1) ==  NFD(c2) ==  NFD(c3)
//      c5 ==  NFD(c4) ==  NFD(c5)
//
//    NFKC
//      c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)
//
//    NFKD
//      c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)
//
// 2. For every code point X assigned in this version of Unicode that is not
//    specifically listed in Part 1, the following invariants must be true
//    for all conformant implementations:
//
//      X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X)
//

// Column types.
const (
	cRaw = iota
	cNFC
	cNFD
	cNFKC
	cNFKD
	cMaxColumns
)

// Holds data from NormalizationTest.txt
var part []Part

type Part struct {
	name   string
	number int
	tests  []Test
}

type Test struct {
	name   string
	partnr int
	number int
	r      rune                // used for character by character test
	cols   [cMaxColumns]string // Each has 5 entries, see below.
}

func (t Test) Name() string {
	if t.number < 0 {
		return part[t.partnr].name
	}
	return fmt.Sprintf("%s:%d", part[t.partnr].name, t.number)
}

var partRe = regexp.MustCompile(`@Part(\d) # (.*)\n$`)
var testRe = regexp.MustCompile(`^` + strings.Repeat(`([\dA-F ]+);`, 5) + ` # (.*)\n?$`)

var counter int

// Load the data form NormalizationTest.txt
func loadTestData() {
	if *localFiles {
		pwd, _ := os.Getwd()
		*url = "file://" + path.Join(pwd, file)
	}
	t := &http.Transport{}
	t.RegisterProtocol("file", http.NewFileTransport(http.Dir("/")))
	c := &http.Client{Transport: t}
	resp, err := c.Get(*url)
	if err != nil {
		logger.Fatal(err)
	}
	if resp.StatusCode != 200 {
		logger.Fatal("bad GET status for "+file, resp.Status)
	}
	f := resp.Body
	defer f.Close()
	input := bufio.NewReader(f)
	for {
		line, err := input.ReadString('\n')
		if err != nil {
			if err == io.EOF {
				break
			}
			logger.Fatal(err)
		}
		if len(line) == 0 || line[0] == '#' {
			continue
		}
		m := partRe.FindStringSubmatch(line)
		if m != nil {
			if len(m) < 3 {
				logger.Fatal("Failed to parse Part: ", line)
			}
			i, err := strconv.Atoi(m[1])
			if err != nil {
				logger.Fatal(err)
			}
			name := m[2]
			part = append(part, Part{name: name[:len(name)-1], number: i})
			continue
		}
		m = testRe.FindStringSubmatch(line)
		if m == nil || len(m) < 7 {
			logger.Fatalf(`Failed to parse: "%s" result: %#v`, line, m)
		}
		test := Test{name: m[6], partnr: len(part) - 1, number: counter}
		counter++
		for j := 1; j < len(m)-1; j++ {
			for _, split := range strings.Split(m[j], " ") {
				r, err := strconv.ParseUint(split, 16, 64)
				if err != nil {
					logger.Fatal(err)
				}
				if test.r == 0 {
					// save for CharacterByCharacterTests
					test.r = int(r)
				}
				var buf [utf8.UTFMax]byte
				sz := utf8.EncodeRune(buf[:], rune(r))
				test.cols[j-1] += string(buf[:sz])
			}
		}
		part := &part[len(part)-1]
		part.tests = append(part.tests, test)
	}
}

var fstr = []string{"NFC", "NFD", "NFKC", "NFKD"}

var errorCount int

func cmpResult(t *Test, name string, f norm.Form, gold, test, result string) {
	if gold != result {
		errorCount++
		if errorCount > 20 {
			return
		}
		st, sr, sg := []rune(test), []rune(result), []rune(gold)
		logger.Printf("%s:%s: %s(%X)=%X; want:%X: %s",
			t.Name(), name, fstr[f], st, sr, sg, t.name)
	}
}

func cmpIsNormal(t *Test, name string, f norm.Form, test string, result, want bool) {
	if result != want {
		errorCount++
		if errorCount > 20 {
			return
		}
		logger.Printf("%s:%s: %s(%X)=%v; want: %v", t.Name(), name, fstr[f], []rune(test), result, want)
	}
}

func doTest(t *Test, f norm.Form, gold, test string) {
	result := f.Bytes([]byte(test))
	cmpResult(t, "Bytes", f, gold, test, string(result))
	for i := range test {
		out := f.Append(f.Bytes([]byte(test[:i])), []byte(test[i:])...)
		cmpResult(t, fmt.Sprintf(":Append:%d", i), f, gold, test, string(out))
	}
	cmpIsNormal(t, "IsNormal", f, test, f.IsNormal([]byte(test)), test == gold)
}

func doConformanceTests(t *Test, partn int) {
	for i := 0; i <= 2; i++ {
		doTest(t, norm.NFC, t.cols[1], t.cols[i])
		doTest(t, norm.NFD, t.cols[2], t.cols[i])
		doTest(t, norm.NFKC, t.cols[3], t.cols[i])
		doTest(t, norm.NFKD, t.cols[4], t.cols[i])
	}
	for i := 3; i <= 4; i++ {
		doTest(t, norm.NFC, t.cols[3], t.cols[i])
		doTest(t, norm.NFD, t.cols[4], t.cols[i])
		doTest(t, norm.NFKC, t.cols[3], t.cols[i])
		doTest(t, norm.NFKD, t.cols[4], t.cols[i])
	}
}

func CharacterByCharacterTests() {
	tests := part[1].tests
	last := 0
	for i := 0; i <= len(tests); i++ { // last one is special case
		var r int
		if i == len(tests) {
			r = 0x2FA1E // Don't have to go to 0x10FFFF
		} else {
			r = tests[i].r
		}
		for last++; last < r; last++ {
			// Check all characters that were not explicitly listed in the test.
			t := &Test{partnr: 1, number: -1}
			char := string(last)
			doTest(t, norm.NFC, char, char)
			doTest(t, norm.NFD, char, char)
			doTest(t, norm.NFKC, char, char)
			doTest(t, norm.NFKD, char, char)
		}
		if i < len(tests) {
			doConformanceTests(&tests[i], 1)
		}
	}
}

func StandardTests() {
	for _, j := range []int{0, 2, 3} {
		for _, test := range part[j].tests {
			doConformanceTests(&test, j)
		}
	}
}

// PerformanceTest verifies that normalization is O(n). If any of the
// code does not properly check for maxCombiningChars, normalization
// may exhibit O(n**2) behavior.
func PerformanceTest() {
	runtime.GOMAXPROCS(2)
	success := make(chan bool, 1)
	go func() {
		buf := bytes.Repeat([]byte("\u035D"), 1024*1024)
		buf = append(buf, "\u035B"...)
		norm.NFC.Append(nil, buf...)
		success <- true
	}()
	timeout := time.After(1e9)
	select {
	case <-success:
		// test completed before the timeout
	case <-timeout:
		errorCount++
		logger.Printf(`unexpectedly long time to complete PerformanceTest`)
	}
}
<commit_msg>exp/norm: fix rune/int types in test<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"exp/norm"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"
)

func main() {
	flag.Parse()
	loadTestData()
	CharacterByCharacterTests()
	StandardTests()
	PerformanceTest()
	if errorCount == 0 {
		fmt.Println("PASS")
	}
}

const file = "NormalizationTest.txt"

var url = flag.String("url",
	"http://www.unicode.org/Public/6.0.0/ucd/"+file,
	"URL of Unicode database directory")
var localFiles = flag.Bool("local",
	false,
	"data files have been copied to the current directory; for debugging only")

var logger = log.New(os.Stderr, "", log.Lshortfile)

// This regression test runs the test set in NormalizationTest.txt
// (taken from http://www.unicode.org/Public/6.0.0/ucd/).
//
// NormalizationTest.txt has form:
// @Part0 # Specific cases
// #
// 1E0A;1E0A;0044 0307;1E0A;0044 0307; # (Ḋ; Ḋ; D◌̇; Ḋ; D◌̇; ) LATIN CAPITAL LETTER D WITH DOT ABOVE
// 1E0C;1E0C;0044 0323;1E0C;0044 0323; # (Ḍ; Ḍ; D◌̣; Ḍ; D◌̣; ) LATIN CAPITAL LETTER D WITH DOT BELOW
//
// Each test has 5 columns (c1, c2, c3, c4, c5), where
// (c1, c2, c3, c4, c5) == (c1, NFC(c1), NFD(c1), NFKC(c1), NFKD(c1))
//
// CONFORMANCE:
// 1. The following invariants must be true for all conformant implementations
//
//    NFC
//      c2 ==  NFC(c1) ==  NFC(c2) ==  NFC(c3)
//      c4 ==  NFC(c4) ==  NFC(c5)
//
//    NFD
//      c3 ==  NFD(c1) ==  NFD(c2) ==  NFD(c3)
//      c5 ==  NFD(c4) ==  NFD(c5)
//
//    NFKC
//      c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)
//
//    NFKD
//      c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)
//
// 2. For every code point X assigned in this version of Unicode that is not
//    specifically listed in Part 1, the following invariants must be true
//    for all conformant implementations:
//
//      X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X)
//

// Column types.
const (
	cRaw = iota
	cNFC
	cNFD
	cNFKC
	cNFKD
	cMaxColumns
)

// Holds data from NormalizationTest.txt
var part []Part

type Part struct {
	name   string
	number int
	tests  []Test
}

type Test struct {
	name   string
	partnr int
	number int
	r      rune                // used for character by character test
	cols   [cMaxColumns]string // Each has 5 entries, see below.
}

func (t Test) Name() string {
	if t.number < 0 {
		return part[t.partnr].name
	}
	return fmt.Sprintf("%s:%d", part[t.partnr].name, t.number)
}

var partRe = regexp.MustCompile(`@Part(\d) # (.*)\n$`)
var testRe = regexp.MustCompile(`^` + strings.Repeat(`([\dA-F ]+);`, 5) + ` # (.*)\n?$`)

var counter int

// Load the data form NormalizationTest.txt
func loadTestData() {
	if *localFiles {
		pwd, _ := os.Getwd()
		*url = "file://" + path.Join(pwd, file)
	}
	t := &http.Transport{}
	t.RegisterProtocol("file", http.NewFileTransport(http.Dir("/")))
	c := &http.Client{Transport: t}
	resp, err := c.Get(*url)
	if err != nil {
		logger.Fatal(err)
	}
	if resp.StatusCode != 200 {
		logger.Fatal("bad GET status for "+file, resp.Status)
	}
	f := resp.Body
	defer f.Close()
	input := bufio.NewReader(f)
	for {
		line, err := input.ReadString('\n')
		if err != nil {
			if err == io.EOF {
				break
			}
			logger.Fatal(err)
		}
		if len(line) == 0 || line[0] == '#' {
			continue
		}
		m := partRe.FindStringSubmatch(line)
		if m != nil {
			if len(m) < 3 {
				logger.Fatal("Failed to parse Part: ", line)
			}
			i, err := strconv.Atoi(m[1])
			if err != nil {
				logger.Fatal(err)
			}
			name := m[2]
			part = append(part, Part{name: name[:len(name)-1], number: i})
			continue
		}
		m = testRe.FindStringSubmatch(line)
		if m == nil || len(m) < 7 {
			logger.Fatalf(`Failed to parse: "%s" result: %#v`, line, m)
		}
		test := Test{name: m[6], partnr: len(part) - 1, number: counter}
		counter++
		for j := 1; j < len(m)-1; j++ {
			for _, split := range strings.Split(m[j], " ") {
				r, err := strconv.ParseUint(split, 16, 64)
				if err != nil {
					logger.Fatal(err)
				}
				if test.r == 0 {
					// save for CharacterByCharacterTests
					test.r = rune(r)
				}
				var buf [utf8.UTFMax]byte
				sz := utf8.EncodeRune(buf[:], rune(r))
				test.cols[j-1] += string(buf[:sz])
			}
		}
		part := &part[len(part)-1]
		part.tests = append(part.tests, test)
	}
}

var fstr = []string{"NFC", "NFD", "NFKC", "NFKD"}

var errorCount int

func cmpResult(t *Test, name string, f norm.Form, gold, test, result string) {
	if gold != result {
		errorCount++
		if errorCount > 20 {
			return
		}
		st, sr, sg := []rune(test), []rune(result), []rune(gold)
		logger.Printf("%s:%s: %s(%X)=%X; want:%X: %s",
			t.Name(), name, fstr[f], st, sr, sg, t.name)
	}
}

func cmpIsNormal(t *Test, name string, f norm.Form, test string, result, want bool) {
	if result != want {
		errorCount++
		if errorCount > 20 {
			return
		}
		logger.Printf("%s:%s: %s(%X)=%v; want: %v", t.Name(), name, fstr[f], []rune(test), result, want)
	}
}

func doTest(t *Test, f norm.Form, gold, test string) {
	result := f.Bytes([]byte(test))
	cmpResult(t, "Bytes", f, gold, test, string(result))
	for i := range test {
		out := f.Append(f.Bytes([]byte(test[:i])), []byte(test[i:])...)
		cmpResult(t, fmt.Sprintf(":Append:%d", i), f, gold, test, string(out))
	}
	cmpIsNormal(t, "IsNormal", f, test, f.IsNormal([]byte(test)), test == gold)
}

func doConformanceTests(t *Test, partn int) {
	for i := 0; i <= 2; i++ {
		doTest(t, norm.NFC, t.cols[1], t.cols[i])
		doTest(t, norm.NFD, t.cols[2], t.cols[i])
		doTest(t, norm.NFKC, t.cols[3], t.cols[i])
		doTest(t, norm.NFKD, t.cols[4], t.cols[i])
	}
	for i := 3; i <= 4; i++ {
		doTest(t, norm.NFC, t.cols[3], t.cols[i])
		doTest(t, norm.NFD, t.cols[4], t.cols[i])
		doTest(t, norm.NFKC, t.cols[3], t.cols[i])
		doTest(t, norm.NFKD, t.cols[4], t.cols[i])
	}
}

func CharacterByCharacterTests() {
	tests := part[1].tests
	var last rune = 0
	for i := 0; i <= len(tests); i++ { // last one is special case
		var r rune
		if i == len(tests) {
			r = 0x2FA1E // Don't have to go to 0x10FFFF
		} else {
			r = tests[i].r
		}
		for last++; last < r; last++ {
			// Check all characters that were not explicitly listed in the test.
			t := &Test{partnr: 1, number: -1}
			char := string(last)
			doTest(t, norm.NFC, char, char)
			doTest(t, norm.NFD, char, char)
			doTest(t, norm.NFKC, char, char)
			doTest(t, norm.NFKD, char, char)
		}
		if i < len(tests) {
			doConformanceTests(&tests[i], 1)
		}
	}
}

func StandardTests() {
	for _, j := range []int{0, 2, 3} {
		for _, test := range part[j].tests {
			doConformanceTests(&test, j)
		}
	}
}

// PerformanceTest verifies that normalization is O(n). If any of the
// code does not properly check for maxCombiningChars, normalization
// may exhibit O(n**2) behavior.
func PerformanceTest() {
	runtime.GOMAXPROCS(2)
	success := make(chan bool, 1)
	go func() {
		buf := bytes.Repeat([]byte("\u035D"), 1024*1024)
		buf = append(buf, "\u035B"...)
		norm.NFC.Append(nil, buf...)
		success <- true
	}()
	timeout := time.After(1e9)
	select {
	case <-success:
		// test completed before the timeout
	case <-timeout:
		errorCount++
		logger.Printf(`unexpectedly long time to complete PerformanceTest`)
	}
}
<|endoftext|>


<commit_before>package f5

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
)

// Paths for file upload.
const (
	PathUploadImage = "/mgmt/cm/autodeploy/software-image-uploads"
	PathUploadFile  = "/mgmt/shared/file-transfer/uploads"

	// For backward compatibility
	// DEPRECATED
	UploadRESTPath = PathUploadFile
)

// An UploadResponse holds the responses send by the BigIP API while uploading
// files.
type UploadResponse struct {
	RemainingByteCount int64          `json:"remainingByteCount"`
	UsedChunks         map[string]int `json:"usedChunks"`
	TotalByteCount     int64          `json:"totalByteCount"`
	LocalFilePath      string         `json:"localFilePath"`
	TemporaryFilePath  string         `json:"temporaryFilePath"`
	Generation         int64          `json:"generation"`
	LastUpdateMicros   int64          `json:"lastUpdateMicros"`
}

// UploadFile reads the content of a file from r and uploads it to the BigIP.
// The uploaded file will be named according to the provided filename.
//
// filesize must be the exact file of the file.
//
// The file is split into small chunk, therefore this method may send multiple
// request.
//
// This method returns the latest upload response received.
func (c *Client) UploadFile(r io.Reader, filename string, filesize int64) (*UploadResponse, error) {
	var uploadResp UploadResponse
	for bytesSent := int64(0); bytesSent < filesize; {
		var chunk int64
		if remainingBytes := filesize - bytesSent; remainingBytes >= 512*1024 {
			chunk = 512 * 1024
		} else {
			chunk = remainingBytes
		}

		req, err := c.MakeUploadRequest(PathUploadFile+"/"+filename, io.LimitReader(r, chunk), bytesSent, chunk, filesize)
		if err != nil {
			return nil, err
		}
		resp, err := c.Do(req)
		if err != nil {
			return nil, err
		}
		if err := c.ReadError(resp); err != nil {
			resp.Body.Close()
			return nil, err
		}

		if filesize-bytesSent <= 512*1024 {
			dec := json.NewDecoder(resp.Body)
			if err := dec.Decode(&uploadResp); err != nil {
				resp.Body.Close()
				return nil, err
			}
		}

		bytesSent += chunk
	}
	return &uploadResp, nil
}

// MakeUploadRequest constructs a single upload request.
//
// restPath can be either PathUploadImage or PathUploadFile.
//
// The file to be uploaded is read from r and must not exceed 524288 bytes.
//
// off represents the number of bytes already sent while chunk is the size of
// chunk to be send in this request.
//
// filesize denotes the size of the entire file.
func (c *Client) MakeUploadRequest(restPath string, r io.Reader, off, chunk, filesize int64) (*http.Request, error) {
	if chunk > 512*1024 {
		return nil, fmt.Errorf("chunk size greater than %d is not supported", 512*1024)
	}
	req, err := http.NewRequest("POST", c.makeURL(restPath), r)
	if err != nil {
		return nil, fmt.Errorf("failed to create F5 authenticated request: %v", err)
	}
	req.Header.Add("Accept", "application/json")
	req.Header.Set("Content-Range", fmt.Sprintf("%d-%d/%d", off, off+chunk-1, filesize))
	req.Header.Set("Content-Type", "application/octet-stream")
	if err := c.makeAuth(req); err != nil {
		return nil, err
	}
	return req, nil
}
<commit_msg>f5: implement UCS file download<commit_after>package f5

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
)

// Paths for file upload.
const (
	PathUploadImage = "/mgmt/cm/autodeploy/software-image-uploads"
	PathUploadFile  = "/mgmt/shared/file-transfer/uploads"
	PathUploadUCS   = "mgmt/shared/file-transfer/ucs-uploads"

	// For backward compatibility
	// DEPRECATED
	UploadRESTPath = PathUploadFile
)

// Paths for file download.
const (
	PathDownloadUCS = "/mgmt/shared/file-transfer/ucs-downloads"
)

// DownloadUCS downloads an UCS file and writes its content to w.
func (c *Client) DownloadUCS(w io.Writer, filename string) (n int64, err error) {
	resp, err := c.SendRequest("GET", PathDownloadUCS+"/"+filename, nil)
	if err != nil {
		return 0, fmt.Errorf("error while requesting ucs file download: %v", err)
	}
	defer resp.Body.Close()

	if err := c.ReadError(resp); err != nil {
		return 0, fmt.Errorf("ucs download api returned an error: %v", err)
	}

	if n, err = io.Copy(w, resp.Body); err != nil {
		return 0, fmt.Errorf("cannot download ucs file: %v", err)
	}

	return
}

// An UploadResponse holds the responses send by the BigIP API while uploading
// files.
type UploadResponse struct {
	RemainingByteCount int64          `json:"remainingByteCount"`
	UsedChunks         map[string]int `json:"usedChunks"`
	TotalByteCount     int64          `json:"totalByteCount"`
	LocalFilePath      string         `json:"localFilePath"`
	TemporaryFilePath  string         `json:"temporaryFilePath"`
	Generation         int64          `json:"generation"`
	LastUpdateMicros   int64          `json:"lastUpdateMicros"`
}

// UploadFile reads the content of a file from r and uploads it to the BigIP.
// The uploaded file will be named according to the provided filename.
//
// filesize must be the exact file of the file.
//
// The file is split into small chunk, therefore this method may send multiple
// request.
//
// This method returns the latest upload response received.
func (c *Client) UploadFile(r io.Reader, filename string, filesize int64) (*UploadResponse, error) {
	return c.upload(r, PathUploadFile, filename, filesize)
}

// UploadImage reads the content of an disk image from r and uploads it to the
// BigIP.
//
// The uploaded image will be named according to the provided filename.
//
// filesize must be the exact file of the file.
//
// The file is split into small chunk, therefore this method may send multiple
// request.
//
// This method returns the latest upload response received.
func (c *Client) UploadImage(r io.Reader, filename string, filesize int64) (*UploadResponse, error) {
	return c.upload(r, PathUploadImage, filename, filesize)
}

// UploadUCS reads the content of an UCS archive from r and uploads it to the
// BigIP.
//
// The uploaded UCS archive will be named according to the provided filename.
//
// filesize must be the exact file of the file.
//
// The file is split into small chunk, therefore this method may send multiple
// request.
//
// This method returns the latest upload response received.
func (c *Client) UploadUCS(r io.Reader, filename string, filesize int64) (*UploadResponse, error) {
	return c.upload(r, PathUploadUCS, filename, filesize)
}

func (c *Client) upload(r io.Reader, restPath, filename string, filesize int64) (*UploadResponse, error) {
	var uploadResp UploadResponse
	for bytesSent := int64(0); bytesSent < filesize; {
		var chunk int64
		if remainingBytes := filesize - bytesSent; remainingBytes >= 512*1024 {
			chunk = 512 * 1024
		} else {
			chunk = remainingBytes
		}

		req, err := c.makeUploadRequest(restPath+"/"+filename, io.LimitReader(r, chunk), bytesSent, chunk, filesize)
		if err != nil {
			return nil, err
		}
		resp, err := c.Do(req)
		if err != nil {
			return nil, err
		}
		if err := c.ReadError(resp); err != nil {
			resp.Body.Close()
			return nil, err
		}

		if filesize-bytesSent <= 512*1024 {
			dec := json.NewDecoder(resp.Body)
			if err := dec.Decode(&uploadResp); err != nil {
				resp.Body.Close()
				return nil, err
			}
		}

		bytesSent += chunk
	}
	return &uploadResp, nil
}

// makeUploadRequest constructs a single upload request.
//
// restPath can be any of the Path* constants defined at the top of this file.
//
// The file to be uploaded is read from r and must not exceed 524288 bytes.
//
// off represents the number of bytes already sent while chunk is the size of
// chunk to be send in this request.
//
// filesize denotes the size of the entire file.
func (c *Client) makeUploadRequest(restPath string, r io.Reader, off, chunk, filesize int64) (*http.Request, error) {
	if chunk > 512*1024 {
		return nil, fmt.Errorf("chunk size greater than %d is not supported", 512*1024)
	}
	req, err := http.NewRequest("POST", c.makeURL(restPath), r)
	if err != nil {
		return nil, fmt.Errorf("failed to create F5 authenticated request: %v", err)
	}
	req.Header.Add("Accept", "application/json")
	req.Header.Set("Content-Range", fmt.Sprintf("%d-%d/%d", off, off+chunk-1, filesize))
	req.Header.Set("Content-Type", "application/octet-stream")
	if err := c.makeAuth(req); err != nil {
		return nil, err
	}
	return req, nil
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package netchan

import "testing"

const count = 10     // number of items in most tests
const closeCount = 5 // number of items when sender closes early

func exportSend(exp *Exporter, n int, t *testing.T) {
	ch := make(chan int)
	err := exp.Export("exportedSend", ch, Send)
	if err != nil {
		t.Fatal("exportSend:", err)
	}
	for i := 0; i < n; i++ {
		ch <- 23+i
	}
	close(ch)
}

func exportReceive(exp *Exporter, t *testing.T) {
	ch := make(chan int)
	err := exp.Export("exportedRecv", ch, Recv)
	if err != nil {
		t.Fatal("exportReceive:", err)
	}
	for i := 0; i < count; i++ {
		v := <-ch
		if v != 45+i {
			t.Errorf("export Receive: bad value: expected 4%d; got %d", 45+i, v)
		}
	}
}

func importReceive(imp *Importer, t *testing.T) {
	ch := make(chan int)
	err := imp.ImportNValues("exportedSend", ch, Recv, count)
	if err != nil {
		t.Fatal("importReceive:", err)
	}
	for i := 0; i < count; i++ {
		v := <-ch
		if closed(ch) {
			if i != closeCount {
				t.Errorf("expected close at %d; got one at %d\n", closeCount, i)
			}
			break
		}
		if v != 23+i {
			t.Errorf("importReceive: bad value: expected %d; got %+d", 23+i, v)
		}
	}
}

func importSend(imp *Importer, t *testing.T) {
	ch := make(chan int)
	err := imp.ImportNValues("exportedRecv", ch, Send, count)
	if err != nil {
		t.Fatal("importSend:", err)
	}
	for i := 0; i < count; i++ {
		ch <- 45+i
	}
}

func TestExportSendImportReceive(t *testing.T) {
	exp, err := NewExporter("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal("new exporter:", err)
	}
	imp, err := NewImporter("tcp", exp.Addr().String())
	if err != nil {
		t.Fatal("new importer:", err)
	}
	go exportSend(exp, count, t)
	importReceive(imp, t)
}

func TestExportReceiveImportSend(t *testing.T) {
	exp, err := NewExporter("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal("new exporter:", err)
	}
	imp, err := NewImporter("tcp", exp.Addr().String())
	if err != nil {
		t.Fatal("new importer:", err)
	}
	go importSend(imp, t)
	exportReceive(exp, t)
}

func TestClosingExportSendImportReceive(t *testing.T) {
	exp, err := NewExporter("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal("new exporter:", err)
	}
	imp, err := NewImporter("tcp", exp.Addr().String())
	if err != nil {
		t.Fatal("new importer:", err)
	}
	go exportSend(exp, closeCount, t)
	importReceive(imp, t)
}
<commit_msg>netchan: Fix race condition in test.<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package netchan

import "testing"

const count = 10     // number of items in most tests
const closeCount = 5 // number of items when sender closes early

func exportSend(exp *Exporter, n int, t *testing.T) {
	ch := make(chan int)
	err := exp.Export("exportedSend", ch, Send)
	if err != nil {
		t.Fatal("exportSend:", err)
	}
	go func() {
		for i := 0; i < n; i++ {
			ch <- 23+i
		}
		close(ch)
	}()
}

func exportReceive(exp *Exporter, t *testing.T) {
	ch := make(chan int)
	err := exp.Export("exportedRecv", ch, Recv)
	if err != nil {
		t.Fatal("exportReceive:", err)
	}
	for i := 0; i < count; i++ {
		v := <-ch
		if v != 45+i {
			t.Errorf("export Receive: bad value: expected 4%d; got %d", 45+i, v)
		}
	}
}

func importReceive(imp *Importer, t *testing.T) {
	ch := make(chan int)
	err := imp.ImportNValues("exportedSend", ch, Recv, count)
	if err != nil {
		t.Fatal("importReceive:", err)
	}
	for i := 0; i < count; i++ {
		v := <-ch
		if closed(ch) {
			if i != closeCount {
				t.Errorf("expected close at %d; got one at %d\n", closeCount, i)
			}
			break
		}
		if v != 23+i {
			t.Errorf("importReceive: bad value: expected %d; got %+d", 23+i, v)
		}
	}
}

func importSend(imp *Importer, t *testing.T) {
	ch := make(chan int)
	err := imp.ImportNValues("exportedRecv", ch, Send, count)
	if err != nil {
		t.Fatal("importSend:", err)
	}
	for i := 0; i < count; i++ {
		ch <- 45+i
	}
}

func TestExportSendImportReceive(t *testing.T) {
	exp, err := NewExporter("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal("new exporter:", err)
	}
	imp, err := NewImporter("tcp", exp.Addr().String())
	if err != nil {
		t.Fatal("new importer:", err)
	}
	exportSend(exp, count, t)
	importReceive(imp, t)
}

func TestExportReceiveImportSend(t *testing.T) {
	exp, err := NewExporter("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal("new exporter:", err)
	}
	imp, err := NewImporter("tcp", exp.Addr().String())
	if err != nil {
		t.Fatal("new importer:", err)
	}
	go importSend(imp, t)
	exportReceive(exp, t)
}

func TestClosingExportSendImportReceive(t *testing.T) {
	exp, err := NewExporter("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal("new exporter:", err)
	}
	imp, err := NewImporter("tcp", exp.Addr().String())
	if err != nil {
		t.Fatal("new importer:", err)
	}
	exportSend(exp, closeCount, t)
	importReceive(imp, t)
}
<|endoftext|>


<commit_before>package web

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"math/rand"
	"net/http"
	"os"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	google_auth2 "google.golang.org/api/oauth2/v2"

	"github.com/oinume/lekcije/server/model"
)

var googleOAuthConfig = &oauth2.Config{
	ClientID:     os.Getenv("GOOGLE_CLIENT_ID"),
	ClientSecret: os.Getenv("GOOGLE_CLIENT_SECRET"),
	Endpoint:     google.Endpoint,
	RedirectURL:  fmt.Sprintf("http://localhost:%d/oauth/google/callback", 4000),
	Scopes: []string{
		"openid email",
		"openid profile",
	},
}

func OAuthGoogle(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	state := randomString(32)
	cookie := &http.Cookie{
		Name:     "oauthState",
		Value:    state,
		Path:     "/",
		Expires:  time.Now().Add(time.Minute * 30),
		HttpOnly: true,
	}
	http.SetCookie(w, cookie)
	http.Redirect(w, r, googleOAuthConfig.AuthCodeURL(state), http.StatusFound)
}

func OAuthGoogleCallback(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	checkState(w, r)
	token, idToken, err := exchange(r)
	if err != nil {
		internalServerError(w, err.Error())
	}
	name, email, err := getNameAndEmail(token, idToken)
	if err != nil {
		internalServerError(w, err.Error())
	}
	db, err := model.Open()
	if err != nil {
		internalServerError(w, fmt.Sprintf("Failed to connect db: %v", err))
	}

	user := model.User{Name: name, Email: email}
	if err := db.FirstOrCreate(&user, model.User{Email: email}).Error; err != nil {
		internalServerError(w, fmt.Sprintf("Failed to access user: %v", err))
	}

	data := map[string]interface{}{
		"id":          user.Id,
		"name":        user.Name,
		"email":       user.Email,
		"accessToken": token.AccessToken,
		"idToken":     idToken,
	}
	if err := json.NewEncoder(w).Encode(data); err != nil {
		http.Error(w, fmt.Sprintf("Failed to encode JSON"), http.StatusInternalServerError)
		return
	}
}

func randomString(length int) string {
	b := make([]byte, length)
	rand.Read(b)
	return base64.StdEncoding.EncodeToString(b)
}

func checkState(w http.ResponseWriter, r *http.Request) {
	state := r.FormValue("state")
	oauthState, err := r.Cookie("oauthState")
	if err != nil {
		internalServerError(w, fmt.Sprintf("Failed to get cookie oauthState: %s", err))
		return
	}
	if state != oauthState.Value {
		internalServerError(w, "state mismatch")
		return
	}
}

func exchange(r *http.Request) (*oauth2.Token, string, error) {
	code := r.FormValue("code")
	token, err := googleOAuthConfig.Exchange(oauth2.NoContext, code)
	if err != nil {
		return nil, "", err
	}
	idToken, ok := token.Extra("id_token").(string)
	if !ok {
		return nil, "", fmt.Errorf("Failed to get id_token")
	}
	return token, idToken, nil
}

func getNameAndEmail(token *oauth2.Token, idToken string) (string, string, error) {
	oauth2Client := oauth2.NewClient(oauth2.NoContext, oauth2.StaticTokenSource(token))
	service, err := google_auth2.New(oauth2Client)
	if err != nil {
		return "", "", fmt.Errorf("Failed to create oauth2.Client: %v", err)
	}

	userinfo, err := service.Userinfo.V2.Me.Get().Do()
	if err != nil {
		return "", "", err
	}

	tokeninfo, err := service.Tokeninfo().IdToken(idToken).Do()
	if err != nil {
		return "", "", err
	}

	return userinfo.Name, tokeninfo.Email, nil
}
<commit_msg>Fix bug of checkState()<commit_after>package web

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"math/rand"
	"net/http"
	"os"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	google_auth2 "google.golang.org/api/oauth2/v2"

	"github.com/oinume/lekcije/server/model"
)

var googleOAuthConfig = &oauth2.Config{
	ClientID:     os.Getenv("GOOGLE_CLIENT_ID"),
	ClientSecret: os.Getenv("GOOGLE_CLIENT_SECRET"),
	Endpoint:     google.Endpoint,
	RedirectURL:  fmt.Sprintf("http://localhost:%d/oauth/google/callback", 4000),
	Scopes: []string{
		"openid email",
		"openid profile",
	},
}

func OAuthGoogle(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	state := randomString(32)
	cookie := &http.Cookie{
		Name:     "oauthState",
		Value:    state,
		Path:     "/",
		Expires:  time.Now().Add(time.Minute * 30),
		HttpOnly: true,
	}
	http.SetCookie(w, cookie)
	http.Redirect(w, r, googleOAuthConfig.AuthCodeURL(state), http.StatusFound)
}

func OAuthGoogleCallback(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	if err := checkState(r); err != nil {
		internalServerError(w, err.Error())
		return
	}
	token, idToken, err := exchange(r)
	if err != nil {
		internalServerError(w, err.Error())
		return
	}
	name, email, err := getNameAndEmail(token, idToken)
	if err != nil {
		internalServerError(w, err.Error())
		return
	}
	db, err := model.Open()
	if err != nil {
		internalServerError(w, fmt.Sprintf("Failed to connect db: %v", err))
		return
	}

	user := model.User{Name: name, Email: email}
	if err := db.FirstOrCreate(&user, model.User{Email: email}).Error; err != nil {
		internalServerError(w, fmt.Sprintf("Failed to access user: %v", err))
		return
	}

	data := map[string]interface{}{
		"id":          user.Id,
		"name":        user.Name,
		"email":       user.Email,
		"accessToken": token.AccessToken,
		"idToken":     idToken,
	}
	if err := json.NewEncoder(w).Encode(data); err != nil {
		internalServerError(w, fmt.Sprintf("Failed to encode JSON"))
		return
	}
}

func randomString(length int) string {
	b := make([]byte, length)
	rand.Read(b)
	return base64.StdEncoding.EncodeToString(b)
}

func checkState(r *http.Request) error {
	state := r.FormValue("state")
	oauthState, err := r.Cookie("oauthState")
	if err != nil {
		return fmt.Errorf("Failed to get cookie oauthState: %s", err)
	}
	if state != oauthState.Value {
		return fmt.Errorf("state mismatch", err)
	}
	return nil
}

func exchange(r *http.Request) (*oauth2.Token, string, error) {
	code := r.FormValue("code")
	token, err := googleOAuthConfig.Exchange(oauth2.NoContext, code)
	if err != nil {
		return nil, "", err
	}
	idToken, ok := token.Extra("id_token").(string)
	if !ok {
		return nil, "", fmt.Errorf("Failed to get id_token")
	}
	return token, idToken, nil
}

func getNameAndEmail(token *oauth2.Token, idToken string) (string, string, error) {
	oauth2Client := oauth2.NewClient(oauth2.NoContext, oauth2.StaticTokenSource(token))
	service, err := google_auth2.New(oauth2Client)
	if err != nil {
		return "", "", fmt.Errorf("Failed to create oauth2.Client: %v", err)
	}

	userinfo, err := service.Userinfo.V2.Me.Get().Do()
	if err != nil {
		return "", "", err
	}

	tokeninfo, err := service.Tokeninfo().IdToken(idToken).Do()
	if err != nil {
		return "", "", err
	}

	return userinfo.Name, tokeninfo.Email, nil
}
<|endoftext|>


<commit_before>package service

import (
	"encoding/json"
	"fmt"
	"time"
	"strings"

	"github.com/kyleterry/tenyks/irc"
	. "github.com/kyleterry/tenyks/version"
)

type servicefn func(*Connection, *Message)

func (self *ServiceEngine) AddHandler(name string, fn servicefn) {
	handler := irc.NewHandler(func(p ...interface{}) {
		fn(p[0].(*Connection), p[1].(*Message))
	})
	self.CommandRg.AddHandler(name, handler)
}

func (self *ServiceEngine) addBaseHandlers() {
	self.AddHandler("PRIVMSG", (*Connection).PrivmsgServiceHandler)
	self.AddHandler("REGISTER", (*Connection).RegisterServiceHandler)
	self.AddHandler("BYE", (*Connection).ByeServiceHandler)
	self.AddHandler("PONG", (*Connection).PongServiceHandler)
}

func (self *Connection) PrivmsgIrcHandler(conn *irc.Connection, msg *irc.Message) {
	serviceMsg := Message{}
	serviceMsg.Target = msg.Params[0]
	serviceMsg.Command = msg.Command
	serviceMsg.Mask = msg.Host
	serviceMsg.Direct = irc.IsDirect(msg.Trail, conn.GetCurrentNick())
	serviceMsg.Nick = msg.Nick
	serviceMsg.Host = msg.Host
	serviceMsg.Full_message = msg.RawMsg
	serviceMsg.User = msg.Ident
	serviceMsg.From_channel = irc.IsChannel(msg.Params[0])
	serviceMsg.Connection = conn.Name
	serviceMsg.Meta = &Meta{"Tenyks", TenyksVersion, nil, ""}
	if serviceMsg.Direct {
		serviceMsg.Payload = irc.StripNickOnDirect(msg.Trail, conn.GetCurrentNick())
	} else {
		serviceMsg.Payload = msg.Trail
	}

	jsonBytes, err := json.Marshal(serviceMsg)
	if err != nil {
		log.Fatal(err)
	}
	self.Out <- string(jsonBytes[:])
}

func (self *Connection) ListServicesIrcHandler(conn *irc.Connection, msg *irc.Message) {
	if irc.IsDirect(msg.Trail, conn.GetCurrentNick()) {
		if strings.Contains(msg.RawMsg, "!services") {
			log.Debug("[service] List services triggered")
			if len(self.engine.ServiceRg.services) > 0 {
				for _, service := range self.engine.ServiceRg.services {
					outMessage := fmt.Sprintf("%s", service)
					conn.Out <- msg.GetDMString(outMessage)
				}
			} else {
				conn.Out <- msg.GetDMString("No services registered")
			}
		}
	}
}

func (self *Connection) HelpIrcHandler(conn *irc.Connection, msg *irc.Message) {
	if irc.IsDirect(msg.Trail, conn.GetCurrentNick()) {
		trail := irc.StripNickOnDirect(msg.Trail, conn.GetCurrentNick())
		if strings.HasPrefix(trail, "!help") {
			trail_pieces := strings.Fields(trail)
			if len(trail_pieces) > 1 {
				if self.engine.ServiceRg.IsService(trail_pieces[1]) {
					service := self.engine.ServiceRg.GetServiceByName(trail_pieces[1])
					if service == nil {
						conn.Out <- msg.GetDMString(
							fmt.Sprintf("No such service `%s`", trail_pieces[1]))
						return
					}
					serviceMsg := &Message{
						Target: msg.Nick,
						Nick: msg.Nick,
						Direct: true,
						From_channel: false,
						Command: "PRIVMSG",
						Connection: conn.Name,
						Payload: fmt.Sprintf("!help %s", service.UUID.String()),
					}
					jsonBytes, err := json.Marshal(serviceMsg)
					if err != nil {
						log.Error("Cannot marshal help message")
						return
					}
					self.Out <- string(jsonBytes[:])
				} else {
					conn.Out <- msg.GetDMString(
						fmt.Sprintf("No such service `%s`", trail[1]))
				}
			} else {
				conn.Out <- msg.GetDMString(
					fmt.Sprintf("%s: !help - This help message", conn.GetCurrentNick()))
				conn.Out <- msg.GetDMString(
					fmt.Sprintf("%s: !services - List services", conn.GetCurrentNick()))
				conn.Out <- msg.GetDMString(
					fmt.Sprintf("%s: !help <servicename> - Get help for a service", conn.GetCurrentNick()))
			}
		}
	}
}

func (self *Connection) PrivmsgServiceHandler(msg *Message) {
	conn := self.getIrcConnByName(msg.Connection)
	if conn != nil {
		msgStr := fmt.Sprintf("%s %s :%s", msg.Command, msg.Target, msg.Payload)
		conn.Out <- msgStr
	} else {
		log.Debug("[service] No such connection `%s`. Ignoring.",
			msg.Connection)
	}
}

func (self *Connection) RegisterServiceHandler(msg *Message) {
	meta := msg.Meta
	if meta.SID == nil || meta.SID.UUID == nil {
		log.Error("[service] ERROR: UUID required to register with Tenyks")
		return
	}
	log.Debug("[service] %s (%s) wants to register", meta.SID.UUID.String(), meta.Name)
	srv := &Service{}
	srv.Name = meta.Name
	srv.Version = meta.Version
	srv.Description = meta.Description
	srv.Online = true
	srv.LastPing = time.Now()
	srv.UUID = meta.SID.UUID
	self.engine.ServiceRg.RegisterService(srv)
}

func (self *Connection) ByeServiceHandler(msg *Message) {
	meta := msg.Meta
	if meta.SID != nil && meta.SID.UUID != nil {
		log.Debug("[service] %s (%s) is hanging up", meta.SID.UUID.String(), meta.Name)
		srv := self.engine.ServiceRg.GetServiceByUUID(meta.SID.UUID.String())
		if srv != nil {
			log.Debug("[service] Settings state to `offline` for `%s`", srv.Name)
			srv.Online = false
		}
	}
}

const (
	ServiceOnline = true
	ServiceOffline = false
)

func (self *Connection) PingServices() {
	log.Debug("[service] Starting pinger")
	for {
		<-time.After(time.Second * 120)
		log.Debug("[service] PINGing services")
		msg := &Message{
			Command: "PING",
			Payload: "!tenyks",
		}
		jsonBytes, err := json.Marshal(msg)
		if err != nil {
			log.Error("Cannot marshal PING message")
			continue
		}
		self.Out <- string(jsonBytes[:])

		services := self.engine.ServiceRg.services
		for _, service := range services {
			if service.Online {
				service.LastPing = time.Now()
			}
		}
	}
}

func (self *Connection) PongServiceHandler(msg *Message) {
	meta := msg.Meta
	if meta.SID != nil && meta.SID.UUID != nil {
		self.engine.UpdateService(meta.SID.UUID.String(), ServiceOnline)
	}
}
<commit_msg>Quick and dirty private message support<commit_after>package service

import (
	"encoding/json"
	"fmt"
	"time"
	"strings"

	"github.com/kyleterry/tenyks/irc"
	. "github.com/kyleterry/tenyks/version"
)

type servicefn func(*Connection, *Message)

func (self *ServiceEngine) AddHandler(name string, fn servicefn) {
	handler := irc.NewHandler(func(p ...interface{}) {
		fn(p[0].(*Connection), p[1].(*Message))
	})
	self.CommandRg.AddHandler(name, handler)
}

func (self *ServiceEngine) addBaseHandlers() {
	self.AddHandler("PRIVMSG", (*Connection).PrivmsgServiceHandler)
	self.AddHandler("REGISTER", (*Connection).RegisterServiceHandler)
	self.AddHandler("BYE", (*Connection).ByeServiceHandler)
	self.AddHandler("PONG", (*Connection).PongServiceHandler)
}

func (self *Connection) PrivmsgIrcHandler(conn *irc.Connection, msg *irc.Message) {
	serviceMsg := Message{}
	if !irc.IsChannel(msg.Params[0]) {
		serviceMsg.Target = msg.Nick
	} else {
		serviceMsg.Target = msg.Params[0]
	}
	serviceMsg.Command = msg.Command
	serviceMsg.Mask = msg.Host
	if irc.IsDirect(msg.Trail, conn.GetCurrentNick()) || !irc.IsChannel(msg.Params[0]) {
		serviceMsg.Direct = true
	} else {
		serviceMsg.Direct = false
	}
	serviceMsg.Nick = msg.Nick
	serviceMsg.Host = msg.Host
	serviceMsg.Full_message = msg.RawMsg
	serviceMsg.User = msg.Ident
	serviceMsg.From_channel = irc.IsChannel(msg.Params[0])
	serviceMsg.Connection = conn.Name
	serviceMsg.Meta = &Meta{"Tenyks", TenyksVersion, nil, ""}
	if serviceMsg.Direct && serviceMsg.From_channel {
		serviceMsg.Payload = irc.StripNickOnDirect(msg.Trail, conn.GetCurrentNick())
	} else {
		serviceMsg.Payload = msg.Trail
	}

	jsonBytes, err := json.Marshal(serviceMsg)
	if err != nil {
		log.Fatal(err)
	}
	self.Out <- string(jsonBytes[:])
}

func (self *Connection) ListServicesIrcHandler(conn *irc.Connection, msg *irc.Message) {
	if irc.IsDirect(msg.Trail, conn.GetCurrentNick()) {
		if strings.Contains(msg.RawMsg, "!services") {
			log.Debug("[service] List services triggered")
			if len(self.engine.ServiceRg.services) > 0 {
				for _, service := range self.engine.ServiceRg.services {
					outMessage := fmt.Sprintf("%s", service)
					conn.Out <- msg.GetDMString(outMessage)
				}
			} else {
				conn.Out <- msg.GetDMString("No services registered")
			}
		}
	}
}

func (self *Connection) HelpIrcHandler(conn *irc.Connection, msg *irc.Message) {
	if irc.IsDirect(msg.Trail, conn.GetCurrentNick()) {
		trail := irc.StripNickOnDirect(msg.Trail, conn.GetCurrentNick())
		if strings.HasPrefix(trail, "!help") {
			trail_pieces := strings.Fields(trail)
			if len(trail_pieces) > 1 {
				if self.engine.ServiceRg.IsService(trail_pieces[1]) {
					service := self.engine.ServiceRg.GetServiceByName(trail_pieces[1])
					if service == nil {
						conn.Out <- msg.GetDMString(
							fmt.Sprintf("No such service `%s`", trail_pieces[1]))
						return
					}
					serviceMsg := &Message{
						Target: msg.Nick,
						Nick: msg.Nick,
						Direct: true,
						From_channel: false,
						Command: "PRIVMSG",
						Connection: conn.Name,
						Payload: fmt.Sprintf("!help %s", service.UUID.String()),
					}
					jsonBytes, err := json.Marshal(serviceMsg)
					if err != nil {
						log.Error("Cannot marshal help message")
						return
					}
					self.Out <- string(jsonBytes[:])
				} else {
					conn.Out <- msg.GetDMString(
						fmt.Sprintf("No such service `%s`", trail[1]))
				}
			} else {
				conn.Out <- msg.GetDMString(
					fmt.Sprintf("%s: !help - This help message", conn.GetCurrentNick()))
				conn.Out <- msg.GetDMString(
					fmt.Sprintf("%s: !services - List services", conn.GetCurrentNick()))
				conn.Out <- msg.GetDMString(
					fmt.Sprintf("%s: !help <servicename> - Get help for a service", conn.GetCurrentNick()))
			}
		}
	}
}

func (self *Connection) PrivmsgServiceHandler(msg *Message) {
	conn := self.getIrcConnByName(msg.Connection)
	if conn != nil {
		msgStr := fmt.Sprintf("%s %s :%s", msg.Command, msg.Target, msg.Payload)
		conn.Out <- msgStr
	} else {
		log.Debug("[service] No such connection `%s`. Ignoring.",
			msg.Connection)
	}
}

func (self *Connection) RegisterServiceHandler(msg *Message) {
	meta := msg.Meta
	if meta.SID == nil || meta.SID.UUID == nil {
		log.Error("[service] ERROR: UUID required to register with Tenyks")
		return
	}
	log.Debug("[service] %s (%s) wants to register", meta.SID.UUID.String(), meta.Name)
	srv := &Service{}
	srv.Name = meta.Name
	srv.Version = meta.Version
	srv.Description = meta.Description
	srv.Online = true
	srv.LastPing = time.Now()
	srv.UUID = meta.SID.UUID
	self.engine.ServiceRg.RegisterService(srv)
}

func (self *Connection) ByeServiceHandler(msg *Message) {
	meta := msg.Meta
	if meta.SID != nil && meta.SID.UUID != nil {
		log.Debug("[service] %s (%s) is hanging up", meta.SID.UUID.String(), meta.Name)
		srv := self.engine.ServiceRg.GetServiceByUUID(meta.SID.UUID.String())
		if srv != nil {
			log.Debug("[service] Settings state to `offline` for `%s`", srv.Name)
			srv.Online = false
		}
	}
}

const (
	ServiceOnline = true
	ServiceOffline = false
)

func (self *Connection) PingServices() {
	log.Debug("[service] Starting pinger")
	for {
		<-time.After(time.Second * 120)
		log.Debug("[service] PINGing services")
		msg := &Message{
			Command: "PING",
			Payload: "!tenyks",
		}
		jsonBytes, err := json.Marshal(msg)
		if err != nil {
			log.Error("Cannot marshal PING message")
			continue
		}
		self.Out <- string(jsonBytes[:])

		services := self.engine.ServiceRg.services
		for _, service := range services {
			if service.Online {
				service.LastPing = time.Now()
			}
		}
	}
}

func (self *Connection) PongServiceHandler(msg *Message) {
	meta := msg.Meta
	if meta.SID != nil && meta.SID.UUID != nil {
		self.engine.UpdateService(meta.SID.UUID.String(), ServiceOnline)
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 The SurgeMQ Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"fmt"
	"io"
	"net"
	"time"

	"github.com/surgemq/message"
	"runtime"
)

type netReader interface {
	io.Reader
	SetReadDeadline(t time.Time) error
}

type timeoutReader struct {
	d    time.Duration
	conn netReader
}

func (r timeoutReader) Read(b []byte) (int, error) {
	if err := r.conn.SetReadDeadline(time.Now().Add(r.d)); err != nil {
		return 0, err
	}
	return r.conn.Read(b)
}

// receiver() reads data from the network, and writes the data into the incoming buffer
func (this *service) receiver() {
	defer func() {
		// Let's recover from panic
		if r := recover(); r != nil {
			Log.Errorc(func() string {
				return fmt.Sprintf("(%s) Recovering from panic(receiver): %v", this.cid(), r)
			})
		}

		this.wgStopped.Done()

		Log.Debugc(func() string {
			return fmt.Sprintf("(%s) Stopping receiver", this.cid())
		})
	}()

	//   Log.Debugc(func() string{ return fmt.Sprintf("(%s) Starting receiver", this.cid())})

	this.wgStarted.Done()

	switch conn := this.conn.(type) {
	case net.Conn:
		//Log.Debugc(func() string{ return fmt.Sprintf("server/handleConnection: Setting read deadline to %d", time.Second*time.Duration(this.keepAlive))})
		keepAlive := time.Second * time.Duration(this.keepAlive)
		r := timeoutReader{
			d:    keepAlive + (keepAlive / 2),
			conn: conn,
		}

		for {
			_, err := this.in.ReadFrom(r)
			//       Log.Errorc(func() string{ return fmt.Sprintf("this.sess is: %v", this.sess)})
			//       Log.Errorc(func() string{ return fmt.Sprintf("this.sessMgr is: %v", this.sessMgr)})

			if err != nil {
				Log.Infoc(func() string {
					return fmt.Sprintf("(%s) error reading from connection: %v", this.cid(), err)
				})
				//         if err != io.EOF {
				//         }
				return
			}
		}

	//case *websocket.Conn:
	//	Log.Errorc(func() string{ return fmt.Sprintf("(%s) Websocket: %v", this.cid(), ErrInvalidConnectionType)})

	default:
		Log.Errorc(func() string {
			return fmt.Sprintf("(%s) %v", this.cid(), ErrInvalidConnectionType)
		})
	}
}

// sender() writes data from the outgoing buffer to the network
func (this *service) sender() {
	defer func() {
		// Let's recover from panic
		if r := recover(); r != nil {
			Log.Errorc(func() string {
				return fmt.Sprintf("(%s) Recovering from panic(sender): %v", this.cid(), r)
			})
		}

		this.wgStopped.Done()

		Log.Debugc(func() string {
			return fmt.Sprintf("(%s) Stopping sender", this.cid())
		})
	}()

	//Log.Debugc(func() string {
	//	return fmt.Sprintf("(%s) Starting sender", this.cid())
	//})

	this.wgStarted.Done()
	//Log.Debugc(func() string {
	//	return fmt.Sprintf("sender_1(%s)", this.cid())
	//})
	switch conn := this.conn.(type) {
	case net.Conn:
		for {
			//Log.Debugc(func() string {
			//	return fmt.Sprintf("sender_2(%s)", this.cid())
			//})
			_, err := this.out.WriteTo(conn)
			Log.Debugc(func() string {
				return fmt.Sprintf("sender_3(%s)", this.cid())
			})
			if err != nil {
				Log.Debugc(func() string {
					return fmt.Sprintf("sender_4(%s)", this.cid())
				})
				if err != io.EOF {
					Log.Errorc(func() string {
						return fmt.Sprintf("(%s) error writing data: %v", this.cid(), err)
					})
				}
				return
			}else {
				Log.Debugc(func() string {
					return fmt.Sprintf("sender_5(%s)", this.cid())
				})
				Log.Infoc(func() string {
					return fmt.Sprintf("向conn些数据成功！")
				})
			}
		}

	//case *websocket.Conn:
	//	Log.Errorc(func() string{ return fmt.Sprintf("(%s) Websocket not supported", this.cid())})

	default:
		Log.Errorc(func() string {
			return fmt.Sprintf("(%s) Invalid connection type", this.cid())
		})
	}
	Log.Debugc(func() string {
		return fmt.Sprintf("sender_6(%s)", this.cid())
	})
}

// readMessage() reads and copies a message from the buffer. The buffer bytes are
// committed as a result of the read.
func (this *service) readMessage(mtype message.MessageType, total int) (message.Message, error) {
	var (
		err error
		msg message.Message
	)

	if this.in == nil {
		err = ErrBufferNotReady
		return nil, err
	}

	var b []byte
	var index int64
	var ok bool

	for i := 0; i < 99; i++ {
		if this.isDone() {
			return nil, err
		}
		b, index, ok = this.in.ReadBuffer()
		if ok {
			break
		}
		runtime.Gosched()
	}
	defer this.in.ReadCommit(index)
	msg, err = mtype.New()
	if err != nil {
		Log.Errorc(func() string {
			return fmt.Sprintf("(%s)NewMessage  Error processing: %v", this.cid(), err)
		})
		return nil, err
	}

	_, err = msg.Decode(b)
	if err != nil {
		Log.Errorc(func() string {
			return fmt.Sprintf("(%s) Decode Error processing: %v", this.cid(), err)
		})
		return nil, err
	}

	return msg, err
}

// writeMessage() writes a message to the outgoing buffer
func (this *service) writeMessage(msg message.Message) (error) {
	if this.out == nil {
		return ErrBufferNotReady
	}

	// This is to serialize writes to the underlying buffer. Multiple goroutines could
	// potentially get here because of calling Publish() or Subscribe() or other
	// functions that will send messages. For example, if a message is received in
	// another connetion, and the message needs to be published to this client, then
	// the Publish() function is called, and at the same time, another client could
	// do exactly the same thing.
	//
	// Not an ideal fix though. If possible we should remove mutex and be lockfree.
	// Mainly because when there's a large number of goroutines that want to publish
	// to this client, then they will all block. However, this will do for now.
	//
	// FIXME: Try to find a better way than a mutex...if possible.
	b := make([]byte, msg.Len())
	_, err := msg.Encode(b)
	if err != nil {
		return err
	}

	for i := 0; i < 100; i++ {
		if this.isDone() {
			return io.EOF
		}
		if this.out.WriteBuffer(b) {
			break
		}
		runtime.Gosched()
	}

	this.outStat.increment(int64(1))

	return nil
}
<commit_msg>修改buffer读写限制<commit_after>// Copyright (c) 2014 The SurgeMQ Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"fmt"
	"io"
	"net"
	"time"

	"github.com/surgemq/message"
	"runtime"
)

type netReader interface {
	io.Reader
	SetReadDeadline(t time.Time) error
}

type timeoutReader struct {
	d    time.Duration
	conn netReader
}

func (r timeoutReader) Read(b []byte) (int, error) {
	if err := r.conn.SetReadDeadline(time.Now().Add(r.d)); err != nil {
		return 0, err
	}
	return r.conn.Read(b)
}

// receiver() reads data from the network, and writes the data into the incoming buffer
func (this *service) receiver() {
	defer func() {
		// Let's recover from panic
		if r := recover(); r != nil {
			Log.Errorc(func() string {
				return fmt.Sprintf("(%s) Recovering from panic(receiver): %v", this.cid(), r)
			})
		}

		this.wgStopped.Done()

		Log.Debugc(func() string {
			return fmt.Sprintf("(%s) Stopping receiver", this.cid())
		})
	}()

	//   Log.Debugc(func() string{ return fmt.Sprintf("(%s) Starting receiver", this.cid())})

	this.wgStarted.Done()

	switch conn := this.conn.(type) {
	case net.Conn:
		//Log.Debugc(func() string{ return fmt.Sprintf("server/handleConnection: Setting read deadline to %d", time.Second*time.Duration(this.keepAlive))})
		keepAlive := time.Second * time.Duration(this.keepAlive)
		r := timeoutReader{
			d:    keepAlive + (keepAlive / 2),
			conn: conn,
		}

		for {
			_, err := this.in.ReadFrom(r)
			//       Log.Errorc(func() string{ return fmt.Sprintf("this.sess is: %v", this.sess)})
			//       Log.Errorc(func() string{ return fmt.Sprintf("this.sessMgr is: %v", this.sessMgr)})

			if err != nil {
				Log.Infoc(func() string {
					return fmt.Sprintf("(%s) error reading from connection: %v", this.cid(), err)
				})
				//         if err != io.EOF {
				//         }
				return
			}
		}

	//case *websocket.Conn:
	//	Log.Errorc(func() string{ return fmt.Sprintf("(%s) Websocket: %v", this.cid(), ErrInvalidConnectionType)})

	default:
		Log.Errorc(func() string {
			return fmt.Sprintf("(%s) %v", this.cid(), ErrInvalidConnectionType)
		})
	}
}

// sender() writes data from the outgoing buffer to the network
func (this *service) sender() {
	defer func() {
		// Let's recover from panic
		if r := recover(); r != nil {
			Log.Errorc(func() string {
				return fmt.Sprintf("(%s) Recovering from panic(sender): %v", this.cid(), r)
			})
		}

		this.wgStopped.Done()

		Log.Debugc(func() string {
			return fmt.Sprintf("(%s) Stopping sender", this.cid())
		})
	}()

	//Log.Debugc(func() string {
	//	return fmt.Sprintf("(%s) Starting sender", this.cid())
	//})

	this.wgStarted.Done()
	//Log.Debugc(func() string {
	//	return fmt.Sprintf("sender_1(%s)", this.cid())
	//})
	switch conn := this.conn.(type) {
	case net.Conn:
		for {
			//Log.Debugc(func() string {
			//	return fmt.Sprintf("sender_2(%s)", this.cid())
			//})
			_, err := this.out.WriteTo(conn)
			Log.Debugc(func() string {
				return fmt.Sprintf("sender_3(%s)", this.cid())
			})
			if err != nil {
				Log.Debugc(func() string {
					return fmt.Sprintf("sender_4(%s)", this.cid())
				})
				if err != io.EOF {
					Log.Errorc(func() string {
						return fmt.Sprintf("(%s) error writing data: %v", this.cid(), err)
					})
				}
				return
			}else {
				Log.Debugc(func() string {
					return fmt.Sprintf("sender_5(%s)", this.cid())
				})
				Log.Infoc(func() string {
					return fmt.Sprintf("向conn些数据成功！")
				})
			}
		}

	//case *websocket.Conn:
	//	Log.Errorc(func() string{ return fmt.Sprintf("(%s) Websocket not supported", this.cid())})

	default:
		Log.Errorc(func() string {
			return fmt.Sprintf("(%s) Invalid connection type", this.cid())
		})
	}
	Log.Debugc(func() string {
		return fmt.Sprintf("sender_6(%s)", this.cid())
	})
}

// readMessage() reads and copies a message from the buffer. The buffer bytes are
// committed as a result of the read.
func (this *service) readMessage(mtype message.MessageType, total int) (message.Message, error) {
	var (
		err error
		msg message.Message
	)

	if this.in == nil {
		err = ErrBufferNotReady
		return nil, err
	}

	var b []byte
	var index int64
	var ok bool

	for i := 0; i < 99; i++ {
		if this.isDone() {
			return nil, err
		}
		b, index, ok = this.in.ReadBuffer()
		if ok {
			break
		}
		runtime.Gosched()
	}
	defer this.in.ReadCommit(index)
	msg, err = mtype.New()
	if err != nil {
		Log.Errorc(func() string {
			return fmt.Sprintf("(%s)NewMessage  Error processing: %v", this.cid(), err)
		})
		return nil, err
	}

	_, err = msg.Decode(b)
	if err != nil {
		Log.Errorc(func() string {
			return fmt.Sprintf("(%s) Decode Error processing: %v", this.cid(), err)
		})
		return nil, err
	}

	return msg, err
}

// writeMessage() writes a message to the outgoing buffer
func (this *service) writeMessage(msg message.Message) (error) {
	if this.out == nil {
		return ErrBufferNotReady
	}

	// This is to serialize writes to the underlying buffer. Multiple goroutines could
	// potentially get here because of calling Publish() or Subscribe() or other
	// functions that will send messages. For example, if a message is received in
	// another connetion, and the message needs to be published to this client, then
	// the Publish() function is called, and at the same time, another client could
	// do exactly the same thing.
	//
	// Not an ideal fix though. If possible we should remove mutex and be lockfree.
	// Mainly because when there's a large number of goroutines that want to publish
	// to this client, then they will all block. However, this will do for now.
	//
	// FIXME: Try to find a better way than a mutex...if possible.
	b := make([]byte, msg.Len())
	n, err := msg.Encode(b)
	if err != nil {
		return err
	}

	for i := 0; i < 100; i++ {
		if this.isDone() {
			return io.EOF
		}
		if this.out.WriteBuffer(b) {
			break
		}
		runtime.Gosched()
	}

	this.outStat.increment(int64(n))

	return nil
}
<|endoftext|>