<commit_before>package qln

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"reflect"
	"strings"

	"github.com/boltdb/bolt"
	"github.com/mit-dci/lit/btcutil"
	"github.com/mit-dci/lit/crypto/fastsha256"
	"github.com/mit-dci/lit/crypto/koblitz"
	"github.com/mit-dci/lit/lnutil"
	"github.com/mit-dci/lit/logging"
	"github.com/mit-dci/lit/sig64"
)

// RCRequestAuthArgs contains the required parameters
// to request authorization for remote control. Only
// the pub key is necessary
type RCRequestAuthArgs struct {
	// Public Key to authorize
	PubKey [33]byte
}

// RemoteControlRequestHandler handles an incoming remote control request
func (nd *LitNode) RemoteControlRequestHandler(msg lnutil.RemoteControlRpcRequestMsg, peer *RemotePeer) error {
	var pubKey [33]byte
	// transportAuthenticated will store whether the request authorization
	// should be checked against the transport's public key, or the one
	// that's included in the message.
	transportAuthenticated := true
	copy(pubKey[:], peer.Con.RemotePub().SerializeCompressed())

	if msg.PubKey != [33]byte{} {
		// The message contains a pubkey. So use that to authorize. We also
		// need to verify the signature inside the message then. If the
		// transport authorization is used, we can skip that because the
		// transport is already secured using signatures.
		pubKey = msg.PubKey
		transportAuthenticated = false
	}
	logging.Infof("Received remote control request [%s] from [%x]\n\n%s", msg.Method, pubKey, string(msg.Args))

	// Fetch the remote control authorization based on the used public key
	auth, err := nd.GetRemoteControlAuthorization(pubKey)
	if err != nil {
		logging.Errorf("Error while checking authorization for remote control: %s", err.Error())
		return err
	}

	// Whitelisted method(s) - Methods that don't require authorization.
	whitelisted := false
	if msg.Method == "LitRPC.RequestRemoteControlAuthorization" {
		// Request for authorization. You do not need to be authorized for this
		// All this method does is include the remote caller's public key into
		// the remote control authorization database as "requested authorization"
		// It will remain unauthorized until approved.
		whitelisted = true
		args := new(RCRequestAuthArgs)
		args.PubKey = pubKey
		msg.Args, err = json.Marshal(args)
		if err != nil {
			logging.Errorf("Error while updating RequestRemoteControlAuthorization arguments: %s", err.Error())
			return err
		}
	}

	// Method to check if the remote caller is authorized. This is not actually
	// part of the RPC surface, but only for the remote control. Hence it being
	// in a different namespace and returning a result from this method directly
	// without calling RPC methods.
	if msg.Method == "RemoteControl.CheckAuthorizationStatus" {
		resp, err := json.Marshal(map[string]interface{}{"Authorized": auth.Allowed})
		if err != nil {
			return err
		}
		outMsg := lnutil.NewRemoteControlRpcResponseMsg(msg.Peer(), msg.Idx, false, resp)
		nd.tmpSendLitMsg(outMsg)
		return nil
	}

	// If i'm not authorized, and it's not a whitelisted method then we fail the
	// request with an 'unauthorized' error
	if !auth.Allowed && !whitelisted {
		err = fmt.Errorf("Received remote control request from unauthorized peer: %x", pubKey)
		logging.Errorf(err.Error())

		outMsg := lnutil.NewRemoteControlRpcResponseMsg(msg.Peer(), msg.Idx, true, []byte("Unauthorized"))
		nd.tmpSendLitMsg(outMsg)

		return err
	}

	// If the transport is not authenticated, and we're using the key provided
	// in the message, then a signature is required to verify the caller actually
	// controls that given key.
	if !transportAuthenticated {
		msgSig := msg.Sig

		// Make the signature empty. The caller signed the message byte slice
		// with an empty signature.
		msg.Sig = [64]byte{}

		var digest []byte
		if msg.DigestType == lnutil.DIGEST_TYPE_SHA256 {
			hash := fastsha256.Sum256(msg.Bytes())
			digest = make([]byte, len(hash))
			copy(digest[:], hash[:])
		} else if msg.DigestType == lnutil.DIGEST_TYPE_RIPEMD160 {
			hash := btcutil.Hash160(msg.Bytes())
			digest = make([]byte, len(hash))
			copy(digest[:], hash[:])
		}

		pub, err := koblitz.ParsePubKey(msg.PubKey[:], koblitz.S256())
		if err != nil {
			logging.Errorf("Error parsing public key for remote control: %s", err.Error())
			return err
		}
		sig := sig64.SigDecompress(msgSig)
		signature, err := koblitz.ParseDERSignature(sig, koblitz.S256())
		if err != nil {
			logging.Errorf("Error parsing signature for remote control: %s", err.Error())
			return err
		}

		if !signature.Verify(digest, pub) {
			err = fmt.Errorf("Signature verification failed in remote control request")
			logging.Errorf(err.Error())
			return err
		}
	}

	// Check if the arguments property is valid JSON by deserializing it
	obj := map[string]interface{}{}
	err = json.Unmarshal(msg.Args, &obj)
	if err != nil {
		logging.Errorf("Could not parse JSON: %s", err.Error())
		return err
	}

	// Handle the request in a goroutine
	go func() {
		// Sanity check for the method
		if !strings.HasPrefix(msg.Method, "LitRPC.") {
			logging.Warnf("Remote control method does not start with `LitRPC.`. We don't know any better. Yet.")
			return
		}

		// Use reflection to call the method on the RPC object
		methodName := strings.TrimPrefix(msg.Method, "LitRPC.")
		rpcType := reflect.ValueOf(nd.RPC)
		if rpcType.IsValid() {
			method := rpcType.MethodByName(methodName)
			if method.IsValid() {

				// Our RPC calls always have params as (args, reply)
				argsType := method.Type().In(0)
				argsPointer := false

				// Unfortunately, sometimes the arguments to the function need
				// a pointer, and sometimes a value. So we check and adjust.
				if argsType.Kind() == reflect.Ptr {
					argsPointer = true
					argsType = argsType.Elem()
				}

				argsPayload := reflect.New(argsType)

				// Unmarshal the JSON into the args object
				err = json.Unmarshal(msg.Args, argsPayload.Interface())
				if err != nil {
					logging.Errorf("Error parsing json argument: %s", err.Error())
					return
				}

				replyType := method.Type().In(1).Elem()
				replyPayload := reflect.New(replyType)

				if !argsPointer {
					argsPayload = argsPayload.Elem()
				}
				result := method.Call([]reflect.Value{argsPayload, replyPayload})

				var reply []byte
				replyIsError := false
				if !result[0].IsNil() {
					replyIsError = true
					err = result[0].Interface().(error)
					reply = []byte(err.Error())
				} else {
					reply, err = json.Marshal(replyPayload.Interface())
					if err != nil {
						replyIsError = true
						reply = []byte(err.Error())
					}
				}

				outMsg := lnutil.NewRemoteControlRpcResponseMsg(msg.Peer(), msg.Idx, replyIsError, reply)
				nd.tmpSendLitMsg(outMsg)
			}
		}
	}()
	return nil
}

// RemoteControlResponseHandler handles the remote control response messages.
// At this time, this is not used in lit itself. Remote control messages are
// sent from the LndcRpcClient and responses are handled there. In normal operation
// two regular lit nodes do not talk to each other using remote control. But
// just in case someone sends us one, we print it out here.
func (nd *LitNode) RemoteControlResponseHandler(msg lnutil.RemoteControlRpcResponseMsg, peer *RemotePeer) error {
	logging.Debugf("Received remote control reply from peer %d:\n%s", msg.Peer(), string(msg.Result))
	return nil
}

// For now, this is simple: Allowed yes or no
// In the future allow more finegrained control
// over which RPCs are allowed and which are not,
// and perhaps authorize up to a certain amount for
// commands like send / push
type RemoteControlAuthorization struct {
	PubKey            [33]byte
	Allowed           bool
	UnansweredRequest bool
}

// Bytes serializes a remotecontrol authorization into a byteslice
func (r *RemoteControlAuthorization) Bytes() []byte {
	var buf bytes.Buffer

	binary.Write(&buf, binary.BigEndian, r.Allowed)
	binary.Write(&buf, binary.BigEndian, r.UnansweredRequest)
	return buf.Bytes()
}

// RemoteControlAuthorizationFromBytes parses a byteslice into a
// RemoteControlAuthorization object
func RemoteControlAuthorizationFromBytes(b []byte, pubKey [33]byte) *RemoteControlAuthorization {
	r := new(RemoteControlAuthorization)
	buf := bytes.NewBuffer(b)
	binary.Read(buf, binary.BigEndian, &r.Allowed)
	binary.Read(buf, binary.BigEndian, &r.UnansweredRequest)
	r.PubKey = pubKey
	return r
}

// SaveRemoteControlAuthorization saves the authorization for a specific
// pubkey into the database.
func (nd *LitNode) SaveRemoteControlAuthorization(pub [33]byte, auth *RemoteControlAuthorization) error {
	return nd.LitDB.Update(func(btx *bolt.Tx) error {
		cbk := btx.Bucket(BKTRCAuth)
		// serialize state
		b := auth.Bytes()
		return cbk.Put(pub[:], b)
	})
}

// GetRemoteControlAuthorization retrieves the remote controlauthorizzation for
// a specific pubkey from the database.
func (nd *LitNode) GetRemoteControlAuthorization(pub [33]byte) (*RemoteControlAuthorization, error) {
	r := new(RemoteControlAuthorization)

	// If the client uses our default remote control key (derived from our root priv)
	// then it has access to our private key (file) and is most likely running from our
	// localhost. So we always accept this.
	if bytes.Equal(pub[:], nd.DefaultRemoteControlKey.SerializeCompressed()) {
		r.Allowed = true
		return r, nil
	}

	// Fetch the authorization from the database and return it.
	err := nd.LitDB.View(func(btx *bolt.Tx) error {
		cbk := btx.Bucket(BKTRCAuth)
		// serialize state
		b := cbk.Get(pub[:])
		r = RemoteControlAuthorizationFromBytes(b, pub)
		return nil
	})
	return r, err
}

// GetPendingRemoteControlRequests retrieves all pending remote control
// authorization requests, so that a GUI can print them out for the user to
// authorize or not.
func (nd *LitNode) GetPendingRemoteControlRequests() ([]*RemoteControlAuthorization, error) {
	r := make([]*RemoteControlAuthorization, 0)
	err := nd.LitDB.View(func(btx *bolt.Tx) error {
		cbk := btx.Bucket(BKTRCAuth)
		// serialize state
		err := cbk.ForEach(func(k, v []byte) error {
			logging.Debugf("%x : %s\n", k, v)
			if len(v) >= 2 {
				if v[1] != 0x00 {
					var pubKey [33]byte
					copy(pubKey[:], k)
					r = append(r, RemoteControlAuthorizationFromBytes(v, pubKey))
				}
			}
			return nil
		})
		return err
	})
	return r, err
}
<commit_msg>Mismatch between peer.Idx and msg.Peer() after refactor (#382)<commit_after>package qln

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"reflect"
	"strings"

	"github.com/boltdb/bolt"
	"github.com/mit-dci/lit/btcutil"
	"github.com/mit-dci/lit/crypto/fastsha256"
	"github.com/mit-dci/lit/crypto/koblitz"
	"github.com/mit-dci/lit/lnutil"
	"github.com/mit-dci/lit/logging"
	"github.com/mit-dci/lit/sig64"
)

// RCRequestAuthArgs contains the required parameters
// to request authorization for remote control. Only
// the pub key is necessary
type RCRequestAuthArgs struct {
	// Public Key to authorize
	PubKey [33]byte
}

// RemoteControlRequestHandler handles an incoming remote control request
func (nd *LitNode) RemoteControlRequestHandler(msg lnutil.RemoteControlRpcRequestMsg, peer *RemotePeer) error {
	var pubKey [33]byte
	// transportAuthenticated will store whether the request authorization
	// should be checked against the transport's public key, or the one
	// that's included in the message.
	transportAuthenticated := true
	copy(pubKey[:], peer.Con.RemotePub().SerializeCompressed())

	if msg.PubKey != [33]byte{} {
		// The message contains a pubkey. So use that to authorize. We also
		// need to verify the signature inside the message then. If the
		// transport authorization is used, we can skip that because the
		// transport is already secured using signatures.
		pubKey = msg.PubKey
		transportAuthenticated = false
	}
	logging.Infof("Received remote control request [%s] from [%x]\n\n%s", msg.Method, pubKey, string(msg.Args))

	// Fetch the remote control authorization based on the used public key
	auth, err := nd.GetRemoteControlAuthorization(pubKey)
	if err != nil {
		logging.Errorf("Error while checking authorization for remote control: %s", err.Error())
		return err
	}

	// Whitelisted method(s) - Methods that don't require authorization.
	whitelisted := false
	if msg.Method == "LitRPC.RequestRemoteControlAuthorization" {
		// Request for authorization. You do not need to be authorized for this
		// All this method does is include the remote caller's public key into
		// the remote control authorization database as "requested authorization"
		// It will remain unauthorized until approved.
		whitelisted = true
		args := new(RCRequestAuthArgs)
		args.PubKey = pubKey
		msg.Args, err = json.Marshal(args)
		if err != nil {
			logging.Errorf("Error while updating RequestRemoteControlAuthorization arguments: %s", err.Error())
			return err
		}
	}

	// Method to check if the remote caller is authorized. This is not actually
	// part of the RPC surface, but only for the remote control. Hence it being
	// in a different namespace and returning a result from this method directly
	// without calling RPC methods.
	if msg.Method == "RemoteControl.CheckAuthorizationStatus" {
		resp, err := json.Marshal(map[string]interface{}{"Authorized": auth.Allowed})
		if err != nil {
			return err
		}
		outMsg := lnutil.NewRemoteControlRpcResponseMsg(peer.Idx, msg.Idx, false, resp)
		nd.tmpSendLitMsg(outMsg)
		return nil
	}

	// If i'm not authorized, and it's not a whitelisted method then we fail the
	// request with an 'unauthorized' error
	if !auth.Allowed && !whitelisted {
		err = fmt.Errorf("Received remote control request from unauthorized peer: %x", pubKey)
		logging.Errorf(err.Error())

		outMsg := lnutil.NewRemoteControlRpcResponseMsg(peer.Idx, msg.Idx, true, []byte("Unauthorized"))
		nd.tmpSendLitMsg(outMsg)

		return err
	}

	// If the transport is not authenticated, and we're using the key provided
	// in the message, then a signature is required to verify the caller actually
	// controls that given key.
	if !transportAuthenticated {
		msgSig := msg.Sig

		// Make the signature empty. The caller signed the message byte slice
		// with an empty signature.
		msg.Sig = [64]byte{}

		var digest []byte
		if msg.DigestType == lnutil.DIGEST_TYPE_SHA256 {
			hash := fastsha256.Sum256(msg.Bytes())
			digest = make([]byte, len(hash))
			copy(digest[:], hash[:])
		} else if msg.DigestType == lnutil.DIGEST_TYPE_RIPEMD160 {
			hash := btcutil.Hash160(msg.Bytes())
			digest = make([]byte, len(hash))
			copy(digest[:], hash[:])
		}

		pub, err := koblitz.ParsePubKey(msg.PubKey[:], koblitz.S256())
		if err != nil {
			logging.Errorf("Error parsing public key for remote control: %s", err.Error())
			return err
		}
		sig := sig64.SigDecompress(msgSig)
		signature, err := koblitz.ParseDERSignature(sig, koblitz.S256())
		if err != nil {
			logging.Errorf("Error parsing signature for remote control: %s", err.Error())
			return err
		}

		if !signature.Verify(digest, pub) {
			err = fmt.Errorf("Signature verification failed in remote control request")
			logging.Errorf(err.Error())
			return err
		}
	}

	// Check if the arguments property is valid JSON by deserializing it
	obj := map[string]interface{}{}
	err = json.Unmarshal(msg.Args, &obj)
	if err != nil {
		logging.Errorf("Could not parse JSON: %s", err.Error())
		return err
	}

	// Handle the request in a goroutine
	go func() {
		// Sanity check for the method
		if !strings.HasPrefix(msg.Method, "LitRPC.") {
			logging.Warnf("Remote control method does not start with `LitRPC.`. We don't know any better. Yet.")
			return
		}

		// Use reflection to call the method on the RPC object
		methodName := strings.TrimPrefix(msg.Method, "LitRPC.")
		rpcType := reflect.ValueOf(nd.RPC)
		if rpcType.IsValid() {
			method := rpcType.MethodByName(methodName)
			if method.IsValid() {

				// Our RPC calls always have params as (args, reply)
				argsType := method.Type().In(0)
				argsPointer := false

				// Unfortunately, sometimes the arguments to the function need
				// a pointer, and sometimes a value. So we check and adjust.
				if argsType.Kind() == reflect.Ptr {
					argsPointer = true
					argsType = argsType.Elem()
				}

				argsPayload := reflect.New(argsType)

				// Unmarshal the JSON into the args object
				err = json.Unmarshal(msg.Args, argsPayload.Interface())
				if err != nil {
					logging.Errorf("Error parsing json argument: %s", err.Error())
					return
				}

				replyType := method.Type().In(1).Elem()
				replyPayload := reflect.New(replyType)

				if !argsPointer {
					argsPayload = argsPayload.Elem()
				}
				result := method.Call([]reflect.Value{argsPayload, replyPayload})

				var reply []byte
				replyIsError := false
				if !result[0].IsNil() {
					replyIsError = true
					err = result[0].Interface().(error)
					reply = []byte(err.Error())
				} else {
					reply, err = json.Marshal(replyPayload.Interface())
					if err != nil {
						replyIsError = true
						reply = []byte(err.Error())
					}
				}

				outMsg := lnutil.NewRemoteControlRpcResponseMsg(peer.Idx, msg.Idx, replyIsError, reply)
				nd.tmpSendLitMsg(outMsg)
			}
		}
	}()
	return nil
}

// RemoteControlResponseHandler handles the remote control response messages.
// At this time, this is not used in lit itself. Remote control messages are
// sent from the LndcRpcClient and responses are handled there. In normal operation
// two regular lit nodes do not talk to each other using remote control. But
// just in case someone sends us one, we print it out here.
func (nd *LitNode) RemoteControlResponseHandler(msg lnutil.RemoteControlRpcResponseMsg, peer *RemotePeer) error {
	logging.Debugf("Received remote control reply from peer %d:\n%s", peer.Idx, string(msg.Result))
	return nil
}

// For now, this is simple: Allowed yes or no
// In the future allow more finegrained control
// over which RPCs are allowed and which are not,
// and perhaps authorize up to a certain amount for
// commands like send / push
type RemoteControlAuthorization struct {
	PubKey            [33]byte
	Allowed           bool
	UnansweredRequest bool
}

// Bytes serializes a remotecontrol authorization into a byteslice
func (r *RemoteControlAuthorization) Bytes() []byte {
	var buf bytes.Buffer

	binary.Write(&buf, binary.BigEndian, r.Allowed)
	binary.Write(&buf, binary.BigEndian, r.UnansweredRequest)
	return buf.Bytes()
}

// RemoteControlAuthorizationFromBytes parses a byteslice into a
// RemoteControlAuthorization object
func RemoteControlAuthorizationFromBytes(b []byte, pubKey [33]byte) *RemoteControlAuthorization {
	r := new(RemoteControlAuthorization)
	buf := bytes.NewBuffer(b)
	binary.Read(buf, binary.BigEndian, &r.Allowed)
	binary.Read(buf, binary.BigEndian, &r.UnansweredRequest)
	r.PubKey = pubKey
	return r
}

// SaveRemoteControlAuthorization saves the authorization for a specific
// pubkey into the database.
func (nd *LitNode) SaveRemoteControlAuthorization(pub [33]byte, auth *RemoteControlAuthorization) error {
	return nd.LitDB.Update(func(btx *bolt.Tx) error {
		cbk := btx.Bucket(BKTRCAuth)
		// serialize state
		b := auth.Bytes()
		return cbk.Put(pub[:], b)
	})
}

// GetRemoteControlAuthorization retrieves the remote controlauthorizzation for
// a specific pubkey from the database.
func (nd *LitNode) GetRemoteControlAuthorization(pub [33]byte) (*RemoteControlAuthorization, error) {
	r := new(RemoteControlAuthorization)

	// If the client uses our default remote control key (derived from our root priv)
	// then it has access to our private key (file) and is most likely running from our
	// localhost. So we always accept this.
	if bytes.Equal(pub[:], nd.DefaultRemoteControlKey.SerializeCompressed()) {
		r.Allowed = true
		return r, nil
	}

	// Fetch the authorization from the database and return it.
	err := nd.LitDB.View(func(btx *bolt.Tx) error {
		cbk := btx.Bucket(BKTRCAuth)
		// serialize state
		b := cbk.Get(pub[:])
		r = RemoteControlAuthorizationFromBytes(b, pub)
		return nil
	})
	return r, err
}

// GetPendingRemoteControlRequests retrieves all pending remote control
// authorization requests, so that a GUI can print them out for the user to
// authorize or not.
func (nd *LitNode) GetPendingRemoteControlRequests() ([]*RemoteControlAuthorization, error) {
	r := make([]*RemoteControlAuthorization, 0)
	err := nd.LitDB.View(func(btx *bolt.Tx) error {
		cbk := btx.Bucket(BKTRCAuth)
		// serialize state
		err := cbk.ForEach(func(k, v []byte) error {
			logging.Debugf("%x : %s\n", k, v)
			if len(v) >= 2 {
				if v[1] != 0x00 {
					var pubKey [33]byte
					copy(pubKey[:], k)
					r = append(r, RemoteControlAuthorizationFromBytes(v, pubKey))
				}
			}
			return nil
		})
		return err
	})
	return r, err
}
<|endoftext|>


<commit_before>package cwl

import (
	"fmt"
	"reflect"

	"github.com/davecgh/go-spew/spew"
	"github.com/mitchellh/mapstructure"
)

// CommandOutputParameter https://www.commonwl.org/v1.0/CommandLineTool.html#CommandOutputParameter
type CommandOutputParameter struct {
	OutputParameter `yaml:",inline" json:",inline" bson:",inline" mapstructure:",squash"` // provides Id, Label, SecondaryFiles, Format, Streamable, OutputBinding, Type

	Description string `yaml:"description,omitempty" bson:"description,omitempty" json:"description,omitempty" mapstructure:"description,omitempty"`
}

// NewCommandOutputParameter _
func NewCommandOutputParameter(original interface{}, thisID string, schemata []CWLType_Type, context *WorkflowContext) (output_parameter *CommandOutputParameter, err error) {

	original, err = MakeStringMap(original, context)
	if err != nil {
		return
	}

	switch original.(type) {

	case string:
		err = fmt.Errorf("(NewCommandOutputParameter) string not supported yet, please implement")

	case map[string]interface{}:

		var op *OutputParameter

		op, err = NewOutputParameterFromInterface(original, thisID, schemata, "CommandOutput", context)
		if err != nil {
			err = fmt.Errorf("(NewCommandOutputParameter) NewOutputParameterFromInterface returns %s", err.Error())
			return
		}

		if op.Id == "" {
			err = fmt.Errorf("(NewCommandOutputParameter) A id is empty !?")
			return
		}

		output_parameter = &CommandOutputParameter{}
		err = mapstructure.Decode(original, output_parameter)
		if err != nil {
			err = fmt.Errorf("(NewCommandOutputParameter) mapstructure returned: %s", err.Error())
			return
		}

		if output_parameter.Id == "" {
			output_parameter.Id = thisID
			if output_parameter.Id == "" {
				err = fmt.Errorf("(NewCommandOutputParameter) B id is empty !?")
				return
			}
		}

		output_parameter.OutputParameter = *op
	default:
		spew.Dump(original)
		err = fmt.Errorf("NewCommandOutputParameter, unknown type %s", reflect.TypeOf(original))
	}
	//spew.Dump(new_array)
	return
}

// NewCommandOutputParameterArray _
func NewCommandOutputParameterArray(original interface{}, schemata []CWLType_Type, context *WorkflowContext) (copa []interface{}, err error) {

	original, err = MakeStringMap(original, context)
	if err != nil {
		return
	}

	switch original.(type) {
	case map[string]interface{}:
		//fmt.Println("(NewCommandOutputParameterArray) map")
		originalMap := original.(map[string]interface{})

		copa = []interface{}{}
		for key, element := range originalMap {
			//fmt.Printf("(NewCommandOutputParameterArray) map element %s\n", key)
			//spew.Dump(element)
			var cop *CommandOutputParameter
			cop, err = NewCommandOutputParameter(element, key, schemata, context)
			if err != nil {
				err = fmt.Errorf("(NewCommandOutputParameterArray) c NewCommandOutputParameter returns: %s", err.Error())
				return
			}
			copa = append(copa, *cop)
		}

	case []interface{}:
		//fmt.Println("(NewCommandOutputParameterArray) array")
		copa = []interface{}{}

		originalArray := original.([]interface{})

		for _, element := range originalArray {
			//fmt.Println("(NewCommandOutputParameterArray) array element")
			var elementStr string
			var ok bool
			elementStr, ok = element.(string)

			if ok {
				//fmt.Println("(NewCommandOutputParameterArray) array element is a string")
				var result CWLType_Type
				result, err = NewCWLType_TypeFromString(schemata, elementStr, "CommandOutput")
				if err != nil {
					err = fmt.Errorf("(NewCommandOutputParameterArray) NewCWLType_TypeFromString returns: %s", err.Error())
					return
				}
				copa = append(copa, result)
				continue
			}
			//fmt.Println("(NewCommandOutputParameterArray) array element is NOT a string")
			var cop *CommandOutputParameter
			cop, err = NewCommandOutputParameter(element, "", schemata, context)
			if err != nil {
				err = fmt.Errorf("(NewCommandOutputParameterArray) b NewCommandOutputParameter returns: %s", err.Error())
				return
			}
			copa = append(copa, *cop)
		}

	default:
		err = fmt.Errorf("NewCommandOutputParameterArray, unknown type %s", reflect.TypeOf(original))
	}
	return

}
<commit_msg>fixed CommandOutputParameter<commit_after>package cwl

import (
	"fmt"
	"reflect"

	"github.com/davecgh/go-spew/spew"
	"github.com/mitchellh/mapstructure"
)

// CommandOutputParameter https://www.commonwl.org/v1.0/CommandLineTool.html#CommandOutputParameter
type CommandOutputParameter struct {
	OutputParameter `yaml:",inline" json:",inline" bson:",inline" mapstructure:",squash"` // provides Id, Label, SecondaryFiles, Format, Streamable, OutputBinding, Type

	Description string `yaml:"description,omitempty" bson:"description,omitempty" json:"description,omitempty" mapstructure:"description,omitempty"`
}

// NewCommandOutputParameter _
func NewCommandOutputParameter(original interface{}, thisID string, schemata []CWLType_Type, context *WorkflowContext) (cmdOutputParameter *CommandOutputParameter, err error) {

	cmdOutputParameter = &CommandOutputParameter{}

	original, err = MakeStringMap(original, context)
	if err != nil {
		return
	}

	var op *OutputParameter
	op, err = NewOutputParameterFromInterface(original, thisID, schemata, "CommandOutput", context)
	if err != nil {
		err = fmt.Errorf("(NewCommandOutputParameter) NewOutputParameterFromInterface returns %s", err.Error())
		return
	}

	switch original.(type) {

	case string:

		cmdOutputParameter.OutputParameter = *op

	case map[string]interface{}:

		err = mapstructure.Decode(original, cmdOutputParameter)
		if err != nil {
			err = fmt.Errorf("(NewCommandOutputParameter) mapstructure returned: %s", err.Error())
			return
		}

		cmdOutputParameter.OutputParameter = *op
	default:
		spew.Dump(original)
		err = fmt.Errorf("NewCommandOutputParameter, unknown type %s", reflect.TypeOf(original))
	}
	//spew.Dump(new_array)
	return
}

// NewCommandOutputParameterArray _
func NewCommandOutputParameterArray(original interface{}, schemata []CWLType_Type, context *WorkflowContext) (copa []interface{}, err error) {

	original, err = MakeStringMap(original, context)
	if err != nil {
		return
	}

	switch original.(type) {
	case map[string]interface{}:
		//fmt.Println("(NewCommandOutputParameterArray) map")
		originalMap := original.(map[string]interface{})

		copa = []interface{}{}
		for key, element := range originalMap {
			//fmt.Printf("(NewCommandOutputParameterArray) map element %s\n", key)
			//spew.Dump(element)
			var cop *CommandOutputParameter
			cop, err = NewCommandOutputParameter(element, key, schemata, context)
			if err != nil {
				err = fmt.Errorf("(NewCommandOutputParameterArray) c NewCommandOutputParameter returns: %s", err.Error())
				return
			}
			copa = append(copa, *cop)
		}

	case []interface{}:
		//fmt.Println("(NewCommandOutputParameterArray) array")
		copa = []interface{}{}

		originalArray := original.([]interface{})

		for _, element := range originalArray {
			//fmt.Println("(NewCommandOutputParameterArray) array element")
			var elementStr string
			var ok bool
			elementStr, ok = element.(string)

			if ok {
				//fmt.Println("(NewCommandOutputParameterArray) array element is a string")
				var result CWLType_Type
				result, err = NewCWLType_TypeFromString(schemata, elementStr, "CommandOutput")
				if err != nil {
					err = fmt.Errorf("(NewCommandOutputParameterArray) NewCWLType_TypeFromString returns: %s", err.Error())
					return
				}
				copa = append(copa, result)
				continue
			}
			//fmt.Println("(NewCommandOutputParameterArray) array element is NOT a string")
			var cop *CommandOutputParameter
			cop, err = NewCommandOutputParameter(element, "", schemata, context)
			if err != nil {
				err = fmt.Errorf("(NewCommandOutputParameterArray) b NewCommandOutputParameter returns: %s", err.Error())
				return
			}
			copa = append(copa, *cop)
		}

	default:
		err = fmt.Errorf("NewCommandOutputParameterArray, unknown type %s", reflect.TypeOf(original))
	}
	return

}
<|endoftext|>


<commit_before>package cmd

import (
	"errors"
	"fmt"
	"io/ioutil"

	"gopkg.in/yaml.v2"
)

type Config struct {
	Regurgitate Regurgitate `yaml:"regurgitate"`
	Ruminate    Ruminate    `yaml:"ruminate"`
	Gulp        Gulp        `yaml:"gulp"`
}

type Regurgitate struct {
	Host  string `yaml:"host"`
	Port  int    `yaml:"port"`
	Proto string `yaml:"proto"`
	Index string `yaml:"index"`
	Type  string `yaml:"type"`
	Query string `yaml:"query"`
}

type Gulp struct {
	Host   string `yaml:"host"`
	Port   int    `yaml:"port"`
	Db     string `yaml:"db"`
	Proto  string `yaml:"proto"`
	Series string `yaml:"series"`
	User   string `yaml:"user"`
	Pass   string `yaml:"Pass"`
}

type Ruminate struct {
	Iterator Iterator `yaml:"iterator"`
}

type Iterator struct {
	Selector  string            `yaml:"selector"`
	Time      string            `yaml:"time"`
	Tags      map[string]string `yaml:"tags"`
	Values    map[string]string `yaml:"values"`
	Iterators []Iterator        `yaml:"iterators"`
}

func Conf() (Config, error) {
	conf := Config{}
	file, err := ioutil.ReadFile(cfgFile)
	if err != nil {
		err = errors.New(fmt.Sprintf("Error while reading %s: %s", cfgFile, err.Error()))
		return conf, err
	}

	err = yaml.Unmarshal(file, &conf)
	if err != nil {
		err = errors.New(fmt.Sprintf("Error while parsing %s: %s", cfgFile, err.Error()))
		return conf, err
	}

	return conf, nil
}
<commit_msg>sampler implemented<commit_after>package cmd

import (
	"errors"
	"fmt"
	"io/ioutil"
	"time"

	"gopkg.in/yaml.v2"
)

type Config struct {
	Regurgitate Regurgitate `yaml:"regurgitate"`
	Ruminate    Ruminate    `yaml:"ruminate"`
	Gulp        Gulp        `yaml:"gulp"`
}

type Regurgitate struct {
	Host    string        `yaml:"host"`
	Port    int           `yaml:"port"`
	Proto   string        `yaml:"proto"`
	Index   string        `yaml:"index"`
	Type    string        `yaml:"type"`
	Query   string        `yaml:"query"`
	Sampler SamplerConfig `yaml:"sampler"`
}

type SamplerConfig struct {
	Offset       time.Duration `yaml:"offset"`
	Samples      int           `yaml:"samples"`
	SampleOffset time.Duration `yaml:"sample_offset"`
	Interval     string        `yaml:"interval"`
}

type Gulp struct {
	Host   string `yaml:"host"`
	Port   int    `yaml:"port"`
	Db     string `yaml:"db"`
	Proto  string `yaml:"proto"`
	Series string `yaml:"series"`
	User   string `yaml:"user"`
	Pass   string `yaml:"Pass"`
}

type Ruminate struct {
	Iterator Iterator `yaml:"iterator"`
}

type Iterator struct {
	Selector  string            `yaml:"selector"`
	Time      string            `yaml:"time"`
	Tags      map[string]string `yaml:"tags"`
	Values    map[string]string `yaml:"values"`
	Iterators []Iterator        `yaml:"iterators"`
}

func Conf() (Config, error) {
	conf := Config{}
	file, err := ioutil.ReadFile(cfgFile)
	if err != nil {
		err = errors.New(fmt.Sprintf("Error while reading %s: %s", cfgFile, err.Error()))
		return conf, err
	}

	err = yaml.Unmarshal(file, &conf)
	if err != nil {
		err = errors.New(fmt.Sprintf("Error while parsing %s: %s", cfgFile, err.Error()))
		return conf, err
	}

	return conf, nil
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/driusan/go-git/git"
)

type stagedFile struct {
	Filename git.IndexPath
	New      bool
	Removed  bool
}

func findUntrackedFilesFromDir(c *git.Client, root, parent, dir string, tracked map[git.IndexPath]bool) (untracked []git.File) {
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil
	}

	for _, fi := range files {
		if fi.IsDir() {
			if fi.Name() == ".git" {
				continue
			}
			recurseFiles := findUntrackedFilesFromDir(c, root, parent+"/"+fi.Name(), dir+"/"+fi.Name(), tracked)
			untracked = append(untracked, recurseFiles...)
		} else {
			indexPath := git.IndexPath(strings.TrimPrefix(parent+"/"+fi.Name(), root))
			if _, ok := tracked[indexPath]; !ok {
				rel, err := indexPath.FilePath(c)
				if err != nil {
					panic(err)
				}
				untracked = append(untracked, rel)
			}
		}
	}
	return
}
func findUntrackedFiles(c *git.Client, tracked map[git.IndexPath]bool) []git.File {
	if c.WorkDir == "" {
		return nil
	}
	wd := string(c.WorkDir)
	return findUntrackedFilesFromDir(c, wd+"/", wd, wd, tracked)
}

// The standard git "status" command doesn't provide any kind of --prefix, so
// this does the work of status, and adds a --prefix for commit to share the
// same code as Status. Status() just parses command line options and calls
// this.
func getStatus(c *git.Client, prefix string) (string, error) {

	idx, err := c.GitDir.ReadIndex()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
	}
	fileInIndex := make(map[git.IndexPath]bool)
	stagedFiles := make([]stagedFile, 0)
	unstagedFiles := make([]stagedFile, 0)

	headFiles := make(map[git.IndexPath]git.Sha1)
	// This isn't very efficiently implemented, but it works(ish).
	head, err := git.ExpandTreeIntoIndexesById(c, "HEAD", true, false)
	if err != nil {
		return "", err
	}
	for _, head := range head {
		headFiles[head.PathName] = head.Sha1
	}

	for _, file := range idx.Objects {
		fileInIndex[file.PathName] = true
		idxsha1 := file.Sha1

		relname, err := file.PathName.FilePath(c)
		if err != nil {
			panic(err)
		}
		fssha1, _, err := git.HashFile("blob", relname.String())
		if err != nil {
			if os.IsNotExist(err) {
				fssha1 = git.Sha1{}
			} else {
				fmt.Fprintf(os.Stderr, "%v\n", err)
				continue
			}
		}

		headSha1, headExists := headFiles[file.PathName]
		if !headExists {
			stagedFiles = append(stagedFiles,
				stagedFile{
					Filename: file.PathName,
					New:      true,
					Removed:  false,
				},
			)
			continue
		}
		if fssha1 != idxsha1 {
			_, err := os.Stat(relname.String())
			if os.IsNotExist(err) {
				unstagedFiles = append(unstagedFiles,
					stagedFile{file.PathName, false, true},
				)
			} else {
				unstagedFiles = append(unstagedFiles,
					stagedFile{file.PathName, false, false},
				)
			}

		}
		if headSha1 != idxsha1 {
			_, err := os.Stat(relname.String())
			if os.IsNotExist(err) {
				stagedFiles = append(stagedFiles,
					stagedFile{file.PathName, false, true},
				)
			} else {
				stagedFiles = append(stagedFiles,
					stagedFile{file.PathName, false, false},
				)
			}
		}
	}

	for file, _ := range headFiles {
		if _, ok := fileInIndex[file]; !ok {
			stagedFiles = append(stagedFiles,
				stagedFile{file, false, true},
			)
		}
	}

	var msg string

	untracked := findUntrackedFiles(c, fileInIndex)
	if len(stagedFiles) != 0 {
		msg += fmt.Sprintf("%sChanges to be committed:\n", prefix)
		msg += fmt.Sprintf("%s (use \"git reset HEAD <file>...\" to unstage)\n", prefix)
		msg += fmt.Sprintf("%s\n", prefix)
		for _, f := range stagedFiles {
			file, err := f.Filename.FilePath(c)
			if err != nil {
				panic(err)
			}
			if f.New {
				msg += fmt.Sprintf("%s\tnew file:\t%s\n", prefix, file)
			} else if f.Removed {
				msg += fmt.Sprintf("%s\tdeleted:\t%s\n", prefix, file)
			} else {
				msg += fmt.Sprintf("%s\tmodified:\t%s\n", prefix, file)
			}
		}
	}
	if len(unstagedFiles) != 0 {
		msg += fmt.Sprintf("%s\n", prefix)
		msg += fmt.Sprintf("%sChanges not staged for commit:\n", prefix)
		msg += fmt.Sprintf("%s\n", prefix)
		msg += fmt.Sprintf("%s  (use \"git add <file>...\" to update what will be committed)\n", prefix)
		msg += fmt.Sprintf("%s  (use \"git checkout -- <file>...\" to discard changes in working directory)\n", prefix)
		msg += fmt.Sprintf("%s\n", prefix)
		for _, f := range unstagedFiles {
			file, err := f.Filename.FilePath(c)
			if err != nil {
				panic(err)
			}

			if f.Removed {
				msg += fmt.Sprintf("\tdeleted:\t%s\n", file)
			} else {
				msg += fmt.Sprintf("\tmodified:\t%s\n", file)
			}
		}
	}

	if len(untracked) != 0 {
		msg += fmt.Sprintf("%s\n", prefix)
		msg += fmt.Sprintf("%sUntracked files:\n", prefix)
		msg += fmt.Sprintf("%s  (use \"git add <file>...\" to include in what will be committed)\n", prefix)
		msg += fmt.Sprintf("%s\n", prefix)
		for _, f := range untracked {
			msg += fmt.Sprintf("%s\t%s\n", prefix, f)
		}
	}

	if len(unstagedFiles) == 0 && len(stagedFiles) == 0 && len(untracked) == 0 {
		return "", fmt.Errorf("nothing to commit, working tree clean")
	}
	return msg, nil
}
func Status(c *git.Client, args []string) error {
	s, err := getStatus(c, "")
	if err != nil {
		return err
	}
	fmt.Printf("%s", s)
	return nil
}
<commit_msg>Fixed display of conflicting paths in git status<commit_after>package cmd

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/driusan/go-git/git"
)

type stagedFile struct {
	Filename git.IndexPath
	New      bool
	Removed  bool
}

type unmergedFile struct {
	Stage1, Stage2, Stage3 git.Sha1
}

func findUntrackedFilesFromDir(c *git.Client, root, parent, dir string, tracked map[git.IndexPath]bool) (untracked []git.File) {
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil
	}

	for _, fi := range files {
		if fi.IsDir() {
			if fi.Name() == ".git" {
				continue
			}
			recurseFiles := findUntrackedFilesFromDir(c, root, parent+"/"+fi.Name(), dir+"/"+fi.Name(), tracked)
			untracked = append(untracked, recurseFiles...)
		} else {
			indexPath := git.IndexPath(strings.TrimPrefix(parent+"/"+fi.Name(), root))
			if _, ok := tracked[indexPath]; !ok {
				rel, err := indexPath.FilePath(c)
				if err != nil {
					panic(err)
				}
				untracked = append(untracked, rel)
			}
		}
	}
	return
}
func findUntrackedFiles(c *git.Client, tracked map[git.IndexPath]bool) []git.File {
	if c.WorkDir == "" {
		return nil
	}
	wd := string(c.WorkDir)
	return findUntrackedFilesFromDir(c, wd+"/", wd, wd, tracked)
}

// The standard git "status" command doesn't provide any kind of --prefix, so
// this does the work of status, and adds a --prefix for commit to share the
// same code as Status. Status() just parses command line options and calls
// this.
func getStatus(c *git.Client, prefix string) (string, error) {

	idx, err := c.GitDir.ReadIndex()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
	}
	fileInIndex := make(map[git.IndexPath]bool)
	stagedFiles := make([]stagedFile, 0)
	unstagedFiles := make([]stagedFile, 0)
	unmergedFiles := make(map[git.IndexPath]*unmergedFile)

	headFiles := make(map[git.IndexPath]git.Sha1)
	// This isn't very efficiently implemented, but it works(ish).
	head, err := git.ExpandTreeIntoIndexesById(c, "HEAD", true, false)
	if err != nil {
		return "", err
	}
	for _, head := range head {
		headFiles[head.PathName] = head.Sha1
	}

	for _, file := range idx.Objects {
		fileInIndex[file.PathName] = true
		idxsha1 := file.Sha1

		switch file.Stage() {
		case git.Stage0:
			break
		case git.Stage1:
			if uf, ok := unmergedFiles[file.PathName]; ok {
				uf.Stage1 = file.Sha1
			} else {
				unmergedFiles[file.PathName] = &unmergedFile{
					Stage1: file.Sha1,
				}
			}
			continue
		case git.Stage2:
			if uf, ok := unmergedFiles[file.PathName]; ok {
				uf.Stage2 = file.Sha1
			} else {
				unmergedFiles[file.PathName] = &unmergedFile{
					Stage2: file.Sha1,
				}
			}
			continue
		case git.Stage3:
			if uf, ok := unmergedFiles[file.PathName]; ok {
				uf.Stage3 = file.Sha1
			} else {
				unmergedFiles[file.PathName] = &unmergedFile{
					Stage3: file.Sha1,
				}
			}
			continue
		default:
			panic("Invalid stage")
		}
		relname, err := file.PathName.FilePath(c)
		if err != nil {
			panic(err)
		}
		fssha1, _, err := git.HashFile("blob", relname.String())
		if err != nil {
			if os.IsNotExist(err) {
				fssha1 = git.Sha1{}
			} else {
				fmt.Fprintf(os.Stderr, "%v\n", err)
				continue
			}
		}

		headSha1, headExists := headFiles[file.PathName]
		if !headExists {
			stagedFiles = append(stagedFiles,
				stagedFile{
					Filename: file.PathName,
					New:      true,
					Removed:  false,
				},
			)
			continue
		}
		if fssha1 != idxsha1 {
			_, err := os.Stat(relname.String())
			if os.IsNotExist(err) {
				unstagedFiles = append(unstagedFiles,
					stagedFile{file.PathName, false, true},
				)
			} else {
				unstagedFiles = append(unstagedFiles,
					stagedFile{file.PathName, false, false},
				)
			}

		}
		if headSha1 != idxsha1 {
			_, err := os.Stat(relname.String())
			if os.IsNotExist(err) {
				stagedFiles = append(stagedFiles,
					stagedFile{file.PathName, false, true},
				)
			} else {
				stagedFiles = append(stagedFiles,
					stagedFile{file.PathName, false, false},
				)
			}
		}
	}

	for file, _ := range headFiles {
		if _, ok := fileInIndex[file]; !ok {
			stagedFiles = append(stagedFiles,
				stagedFile{file, false, true},
			)
		}
	}

	var msg string

	untracked := findUntrackedFiles(c, fileInIndex)
	if len(stagedFiles) != 0 {
		msg += fmt.Sprintf("%sChanges to be committed:\n", prefix)
		msg += fmt.Sprintf("%s (use \"git reset HEAD <file>...\" to unstage)\n", prefix)
		msg += fmt.Sprintf("%s\n", prefix)
		for _, f := range stagedFiles {
			file, err := f.Filename.FilePath(c)
			if err != nil {
				panic(err)
			}
			if f.New {
				msg += fmt.Sprintf("%s\tnew file:\t%s\n", prefix, file)
			} else if f.Removed {
				msg += fmt.Sprintf("%s\tdeleted:\t%s\n", prefix, file)
			} else {
				msg += fmt.Sprintf("%s\tmodified:\t%s\n", prefix, file)
			}
		}
	}

	if len(unmergedFiles) != 0 {
		msg += fmt.Sprintf("%s\n", prefix)
		msg += fmt.Sprintf("%sUnmerged paths:\n", prefix)
		msg += fmt.Sprintf("%s (use \"git reset HEAD <file>...\" to unstage)\n", prefix)
		msg += fmt.Sprintf("%s (use \"git add/rm <file>...\" as appropriate to mark resolution)\n", prefix)
		msg += fmt.Sprintf("%s\n", prefix)

		for path, status := range unmergedFiles {
			file, err := path.FilePath(c)
			if err != nil {
				panic(err)
			}

			var empty git.Sha1
			if status.Stage2 == empty && status.Stage3 != empty {
				msg += fmt.Sprintf("%s\tdeleted by us:\t%s\n", prefix, file)
			} else if status.Stage2 != empty && status.Stage3 == empty {
				msg += fmt.Sprintf("%s\tdeleted by them:\t%s\n", prefix, file)
			} else if status.Stage2 == empty && status.Stage3 == empty {
				msg += fmt.Sprintf("%s\tdeleted by both:\t%s\n", prefix, file)
			} else if status.Stage2 != status.Stage3 {
				msg += fmt.Sprintf("%s\tmodified by both:\t%s\n", prefix, file)
			}
		}
	}

	if len(unstagedFiles) != 0 {
		msg += fmt.Sprintf("%s\n", prefix)
		msg += fmt.Sprintf("%sChanges not staged for commit:\n", prefix)
		msg += fmt.Sprintf("%s\n", prefix)
		msg += fmt.Sprintf("%s  (use \"git add <file>...\" to update what will be committed)\n", prefix)
		msg += fmt.Sprintf("%s  (use \"git checkout -- <file>...\" to discard changes in working directory)\n", prefix)
		msg += fmt.Sprintf("%s\n", prefix)
		for _, f := range unstagedFiles {
			file, err := f.Filename.FilePath(c)
			if err != nil {
				panic(err)
			}

			if f.Removed {
				msg += fmt.Sprintf("%s\tdeleted:\t%s\n", prefix, file)
			} else {
				msg += fmt.Sprintf("%s\tmodified:\t%s\n", prefix, file)
			}
		}
	}

	if len(untracked) != 0 {
		msg += fmt.Sprintf("%s\n", prefix)
		msg += fmt.Sprintf("%sUntracked files:\n", prefix)
		msg += fmt.Sprintf("%s  (use \"git add <file>...\" to include in what will be committed)\n", prefix)
		msg += fmt.Sprintf("%s\n", prefix)
		for _, f := range untracked {
			msg += fmt.Sprintf("%s\t%s\n", prefix, f)
		}
	}

	if len(unstagedFiles) == 0 && len(stagedFiles) == 0 && len(untracked) == 0 {
		return "", fmt.Errorf("nothing to commit, working tree clean")
	}
	return msg, nil
}
func Status(c *git.Client, args []string) error {
	s, err := getStatus(c, "")
	if err != nil {
		return err
	}
	fmt.Printf("%s", s)
	return nil
}
<|endoftext|>


<commit_before>// +build linux

package keyring

import (
	"encoding/json"
	"errors"

	"github.com/godbus/dbus"
	"github.com/gsterjov/go-libsecret"
)

func init() {
	// silently fail if dbus isn't available
	_, err := dbus.SessionBus()
	if err != nil {
		return
	}

	supportedBackends[SecretServiceBackend] = opener(func(cfg Config) (Keyring, error) {
		if cfg.ServiceName == "" {
			cfg.ServiceName = "secret-service"
		}
		if cfg.LibSecretCollectionName == "" {
			cfg.LibSecretCollectionName = cfg.ServiceName
		}

		service, err := libsecret.NewService()
		if err != nil {
			return &secretsKeyring{}, err
		}

		ring := &secretsKeyring{
			name:    cfg.LibSecretCollectionName,
			service: service,
		}

		return ring, ring.openSecrets()
	})
}

type secretsKeyring struct {
	name       string
	service    *libsecret.Service
	collection *libsecret.Collection
	session    *libsecret.Session
}

type secretsError struct {
	message string
}

func (e *secretsError) Error() string {
	return e.message
}

var errCollectionNotFound = errors.New("The collection does not exist. Please add a key first")

func (k *secretsKeyring) openSecrets() error {
	session, err := k.service.Open()
	if err != nil {
		return err
	}
	k.session = session

	// get the collection if it already exists
	collections, err := k.service.Collections()
	if err != nil {
		return err
	}

	path := libsecret.DBusPath + "/collection/" + k.name

	for _, collection := range collections {
		if string(collection.Path()) == path {
			k.collection = &collection
			return nil
		}
	}

	return nil
}

func (k *secretsKeyring) openCollection() error {
	if err := k.openSecrets(); err != nil {
		return err
	}

	if k.collection == nil {
		return errCollectionNotFound
		// return &secretsError{fmt.Sprintf(
		// 	"The collection %q does not exist. Please add a key first",
		// 	k.name,
		// )}
	}

	return nil
}

func (k *secretsKeyring) Get(key string) (Item, error) {
	if err := k.openCollection(); err != nil {
		if err == errCollectionNotFound {
			return Item{}, ErrKeyNotFound
		}
		return Item{}, err
	}

	items, err := k.collection.SearchItems(key)
	if err != nil {
		return Item{}, err
	}

	if len(items) == 0 {
		return Item{}, ErrKeyNotFound
	}

	// use the first item whenever there are multiples
	// with the same profile name
	item := items[0]

	locked, err := item.Locked()
	if err != nil {
		return Item{}, err
	}

	if locked {
		if err := k.service.Unlock(item); err != nil {
			return Item{}, err
		}
	}

	secret, err := item.GetSecret(k.session)
	if err != nil {
		return Item{}, err
	}

	// pack the secret into the item
	var ret Item
	if err = json.Unmarshal(secret.Value, &ret); err != nil {
		return Item{}, err
	}

	return ret, err
}

// GetMetadata for libsecret returns an error indicating that it's unsupported
// for this backend.
//
// libsecret actually implements a metadata system which we could use, "Secret
// Attributes"; I found no indication in documentation of anything like an
// automatically maintained last-modification timestamp, so to use this we'd
// need to have a SetMetadata API too.  Which we're not yet doing, but feel
// free to contribute patches.
func (k *secretsKeyring) GetMetadata(key string) (Metadata, error) {
	return Metadata{}, ErrMetadataNeedsCredentials
}

func (k *secretsKeyring) Set(item Item) error {
	err := k.openSecrets()
	if err != nil {
		return err
	}

	// create the collection if it doesn't already exist
	if k.collection == nil {
		collection, err := k.service.CreateCollection(k.name)
		if err != nil {
			return err
		}

		k.collection = collection
	}

	// create the new item
	data, err := json.Marshal(item)
	if err != nil {
		return err
	}

	secret := libsecret.NewSecret(k.session, []byte{}, data, "application/json")

	// unlock the collection first
	locked, err := k.collection.Locked()
	if err != nil {
		return err
	}

	if locked {
		if err := k.service.Unlock(k.collection); err != nil {
			return err
		}
	}

	if _, err := k.collection.CreateItem(item.Key, secret, true); err != nil {
		return err
	}

	return nil
}

func (k *secretsKeyring) Remove(key string) error {
	if err := k.openCollection(); err != nil {
		if err == errCollectionNotFound {
			return ErrKeyNotFound
		}
		return err
	}

	items, err := k.collection.SearchItems(key)
	if err != nil {
		return err
	}

	// nothing to delete
	if len(items) == 0 {
		return nil
	}

	// we dont want to delete more than one anyway
	// so just get the first item found
	item := items[0]

	locked, err := item.Locked()
	if err != nil {
		return err
	}

	if locked {
		if err := k.service.Unlock(item); err != nil {
			return err
		}
	}

	if err := item.Delete(); err != nil {
		return err
	}

	return nil
}

func (k *secretsKeyring) Keys() ([]string, error) {
	if err := k.openCollection(); err != nil {
		if err == errCollectionNotFound {
			return []string{}, nil
		}
		return []string{}, err
	}

	items, err := k.collection.Items()
	if err != nil {
		return []string{}, err
	}

	keys := []string{}

	for _, item := range items {
		label, err := item.Label()
		if err == nil {
			keys = append(keys, label)
		}
	}

	return keys, nil
}

// deleteCollection deletes the keyring's collection if it exists. This is mainly to support testing.
func (k *secretsKeyring) deleteCollection() error {
	if err := k.openCollection(); err != nil {
		return err
	}
	return k.collection.Delete()
}
<commit_msg>libsecret: Keys() unlocks collection first<commit_after>// +build linux

package keyring

import (
	"encoding/json"
	"errors"

	"github.com/godbus/dbus"
	"github.com/gsterjov/go-libsecret"
)

func init() {
	// silently fail if dbus isn't available
	_, err := dbus.SessionBus()
	if err != nil {
		return
	}

	supportedBackends[SecretServiceBackend] = opener(func(cfg Config) (Keyring, error) {
		if cfg.ServiceName == "" {
			cfg.ServiceName = "secret-service"
		}
		if cfg.LibSecretCollectionName == "" {
			cfg.LibSecretCollectionName = cfg.ServiceName
		}

		service, err := libsecret.NewService()
		if err != nil {
			return &secretsKeyring{}, err
		}

		ring := &secretsKeyring{
			name:    cfg.LibSecretCollectionName,
			service: service,
		}

		return ring, ring.openSecrets()
	})
}

type secretsKeyring struct {
	name       string
	service    *libsecret.Service
	collection *libsecret.Collection
	session    *libsecret.Session
}

type secretsError struct {
	message string
}

func (e *secretsError) Error() string {
	return e.message
}

var errCollectionNotFound = errors.New("The collection does not exist. Please add a key first")

func (k *secretsKeyring) openSecrets() error {
	session, err := k.service.Open()
	if err != nil {
		return err
	}
	k.session = session

	// get the collection if it already exists
	collections, err := k.service.Collections()
	if err != nil {
		return err
	}

	path := libsecret.DBusPath + "/collection/" + k.name

	for _, collection := range collections {
		if string(collection.Path()) == path {
			k.collection = &collection
			return nil
		}
	}

	return nil
}

func (k *secretsKeyring) openCollection() error {
	if err := k.openSecrets(); err != nil {
		return err
	}

	if k.collection == nil {
		return errCollectionNotFound
		// return &secretsError{fmt.Sprintf(
		// 	"The collection %q does not exist. Please add a key first",
		// 	k.name,
		// )}
	}

	return nil
}

func (k *secretsKeyring) Get(key string) (Item, error) {
	if err := k.openCollection(); err != nil {
		if err == errCollectionNotFound {
			return Item{}, ErrKeyNotFound
		}
		return Item{}, err
	}

	items, err := k.collection.SearchItems(key)
	if err != nil {
		return Item{}, err
	}

	if len(items) == 0 {
		return Item{}, ErrKeyNotFound
	}

	// use the first item whenever there are multiples
	// with the same profile name
	item := items[0]

	locked, err := item.Locked()
	if err != nil {
		return Item{}, err
	}

	if locked {
		if err := k.service.Unlock(item); err != nil {
			return Item{}, err
		}
	}

	secret, err := item.GetSecret(k.session)
	if err != nil {
		return Item{}, err
	}

	// pack the secret into the item
	var ret Item
	if err = json.Unmarshal(secret.Value, &ret); err != nil {
		return Item{}, err
	}

	return ret, err
}

// GetMetadata for libsecret returns an error indicating that it's unsupported
// for this backend.
//
// libsecret actually implements a metadata system which we could use, "Secret
// Attributes"; I found no indication in documentation of anything like an
// automatically maintained last-modification timestamp, so to use this we'd
// need to have a SetMetadata API too.  Which we're not yet doing, but feel
// free to contribute patches.
func (k *secretsKeyring) GetMetadata(key string) (Metadata, error) {
	return Metadata{}, ErrMetadataNeedsCredentials
}

func (k *secretsKeyring) Set(item Item) error {
	err := k.openSecrets()
	if err != nil {
		return err
	}

	// create the collection if it doesn't already exist
	if k.collection == nil {
		collection, err := k.service.CreateCollection(k.name)
		if err != nil {
			return err
		}

		k.collection = collection
	}

	if err := k.ensureCollectionUnlocked(); err != nil {
		return err
	}

	// create the new item
	data, err := json.Marshal(item)
	if err != nil {
		return err
	}

	secret := libsecret.NewSecret(k.session, []byte{}, data, "application/json")

	if _, err := k.collection.CreateItem(item.Key, secret, true); err != nil {
		return err
	}

	return nil
}

func (k *secretsKeyring) Remove(key string) error {
	if err := k.openCollection(); err != nil {
		if err == errCollectionNotFound {
			return ErrKeyNotFound
		}
		return err
	}

	items, err := k.collection.SearchItems(key)
	if err != nil {
		return err
	}

	// nothing to delete
	if len(items) == 0 {
		return nil
	}

	// we dont want to delete more than one anyway
	// so just get the first item found
	item := items[0]

	locked, err := item.Locked()
	if err != nil {
		return err
	}

	if locked {
		if err := k.service.Unlock(item); err != nil {
			return err
		}
	}

	if err := item.Delete(); err != nil {
		return err
	}

	return nil
}

func (k *secretsKeyring) Keys() ([]string, error) {
	if err := k.openCollection(); err != nil {
		if err == errCollectionNotFound {
			return []string{}, nil
		}
		return nil, err
	}
	if err := k.ensureCollectionUnlocked(); err != nil {
		return nil, err
	}
	items, err := k.collection.Items()
	if err != nil {
		return nil, err
	}
	keys := []string{}
	for _, item := range items {
		label, err := item.Label()
		if err == nil {
			keys = append(keys, label)
		} else {
			// err is being silently ignored here, not sure if that's good or bad
		}
	}
	return keys, nil
}

// deleteCollection deletes the keyring's collection if it exists. This is mainly to support testing.
func (k *secretsKeyring) deleteCollection() error {
	if err := k.openCollection(); err != nil {
		return err
	}
	return k.collection.Delete()
}

// unlock the collection if it's locked
func (k *secretsKeyring) ensureCollectionUnlocked() error {
	locked, err := k.collection.Locked()
	if err != nil {
		return err
	}
	if !locked {
		return nil
	}
	return k.service.Unlock(k.collection)
}
<|endoftext|>


<commit_before>package qemu

import (
	"bytes"
	"encoding/binary"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"path/filepath"
	"strconv"
	"testing"
	"time"

	"github.com/waterfall/testutils"
	"golang.org/x/net/context"
	"golang.org/x/sync/errgroup"
)

var (
	// test args
	launcher = flag.String("launcher", "", "The path to the emulator launcher")
	adbTurbo = flag.String("adb_turbo", "", "The path to abd.turbo binary")
	server   = flag.String("server", "", "The path to test server")
)

const emuWorkingDir = "images/session"
const socketName = "sockets/h2o"

func init() {
	flag.Parse()

	if *launcher == "" || *adbTurbo == "" || *server == "" {
		log.Fatalf("launcher, adb and server args need to be provided")
	}
}

func testBytes(size int) []byte {
	bb := new(bytes.Buffer)
	var i uint32
	for i = 0; i < uint32(size); i += 4 {
		binary.Write(bb, binary.LittleEndian, i)
	}
	return bb.Bytes()
}

func runServer(ctx context.Context, adbTurbo, adbPort, server string, n, bs int) (chan string, chan error, error) {
	s := "localhost:" + adbPort
	_, err := testutils.ExecOnDevice(
		ctx, adbTurbo, s, "push", []string{server, "/data/local/tmp/server"})
	if err != nil {
		return nil, nil, err
	}

	_, err = testutils.ExecOnDevice(
		ctx, adbTurbo, s, "shell", []string{"chmod", "+x", "/data/local/tmp/server"})
	if err != nil {
		return nil, nil, err
	}

	outChan := make(chan string, 1)
	errChan := make(chan error, 1)
	go func() {
		out, err := testutils.ExecOnDevice(
			ctx, adbTurbo, s, "shell", []string{"/data/local/tmp/server",
				"-conns", strconv.Itoa(n), "-rec_n", strconv.Itoa(bs)})
		outChan <- out
		errChan <- err
	}()
	return outChan, errChan, nil
}

func read(c net.Conn, buff []byte) ([]byte, error) {
	b := bytes.NewBuffer(buff)
	if _, err := io.Copy(bytes.NewBuffer(buff), c); err != nil {
		return nil, err
	}
	return b.Bytes(), nil
}

func TestSingleConn(t *testing.T) {
	adbServerPort, adbPort, emuPort, err := testutils.GetAdbPorts()
	if err != nil {
		t.Fatal(err)
	}

	l := filepath.Join(testutils.RunfilesRoot(), *launcher)
	a := filepath.Join(testutils.RunfilesRoot(), *adbTurbo)
	svr := filepath.Join(testutils.RunfilesRoot(), *server)

	emuDir, err := testutils.SetupEmu(l, adbServerPort, adbPort, emuPort)
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(emuDir)
	defer testutils.KillEmu(l, adbServerPort, adbPort, emuPort)

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()

	s := 4 * 1024 * 1024
	tb := testBytes(s)
	outChan, errChan, err := runServer(ctx, a, adbPort, svr, 1, s)
	if err != nil {
		t.Fatalf("error starting server: %v", err)
	}

	cb, err := MakeConnBuilder(filepath.Join(emuDir, emuWorkingDir), socketName)
	if err != nil {
		t.Fatalf("error opening qemu connection: %v", err)
	}
	defer cb.Close()

	c, err := cb.Accept()
	if err != nil {
		t.Fatal(err)
	}

	eg, ctx := errgroup.WithContext(ctx)
	eg.Go(func() error {
		defer c.Close()
		wr, err := io.Copy(c, bytes.NewBuffer(tb))
		if err != nil {
			return err
		}
		if wr != int64(len(tb)) {
			return fmt.Errorf("wrote %d bytes but had %d", wr, len(tb))
		}
		return nil
	})

	eg.Go(func() error {
		bb := new(bytes.Buffer)
		rr, err := io.Copy(bb, c)
		if err != nil {
			return err
		}
		if rr != int64(len(tb)) {
			return fmt.Errorf("read %d but only sent %d", rr, len(tb))
		}
		if bytes.Compare(tb, bb.Bytes()) != 0 {
			return fmt.Errorf("sent bytes not the same as received")
		}
		return err
	})

	if err := eg.Wait(); err != nil {
		t.Errorf("unexpected error: %v", err)
	}

	out := <-outChan
	if err := <-errChan; err != nil {
		t.Errorf("server died with error: %v out %v", err, out)
	}

}

func TestMultipleConn(t *testing.T) {
	adbServerPort, adbPort, emuPort, err := testutils.GetAdbPorts()
	if err != nil {
		t.Fatal(err)
	}

	l := filepath.Join(testutils.RunfilesRoot(), *launcher)
	a := filepath.Join(testutils.RunfilesRoot(), *adbTurbo)
	svr := filepath.Join(testutils.RunfilesRoot(), *server)

	emuDir, err := testutils.SetupEmu(l, adbServerPort, adbPort, emuPort)
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(emuDir)
	defer testutils.KillEmu(l, adbServerPort, adbPort, emuPort)

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()

	conns := 256
	s := 4 * 1024 * 1024
	tb := testBytes(s)
	outChan, errChan, err := runServer(ctx, a, adbPort, svr, conns, s)
	if err != nil {
		t.Fatalf("error starting server: %v", err)
	}

	cb, err := MakeConnBuilder(filepath.Join(emuDir, emuWorkingDir), socketName)
	if err != nil {
		t.Fatalf("error opening qemu connection: %v", err)
	}
	defer cb.Close()

	eg, _ := errgroup.WithContext(ctx)
	for i := 0; i < conns; i++ {

		// Lets avoid variable aliasing
		func() {
			c, err := cb.Accept()
			if err != nil {
				t.Fatal(err)
			}

			eg.Go(func() error {
				defer c.Close()
				wr, err := io.Copy(c, bytes.NewBuffer(tb))
				if err != nil {
					return err
				}
				if wr != int64(len(tb)) {
					return fmt.Errorf("wrote %d bytes but had %d", wr, len(tb))
				}
				return nil
			})

			eg.Go(func() error {
				bb := new(bytes.Buffer)
				rr, err := io.Copy(bb, c)
				if err != nil {
					return err
				}
				if rr != int64(len(tb)) {
					return fmt.Errorf("read %d but only sent %d", rr, len(tb))
				}
				if bytes.Compare(tb, bb.Bytes()) != 0 {
					return fmt.Errorf("sent bytes not the same as received")
				}
				return err
			})
		}()

	}

	if err := eg.Wait(); err != nil {
		t.Errorf("got unexpected error: %v", err)
	}
	out := <-outChan
	if err := <-errChan; err != nil {
		t.Errorf("got unexpected error: %v out %v", err, out)
	}
}
<commit_msg>Fix qemu conn tests<commit_after>package qemu

import (
	"bytes"
	"encoding/binary"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"path/filepath"
	"strconv"
	"testing"
	"time"

	"github.com/waterfall/testutils"
	"golang.org/x/net/context"
	"golang.org/x/sync/errgroup"
)

var (
	// test args
	launcher = flag.String("launcher", "", "The path to the emulator launcher")
	adbTurbo = flag.String("adb_turbo", "", "The path to abd.turbo binary")
	server   = flag.String("server", "", "The path to test server")
)

const emuWorkingDir = "images/session"
const socketName = "sockets/h2o"

func init() {
	flag.Parse()

	if *launcher == "" || *adbTurbo == "" || *server == "" {
		log.Fatalf("launcher, adb and server args need to be provided")
	}
}

func testBytes(size int) []byte {
	bb := new(bytes.Buffer)
	var i uint32
	for i = 0; i < uint32(size); i += 4 {
		binary.Write(bb, binary.LittleEndian, i)
	}
	return bb.Bytes()
}

func runServer(ctx context.Context, adbTurbo, adbPort, server string, n, bs int) (chan string, chan error, error) {
	s := "localhost:" + adbPort
	_, err := testutils.ExecOnDevice(
		ctx, adbTurbo, s, "push", []string{server, "/data/local/tmp/server"})
	if err != nil {
		return nil, nil, err
	}

	if _, err = testutils.ExecOnDevice(
		ctx, adbTurbo, s, "shell", []string{"chmod", "+x", "/data/local/tmp/server"}); err != nil {
		return nil, nil, err
	}

	// stop waterfall service as it will interfere with this test
	if _, err = testutils.ExecOnDevice(
		ctx, adbTurbo, s, "shell", []string{"stop", "waterfall"}); err != nil {
		return nil, nil, err
	}

	outChan := make(chan string, 1)
	errChan := make(chan error, 1)
	go func() {
		out, err := testutils.ExecOnDevice(
			ctx, adbTurbo, s, "shell", []string{"/data/local/tmp/server",
				"-conns", strconv.Itoa(n), "-rec_n", strconv.Itoa(bs)})
		outChan <- out
		errChan <- err
	}()
	return outChan, errChan, nil
}

func read(c net.Conn, buff []byte) ([]byte, error) {
	b := bytes.NewBuffer(buff)
	if _, err := io.Copy(bytes.NewBuffer(buff), c); err != nil {
		return nil, err
	}
	return b.Bytes(), nil
}

func TestSingleConn(t *testing.T) {
	adbServerPort, adbPort, emuPort, err := testutils.GetAdbPorts()
	if err != nil {
		t.Fatal(err)
	}

	l := filepath.Join(testutils.RunfilesRoot(), *launcher)
	a := filepath.Join(testutils.RunfilesRoot(), *adbTurbo)
	svr := filepath.Join(testutils.RunfilesRoot(), *server)

	emuDir, err := testutils.SetupEmu(l, adbServerPort, adbPort, emuPort)
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(emuDir)
	defer testutils.KillEmu(l, adbServerPort, adbPort, emuPort)

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()

	s := 4 * 1024 * 1024
	tb := testBytes(s)
	outChan, errChan, err := runServer(ctx, a, adbPort, svr, 1, s)
	if err != nil {
		t.Fatalf("error starting server: %v", err)
	}

	cb, err := MakeConnBuilder(filepath.Join(emuDir, emuWorkingDir), socketName)
	if err != nil {
		t.Fatalf("error opening qemu connection: %v", err)
	}
	defer cb.Close()

	c, err := cb.Accept()
	if err != nil {
		t.Fatal(err)
	}

	eg, ctx := errgroup.WithContext(ctx)
	eg.Go(func() error {
		defer c.(*Conn).CloseWrite()
		wr, err := io.Copy(c, bytes.NewBuffer(tb))
		if err != nil {
			return err
		}
		if wr != int64(len(tb)) {
			return fmt.Errorf("wrote %d bytes but had %d", wr, len(tb))
		}
		return nil
	})

	eg.Go(func() error {
		bb := new(bytes.Buffer)
		rr, err := io.Copy(bb, c)
		if err != nil {
			return err
		}
		if rr != int64(len(tb)) {
			return fmt.Errorf("read %d but only sent %d", rr, len(tb))
		}
		if bytes.Compare(tb, bb.Bytes()) != 0 {
			return fmt.Errorf("sent bytes not the same as received")
		}
		return err
	})

	if err := eg.Wait(); err != nil {
		t.Errorf("unexpected error: %v", err)
	}

	out := <-outChan
	if err := <-errChan; err != nil {
		t.Errorf("server died with error: %v out %v", err, out)
	}

}

func TestMultipleConn(t *testing.T) {
	adbServerPort, adbPort, emuPort, err := testutils.GetAdbPorts()
	if err != nil {
		t.Fatal(err)
	}

	l := filepath.Join(testutils.RunfilesRoot(), *launcher)
	a := filepath.Join(testutils.RunfilesRoot(), *adbTurbo)
	svr := filepath.Join(testutils.RunfilesRoot(), *server)

	emuDir, err := testutils.SetupEmu(l, adbServerPort, adbPort, emuPort)
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(emuDir)
	defer testutils.KillEmu(l, adbServerPort, adbPort, emuPort)

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()

	conns := 256
	s := 4 * 1024 * 1024
	tb := testBytes(s)
	outChan, errChan, err := runServer(ctx, a, adbPort, svr, conns, s)
	if err != nil {
		t.Fatalf("error starting server: %v", err)
	}

	cb, err := MakeConnBuilder(filepath.Join(emuDir, emuWorkingDir), socketName)
	if err != nil {
		t.Fatalf("error opening qemu connection: %v", err)
	}
	defer cb.Close()

	eg, _ := errgroup.WithContext(ctx)
	for i := 0; i < conns; i++ {

		// Lets avoid variable aliasing
		func() {
			c, err := cb.Accept()
			if err != nil {
				t.Fatal(err)
			}

			eg.Go(func() error {
				defer c.(*Conn).CloseWrite()
				wr, err := io.Copy(c, bytes.NewBuffer(tb))
				if err != nil {
					return err
				}
				if wr != int64(len(tb)) {
					return fmt.Errorf("wrote %d bytes but had %d", wr, len(tb))
				}
				return nil
			})

			eg.Go(func() error {
				bb := new(bytes.Buffer)
				rr, err := io.Copy(bb, c)
				if err != nil {
					return err
				}
				if rr != int64(len(tb)) {
					return fmt.Errorf("read %d but sent %d", rr, len(tb))
				}
				if bytes.Compare(tb, bb.Bytes()) != 0 {
					return fmt.Errorf("sent bytes not the same as received")
				}
				return err
			})
		}()

	}

	if err := eg.Wait(); err != nil {
		t.Errorf("got unexpected error: %v", err)
	}
	out := <-outChan
	if err := <-errChan; err != nil {
		t.Errorf("got unexpected error: %v out %v", err, out)
	}
}
<|endoftext|>


<commit_before>// Copyright 2013-2017 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Print process information.
//
// Synopsis:
//     id
//
// Description:
//     id displays the uid, guid and groups of the calling process
//
// Options:
package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"
	"syscall"
)

var (
	GROUP_FILE  = "/etc/group"
	PASSWD_FILE = "/etc/passwd"
)

type Group struct {
	Name   string
	Number int
}

type User struct {
	Name   string
	Uid    int
	Euid   int
	Groups []*Group
}

func (u *User) pullUid() {
	u.Uid = syscall.Getuid()
}

func (u *User) pullEuid() {
	u.Uid = syscall.Getieuid()
}

func (u *User) pullName() {
	passwdFile, err := os.Open(PASSWD_FILE)
	if err != nil {
		log.Fatal(err)
	}

	var passwdInfo []string

	passwdScanner = bufio.NewScanner(PASSWD_FILE)

	for passwdScanner.Scan() {
		passwdInfo = strings.Split(passwdScanner.Text(), ":")
		if passwdInfo[2] == u.getUid() {
			u.Name = passwdInfo[0]
			return
		}
	}

	log.Fatal()
}

func (u *User) pullGroups() {
	groupsNumbers, err := syscall.Getgroups()
	if err != nil {
		log.Fatal(err)
	}

	groupsMap, err := readGroups()
	if err != nil {
		log.Fatal(err)
	}

	for _, groupNum := range groupsNumbers {
		u.Groups = append(u.Groups, Group{
			Name:   groupsMap[groupNum],
			Number: groupNum,
		})
	}

}

func readGroups() (map[int]string, error) {
	groupFile, err := os.Open(GROUP_FILE)
	if err != nil {
		return nil, err
	}

	var groupInfo []string

	groupsMap := make(map[int]string)
	groupScanner := bufio.NewScanner(groupFile)

	for groupScanner.Scan() {
		groupInfo = strings.Split(groupScanner.Text(), ":")
		groupsMap[strconv.Atoi(groupInfo[2])] = groupInfo[0]
	}

	return groupMap, nil
}

func (u *User) Init() {
	u.pullUid()
	u.pullEuid()
	u.pullGroups()
	u.pullName()
}

func main() {
	uid := syscall.Getuid()
	gid := syscall.Getgid()
	groups, err := syscall.Getgroups()
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("uid: %d\n", uid)
	fmt.Printf("gid: %d\n", gid)

	fmt.Print("groups: ")
	for _, group := range groups {
		fmt.Printf("%d ", group)
	}
	fmt.Println()

}
<commit_msg>Add get methods and implement Creator<commit_after>// Copyright 2013-2017 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Print process information.
//
// Synopsis:
//     id
//
// Description:
//     id displays the uid, guid and groups of the calling process
//
// Options:
package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"
	"syscall"
)

var (
	GROUP_FILE  = "/etc/group"
	PASSWD_FILE = "/etc/passwd"
)

type Group struct {
	Name   string
	Number int
}

type User struct {
	Name   string
	Uid    int
	Euid   int
	Groups []*Group
}

func (u *User) getUid() int {
	return u.Uid
}

func (u *User) pullUid() {
	u.Uid = syscall.Getuid()
}

func (u *User) getEuid() int {
	return u.Euid
}

func (u *User) pullEuid() {
	u.Uid = syscall.Geteuid()
}

func (u *User) getName() string {
	return u.Name
}

func (u *User) pullName() {
	passwdFile, err := os.Open(PASSWD_FILE)
	if err != nil {
		log.Fatal(err)
	}

	var passwdInfo []string

	passwdScanner := bufio.NewScanner(passwdFile)

	for passwdScanner.Scan() {
		passwdInfo = strings.Split(passwdScanner.Text(), ":")
		if val, err := strconv.Atoi(passwdInfo[2]); err != nil {
			log.Fatal(err)
		} else if val == u.getUid() {
			u.Name = passwdInfo[0]
			return
		}
	}

	log.Fatal()
}

func (u *User) getGroups() []*Group {
	return u.Groups
}

func (u *User) pullGroups() {
	groupsNumbers, err := syscall.Getgroups()
	if err != nil {
		log.Fatal(err)
	}

	groupsMap, err := readGroups()
	if err != nil {
		log.Fatal(err)
	}

	for _, groupNum := range groupsNumbers {
		u.Groups = append(u.Groups, &Group{
			Name:   groupsMap[groupNum],
			Number: groupNum,
		})
	}

}

func readGroups() (map[int]string, error) {
	groupFile, err := os.Open(GROUP_FILE)
	if err != nil {
		return nil, err
	}

	var groupInfo []string

	groupsMap := make(map[int]string)
	groupScanner := bufio.NewScanner(groupFile)

	for groupScanner.Scan() {
		groupInfo = strings.Split(groupScanner.Text(), ":")
		groupNum, err := strconv.Atoi(groupInfo[2])
		if err != nil {
			log.Fatal(err)
		}

		groupsMap[groupNum] = groupInfo[0]
	}

	return groupsMap, nil
}

func (u *User) Init() {
	u.pullUid()
	u.pullEuid()
	u.pullGroups()
	u.pullName()
}

func main() {
	uid := syscall.Getuid()
	gid := syscall.Getgid()
	groups, err := syscall.Getgroups()
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("uid: %d\n", uid)
	fmt.Printf("gid: %d\n", gid)

	fmt.Print("groups: ")
	for _, group := range groups {
		fmt.Printf("%d ", group)
	}
	fmt.Println()

}
<|endoftext|>


<commit_before>package mount

import (
	"context"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/mount/meta_cache"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/hanwen/go-fuse/v2/fuse"
)

// Lookup is called by the kernel when the VFS wants to know
// about a file inside a directory. Many lookup calls can
// occur in parallel, but only one call happens for each (dir,
// name) pair.

func (wfs *WFS) Lookup(cancel <-chan struct{}, header *fuse.InHeader, name string, out *fuse.EntryOut) (code fuse.Status) {

	if s := checkName(name); s != fuse.OK {
		return s
	}

	dirPath, code := wfs.inodeToPath.GetPath(header.NodeId)
	if code != fuse.OK {
		return
	}

	fullFilePath := dirPath.Child(name)

	visitErr := meta_cache.EnsureVisited(wfs.metaCache, wfs, dirPath, nil)
	if visitErr != nil {
		glog.Errorf("dir Lookup %s: %v", dirPath, visitErr)
		return fuse.EIO
	}
	localEntry, cacheErr := wfs.metaCache.FindEntry(context.Background(), fullFilePath)
	if cacheErr == filer_pb.ErrNotFound {
		return fuse.ENOENT
	}

	if localEntry == nil {
		// glog.V(3).Infof("dir Lookup cache miss %s", fullFilePath)
		entry, err := filer_pb.GetEntry(wfs, fullFilePath)
		if err != nil {
			glog.V(1).Infof("dir GetEntry %s: %v", fullFilePath, err)
			return fuse.ENOENT
		}
		localEntry = filer.FromPbEntry(string(dirPath), entry)
	} else {
		glog.V(4).Infof("dir Lookup cache hit %s", fullFilePath)
	}

	if localEntry == nil {
		return fuse.ENOENT
	}

	inode := wfs.inodeToPath.Lookup(fullFilePath, localEntry.Mode, len(localEntry.HardLinkId) > 0, localEntry.Inode, true)

	if fh, found := wfs.fhmap.FindFileHandle(inode); found {
		glog.V(4).Infof("lookup opened file %s size %d", dirPath.Child(localEntry.Name()), filer.FileSize(fh.entry))
		localEntry = filer.FromPbEntry(string(dirPath), fh.entry)
	}

	wfs.outputFilerEntry(out, inode, localEntry)

	return fuse.OK

}
<commit_msg>fix nil entry<commit_after>package mount

import (
	"context"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/mount/meta_cache"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/hanwen/go-fuse/v2/fuse"
)

// Lookup is called by the kernel when the VFS wants to know
// about a file inside a directory. Many lookup calls can
// occur in parallel, but only one call happens for each (dir,
// name) pair.

func (wfs *WFS) Lookup(cancel <-chan struct{}, header *fuse.InHeader, name string, out *fuse.EntryOut) (code fuse.Status) {

	if s := checkName(name); s != fuse.OK {
		return s
	}

	dirPath, code := wfs.inodeToPath.GetPath(header.NodeId)
	if code != fuse.OK {
		return
	}

	fullFilePath := dirPath.Child(name)

	visitErr := meta_cache.EnsureVisited(wfs.metaCache, wfs, dirPath, nil)
	if visitErr != nil {
		glog.Errorf("dir Lookup %s: %v", dirPath, visitErr)
		return fuse.EIO
	}
	localEntry, cacheErr := wfs.metaCache.FindEntry(context.Background(), fullFilePath)
	if cacheErr == filer_pb.ErrNotFound {
		return fuse.ENOENT
	}

	if localEntry == nil {
		// glog.V(3).Infof("dir Lookup cache miss %s", fullFilePath)
		entry, err := filer_pb.GetEntry(wfs, fullFilePath)
		if err != nil {
			glog.V(1).Infof("dir GetEntry %s: %v", fullFilePath, err)
			return fuse.ENOENT
		}
		localEntry = filer.FromPbEntry(string(dirPath), entry)
	} else {
		glog.V(4).Infof("dir Lookup cache hit %s", fullFilePath)
	}

	if localEntry == nil {
		return fuse.ENOENT
	}

	inode := wfs.inodeToPath.Lookup(fullFilePath, localEntry.Mode, len(localEntry.HardLinkId) > 0, localEntry.Inode, true)

	if fh, found := wfs.fhmap.FindFileHandle(inode); found && fh.entry != nil {
		glog.V(4).Infof("lookup opened file %s size %d", dirPath.Child(localEntry.Name()), filer.FileSize(fh.entry))
		localEntry = filer.FromPbEntry(string(dirPath), fh.entry)
	}

	wfs.outputFilerEntry(out, inode, localEntry)

	return fuse.OK

}
<|endoftext|>


<commit_before>package s3api

// the related code is copied and modified from minio source code

/*
 * Minio Cloud Storage, (C) 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import (
	"bufio"
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
	"hash"
	"io"
	"net/http"
	"time"

	"github.com/dustin/go-humanize"
)

// getChunkSignature - get chunk signature.
func getChunkSignature(secretKey string, seedSignature string, region string, date time.Time, hashedChunk string) string {

	// Calculate string to sign.
	stringToSign := signV4ChunkedAlgorithm + "\n" +
		date.Format(iso8601Format) + "\n" +
		getScope(date, region) + "\n" +
		seedSignature + "\n" +
		emptySHA256 + "\n" +
		hashedChunk

	// Get hmac signing key.
	signingKey := getSigningKey(secretKey, date, region, "s3")

	// Calculate signature.
	newSignature := getSignature(signingKey, stringToSign)

	return newSignature
}

// calculateSeedSignature - Calculate seed signature in accordance with
//     - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
// returns signature, error otherwise if the signature mismatches or any other
// error while parsing and validating.
func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cred *Credential, signature string, region string, date time.Time, errCode s3err.ErrorCode) {

	// Copy request.
	req := *r

	// Save authorization header.
	v4Auth := req.Header.Get("Authorization")

	// Parse signature version '4' header.
	signV4Values, errCode := parseSignV4(v4Auth)
	if errCode != s3err.ErrNone {
		return nil, "", "", time.Time{}, errCode
	}

	// Payload streaming.
	payload := streamingContentSHA256

	// Payload for STREAMING signature should be 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'
	if payload != req.Header.Get("X-Amz-Content-Sha256") {
		return nil, "", "", time.Time{}, s3err.ErrContentSHA256Mismatch
	}

	// Extract all the signed headers along with its values.
	extractedSignedHeaders, errCode := extractSignedHeaders(signV4Values.SignedHeaders, r)
	if errCode != s3err.ErrNone {
		return nil, "", "", time.Time{}, errCode
	}
	// Verify if the access key id matches.
	_, cred, found := iam.lookupByAccessKey(signV4Values.Credential.accessKey)
	if !found {
		return nil, "", "", time.Time{}, s3err.ErrInvalidAccessKeyID
	}

	// Verify if region is valid.
	region = signV4Values.Credential.scope.region

	// Extract date, if not present throw error.
	var dateStr string
	if dateStr = req.Header.Get(http.CanonicalHeaderKey("x-amz-date")); dateStr == "" {
		if dateStr = r.Header.Get("Date"); dateStr == "" {
			return nil, "", "", time.Time{}, s3err.ErrMissingDateHeader
		}
	}
	// Parse date header.
	var err error
	date, err = time.Parse(iso8601Format, dateStr)
	if err != nil {
		return nil, "", "", time.Time{}, s3err.ErrMalformedDate
	}

	// Query string.
	queryStr := req.URL.Query().Encode()

	// Get canonical request.
	canonicalRequest := getCanonicalRequest(extractedSignedHeaders, payload, queryStr, req.URL.Path, req.Method)

	// Get string to sign from canonical request.
	stringToSign := getStringToSign(canonicalRequest, date, signV4Values.Credential.getScope())

	// Get hmac signing key.
	signingKey := getSigningKey(cred.SecretKey, signV4Values.Credential.scope.date, region, "s3")

	// Calculate signature.
	newSignature := getSignature(signingKey, stringToSign)

	// Verify if signature match.
	if !compareSignatureV4(newSignature, signV4Values.Signature) {
		return nil, "", "", time.Time{}, s3err.ErrSignatureDoesNotMatch
	}

	// Return caculated signature.
	return cred, newSignature, region, date, s3err.ErrNone
}

const maxLineLength = 4 * humanize.KiByte // assumed <= bufio.defaultBufSize 4KiB

// lineTooLong is generated as chunk header is bigger than 4KiB.
var errLineTooLong = errors.New("header line too long")

// Malformed encoding is generated when chunk header is wrongly formed.
var errMalformedEncoding = errors.New("malformed chunked encoding")

// newSignV4ChunkedReader returns a new s3ChunkedReader that translates the data read from r
// out of HTTP "chunked" format before returning it.
// The s3ChunkedReader returns io.EOF when the final 0-length chunk is read.
func (iam *IdentityAccessManagement) newSignV4ChunkedReader(req *http.Request) (io.ReadCloser, s3err.ErrorCode) {
	ident, seedSignature, region, seedDate, errCode := iam.calculateSeedSignature(req)
	if errCode != s3err.ErrNone {
		return nil, errCode
	}
	return &s3ChunkedReader{
		cred:              ident,
		reader:            bufio.NewReader(req.Body),
		seedSignature:     seedSignature,
		seedDate:          seedDate,
		region:            region,
		chunkSHA256Writer: sha256.New(),
		state:             readChunkHeader,
	}, s3err.ErrNone
}

// Represents the overall state that is required for decoding a
// AWS Signature V4 chunked reader.
type s3ChunkedReader struct {
	cred              *Credential
	reader            *bufio.Reader
	seedSignature     string
	seedDate          time.Time
	region            string
	state             chunkState
	lastChunk         bool
	chunkSignature    string
	chunkSHA256Writer hash.Hash // Calculates sha256 of chunk data.
	n                 uint64    // Unread bytes in chunk
	err               error
}

// Read chunk reads the chunk token signature portion.
func (cr *s3ChunkedReader) readS3ChunkHeader() {
	// Read the first chunk line until CRLF.
	var hexChunkSize, hexChunkSignature []byte
	hexChunkSize, hexChunkSignature, cr.err = readChunkLine(cr.reader)
	if cr.err != nil {
		return
	}
	// <hex>;token=value - converts the hex into its uint64 form.
	cr.n, cr.err = parseHexUint(hexChunkSize)
	if cr.err != nil {
		return
	}
	if cr.n == 0 {
		cr.err = io.EOF
	}
	// Save the incoming chunk signature.
	cr.chunkSignature = string(hexChunkSignature)
}

type chunkState int

const (
	readChunkHeader chunkState = iota
	readChunkTrailer
	readChunk
	verifyChunk
	eofChunk
)

func (cs chunkState) String() string {
	stateString := ""
	switch cs {
	case readChunkHeader:
		stateString = "readChunkHeader"
	case readChunkTrailer:
		stateString = "readChunkTrailer"
	case readChunk:
		stateString = "readChunk"
	case verifyChunk:
		stateString = "verifyChunk"
	case eofChunk:
		stateString = "eofChunk"

	}
	return stateString
}

func (cr *s3ChunkedReader) Close() (err error) {
	return nil
}

// Read - implements `io.Reader`, which transparently decodes
// the incoming AWS Signature V4 streaming signature.
func (cr *s3ChunkedReader) Read(buf []byte) (n int, err error) {
	for {
		switch cr.state {
		case readChunkHeader:
			cr.readS3ChunkHeader()
			// If we're at the end of a chunk.
			if cr.n == 0 && cr.err == io.EOF {
				cr.state = readChunkTrailer
				cr.lastChunk = true
				continue
			}
			if cr.err != nil {
				return 0, cr.err
			}
			cr.state = readChunk
		case readChunkTrailer:
			cr.err = readCRLF(cr.reader)
			if cr.err != nil {
				return 0, errMalformedEncoding
			}
			cr.state = verifyChunk
		case readChunk:
			// There is no more space left in the request buffer.
			if len(buf) == 0 {
				return n, nil
			}
			rbuf := buf
			// The request buffer is larger than the current chunk size.
			// Read only the current chunk from the underlying reader.
			if uint64(len(rbuf)) > cr.n {
				rbuf = rbuf[:cr.n]
			}
			var n0 int
			n0, cr.err = cr.reader.Read(rbuf)
			if cr.err != nil {
				// We have lesser than chunk size advertised in chunkHeader, this is 'unexpected'.
				if cr.err == io.EOF {
					cr.err = io.ErrUnexpectedEOF
				}
				return 0, cr.err
			}

			// Calculate sha256.
			cr.chunkSHA256Writer.Write(rbuf[:n0])

			// Update the bytes read into request buffer so far.
			n += n0
			buf = buf[n0:]
			// Update bytes to be read of the current chunk before verifying chunk's signature.
			cr.n -= uint64(n0)

			// If we're at the end of a chunk.
			if cr.n == 0 {
				cr.state = readChunkTrailer
				continue
			}
		case verifyChunk:
			// Calculate the hashed chunk.
			hashedChunk := hex.EncodeToString(cr.chunkSHA256Writer.Sum(nil))
			// Calculate the chunk signature.
			newSignature := getChunkSignature(cr.cred.SecretKey, cr.seedSignature, cr.region, cr.seedDate, hashedChunk)
			if !compareSignatureV4(cr.chunkSignature, newSignature) {
				// Chunk signature doesn't match we return signature does not match.
				cr.err = errors.New("chunk signature does not match")
				return 0, cr.err
			}
			// Newly calculated signature becomes the seed for the next chunk
			// this follows the chaining.
			cr.seedSignature = newSignature
			cr.chunkSHA256Writer.Reset()
			if cr.lastChunk {
				cr.state = eofChunk
			} else {
				cr.state = readChunkHeader
			}
		case eofChunk:
			return n, io.EOF
		}
	}
}

// readCRLF - check if reader only has '\r\n' CRLF character.
// returns malformed encoding if it doesn't.
func readCRLF(reader io.Reader) error {
	buf := make([]byte, 2)
	_, err := io.ReadFull(reader, buf[:2])
	if err != nil {
		return err
	}
	if buf[0] != '\r' || buf[1] != '\n' {
		return errMalformedEncoding
	}
	return nil
}

// Read a line of bytes (up to \n) from b.
// Give up if the line exceeds maxLineLength.
// The returned bytes are owned by the bufio.Reader
// so they are only valid until the next bufio read.
func readChunkLine(b *bufio.Reader) ([]byte, []byte, error) {
	buf, err := b.ReadSlice('\n')
	if err != nil {
		// We always know when EOF is coming.
		// If the caller asked for a line, there should be a line.
		if err == io.EOF {
			err = io.ErrUnexpectedEOF
		} else if err == bufio.ErrBufferFull {
			err = errLineTooLong
		}
		return nil, nil, err
	}
	if len(buf) >= maxLineLength {
		return nil, nil, errLineTooLong
	}
	// Parse s3 specific chunk extension and fetch the values.
	hexChunkSize, hexChunkSignature := parseS3ChunkExtension(buf)
	return hexChunkSize, hexChunkSignature, nil
}

// trimTrailingWhitespace - trim trailing white space.
func trimTrailingWhitespace(b []byte) []byte {
	for len(b) > 0 && isASCIISpace(b[len(b)-1]) {
		b = b[:len(b)-1]
	}
	return b
}

// isASCIISpace - is ascii space?
func isASCIISpace(b byte) bool {
	return b == ' ' || b == '\t' || b == '\n' || b == '\r'
}

// Constant s3 chunk encoding signature.
const s3ChunkSignatureStr = ";chunk-signature="

// parses3ChunkExtension removes any s3 specific chunk-extension from buf.
// For example,
//     "10000;chunk-signature=..." => "10000", "chunk-signature=..."
func parseS3ChunkExtension(buf []byte) ([]byte, []byte) {
	buf = trimTrailingWhitespace(buf)
	semi := bytes.Index(buf, []byte(s3ChunkSignatureStr))
	// Chunk signature not found, return the whole buffer.
	if semi == -1 {
		return buf, nil
	}
	return buf[:semi], parseChunkSignature(buf[semi:])
}

// parseChunkSignature - parse chunk signature.
func parseChunkSignature(chunk []byte) []byte {
	chunkSplits := bytes.SplitN(chunk, []byte(s3ChunkSignatureStr), 2)
	return chunkSplits[1]
}

// parse hex to uint64.
func parseHexUint(v []byte) (n uint64, err error) {
	for i, b := range v {
		switch {
		case '0' <= b && b <= '9':
			b = b - '0'
		case 'a' <= b && b <= 'f':
			b = b - 'a' + 10
		case 'A' <= b && b <= 'F':
			b = b - 'A' + 10
		default:
			return 0, errors.New("invalid byte in chunk length")
		}
		if i == 16 {
			return 0, errors.New("http chunk length too large")
		}
		n <<= 4
		n |= uint64(b)
	}
	return
}
<commit_msg>S3 authorization: StreamingSigned enforces access control<commit_after>package s3api

// the related code is copied and modified from minio source code

/*
 * Minio Cloud Storage, (C) 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import (
	"bufio"
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
	"hash"
	"io"
	"net/http"
	"time"

	"github.com/dustin/go-humanize"
)

// getChunkSignature - get chunk signature.
func getChunkSignature(secretKey string, seedSignature string, region string, date time.Time, hashedChunk string) string {

	// Calculate string to sign.
	stringToSign := signV4ChunkedAlgorithm + "\n" +
		date.Format(iso8601Format) + "\n" +
		getScope(date, region) + "\n" +
		seedSignature + "\n" +
		emptySHA256 + "\n" +
		hashedChunk

	// Get hmac signing key.
	signingKey := getSigningKey(secretKey, date, region, "s3")

	// Calculate signature.
	newSignature := getSignature(signingKey, stringToSign)

	return newSignature
}

// calculateSeedSignature - Calculate seed signature in accordance with
//     - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
// returns signature, error otherwise if the signature mismatches or any other
// error while parsing and validating.
func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cred *Credential, signature string, region string, date time.Time, errCode s3err.ErrorCode) {

	// Copy request.
	req := *r

	// Save authorization header.
	v4Auth := req.Header.Get("Authorization")

	// Parse signature version '4' header.
	signV4Values, errCode := parseSignV4(v4Auth)
	if errCode != s3err.ErrNone {
		return nil, "", "", time.Time{}, errCode
	}

	// Payload streaming.
	payload := streamingContentSHA256

	// Payload for STREAMING signature should be 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'
	if payload != req.Header.Get("X-Amz-Content-Sha256") {
		return nil, "", "", time.Time{}, s3err.ErrContentSHA256Mismatch
	}

	// Extract all the signed headers along with its values.
	extractedSignedHeaders, errCode := extractSignedHeaders(signV4Values.SignedHeaders, r)
	if errCode != s3err.ErrNone {
		return nil, "", "", time.Time{}, errCode
	}
	// Verify if the access key id matches.
	identity, cred, found := iam.lookupByAccessKey(signV4Values.Credential.accessKey)
	if !found {
		return nil, "", "", time.Time{}, s3err.ErrInvalidAccessKeyID
	}

	bucket, _ := getBucketAndObject(r)
	if !identity.canDo("Write", bucket) {
		errCode = s3err.ErrAccessDenied
		return
	}

	// Verify if region is valid.
	region = signV4Values.Credential.scope.region

	// Extract date, if not present throw error.
	var dateStr string
	if dateStr = req.Header.Get(http.CanonicalHeaderKey("x-amz-date")); dateStr == "" {
		if dateStr = r.Header.Get("Date"); dateStr == "" {
			return nil, "", "", time.Time{}, s3err.ErrMissingDateHeader
		}
	}
	// Parse date header.
	var err error
	date, err = time.Parse(iso8601Format, dateStr)
	if err != nil {
		return nil, "", "", time.Time{}, s3err.ErrMalformedDate
	}

	// Query string.
	queryStr := req.URL.Query().Encode()

	// Get canonical request.
	canonicalRequest := getCanonicalRequest(extractedSignedHeaders, payload, queryStr, req.URL.Path, req.Method)

	// Get string to sign from canonical request.
	stringToSign := getStringToSign(canonicalRequest, date, signV4Values.Credential.getScope())

	// Get hmac signing key.
	signingKey := getSigningKey(cred.SecretKey, signV4Values.Credential.scope.date, region, "s3")

	// Calculate signature.
	newSignature := getSignature(signingKey, stringToSign)

	// Verify if signature match.
	if !compareSignatureV4(newSignature, signV4Values.Signature) {
		return nil, "", "", time.Time{}, s3err.ErrSignatureDoesNotMatch
	}

	// Return caculated signature.
	return cred, newSignature, region, date, s3err.ErrNone
}

const maxLineLength = 4 * humanize.KiByte // assumed <= bufio.defaultBufSize 4KiB

// lineTooLong is generated as chunk header is bigger than 4KiB.
var errLineTooLong = errors.New("header line too long")

// Malformed encoding is generated when chunk header is wrongly formed.
var errMalformedEncoding = errors.New("malformed chunked encoding")

// newSignV4ChunkedReader returns a new s3ChunkedReader that translates the data read from r
// out of HTTP "chunked" format before returning it.
// The s3ChunkedReader returns io.EOF when the final 0-length chunk is read.
func (iam *IdentityAccessManagement) newSignV4ChunkedReader(req *http.Request) (io.ReadCloser, s3err.ErrorCode) {
	ident, seedSignature, region, seedDate, errCode := iam.calculateSeedSignature(req)
	if errCode != s3err.ErrNone {
		return nil, errCode
	}
	return &s3ChunkedReader{
		cred:              ident,
		reader:            bufio.NewReader(req.Body),
		seedSignature:     seedSignature,
		seedDate:          seedDate,
		region:            region,
		chunkSHA256Writer: sha256.New(),
		state:             readChunkHeader,
	}, s3err.ErrNone
}

// Represents the overall state that is required for decoding a
// AWS Signature V4 chunked reader.
type s3ChunkedReader struct {
	cred              *Credential
	reader            *bufio.Reader
	seedSignature     string
	seedDate          time.Time
	region            string
	state             chunkState
	lastChunk         bool
	chunkSignature    string
	chunkSHA256Writer hash.Hash // Calculates sha256 of chunk data.
	n                 uint64    // Unread bytes in chunk
	err               error
}

// Read chunk reads the chunk token signature portion.
func (cr *s3ChunkedReader) readS3ChunkHeader() {
	// Read the first chunk line until CRLF.
	var hexChunkSize, hexChunkSignature []byte
	hexChunkSize, hexChunkSignature, cr.err = readChunkLine(cr.reader)
	if cr.err != nil {
		return
	}
	// <hex>;token=value - converts the hex into its uint64 form.
	cr.n, cr.err = parseHexUint(hexChunkSize)
	if cr.err != nil {
		return
	}
	if cr.n == 0 {
		cr.err = io.EOF
	}
	// Save the incoming chunk signature.
	cr.chunkSignature = string(hexChunkSignature)
}

type chunkState int

const (
	readChunkHeader chunkState = iota
	readChunkTrailer
	readChunk
	verifyChunk
	eofChunk
)

func (cs chunkState) String() string {
	stateString := ""
	switch cs {
	case readChunkHeader:
		stateString = "readChunkHeader"
	case readChunkTrailer:
		stateString = "readChunkTrailer"
	case readChunk:
		stateString = "readChunk"
	case verifyChunk:
		stateString = "verifyChunk"
	case eofChunk:
		stateString = "eofChunk"

	}
	return stateString
}

func (cr *s3ChunkedReader) Close() (err error) {
	return nil
}

// Read - implements `io.Reader`, which transparently decodes
// the incoming AWS Signature V4 streaming signature.
func (cr *s3ChunkedReader) Read(buf []byte) (n int, err error) {
	for {
		switch cr.state {
		case readChunkHeader:
			cr.readS3ChunkHeader()
			// If we're at the end of a chunk.
			if cr.n == 0 && cr.err == io.EOF {
				cr.state = readChunkTrailer
				cr.lastChunk = true
				continue
			}
			if cr.err != nil {
				return 0, cr.err
			}
			cr.state = readChunk
		case readChunkTrailer:
			cr.err = readCRLF(cr.reader)
			if cr.err != nil {
				return 0, errMalformedEncoding
			}
			cr.state = verifyChunk
		case readChunk:
			// There is no more space left in the request buffer.
			if len(buf) == 0 {
				return n, nil
			}
			rbuf := buf
			// The request buffer is larger than the current chunk size.
			// Read only the current chunk from the underlying reader.
			if uint64(len(rbuf)) > cr.n {
				rbuf = rbuf[:cr.n]
			}
			var n0 int
			n0, cr.err = cr.reader.Read(rbuf)
			if cr.err != nil {
				// We have lesser than chunk size advertised in chunkHeader, this is 'unexpected'.
				if cr.err == io.EOF {
					cr.err = io.ErrUnexpectedEOF
				}
				return 0, cr.err
			}

			// Calculate sha256.
			cr.chunkSHA256Writer.Write(rbuf[:n0])

			// Update the bytes read into request buffer so far.
			n += n0
			buf = buf[n0:]
			// Update bytes to be read of the current chunk before verifying chunk's signature.
			cr.n -= uint64(n0)

			// If we're at the end of a chunk.
			if cr.n == 0 {
				cr.state = readChunkTrailer
				continue
			}
		case verifyChunk:
			// Calculate the hashed chunk.
			hashedChunk := hex.EncodeToString(cr.chunkSHA256Writer.Sum(nil))
			// Calculate the chunk signature.
			newSignature := getChunkSignature(cr.cred.SecretKey, cr.seedSignature, cr.region, cr.seedDate, hashedChunk)
			if !compareSignatureV4(cr.chunkSignature, newSignature) {
				// Chunk signature doesn't match we return signature does not match.
				cr.err = errors.New("chunk signature does not match")
				return 0, cr.err
			}
			// Newly calculated signature becomes the seed for the next chunk
			// this follows the chaining.
			cr.seedSignature = newSignature
			cr.chunkSHA256Writer.Reset()
			if cr.lastChunk {
				cr.state = eofChunk
			} else {
				cr.state = readChunkHeader
			}
		case eofChunk:
			return n, io.EOF
		}
	}
}

// readCRLF - check if reader only has '\r\n' CRLF character.
// returns malformed encoding if it doesn't.
func readCRLF(reader io.Reader) error {
	buf := make([]byte, 2)
	_, err := io.ReadFull(reader, buf[:2])
	if err != nil {
		return err
	}
	if buf[0] != '\r' || buf[1] != '\n' {
		return errMalformedEncoding
	}
	return nil
}

// Read a line of bytes (up to \n) from b.
// Give up if the line exceeds maxLineLength.
// The returned bytes are owned by the bufio.Reader
// so they are only valid until the next bufio read.
func readChunkLine(b *bufio.Reader) ([]byte, []byte, error) {
	buf, err := b.ReadSlice('\n')
	if err != nil {
		// We always know when EOF is coming.
		// If the caller asked for a line, there should be a line.
		if err == io.EOF {
			err = io.ErrUnexpectedEOF
		} else if err == bufio.ErrBufferFull {
			err = errLineTooLong
		}
		return nil, nil, err
	}
	if len(buf) >= maxLineLength {
		return nil, nil, errLineTooLong
	}
	// Parse s3 specific chunk extension and fetch the values.
	hexChunkSize, hexChunkSignature := parseS3ChunkExtension(buf)
	return hexChunkSize, hexChunkSignature, nil
}

// trimTrailingWhitespace - trim trailing white space.
func trimTrailingWhitespace(b []byte) []byte {
	for len(b) > 0 && isASCIISpace(b[len(b)-1]) {
		b = b[:len(b)-1]
	}
	return b
}

// isASCIISpace - is ascii space?
func isASCIISpace(b byte) bool {
	return b == ' ' || b == '\t' || b == '\n' || b == '\r'
}

// Constant s3 chunk encoding signature.
const s3ChunkSignatureStr = ";chunk-signature="

// parses3ChunkExtension removes any s3 specific chunk-extension from buf.
// For example,
//     "10000;chunk-signature=..." => "10000", "chunk-signature=..."
func parseS3ChunkExtension(buf []byte) ([]byte, []byte) {
	buf = trimTrailingWhitespace(buf)
	semi := bytes.Index(buf, []byte(s3ChunkSignatureStr))
	// Chunk signature not found, return the whole buffer.
	if semi == -1 {
		return buf, nil
	}
	return buf[:semi], parseChunkSignature(buf[semi:])
}

// parseChunkSignature - parse chunk signature.
func parseChunkSignature(chunk []byte) []byte {
	chunkSplits := bytes.SplitN(chunk, []byte(s3ChunkSignatureStr), 2)
	return chunkSplits[1]
}

// parse hex to uint64.
func parseHexUint(v []byte) (n uint64, err error) {
	for i, b := range v {
		switch {
		case '0' <= b && b <= '9':
			b = b - '0'
		case 'a' <= b && b <= 'f':
			b = b - 'a' + 10
		case 'A' <= b && b <= 'F':
			b = b - 'A' + 10
		default:
			return 0, errors.New("invalid byte in chunk length")
		}
		if i == 16 {
			return 0, errors.New("http chunk length too large")
		}
		n <<= 4
		n |= uint64(b)
	}
	return
}
<|endoftext|>