<commit_before>package ipfix

import (
	"bytes"
	"encoding/binary"
	"errors"
	"io"
	"runtime"
)

// The version field in IPFIX messages should always have the value 10. If it
// does not, you get this error. It's probably a sign of a bug in the parser or
// the exporter and that we have lost synchronization with the data stream.
// Reestablishing the session is the only way forward at this point.
var ErrVersion = errors.New("incorrect version field in message header - out of sync?")
var ErrRead = errors.New("short read - malformed packet?")

// A Message is the top level construct representing an IPFIX message. A well
// formed message contains one or more sets of data or template information.
type Message struct {
	Header          MessageHeader
	DataRecords     []DataRecord
	TemplateRecords []TemplateRecord
}

// The MessageHeader provides metadata for the entire Message. The sequence
// number and domain ID can be used to gain knowledge of messages lost on an
// unreliable transport such as UDP.
type MessageHeader struct {
	Version        uint16 // Always 0x0a
	Length         uint16
	ExportTime     uint32 // Epoch seconds
	SequenceNumber uint32
	DomainId       uint32
}

type setHeader struct {
	SetId  uint16
	Length uint16
}

type templateHeader struct {
	TemplateId uint16
	FieldCount uint16
}

// The DataRecord represents a single exported flow. The Fields each describe
// different aspects of the flow (source and destination address, counters,
// service, etc.).
type DataRecord struct {
	TemplateId uint16
	Fields     [][]byte
}

// The TemplateRecord describes a data template, as used by DataRecords.
type TemplateRecord struct {
	TemplateId      uint16
	FieldSpecifiers []TemplateFieldSpecifier
}

// The TemplateFieldSpecifier describes the ID and size of the corresponding
// Fields in a DataRecord.
type TemplateFieldSpecifier struct {
	EnterpriseId uint32
	FieldId      uint16
	Length       uint16
}

// The Session is the context for IPFIX messages.
type Session struct {
	templates [][]TemplateFieldSpecifier
	reader    io.Reader
	minRecord []uint16
}

// NewSession initializes a new Session based on the provided io.Reader.
func NewSession(reader io.Reader) *Session {
	s := Session{}
	s.templates = make([][]TemplateFieldSpecifier, 65536)
	s.reader = reader
	s.minRecord = make([]uint16, 65536)
	return &s
}

var msgHeaderLength = binary.Size(MessageHeader{})
var setHeaderLength = binary.Size(setHeader{})
var templateHeaderLength = binary.Size(templateHeader{})

// ReadMessage extracts and returns one message from the IPFIX stream. As long
// as err is nil, further messages can be read from the stream. Errors are not
// recoverable -- once an error has been returned, ReadMessage should not be
// called again on the same session.
func (s *Session) ReadMessage() (msg *Message, err error) {
	defer func() {
		if r := recover(); r != nil {
			if _, ok := r.(runtime.Error); ok {
				panic(r)
			}

			msg = nil
			err = r.(error)
		}
	}()

	msg = &Message{}
	msg.DataRecords = make([]DataRecord, 0)
	msg.TemplateRecords = make([]TemplateRecord, 0)

	msgHdr := MessageHeader{}
	err = binary.Read(s.reader, binary.BigEndian, &msgHdr)
	errorIf(err)
	if msgHdr.Version != 10 {
		errorIf(ErrVersion)
	}
	msg.Header = msgHdr

	msgSlice := make([]byte, int(msgHdr.Length)-msgHeaderLength)
	io.ReadFull(s.reader, msgSlice)
	r := bytes.NewBuffer(msgSlice)

	for r.Len() > 0 {
		trecs, drecs := s.readSet(r)
		msg.TemplateRecords = append(msg.TemplateRecords, trecs...)
		msg.DataRecords = append(msg.DataRecords, drecs...)
	}

	return
}

func (s *Session) readSet(r *bytes.Buffer) (trecs []TemplateRecord, drecs []DataRecord) {
	trecs = make([]TemplateRecord, 0)
	drecs = make([]DataRecord, 0)

	setHdr := setHeader{}
	err := binary.Read(r, binary.BigEndian, &setHdr)
	errorIf(err)

	for r.Len() > 0 {
		if r.Len() < int(s.minRecord[setHdr.SetId]) {
			// Padding
			return
		} else if setHdr.SetId == 2 {
			// Template Set
			ts := s.readTemplateRecord(r)
			trecs = append(trecs, *ts)

			// Update the template cache
			tid := ts.TemplateId
			if len(ts.FieldSpecifiers) == 0 {
				// Set was withdrawn
				s.templates[tid] = nil
			} else {
				s.templates[tid] = ts.FieldSpecifiers
			}

			// Update the minimum record length cache
			var minLength uint16
			for i := range ts.FieldSpecifiers {
				if ts.FieldSpecifiers[i].Length == 65535 {
					minLength += 1
				} else {
					minLength += ts.FieldSpecifiers[i].Length
				}
			}
			s.minRecord[tid] = minLength
		} else if setHdr.SetId == 3 {
			// Options Template Set, not handled
			r.Read(make([]byte, int(setHdr.Length)-setHeaderLength))
		} else {
			if tpl := s.templates[setHdr.SetId]; tpl != nil {
				// Data set
				ds := s.readDataRecord(r, tpl)
				ds.TemplateId = setHdr.SetId
				drecs = append(drecs, *ds)
			} else {
				// Data set with unknown template
				// We can't trust set length, because we might be out of sync.
				// Consume rest of message.
				r.Read(make([]byte, r.Len()))
				return
			}
		}
	}

	return
}

func (s *Session) readDataRecord(r *bytes.Buffer, tpl []TemplateFieldSpecifier) *DataRecord {
	ds := DataRecord{}
	ds.Fields = make([][]byte, len(tpl))

	for i := range tpl {
		var bs []byte
		if tpl[i].Length == 65535 {
			bs = s.readVariableLength(r)
		} else {
			bs = s.readFixedLength(r, int(tpl[i].Length))
		}
		ds.Fields[i] = bs
	}

	return &ds
}

func (s *Session) readTemplateRecord(r *bytes.Buffer) *TemplateRecord {
	ts := TemplateRecord{}
	th := templateHeader{}
	err := binary.Read(r, binary.BigEndian, &th)
	errorIf(err)

	ts.TemplateId = th.TemplateId
	ts.FieldSpecifiers = make([]TemplateFieldSpecifier, th.FieldCount)
	for i := 0; i < int(th.FieldCount); i++ {
		f := TemplateFieldSpecifier{}
		err = binary.Read(r, binary.BigEndian, &f.FieldId)
		errorIf(err)
		err = binary.Read(r, binary.BigEndian, &f.Length)
		errorIf(err)
		if f.FieldId >= 0x8000 {
			f.FieldId -= 0x8000
			err = binary.Read(r, binary.BigEndian, &f.EnterpriseId)
			errorIf(err)
		}
		ts.FieldSpecifiers[i] = f
	}

	return &ts
}

func (s *Session) readFixedLength(r *bytes.Buffer, n int) []byte {
	bs := make([]byte, n)
	n1, err := r.Read(bs)
	errorIf(err)
	if n1 != n {
		panic(ErrRead)
	}
	return bs
}

func (s *Session) readVariableLength(r *bytes.Buffer) []byte {
	var l int

	var l0 uint8
	err := binary.Read(r, binary.BigEndian, &l0)
	errorIf(err)

	if l0 < 255 {
		l = int(l0)
	} else {
		var l1 uint16
		err := binary.Read(r, binary.BigEndian, &l1)
		errorIf(err)
		l = int(l1)
	}

	return s.readFixedLength(r, l)
}

func errorIf(err error) {
	if err != nil {
		panic(err)
	}
}
<commit_msg>Correctly handle multiple sets per packet<commit_after>package ipfix

import (
	"bytes"
	"encoding/binary"
	"errors"
	"io"
	"runtime"
)

// The version field in IPFIX messages should always have the value 10. If it
// does not, you get this error. It's probably a sign of a bug in the parser or
// the exporter and that we have lost synchronization with the data stream.
// Reestablishing the session is the only way forward at this point.
var ErrVersion = errors.New("incorrect version field in message header - out of sync?")
var ErrRead = errors.New("short read - malformed packet?")

// A Message is the top level construct representing an IPFIX message. A well
// formed message contains one or more sets of data or template information.
type Message struct {
	Header          MessageHeader
	DataRecords     []DataRecord
	TemplateRecords []TemplateRecord
}

// The MessageHeader provides metadata for the entire Message. The sequence
// number and domain ID can be used to gain knowledge of messages lost on an
// unreliable transport such as UDP.
type MessageHeader struct {
	Version        uint16 // Always 0x0a
	Length         uint16
	ExportTime     uint32 // Epoch seconds
	SequenceNumber uint32
	DomainId       uint32
}

type setHeader struct {
	SetId  uint16
	Length uint16
}

type templateHeader struct {
	TemplateId uint16
	FieldCount uint16
}

// The DataRecord represents a single exported flow. The Fields each describe
// different aspects of the flow (source and destination address, counters,
// service, etc.).
type DataRecord struct {
	TemplateId uint16
	Fields     [][]byte
}

// The TemplateRecord describes a data template, as used by DataRecords.
type TemplateRecord struct {
	TemplateId      uint16
	FieldSpecifiers []TemplateFieldSpecifier
}

// The TemplateFieldSpecifier describes the ID and size of the corresponding
// Fields in a DataRecord.
type TemplateFieldSpecifier struct {
	EnterpriseId uint32
	FieldId      uint16
	Length       uint16
}

// The Session is the context for IPFIX messages.
type Session struct {
	templates [][]TemplateFieldSpecifier
	reader    io.Reader
	minRecord []uint16
}

// NewSession initializes a new Session based on the provided io.Reader.
func NewSession(reader io.Reader) *Session {
	s := Session{}
	s.templates = make([][]TemplateFieldSpecifier, 65536)
	s.reader = reader
	s.minRecord = make([]uint16, 65536)
	return &s
}

var msgHeaderLength = binary.Size(MessageHeader{})
var setHeaderLength = binary.Size(setHeader{})
var templateHeaderLength = binary.Size(templateHeader{})

// ReadMessage extracts and returns one message from the IPFIX stream. As long
// as err is nil, further messages can be read from the stream. Errors are not
// recoverable -- once an error has been returned, ReadMessage should not be
// called again on the same session.
func (s *Session) ReadMessage() (msg *Message, err error) {
	defer func() {
		if r := recover(); r != nil {
			if _, ok := r.(runtime.Error); ok {
				panic(r)
			}

			msg = nil
			err = r.(error)
		}
	}()

	msg = &Message{}
	msg.DataRecords = make([]DataRecord, 0)
	msg.TemplateRecords = make([]TemplateRecord, 0)

	msgHdr := MessageHeader{}
	err = binary.Read(s.reader, binary.BigEndian, &msgHdr)
	errorIf(err)
	if msgHdr.Version != 10 {
		errorIf(ErrVersion)
	}
	msg.Header = msgHdr

	msgLen := int(msgHdr.Length) - msgHeaderLength
	msgSlice := make([]byte, msgLen)
	_, err = io.ReadFull(s.reader, msgSlice)
	errorIf(err)
	r := bytes.NewBuffer(msgSlice)

	for r.Len() > 0 {
		trecs, drecs := s.readSet(r)
		msg.TemplateRecords = append(msg.TemplateRecords, trecs...)
		msg.DataRecords = append(msg.DataRecords, drecs...)
	}

	return
}

func (s *Session) readSet(r *bytes.Buffer) (trecs []TemplateRecord, drecs []DataRecord) {
	trecs = make([]TemplateRecord, 0)
	drecs = make([]DataRecord, 0)

	setHdr := setHeader{}
	err := binary.Read(r, binary.BigEndian, &setHdr)
	setEnd := r.Len() - int(setHdr.Length) + setHeaderLength
	errorIf(err)

	for r.Len() > setEnd {
		if r.Len()-setEnd < int(s.minRecord[setHdr.SetId]) {
			// Padding
			return
		} else if setHdr.SetId == 2 {
			// Template Set
			ts := s.readTemplateRecord(r)
			trecs = append(trecs, *ts)

			// Update the template cache
			tid := ts.TemplateId
			if len(ts.FieldSpecifiers) == 0 {
				// Set was withdrawn
				s.templates[tid] = nil
			} else {
				s.templates[tid] = ts.FieldSpecifiers
			}

			// Update the minimum record length cache
			var minLength uint16
			for i := range ts.FieldSpecifiers {
				if ts.FieldSpecifiers[i].Length == 65535 {
					minLength += 1
				} else {
					minLength += ts.FieldSpecifiers[i].Length
				}
			}
			s.minRecord[tid] = minLength
		} else if setHdr.SetId == 3 {
			// Options Template Set, not handled
			r.Read(make([]byte, int(setHdr.Length)-setHeaderLength))
		} else {
			if tpl := s.templates[setHdr.SetId]; tpl != nil {
				// Data set
				ds := s.readDataRecord(r, tpl)
				ds.TemplateId = setHdr.SetId
				drecs = append(drecs, *ds)
			} else {
				// Data set with unknown template
				// We can't trust set length, because we might be out of sync.
				// Consume rest of message.
				r.Read(make([]byte, r.Len()))
				return
			}
		}
	}

	return
}

func (s *Session) readDataRecord(r *bytes.Buffer, tpl []TemplateFieldSpecifier) *DataRecord {
	ds := DataRecord{}
	ds.Fields = make([][]byte, len(tpl))

	for i := range tpl {
		var bs []byte
		if tpl[i].Length == 65535 {
			bs = s.readVariableLength(r)
		} else {
			bs = s.readFixedLength(r, int(tpl[i].Length))
		}
		ds.Fields[i] = bs
	}

	return &ds
}

func (s *Session) readTemplateRecord(r *bytes.Buffer) *TemplateRecord {
	ts := TemplateRecord{}
	th := templateHeader{}
	err := binary.Read(r, binary.BigEndian, &th)
	errorIf(err)

	ts.TemplateId = th.TemplateId
	ts.FieldSpecifiers = make([]TemplateFieldSpecifier, th.FieldCount)
	for i := 0; i < int(th.FieldCount); i++ {
		f := TemplateFieldSpecifier{}
		err = binary.Read(r, binary.BigEndian, &f.FieldId)
		errorIf(err)
		err = binary.Read(r, binary.BigEndian, &f.Length)
		errorIf(err)
		if f.FieldId >= 0x8000 {
			f.FieldId -= 0x8000
			err = binary.Read(r, binary.BigEndian, &f.EnterpriseId)
			errorIf(err)
		}
		ts.FieldSpecifiers[i] = f
	}

	return &ts
}

func (s *Session) readFixedLength(r *bytes.Buffer, n int) []byte {
	bs := make([]byte, n)
	n1, err := r.Read(bs)
	errorIf(err)
	if n1 != n {
		panic(ErrRead)
	}
	return bs
}

func (s *Session) readVariableLength(r *bytes.Buffer) []byte {
	var l int

	var l0 uint8
	err := binary.Read(r, binary.BigEndian, &l0)
	errorIf(err)

	if l0 < 255 {
		l = int(l0)
	} else {
		var l1 uint16
		err := binary.Read(r, binary.BigEndian, &l1)
		errorIf(err)
		l = int(l1)
	}

	return s.readFixedLength(r, l)
}

func errorIf(err error) {
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package lib

import (
    "fmt"
    "os"
    "time"

    . "gopkg.in/check.v1"
)

func (s *OssutilCommandSuite) rawList(args []string, shortFormat, directory bool) (bool, error) {
    command := "ls"
    str := ""
    options := OptionMapType{
        "endpoint": &str,
        "accessKeyID": &str,
        "accessKeySecret": &str,
        "stsToken": &str,
        "configFile": &configFile,
        "shortFormat": &shortFormat,
        "directory": &directory,
    }
    showElapse, err := cm.RunCommand(command, args, options)
    return showElapse, err
}

// test list buckets
func (s *OssutilCommandSuite) TestListLoadConfig(c *C) {
    command := "ls"
    var args []string
    str := ""
    options := OptionMapType{
        "endpoint": &str,
        "accessKeyID": &str,
        "accessKeySecret": &str,
        "stsToken": &str,
        "configFile": &configFile,
    }
    showElapse, err := cm.RunCommand(command, args, options)
    c.Assert(err, IsNil)
    c.Assert(showElapse, Equals, true)

    args = []string{"oss://"}
    options = OptionMapType{
        "endpoint": &str,
        "accessKeyID": &str,
        "accessKeySecret": &str,
        "stsToken": &str,
        "configFile": &configFile,
    }
    showElapse, err = cm.RunCommand(command, args, options)
    c.Assert(err, IsNil)
    c.Assert(showElapse, Equals, true)
}

func (s *OssutilCommandSuite) TestListNotExistConfigFile(c *C) {
    command := "ls"
    var args []string
    str := ""
    cfile := "notexistfile"
    options := OptionMapType{
        "endpoint": &str,
        "accessKeyID": &str,
        "accessKeySecret": &str,
        "stsToken": &str,
        "configFile": &cfile,
    }
    showElapse, err := cm.RunCommand(command, args, options)
    c.Assert(err, NotNil)
    c.Assert(showElapse, Equals, false)
}

func (s *OssutilCommandSuite) TestListErrConfigFile(c *C) {
    cfile := "ossutil_test.config_boto"
    s.createFile(cfile, content, c)

    command := "ls"
    var args []string
    str := ""
    options := OptionMapType{
        "endpoint": &str,
        "accessKeyID": &str,
        "accessKeySecret": &str,
        "stsToken": &str,
        "configFile": &cfile,
    }
    showElapse, err := cm.RunCommand(command, args, options)
    c.Assert(err, NotNil)
    c.Assert(showElapse, Equals, false)

    _ = os.Remove(cfile)
}

func (s *OssutilCommandSuite) TestListConfigFile(c *C) {
    cfile := "ossutil_test.config_boto"
    data := fmt.Sprintf("[Credentials]\nendpoint=%s\naccessKeyID=%s\naccessKeySecret=%s\nretryTimes=%s", endpoint, accessKeyID, accessKeySecret, "errretry")
    s.createFile(cfile, data, c)

    command := "ls"
    var args []string
    str := ""
    options := OptionMapType{
        "endpoint": &str,
        "accessKeyID": &str,
        "accessKeySecret": &str,
        "stsToken": &str,
        "configFile": &cfile,
    }
    showElapse, err := cm.RunCommand(command, args, options)
    c.Assert(err, IsNil)
    c.Assert(showElapse, Equals, true)

    _ = os.Remove(cfile)
}

func (s *OssutilCommandSuite) TestListWithBucketEndpoint(c *C) {
    bucket := bucketNameExist

    cfile := "ossutil_test.config_boto"
    data := fmt.Sprintf("[Credentials]\nendpoint=%s\naccessKeyID=%s\naccessKeySecret=%s\n[Bucket-Endpoint]\n%s=%s", "abc", accessKeyID, accessKeySecret, bucket, endpoint)
    s.createFile(cfile, data, c)

    command := "ls"
    args := []string{CloudURLToString(bucket, "")}
    str := ""
    options := OptionMapType{
        "endpoint": &str,
        "accessKeyID": &str,
        "accessKeySecret": &str,
        "stsToken": &str,
        "configFile": &cfile,
    }
    showElapse, err := cm.RunCommand(command, args, options)
    c.Assert(err, IsNil)
    c.Assert(showElapse, Equals, true)

    _ = os.Remove(cfile)
}

func (s *OssutilCommandSuite) TestListWithBucketCname(c *C) {
    bucket := bucketNamePrefix + "ls1"
    s.putBucket(bucket, c)

    cfile := "ossutil_test.config_boto"
    data := fmt.Sprintf("[Credentials]\nendpoint=%s\naccessKeyID=%s\naccessKeySecret=%s\n[Bucket-Endpoint]\n%s=%s\n[Bucket-Cname]\n%s=%s", "abc", accessKeyID, accessKeySecret, bucket, "abc", bucket, bucket + "." + endpoint)
    s.createFile(cfile, data, c)

    command := "ls"
    args := []string{CloudURLToString(bucket, "")}
    str := ""
    options := OptionMapType{
        "endpoint": &str,
        "accessKeyID": &str,
        "accessKeySecret": &str,
        "stsToken": &str,
        "configFile": &cfile,
    }
    showElapse, err := cm.RunCommand(command, args, options)
    c.Assert(err, IsNil)
    c.Assert(showElapse, Equals, true)

    _ = os.Remove(cfile)
    s.removeBucket(bucket, true, c)
    time.Sleep(7*time.Second)
}

func (s *OssutilCommandSuite) TestListBuckets(c *C) {
    // "ls"
    bucket := bucketNamePrefix + "ls2"
    s.removeBucket(bucket, true, c)
    time.Sleep(3*7*time.Second)

    buckets := s.listBuckets(false, c)
    c.Assert(FindPos(bucket, buckets) == -1, Equals, true)
    bucketNum := len(buckets)

    // "ls -s"
    buckets = s.listBuckets(true, c)
    c.Assert(FindPos(bucket, buckets) == -1, Equals, true)

    // put bucket
    s.putBucket(bucket, c)
    time.Sleep(14*time.Second)

    // get result
    buckets = s.listBuckets(false, c)
    c.Assert(FindPos(bucket, buckets) != -1, Equals, true)
    c.Assert(len(buckets) >= bucketNum, Equals, true)

    // remove bucket
    s.removeBucket(bucket, true, c)
    time.Sleep(7*time.Second)

    // get result
    buckets = s.listBuckets(false, c)
    c.Assert(FindPos(bucket, buckets) == -1, Equals, true)
}

// list objects with not exist bucket
func (s *OssutilCommandSuite) TestListObjectsBucketNotExist(c *C) {
    bucket := bucketNameNotExist
    command := "ls"
    args := []string{CloudURLToString(bucket, "")}
    str := ""
    options := OptionMapType{
        "endpoint": &str,
        "accessKeyID": &str,
        "accessKeySecret": &str,
        "stsToken": &str,
        "configFile": &configFile,
    }
    showElapse, err := cm.RunCommand(command, args, options)
    c.Assert(err, NotNil)
    c.Assert(showElapse, Equals, false)
}

// list objects
func (s *OssutilCommandSuite) TestListObjects(c *C) {
    bucket := bucketNameList
    s.removeObjects(bucket, "", true, true, c)
    time.Sleep(3*sleepTime)

    // "ls oss://bucket"
    objects := s.listObjects(bucket, "", false, false, c)
    c.Assert(len(objects), Equals, 0)

    // put objects
    num := 10
    objectNames := []string{}
    for i := 0; i < num; i++ {
        object := fmt.Sprintf("lstest:#%d", i)
        s.putObject(bucket, object, uploadFileName, c)
        objectNames = append(objectNames, object)
    }

    object := "another_object"
    s.putObject(bucket, object, uploadFileName, c)
    objectNames = append(objectNames, object)

    // "ls oss://bucket -s"
    objects = s.listObjects(bucket, "", true, false, c)
    c.Assert(len(objects), Equals, len(objectNames))

    // "ls oss://bucket/prefix -s"
    objects = s.listObjects(bucket, "lstest:", true, false, c)
    c.Assert(len(objects), Equals, len(objectNames) - 1)

    //put directories
    num = 5
    objectNames = []string{}
    for i := 0; i < num; i++ {
        object := fmt.Sprintf("lstest:#%d/", i)
        s.putObject(bucket, object, uploadFileName, c)

        object = fmt.Sprintf("lstest:#%d/%d/", i, i)
        s.putObject(bucket, object, uploadFileName, c)
        objectNames = append(objectNames, object)
    }

    // "ls oss://bucket/prefix"
    objects = s.listObjects(bucket, "lstest:#", false, false, c)
    c.Assert(len(objects), Equals, 20)

    // "ls oss://bucket/prefix -d"
    objects = s.listObjects(bucket, "lstest:#", false, true, c)
    c.Assert(len(objects), Equals, 15)

    objects = s.listObjects(bucket, "lstest:#1/", false, true, c)
    c.Assert(len(objects), Equals, 2)
}

func (s *OssutilCommandSuite) TestErrList(c *C) {
    showElapse, err := s.rawList([]string{"../"}, true, false)
    c.Assert(err, NotNil)
    c.Assert(showElapse, Equals, false)

    bucket := bucketNameNotExist
    showElapse, err = s.rawList([]string{CloudURLToString(bucket, "")}, false, true)
    c.Assert(err, NotNil)
    c.Assert(showElapse, Equals, false)

    // list buckets with -d
    showElapse, err = s.rawList([]string{"oss://"}, false, true)
    c.Assert(err, NotNil)
    c.Assert(showElapse, Equals, false)
}

func (s *OssutilCommandSuite) TestListIDKey(c *C) {
    bucket := bucketNamePrefix + "lsidkey"

    cfile := "ossutil_test.config_boto"
    data := fmt.Sprintf("[Credentials]\nendpoint=%s\naccessKeyID=%s\naccessKeySecret=%s\n[Bucket-Endpoint]\n%s=%s[Bucket-Cname]\n%s=%s", "abc", "def", "ghi", bucket, "abc", bucket, "abc")
    s.createFile(cfile, data, c)

    command := "ls"
    str := ""
    args := []string{"oss://"}
    options := OptionMapType{
        "endpoint": &str,
        "accessKeyID": &str,
        "accessKeySecret": &str,
        "stsToken": &str,
        "configFile": &cfile,
    }
    showElapse, err := cm.RunCommand(command, args, options)
    c.Assert(err, NotNil)

    options = OptionMapType{
        "endpoint": &endpoint,
        "accessKeyID": &accessKeyID,
        "accessKeySecret": &accessKeySecret,
        "stsToken": &str,
        "configFile": &cfile,
    }
    showElapse, err = cm.RunCommand(command, args, options)
    c.Assert(err, IsNil)
    c.Assert(showElapse, Equals, true)

    _ = os.Remove(cfile)
}

func (s *OssutilCommandSuite) TestListBucketIDKey(c *C) {
    bucket := bucketNameExist

    cfile := "ossutil_test.config_boto"
    data := fmt.Sprintf("[Credentials]\nendpoint=%s\naccessKeyID=%s\naccessKeySecret=%s\n[Bucket-Endpoint]\n%s=%s[Bucket-Cname]\n%s=%s", "abc", "def", "ghi", bucket, "abc", bucket, "abc")
    s.createFile(cfile, data, c)

    command := "ls"
    str := ""
    args := []string{CloudURLToString(bucket, "")}
    options := OptionMapType{
        "endpoint": &str,
        "accessKeyID": &str,
        "accessKeySecret": &str,
        "stsToken": &str,
        "configFile": &cfile,
    }
    showElapse, err := cm.RunCommand(command, args, options)
    c.Assert(err, NotNil)

    options = OptionMapType{
        "endpoint": &endpoint,
        "accessKeyID": &accessKeyID,
        "accessKeySecret": &accessKeySecret,
        "stsToken": &str,
        "configFile": &cfile,
    }
    showElapse, err = cm.RunCommand(command, args, options)
    c.Assert(err, IsNil)
    c.Assert(showElapse, Equals, true)

    _ = os.Remove(cfile)
}
<commit_msg>modify case<commit_after>package lib

import (
    "fmt"
    "os"
    "time"

    . "gopkg.in/check.v1"
)

func (s *OssutilCommandSuite) rawList(args []string, shortFormat, directory bool) (bool, error) {
    command := "ls"
    str := ""
    options := OptionMapType{
        "endpoint": &str,
        "accessKeyID": &str,
        "accessKeySecret": &str,
        "stsToken": &str,
        "configFile": &configFile,
        "shortFormat": &shortFormat,
        "directory": &directory,
    }
    showElapse, err := cm.RunCommand(command, args, options)
    return showElapse, err
}

// test list buckets
func (s *OssutilCommandSuite) TestListLoadConfig(c *C) {
    command := "ls"
    var args []string
    str := ""
    options := OptionMapType{
        "endpoint": &str,
        "accessKeyID": &str,
        "accessKeySecret": &str,
        "stsToken": &str,
        "configFile": &configFile,
    }
    showElapse, err := cm.RunCommand(command, args, options)
    c.Assert(err, IsNil)
    c.Assert(showElapse, Equals, true)

    args = []string{"oss://"}
    options = OptionMapType{
        "endpoint": &str,
        "accessKeyID": &str,
        "accessKeySecret": &str,
        "stsToken": &str,
        "configFile": &configFile,
    }
    showElapse, err = cm.RunCommand(command, args, options)
    c.Assert(err, IsNil)
    c.Assert(showElapse, Equals, true)
}

func (s *OssutilCommandSuite) TestListNotExistConfigFile(c *C) {
    command := "ls"
    var args []string
    str := ""
    cfile := "notexistfile"
    options := OptionMapType{
        "endpoint": &str,
        "accessKeyID": &str,
        "accessKeySecret": &str,
        "stsToken": &str,
        "configFile": &cfile,
    }
    showElapse, err := cm.RunCommand(command, args, options)
    c.Assert(err, NotNil)
    c.Assert(showElapse, Equals, false)
}

func (s *OssutilCommandSuite) TestListErrConfigFile(c *C) {
    cfile := "ossutil_test.config_boto"
    s.createFile(cfile, content, c)

    command := "ls"
    var args []string
    str := ""
    options := OptionMapType{
        "endpoint": &str,
        "accessKeyID": &str,
        "accessKeySecret": &str,
        "stsToken": &str,
        "configFile": &cfile,
    }
    showElapse, err := cm.RunCommand(command, args, options)
    c.Assert(err, NotNil)
    c.Assert(showElapse, Equals, false)

    _ = os.Remove(cfile)
}

func (s *OssutilCommandSuite) TestListConfigFile(c *C) {
    cfile := "ossutil_test.config_boto"
    data := fmt.Sprintf("[Credentials]\nendpoint=%s\naccessKeyID=%s\naccessKeySecret=%s\nretryTimes=%s", endpoint, accessKeyID, accessKeySecret, "errretry")
    s.createFile(cfile, data, c)

    command := "ls"
    var args []string
    str := ""
    options := OptionMapType{
        "endpoint": &str,
        "accessKeyID": &str,
        "accessKeySecret": &str,
        "stsToken": &str,
        "configFile": &cfile,
    }
    showElapse, err := cm.RunCommand(command, args, options)
    c.Assert(err, IsNil)
    c.Assert(showElapse, Equals, true)

    _ = os.Remove(cfile)
}

func (s *OssutilCommandSuite) TestListWithBucketEndpoint(c *C) {
    bucket := bucketNameExist

    cfile := "ossutil_test.config_boto"
    data := fmt.Sprintf("[Credentials]\nendpoint=%s\naccessKeyID=%s\naccessKeySecret=%s\n[Bucket-Endpoint]\n%s=%s", "abc", accessKeyID, accessKeySecret, bucket, endpoint)
    s.createFile(cfile, data, c)

    command := "ls"
    args := []string{CloudURLToString(bucket, "")}
    str := ""
    options := OptionMapType{
        "endpoint": &str,
        "accessKeyID": &str,
        "accessKeySecret": &str,
        "stsToken": &str,
        "configFile": &cfile,
    }
    showElapse, err := cm.RunCommand(command, args, options)
    c.Assert(err, IsNil)
    c.Assert(showElapse, Equals, true)

    _ = os.Remove(cfile)
}

func (s *OssutilCommandSuite) TestListWithBucketCname(c *C) {
    bucket := bucketNamePrefix + "ls1"
    s.putBucket(bucket, c)

    cfile := "ossutil_test.config_boto"
    data := fmt.Sprintf("[Credentials]\nendpoint=%s\naccessKeyID=%s\naccessKeySecret=%s\n[Bucket-Endpoint]\n%s=%s\n[Bucket-Cname]\n%s=%s", "abc", accessKeyID, accessKeySecret, bucket, "abc", bucket, bucket + "." + endpoint)
    s.createFile(cfile, data, c)

    command := "ls"
    args := []string{CloudURLToString(bucket, "")}
    str := ""
    options := OptionMapType{
        "endpoint": &str,
        "accessKeyID": &str,
        "accessKeySecret": &str,
        "stsToken": &str,
        "configFile": &cfile,
    }
    showElapse, err := cm.RunCommand(command, args, options)
    c.Assert(err, IsNil)
    c.Assert(showElapse, Equals, true)

    _ = os.Remove(cfile)
    s.removeBucket(bucket, true, c)
    time.Sleep(7*time.Second)
}

func (s *OssutilCommandSuite) TestListBuckets(c *C) {
    // "ls"
    bucket := bucketNamePrefix + "ls2"
    s.removeBucket(bucket, true, c)
    time.Sleep(21*time.Second)

    buckets := s.listBuckets(false, c)
    c.Assert(FindPos(bucket, buckets) == -1, Equals, true)
    bucketNum := len(buckets)

    // "ls -s"
    buckets = s.listBuckets(true, c)
    c.Assert(FindPos(bucket, buckets) == -1, Equals, true)

    // put bucket
    s.putBucket(bucket, c)
    time.Sleep(14*time.Second)

    // get result
    buckets = s.listBuckets(false, c)
    c.Assert(FindPos(bucket, buckets) != -1, Equals, true)
    c.Assert(len(buckets) >= bucketNum, Equals, true)

    // remove bucket
    s.removeBucket(bucket, true, c)
    time.Sleep(7*time.Second)

    // get result
    buckets = s.listBuckets(false, c)
    c.Assert(FindPos(bucket, buckets) == -1, Equals, true)
}

// list objects with not exist bucket
func (s *OssutilCommandSuite) TestListObjectsBucketNotExist(c *C) {
    bucket := bucketNameNotExist
    command := "ls"
    args := []string{CloudURLToString(bucket, "")}
    str := ""
    options := OptionMapType{
        "endpoint": &str,
        "accessKeyID": &str,
        "accessKeySecret": &str,
        "stsToken": &str,
        "configFile": &configFile,
    }
    showElapse, err := cm.RunCommand(command, args, options)
    c.Assert(err, NotNil)
    c.Assert(showElapse, Equals, false)
}

// list objects
func (s *OssutilCommandSuite) TestListObjects(c *C) {
    bucket := bucketNameList
    s.removeObjects(bucket, "", true, true, c)
    time.Sleep(3*sleepTime)

    // "ls oss://bucket"
    objects := s.listObjects(bucket, "", false, false, c)
    c.Assert(len(objects), Equals, 0)

    // put objects
    num := 10
    objectNames := []string{}
    for i := 0; i < num; i++ {
        object := fmt.Sprintf("lstest:#%d", i)
        s.putObject(bucket, object, uploadFileName, c)
        objectNames = append(objectNames, object)
    }

    object := "another_object"
    s.putObject(bucket, object, uploadFileName, c)
    objectNames = append(objectNames, object)

    // "ls oss://bucket -s"
    objects = s.listObjects(bucket, "", true, false, c)
    c.Assert(len(objects), Equals, len(objectNames))

    // "ls oss://bucket/prefix -s"
    objects = s.listObjects(bucket, "lstest:", true, false, c)
    c.Assert(len(objects), Equals, len(objectNames) - 1)

    //put directories
    num = 5
    objectNames = []string{}
    for i := 0; i < num; i++ {
        object := fmt.Sprintf("lstest:#%d/", i)
        s.putObject(bucket, object, uploadFileName, c)

        object = fmt.Sprintf("lstest:#%d/%d/", i, i)
        s.putObject(bucket, object, uploadFileName, c)
        objectNames = append(objectNames, object)
    }

    // "ls oss://bucket/prefix"
    objects = s.listObjects(bucket, "lstest:#", false, false, c)
    c.Assert(len(objects), Equals, 20)

    // "ls oss://bucket/prefix -d"
    objects = s.listObjects(bucket, "lstest:#", false, true, c)
    c.Assert(len(objects), Equals, 15)

    objects = s.listObjects(bucket, "lstest:#1/", false, true, c)
    c.Assert(len(objects), Equals, 2)
}

func (s *OssutilCommandSuite) TestErrList(c *C) {
    showElapse, err := s.rawList([]string{"../"}, true, false)
    c.Assert(err, NotNil)
    c.Assert(showElapse, Equals, false)

    bucket := bucketNameNotExist
    showElapse, err = s.rawList([]string{CloudURLToString(bucket, "")}, false, true)
    c.Assert(err, NotNil)
    c.Assert(showElapse, Equals, false)

    // list buckets with -d
    showElapse, err = s.rawList([]string{"oss://"}, false, true)
    c.Assert(err, NotNil)
    c.Assert(showElapse, Equals, false)
}

func (s *OssutilCommandSuite) TestListIDKey(c *C) {
    bucket := bucketNamePrefix + "lsidkey"

    cfile := "ossutil_test.config_boto"
    data := fmt.Sprintf("[Credentials]\nendpoint=%s\naccessKeyID=%s\naccessKeySecret=%s\n[Bucket-Endpoint]\n%s=%s[Bucket-Cname]\n%s=%s", "abc", "def", "ghi", bucket, "abc", bucket, "abc")
    s.createFile(cfile, data, c)

    command := "ls"
    str := ""
    args := []string{"oss://"}
    options := OptionMapType{
        "endpoint": &str,
        "accessKeyID": &str,
        "accessKeySecret": &str,
        "stsToken": &str,
        "configFile": &cfile,
    }
    showElapse, err := cm.RunCommand(command, args, options)
    c.Assert(err, NotNil)

    options = OptionMapType{
        "endpoint": &endpoint,
        "accessKeyID": &accessKeyID,
        "accessKeySecret": &accessKeySecret,
        "stsToken": &str,
        "configFile": &cfile,
    }
    showElapse, err = cm.RunCommand(command, args, options)
    c.Assert(err, IsNil)
    c.Assert(showElapse, Equals, true)

    _ = os.Remove(cfile)
}

func (s *OssutilCommandSuite) TestListBucketIDKey(c *C) {
    bucket := bucketNameExist

    cfile := "ossutil_test.config_boto"
    data := fmt.Sprintf("[Credentials]\nendpoint=%s\naccessKeyID=%s\naccessKeySecret=%s\n[Bucket-Endpoint]\n%s=%s[Bucket-Cname]\n%s=%s", "abc", "def", "ghi", bucket, "abc", bucket, "abc")
    s.createFile(cfile, data, c)

    command := "ls"
    str := ""
    args := []string{CloudURLToString(bucket, "")}
    options := OptionMapType{
        "endpoint": &str,
        "accessKeyID": &str,
        "accessKeySecret": &str,
        "stsToken": &str,
        "configFile": &cfile,
    }
    showElapse, err := cm.RunCommand(command, args, options)
    c.Assert(err, NotNil)

    options = OptionMapType{
        "endpoint": &endpoint,
        "accessKeyID": &accessKeyID,
        "accessKeySecret": &accessKeySecret,
        "stsToken": &str,
        "configFile": &cfile,
    }
    showElapse, err = cm.RunCommand(command, args, options)
    c.Assert(err, IsNil)
    c.Assert(showElapse, Equals, true)

    _ = os.Remove(cfile)
}
<|endoftext|>


<commit_before>// +build windows

package pb

func bold(str string) string {
    return str
}

func terminalWidth() (int, error) {
    return 80, nil
}
<commit_msg>Hardcode fix for windows<commit_after>// +build windows

package pb

func bold(str string) string {
    return str
}

func terminalWidth() (int, error) {
    return 81, nil
}
<|endoftext|>


<commit_before>// git.go
package translation

import (
	"bitbucket.org/kardianos/rsync"
	"github.com/libgit2/git2go"
	"github.com/rgeorgiev583/gonflator/remote"
)

const (
	chanCap  = 1000
	sliceCap = 1000
)

type SyntheticRemote struct {
	URL      url.URL
	Protocol remote.Protocol
}

type SyntheticRemoteCollection map[string]*SyntheticRemote

type GitRepository struct {
	git2go.Repository
	Tree             ConfigurationTree
	SyntheticRemotes SyntheticRemoteCollection
}

func (gr *GitRepository) GetRdiff(diff chan<- git2go.DiffDelta) (rdiff <-chan Delta, err error) {
	deltas := make(chan Delta, chanCap)

	go func() {
		defer func() {
			close(deltas)
			rdiff = deltas
		}()

		for delta := range diff {
			switch delta.Status {
			case git2go.DeltaUnmodified:
				deltas <- &Delta{
					NewPath: delta.NewFile.Path,
					Type:    DeltaUnmodified,
				}
			case git2go.DeltaAdded:
				blob, err := gr.LookupBlob(delta.NewFile.Oid)
				if err != nil {
					return
				}

				deltas <- &Delta{
					Operation: &rsync.Operation{Data: blob.Contents()},
					NewPath:   delta.NewFile.Path,
					Type:      DeltaAdded,
				}
			case git2go.DeltaDeleted:
				deltas <- &Delta{
					OldPath: delta.OldFile.Path,
					Type:    DeltaDeleted,
				}
			case git2go.DeltaModified:
				newBlob, err := gr.LookupBlob(delta.NewFile.Oid)
				if err != nil {
					return
				}

				oldBlob, err := gr.LookupBlob(delta.OldFile.Oid)
				if err != nil {
					return
				}

				rdiffMaker := &rsync.Rsync{}
				oldReader := bytes.NewReader(oldBlob.Contents())
				newReader := bytes.NewReader(newBlob.Contents())
				signature := new([]BlockHash, 0, sliceCapacity)
				err = rsync.CreateSignature(oldReader, func(bh BlockHash) error {
					append(signature, bh)
					return nil
				})
				if err != nil {
					return
				}

				err = rsync.CreateDelta(newReader, signature, func(op Operation) error {
					deltas <- &Delta{
						Operation: op,
						OldPath:   delta.OldFile.Path,
						NewPath:   delta.NewFile.Path,
						Type:      DeltaModified,
					}
					return nil
				})
			case git2go.DeltaRenamed:
				deltas <- &Delta{
					OldPath: delta.OldFile.Path,
					NewPath: delta.NewFile.Path,
					Type:    DeltaRenamed,
				}
			case git2go.DeltaCopied:
				deltas <- &Delta{
					OldPath: delta.OldFile.Path,
					Path:    delta.NewFile.Path,
					Type:    DeltaCopied,
				}
			}
		}
	}()

	return
}

func (gr *GitRepository) DiffReference(ref *git2go.Reference) (diff *git2go.Diff, err error) {
	diff, err = gr.DiffTreeToWorkdirWithIndex(ref.Target(), nil)
	return
}

func (gr *GitRepository) DiffRemote(ref *git2go.Reference, remoteName string) (translatedDiff <-chan Delta, err error) {
	diff, err := gr.DiffReference(ref)
	if err != nil {
		return
	}

	deltas := make(chan git2go.DiffDelta, chanCap)
	defer close(deltas)
	deltaCollector := func(delta git2go.DiffDelta, _ float64) (git2go.DiffForEachHunkCallback, error) {
		deltas <- delta
		return nil, nil
	}
	err = diff.ForEach(deltaCollector, git2go.DiffDetailFiles)
	if err != nil {
		return
	}

	remote, err := gr.SyntheticRemotes[remoteName]
	if err != nil {
		return
	}

	comm := remote.DialURL(remote.URL, remote.Protocol)
	return gr.GetRdiff(deltas)
}

func (src *SyntheticRemoteCollection) Add(name string, rawurl string, proto remote.Protocol) (err error) {
	url, err := url.Parse(rawurl)
	if err != nil {
		return
	}

	gr.SyntheticRemotes[name] = &SyntheticRemote{
		URL:      url,
		Protocol: proto,
	}
}
<commit_msg>rename: gave args and vars in the various git diffing methods more adequate names<commit_after>// git.go
package translation

import (
	"bitbucket.org/kardianos/rsync"
	"github.com/libgit2/git2go"
	"github.com/rgeorgiev583/gonflator/remote"
)

const (
	chanCap  = 1000
	sliceCap = 1000
)

type SyntheticRemote struct {
	URL      url.URL
	Protocol remote.Protocol
}

type SyntheticRemoteCollection map[string]*SyntheticRemote

type GitRepository struct {
	git2go.Repository
	Tree             ConfigurationTree
	SyntheticRemotes SyntheticRemoteCollection
}

func (gr *GitRepository) GetRdiff(diff chan<- git2go.DiffDelta) (rdiff <-chan Delta, err error) {
	rdiff = make(chan Delta, chanCap)

	go func() {
		defer func() {
			close(rdiff)
		}()

		for delta := range diff {
			switch delta.Status {
			case git2go.DeltaUnmodified:
				rdiff <- &Delta{
					NewPath: delta.NewFile.Path,
					Type:    DeltaUnmodified,
				}
			case git2go.DeltaAdded:
				blob, err := gr.LookupBlob(delta.NewFile.Oid)
				if err != nil {
					return
				}

				rdiff <- &Delta{
					Operation: &rsync.Operation{Data: blob.Contents()},
					NewPath:   delta.NewFile.Path,
					Type:      DeltaAdded,
				}
			case git2go.DeltaDeleted:
				rdiff <- &Delta{
					OldPath: delta.OldFile.Path,
					Type:    DeltaDeleted,
				}
			case git2go.DeltaModified:
				newBlob, err := gr.LookupBlob(delta.NewFile.Oid)
				if err != nil {
					return
				}

				oldBlob, err := gr.LookupBlob(delta.OldFile.Oid)
				if err != nil {
					return
				}

				rdiffMaker := &rsync.Rsync{}
				oldReader := bytes.NewReader(oldBlob.Contents())
				newReader := bytes.NewReader(newBlob.Contents())
				signature := new([]BlockHash, 0, sliceCapacity)
				err = rsync.CreateSignature(oldReader, func(bh BlockHash) error {
					append(signature, bh)
					return nil
				})
				if err != nil {
					return
				}

				err = rsync.CreateDelta(newReader, signature, func(op Operation) error {
					rdiff <- &Delta{
						Operation: op,
						OldPath:   delta.OldFile.Path,
						NewPath:   delta.NewFile.Path,
						Type:      DeltaModified,
					}
					return nil
				})
			case git2go.DeltaRenamed:
				rdiff <- &Delta{
					OldPath: delta.OldFile.Path,
					NewPath: delta.NewFile.Path,
					Type:    DeltaRenamed,
				}
			case git2go.DeltaCopied:
				rdiff <- &Delta{
					OldPath: delta.OldFile.Path,
					Path:    delta.NewFile.Path,
					Type:    DeltaCopied,
				}
			}
		}
	}()

	return
}

func (gr *GitRepository) DiffReference(ref *git2go.Reference) (diff *git2go.Diff, err error) {
	diff, err = gr.DiffTreeToWorkdirWithIndex(ref.Target(), nil)
	return
}

func (gr *GitRepository) DiffRemote(ref *git2go.Reference, remoteName string) (translatedDiff <-chan Delta, err error) {
	gitDiff, err := gr.DiffReference(ref)
	if err != nil {
		return
	}

	diff := make(chan git2go.DiffDelta, chanCap)
	defer close(diff)
	deltaCollector := func(delta git2go.DiffDelta, _ float64) (git2go.DiffForEachHunkCallback, error) {
		diff <- delta
		return nil, nil
	}
	err = diff.ForEach(deltaCollector, git2go.DiffDetailFiles)
	if err != nil {
		return
	}

	remote, err := gr.SyntheticRemotes[remoteName]
	if err != nil {
		return
	}

	comm := remote.DialURL(remote.URL, remote.Protocol)
	return gr.GetRdiff(rdiff)
}

func (src *SyntheticRemoteCollection) Add(name string, rawurl string, proto remote.Protocol) (err error) {
	url, err := url.Parse(rawurl)
	if err != nil {
		return
	}

	gr.SyntheticRemotes[name] = &SyntheticRemote{
		URL:      url,
		Protocol: proto,
	}
}
<|endoftext|>


<commit_before>package lib

import (
	"fmt"
	"strings"

	oss "github.com/aliyun/aliyun-oss-go-sdk/oss"
)

type batchOptionType struct {
	ctnu     bool
	reporter *Reporter
}

var specChineseRestore = SpecText{

	synopsisText: "恢复冷冻状态的Objects为可读状态",

	paramText: "cloud_url [options]",

	syntaxText: `
    ossutil restore cloud_url [--encoding-type url] [-r] [-f] [--output-dir=odir] [-c file]
`,

	detailHelpText: `
    该命令恢复处于冷冻状态的object进入可读状态，即操作对象object必须为` + StorageArchive + `存储类型
    的object。

    如果是针对处于冷冻状态的object第一次调用restore接口，则返回成功。
    如果已经成功调用过restore接口，且restore没有完全完成，再次调用时同样成功。
    如果已经成功调用过restore接口，且restore已经完成，再次调用时返回成功，且会将object
    的可下载时间延长一天，最多延长7天。

用法：

    该命令有两种用法：

    1) ossutil restore oss://bucket/object [--encoding-type url]
        该用法恢复单个冷冻状态object为可读状态，当指定object不存在时，ossutil会提示错
    误，此时请确保指定的url精确匹配需要设置acl的object，并且不要指定--recursive选项（
    否则ossutil会进行前缀匹配，恢复多个冷冻状态的objects为可读状态）。无论--force选项
    是否指定，都不会进行询问提示。

    2) ossutil restore oss://bucket[/prefix] -r [--encoding-type url] [-f] [--output-dir=odir]
        该用法可批量恢复多个冷冻状态的objects为可读状态，此时必须输入--recursive选项，
    ossutil会查找所有前缀匹配url的objects，恢复它们为可读状态。当一个object操作出现错
    误时，会将出错object的错误信息记录到report文件，并继续操作其他object，成功操作的
    object信息将不会被记录到report文件中（更多信息见cp命令的帮助）。如果--force选项被
    指定，则不会进行询问提示。
`,

	sampleText: `
    1) ossutil restore oss://bucket-restore/object-store
    2) ossutil restore oss://bucket-restore/object-prefix -r
    3) ossutil restore oss://bucket-restore/object-prefix -r -f
    4) ossutil restore oss://bucket-restore/%e4%b8%ad%e6%96%87 --encoding-type url
`,
}

var specEnglishRestore = SpecText{

	synopsisText: "Restore Frozen State Object to Read Ready Status",

	paramText: "cloud_url [options]",

	syntaxText: `
    ossutil restore cloud_url [--encoding-type url] [-r] [-f] [--output-dir=odir] [-c file]
`,

	detailHelpText: `
    The command restore frozen state object to read ready status, the object must be in
    the storage class of ` + StorageArchive + `.

    If it's the first time to restore a frozen state object, the operation will success.
    If the object is in restoring, and the restoring is not finished, do the operation
    again will success.
    If an object has been restored, do the operation again will success, and the time that
    the object can be downloaded will extend one day, we can at most extend seven days.

Usage:

    There are two usages:

    1) ossutil restore oss://bucket/object [--encoding-type url]
        If --recursive option is not specified, ossutil restore the specified single frozen state
    object to read ready status. In the usage, please make sure url exactly specified the object
    you want to restore, if object not exist, error occurs. No matter --force option is specified
    or not, ossutil will not show prompt question.

    2) ossutil restore oss://bucket[/prefix] -r [--encoding-type url] [-f] [--output-dir=odir]
        The usage can restore many objects that in frozen state to read ready status, --recursive
    option is required for the usage, ossutil will search for prefix-matching objects and restore
    those objects. When an error occurs when restore an object, ossutil will record the error message
    to report file, and ossutil will continue to attempt to set acl on the remaining objects(more
    information see help of cp command). If --force option is specified, ossutil will not show
    prompt question.
`,

	sampleText: `
    1) ossutil restore oss://bucket-restore/object-store
    2) ossutil restore oss://bucket-restore/object-prefix -r
    3) ossutil restore oss://bucket-restore/object-prefix -r -f
    4) ossutil restore oss://bucket-restore/%e4%b8%ad%e6%96%87 --encoding-type url
`,
}

// RestoreCommand is the command list buckets or objects
type RestoreCommand struct {
	command  Command
	monitor  Monitor
	reOption batchOptionType
}

var restoreCommand = RestoreCommand{
	command: Command{
		name:        "restore",
		nameAlias:   []string{},
		minArgc:     1,
		maxArgc:     1,
		specChinese: specChineseRestore,
		specEnglish: specEnglishRestore,
		group:       GroupTypeNormalCommand,
		validOptionNames: []string{
			OptionRecursion,
			OptionForce,
			OptionEncodingType,
			OptionConfigFile,
			OptionEndpoint,
			OptionAccessKeyID,
			OptionAccessKeySecret,
			OptionSTSToken,
			OptionRetryTimes,
			OptionRoutines,
			OptionOutputDir,
		},
	},
}

// function for FormatHelper interface
func (rc *RestoreCommand) formatHelpForWhole() string {
	return rc.command.formatHelpForWhole()
}

func (rc *RestoreCommand) formatIndependHelp() string {
	return rc.command.formatIndependHelp()
}

// Init simulate inheritance, and polymorphism
func (rc *RestoreCommand) Init(args []string, options OptionMapType) error {
	return rc.command.Init(args, options, rc)
}

// RunCommand simulate inheritance, and polymorphism
func (rc *RestoreCommand) RunCommand() error {
	rc.monitor.init("Restored")

	encodingType, _ := GetString(OptionEncodingType, rc.command.options)
	recursive, _ := GetBool(OptionRecursion, rc.command.options)

	cloudURL, err := CloudURLFromString(rc.command.args[0], encodingType)
	if err != nil {
		return err
	}

	if err = rc.checkArgs(cloudURL, recursive); err != nil {
		return err
	}

	bucket, err := rc.command.ossBucket(cloudURL.bucket)
	if err != nil {
		return err
	}

	if !recursive {
		return rc.ossRestoreObject(bucket, cloudURL.object)
	}
	return rc.batchRestoreObjects(bucket, cloudURL)
}

func (rc *RestoreCommand) checkArgs(cloudURL CloudURL, recursive bool) error {
	if cloudURL.bucket == "" {
		return fmt.Errorf("invalid cloud url: %s, miss bucket", rc.command.args[0])
	}
	if !recursive && cloudURL.object == "" {
		return fmt.Errorf("restore object invalid cloud url: %s, object empty. Restore bucket is not supported, if you mean batch restore objects, please use --recursive", rc.command.args[0])
	}
	return nil
}

func (rc *RestoreCommand) ossRestoreObject(bucket *oss.Bucket, object string) error {
	retryTimes, _ := GetInt(OptionRetryTimes, rc.command.options)
	for i := 1; ; i++ {
		err := bucket.RestoreObject(object)
		if err == nil {
			return err
		}

		switch err.(type) {
		case oss.ServiceError:
			if err.(oss.ServiceError).StatusCode == 409 && err.(oss.ServiceError).Code == "RestoreAlreadyInProgress" {
				return nil
			}
		}

		if int64(i) >= retryTimes {
			return ObjectError{err, bucket.BucketName, object}
		}
	}
}

func (rc *RestoreCommand) batchRestoreObjects(bucket *oss.Bucket, cloudURL CloudURL) error {
	force, _ := GetBool(OptionForce, rc.command.options)
	if !force {
		var val string
		fmt.Printf("Do you really mean to recursivlly restore objects of %s(y or N)? ", rc.command.args[0])
		if _, err := fmt.Scanln(&val); err != nil || (strings.ToLower(val) != "yes" && strings.ToLower(val) != "y") {
			fmt.Println("operation is canceled.")
			return nil
		}
	}

	rc.reOption.ctnu = true
	outputDir, _ := GetString(OptionOutputDir, rc.command.options)

	// init reporter
	var err error
	if rc.reOption.reporter, err = GetReporter(rc.reOption.ctnu, outputDir, commandLine); err != nil {
		return err
	}
	defer rc.reOption.reporter.Clear()

	return rc.restoreObjects(bucket, cloudURL)
}

func (rc *RestoreCommand) restoreObjects(bucket *oss.Bucket, cloudURL CloudURL) error {
	routines, _ := GetInt(OptionRoutines, rc.command.options)

	chObjects := make(chan string, ChannelBuf)
	chError := make(chan error, routines+1)
	chListError := make(chan error, 1)
	go rc.command.objectStatistic(bucket, cloudURL, &rc.monitor)
	go rc.command.objectProducer(bucket, cloudURL, chObjects, chListError)
	for i := 0; int64(i) < routines; i++ {
		go rc.restoreConsumer(bucket, cloudURL, chObjects, chError)
	}

	return rc.waitRoutinueComplete(chError, chListError, routines)
}

func (rc *RestoreCommand) restoreConsumer(bucket *oss.Bucket, cloudURL CloudURL, chObjects <-chan string, chError chan<- error) {
	for object := range chObjects {
		err := rc.restoreObjectWithReport(bucket, object)
		if err != nil {
			chError <- err
			if !rc.reOption.ctnu {
				return
			}
			continue
		}
	}

	chError <- nil
}

func (rc *RestoreCommand) restoreObjectWithReport(bucket *oss.Bucket, object string) error {
	err := rc.ossRestoreObject(bucket, object)
	rc.command.updateMonitor(err, &rc.monitor)
	msg := fmt.Sprintf("restore %s", CloudURLToString(bucket.BucketName, object))
	rc.command.report(msg, err, &rc.reOption)
	return err
}

func (rc *RestoreCommand) waitRoutinueComplete(chError, chListError <-chan error, routines int64) error {
	completed := 0
	var ferr error
	for int64(completed) <= routines {
		select {
		case err := <-chListError:
			if err != nil {
				return err
			}
			completed++
		case err := <-chError:
			if err == nil {
				completed++
			} else {
				ferr = err
				if !rc.reOption.ctnu {
					fmt.Printf(rc.monitor.progressBar(true, errExit))
					return err
				}
			}
		}
	}
	return rc.formatResultPrompt(ferr)
}

func (rc *RestoreCommand) formatResultPrompt(err error) error {
	fmt.Printf(rc.monitor.progressBar(true, normalExit))
	if err != nil && rc.reOption.ctnu {
		return nil
	}
	return err
}
<commit_msg>modify help of restore<commit_after>package lib

import (
	"fmt"
	"strings"

	oss "github.com/aliyun/aliyun-oss-go-sdk/oss"
)

type batchOptionType struct {
	ctnu     bool
	reporter *Reporter
}

var specChineseRestore = SpecText{

	synopsisText: "恢复冷冻状态的Objects为可读状态",

	paramText: "cloud_url [options]",

	syntaxText: `
    ossutil restore cloud_url [--encoding-type url] [-r] [-f] [--output-dir=odir] [-c file]
`,

	detailHelpText: `
    该命令恢复处于冷冻状态的object进入可读状态，即操作对象object必须为` + StorageArchive + `存储类型
    的object。

    如果是针对处于冷冻状态的object第一次调用restore接口，则返回成功。
    如果已经成功调用过restore接口，且restore没有完全完成，再次调用时同样成功。
    如果已经成功调用过restore接口，且restore已经完成，再次调用时返回成功，且会将object
    的可下载时间延长一天，最多延长7天。

用法：

    该命令有两种用法：

    1) ossutil restore oss://bucket/object [--encoding-type url]
        该用法恢复单个冷冻状态object为可读状态，当指定object不存在时，ossutil会提示错
    误，此时请确保指定的url精确匹配需要设置acl的object，并且不要指定--recursive选项（
    否则ossutil会进行前缀匹配，恢复多个冷冻状态的objects为可读状态）。无论--force选项
    是否指定，都不会进行询问提示。

    2) ossutil restore oss://bucket[/prefix] -r [--encoding-type url] [-f] [--output-dir=odir]
        该用法可批量恢复多个冷冻状态的objects为可读状态，此时必须输入--recursive选项，
    ossutil会查找所有前缀匹配url的objects，恢复它们为可读状态。当一个object操作出现错
    误时，会将出错object的错误信息记录到report文件，并继续操作其他object，成功操作的
    object信息将不会被记录到report文件中（更多信息见cp命令的帮助）。如果--force选项被
    指定，则不会进行询问提示。
`,

	sampleText: `
    1) ossutil restore oss://bucket-restore/object-store
    2) ossutil restore oss://bucket-restore/object-prefix -r
    3) ossutil restore oss://bucket-restore/object-prefix -r -f
    4) ossutil restore oss://bucket-restore/%e4%b8%ad%e6%96%87 --encoding-type url
`,
}

var specEnglishRestore = SpecText{

	synopsisText: "Restore Frozen State Object to Read Ready Status",

	paramText: "cloud_url [options]",

	syntaxText: `
    ossutil restore cloud_url [--encoding-type url] [-r] [-f] [--output-dir=odir] [-c file]
`,

	detailHelpText: `
    The command restore frozen state object to read ready status, the object must be in
    the storage class of ` + StorageArchive + `.

    If it's the first time to restore a frozen state object, the operation will success.
    If the object is in restoring, and the restoring is not finished, do the operation
    again will success.
    If an object has been restored, do the operation again will success, and the time that
    the object can be downloaded will extend one day, we can at most extend seven days.

Usage:

    There are two usages:

    1) ossutil restore oss://bucket/object [--encoding-type url]
        If --recursive option is not specified, ossutil restore the specified frozen state object
    to readable status. In the usage, please make sure url exactly specified the object you want to
    restore, if object not exist, error occurs. No matter --force option is specified or not, ossutil
    will not show prompt question.

    2) ossutil restore oss://bucket[/prefix] -r [--encoding-type url] [-f] [--output-dir=odir]
        The usage restore the objects with the specified prefix and in frozen state to readable status.
    --recursive option is required for the usage, and ossutil will search for prefix-matching objects
    and restore those objects. When an error occurs when restore an object, ossutil will record the
    error message to report file, and ossutil will continue to attempt to set acl on the remaining
    objects(more information see help of cp command). If --force option is specified, ossutil will
    not show prompt question.
`,

	sampleText: `
    1) ossutil restore oss://bucket-restore/object-store
    2) ossutil restore oss://bucket-restore/object-prefix -r
    3) ossutil restore oss://bucket-restore/object-prefix -r -f
    4) ossutil restore oss://bucket-restore/%e4%b8%ad%e6%96%87 --encoding-type url
`,
}

// RestoreCommand is the command list buckets or objects
type RestoreCommand struct {
	command  Command
	monitor  Monitor
	reOption batchOptionType
}

var restoreCommand = RestoreCommand{
	command: Command{
		name:        "restore",
		nameAlias:   []string{},
		minArgc:     1,
		maxArgc:     1,
		specChinese: specChineseRestore,
		specEnglish: specEnglishRestore,
		group:       GroupTypeNormalCommand,
		validOptionNames: []string{
			OptionRecursion,
			OptionForce,
			OptionEncodingType,
			OptionConfigFile,
			OptionEndpoint,
			OptionAccessKeyID,
			OptionAccessKeySecret,
			OptionSTSToken,
			OptionRetryTimes,
			OptionRoutines,
			OptionOutputDir,
		},
	},
}

// function for FormatHelper interface
func (rc *RestoreCommand) formatHelpForWhole() string {
	return rc.command.formatHelpForWhole()
}

func (rc *RestoreCommand) formatIndependHelp() string {
	return rc.command.formatIndependHelp()
}

// Init simulate inheritance, and polymorphism
func (rc *RestoreCommand) Init(args []string, options OptionMapType) error {
	return rc.command.Init(args, options, rc)
}

// RunCommand simulate inheritance, and polymorphism
func (rc *RestoreCommand) RunCommand() error {
	rc.monitor.init("Restored")

	encodingType, _ := GetString(OptionEncodingType, rc.command.options)
	recursive, _ := GetBool(OptionRecursion, rc.command.options)

	cloudURL, err := CloudURLFromString(rc.command.args[0], encodingType)
	if err != nil {
		return err
	}

	if err = rc.checkArgs(cloudURL, recursive); err != nil {
		return err
	}

	bucket, err := rc.command.ossBucket(cloudURL.bucket)
	if err != nil {
		return err
	}

	if !recursive {
		return rc.ossRestoreObject(bucket, cloudURL.object)
	}
	return rc.batchRestoreObjects(bucket, cloudURL)
}

func (rc *RestoreCommand) checkArgs(cloudURL CloudURL, recursive bool) error {
	if cloudURL.bucket == "" {
		return fmt.Errorf("invalid cloud url: %s, miss bucket", rc.command.args[0])
	}
	if !recursive && cloudURL.object == "" {
		return fmt.Errorf("restore object invalid cloud url: %s, object empty. Restore bucket is not supported, if you mean batch restore objects, please use --recursive", rc.command.args[0])
	}
	return nil
}

func (rc *RestoreCommand) ossRestoreObject(bucket *oss.Bucket, object string) error {
	retryTimes, _ := GetInt(OptionRetryTimes, rc.command.options)
	for i := 1; ; i++ {
		err := bucket.RestoreObject(object)
		if err == nil {
			return err
		}

		switch err.(type) {
		case oss.ServiceError:
			if err.(oss.ServiceError).StatusCode == 409 && err.(oss.ServiceError).Code == "RestoreAlreadyInProgress" {
				return nil
			}
		}

		if int64(i) >= retryTimes {
			return ObjectError{err, bucket.BucketName, object}
		}
	}
}

func (rc *RestoreCommand) batchRestoreObjects(bucket *oss.Bucket, cloudURL CloudURL) error {
	force, _ := GetBool(OptionForce, rc.command.options)
	if !force {
		var val string
		fmt.Printf("Do you really mean to recursivlly restore objects of %s(y or N)? ", rc.command.args[0])
		if _, err := fmt.Scanln(&val); err != nil || (strings.ToLower(val) != "yes" && strings.ToLower(val) != "y") {
			fmt.Println("operation is canceled.")
			return nil
		}
	}

	rc.reOption.ctnu = true
	outputDir, _ := GetString(OptionOutputDir, rc.command.options)

	// init reporter
	var err error
	if rc.reOption.reporter, err = GetReporter(rc.reOption.ctnu, outputDir, commandLine); err != nil {
		return err
	}
	defer rc.reOption.reporter.Clear()

	return rc.restoreObjects(bucket, cloudURL)
}

func (rc *RestoreCommand) restoreObjects(bucket *oss.Bucket, cloudURL CloudURL) error {
	routines, _ := GetInt(OptionRoutines, rc.command.options)

	chObjects := make(chan string, ChannelBuf)
	chError := make(chan error, routines+1)
	chListError := make(chan error, 1)
	go rc.command.objectStatistic(bucket, cloudURL, &rc.monitor)
	go rc.command.objectProducer(bucket, cloudURL, chObjects, chListError)
	for i := 0; int64(i) < routines; i++ {
		go rc.restoreConsumer(bucket, cloudURL, chObjects, chError)
	}

	return rc.waitRoutinueComplete(chError, chListError, routines)
}

func (rc *RestoreCommand) restoreConsumer(bucket *oss.Bucket, cloudURL CloudURL, chObjects <-chan string, chError chan<- error) {
	for object := range chObjects {
		err := rc.restoreObjectWithReport(bucket, object)
		if err != nil {
			chError <- err
			if !rc.reOption.ctnu {
				return
			}
			continue
		}
	}

	chError <- nil
}

func (rc *RestoreCommand) restoreObjectWithReport(bucket *oss.Bucket, object string) error {
	err := rc.ossRestoreObject(bucket, object)
	rc.command.updateMonitor(err, &rc.monitor)
	msg := fmt.Sprintf("restore %s", CloudURLToString(bucket.BucketName, object))
	rc.command.report(msg, err, &rc.reOption)
	return err
}

func (rc *RestoreCommand) waitRoutinueComplete(chError, chListError <-chan error, routines int64) error {
	completed := 0
	var ferr error
	for int64(completed) <= routines {
		select {
		case err := <-chListError:
			if err != nil {
				return err
			}
			completed++
		case err := <-chError:
			if err == nil {
				completed++
			} else {
				ferr = err
				if !rc.reOption.ctnu {
					fmt.Printf(rc.monitor.progressBar(true, errExit))
					return err
				}
			}
		}
	}
	return rc.formatResultPrompt(ferr)
}

func (rc *RestoreCommand) formatResultPrompt(err error) error {
	fmt.Printf(rc.monitor.progressBar(true, normalExit))
	if err != nil && rc.reOption.ctnu {
		return nil
	}
	return err
}
<|endoftext|>


<commit_before>package vaulted

import (
	"crypto/rand"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"time"

	"github.com/miquella/xdg"
	"golang.org/x/crypto/nacl/secretbox"
)

var (
	ErrInvalidPassword         = errors.New("Invalid password")
	ErrInvalidKeyConfig        = errors.New("Invalid key configuration")
	ErrInvalidEncryptionConfig = errors.New("Invalid encryption configuration")
)

func VaultExists(name string) bool {
	existing := xdg.DATA.Find(filepath.Join("vaulted", name))
	if len(existing) == 0 {
		return false
	}

	return true
}

func ListVaults() ([]string, error) {
	vaults, err := xdg.DATA.Glob(filepath.Join("vaulted", "*"))
	if err != nil {
		return nil, err
	}

	var found []string
	emitted := map[string]bool{}
	for _, vault := range vaults {
		info, err := os.Stat(vault)
		if err != nil {
			return nil, err
		}
		if !info.Mode().IsRegular() {
			continue
		}

		if !emitted[info.Name()] {
			emitted[info.Name()] = true
			found = append(found, info.Name())
		}
	}

	return found, nil
}

func SealVault(name, password string, vault *Vault) error {
	vf := &VaultFile{
		Method:  "secretbox",
		Details: make(Details),
	}

	// generate a new key (while trying to keeping the existing key derivation and encryption methods)
	existingVaultFile, err := readVaultFile(name)
	if err == nil {
		vf.Method = existingVaultFile.Method
		vf.Key = existingVaultFile.Key
	}

	vf.Key = newVaultKey(vf.Key)

	// marshal the vault content
	content, err := json.Marshal(vault)
	if err != nil {
		return err
	}

	// encrypt the vault
	if vf.Method == "" {
		vf.Method = "secretbox"
	}

	switch vf.Method {
	case "secretbox":
		nonce := [24]byte{}
		_, err = rand.Read(nonce[:])
		if err != nil {
			return err
		}
		vf.Details.SetBytes("nonce", nonce[:])

		key := [32]byte{}
		derivedKey, err := vf.Key.key(password, len(key))
		if err != nil {
			return err
		}
		copy(key[:], derivedKey[:])

		vf.Ciphertext = secretbox.Seal(nil, content, &nonce, &key)

	default:
		return fmt.Errorf("Invalid encryption method: %s", vf.Method)
	}

	writeVaultFile(name, vf)

	return nil
}

func OpenVault(name, password string) (*Vault, error) {
	vf, err := readVaultFile(name)
	if err != nil {
		return nil, err
	}

	v := Vault{}

	switch vf.Method {
	case "secretbox":
		if vf.Key == nil {
			return nil, ErrInvalidKeyConfig
		}

		nonce := vf.Details.Bytes("nonce")
		if len(nonce) == 0 {
			return nil, ErrInvalidEncryptionConfig
		}
		boxNonce := [24]byte{}
		copy(boxNonce[:], nonce)

		boxKey := [32]byte{}
		derivedKey, err := vf.Key.key(password, len(boxKey))
		if err != nil {
			return nil, err
		}
		copy(boxKey[:], derivedKey[:])

		plaintext, ok := secretbox.Open(nil, vf.Ciphertext, &boxNonce, &boxKey)
		if !ok {
			return nil, ErrInvalidPassword
		}

		err = json.Unmarshal(plaintext, &v)
		if err != nil {
			return nil, err
		}

	default:
		return nil, fmt.Errorf("Invalid encryption method: %s", vf.Method)
	}

	return &v, nil
}

func RemoveVault(name string) error {
	existing := xdg.DATA_HOME.Find(filepath.Join("vaulted", name))
	if existing == "" {
		untouchable := xdg.DATA_DIRS.Find(filepath.Join("vaulted", name))
		if len(untouchable) == 0 {
			return os.ErrNotExist
		}

		return fmt.Errorf("Because %s is outside the vaulted managed directory (%s), it must be removed manually", untouchable[0], xdg.DATA_HOME.Join("vaulted"))
	}

	removeSession(name)

	return os.Remove(existing)
}

func GetSession(name, password string) (*Session, error) {
	v, err := OpenVault(name, password)
	if err != nil {
		return nil, err
	}

	session, err := getSession(v, name, password)
	if err != nil {
		return nil, err
	}

	if v.AWSKey != nil && v.AWSKey.Role != "" {
		session, err = session.Assume(v.AWSKey.Role)
		if err != nil {
			return nil, err
		}
	}

	return session, nil
}

func getSession(v *Vault, name, password string) (*Session, error) {
	session, err := openSession(name, password)
	if err == nil {
		expired := time.Now().Add(15 * time.Minute).After(session.Expiration)
		if !expired {
			return session, nil
		}
	}

	// the session isn't valid (possibly expired), so remove it
	removeSession(name)

	session, err = v.CreateSession(name)
	if err != nil {
		return nil, err
	}

	// we have a valid session, so if saving fails, ignore the failure
	sealSession(name, password, session)
	return session, nil
}

func sealSession(name, password string, session *Session) error {
	// read the vault file (to get key details)
	vf, err := readVaultFile(name)
	if err != nil {
		return err
	}

	// marshal the session content
	content, err := json.Marshal(session)
	if err != nil {
		return err
	}

	// encrypt the session
	sf := &SessionFile{
		Method:  "secretbox",
		Details: make(Details),
	}

	switch sf.Method {
	case "secretbox":
		nonce := [24]byte{}
		_, err = rand.Read(nonce[:])
		if err != nil {
			return err
		}
		sf.Details.SetBytes("nonce", nonce[:])

		key := [32]byte{}
		derivedKey, err := vf.Key.key(password, len(key))
		if err != nil {
			return err
		}
		copy(key[:], derivedKey[:])

		sf.Ciphertext = secretbox.Seal(nil, content, &nonce, &key)

	default:
		return err
	}

	return writeSessionFile(name, sf)
}

func openSession(name, password string) (*Session, error) {
	vf, err := readVaultFile(name)
	if err != nil {
		return nil, err
	}

	sf, err := readSessionFile(name)
	if err != nil {
		return nil, err
	}

	s := Session{}

	switch sf.Method {
	case "secretbox":
		if vf.Key == nil {
			return nil, ErrInvalidKeyConfig
		}

		nonce := sf.Details.Bytes("nonce")
		if len(nonce) == 0 {
			return nil, ErrInvalidEncryptionConfig
		}
		boxNonce := [24]byte{}
		copy(boxNonce[:], nonce)

		boxKey := [32]byte{}
		derivedKey, err := vf.Key.key(password, len(boxKey))
		if err != nil {
			return nil, err
		}
		copy(boxKey[:], derivedKey[:])

		plaintext, ok := secretbox.Open(nil, sf.Ciphertext, &boxNonce, &boxKey)
		if !ok {
			return nil, ErrInvalidPassword
		}

		err = json.Unmarshal(plaintext, &s)
		if err != nil {
			return nil, err
		}

	default:
		return nil, fmt.Errorf("Invalid encryption method: %s", sf.Method)
	}

	return &s, nil
}
<commit_msg>Only remove cached sessions that fail to load<commit_after>package vaulted

import (
	"crypto/rand"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"time"

	"github.com/miquella/xdg"
	"golang.org/x/crypto/nacl/secretbox"
)

var (
	ErrInvalidPassword         = errors.New("Invalid password")
	ErrInvalidKeyConfig        = errors.New("Invalid key configuration")
	ErrInvalidEncryptionConfig = errors.New("Invalid encryption configuration")
)

func VaultExists(name string) bool {
	existing := xdg.DATA.Find(filepath.Join("vaulted", name))
	if len(existing) == 0 {
		return false
	}

	return true
}

func ListVaults() ([]string, error) {
	vaults, err := xdg.DATA.Glob(filepath.Join("vaulted", "*"))
	if err != nil {
		return nil, err
	}

	var found []string
	emitted := map[string]bool{}
	for _, vault := range vaults {
		info, err := os.Stat(vault)
		if err != nil {
			return nil, err
		}
		if !info.Mode().IsRegular() {
			continue
		}

		if !emitted[info.Name()] {
			emitted[info.Name()] = true
			found = append(found, info.Name())
		}
	}

	return found, nil
}

func SealVault(name, password string, vault *Vault) error {
	vf := &VaultFile{
		Method:  "secretbox",
		Details: make(Details),
	}

	// generate a new key (while trying to keeping the existing key derivation and encryption methods)
	existingVaultFile, err := readVaultFile(name)
	if err == nil {
		vf.Method = existingVaultFile.Method
		vf.Key = existingVaultFile.Key
	}

	vf.Key = newVaultKey(vf.Key)

	// marshal the vault content
	content, err := json.Marshal(vault)
	if err != nil {
		return err
	}

	// encrypt the vault
	if vf.Method == "" {
		vf.Method = "secretbox"
	}

	switch vf.Method {
	case "secretbox":
		nonce := [24]byte{}
		_, err = rand.Read(nonce[:])
		if err != nil {
			return err
		}
		vf.Details.SetBytes("nonce", nonce[:])

		key := [32]byte{}
		derivedKey, err := vf.Key.key(password, len(key))
		if err != nil {
			return err
		}
		copy(key[:], derivedKey[:])

		vf.Ciphertext = secretbox.Seal(nil, content, &nonce, &key)

	default:
		return fmt.Errorf("Invalid encryption method: %s", vf.Method)
	}

	writeVaultFile(name, vf)

	return nil
}

func OpenVault(name, password string) (*Vault, error) {
	vf, err := readVaultFile(name)
	if err != nil {
		return nil, err
	}

	v := Vault{}

	switch vf.Method {
	case "secretbox":
		if vf.Key == nil {
			return nil, ErrInvalidKeyConfig
		}

		nonce := vf.Details.Bytes("nonce")
		if len(nonce) == 0 {
			return nil, ErrInvalidEncryptionConfig
		}
		boxNonce := [24]byte{}
		copy(boxNonce[:], nonce)

		boxKey := [32]byte{}
		derivedKey, err := vf.Key.key(password, len(boxKey))
		if err != nil {
			return nil, err
		}
		copy(boxKey[:], derivedKey[:])

		plaintext, ok := secretbox.Open(nil, vf.Ciphertext, &boxNonce, &boxKey)
		if !ok {
			return nil, ErrInvalidPassword
		}

		err = json.Unmarshal(plaintext, &v)
		if err != nil {
			return nil, err
		}

	default:
		return nil, fmt.Errorf("Invalid encryption method: %s", vf.Method)
	}

	return &v, nil
}

func RemoveVault(name string) error {
	existing := xdg.DATA_HOME.Find(filepath.Join("vaulted", name))
	if existing == "" {
		untouchable := xdg.DATA_DIRS.Find(filepath.Join("vaulted", name))
		if len(untouchable) == 0 {
			return os.ErrNotExist
		}

		return fmt.Errorf("Because %s is outside the vaulted managed directory (%s), it must be removed manually", untouchable[0], xdg.DATA_HOME.Join("vaulted"))
	}

	removeSession(name)

	return os.Remove(existing)
}

func GetSession(name, password string) (*Session, error) {
	v, err := OpenVault(name, password)
	if err != nil {
		return nil, err
	}

	session, err := getSession(v, name, password)
	if err != nil {
		return nil, err
	}

	if v.AWSKey != nil && v.AWSKey.Role != "" {
		session, err = session.Assume(v.AWSKey.Role)
		if err != nil {
			return nil, err
		}
	}

	return session, nil
}

func getSession(v *Vault, name, password string) (*Session, error) {
	session, err := openSession(name, password)
	if err != nil {
		removeSession(name)
	} else if session.Expiration.After(time.Now().Add(15 * time.Minute)) {
		return session, nil
	}

	session, err = v.CreateSession(name)
	if err != nil {
		return nil, err
	}

	// we have a valid session, so if saving fails, ignore the failure
	sealSession(name, password, session)
	return session, nil
}

func sealSession(name, password string, session *Session) error {
	// read the vault file (to get key details)
	vf, err := readVaultFile(name)
	if err != nil {
		return err
	}

	// marshal the session content
	content, err := json.Marshal(session)
	if err != nil {
		return err
	}

	// encrypt the session
	sf := &SessionFile{
		Method:  "secretbox",
		Details: make(Details),
	}

	switch sf.Method {
	case "secretbox":
		nonce := [24]byte{}
		_, err = rand.Read(nonce[:])
		if err != nil {
			return err
		}
		sf.Details.SetBytes("nonce", nonce[:])

		key := [32]byte{}
		derivedKey, err := vf.Key.key(password, len(key))
		if err != nil {
			return err
		}
		copy(key[:], derivedKey[:])

		sf.Ciphertext = secretbox.Seal(nil, content, &nonce, &key)

	default:
		return err
	}

	return writeSessionFile(name, sf)
}

func openSession(name, password string) (*Session, error) {
	vf, err := readVaultFile(name)
	if err != nil {
		return nil, err
	}

	sf, err := readSessionFile(name)
	if err != nil {
		return nil, err
	}

	s := Session{}

	switch sf.Method {
	case "secretbox":
		if vf.Key == nil {
			return nil, ErrInvalidKeyConfig
		}

		nonce := sf.Details.Bytes("nonce")
		if len(nonce) == 0 {
			return nil, ErrInvalidEncryptionConfig
		}
		boxNonce := [24]byte{}
		copy(boxNonce[:], nonce)

		boxKey := [32]byte{}
		derivedKey, err := vf.Key.key(password, len(boxKey))
		if err != nil {
			return nil, err
		}
		copy(boxKey[:], derivedKey[:])

		plaintext, ok := secretbox.Open(nil, sf.Ciphertext, &boxNonce, &boxKey)
		if !ok {
			return nil, ErrInvalidPassword
		}

		err = json.Unmarshal(plaintext, &s)
		if err != nil {
			return nil, err
		}

	default:
		return nil, fmt.Errorf("Invalid encryption method: %s", sf.Method)
	}

	return &s, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 The appc Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package docker2aci

import "github.com/appc/spec/schema"

var Version = "0.15.0"
var AppcVersion = schema.AppContainerVersion
<commit_msg>version: bump to v0.15.0+git<commit_after>// Copyright 2016 The appc Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package docker2aci

import "github.com/appc/spec/schema"

var Version = "0.15.0+git"
var AppcVersion = schema.AppContainerVersion
<|endoftext|>


<commit_before>package libs

import (
	"context"
	"fmt"
	"strconv"
	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
)

type GithubRepo struct {
	Name        *string `json:"name,omitempty"`
	Description *string `json:"description,omitempty"`
	Fullname    *string `json:"fullname,omitempty"`
	Branch      *string `json:"branch,omitempty"`
	Url         *string `json:"url,omitempty"`
	Language    *string `json:"language,omitempty"`
	Forks       *int    `json:"forks,omitempty"`
	Stars       *int    `json:"stars,omitempty"`
	Watches     *int    `json:"watches,omitempty"`
	Badge       *string `json:"badge,omitempty"`
}

type GithubRepos []GithubRepo
type GithubContribs map[string]int
type docGetter func(*Config) (*goquery.Document, error)

func getGithubContribsDoc(c *Config) (*goquery.Document, error) {
	url := fmt.Sprintf(
		"https://github.com/users/%s/contributions",
		c.Github.Username,
	)
	return goquery.NewDocument(url)
}

type Github struct {
	Client    *github.Client
	DocGetter docGetter
	config    *Config
}

func NewGithub(config *Config) *Github {
	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: config.Github.Token},
	)
	tc := oauth2.NewClient(oauth2.NoContext, ts)
	return &Github{
		Client:    github.NewClient(tc),
		DocGetter: getGithubContribsDoc,
		config:    config,
	}
}

func (g *Github) GetRepos() (*GithubRepos, error) {
	opt := &github.RepositoryListOptions{
		Type:      "all",
		Sort:      "pushed",
		Direction: "desc",
	}
	repos, _, err := g.Client.Repositories.List(context.Background(), "", opt)
	if err != nil {
		return nil, err
	}

	var gr GithubRepos
	for _, repo := range repos {
		badge := fmt.Sprintf(
			"https://api.travis-ci.org/%s.svg?branch=%s",
			*repo.FullName,
			*repo.DefaultBranch,
		)
		gr = append(gr, GithubRepo{
			Name:        repo.Name,
			Description: repo.Description,
			Fullname:    repo.FullName,
			Branch:      repo.DefaultBranch,
			Url:         repo.HTMLURL,
			Language:    repo.Language,
			Forks:       repo.ForksCount,
			Stars:       repo.StargazersCount,
			Watches:     repo.SubscribersCount,
			Badge:       &badge,
		})
	}

	return &gr, nil
}

func (g *Github) GetContribs() (*GithubContribs, error) {
	doc, err := g.DocGetter(g.config)
	if err != nil {
		return nil, err
	}
	gc := make(GithubContribs)
	doc.Find("g > g > rect").Each(func(i int, s *goquery.Selection) {
		rawDate, _ := s.Attr("data-date")
		rawCount, _ := s.Attr("data-count")

		var (
			t         time.Time
			err       error
			timestamp string = "0"
			count     int    = 0
		)

		t, err = time.Parse("2006-01-02", rawDate)
		if err == nil {
			timestamp = strconv.FormatInt(t.Unix(), 10)
		}
		count, _ = strconv.Atoi(rawCount)

		gc[timestamp] = count
	})
	return &gc, nil
}
<commit_msg>github: only show public projects<commit_after>package libs

import (
	"context"
	"fmt"
	"strconv"
	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
)

type GithubRepo struct {
	Name        *string `json:"name,omitempty"`
	Description *string `json:"description,omitempty"`
	Fullname    *string `json:"fullname,omitempty"`
	Branch      *string `json:"branch,omitempty"`
	Url         *string `json:"url,omitempty"`
	Language    *string `json:"language,omitempty"`
	Forks       *int    `json:"forks,omitempty"`
	Stars       *int    `json:"stars,omitempty"`
	Watches     *int    `json:"watches,omitempty"`
	Badge       *string `json:"badge,omitempty"`
}

type GithubRepos []GithubRepo
type GithubContribs map[string]int
type docGetter func(*Config) (*goquery.Document, error)

func getGithubContribsDoc(c *Config) (*goquery.Document, error) {
	url := fmt.Sprintf(
		"https://github.com/users/%s/contributions",
		c.Github.Username,
	)
	return goquery.NewDocument(url)
}

type Github struct {
	Client    *github.Client
	DocGetter docGetter
	config    *Config
}

func NewGithub(config *Config) *Github {
	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: config.Github.Token},
	)
	tc := oauth2.NewClient(oauth2.NoContext, ts)
	return &Github{
		Client:    github.NewClient(tc),
		DocGetter: getGithubContribsDoc,
		config:    config,
	}
}

func (g *Github) GetRepos() (*GithubRepos, error) {
	opt := &github.RepositoryListOptions{
		Type:      "public",
		Sort:      "pushed",
		Direction: "desc",
	}
	repos, _, err := g.Client.Repositories.List(context.Background(), "", opt)
	if err != nil {
		return nil, err
	}

	var gr GithubRepos
	for _, repo := range repos {
		badge := fmt.Sprintf(
			"https://api.travis-ci.org/%s.svg?branch=%s",
			*repo.FullName,
			*repo.DefaultBranch,
		)
		gr = append(gr, GithubRepo{
			Name:        repo.Name,
			Description: repo.Description,
			Fullname:    repo.FullName,
			Branch:      repo.DefaultBranch,
			Url:         repo.HTMLURL,
			Language:    repo.Language,
			Forks:       repo.ForksCount,
			Stars:       repo.StargazersCount,
			Watches:     repo.SubscribersCount,
			Badge:       &badge,
		})
	}

	return &gr, nil
}

func (g *Github) GetContribs() (*GithubContribs, error) {
	doc, err := g.DocGetter(g.config)
	if err != nil {
		return nil, err
	}
	gc := make(GithubContribs)
	doc.Find("g > g > rect").Each(func(i int, s *goquery.Selection) {
		rawDate, _ := s.Attr("data-date")
		rawCount, _ := s.Attr("data-count")

		var (
			t         time.Time
			err       error
			timestamp string = "0"
			count     int    = 0
		)

		t, err = time.Parse("2006-01-02", rawDate)
		if err == nil {
			timestamp = strconv.FormatInt(t.Unix(), 10)
		}
		count, _ = strconv.Atoi(rawCount)

		gc[timestamp] = count
	})
	return &gc, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
)

//check() checks and panics if an error has occured
func check(e error) {
	if e != nil {
		panic(e)
	}
}

//help() prints usage, commands and examples for licensethis.
func help() {
	helptext := `licensethis - Choose an OSS license to your project with ease.

Usage:
licensethis help
licensethis list
licensethis info <license-name>
licensethis generate <license-name>

Commands:
help		Show this help text.
list		Show a list of all available OSS licenses.
info		Show more information for a license.
generate	Generate LICENSE.txt file in current folder after asking for author name.

Examples:
licensethis help		Show this help text.
licensethis info mit		Show more information for MIT license.
licensethis generate mit	Geneate MIT license text file in current directory.\n`

	fmt.Printf("%v", helptext)
}

func info(licensename []string) {
	//fmt.Printf("%v\n", licensename[0])

	file, e := ioutil.ReadFile("licenses.json")
	if e != nil {
		log.Fatal(e)
	}
	var jsontype map[string][]map[string]interface{}
	err := json.Unmarshal(file, &jsontype)
	if err != nil {
		log.Fatal(err)
	}
	//fmt.Printf("Results: %v \n", jsontype["licenses"])
	for i := range jsontype["licenses"] {
		item := jsontype["licenses"][i]
		//fmt.Printf("%v\n",item["name"])
		if item["name"] == licensename[0] {
			fmt.Printf("%v\n%v\n%v", item["name"], item["version"], item["text"])
		}
	}
}

//parseArgs parses command line arguments and calls appropriate functions.
func parseArgs(args []string) {
	if len(args) != 0 {
		switch args[0] {
		case "help":
			help()
		case "info":
			info(args[1:])
		case "list":
			fmt.Println("list")
		case "generate":
			fmt.Println("generate")
		default:
			fmt.Println("Incorrect usage!")
			fmt.Println("Type `licensethis help` for help on proper usage.")
		}
	} else {
		fmt.Println("Incorrect usage!")
		fmt.Println("Type `licensethis help` for help on proper usage.")
	}
}

func main() {
	args := os.Args[1:]
	parseArgs(args)
}
<commit_msg>changes to argument parsing:better structured<commit_after>package main

import (
	"fmt"
	"os"
)

//check() checks and panics if an error has occured
func check(e error) {
	if e != nil {
		panic(e)
	}
}

//help() prints usage, commands and examples for licensethis.
func help() {
	helptext := `licensethis - Choose an OSS license for your project with ease.

Usage:
licensethis help
licensethis list
licensethis info <license-name>
licensethis generate <license-name>

Commands:
help		Show this help text.
list		Show a list of all available OSS licenses.
info		Show more information for a license.
generate	Generate LICENSE.txt file in current folder after asking for author name.

Examples:
licensethis help		Show this help text.
licensethis info mit		Show more information for MIT license.
licensethis generate mit	Geneate MIT license text file in current directory.`

	fmt.Printf("%v\n", helptext)
}

func info(licensename string) {
	//fmt.Printf("%v\n", licensename[0])
	fmt.Println(licensename)
	// file, e := ioutil.ReadFile("licenses.json")
	// if e != nil {
	// 	log.Fatal(e)
	// }
	// var jsontype map[string][]map[string]interface{}
	// err := json.Unmarshal(file, &jsontype)
	// if err != nil {
	// 	log.Fatal(err)
	// }
	// //fmt.Printf("Results: %v \n", jsontype["licenses"])
	// for i := range jsontype["licenses"] {
	// 	item := jsontype["licenses"][i]
	// 	//fmt.Printf("%v\n",item["name"])
	// 	if item["name"] == licensename[0] {
	// 		fmt.Printf("%v\n%v\n%v", item["name"], item["version"], item["text"])
	// 	}
	// }
}

func list() {
	fmt.Println("List of all available OSS licenses:")
}

func generate(licensename string) {
	fmt.Println(licensename)
}

//parseArgs parses command line arguments and calls appropriate functions.
func parseArgs(args []string) {
	// if len(args) != 0 {
	// 	switch args[0] {
	// 	case "help":
	// 		help()
	// 	case "info":
	// 		if len(args) != 2 {
	// 			fmt.Println("Incorrect usage!")
	// 			fmt.Println("Please provide name of the license for which more information is required.")
	// 			fmt.Println("Type `licensethis help` for help on proper usage.")
	// 		} else {
	// 			info(args[1])
	// 		}
	// 	case "list":
	// 		list()
	// 	case "generate":
	// 		if len(args) != 2 {
	// 			fmt.Println("Incorrect usage!")
	// 			fmt.Println("Please provide name of the license to generate a license file.")
	// 			fmt.Println("Type `licensethis help` for help on proper usage.")
	// 		} else {
	// 			generate(args[1])
	// 		}
	// 	default:
	// 		fmt.Println("Incorrect usage!")
	// 		fmt.Println("Type `licensethis help` for help on proper usage.")
	// 	}
	// }

	//better structed version of the above block. but very strict. does not give personalized error messages.
	//for example, generate should print more help regarding generate usage

	// !!---FUTURE---!!
	//might have to look into a way to accept more than one license name for `info`.
	if len(args) == 1 {
		switch args[0] {
		case "help":
			help()
		case "list":
			list()
		default:
			fmt.Println("Incorrect usage!")
			fmt.Println("Type `licensethis help` for help on proper usage.")
		}
	} else if len(args) == 2 {
		switch args[0] {
		case "info":
			info(args[1])
		case "generate":
			generate(args[1])
		default:
			fmt.Println("Incorrect usage!")
			fmt.Println("Type `licensethis help` for help on proper usage.")
		}
	} else {
		fmt.Println("Incorrect usage!")
		fmt.Println("Type `licensethis help` for help on proper usage.")
	}
}

func main() {
	args := os.Args[1:]
	parseArgs(args)
}
<|endoftext|>