<commit_before>// Package go-oui provides functions to work with MAC and OUI's
package ouidb

import (
	"bufio"
	"bytes"
	"errors"
	"net"
	"os"
	"regexp"
	"strconv"
)

// https://code.wireshark.org/review/gitweb?p=wireshark.git;a=blob_plain;f=manuf
// Bigger than we need, not too big to worry about overflow
const big = 0xFFFFFF

var ErrInvalidMACAddress = errors.New("invalid MAC address")

// Hexadecimal to integer starting at &s[i0].
// Returns number, new offset, success.
func xtoi(s string, i0 int) (n int, i int, ok bool) {
	n = 0
	for i = i0; i < len(s); i++ {
		if '0' <= s[i] && s[i] <= '9' {
			n *= 16
			n += int(s[i] - '0')
		} else if 'a' <= s[i] && s[i] <= 'f' {
			n *= 16
			n += int(s[i]-'a') + 10
		} else if 'A' <= s[i] && s[i] <= 'F' {
			n *= 16
			n += int(s[i]-'A') + 10
		} else {
			break
		}
		if n >= big {
			return 0, i, false
		}
	}
	if i == i0 {
		return 0, i, false
	}
	return n, i, true
}

// xtoi2 converts the next two hex digits of s into a byte.
// If s is longer than 2 bytes then the third byte must be e.
// If the first two bytes of s are not hex digits or the third byte
// does not match e, false is returned.
func xtoi2(s string, e byte) (byte, bool) {
	if len(s) > 2 && s[2] != e {
		return 0, false
	}
	n, ei, ok := xtoi(s[:2], 0)
	return byte(n), ok && ei == 2
}

const hexDigit = "0123456789abcdef"

type HardwareAddr net.HardwareAddr

// ParseMAC parses s as an IEEE 802 MAC-48, EUI-48, or EUI-64 using one of the
// following formats:
//   01:23:45:67:89:ab
//   01:23:45:67:89:ab:cd:ef
//   01-23-45-67-89-ab
//   01-23-45-67-89-ab-cd-ef
//   0123.4567.89ab
//   0123.4567.89ab.cdef
func ParseOUI(s string, size int) (hw HardwareAddr, err error) {
	if s[2] == ':' || s[2] == '-' {
		if (len(s)+1)%3 != 0 {
			goto error
		}

		n := (len(s) + 1) / 3

		hw = make(HardwareAddr, size)
		for x, i := 0, 0; i < n; i++ {
			var ok bool
			if hw[i], ok = xtoi2(s[x:], s[2]); !ok {
				goto error
			}
			x += 3
		}
	} else {
		goto error
	}
	return hw, nil

error:
	return nil, ErrInvalidMACAddress
}

// Mask returns the result of masking the address with mask.
func (address HardwareAddr) Mask(mask []byte) []byte {
	n := len(address)
	if n != len(mask) {
		return nil
	}
	out := make([]byte, n)
	for i := 0; i < n; i++ {
		out[i] = address[i] & mask[i]
	}
	return out
}

type t2 struct {
	T3    map[byte]t2
	Block *AddressBlock
}

type OuiDb struct {
	hw   [6]byte
	mask int

	dict   [][]byte
	Blocks []AddressBlock

	t map[int]t2
}

// New returns a new OUI database loaded from the specified file.
func New(file string) *OuiDb {
	db := &OuiDb{}
	if err := db.Load(file); err != nil {
		return nil
	}
	return db
}

// Lookup finds the OUI the address belongs to
func (m *OuiDb) Lookup(address HardwareAddr) *AddressBlock {
	for _, block := range m.Blocks {
		if block.Contains(address) {
			return &block
		}
	}

	return nil
}

// VendorLookup obtains the vendor organization name from the MAC address s.
func (m *OuiDb) VendorLookup(s string) (string, error) {
	addr, err := net.ParseMAC(s)
	if err != nil {
		return "", err
	}
	block := m.Lookup(HardwareAddr(addr))
	if block == nil {
		return "", ErrInvalidMACAddress
	}
	return block.Organization, nil
}

func byteIndex(s string, c byte) int {
	for i := 0; i < len(s); i++ {
		if s[i] == c {
			return i
		}
	}
	return -1
}

func (m *OuiDb) Load(path string) error {
	file, err := os.Open(path)
	if err != nil {
		return (err)
	}

	fieldsRe := regexp.MustCompile(`^(\S+)\t+(\S+)(\s+#\s+(\S.*))?`)

	re := regexp.MustCompile(`((?:(?:[0-9a-zA-Z]{2})[-:]){2,5}(?:[0-9a-zA-Z]{2}))(?:/(\w{1,2}))?`)

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		text := scanner.Text()
		if text == "" || text[0] == '#' || text[0] == '\t' {
			continue
		}

		block := AddressBlock{}

		// Split input text into address, short organization name
		// and full organization name
		fields := fieldsRe.FindAllStringSubmatch(text, -1)
		addr := fields[0][1]
		if fields[0][4] != "" {
			block.Organization = fields[0][4]
		} else {
			block.Organization = fields[0][2]
		}

		matches := re.FindAllStringSubmatch(addr, -1)
		if len(matches) == 0 {
			continue
		}

		s := matches[0][1]

		i := byteIndex(s, '/')

		if i == -1 {
			block.Oui, err = ParseOUI(s, 6)
			block.Mask = 24 // len(block.Oui) * 8
		} else {
			block.Oui, err = ParseOUI(s[:i], 6)
			block.Mask, err = strconv.Atoi(s[i+1:])
		}

		//fmt.Println("OUI:", block.Oui, block.Mask, err)

		m.Blocks = append(m.Blocks, block)

		// create smart map
		for i := len(block.Oui) - 1; i >= 0; i-- {
			_ = block.Oui[i]

		}

		// fmt.Printf("BLA %v %v ALB", m.hw, m.mask)
	}

	if err := scanner.Err(); err != nil {
		return (err)
	}

	return (nil)
}

func CIDRMask(ones, bits int) []byte {
	l := bits / 8
	m := make([]byte, l)

	n := uint(ones)
	for i := 0; i < l; i++ {
		if n >= 8 {
			m[i] = 0xff
			n -= 8
			continue
		}
		m[i] = ^byte(0xff >> n)
		n = 0
	}

	return (m)
}

// oui, mask, organization
type AddressBlock struct {
	Oui          HardwareAddr
	Mask         int
	Organization string
}

// Contains reports whether the mac address belongs to the OUI
func (b *AddressBlock) Contains(address HardwareAddr) bool {
	//fmt.Println("%v %v %v %v", b.Oui, len(b.Oui), address.Mask(CIDRMask(b.Mask, len(b.Oui)*8)), CIDRMask(b.Mask, len(b.Oui)*8))

	return (bytes.Equal(address.Mask(CIDRMask(b.Mask, len(b.Oui)*8)), b.Oui))
}
<commit_msg>Change internal data to store OUIs as byte array<commit_after>// Package go-oui provides functions to work with MAC and OUI's
package ouidb

import (
	"bufio"
	"encoding/hex"
	"errors"
	"net"
	"os"
	"regexp"
	"strconv"
	"strings"
)

// https://code.wireshark.org/review/gitweb?p=wireshark.git;a=blob_plain;f=manuf
// Bigger than we need, not too big to worry about overflow
const big = 0xFFFFFF

var ErrInvalidMACAddress = errors.New("invalid MAC address")

type HardwareAddr net.HardwareAddr

func parseMAC(s string) ([6]byte, error) {
	var hw [6]byte

	oct := strings.FieldsFunc(s, func(r rune) bool { return r == ':' || r == '-' })

	_, err := hex.Decode(hw[:], []byte(strings.Join(oct, "")))
	if err != nil {
		return hw, err
	}

	return hw, nil
}

// Mask returns the result of masking the address with mask.
func (address HardwareAddr) Mask(mask []byte) []byte {
	n := len(address)
	if n != len(mask) {
		return nil
	}
	out := make([]byte, n)
	for i := 0; i < n; i++ {
		out[i] = address[i] & mask[i]
	}
	return out
}

type t2 struct {
	T3    map[byte]t2
	Block *AddressBlock
}

type OuiDb struct {
	hw   [6]byte
	mask int

	Blocks []AddressBlock

	t map[int]t2
}

// New returns a new OUI database loaded from the specified file.
func New(file string) *OuiDb {
	db := &OuiDb{}
	if err := db.Load(file); err != nil {
		return nil
	}
	return db
}

// Lookup finds the OUI the address belongs to
func (m *OuiDb) lookup(address [6]byte) *AddressBlock {
	a := macToUint64(address)
	for _, block := range m.Blocks {
		o := macToUint64(block.Oui)
		m := maskToUint64(block.Mask)

		if a &m == o {
			return &block
		}
	}

	return nil
}

// VendorLookup obtains the vendor organization name from the MAC address s.
func (m *OuiDb) VendorLookup(s string) (string, error) {
	addr, err := parseMAC(s)
	if err != nil {
		return "", err
	}
	block := m.lookup(addr)
	if block == nil {
		return "", ErrInvalidMACAddress
	}
	return block.Organization, nil
}

func byteIndex(s string, c byte) int {
	for i := 0; i < len(s); i++ {
		if s[i] == c {
			return i
		}
	}
	return -1
}

func (m *OuiDb) Load(path string) error {
	file, err := os.Open(path)
	if err != nil {
		return (err)
	}

	fieldsRe := regexp.MustCompile(`^(\S+)\t+(\S+)(\s+#\s+(\S.*))?`)

	re := regexp.MustCompile(`((?:(?:[0-9a-zA-Z]{2})[-:]){2,5}(?:[0-9a-zA-Z]{2}))(?:/(\w{1,2}))?`)

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		text := scanner.Text()
		if text == "" || text[0] == '#' || text[0] == '\t' {
			continue
		}

		block := AddressBlock{}

		// Split input text into address, short organization name
		// and full organization name
		fields := fieldsRe.FindAllStringSubmatch(text, -1)
		addr := fields[0][1]
		if fields[0][4] != "" {
			block.Organization = fields[0][4]
		} else {
			block.Organization = fields[0][2]
		}

		matches := re.FindAllStringSubmatch(addr, -1)
		if len(matches) == 0 {
			continue
		}

		s := matches[0][1]

		if i := byteIndex(s, '/'); i < 0 {
			block.Oui, err = parseMAC(s)
			block.Mask = 24 // len(block.Oui) * 8
		} else {
			var mask int
			block.Oui, err = parseMAC(s[:i])
			mask, err = strconv.Atoi(s[i+1:])
			block.Mask = uint8(mask)
		}

		if err != nil {
			continue
		}

		//fmt.Println("OUI:", block.Oui, block.Mask, err)

		m.Blocks = append(m.Blocks, block)

		// create smart map
		for i := len(block.Oui) - 1; i >= 0; i-- {
			_ = block.Oui[i]

		}

		// fmt.Printf("BLA %v %v ALB", m.hw, m.mask)
	}

	if err := scanner.Err(); err != nil {
		return err
	}

	return nil
}

func CIDRMask(ones, bits int) []byte {
	l := bits / 8
	m := make([]byte, l)

	n := uint(ones)
	for i := 0; i < l; i++ {
		if n >= 8 {
			m[i] = 0xff
			n -= 8
			continue
		}
		m[i] = ^byte(0xff >> n)
		n = 0
	}

	return (m)
}

// oui, mask, organization
type AddressBlock struct {
	Oui          [6]uint8
	Mask         uint8
	Organization string
}

func macToUint64(address [6]byte) uint64 {
	var a uint64
	for _, x := range address {
		a <<= 8
		a |= uint64(x)
	}
	return a
}

func maskToUint64(mask uint8) uint64 {
	return ^(uint64(1)<<(48-mask) - 1)
}

<|endoftext|>


<commit_before>package outbound

import (
	"bufio"
	"bytes"
	"compress/flate"
	"compress/gzip"
	"compress/zlib"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"reflect"
	"strings"
	"time"
)

type Request struct {
	headers           []headerTuple
	Method            string
	Uri               string
	Body              interface{}
	QueryString       interface{}
	Timeout           time.Duration
	ContentType       string
	Accept            string
	Host              string
	UserAgent         string
	Insecure          bool
	MaxRedirects      int
	Proxy             string
	Compression       *compression
	BasicAuthUsername string
	BasicAuthPassword string
}

type Response struct {
	StatusCode    int
	ContentLength int64
	Body          *Body
	Header        http.Header
}

type Body struct {
	reader           io.ReadCloser
	compressedReader io.ReadCloser
}

type Error struct {
	timeout bool
	Err     error
}

func (e *Error) Timeout() bool {
	return e.timeout
}

func (e *Error) Error() string {
	return e.Err.Error()
}

func (b *Body) Read(p []byte) (int, error) {
	if b.compressedReader != nil {
		return b.compressedReader.Read(p)
	}
	return b.reader.Read(p)
}

func (b *Body) Close() error {
	err := b.reader.Close()
	if b.compressedReader != nil {
		return b.compressedReader.Close()
	}
	return err
}

func paramParse(query interface{}) (string, error) {
	var (
		v = &url.Values{}
		s = reflect.ValueOf(query)
		t = reflect.TypeOf(query)
	)

	switch query.(type) {
	case url.Values:
		return query.(url.Values).Encode(), nil
	default:
		for i := 0; i < s.NumField(); i++ {
			v.Add(strings.ToLower(t.Field(i).Name), fmt.Sprintf("%v", s.Field(i).Interface()))
		}
		return v.Encode(), nil
	}
}

func prepareRequestBody(b interface{}) (io.Reader, error) {
	switch b.(type) {
	case string:
		return strings.NewReader(b.(string)), nil
	case io.Reader:
		return b.(io.Reader), nil
	case []byte:
		return bytes.NewReader(b.([]byte)), nil
	case nil:
		return nil, nil
	default:
		j, err := json.Marshal(b)
		if err == nil {
			return bytes.NewReader(j), nil
		}
		return nil, err
	}
}
func (r Request) Send() (*Response, error) {
	var req *http.Request
	var er error
	var transport = defaultTransport
	var client = defaultClient

	if r.Insecure {
		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
	} else if transport.TLSClientConfig != nil {
		transport.TLSClientConfig.InsecureSkipVerify = false
	}

	b, e := prepareRequestBody(r.Body)
	if e != nil {
		return nil, &Error{Err: e}
	}

	if r.QueryString != nil {
		param, e := paramParse(r.QueryString)
		if e != nil {
			return nil, &Error{Err: e}
		}
		r.Uri = r.Uri + "?" + param
	}

	var bodyReader io.Reader
	if b != nil && r.Compression != nil {
		buffer := bytes.NewBuffer([]byte{})
		readBuffer := bufio.NewReader(b)
		writer, err := r.Compression.writer(buffer)
		if err != nil {
			return nil, &Error{Err: err}
		}
		_, e = readBuffer.WriteTo(writer)
		writer.Close()
		if e != nil {
			return nil, &Error{Err: e}
		}
		bodyReader = buffer
	} else {
		bodyReader = b
	}
	req, er = http.NewRequest(r.Method, r.Uri, bodyReader)

	if er != nil {
		return nil, &Error{Err: er}
	}

	req.Host = r.Host
	req.Header.Add("User-Agent", r.UserAgent)
	req.Header.Add("Content-Type", r.ContentType)
	req.Header.Add("Accept", r.Accept)
	if r.Compression != nil {
		req.Header.Add("Content-Encoding", r.Compression.ContentEncoding)
		req.Header.Add("Accept-Encoding", r.Compression.ContentEncoding)
	}
	if r.headers != nil {
		for _, header := range r.headers {
			req.Header.Add(header.name, header.value)
		}
	}

	if r.BasicAuthUsername != "" && r.BasicAuthPassword != "" {
		req.SetBasicAuth(r.BasicAuthUsername, r.BasicAuthPassword)
	}

	timeout := false
	var timer *time.Timer
	if r.Timeout > 0 {
		timer = time.AfterFunc(r.Timeout, func() {
			transport.CancelRequest(req)
			timeout = true
		})
	}

	res, err := client.Do(req)
	if timer != nil {
		timer.Stop()
	}

	if err != nil {
		if !timeout {
			switch err := err.(type) {
			case *net.OpError:
				timeout = err.Timeout()
			case *url.Error:
				if op, ok := err.Err.(*net.OpError); ok {
					timeout = op.Timeout()
				}
			}
		}

		return nil, &Error{timeout: timeout, Err: err}
	}

	if isRedirect(res.StatusCode) && r.MaxRedirects > 0 {
		loc, _ := res.Location()
		r.MaxRedirects--
		r.Uri = loc.String()
		return r.Send()
	}

	if r.Compression != nil && strings.Contains(res.Header.Get("Content-Encoding"), r.Compression.ContentEncoding) {
		compressedReader, err := r.Compression.reader(res.Body)
		if err != nil {
			return nil, &Error{Err: err}
		}
		return &Response{StatusCode: res.StatusCode, ContentLength: res.ContentLength, Header: res.Header, Body: &Body{reader: res.Body, compressedReader: compressedReader}}, nil
	} else {
		return &Response{StatusCode: res.StatusCode, ContentLength: res.ContentLength, Header: res.Header, Body: &Body{reader: res.Body}}, nil
	}
}

func isRedirect(status int) bool {
	switch status {
	case http.StatusMovedPermanently:
		return true
	case http.StatusFound:
		return true
	case http.StatusSeeOther:
		return true
	case http.StatusTemporaryRedirect:
		return true
	default:
		return false
	}
}
<commit_msg>proxy stuffs<commit_after>package outbound

import (
	"bufio"
	"bytes"
	"compress/flate"
	"compress/gzip"
	"compress/zlib"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"reflect"
	"strings"
	"time"
)

type Request struct {
	headers           []headerTuple
	Method            string
	Uri               string
	Body              interface{}
	QueryString       interface{}
	Timeout           time.Duration
	ContentType       string
	Accept            string
	Host              string
	UserAgent         string
	Insecure          bool
	MaxRedirects      int
	Proxy             string
	Compression       *compression
	BasicAuthUsername string
	BasicAuthPassword string
}

type Response struct {
	StatusCode    int
	ContentLength int64
	Body          *Body
	Header        http.Header
}

type Body struct {
	reader           io.ReadCloser
	compressedReader io.ReadCloser
}

type Error struct {
	timeout bool
	Err     error
}

func (e *Error) Timeout() bool {
	return e.timeout
}

func (e *Error) Error() string {
	return e.Err.Error()
}

func (b *Body) Read(p []byte) (int, error) {
	if b.compressedReader != nil {
		return b.compressedReader.Read(p)
	}
	return b.reader.Read(p)
}

func (b *Body) Close() error {
	err := b.reader.Close()
	if b.compressedReader != nil {
		return b.compressedReader.Close()
	}
	return err
}

func paramParse(query interface{}) (string, error) {
	var (
		v = &url.Values{}
		s = reflect.ValueOf(query)
		t = reflect.TypeOf(query)
	)

	switch query.(type) {
	case url.Values:
		return query.(url.Values).Encode(), nil
	default:
		for i := 0; i < s.NumField(); i++ {
			v.Add(strings.ToLower(t.Field(i).Name), fmt.Sprintf("%v", s.Field(i).Interface()))
		}
		return v.Encode(), nil
	}
}

func prepareRequestBody(b interface{}) (io.Reader, error) {
	switch b.(type) {
	case string:
		return strings.NewReader(b.(string)), nil
	case io.Reader:
		return b.(io.Reader), nil
	case []byte:
		return bytes.NewReader(b.([]byte)), nil
	case nil:
		return nil, nil
	default:
		j, err := json.Marshal(b)
		if err == nil {
			return bytes.NewReader(j), nil
		}
		return nil, err
	}
}
func (r Request) Send() (*Response, error) {
	var req *http.Request
	var er error
	var transport = defaultTransport
	var client = defaultClient

	if r.Proxy != "" {
		transport = proxyTransport
		client = proxyClient
	}

	if r.Insecure {
		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
	} else if transport.TLSClientConfig != nil {
		transport.TLSClientConfig.InsecureSkipVerify = false
	}

	b, e := prepareRequestBody(r.Body)
	if e != nil {
		return nil, &Error{Err: e}
	}

	if r.QueryString != nil {
		param, e := paramParse(r.QueryString)
		if e != nil {
			return nil, &Error{Err: e}
		}
		r.Uri = r.Uri + "?" + param
	}

	var bodyReader io.Reader
	if b != nil && r.Compression != nil {
		buffer := bytes.NewBuffer([]byte{})
		readBuffer := bufio.NewReader(b)
		writer, err := r.Compression.writer(buffer)
		if err != nil {
			return nil, &Error{Err: err}
		}
		_, e = readBuffer.WriteTo(writer)
		writer.Close()
		if e != nil {
			return nil, &Error{Err: e}
		}
		bodyReader = buffer
	} else {
		bodyReader = b
	}
	req, er = http.NewRequest(r.Method, r.Uri, bodyReader)

	if er != nil {
		return nil, &Error{Err: er}
	}

	req.Host = r.Host
	req.Header.Add("User-Agent", r.UserAgent)
	req.Header.Add("Content-Type", r.ContentType)
	req.Header.Add("Accept", r.Accept)
	if r.Compression != nil {
		req.Header.Add("Content-Encoding", r.Compression.ContentEncoding)
		req.Header.Add("Accept-Encoding", r.Compression.ContentEncoding)
	}
	if r.headers != nil {
		for _, header := range r.headers {
			req.Header.Add(header.name, header.value)
		}
	}

	if r.BasicAuthUsername != "" && r.BasicAuthPassword != "" {
		req.SetBasicAuth(r.BasicAuthUsername, r.BasicAuthPassword)
	}

	timeout := false
	var timer *time.Timer
	if r.Timeout > 0 {
		timer = time.AfterFunc(r.Timeout, func() {
			transport.CancelRequest(req)
			timeout = true
		})
	}

	res, err := client.Do(req)
	if timer != nil {
		timer.Stop()
	}

	if err != nil {
		if !timeout {
			switch err := err.(type) {
			case *net.OpError:
				timeout = err.Timeout()
			case *url.Error:
				if op, ok := err.Err.(*net.OpError); ok {
					timeout = op.Timeout()
				}
			}
		}

		return nil, &Error{timeout: timeout, Err: err}
	}

	if isRedirect(res.StatusCode) && r.MaxRedirects > 0 {
		loc, _ := res.Location()
		r.MaxRedirects--
		r.Uri = loc.String()
		return r.Send()
	}

	if r.Compression != nil && strings.Contains(res.Header.Get("Content-Encoding"), r.Compression.ContentEncoding) {
		compressedReader, err := r.Compression.reader(res.Body)
		if err != nil {
			return nil, &Error{Err: err}
		}
		return &Response{StatusCode: res.StatusCode, ContentLength: res.ContentLength, Header: res.Header, Body: &Body{reader: res.Body, compressedReader: compressedReader}}, nil
	} else {
		return &Response{StatusCode: res.StatusCode, ContentLength: res.ContentLength, Header: res.Header, Body: &Body{reader: res.Body}}, nil
	}
}

func isRedirect(status int) bool {
	switch status {
	case http.StatusMovedPermanently:
		return true
	case http.StatusFound:
		return true
	case http.StatusSeeOther:
		return true
	case http.StatusTemporaryRedirect:
		return true
	default:
		return false
	}
}
<|endoftext|>


<commit_before>package cli

import (
	"fmt"
	"os"
)

var version = "v1.4.5"

var optionDocs = map[string]string{
	"-d": "Show debug message",
	"-v": "Show version",
	"-h": "Show help",
}

var cmds = []string{
	"account",
	"dircache",
	"listbucket",
	"alilistbucket",
	"prefop",
	"fput",
	"rput",
	"qupload",
	"qdownload",
	"stat",
	"delete",
	"move",
	"copy",
	"chgm",
	"fetch",
	"prefetch",
	"batchstat",
	"batchdelete",
	"batchchgm",
	"batchcopy",
	"batchmove",
	"batchrename",
	"checkqrsync",
	"b64encode",
	"b64decode",
	"urlencode",
	"urldecode",
	"ts2d",
	"tms2d",
	"tns2d",
	"d2ts",
	"ip",
	"qetag",
	"unzip",
	"privateurl",
	"saveas",
	"reqid",
	"m3u8delete",
	"buckets",
	"domains",
}
var cmdDocs = map[string][]string{
	"account":       []string{"qshell [-d] account [<AccessKey> <SecretKey>]", "Get/Set AccessKey and SecretKey"},
	"dircache":      []string{"qshell [-d] dircache <DirCacheRootPath> <DirCacheResultFile>", "Cache the directory structure of a file path"},
	"listbucket":    []string{"qshell [-d] listbucket <Bucket> [<Prefix>] <ListBucketResultFile>", "List all the file in the bucket by prefix"},
	"alilistbucket": []string{"qshell [-d] alilistbucket <DataCenter> <Bucket> <AccessKeyId> <AccesskeySecret> [Prefix] <ListBucketResultFile>", "List all the file in the bucket of aliyun oss by prefix"},
	"prefop":        []string{"qshell [-d] prefop <PersistentId>", "Query the fop status"},
	"fput":          []string{"qshell [-d] fput <Bucket> <Key> <LocalFile> [MimeType] [UpHost]", "Form upload a local file"},
	"rput":          []string{"qshell [-d] rput <Bucket> <Key> <LocalFile> [MimeType] [UpHost]", "Resumable upload a local file"},
	"qupload":       []string{"qshell [-d] qupload [<ThreadCount>] <LocalUploadConfig>", "Batch upload files to the qiniu bucket"},
	"qdownload":     []string{"qshell [-d] qdownload [<ThreadCount>] <LocalDownloadConfig>", "Batch download files from the qiniu bucket"},
	"stat":          []string{"qshell [-d] stat <Bucket> <Key>", "Get the basic info of a remote file"},
	"delete":        []string{"qshell [-d] delete <Bucket> <Key>", "Delete a remote file in the bucket"},
	"move":          []string{"qshell [-d] move <SrcBucket> <SrcKey> <DestBucket> <DestKey>", "Move/Rename a file and save in bucket"},
	"copy":          []string{"qshell [-d] copy <SrcBucket> <SrcKey> <DestBucket> [<DestKey>]", "Make a copy of a file and save in bucket"},
	"chgm":          []string{"qshell [-d] chgm <Bucket> <Key> <NewMimeType>", "Change the mimeType of a file"},
	"fetch":         []string{"qshell [-d] fetch <RemoteResourceUrl> <Bucket> [<Key>]", "Fetch a remote resource by url and save in bucket"},
	"prefetch":      []string{"qshell [-d] prefetch <Bucket> <Key>", "Fetch and update the file in bucket using mirror storage"},
	"batchstat":     []string{"qshell [-d] batchstat <Bucket> <KeyListFile>", "Batch stat files in bucket"},
	"batchdelete":   []string{"qshell [-d] batchdelete <Bucket> <KeyListFile>", "Batch delete files in bucket"},
	"batchchgm":     []string{"qshell [-d] batchchgm <Bucket> <KeyMimeMapFile>", "Batch chgm files in bucket"},
	"batchcopy":     []string{"qshell [-d] batchcopy <SrcBucket> <DestBucket> <SrcDestKeyMapFile>", "Batch copy files from bucket to bucket"},
	"batchmove":     []string{"qshell [-d] batchmove <SrcBucket> <DestBucket> <SrcDestKeyMapFile>", "Batch move files from bucket to bucket"},
	"batchrename":   []string{"qshell [-d] batchrename <Bucket> <OldNewKeyMapFile>", "Batch rename files in the bucket"},
	"checkqrsync":   []string{"qshell [-d] checkqrsync <DirCacheResultFile> <ListBucketResultFile> <IgnoreLocalDir> [Prefix]", "Check the qrsync result"},
	"b64encode":     []string{"qshell [-d] b64encode [<UrlSafe>] <DataToEncode>", "Base64 Encode"},
	"b64decode":     []string{"qshell [-d] b64decode [<UrlSafe>] <DataToDecode>", "Base64 Decode"},
	"urlencode":     []string{"qshell [-d] urlencode <DataToEncode>", "Url encode"},
	"urldecode":     []string{"qshell [-d] urldecode <DataToDecode>", "Url decode"},
	"ts2d":          []string{"qshell [-d] ts2d <TimestampInSeconds>", "Convert timestamp in seconds to a date (TZ: Local)"},
	"tms2d":         []string{"qshell [-d] tms2d <TimestampInMilliSeconds>", "Convert timestamp in milli-seconds to a date (TZ: Local)"},
	"tns2d":         []string{"qshell [-d] tns2d <TimestampIn100NanoSeconds>", "Convert timestamp in 100 nano-seconds to a date (TZ: Local)"},
	"d2ts":          []string{"qshell [-d] d2ts <SecondsToNow>", "Create a timestamp in seconds using seconds to now"},
	"ip":            []string{"qshell [-d] ip <Ip1> [<Ip2> [<Ip3> ...]]]", "Query the ip information"},
	"qetag":         []string{"qshell [-d] qetag <LocalFilePath>", "Calculate the hash of local file using the algorithm of qiniu qetag"},
	"unzip":         []string{"qshell [-d] unzip <QiniuZipFilePath> [<UnzipToDir>]", "Unzip the archive file created by the qiniu mkzip API"},
	"privateurl":    []string{"qshell [-d] privateurl <PublicUrl> [<Deadline>]", "Create private resource access url"},
	"saveas":        []string{"qshell [-d] saveas <PublicUrlWithFop> <SaveBucket> <SaveKey>", "Create a resource access url with fop and saveas"},
	"reqid":         []string{"qshell [-d] reqid <ReqIdToDecode>", "Decode a qiniu reqid"},
	"m3u8delete":    []string{"qshell [-d] m3u8delete <Bucket> <M3u8Key> [<IsPrivate>]", "Delete m3u8 playlist and the slices it references"},
	"buckets":       []string{"qshell [-d] buckets", "Get all buckets of the account"},
	"domains":       []string{"qshell [-d] domains <Bucket>", "Get all domains of the bucket"},
}

func Version() {
	fmt.Println("qshell", version)
}

func Help(cmd string, params ...string) {
	if len(params) == 0 {
		fmt.Println(CmdList())
	} else {
		CmdHelps(params...)
	}
}

func CmdList() string {
	helpAll := fmt.Sprintf("QShell %s\r\n\r\n", version)
	helpAll += "Options:\r\n"
	for k, v := range optionDocs {
		helpAll += fmt.Sprintf("\t%-20s%-20s\r\n", k, v)
	}
	helpAll += "\r\n"
	helpAll += "Commands:\r\n"
	for _, cmd := range cmds {
		if help, ok := cmdDocs[cmd]; ok {
			cmdDesc := help[1]
			helpAll += fmt.Sprintf("\t%-20s%-20s\r\n", cmd, cmdDesc)
		}
	}
	return helpAll
}

func CmdHelps(cmds ...string) {
	defer os.Exit(1)
	if len(cmds) == 0 {
		fmt.Println(CmdList())
	} else {
		for _, cmd := range cmds {
			CmdHelp(cmd)
		}
	}
}

func CmdHelp(cmd string) {
	docStr := fmt.Sprintf("Unknow cmd `%s'", cmd)
	if cmdDoc, ok := cmdDocs[cmd]; ok {
		docStr = fmt.Sprintf("Usage: %s\r\n  %s\r\n", cmdDoc[0], cmdDoc[1])
	}
	fmt.Println(docStr)
}
<commit_msg>Update version code.<commit_after>package cli

import (
	"fmt"
	"os"
)

var version = "v1.4.6"

var optionDocs = map[string]string{
	"-d": "Show debug message",
	"-v": "Show version",
	"-h": "Show help",
}

var cmds = []string{
	"account",
	"dircache",
	"listbucket",
	"alilistbucket",
	"prefop",
	"fput",
	"rput",
	"qupload",
	"qdownload",
	"stat",
	"delete",
	"move",
	"copy",
	"chgm",
	"fetch",
	"prefetch",
	"batchstat",
	"batchdelete",
	"batchchgm",
	"batchcopy",
	"batchmove",
	"batchrename",
	"checkqrsync",
	"b64encode",
	"b64decode",
	"urlencode",
	"urldecode",
	"ts2d",
	"tms2d",
	"tns2d",
	"d2ts",
	"ip",
	"qetag",
	"unzip",
	"privateurl",
	"saveas",
	"reqid",
	"m3u8delete",
	"buckets",
	"domains",
}
var cmdDocs = map[string][]string{
	"account":       []string{"qshell [-d] account [<AccessKey> <SecretKey>]", "Get/Set AccessKey and SecretKey"},
	"dircache":      []string{"qshell [-d] dircache <DirCacheRootPath> <DirCacheResultFile>", "Cache the directory structure of a file path"},
	"listbucket":    []string{"qshell [-d] listbucket <Bucket> [<Prefix>] <ListBucketResultFile>", "List all the file in the bucket by prefix"},
	"alilistbucket": []string{"qshell [-d] alilistbucket <DataCenter> <Bucket> <AccessKeyId> <AccesskeySecret> [Prefix] <ListBucketResultFile>", "List all the file in the bucket of aliyun oss by prefix"},
	"prefop":        []string{"qshell [-d] prefop <PersistentId>", "Query the fop status"},
	"fput":          []string{"qshell [-d] fput <Bucket> <Key> <LocalFile> [MimeType] [UpHost]", "Form upload a local file"},
	"rput":          []string{"qshell [-d] rput <Bucket> <Key> <LocalFile> [MimeType] [UpHost]", "Resumable upload a local file"},
	"qupload":       []string{"qshell [-d] qupload [<ThreadCount>] <LocalUploadConfig>", "Batch upload files to the qiniu bucket"},
	"qdownload":     []string{"qshell [-d] qdownload [<ThreadCount>] <LocalDownloadConfig>", "Batch download files from the qiniu bucket"},
	"stat":          []string{"qshell [-d] stat <Bucket> <Key>", "Get the basic info of a remote file"},
	"delete":        []string{"qshell [-d] delete <Bucket> <Key>", "Delete a remote file in the bucket"},
	"move":          []string{"qshell [-d] move <SrcBucket> <SrcKey> <DestBucket> <DestKey>", "Move/Rename a file and save in bucket"},
	"copy":          []string{"qshell [-d] copy <SrcBucket> <SrcKey> <DestBucket> [<DestKey>]", "Make a copy of a file and save in bucket"},
	"chgm":          []string{"qshell [-d] chgm <Bucket> <Key> <NewMimeType>", "Change the mimeType of a file"},
	"fetch":         []string{"qshell [-d] fetch <RemoteResourceUrl> <Bucket> [<Key>]", "Fetch a remote resource by url and save in bucket"},
	"prefetch":      []string{"qshell [-d] prefetch <Bucket> <Key>", "Fetch and update the file in bucket using mirror storage"},
	"batchstat":     []string{"qshell [-d] batchstat <Bucket> <KeyListFile>", "Batch stat files in bucket"},
	"batchdelete":   []string{"qshell [-d] batchdelete <Bucket> <KeyListFile>", "Batch delete files in bucket"},
	"batchchgm":     []string{"qshell [-d] batchchgm <Bucket> <KeyMimeMapFile>", "Batch chgm files in bucket"},
	"batchcopy":     []string{"qshell [-d] batchcopy <SrcBucket> <DestBucket> <SrcDestKeyMapFile>", "Batch copy files from bucket to bucket"},
	"batchmove":     []string{"qshell [-d] batchmove <SrcBucket> <DestBucket> <SrcDestKeyMapFile>", "Batch move files from bucket to bucket"},
	"batchrename":   []string{"qshell [-d] batchrename <Bucket> <OldNewKeyMapFile>", "Batch rename files in the bucket"},
	"checkqrsync":   []string{"qshell [-d] checkqrsync <DirCacheResultFile> <ListBucketResultFile> <IgnoreLocalDir> [Prefix]", "Check the qrsync result"},
	"b64encode":     []string{"qshell [-d] b64encode [<UrlSafe>] <DataToEncode>", "Base64 Encode"},
	"b64decode":     []string{"qshell [-d] b64decode [<UrlSafe>] <DataToDecode>", "Base64 Decode"},
	"urlencode":     []string{"qshell [-d] urlencode <DataToEncode>", "Url encode"},
	"urldecode":     []string{"qshell [-d] urldecode <DataToDecode>", "Url decode"},
	"ts2d":          []string{"qshell [-d] ts2d <TimestampInSeconds>", "Convert timestamp in seconds to a date (TZ: Local)"},
	"tms2d":         []string{"qshell [-d] tms2d <TimestampInMilliSeconds>", "Convert timestamp in milli-seconds to a date (TZ: Local)"},
	"tns2d":         []string{"qshell [-d] tns2d <TimestampIn100NanoSeconds>", "Convert timestamp in 100 nano-seconds to a date (TZ: Local)"},
	"d2ts":          []string{"qshell [-d] d2ts <SecondsToNow>", "Create a timestamp in seconds using seconds to now"},
	"ip":            []string{"qshell [-d] ip <Ip1> [<Ip2> [<Ip3> ...]]]", "Query the ip information"},
	"qetag":         []string{"qshell [-d] qetag <LocalFilePath>", "Calculate the hash of local file using the algorithm of qiniu qetag"},
	"unzip":         []string{"qshell [-d] unzip <QiniuZipFilePath> [<UnzipToDir>]", "Unzip the archive file created by the qiniu mkzip API"},
	"privateurl":    []string{"qshell [-d] privateurl <PublicUrl> [<Deadline>]", "Create private resource access url"},
	"saveas":        []string{"qshell [-d] saveas <PublicUrlWithFop> <SaveBucket> <SaveKey>", "Create a resource access url with fop and saveas"},
	"reqid":         []string{"qshell [-d] reqid <ReqIdToDecode>", "Decode a qiniu reqid"},
	"m3u8delete":    []string{"qshell [-d] m3u8delete <Bucket> <M3u8Key> [<IsPrivate>]", "Delete m3u8 playlist and the slices it references"},
	"buckets":       []string{"qshell [-d] buckets", "Get all buckets of the account"},
	"domains":       []string{"qshell [-d] domains <Bucket>", "Get all domains of the bucket"},
}

func Version() {
	fmt.Println("qshell", version)
}

func Help(cmd string, params ...string) {
	if len(params) == 0 {
		fmt.Println(CmdList())
	} else {
		CmdHelps(params...)
	}
}

func CmdList() string {
	helpAll := fmt.Sprintf("QShell %s\r\n\r\n", version)
	helpAll += "Options:\r\n"
	for k, v := range optionDocs {
		helpAll += fmt.Sprintf("\t%-20s%-20s\r\n", k, v)
	}
	helpAll += "\r\n"
	helpAll += "Commands:\r\n"
	for _, cmd := range cmds {
		if help, ok := cmdDocs[cmd]; ok {
			cmdDesc := help[1]
			helpAll += fmt.Sprintf("\t%-20s%-20s\r\n", cmd, cmdDesc)
		}
	}
	return helpAll
}

func CmdHelps(cmds ...string) {
	defer os.Exit(1)
	if len(cmds) == 0 {
		fmt.Println(CmdList())
	} else {
		for _, cmd := range cmds {
			CmdHelp(cmd)
		}
	}
}

func CmdHelp(cmd string) {
	docStr := fmt.Sprintf("Unknow cmd `%s'", cmd)
	if cmdDoc, ok := cmdDocs[cmd]; ok {
		docStr = fmt.Sprintf("Usage: %s\r\n  %s\r\n", cmdDoc[0], cmdDoc[1])
	}
	fmt.Println(docStr)
}
<|endoftext|>


<commit_before>package cli

import (
	"fmt"
	"os"
	"runtime"
)

var version = "v1.5.3"

var optionDocs = map[string]string{
	"-d": "Show debug message",
	"-v": "Show version",
	"-h": "Show help",
}

var cmds = []string{
	"account",
	"dircache",
	"listbucket",
	"alilistbucket",
	"prefop",
	"fput",
	"rput",
	"qupload",
	"qdownload",
	"stat",
	"delete",
	"move",
	"copy",
	"chgm",
	"fetch",
	"prefetch",
	"batchstat",
	"batchdelete",
	"batchchgm",
	"batchcopy",
	"batchmove",
	"batchrename",
	"batchrefresh",
	"batchsign",
	"checkqrsync",
	"b64encode",
	"b64decode",
	"urlencode",
	"urldecode",
	"ts2d",
	"tms2d",
	"tns2d",
	"d2ts",
	"ip",
	"qetag",
	"unzip",
	"privateurl",
	"saveas",
	"reqid",
	"m3u8delete",
	"buckets",
	"domains",
}
var cmdDocs = map[string][]string{
	"account":       []string{"qshell [-d] account [<AccessKey> <SecretKey>]", "Get/Set AccessKey and SecretKey"},
	"dircache":      []string{"qshell [-d] dircache <DirCacheRootPath> <DirCacheResultFile>", "Cache the directory structure of a file path"},
	"listbucket":    []string{"qshell [-d] listbucket <Bucket> [<Prefix>] <ListBucketResultFile>", "List all the file in the bucket by prefix"},
	"alilistbucket": []string{"qshell [-d] alilistbucket <DataCenter> <Bucket> <AccessKeyId> <AccesskeySecret> [Prefix] <ListBucketResultFile>", "List all the file in the bucket of aliyun oss by prefix"},
	"prefop":        []string{"qshell [-d] prefop <PersistentId>", "Query the fop status"},
	"fput":          []string{"qshell [-d] fput <Bucket> <Key> <LocalFile> [MimeType] [UpHost]", "Form upload a local file"},
	"rput":          []string{"qshell [-d] rput <Bucket> <Key> <LocalFile> [MimeType] [UpHost]", "Resumable upload a local file"},
	"qupload":       []string{"qshell [-d] qupload [<ThreadCount>] <LocalUploadConfig>", "Batch upload files to the qiniu bucket"},
	"qdownload":     []string{"qshell [-d] qdownload [<ThreadCount>] <LocalDownloadConfig>", "Batch download files from the qiniu bucket"},
	"stat":          []string{"qshell [-d] stat <Bucket> <Key>", "Get the basic info of a remote file"},
	"delete":        []string{"qshell [-d] delete <Bucket> <Key>", "Delete a remote file in the bucket"},
	"move":          []string{"qshell [-d] move <SrcBucket> <SrcKey> <DestBucket> <DestKey>", "Move/Rename a file and save in bucket"},
	"copy":          []string{"qshell [-d] copy <SrcBucket> <SrcKey> <DestBucket> [<DestKey>]", "Make a copy of a file and save in bucket"},
	"chgm":          []string{"qshell [-d] chgm <Bucket> <Key> <NewMimeType>", "Change the mimeType of a file"},
	"fetch":         []string{"qshell [-d] fetch <RemoteResourceUrl> <Bucket> [<Key>]", "Fetch a remote resource by url and save in bucket"},
	"prefetch":      []string{"qshell [-d] prefetch <Bucket> <Key>", "Fetch and update the file in bucket using mirror storage"},
	"batchstat":     []string{"qshell [-d] batchstat <Bucket> <KeyListFile>", "Batch stat files in bucket"},
	"batchdelete":   []string{"qshell [-d] batchdelete <Bucket> <KeyListFile>", "Batch delete files in bucket"},
	"batchchgm":     []string{"qshell [-d] batchchgm <Bucket> <KeyMimeMapFile>", "Batch chgm files in bucket"},
	"batchcopy":     []string{"qshell [-d] batchcopy <SrcBucket> <DestBucket> <SrcDestKeyMapFile>", "Batch copy files from bucket to bucket"},
	"batchmove":     []string{"qshell [-d] batchmove <SrcBucket> <DestBucket> <SrcDestKeyMapFile>", "Batch move files from bucket to bucket"},
	"batchrename":   []string{"qshell [-d] batchrename <Bucket> <OldNewKeyMapFile>", "Batch rename files in the bucket"},
	"batchrefresh":  []string{"qshell [-d] batchrefresh <UrlListFile>", "Batch refresh the cdn cache by the url list file"},
	"batchsign":     []string{"qshell [-d] batchsign <UrlListFile> [<Deadline>]", "Batch create the private url from the public url list file"},
	"checkqrsync":   []string{"qshell [-d] checkqrsync <DirCacheResultFile> <ListBucketResultFile> <IgnoreLocalDir> [Prefix]", "Check the qrsync result"},
	"b64encode":     []string{"qshell [-d] b64encode [<UrlSafe>] <DataToEncode>", "Base64 Encode"},
	"b64decode":     []string{"qshell [-d] b64decode [<UrlSafe>] <DataToDecode>", "Base64 Decode"},
	"urlencode":     []string{"qshell [-d] urlencode <DataToEncode>", "Url encode"},
	"urldecode":     []string{"qshell [-d] urldecode <DataToDecode>", "Url decode"},
	"ts2d":          []string{"qshell [-d] ts2d <TimestampInSeconds>", "Convert timestamp in seconds to a date (TZ: Local)"},
	"tms2d":         []string{"qshell [-d] tms2d <TimestampInMilliSeconds>", "Convert timestamp in milli-seconds to a date (TZ: Local)"},
	"tns2d":         []string{"qshell [-d] tns2d <TimestampIn100NanoSeconds>", "Convert timestamp in 100 nano-seconds to a date (TZ: Local)"},
	"d2ts":          []string{"qshell [-d] d2ts <SecondsToNow>", "Create a timestamp in seconds using seconds to now"},
	"ip":            []string{"qshell [-d] ip <Ip1> [<Ip2> [<Ip3> ...]]]", "Query the ip information"},
	"qetag":         []string{"qshell [-d] qetag <LocalFilePath>", "Calculate the hash of local file using the algorithm of qiniu qetag"},
	"unzip":         []string{"qshell [-d] unzip <QiniuZipFilePath> [<UnzipToDir>]", "Unzip the archive file created by the qiniu mkzip API"},
	"privateurl":    []string{"qshell [-d] privateurl <PublicUrl> [<Deadline>]", "Create private resource access url"},
	"saveas":        []string{"qshell [-d] saveas <PublicUrlWithFop> <SaveBucket> <SaveKey>", "Create a resource access url with fop and saveas"},
	"reqid":         []string{"qshell [-d] reqid <ReqIdToDecode>", "Decode a qiniu reqid"},
	"m3u8delete":    []string{"qshell [-d] m3u8delete <Bucket> <M3u8Key> [<IsPrivate>]", "Delete m3u8 playlist and the slices it references"},
	"buckets":       []string{"qshell [-d] buckets", "Get all buckets of the account"},
	"domains":       []string{"qshell [-d] domains <Bucket>", "Get all domains of the bucket"},
}

func Version() {
	fmt.Println("qshell", version)
}

func Help(cmd string, params ...string) {
	if len(params) == 0 {
		fmt.Println(CmdList())
	} else {
		CmdHelps(params...)
	}
}

func CmdList() string {
	helpAll := fmt.Sprintf("QShell %s\r\n\r\n", version)
	helpAll += "Options:\r\n"
	for k, v := range optionDocs {
		helpAll += fmt.Sprintf("\t%-20s%-20s\r\n", k, v)
	}
	helpAll += "\r\n"
	helpAll += "Commands:\r\n"
	for _, cmd := range cmds {
		if help, ok := cmdDocs[cmd]; ok {
			cmdDesc := help[1]
			helpAll += fmt.Sprintf("\t%-20s%-20s\r\n", cmd, cmdDesc)
		}
	}
	return helpAll
}

func CmdHelps(cmds ...string) {
	defer os.Exit(1)
	if len(cmds) == 0 {
		fmt.Println(CmdList())
	} else {
		for _, cmd := range cmds {
			CmdHelp(cmd)
		}
	}
}

func CmdHelp(cmd string) {
	docStr := fmt.Sprintf("Unknow cmd `%s'", cmd)
	if cmdDoc, ok := cmdDocs[cmd]; ok {
		docStr = fmt.Sprintf("Usage: %s\r\n  %s\r\n", cmdDoc[0], cmdDoc[1])
	}
	fmt.Println(docStr)
}

func UserAgent() string {
	return fmt.Sprintf("QShell/%s (%s; %s; %s)", version, runtime.GOOS, runtime.GOARCH, runtime.Version())
}
<commit_msg>add version 1.5.4<commit_after>package cli

import (
	"fmt"
	"os"
	"runtime"
)

var version = "v1.5.4"

var optionDocs = map[string]string{
	"-d": "Show debug message",
	"-v": "Show version",
	"-h": "Show help",
}

var cmds = []string{
	"account",
	"dircache",
	"listbucket",
	"alilistbucket",
	"prefop",
	"fput",
	"rput",
	"qupload",
	"qdownload",
	"stat",
	"delete",
	"move",
	"copy",
	"chgm",
	"fetch",
	"prefetch",
	"batchstat",
	"batchdelete",
	"batchchgm",
	"batchcopy",
	"batchmove",
	"batchrename",
	"batchrefresh",
	"batchsign",
	"checkqrsync",
	"b64encode",
	"b64decode",
	"urlencode",
	"urldecode",
	"ts2d",
	"tms2d",
	"tns2d",
	"d2ts",
	"ip",
	"qetag",
	"unzip",
	"privateurl",
	"saveas",
	"reqid",
	"m3u8delete",
	"buckets",
	"domains",
}
var cmdDocs = map[string][]string{
	"account":       []string{"qshell [-d] account [<AccessKey> <SecretKey>]", "Get/Set AccessKey and SecretKey"},
	"dircache":      []string{"qshell [-d] dircache <DirCacheRootPath> <DirCacheResultFile>", "Cache the directory structure of a file path"},
	"listbucket":    []string{"qshell [-d] listbucket <Bucket> [<Prefix>] <ListBucketResultFile>", "List all the file in the bucket by prefix"},
	"alilistbucket": []string{"qshell [-d] alilistbucket <DataCenter> <Bucket> <AccessKeyId> <AccesskeySecret> [Prefix] <ListBucketResultFile>", "List all the file in the bucket of aliyun oss by prefix"},
	"prefop":        []string{"qshell [-d] prefop <PersistentId>", "Query the fop status"},
	"fput":          []string{"qshell [-d] fput <Bucket> <Key> <LocalFile> [MimeType] [UpHost]", "Form upload a local file"},
	"rput":          []string{"qshell [-d] rput <Bucket> <Key> <LocalFile> [MimeType] [UpHost]", "Resumable upload a local file"},
	"qupload":       []string{"qshell [-d] qupload [<ThreadCount>] <LocalUploadConfig>", "Batch upload files to the qiniu bucket"},
	"qdownload":     []string{"qshell [-d] qdownload [<ThreadCount>] <LocalDownloadConfig>", "Batch download files from the qiniu bucket"},
	"stat":          []string{"qshell [-d] stat <Bucket> <Key>", "Get the basic info of a remote file"},
	"delete":        []string{"qshell [-d] delete <Bucket> <Key>", "Delete a remote file in the bucket"},
	"move":          []string{"qshell [-d] move <SrcBucket> <SrcKey> <DestBucket> <DestKey>", "Move/Rename a file and save in bucket"},
	"copy":          []string{"qshell [-d] copy <SrcBucket> <SrcKey> <DestBucket> [<DestKey>]", "Make a copy of a file and save in bucket"},
	"chgm":          []string{"qshell [-d] chgm <Bucket> <Key> <NewMimeType>", "Change the mimeType of a file"},
	"fetch":         []string{"qshell [-d] fetch <RemoteResourceUrl> <Bucket> [<Key>]", "Fetch a remote resource by url and save in bucket"},
	"prefetch":      []string{"qshell [-d] prefetch <Bucket> <Key>", "Fetch and update the file in bucket using mirror storage"},
	"batchstat":     []string{"qshell [-d] batchstat <Bucket> <KeyListFile>", "Batch stat files in bucket"},
	"batchdelete":   []string{"qshell [-d] batchdelete <Bucket> <KeyListFile>", "Batch delete files in bucket"},
	"batchchgm":     []string{"qshell [-d] batchchgm <Bucket> <KeyMimeMapFile>", "Batch chgm files in bucket"},
	"batchcopy":     []string{"qshell [-d] batchcopy <SrcBucket> <DestBucket> <SrcDestKeyMapFile>", "Batch copy files from bucket to bucket"},
	"batchmove":     []string{"qshell [-d] batchmove <SrcBucket> <DestBucket> <SrcDestKeyMapFile>", "Batch move files from bucket to bucket"},
	"batchrename":   []string{"qshell [-d] batchrename <Bucket> <OldNewKeyMapFile>", "Batch rename files in the bucket"},
	"batchrefresh":  []string{"qshell [-d] batchrefresh <UrlListFile>", "Batch refresh the cdn cache by the url list file"},
	"batchsign":     []string{"qshell [-d] batchsign <UrlListFile> [<Deadline>]", "Batch create the private url from the public url list file"},
	"checkqrsync":   []string{"qshell [-d] checkqrsync <DirCacheResultFile> <ListBucketResultFile> <IgnoreLocalDir> [Prefix]", "Check the qrsync result"},
	"b64encode":     []string{"qshell [-d] b64encode [<UrlSafe>] <DataToEncode>", "Base64 Encode"},
	"b64decode":     []string{"qshell [-d] b64decode [<UrlSafe>] <DataToDecode>", "Base64 Decode"},
	"urlencode":     []string{"qshell [-d] urlencode <DataToEncode>", "Url encode"},
	"urldecode":     []string{"qshell [-d] urldecode <DataToDecode>", "Url decode"},
	"ts2d":          []string{"qshell [-d] ts2d <TimestampInSeconds>", "Convert timestamp in seconds to a date (TZ: Local)"},
	"tms2d":         []string{"qshell [-d] tms2d <TimestampInMilliSeconds>", "Convert timestamp in milli-seconds to a date (TZ: Local)"},
	"tns2d":         []string{"qshell [-d] tns2d <TimestampIn100NanoSeconds>", "Convert timestamp in 100 nano-seconds to a date (TZ: Local)"},
	"d2ts":          []string{"qshell [-d] d2ts <SecondsToNow>", "Create a timestamp in seconds using seconds to now"},
	"ip":            []string{"qshell [-d] ip <Ip1> [<Ip2> [<Ip3> ...]]]", "Query the ip information"},
	"qetag":         []string{"qshell [-d] qetag <LocalFilePath>", "Calculate the hash of local file using the algorithm of qiniu qetag"},
	"unzip":         []string{"qshell [-d] unzip <QiniuZipFilePath> [<UnzipToDir>]", "Unzip the archive file created by the qiniu mkzip API"},
	"privateurl":    []string{"qshell [-d] privateurl <PublicUrl> [<Deadline>]", "Create private resource access url"},
	"saveas":        []string{"qshell [-d] saveas <PublicUrlWithFop> <SaveBucket> <SaveKey>", "Create a resource access url with fop and saveas"},
	"reqid":         []string{"qshell [-d] reqid <ReqIdToDecode>", "Decode a qiniu reqid"},
	"m3u8delete":    []string{"qshell [-d] m3u8delete <Bucket> <M3u8Key> [<IsPrivate>]", "Delete m3u8 playlist and the slices it references"},
	"buckets":       []string{"qshell [-d] buckets", "Get all buckets of the account"},
	"domains":       []string{"qshell [-d] domains <Bucket>", "Get all domains of the bucket"},
}

func Version() {
	fmt.Println("qshell", version)
}

func Help(cmd string, params ...string) {
	if len(params) == 0 {
		fmt.Println(CmdList())
	} else {
		CmdHelps(params...)
	}
}

func CmdList() string {
	helpAll := fmt.Sprintf("QShell %s\r\n\r\n", version)
	helpAll += "Options:\r\n"
	for k, v := range optionDocs {
		helpAll += fmt.Sprintf("\t%-20s%-20s\r\n", k, v)
	}
	helpAll += "\r\n"
	helpAll += "Commands:\r\n"
	for _, cmd := range cmds {
		if help, ok := cmdDocs[cmd]; ok {
			cmdDesc := help[1]
			helpAll += fmt.Sprintf("\t%-20s%-20s\r\n", cmd, cmdDesc)
		}
	}
	return helpAll
}

func CmdHelps(cmds ...string) {
	defer os.Exit(1)
	if len(cmds) == 0 {
		fmt.Println(CmdList())
	} else {
		for _, cmd := range cmds {
			CmdHelp(cmd)
		}
	}
}

func CmdHelp(cmd string) {
	docStr := fmt.Sprintf("Unknow cmd `%s'", cmd)
	if cmdDoc, ok := cmdDocs[cmd]; ok {
		docStr = fmt.Sprintf("Usage: %s\r\n  %s\r\n", cmdDoc[0], cmdDoc[1])
	}
	fmt.Println(docStr)
}

func UserAgent() string {
	return fmt.Sprintf("QShell/%s (%s; %s; %s)", version, runtime.GOOS, runtime.GOARCH, runtime.Version())
}
<|endoftext|>


<commit_before>package leakybucket

import "testing"

func Test_Open_DB(t *testing.T) {
}
<commit_msg>Added first test case to leaky buckegt<commit_after>package leakybucket

import "testing"
import "../db/sqlconstants"
import "fmt"

func TestNewBucket(t *testing.T) {
	s, err := NewBucket("A11", 10000, 3600)
	if err != nil {
		t.Error(err)
	}

	fmt.Println(s.Name())

	conn, err := newConnection()
	if err != nil {
		t.Error(err)
	}

	goData, err := conn.Select(sqlconstants.SQLITE3_SELECT_ALL_BUCKETS, s.Name())

	if err != nil {
		t.Error(err)
	}

	fmt.Println(goData.GetRows()[0]["CREATED_TIMESTAMP"])

	err = conn.Drop(sqlconstants.SQLITE3_DROP_BUCKETS)

	if err != nil {
		t.Error(err)
	}

	err = conn.Close()

	if err != nil {
		t.Error(err)
	}

}
<|endoftext|>


<commit_before>// The colors package provide a simple way to bring colorful characters to terminal interface.
//
// This example will output the text with a Blue foreground and a Black background
//      color.Println("@{bK}Example Text")
//
// This one will output the text with a red foreground
//      color.Println("@rExample Text")
//
// This one will escape the @
//      color.Println("@@")
//
// Full color syntax code
//      @{rgbcmykwRGBCMYKW}  foreground/background color
//        r/R:  Red
//        g/G:  Green
//        b/B:  Blue
//        c/C:  Cyan
//        m/M:  Magenta
//        y/Y:  Yellow
//        k/K:  Black
//        w/W:  White
//      @{|}  Reset format style
//      @{!./_} Bold / Dim / Italic / Underline
//      @{^&} Blink / Fast blink
//      @{?} Reverse the foreground and background color
//      @{-} Hide the text
// Note some of the functions are not widely supported, like "Fast blink" and "Italic".
package color

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"log"
)

const (
	EscapeChar = '@'       // Escape character for color syntax
	ResetCode  = "\033[0m" // Short for reset to default style
)

// Mapping from character to concrete escape code.
var codeMap = map[int]int{
	'|': 0,
	'!': 1,
	'.': 2,
	'/': 3,
	'_': 4,
	'^': 5,
	'&': 6,
	'?': 7,
	'-': 8,

	'k': 30,
	'r': 31,
	'g': 32,
	'y': 33,
	'b': 34,
	'm': 35,
	'c': 36,
	'w': 37,
	'd': 39,

	'K': 40,
	'R': 41,
	'G': 42,
	'Y': 43,
	'B': 44,
	'M': 45,
	'C': 46,
	'W': 47,
	'D': 49,
}

// Compile color syntax string like "rG" to escape code.
func Colorize(x string) string {
	attr := 0
	fg := 39
	bg := 49

	for _, key := range x {
		c, ok := codeMap[int(key)]
		switch {
		case !ok:
			log.Printf("Wrong color syntax: %c", key)
		case 0 <= c && c <= 8:
			attr = c
		case 30 <= c && c <= 37:
			fg = c
		case 40 <= c && c <= 47:
			bg = c
		}
	}
	return fmt.Sprintf("\033[%d;%d;%dm", attr, fg, bg)
}

// Handle state after meeting one '@'
func compileColorSyntax(input, output *bytes.Buffer) {
	i, _, err := input.ReadRune()
	if err != nil {
		// EOF got
		log.Print("Parse failed on color syntax")
		return
	}

	switch i {
	default:
		output.WriteString(Colorize(string(i)))
	case '{':
		color := bytes.NewBufferString("")
		for {
			i, _, err := input.ReadRune()
			if err != nil {
				log.Print("Parse failed on color syntax")
				break
			}
			if i == '}' {
				break
			}
			color.WriteRune(i)
		}
		output.WriteString(Colorize(color.String()))
	case EscapeChar:
		output.WriteRune(EscapeChar)
	}
}

// Compile the string and replace color syntax with concrete escape code.
func compile(x string) string {
	if x == "" {
		return ""
	}

	input := bytes.NewBufferString(x)
	output := bytes.NewBufferString("")

	for {
		i, _, err := input.ReadRune()
		if err != nil {
			break
		}
		switch i {
		default:
			output.WriteRune(i)
		case EscapeChar:
			compileColorSyntax(input, output)
		}
	}
	return output.String()
}

// Compile multiple values, only do compiling on string type.
func compileValues(a *[]interface{}) {
	for i, x := range *a {
		if str, ok := x.(string); ok {
			(*a)[i] = compile(str)
		}
	}
}

// Similar to fmt.Print, will reset the color at the end.
func Print(a ...interface{}) (int, error) {
	a = append(a, ResetCode)
	compileValues(&a)
	return fmt.Print(a...)
}

// Similar to fmt.Println, will reset the color at the end.
func Println(a ...interface{}) (int, error) {
	a = append(a, ResetCode)
	compileValues(&a)
	return fmt.Println(a...)
}

// Similar to fmt.Printf, will reset the color at the end.
func Printf(format string, a ...interface{}) (int, error) {
	format += ResetCode
	format = compile(format)
	return fmt.Printf(format, a...)
}

// Similar to fmt.Fprint, will reset the color at the end.
func Fprint(w io.Writer, a ...interface{}) (int, error) {
	a = append(a, ResetCode)
	compileValues(&a)
	return fmt.Fprint(w, a...)
}

// Similar to fmt.Fprintln, will reset the color at the end.
func Fprintln(w io.Writer, a ...interface{}) (int, error) {
	a = append(a, ResetCode)
	compileValues(&a)
	return fmt.Fprintln(w, a...)
}

// Similar to fmt.Fprintf, will reset the color at the end.
func Fprintf(w io.Writer, format string, a ...interface{}) (int, error) {
	format += ResetCode
	format = compile(format)
	return fmt.Fprintf(w, format, a...)
}

// Similar to fmt.Sprint, will reset the color at the end.
func Sprint(a ...interface{}) string {
	a = append(a, ResetCode)
	compileValues(&a)
	return fmt.Sprint(a...)
}

// Similar to fmt.Sprintf, will reset the color at the end.
func Sprintf(format string, a ...interface{}) string {
	format += ResetCode
	format = compile(format)
	return fmt.Sprintf(format, a...)
}

// Similar to fmt.Errorf, will reset the color at the end.
func Errorf(format string, a ...interface{}) error {
	return errors.New(Sprintf(format, a...))
}
<commit_msg>Added support for high intensity foreground.<commit_after>// The colors package provide a simple way to bring colorful characters to terminal interface.
//
// This example will output the text with a Blue foreground and a Black background
//      color.Println("@{bK}Example Text")
//
// This one will output the text with a red foreground
//      color.Println("@rExample Text")
//
// This one will escape the @
//      color.Println("@@")
//
// Full color syntax code
//      @{rgbcmykwRGBCMYKW}  foreground/background color
//        r/R:  Red
//        g/G:  Green
//        b/B:  Blue
//        c/C:  Cyan
//        m/M:  Magenta
//        y/Y:  Yellow
//        k/K:  Black
//        w/W:  White
//      @{|}  Reset format style
//      @{!./_} Bold / Dim / Italic / Underline
//      @{^&} Blink / Fast blink
//      @{*} High intensity foreground color
//      @{?} Reverse the foreground and background color
//      @{-} Hide the text
// Note some of the functions are not widely supported, like "Fast blink" and "Italic".
package color

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"log"
)

const (
	EscapeChar = '@'       // Escape character for color syntax
	ResetCode  = "\033[0m" // Short for reset to default style
)

// Mapping from character to concrete escape code.
var codeMap = map[int]int{
	'|': 0,
	'!': 1,
	'.': 2,
	'/': 3,
	'_': 4,
	'^': 5,
	'&': 6,
	'?': 7,
	'-': 8,
	'*': 60,

	'k': 30,
	'r': 31,
	'g': 32,
	'y': 33,
	'b': 34,
	'm': 35,
	'c': 36,
	'w': 37,
	'd': 39,

	'K': 40,
	'R': 41,
	'G': 42,
	'Y': 43,
	'B': 44,
	'M': 45,
	'C': 46,
	'W': 47,
	'D': 49,
}

// Compile color syntax string like "rG" to escape code.
func Colorize(x string) string {
	attr := 0
	fg := 39
	bg := 49

	for _, key := range x {
		c, ok := codeMap[int(key)]
		switch {
		case !ok:
			log.Printf("Wrong color syntax: %c", key)
		case 0 <= c && c <= 8:
			attr = c
		case 30 <= c && c <= 37:
			fg = c
		case 40 <= c && c <= 47:
			bg = c
		case c == 60:
			fg += c
		}
	}
	return fmt.Sprintf("\033[%d;%d;%dm", attr, fg, bg)
}

// Handle state after meeting one '@'
func compileColorSyntax(input, output *bytes.Buffer) {
	i, _, err := input.ReadRune()
	if err != nil {
		// EOF got
		log.Print("Parse failed on color syntax")
		return
	}

	switch i {
	default:
		output.WriteString(Colorize(string(i)))
	case '{':
		color := bytes.NewBufferString("")
		for {
			i, _, err := input.ReadRune()
			if err != nil {
				log.Print("Parse failed on color syntax")
				break
			}
			if i == '}' {
				break
			}
			color.WriteRune(i)
		}
		output.WriteString(Colorize(color.String()))
	case EscapeChar:
		output.WriteRune(EscapeChar)
	}
}

// Compile the string and replace color syntax with concrete escape code.
func compile(x string) string {
	if x == "" {
		return ""
	}

	input := bytes.NewBufferString(x)
	output := bytes.NewBufferString("")

	for {
		i, _, err := input.ReadRune()
		if err != nil {
			break
		}
		switch i {
		default:
			output.WriteRune(i)
		case EscapeChar:
			compileColorSyntax(input, output)
		}
	}
	return output.String()
}

// Compile multiple values, only do compiling on string type.
func compileValues(a *[]interface{}) {
	for i, x := range *a {
		if str, ok := x.(string); ok {
			(*a)[i] = compile(str)
		}
	}
}

// Similar to fmt.Print, will reset the color at the end.
func Print(a ...interface{}) (int, error) {
	a = append(a, ResetCode)
	compileValues(&a)
	return fmt.Print(a...)
}

// Similar to fmt.Println, will reset the color at the end.
func Println(a ...interface{}) (int, error) {
	a = append(a, ResetCode)
	compileValues(&a)
	return fmt.Println(a...)
}

// Similar to fmt.Printf, will reset the color at the end.
func Printf(format string, a ...interface{}) (int, error) {
	format += ResetCode
	format = compile(format)
	return fmt.Printf(format, a...)
}

// Similar to fmt.Fprint, will reset the color at the end.
func Fprint(w io.Writer, a ...interface{}) (int, error) {
	a = append(a, ResetCode)
	compileValues(&a)
	return fmt.Fprint(w, a...)
}

// Similar to fmt.Fprintln, will reset the color at the end.
func Fprintln(w io.Writer, a ...interface{}) (int, error) {
	a = append(a, ResetCode)
	compileValues(&a)
	return fmt.Fprintln(w, a...)
}

// Similar to fmt.Fprintf, will reset the color at the end.
func Fprintf(w io.Writer, format string, a ...interface{}) (int, error) {
	format += ResetCode
	format = compile(format)
	return fmt.Fprintf(w, format, a...)
}

// Similar to fmt.Sprint, will reset the color at the end.
func Sprint(a ...interface{}) string {
	a = append(a, ResetCode)
	compileValues(&a)
	return fmt.Sprint(a...)
}

// Similar to fmt.Sprintf, will reset the color at the end.
func Sprintf(format string, a ...interface{}) string {
	format += ResetCode
	format = compile(format)
	return fmt.Sprintf(format, a...)
}

// Similar to fmt.Errorf, will reset the color at the end.
func Errorf(format string, a ...interface{}) error {
	return errors.New(Sprintf(format, a...))
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"github.com/jessevdk/go-flags"
	"github.com/takumakanari/cronv"
	"os"
)

const (
	version = "0.3.4"
	name    = "Cronv"
)

func main() {
	opts := cronv.NewCronvCommand()

	parser := flags.NewParser(opts, flags.Default)
	parser.Name = fmt.Sprintf("%s v%s", name, version)
	if _, err := parser.Parse(); err != nil {
		os.Exit(0)
	}

	ctx, err := cronv.NewCtx(opts)
	if err != nil {
		panic(err)
	}

	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		if _, err := ctx.AppendNewLine(scanner.Text()); err != nil {
			panic(err)
		}
	}
	if err := scanner.Err(); err != nil {
		panic(err)
	}

	path, err := ctx.Dump()
	if err != nil {
		panic(err)
	}

	fmt.Printf("[%s] %d tasks.\n", opts.Title, len(ctx.CronEntries))
	fmt.Printf("[%s] '%s' generated.\n", opts.Title, path)
}
<commit_msg>Bump up<commit_after>package main

import (
	"bufio"
	"fmt"
	"github.com/jessevdk/go-flags"
	"github.com/takumakanari/cronv"
	"os"
)

const (
	version = "0.4.0"
	name    = "Cronv"
)

func main() {
	opts := cronv.NewCronvCommand()

	parser := flags.NewParser(opts, flags.Default)
	parser.Name = fmt.Sprintf("%s v%s", name, version)
	if _, err := parser.Parse(); err != nil {
		os.Exit(0)
	}

	ctx, err := cronv.NewCtx(opts)
	if err != nil {
		panic(err)
	}

	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		if _, err := ctx.AppendNewLine(scanner.Text()); err != nil {
			panic(err)
		}
	}
	if err := scanner.Err(); err != nil {
		panic(err)
	}

	path, err := ctx.Dump()
	if err != nil {
		panic(err)
	}

	fmt.Printf("[%s] %d tasks.\n", opts.Title, len(ctx.CronEntries))
	fmt.Printf("[%s] '%s' generated.\n", opts.Title, path)
}
<|endoftext|>


<commit_before>package git

import (
	"os"

	"gopkg.in/src-d/go-git.v2/core"
	"gopkg.in/src-d/go-git.v2/formats/packfile"

	. "gopkg.in/check.v1"
)

type SuiteCommit struct {
	repos map[string]*Repository
}

var _ = Suite(&SuiteCommit{})

// create the repositories of the fixtures
func (s *SuiteCommit) SetUpSuite(c *C) {
	fixtureRepos := [...]struct {
		url      string
		packfile string
	}{
		{"https://github.com/tyba/git-fixture.git", "formats/packfile/fixtures/git-fixture.ofs-delta"},
	}
	s.repos = make(map[string]*Repository, 0)
	for _, fixRepo := range fixtureRepos {
		s.repos[fixRepo.url] = NewPlainRepository()

		d, err := os.Open(fixRepo.packfile)
		c.Assert(err, IsNil)

		r := packfile.NewReader(d)
		r.Format = packfile.OFSDeltaFormat // TODO: how to know the format of a pack file ahead of time?

		_, err = r.Read(s.repos[fixRepo.url].Storage)
		c.Assert(err, IsNil)

		c.Assert(d.Close(), IsNil)
	}
}

// FIXME: Test the new CommitIter
/*
func (s *SuiteCommit) TestIterClose(c *C) {
	i := &iter{ch: make(chan core.Object, 1)}
	i.Close()
	i.Close()
}
*/

var fileTests = []struct {
	repo     string // the repo name as in localRepos
	commit   string // the commit to search for the file
	path     string // the path of the file to find
	blobHash string // expected hash of the returned file
	found    bool   // expected found value
}{
	// use git ls-tree commit to get the hash of the blobs
	{"https://github.com/tyba/git-fixture.git", "b029517f6300c2da0f4b651b8642506cd6aaf45d", "not-found",
		"", false},
	{"https://github.com/tyba/git-fixture.git", "b029517f6300c2da0f4b651b8642506cd6aaf45d", ".gitignore",
		"32858aad3c383ed1ff0a0f9bdf231d54a00c9e88", true},
	{"https://github.com/tyba/git-fixture.git", "b029517f6300c2da0f4b651b8642506cd6aaf45d", "LICENSE",
		"c192bd6a24ea1ab01d78686e417c8bdc7c3d197f", true},

	{"https://github.com/tyba/git-fixture.git", "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "not-found",
		"", false},
	{"https://github.com/tyba/git-fixture.git", "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", ".gitignore",
		"32858aad3c383ed1ff0a0f9bdf231d54a00c9e88", true},
	{"https://github.com/tyba/git-fixture.git", "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "binary.jpg",
		"d5c0f4ab811897cadf03aec358ae60d21f91c50d", true},
	{"https://github.com/tyba/git-fixture.git", "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "LICENSE",
		"c192bd6a24ea1ab01d78686e417c8bdc7c3d197f", true},

	{"https://github.com/tyba/git-fixture.git", "35e85108805c84807bc66a02d91535e1e24b38b9", "binary.jpg",
		"d5c0f4ab811897cadf03aec358ae60d21f91c50d", true},
	{"https://github.com/tyba/git-fixture.git", "b029517f6300c2da0f4b651b8642506cd6aaf45d", "binary.jpg",
		"", false},

	{"https://github.com/tyba/git-fixture.git", "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "CHANGELOG",
		"d3ff53e0564a9f87d8e84b6e28e5060e517008aa", true},
	{"https://github.com/tyba/git-fixture.git", "1669dce138d9b841a518c64b10914d88f5e488ea", "CHANGELOG",
		"d3ff53e0564a9f87d8e84b6e28e5060e517008aa", true},
	{"https://github.com/tyba/git-fixture.git", "a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69", "CHANGELOG",
		"d3ff53e0564a9f87d8e84b6e28e5060e517008aa", true},
	{"https://github.com/tyba/git-fixture.git", "35e85108805c84807bc66a02d91535e1e24b38b9", "CHANGELOG",
		"d3ff53e0564a9f87d8e84b6e28e5060e517008aa", false},
	{"https://github.com/tyba/git-fixture.git", "b8e471f58bcbca63b07bda20e428190409c2db47", "CHANGELOG",
		"d3ff53e0564a9f87d8e84b6e28e5060e517008aa", true},
	{"https://github.com/tyba/git-fixture.git", "b029517f6300c2da0f4b651b8642506cd6aaf45d", "CHANGELOG",
		"d3ff53e0564a9f87d8e84b6e28e5060e517008aa", false},
}

func (s *SuiteCommit) TestFile(c *C) {
	for i, t := range fileTests {
		commit, err := s.repos[t.repo].Commit(core.NewHash(t.commit))
		c.Assert(err, IsNil, Commentf("subtest %d: %v (%s)", i, err, t.commit))

		file, err := commit.File(t.path)
		found := err == nil
		c.Assert(found, Equals, t.found, Commentf("subtest %d, path=%s, commit=%s", i, t.path, t.commit))
		if found {
			c.Assert(file.Hash.String(), Equals, t.blobHash, Commentf("subtest %d, commit=%s, path=%s", i, t.commit, t.path))
		}
	}
}
<commit_msg>Added tests for CommitIter<commit_after>package git

import (
	"io"
	"os"

	"gopkg.in/src-d/go-git.v2/core"
	"gopkg.in/src-d/go-git.v2/formats/packfile"

	. "gopkg.in/check.v1"
)

type SuiteCommit struct {
	repos map[string]*Repository
}

var _ = Suite(&SuiteCommit{})

// create the repositories of the fixtures
func (s *SuiteCommit) SetUpSuite(c *C) {
	fixtureRepos := [...]struct {
		url      string
		packfile string
	}{
		{"https://github.com/tyba/git-fixture.git", "formats/packfile/fixtures/git-fixture.ofs-delta"},
	}
	s.repos = make(map[string]*Repository, 0)
	for _, fixRepo := range fixtureRepos {
		s.repos[fixRepo.url] = NewPlainRepository()

		d, err := os.Open(fixRepo.packfile)
		c.Assert(err, IsNil)

		r := packfile.NewReader(d)
		r.Format = packfile.OFSDeltaFormat // TODO: how to know the format of a pack file ahead of time?

		_, err = r.Read(s.repos[fixRepo.url].Storage)
		c.Assert(err, IsNil)

		c.Assert(d.Close(), IsNil)
	}
}

var iterTests = []struct {
	repo    string   // the repo name in the test suite's map of fixtures
	commits []string // the commit hashes to iterate over in the test
}{
	{"https://github.com/tyba/git-fixture.git", []string{
		"6ecf0ef2c2dffb796033e5a02219af86ec6584e5",
		"918c48b83bd081e863dbe1b80f8998f058cd8294",
		"af2d6a6954d532f8ffb47615169c8fdf9d383a1a",
		"1669dce138d9b841a518c64b10914d88f5e488ea",
		"35e85108805c84807bc66a02d91535e1e24b38b9",
		"b029517f6300c2da0f4b651b8642506cd6aaf45d",
		"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69",
		"b029517f6300c2da0f4b651b8642506cd6aaf45d", // Intentional duplicate
		"b8e471f58bcbca63b07bda20e428190409c2db47",
		"b029517f6300c2da0f4b651b8642506cd6aaf45d"}}, // Intentional duplicate
}

func (s *SuiteCommit) TestIterSlice(c *C) {
	for i, t := range iterTests {
		r := s.repos[t.repo]
		iter := NewCommitIter(r, core.NewObjectSliceIter(makeObjectSlice(t.commits, r.Storage)))
		s.checkIter(c, r, i, iter, t.commits)
	}
}

func (s *SuiteCommit) TestIterLookup(c *C) {
	for i, t := range iterTests {
		r := s.repos[t.repo]
		iter := NewCommitIter(r, core.NewObjectLookupIter(r.Storage, makeHashSlice(t.commits)))
		s.checkIter(c, r, i, iter, t.commits)
	}
}

func (s *SuiteCommit) checkIter(c *C, r *Repository, subtest int, iter *CommitIter, commits []string) {
	for k := 0; k < len(commits); k++ {
		commit, err := iter.Next()
		c.Assert(err, IsNil, Commentf("subtest %d, iter %d, err=%v", subtest, k, err))
		c.Assert(commit.Hash.String(), Equals, commits[k], Commentf("subtest %d, iter %d, hash=%v, expected=%s", subtest, k, commit.Hash.String(), commits[k]))
	}
	_, err := iter.Next()
	c.Assert(err, Equals, io.EOF)
}

func (s *SuiteCommit) TestIterSliceClose(c *C) {
	for i, t := range iterTests {
		r := s.repos[t.repo]
		iter := NewCommitIter(r, core.NewObjectSliceIter(makeObjectSlice(t.commits, r.Storage)))
		s.checkIterClose(c, i, iter)
	}
}

func (s *SuiteCommit) TestIterLookupClose(c *C) {
	for i, t := range iterTests {
		r := s.repos[t.repo]
		iter := NewCommitIter(r, core.NewObjectLookupIter(r.Storage, makeHashSlice(t.commits)))
		s.checkIterClose(c, i, iter)
	}
}

func (s *SuiteCommit) checkIterClose(c *C, subtest int, iter *CommitIter) {
	iter.Close()
	_, err := iter.Next()
	c.Assert(err, Equals, io.EOF, Commentf("subtest %d, close 1, err=%v", subtest, err))

	iter.Close()
	_, err = iter.Next()
	c.Assert(err, Equals, io.EOF, Commentf("subtest %d, close 2, err=%v", subtest, err))
}

var fileTests = []struct {
	repo     string // the repo name as in localRepos
	commit   string // the commit to search for the file
	path     string // the path of the file to find
	blobHash string // expected hash of the returned file
	found    bool   // expected found value
}{
	// use git ls-tree commit to get the hash of the blobs
	{"https://github.com/tyba/git-fixture.git", "b029517f6300c2da0f4b651b8642506cd6aaf45d", "not-found",
		"", false},
	{"https://github.com/tyba/git-fixture.git", "b029517f6300c2da0f4b651b8642506cd6aaf45d", ".gitignore",
		"32858aad3c383ed1ff0a0f9bdf231d54a00c9e88", true},
	{"https://github.com/tyba/git-fixture.git", "b029517f6300c2da0f4b651b8642506cd6aaf45d", "LICENSE",
		"c192bd6a24ea1ab01d78686e417c8bdc7c3d197f", true},

	{"https://github.com/tyba/git-fixture.git", "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "not-found",
		"", false},
	{"https://github.com/tyba/git-fixture.git", "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", ".gitignore",
		"32858aad3c383ed1ff0a0f9bdf231d54a00c9e88", true},
	{"https://github.com/tyba/git-fixture.git", "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "binary.jpg",
		"d5c0f4ab811897cadf03aec358ae60d21f91c50d", true},
	{"https://github.com/tyba/git-fixture.git", "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "LICENSE",
		"c192bd6a24ea1ab01d78686e417c8bdc7c3d197f", true},

	{"https://github.com/tyba/git-fixture.git", "35e85108805c84807bc66a02d91535e1e24b38b9", "binary.jpg",
		"d5c0f4ab811897cadf03aec358ae60d21f91c50d", true},
	{"https://github.com/tyba/git-fixture.git", "b029517f6300c2da0f4b651b8642506cd6aaf45d", "binary.jpg",
		"", false},

	{"https://github.com/tyba/git-fixture.git", "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "CHANGELOG",
		"d3ff53e0564a9f87d8e84b6e28e5060e517008aa", true},
	{"https://github.com/tyba/git-fixture.git", "1669dce138d9b841a518c64b10914d88f5e488ea", "CHANGELOG",
		"d3ff53e0564a9f87d8e84b6e28e5060e517008aa", true},
	{"https://github.com/tyba/git-fixture.git", "a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69", "CHANGELOG",
		"d3ff53e0564a9f87d8e84b6e28e5060e517008aa", true},
	{"https://github.com/tyba/git-fixture.git", "35e85108805c84807bc66a02d91535e1e24b38b9", "CHANGELOG",
		"d3ff53e0564a9f87d8e84b6e28e5060e517008aa", false},
	{"https://github.com/tyba/git-fixture.git", "b8e471f58bcbca63b07bda20e428190409c2db47", "CHANGELOG",
		"d3ff53e0564a9f87d8e84b6e28e5060e517008aa", true},
	{"https://github.com/tyba/git-fixture.git", "b029517f6300c2da0f4b651b8642506cd6aaf45d", "CHANGELOG",
		"d3ff53e0564a9f87d8e84b6e28e5060e517008aa", false},
}

func (s *SuiteCommit) TestFile(c *C) {
	for i, t := range fileTests {
		commit, err := s.repos[t.repo].Commit(core.NewHash(t.commit))
		c.Assert(err, IsNil, Commentf("subtest %d: %v (%s)", i, err, t.commit))

		file, err := commit.File(t.path)
		found := err == nil
		c.Assert(found, Equals, t.found, Commentf("subtest %d, path=%s, commit=%s", i, t.path, t.commit))
		if found {
			c.Assert(file.Hash.String(), Equals, t.blobHash, Commentf("subtest %d, commit=%s, path=%s", i, t.commit, t.path))
		}
	}
}

func makeObjectSlice(hashes []string, storage core.ObjectStorage) []core.Object {
	series := make([]core.Object, 0, len(hashes))
	for _, member := range hashes {
		obj, ok := storage.Get(core.NewHash(member))
		if ok {
			series = append(series, obj)
		}
	}
	return series
}

func makeHashSlice(hashes []string) []core.Hash {
	series := make([]core.Hash, 0, len(hashes))
	for _, member := range hashes {
		series = append(series, core.NewHash(member))
	}
	return series
}
<|endoftext|>


<commit_before>package tesTaskEngineWorker

import (
	"context"
	"fmt"
	"net"
	"os/exec"
	"path/filepath"
	"reflect"
	"strings"
	"syscall"
	pbe "tes/ga4gh"
	"tes/scheduler"
	pbr "tes/server/proto"
	"tes/storage"
	"time"
)

// Engine is responsible for running a job. This includes downloading inputs,
// communicating updates to the scheduler service, running the actual command,
// and uploading outputs.
type Engine interface {
	RunJob(ctx context.Context, job *pbr.JobResponse) error
}

// engine is the internal implementation of a docker job engine.
type engine struct {
	conf Config
}

// NewEngine returns a new Engine instance configured with a given scheduler address,
// working directory, and storage client.
//
// If the working directory can't be initialized, this returns an error.
func NewEngine(conf Config) (Engine, error) {
	dir, err := filepath.Abs(conf.WorkDir)
	if err != nil {
		return nil, err
	}
	ensureDir(dir)

	return &engine{conf}, nil
}

// RunJob is a wrapper for runJob that polls for Cancel requests
// TODO documentation
func (eng *engine) RunJob(parentCtx context.Context, jobR *pbr.JobResponse) error {
	// Get a client for the scheduler service
	sched, schederr := scheduler.NewClient(eng.conf.ServerAddress)
	defer sched.Close()
	// TODO if we're here then we have a serious problem. We have already
	//      told the scheduler that we're running the job, but now we can't
	//      tell it things are broken, so the job is going to stay running
	//      forever. Possibly the scheduler should have a job timeout.
	if schederr != nil {
		return schederr
	}

	jobID := &pbe.JobID{
		Value: jobR.Job.JobID,
	}

	ctx, cancel := context.WithCancel(parentCtx)
	defer cancel()

	joberr := make(chan error, 1)
	go func() {
		joberr <- eng.runJob(ctx, sched, jobR)
	}()

	// Ticker for State polling
	tickChan := time.NewTicker(time.Second * 5).C
	for {
		select {
		case joberr := <-joberr:
			if joberr != nil {
				sched.SetFailed(ctx, jobR.Job)
				return fmt.Errorf("Error running job: %v", joberr)
			}
			sched.SetComplete(ctx, jobR.Job)
			return nil
		case <-tickChan:
			jobDesc, err := sched.GetJobState(ctx, jobID)
			if err != nil {
				return fmt.Errorf("Error trying to get job status: %v", err)
			}
			switch jobDesc.State {
			case pbe.State_Canceled:
				cancel()
			}
		}
	}
}

// runJob runs a job
// TODO documentation
func (eng *engine) runJob(ctx context.Context, sched *scheduler.Client, jobR *pbr.JobResponse) error {
	log := log.WithFields("jobID", jobR.Job.JobID)
	// Initialize job
	sched.SetInitializing(ctx, jobR.Job)
	mapper, merr := eng.getMapper(jobR.Job)

	if merr != nil {
		sched.SetFailed(ctx, jobR.Job)
		return fmt.Errorf("Error during mapper initialization: %s", merr)
	}

	store, serr := eng.getStorage(jobR)
	if serr != nil {
		return fmt.Errorf("Error during store initialization: %s", serr)
	}

	derr := eng.downloadInputs(mapper, store)
	if derr != nil {
		return fmt.Errorf("Error during input provisioning: %s", derr)
	}

	// Run job steps
	sched.SetRunning(ctx, jobR.Job)
	for stepNum, step := range jobR.Job.Task.Docker {
		joberr := eng.runStep(ctx, sched, mapper, jobR.Job.JobID, step, stepNum)
		if joberr != nil {
			return fmt.Errorf("Error running job: %s", joberr)
		}
	}

	// Finalize job
	oerr := eng.uploadOutputs(mapper, store)
	if oerr != nil {
		return fmt.Errorf("Error uploading job outputs: %s", oerr)
	}

	// Job is Complete
	log.Info("Job completed without error")
	return nil
}

// runStep
// TODO documentation
func (eng *engine) runStep(ctx context.Context, sched *scheduler.Client, mapper *FileMapper, id string, step *pbe.DockerExecutor, stepNum int) error {
	stepID := fmt.Sprintf("%v-%v", id, stepNum)
	dcmd, err := eng.setupDockerCmd(mapper, step, stepID)
	if err != nil {
		return fmt.Errorf("Error setting up docker command: %v", err)
	}
	log.Info("Running command", "cmd", strings.Join(dcmd.Cmd.Args, " "))

	// Start task step asynchronously
	dcmd.Cmd.Start()

	// Open channel to track async process
	done := make(chan error, 1)
	go func() {
		done <- dcmd.Cmd.Wait()
	}()

	// Open channel to track container initialization
	metaCh := make(chan []*pbe.Ports, 1)
	go func() {
		metaCh <- dcmd.InspectContainer(ctx)
	}()

	// Initialized to allow for DeepEquals comparison during polling
	stepLog := &pbe.JobLog{}

	// Ticker for polling rate
	tickChan := time.NewTicker(time.Second * 5).C

	for {
		select {

		// ensure containers are stopped if the context is canceled
		// handles cancel request
		case <-ctx.Done():
			err := dcmd.StopContainer()
			if err != nil {
				return err
			}

		// TODO ensure metadata gets logged
		case portMap := <-metaCh:
			ip, err := externalIP()
			if err != nil {
				return err
			}

			// log update with host ip and port mapping
			initLog := &pbe.JobLog{
				HostIP: ip,
				Ports:  portMap,
			}

			statusReq := &pbr.UpdateStatusRequest{
				Id:   id,
				Step: int64(stepNum),
				Log:  initLog,
			}
			sched.UpdateJobStatus(ctx, statusReq)

		// handles docker run failure and success
		case cmdErr := <-done:
			stepLogUpdate := eng.finalizeLogs(dcmd, cmdErr)
			// final log update that includes the exit code
			statusReq := &pbr.UpdateStatusRequest{
				Id:   id,
				Step: int64(stepNum),
				Log:  stepLogUpdate,
			}
			sched.UpdateJobStatus(ctx, statusReq)

			if cmdErr != nil {
				return fmt.Errorf("Docker command error: %v", cmdErr)
			}
			return nil

		// update stdout and stderr in logs every 5 seconds
		case <-tickChan:
			stepLogUpdate := eng.updateLogs(dcmd)
			// check if log update has any new data
			if reflect.DeepEqual(stepLogUpdate, stepLog) == false {
				statusReq := &pbr.UpdateStatusRequest{
					Id:   id,
					Step: int64(stepNum),
					Log:  stepLogUpdate,
				}
				sched.UpdateJobStatus(ctx, statusReq)
			}
		}
	}
}

// getMapper returns a FileMapper instance with volumes, inputs, and outputs
// configured for the given job.
func (eng *engine) getMapper(job *pbe.Job) (*FileMapper, error) {
	mapper := NewJobFileMapper(job.JobID, eng.conf.WorkDir)

	// Iterates through job.Task.Resources.Volumes and add the volume to mapper.
	for _, vol := range job.Task.Resources.Volumes {
		err := mapper.AddVolume(vol.Source, vol.MountPoint)
		if err != nil {
			return nil, err
		}
	}

	// Add all the inputs to the mapper
	for _, input := range job.Task.Inputs {
		err := mapper.AddInput(input)
		if err != nil {
			return nil, err
		}
	}

	// Add all the outputs to the mapper
	for _, output := range job.Task.Outputs {
		err := mapper.AddOutput(output)
		if err != nil {
			return nil, err
		}
	}

	return mapper, nil
}

// getStorage returns a Storage instance configured for the given job.
func (eng *engine) getStorage(jobR *pbr.JobResponse) (*storage.Storage, error) {
	var err error
	storage := new(storage.Storage)

	for _, conf := range eng.conf.Storage {
		storage, err = storage.WithConfig(conf)
		if err != nil {
			return nil, err
		}
	}

	for _, conf := range jobR.Storage {
		storage, err = storage.WithConfig(conf)
		if err != nil {
			return nil, err
		}
	}

	if storage == nil {
		return nil, fmt.Errorf("No storage configured")
	}

	return storage, nil
}

func (eng *engine) downloadInputs(mapper *FileMapper, store *storage.Storage) error {
	// Validate all the input source URLs
	//for _, input := range mapper.Inputs {
	// TODO ?
	//}

	// Download all the inputs from storage
	for _, input := range mapper.Inputs {
		err := store.Get(input.Location, input.Path, input.Class)
		if err != nil {
			return err
		}
	}
	return nil
}

// The bulk of job running happens here.
func (eng *engine) setupDockerCmd(mapper *FileMapper, step *pbe.DockerExecutor, id string) (*DockerCmd, error) {

	dcmd := &DockerCmd{
		ImageName:     step.ImageName,
		CmdString:     step.Cmd,
		Volumes:       mapper.Volumes,
		Workdir:       step.Workdir,
		Ports:         step.Ports,
		ContainerName: id,
		// TODO make RemoveContainer configurable
		RemoveContainer: true,
		Stdin:           nil,
		Stdout:          nil,
		Stderr:          nil,
		Log:             map[string][]byte{},
	}

	// Find the path for job stdin
	if step.Stdin != "" {
		f, err := mapper.OpenHostFile(step.Stdin)
		if err != nil {
			return nil, fmt.Errorf("Error setting up job stdin: %s", err)
		}
		dcmd.Stdin = f
	}

	// Create file for job stdout
	if step.Stdout != "" {
		f, err := mapper.CreateHostFile(step.Stdout)
		if err != nil {
			return nil, fmt.Errorf("Error setting up job stdout: %s", err)
		}
		dcmd.Stdout = f
	}

	// Create file for job stderr
	if step.Stderr != "" {
		f, err := mapper.CreateHostFile(step.Stderr)
		if err != nil {
			return nil, fmt.Errorf("Error setting up job stderr: %s", err)
		}
		dcmd.Stderr = f
	}

	dcmd, err := dcmd.SetupCommand()
	if err != nil {
		return nil, fmt.Errorf("Error setting up job command: %s", err)
	}

	return dcmd, nil
}

func externalIP() (string, error) {
	ifaces, err := net.Interfaces()
	if err != nil {
		return "", err
	}
	for _, iface := range ifaces {
		if iface.Flags&net.FlagUp == 0 {
			continue // interface down

		}
		if iface.Flags&net.FlagLoopback != 0 {
			continue // loopback interface

		}
		addrs, err := iface.Addrs()
		if err != nil {
			return "", err

		}
		for _, addr := range addrs {
			var ip net.IP
			switch v := addr.(type) {
			case *net.IPNet:
				ip = v.IP
			case *net.IPAddr:
				ip = v.IP
			}
			if ip == nil || ip.IsLoopback() {
				continue
			}
			ip = ip.To4()
			if ip == nil {
				continue // not an ipv4 address
			}
			return ip.String(), nil
		}
	}
	return "", fmt.Errorf("Error no network connection")
}

func (eng *engine) updateLogs(dcmd *DockerCmd) *pbe.JobLog {
	stepLog := &pbe.JobLog{}

	if len(dcmd.Log["Stdout"]) > 0 {
		stdoutText := string(dcmd.Log["Stdout"][:])
		dcmd.Log["Stdout"] = []byte{}
		stepLog.Stdout = stdoutText
	}

	if len(dcmd.Log["Stderr"]) > 0 {
		stderrText := string(dcmd.Log["Stderr"][:])
		dcmd.Log["Stderr"] = []byte{}
		stepLog.Stderr = stderrText
	}

	return stepLog
}

func (eng *engine) finalizeLogs(dcmd *DockerCmd, cmdErr error) *pbe.JobLog {
	exitCode := getExitCode(cmdErr)
	log.Info("Exit code", "code", exitCode)
	steplog := eng.updateLogs(dcmd)
	steplog.ExitCode = exitCode
	return steplog
}

func (eng *engine) uploadOutputs(mapper *FileMapper, store *storage.Storage) error {
	// Upload all the outputs to storage
	for _, out := range mapper.Outputs {
		err := store.Put(out.Location, out.Path, out.Class)
		if err != nil {
			return err
		}
	}
	return nil
}

// getExitCode gets the exit status (i.e. exit code) from the result of an executed command.
// The exit code is zero if the command completed without error.
func getExitCode(err error) int32 {
	if err != nil {
		if exiterr, exitOk := err.(*exec.ExitError); exitOk {
			if status, statusOk := exiterr.Sys().(syscall.WaitStatus); statusOk {
				return int32(status.ExitStatus())
			}
		} else {
			log.Info("Could not determine exit code. Using default -999")
			return -999
		}
	}
	// The error is nil, the command returned successfully, so exit status is 0.
	return 0
}
<commit_msg>fixed some of the documentation to address PR comments<commit_after>package tesTaskEngineWorker

import (
	"context"
	"fmt"
	"net"
	"os/exec"
	"path/filepath"
	"reflect"
	"strings"
	"syscall"
	pbe "tes/ga4gh"
	"tes/scheduler"
	pbr "tes/server/proto"
	"tes/storage"
	"time"
)

// Engine is responsible for running a job. This includes downloading inputs,
// communicating updates to the scheduler service, running the actual command,
// and uploading outputs.
type Engine interface {
	RunJob(ctx context.Context, job *pbr.JobResponse) error
}

// engine is the internal implementation of a docker job engine.
type engine struct {
	conf Config
}

// NewEngine returns a new Engine instance configured with a given scheduler address,
// working directory, and storage client.
//
// If the working directory can't be initialized, this returns an error.
func NewEngine(conf Config) (Engine, error) {
	dir, err := filepath.Abs(conf.WorkDir)
	if err != nil {
		return nil, err
	}
	ensureDir(dir)

	return &engine{conf}, nil
}

// RunJob is a wrapper for runJob that polls for Cancel requests
func (eng *engine) RunJob(parentCtx context.Context, jobR *pbr.JobResponse) error {
	// This is essentially a simple helper for runJob() (below).
 	// This ensures that the job state is always updated in the scheduler,
  // without having to do it on 15+ different lines in runJob() and others.
  //
  // Please try to keep this function as simple as possible.
  // New code should probably go in runJob()

	ctx, cancel := context.WithCancel(parentCtx)
	defer cancel()

	// Get a client for the scheduler service
	sched, schederr := scheduler.NewClient(eng.conf.ServerAddress)
	defer sched.Close()
	// TODO if we're here then we have a serious problem. We have already
	//      told the scheduler that we're running the job, but now we can't
	//      tell it things are broken, so the job is going to stay running
	//      forever. Possibly the scheduler should have a job timeout.
	if schederr != nil {
		return schederr
	}

	jobID := &pbe.JobID{
		Value: jobR.Job.JobID,
	}

	sched.SetInitializing(ctx, jobR.Job)

	joberr := make(chan error, 1)
	go func() {
		joberr <- eng.runJob(ctx, sched, jobR)
	}()

	// Ticker for State polling
	tickChan := time.NewTicker(time.Second * 5).C
	for {
		select {
		case joberr := <-joberr:
			if joberr != nil {
				sched.SetFailed(ctx, jobR.Job)
				return fmt.Errorf("Error running job: %v", joberr)
			}
			sched.SetComplete(ctx, jobR.Job)
			return nil
		case <-tickChan:
			jobDesc, err := sched.GetJobState(ctx, jobID)
			if err != nil {
				return fmt.Errorf("Error trying to get job status: %v", err)
			}
			switch jobDesc.State {
			case pbe.State_Canceled:
				cancel()
			}
		}
	}
}

// runJob calls a series of other functions to process a job:
// 1. set up the file mapping between the host and the container
// 2. set up the storage client
// 3. download the inputs
// 4. run the job steps
// 4a. update the scheduler with job status after each step
// 5. upload the outputs
func (eng *engine) runJob(ctx context.Context, sched *scheduler.Client, jobR *pbr.JobResponse) error {
	log := log.WithFields("jobID", jobR.Job.JobID)
	// Initialize job
	mapper, merr := eng.getMapper(jobR.Job)

	if merr != nil {
		sched.SetFailed(ctx, jobR.Job)
		return fmt.Errorf("Error during mapper initialization: %s", merr)
	}

	store, serr := eng.getStorage(jobR)
	if serr != nil {
		return fmt.Errorf("Error during store initialization: %s", serr)
	}

	derr := eng.downloadInputs(mapper, store)
	if derr != nil {
		return fmt.Errorf("Error during input provisioning: %s", derr)
	}

	// Run job steps
	sched.SetRunning(ctx, jobR.Job)
	for stepNum, step := range jobR.Job.Task.Docker {
		joberr := eng.runStep(ctx, sched, mapper, jobR.Job.JobID, step, stepNum)
		if joberr != nil {
			return fmt.Errorf("Error running job: %s", joberr)
		}
	}

	// Finalize job
	oerr := eng.uploadOutputs(mapper, store)
	if oerr != nil {
		return fmt.Errorf("Error uploading job outputs: %s", oerr)
	}

	// Job is Complete
	log.Info("Job completed without error")
	return nil
}

// runStep runs a single docker step of a task
func (eng *engine) runStep(ctx context.Context, sched *scheduler.Client, mapper *FileMapper, id string, step *pbe.DockerExecutor, stepNum int) error {
	stepID := fmt.Sprintf("%v-%v", id, stepNum)
	dcmd, err := eng.setupDockerCmd(mapper, step, stepID)
	if err != nil {
		return fmt.Errorf("Error setting up docker command: %v", err)
	}
	log.Info("Running command", "cmd", strings.Join(dcmd.Cmd.Args, " "))

	// Start task step asynchronously
	dcmd.Cmd.Start()

	// Open channel to track async process
	done := make(chan error, 1)
	go func() {
		done <- dcmd.Cmd.Wait()
	}()

	// Open channel to track container initialization
	metaCh := make(chan []*pbe.Ports, 1)
	go func() {
		metaCh <- dcmd.InspectContainer(ctx)
	}()

	// Initialized to allow for DeepEquals comparison during polling
	stepLog := &pbe.JobLog{}

	// Ticker for polling rate
	tickChan := time.NewTicker(time.Second * 5).C

	for {
		select {

		// ensure containers are stopped if the context is canceled
		// handles cancel request
		case <-ctx.Done():
			err := dcmd.StopContainer()
			if err != nil {
				return err
			}

		// TODO ensure metadata gets logged
		case portMap := <-metaCh:
			ip, err := externalIP()
			if err != nil {
				return err
			}

			// log update with host ip and port mapping
			initLog := &pbe.JobLog{
				HostIP: ip,
				Ports:  portMap,
			}

			statusReq := &pbr.UpdateStatusRequest{
				Id:   id,
				Step: int64(stepNum),
				Log:  initLog,
			}
			sched.UpdateJobStatus(ctx, statusReq)

		// handles docker run failure and success
		case cmdErr := <-done:
			stepLogUpdate := eng.finalizeLogs(dcmd, cmdErr)
			// final log update that includes the exit code
			statusReq := &pbr.UpdateStatusRequest{
				Id:   id,
				Step: int64(stepNum),
				Log:  stepLogUpdate,
			}
			sched.UpdateJobStatus(ctx, statusReq)

			if cmdErr != nil {
				return fmt.Errorf("Docker command error: %v", cmdErr)
			}
			return nil

		// update stdout and stderr in logs every 5 seconds
		case <-tickChan:
			stepLogUpdate := eng.updateLogs(dcmd)
			// check if log update has any new data
			if reflect.DeepEqual(stepLogUpdate, stepLog) == false {
				statusReq := &pbr.UpdateStatusRequest{
					Id:   id,
					Step: int64(stepNum),
					Log:  stepLogUpdate,
				}
				sched.UpdateJobStatus(ctx, statusReq)
			}
		}
	}
}

// getMapper returns a FileMapper instance with volumes, inputs, and outputs
// configured for the given job.
func (eng *engine) getMapper(job *pbe.Job) (*FileMapper, error) {
	mapper := NewJobFileMapper(job.JobID, eng.conf.WorkDir)

	// Iterates through job.Task.Resources.Volumes and add the volume to mapper.
	for _, vol := range job.Task.Resources.Volumes {
		err := mapper.AddVolume(vol.Source, vol.MountPoint)
		if err != nil {
			return nil, err
		}
	}

	// Add all the inputs to the mapper
	for _, input := range job.Task.Inputs {
		err := mapper.AddInput(input)
		if err != nil {
			return nil, err
		}
	}

	// Add all the outputs to the mapper
	for _, output := range job.Task.Outputs {
		err := mapper.AddOutput(output)
		if err != nil {
			return nil, err
		}
	}

	return mapper, nil
}

// getStorage returns a Storage instance configured for the given job.
func (eng *engine) getStorage(jobR *pbr.JobResponse) (*storage.Storage, error) {
	var err error
	storage := new(storage.Storage)

	for _, conf := range eng.conf.Storage {
		storage, err = storage.WithConfig(conf)
		if err != nil {
			return nil, err
		}
	}

	for _, conf := range jobR.Storage {
		storage, err = storage.WithConfig(conf)
		if err != nil {
			return nil, err
		}
	}

	if storage == nil {
		return nil, fmt.Errorf("No storage configured")
	}

	return storage, nil
}

func (eng *engine) downloadInputs(mapper *FileMapper, store *storage.Storage) error {
	// Validate all the input source URLs
	//for _, input := range mapper.Inputs {
	// TODO ?
	//}

	// Download all the inputs from storage
	for _, input := range mapper.Inputs {
		err := store.Get(input.Location, input.Path, input.Class)
		if err != nil {
			return err
		}
	}
	return nil
}

// The bulk of job running happens here.
func (eng *engine) setupDockerCmd(mapper *FileMapper, step *pbe.DockerExecutor, id string) (*DockerCmd, error) {

	dcmd := &DockerCmd{
		ImageName:     step.ImageName,
		CmdString:     step.Cmd,
		Volumes:       mapper.Volumes,
		Workdir:       step.Workdir,
		Ports:         step.Ports,
		ContainerName: id,
		// TODO make RemoveContainer configurable
		RemoveContainer: true,
		Stdin:           nil,
		Stdout:          nil,
		Stderr:          nil,
		Log:             map[string][]byte{},
	}

	// Find the path for job stdin
	if step.Stdin != "" {
		f, err := mapper.OpenHostFile(step.Stdin)
		if err != nil {
			return nil, fmt.Errorf("Error setting up job stdin: %s", err)
		}
		dcmd.Stdin = f
	}

	// Create file for job stdout
	if step.Stdout != "" {
		f, err := mapper.CreateHostFile(step.Stdout)
		if err != nil {
			return nil, fmt.Errorf("Error setting up job stdout: %s", err)
		}
		dcmd.Stdout = f
	}

	// Create file for job stderr
	if step.Stderr != "" {
		f, err := mapper.CreateHostFile(step.Stderr)
		if err != nil {
			return nil, fmt.Errorf("Error setting up job stderr: %s", err)
		}
		dcmd.Stderr = f
	}

	dcmd, err := dcmd.SetupCommand()
	if err != nil {
		return nil, fmt.Errorf("Error setting up job command: %s", err)
	}

	return dcmd, nil
}

func externalIP() (string, error) {
	ifaces, err := net.Interfaces()
	if err != nil {
		return "", err
	}
	for _, iface := range ifaces {
		if iface.Flags&net.FlagUp == 0 {
			continue // interface down

		}
		if iface.Flags&net.FlagLoopback != 0 {
			continue // loopback interface

		}
		addrs, err := iface.Addrs()
		if err != nil {
			return "", err

		}
		for _, addr := range addrs {
			var ip net.IP
			switch v := addr.(type) {
			case *net.IPNet:
				ip = v.IP
			case *net.IPAddr:
				ip = v.IP
			}
			if ip == nil || ip.IsLoopback() {
				continue
			}
			ip = ip.To4()
			if ip == nil {
				continue // not an ipv4 address
			}
			return ip.String(), nil
		}
	}
	return "", fmt.Errorf("Error no network connection")
}

func (eng *engine) updateLogs(dcmd *DockerCmd) *pbe.JobLog {
	stepLog := &pbe.JobLog{}

	if len(dcmd.Log["Stdout"]) > 0 {
		stdoutText := string(dcmd.Log["Stdout"][:])
		dcmd.Log["Stdout"] = []byte{}
		stepLog.Stdout = stdoutText
	}

	if len(dcmd.Log["Stderr"]) > 0 {
		stderrText := string(dcmd.Log["Stderr"][:])
		dcmd.Log["Stderr"] = []byte{}
		stepLog.Stderr = stderrText
	}

	return stepLog
}

func (eng *engine) finalizeLogs(dcmd *DockerCmd, cmdErr error) *pbe.JobLog {
	exitCode := getExitCode(cmdErr)
	log.Info("Exit code", "code", exitCode)
	steplog := eng.updateLogs(dcmd)
	steplog.ExitCode = exitCode
	return steplog
}

func (eng *engine) uploadOutputs(mapper *FileMapper, store *storage.Storage) error {
	// Upload all the outputs to storage
	for _, out := range mapper.Outputs {
		err := store.Put(out.Location, out.Path, out.Class)
		if err != nil {
			return err
		}
	}
	return nil
}

// getExitCode gets the exit status (i.e. exit code) from the result of an executed command.
// The exit code is zero if the command completed without error.
func getExitCode(err error) int32 {
	if err != nil {
		if exiterr, exitOk := err.(*exec.ExitError); exitOk {
			if status, statusOk := exiterr.Sys().(syscall.WaitStatus); statusOk {
				return int32(status.ExitStatus())
			}
		} else {
			log.Info("Could not determine exit code. Using default -999")
			return -999
		}
	}
	// The error is nil, the command returned successfully, so exit status is 0.
	return 0
}
<|endoftext|>


<commit_before>package restic

import (
	"restic/backend"
	"restic/repository"
)

//  FindUsedBlobs traverse the tree ID and adds all seen blobs to blobs.
func findUsedBlobs(repo *repository.Repository, treeID backend.ID, blobs backend.IDSet) error {
	blobs.Insert(treeID)

	tree, err := LoadTree(repo, treeID)
	if err != nil {
		return err
	}

	for _, node := range tree.Nodes {
		switch node.Type {
		case "file":
			for _, blob := range node.Content {
				blobs.Insert(blob)
			}
		case "dir":
			err := findUsedBlobs(repo, *node.Subtree, blobs)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// FindUsedBlobs traverses the tree ID and returns a set of all blobs
// encountered.
func FindUsedBlobs(repo *repository.Repository, treeID backend.ID) (blobs backend.IDSet, err error) {
	blobs = backend.NewIDSet()
	return blobs, findUsedBlobs(repo, treeID, blobs)
}
<commit_msg>Prevent loops when finding used blobs<commit_after>package restic

import (
	"restic/backend"
	"restic/repository"
)

//  FindUsedBlobs traverse the tree ID and adds all seen blobs to blobs.
func findUsedBlobs(repo *repository.Repository, treeID backend.ID, blobs backend.IDSet, seen backend.IDSet) error {
	blobs.Insert(treeID)

	tree, err := LoadTree(repo, treeID)
	if err != nil {
		return err
	}

	for _, node := range tree.Nodes {
		switch node.Type {
		case "file":
			for _, blob := range node.Content {
				blobs.Insert(blob)
			}
		case "dir":
			subtreeID := *node.Subtree
			if seen.Has(subtreeID) {
				continue
			}

			seen.Insert(subtreeID)

			err := findUsedBlobs(repo, subtreeID, blobs, seen)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// FindUsedBlobs traverses the tree ID and returns a set of all blobs
// encountered.
func FindUsedBlobs(repo *repository.Repository, treeID backend.ID) (blobs backend.IDSet, err error) {
	blobs = backend.NewIDSet()
	return blobs, findUsedBlobs(repo, treeID, blobs, backend.NewIDSet())
}
<|endoftext|>