<commit_before>package lexer

import (
	"bufio"
	"fmt"
	"io"
	"strings"
	"unicode"
	"unicode/utf8"
)

//TODO: support other encodings besides utf-8 (conversion before the lexer?)

// ItemType identifies the type of lex Items.
type ItemType int

// Item represents a token or text string returned from the scanner.
type Item struct {
	Type ItemType // The type of this Item.
	Pos  int      // The starting position, in bytes, of this item in the input string.
	Val  string   // The value of this Item.
}

const (
	ItemError ItemType = iota // error occurred; value is text of error
	ItemEOF
	ItemWhitespace
	ItemSingleLineComment
	ItemMultiLineComment
	ItemKeyword        // SQL language keyword like SELECT, INSERT, etc.
	ItemOperator       // operators like '=', '<>', etc.
	ItemStar           // *: identifier that matches every column in a table
	ItemIdentifier     // alphanumeric identifier or complex identifier like `a.b` and `c.*`
	ItemLeftParen      // '('
	ItemNumber         // simple number, including imaginary
	ItemRightParen     // ')'
	ItemSpace          // run of spaces separating arguments
	ItemString         // quoted string (includes quotes)
	ItemComment        // comments
	ItemStatementStart // start of a statement like SELECT
	ItemStetementEnd   // ';'
	// etc.
	//TODO: enumerate all item types
)

const EOF = -1

// StateFn represents the state of the scanner as a function that returns the next state.
type StateFn func(*Lexer) StateFn

// ValidatorFn represents a function that is used to check whether a specific rune matches certain rules.
type ValidatorFn func(rune) bool

// Lexer holds the state of the scanner.
type Lexer struct {
	state             StateFn       // the next lexing function to enter
	input             io.RuneReader // the input source
	inputCurrentStart int           // start position of this item
	buffer            []rune        // a slice of runes that contains the currently lexed item
	bufferPos         int           // the current position in the buffer
	Items             chan Item     // channel of scanned Items
}

// next() returns the next rune in the input.
func (l *Lexer) next() rune {
	if l.bufferPos < len(l.buffer) {
		res := l.buffer[l.bufferPos]
		l.bufferPos++
		return res
	}

	r, _, _ := l.input.ReadRune()
	//TODO: handle EOF, panic on other errors
	l.buffer = append(l.buffer, r)
	l.bufferPos++
	return r
}

// peek() returns but does not consume the next rune in the input.
func (l *Lexer) peek() rune {
	if l.bufferPos < len(l.buffer) {
		return l.buffer[l.bufferPos]
	}

	r, _, _ := l.input.ReadRune()
	//TODO: handle EOF, panic on other errors

	l.buffer = append(l.buffer, r)
	return r
}

// peek() returns but does not consume the next few runes in the input.
func (l *Lexer) peekNext(length int) string {
	lenDiff := l.bufferPos + length - len(l.buffer)
	if lenDiff > 0 {
		for i := 0; i < lenDiff; i++ {
			r, _, _ := l.input.ReadRune()
			//TODO: handle EOF, panic on other errors
			l.buffer = append(l.buffer, r)
		}
	}

	return string(l.buffer[l.bufferPos : l.bufferPos+length])
}

// backup steps back one rune
func (l *Lexer) backup() {
	l.backupWith(1)
}

// backup steps back many runes
func (l *Lexer) backupWith(length int) {
	if l.bufferPos < length {
		panic(fmt.Errorf("lexer: trying to backup with %d when the buffer position is %d", length, l.bufferPos))
	}

	l.bufferPos -= length
}

// emit passes an Item back to the client.
func (l *Lexer) emit(t ItemType) {
	l.Items <- Item{t, l.inputCurrentStart, string(l.buffer[:l.bufferPos])}
	l.ignore()
}

// ignore skips over the pending input before this point.
func (l *Lexer) ignore() {
	itemByteLen := 0
	for i := 0; i < l.bufferPos; i++ {
		itemByteLen += utf8.RuneLen(l.buffer[i])
	}

	l.inputCurrentStart += itemByteLen
	l.buffer = l.buffer[l.bufferPos:] //TODO: check for memory leaks, maybe copy remaining items into a new slice?
	l.bufferPos = 0
}

// accept consumes the next rune if it's from the valid set.
func (l *Lexer) accept(valid string) bool {
	r := l.next()
	if strings.IndexRune(valid, r) >= 0 {
		return true
	}
	l.backup()
	return false
}

// acceptWhile consumes runes while the specified condition is true
func (l *Lexer) acceptWhile(fn ValidatorFn) {
	r := l.next()
	for fn(r) {
		r = l.next()
	}
	l.backup()
}

// acceptUntil consumes runes until the specified contidtion is met
func (l *Lexer) acceptUntil(fn ValidatorFn) {
	r := l.next()
	for !fn(r) {
		r = l.next()
	}
	l.backup()
}

// acceptUntil consumes runes until the specified string is met
func (l *Lexer) acceptUntilMatch(match string) {
	length := len(match)
	next := l.peekNext(length)
	for next != match {
		l.next()
		next = l.peekNext(length)
	}
}

// nextItem returns the next Item from the input.
func (l *Lexer) nextItem() Item {
	return <-l.Items
}

// lex creates a new scanner for the input string.
func Lex(input io.Reader) *Lexer {
	l := &Lexer{
		input:  bufio.NewReader(input),
		buffer: make([]rune, 0, 10),
		Items:  make(chan Item),
	}
	go l.run()
	return l
}

// run runs the state machine for the Lexer.
func (l *Lexer) run() {
	for state := lexWhitespace; state != nil; {
		state = state(l)
	}
	close(l.Items)
}

// isSpace reports whether r is a whitespace character (space or end of line).
func isWhitespace(r rune) bool {
	return r == ' ' || r == '\t' || r == '\r' || r == '\n'
}

// isSpace reports whether r is a space character.
func isSpace(r rune) bool {
	return r == ' ' || r == '\t'
}

// isEndOfLine reports whether r is an end-of-line character.
func isEndOfLine(r rune) bool {
	return r == '\r' || r == '\n'
}

// isAlphaNumeric reports whether r is an alphabetic, digit, or underscore.
func isAlphaNumeric(r rune) bool {
	return r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r)
}

func lexWhitespace(l *Lexer) StateFn {
	l.acceptWhile(isWhitespace)
	if l.bufferPos > 0 {
		l.emit(ItemWhitespace)
	}

	next := l.peek()
	nextTwo := l.peekNext(2)

	switch {
	case next == EOF:
		l.emit(ItemEOF)
		return nil

	case nextTwo == "--":
		return lexSingleLineComment

	case nextTwo == "/*":
		return lexMultiLineComment

	case next == '*':
		//TODO: determine if this is neccessary or should be classified as an identifier
		l.next()
		l.emit(ItemStar)
		return lexWhitespace

	case next == '(':
		l.next()
		l.emit(ItemLeftParen)
		return lexWhitespace

	case next == ')':
		l.next()
		l.emit(ItemRightParen)
		return lexWhitespace

	/*
		//TODO: finish different cases
		case next == '*':
			return lexStar
		case next == '`':
			return lexIdentifier
		case next == '"' || next == '\'':
			return lexString
		case next == '+' || next == '-' || ('0' <= next && next <= '9'):
			return lexNumber
	*/

	case isAlphaNumeric(next):
		return lexKeyWordOrIdentifier

	default:
		//TODO: enable panic :)
		//panic(fmt.Sprintf("don't know what to do with: %q", next))
		l.emit(ItemEOF)
		return nil
	}
}

func lexSingleLineComment(l *Lexer) StateFn {
	l.acceptUntil(isEndOfLine)
	l.emit(ItemSingleLineComment)
	return lexWhitespace
}

func lexMultiLineComment(l *Lexer) StateFn {
	l.acceptUntilMatch("*/")
	l.next()
	l.next()
	l.emit(ItemMultiLineComment)
	return lexWhitespace
}

func lexKeyWordOrIdentifier(l *Lexer) StateFn {
	l.acceptWhile(isAlphaNumeric)
	l.emit(ItemIdentifier)
	//TODO: determine whether this is a keyword
	return lexWhitespace
}
<commit_msg>Alpha version of the lexer - should be almost finished except for tuning.<commit_after>package lexer

import (
	"bufio"
	"fmt"
	"io"
	"strings"
	"unicode"
	"unicode/utf8"
)

//TODO: support other encodings besides utf-8 (conversion before the lexer?)

// ItemType identifies the type of lex Items.
type ItemType int

// Item represents a token or text string returned from the scanner.
type Item struct {
	Type ItemType // The type of this Item.
	Pos  int      // The starting position, in bytes, of this item in the input string.
	Val  string   // The value of this Item.
}

const (
	ItemError             ItemType = iota // error occurred; value is text of error
	ItemEOF                               // end of the file
	ItemWhitespace                        // a run of spaces, tabs and newlines
	ItemSingleLineComment                 // A comment like --
	ItemMultiLineComment                  // A multiline comment like /* ... */
	ItemKeyword                           // SQL language keyword like SELECT, INSERT, etc.
	ItemIdentifier                        // alphanumeric identifier or complex identifier like `a.b` and `c`.*
	ItemOperator                          // operators like '=', '<>', etc.
	ItemLeftParen                         // '('
	ItemRightParen                        // ')'
	ItemComma                             // ','
	ItemDot                               // '.'
	ItemStetementEnd                      // ';'
	ItemNumber                            // simple number, including imaginary
	ItemString                            // quoted string (includes quotes)
)

const EOF = -1

// StateFn represents the state of the scanner as a function that returns the next state.
type StateFn func(*Lexer) StateFn

// ValidatorFn represents a function that is used to check whether a specific rune matches certain rules.
type ValidatorFn func(rune) bool

// Lexer holds the state of the scanner.
type Lexer struct {
	state             StateFn       // the next lexing function to enter
	input             io.RuneReader // the input source
	inputCurrentStart int           // start position of this item
	buffer            []rune        // a slice of runes that contains the currently lexed item
	bufferPos         int           // the current position in the buffer
	Items             chan Item     // channel of scanned Items
}

// next() returns the next rune in the input.
func (l *Lexer) next() rune {
	if l.bufferPos < len(l.buffer) {
		res := l.buffer[l.bufferPos]
		l.bufferPos++
		return res
	}

	r, _, err := l.input.ReadRune()
	if err == io.EOF {
		r = EOF
	} else if err != nil {
		panic(err)
	}

	l.buffer = append(l.buffer, r)
	l.bufferPos++
	return r
}

// peek() returns but does not consume the next rune in the input.
func (l *Lexer) peek() rune {
	if l.bufferPos < len(l.buffer) {
		return l.buffer[l.bufferPos]
	}

	r, _, err := l.input.ReadRune()
	if err == io.EOF {
		r = EOF
	} else if err != nil {
		panic(err)
	}

	l.buffer = append(l.buffer, r)
	return r
}

// peek() returns but does not consume the next few runes in the input.
func (l *Lexer) peekNext(length int) string {
	lenDiff := l.bufferPos + length - len(l.buffer)
	if lenDiff > 0 {
		for i := 0; i < lenDiff; i++ {
			r, _, err := l.input.ReadRune()
			if err == io.EOF {
				r = EOF
			} else if err != nil {
				panic(err)
			}

			l.buffer = append(l.buffer, r)
		}
	}

	return string(l.buffer[l.bufferPos : l.bufferPos+length])
}

// backup steps back one rune
func (l *Lexer) backup() {
	l.backupWith(1)
}

// backup steps back many runes
func (l *Lexer) backupWith(length int) {
	if l.bufferPos < length {
		panic(fmt.Errorf("lexer: trying to backup with %d when the buffer position is %d", length, l.bufferPos))
	}

	l.bufferPos -= length
}

// emit passes an Item back to the client.
func (l *Lexer) emit(t ItemType) {
	l.Items <- Item{t, l.inputCurrentStart, string(l.buffer[:l.bufferPos])}
	l.ignore()
}

// ignore skips over the pending input before this point.
func (l *Lexer) ignore() {
	itemByteLen := 0
	for i := 0; i < l.bufferPos; i++ {
		itemByteLen += utf8.RuneLen(l.buffer[i])
	}

	l.inputCurrentStart += itemByteLen
	l.buffer = l.buffer[l.bufferPos:] //TODO: check for memory leaks, maybe copy remaining items into a new slice?
	l.bufferPos = 0
}

// accept consumes the next rune if it's from the valid set.
func (l *Lexer) accept(valid string) int {
	r := l.next()
	if strings.IndexRune(valid, r) >= 0 {
		return 1
	}
	l.backup()
	return 0
}

// acceptWhile consumes runes while the specified condition is true
func (l *Lexer) acceptWhile(fn ValidatorFn) int {
	r := l.next()
	count := 0
	for fn(r) {
		r = l.next()
		count++
	}
	l.backup()
	return count
}

// acceptUntil consumes runes until the specified contidtion is met
func (l *Lexer) acceptUntil(fn ValidatorFn) int {
	r := l.next()
	count := 0
	for !fn(r) && r != EOF {
		r = l.next()
		count++
	}
	l.backup()
	return count
}

// errorf returns an error token and terminates the scan by passing
// back a nil pointer that will be the next state, terminating l.nextItem.
func (l *Lexer) errorf(format string, args ...interface{}) StateFn {
	l.Items <- Item{ItemError, l.inputCurrentStart, fmt.Sprintf(format, args...)}
	return nil
}

// nextItem returns the next Item from the input.
func (l *Lexer) nextItem() Item {
	return <-l.Items
}

// lex creates a new scanner for the input string.
func Lex(input io.Reader) *Lexer {
	l := &Lexer{
		input:  bufio.NewReader(input),
		buffer: make([]rune, 0, 10),
		Items:  make(chan Item),
	}
	go l.run()
	return l
}

// run runs the state machine for the Lexer.
func (l *Lexer) run() {
	for state := lexWhitespace; state != nil; {
		state = state(l)
	}
	close(l.Items)
}

// isSpace reports whether r is a whitespace character (space or end of line).
func isWhitespace(r rune) bool {
	return r == ' ' || r == '\t' || r == '\r' || r == '\n'
}

// isSpace reports whether r is a space character.
func isSpace(r rune) bool {
	return r == ' ' || r == '\t'
}

// isEndOfLine reports whether r is an end-of-line character.
func isEndOfLine(r rune) bool {
	return r == '\r' || r == '\n' || r == EOF
}

// isAlphaNumeric reports whether r is an alphabetic, digit, or underscore.
func isAlphaNumeric(r rune) bool {
	return r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r)
}

// isOperator reports whether r is an operator.
func isOperator(r rune) bool {
	return r == '+' || r == '-' || r == '*' || r == '/' || r == '=' || r == '>' || r == '<' || r == '~' || r == '|' || r == '^' || r == '&' || r == '%'
}

// isComplexIdentifier reports whether r is an alphabetic, digit, or underscore.
func isComplexIdentifier(r rune) bool {
	return isAlphaNumeric(r) || r == '.' || r == '`' || r == '*'
}

func lexWhitespace(l *Lexer) StateFn {
	l.acceptWhile(isWhitespace)
	if l.bufferPos > 0 {
		l.emit(ItemWhitespace)
	}

	next := l.peek()
	nextTwo := l.peekNext(2)

	switch {
	case next == EOF:
		l.emit(ItemEOF)
		return nil

	case nextTwo == "--":
		return lexSingleLineComment

	case nextTwo == "/*":
		return lexMultiLineComment

	case next == '(':
		l.next()
		l.emit(ItemLeftParen)
		return lexWhitespace

	case next == ')':
		l.next()
		l.emit(ItemRightParen)
		return lexWhitespace

	case next == ',':
		l.next()
		l.emit(ItemComma)
		return lexWhitespace

	case next == ';':
		l.next()
		l.emit(ItemStetementEnd)
		return lexWhitespace

	case isOperator(next):
		return lexOperator

	case next == '"' || next == '\'':
		return lexString

	case ('0' <= next && next <= '9'):
		return lexNumber

	case isAlphaNumeric(next) || next == '`':
		return lexIdentifierOrKeyword

	default:
		l.errorf("don't know what to do with '%s'", nextTwo)
		return nil
	}
}

func lexSingleLineComment(l *Lexer) StateFn {
	l.acceptUntil(isEndOfLine)
	l.emit(ItemSingleLineComment)
	return lexWhitespace
}

func lexMultiLineComment(l *Lexer) StateFn {
	l.next()
	l.next()
	for {
		l.acceptUntil(func(r rune) bool { return r == '*' })
		if l.peekNext(2) == "*/" {
			l.next()
			l.next()
			l.emit(ItemMultiLineComment)
			return lexWhitespace
		}

		if l.peek() == EOF {
			l.errorf("reached EOF when looking for comment end")
			return nil
		}

		l.next()
	}
}

func lexOperator(l *Lexer) StateFn {
	l.acceptWhile(isOperator)
	l.emit(ItemOperator)
	return lexWhitespace
}

func lexNumber(l *Lexer) StateFn {
	count := 0
	count += l.acceptWhile(unicode.IsDigit)
	if l.accept(".") > 0 {
		count += 1 + l.acceptWhile(unicode.IsDigit)
	}
	if l.accept("eE") > 0 {
		count += 1 + l.accept("+-")
		count += l.acceptWhile(unicode.IsDigit)
	}

	if isAlphaNumeric(l.peek()) {
		// We were lexing an identifier all along - backup and pass the ball
		l.backupWith(count)
		return lexIdentifierOrKeyword
	}

	l.emit(ItemNumber)
	return lexWhitespace
}

func lexString(l *Lexer) StateFn {
	quote := l.next()

	for {
		n := l.next()

		if n == EOF {
			return l.errorf("unterminated quoted string")
		}
		if n == '\\' {
			//TODO: fix possible problems with NO_BACKSLASH_ESCAPES mode
			if l.peek() == EOF {
				return l.errorf("unterminated quoted string")
			}
			l.next()
		}

		if n == quote {
			if l.peek() == quote {
				l.next()
			} else {
				l.emit(ItemString)
				return lexWhitespace
			}
		}
	}

}

func lexIdentifierOrKeyword(l *Lexer) StateFn {
	for {
		s := l.next()

		if s == '`' {
			for {
				n := l.next()

				if n == EOF {
					return l.errorf("unterminated quoted string")
				} else if n == '`' {
					if (l.peek() == '`') {
						l.next()
					} else {
						break
					}
				}
			}
			l.emit(ItemIdentifier)
		} else if isAlphaNumeric(s) {
			l.acceptWhile(isAlphaNumeric)

			//TODO: check whether token is a keyword or an identifier
			l.emit(ItemIdentifier)
		}

		l.acceptWhile(isWhitespace)
		if l.bufferPos > 0 {
			l.emit(ItemWhitespace)
		}

		if l.peek() != '.' {
			break
		}

		l.next()
		l.emit(ItemDot)
	}

	return lexWhitespace
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"os/user"
	"strings"
	"text/tabwriter"

	"code.google.com/p/gcfg"

	"github.com/atsaki/golang-cloudstack-library"
	"github.com/codegangsta/cli"
)

const (
	configfile = "~/.cloudmonkey/config"
)

type Config struct {
	User struct {
		Username  string
		Password  string
		Secretkey string
		Apikey    string
	}
	Server struct {
		Protocol string
		Host     string
		Port     string
		Path     string
	}
}

func expandPath(path string) string {
	usr, _ := user.Current()
	home := usr.HomeDir

	if strings.HasPrefix(path, "~/") {
		path = strings.Replace(path, "~/", home+"/", 1)
	}
	return path
}

func main() {

	log.SetOutput(ioutil.Discard)

	cfg := Config{}

	err := gcfg.ReadFileInto(&cfg, expandPath(configfile))
	if err != nil {
		log.Fatal(err)
	}

	if strings.HasPrefix(cfg.Server.Path, "/") {
		cfg.Server.Path = strings.Replace(cfg.Server.Path, "/", "", 1)
	}

	endpoint := url.URL{
		Scheme: cfg.Server.Protocol,
		Host:   fmt.Sprintf("%s:%s", cfg.Server.Host, cfg.Server.Port),
		Path:   cfg.Server.Path,
	}

	client, _ := cloudstack.NewClient(endpoint,
		cfg.User.Apikey, cfg.User.Secretkey, cfg.User.Username, cfg.User.Password)

	sep := '\t'
	tabw := new(tabwriter.Writer)
	tabw.Init(os.Stdout, 0, 8, 0, byte(sep), 0)

	app := cli.NewApp()
	app.Name = "lg"
	app.Usage = "lg comand"

	app.Commands = []cli.Command{
		{
			Name:      "virtualmachines",
			ShortName: "vms",
			Usage:     "list virtualmachines",
			Action: func(c *cli.Context) {
				params := cloudstack.ListVirtualMachinesParameter{}
				resp, _ := client.ListVirtualMachines(params)
				for _, v := range resp.Virtualmachine {
					fmt.Fprintln(
						tabw,
						strings.Join(
							[]string{
								v.Id.String,
								v.Name.String,
								v.Displayname.String,
								v.State.String,
								v.Zonename.String,
								v.Templatename.String,
								v.Serviceofferingname.String,
							}, string(sep)))
				}
				tabw.Flush()
			},
		},
		{
			Name:  "deploy",
			Usage: "deploy virtualmachine",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "zone",
					Value: "",
				},
				cli.StringFlag{
					Name:  "serviceoffering",
					Value: "",
				},
				cli.StringFlag{
					Name:  "template",
					Value: "",
				},
				cli.StringFlag{
					Name:  "displayname",
					Value: "",
				},
			},
			Action: func(c *cli.Context) {
				params := cloudstack.DeployVirtualMachineParameter{}
				if c.String("zone") != "" {
					params.SetZoneid(c.String("zone"))
				}
				if c.String("serviceoffering") != "" {
					params.SetServiceofferingid(c.String("serviceoffering"))
				}
				if c.String("template") != "" {
					params.SetTemplateid(c.String("template"))
				}
				if c.String("displayname") != "" {
					params.SetDisplayname(c.String("displayname"))
				}
				resp, _ := client.DeployVirtualMachine(params)
				v := resp.Virtualmachine
				fmt.Fprintln(
					tabw,
					strings.Join(
						[]string{
							v.Id.String,
							v.Name.String,
							v.Displayname.String,
							v.State.String,
							v.Zonename.String,
							v.Templatename.String,
							v.Serviceofferingname.String,
						}, string(sep)))
				tabw.Flush()
			},
		},
		{
			Name:  "start",
			Usage: "start virtualmachine",
			Action: func(c *cli.Context) {
				params := cloudstack.StartVirtualMachineParameter{}
				if len(c.Args()) == 0 {
					scanner := bufio.NewScanner(os.Stdin)
					for scanner.Scan() {
						id := strings.Split(scanner.Text(), string(sep))[0]
						params.SetId(id)
						resp, _ := client.StartVirtualMachine(params)
						v := resp.Virtualmachine
						fmt.Fprintln(
							tabw,
							strings.Join(
								[]string{
									v.Id.String,
									v.Name.String,
									v.Displayname.String,
									v.State.String,
									v.Zonename.String,
									v.Templatename.String,
									v.Serviceofferingname.String,
								}, string(sep)))
					}
					tabw.Flush()
				} else {
					params.SetId(c.Args()[0])
					resp, _ := client.StartVirtualMachine(params)
					v := resp.Virtualmachine
					fmt.Fprintln(
						tabw,
						strings.Join(
							[]string{
								v.Id.String,
								v.Name.String,
								v.Displayname.String,
								v.State.String,
								v.Zonename.String,
								v.Templatename.String,
								v.Serviceofferingname.String,
							}, string(sep)))
					tabw.Flush()
				}
			},
		},
		{
			Name:  "stop",
			Usage: "stop virtualmachine",
			Action: func(c *cli.Context) {
				params := cloudstack.StopVirtualMachineParameter{}
				if len(c.Args()) == 0 {
					scanner := bufio.NewScanner(os.Stdin)
					for scanner.Scan() {
						id := strings.Split(scanner.Text(), string(sep))[0]
						params.SetId(id)
						resp, _ := client.StopVirtualMachine(params)
						v := resp.Virtualmachine
						fmt.Fprintln(
							tabw,
							strings.Join(
								[]string{
									v.Id.String,
									v.Name.String,
									v.Displayname.String,
									v.State.String,
									v.Zonename.String,
									v.Templatename.String,
									v.Serviceofferingname.String,
								}, string(sep)))
					}
					tabw.Flush()
				} else {
					params.SetId(c.Args()[0])
					resp, _ := client.StopVirtualMachine(params)
					v := resp.Virtualmachine
					fmt.Fprintln(
						tabw,
						strings.Join(
							[]string{
								v.Id.String,
								v.Name.String,
								v.Displayname.String,
								v.State.String,
								v.Zonename.String,
								v.Templatename.String,
								v.Serviceofferingname.String,
							}, string(sep)))
					tabw.Flush()
				}
			},
		},
		{
			Name:  "destroy",
			Usage: "destroy virtualmachine",
			Action: func(c *cli.Context) {
				params := cloudstack.DestroyVirtualMachineParameter{}
				if len(c.Args()) == 0 {
					scanner := bufio.NewScanner(os.Stdin)
					for scanner.Scan() {
						id := strings.Split(scanner.Text(), string(sep))[0]
						params.SetId(id)
						resp, _ := client.DestroyVirtualMachine(params)
						v := resp.Virtualmachine
						fmt.Fprintln(
							tabw,
							strings.Join(
								[]string{
									v.Id.String,
									v.Name.String,
									v.Displayname.String,
									v.State.String,
									v.Zonename.String,
									v.Templatename.String,
									v.Serviceofferingname.String,
								}, string(sep)))
					}
					tabw.Flush()
				} else {
					params.SetId(c.Args()[0])
					resp, _ := client.DestroyVirtualMachine(params)
					v := resp.Virtualmachine
					fmt.Fprintln(
						tabw,
						strings.Join(
							[]string{
								v.Id.String,
								v.Name.String,
								v.Displayname.String,
								v.State.String,
								v.Zonename.String,
								v.Templatename.String,
								v.Serviceofferingname.String,
							}, string(sep)))
					tabw.Flush()
				}
			},
		},
		{
			Name:  "zones",
			Usage: "list zones",
			Action: func(c *cli.Context) {
				params := cloudstack.ListZonesParameter{}
				resp, _ := client.ListZones(params)
				for _, v := range resp.Zone {
					fmt.Fprintln(
						tabw,
						strings.Join(
							[]string{
								v.Id.String,
								v.Name.String,
							}, string(sep)))
				}
				tabw.Flush()
			},
		},
		{
			Name:      "serviceofferings",
			ShortName: "sizes",
			Usage:     "list serviceofferings",
			Action: func(c *cli.Context) {
				params := cloudstack.ListServiceOfferingsParameter{}
				resp, _ := client.ListServiceOfferings(params)
				for _, v := range resp.Serviceoffering {
					fmt.Fprintln(
						tabw,
						strings.Join(
							[]string{
								v.Id.String,
								v.Name.String,
							}, string(sep)))
				}
				tabw.Flush()
			},
		},
		{
			Name:      "templates",
			ShortName: "images",
			Usage:     "list templates",
			Action: func(c *cli.Context) {
				params := cloudstack.ListTemplatesParameter{}
				params.SetTemplatefilter("featured")
				resp, _ := client.ListTemplates(params)
				for _, v := range resp.Template {
					fmt.Fprintln(
						tabw,
						strings.Join(
							[]string{
								v.Id.String,
								v.Name.String,
								v.Displaytext.String,
							}, string(sep)))
				}
				tabw.Flush()
			},
		},
		{
			Name:      "networks",
			ShortName: "nws",
			Usage:     "list network",
			Action: func(c *cli.Context) {
				params := cloudstack.ListNetworksParameter{}
				resp, _ := client.ListNetworks(params)
				for _, v := range resp.Network {
					fmt.Fprintln(
						tabw,
						strings.Join(
							[]string{
								v.Id.String,
								v.Name.String,
								v.Networkofferingname.String,
							}, string(sep)))
				}
				tabw.Flush()
			},
		},
	}

	app.Run(os.Args)
}
<commit_msg>update for library changes<commit_after>package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"os/user"
	"strings"
	"text/tabwriter"

	"github.com/atsaki/golang-cloudstack-library"
	"github.com/codegangsta/cli"
	"github.com/vaughan0/go-ini"
)

const (
	sep = '\t'
)

var (
	client *cloudstack.Client
)

func expandPath(path string) string {
	usr, _ := user.Current()
	home := usr.HomeDir

	if strings.HasPrefix(path, "~/") {
		path = strings.Replace(path, "~/", home+"/", 1)
	}
	return path
}

func setup(c *cli.Context) {
	if !c.GlobalBool("debug") {
		log.SetOutput(ioutil.Discard)
	}
	configfile := expandPath(c.GlobalString("config-file"))
	log.Println("configfile:", configfile)
	cfg, err := ini.LoadFile(configfile)
	if err != nil {
		log.Fatal(err)
	}
	var ok bool

	profile, ok := cfg.Get("core", "profile")
	if !ok {
		profile = "local"
	}
	log.Println("profile:", profile)

	endpointUrl, ok := cfg.Get(profile, "url")
	if !ok {
		log.Fatalf("URL is not specified")
	}
	log.Println("url:", endpointUrl)

	apikey, ok := cfg.Get(profile, "apikey")
	if !ok {
		apikey = ""
	}
	log.Println("apikey:", apikey)

	secretkey, ok := cfg.Get(profile, "secretkey")
	if !ok {
		secretkey = ""
	}
	log.Println("secretkey:", secretkey)

	username, ok := cfg.Get(profile, "username")
	if !ok {
		username = ""
	}
	log.Println("username:", username)

	password, ok := cfg.Get(profile, "password")
	if !ok {
		password = ""
	}
	log.Println("password:", password)

	endpoint, err := url.Parse(endpointUrl)
	if err != nil {
		log.Fatal(err)
	}
	log.Println("endpoint:", endpoint)

	client, err = cloudstack.NewClient(*endpoint, apikey, secretkey,
		username, password)
	if err != nil {
		log.Fatal(err)
	}
}

func main() {

	tabw := new(tabwriter.Writer)
	tabw.Init(os.Stdout, 0, 8, 0, byte(sep), 0)

	app := cli.NewApp()
	app.Name = "lg"
	app.Usage = "lg comand"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "config-file, c",
			Value: "~/.cloudmonkey/config",
			Usage: "Config file path",
		},
		cli.BoolFlag{
			Name:  "debug",
			Usage: "Show debug messages",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:      "virtualmachines",
			ShortName: "vms",
			Usage:     "List virtualmachines",
			Action: func(c *cli.Context) {
				setup(c)
				params := cloudstack.ListVirtualMachinesParameter{}
				vms, err := client.ListVirtualMachines(params)
				if err != nil {
					log.Fatal(err)
				}
				for _, vm := range vms {
					fmt.Fprintln(
						tabw,
						strings.Join(
							[]string{
								vm.Id.String,
								vm.Name.String,
								vm.Displayname.String,
								vm.State.String,
								vm.Zonename.String,
								vm.Templatename.String,
								vm.Serviceofferingname.String,
							}, string(sep)))
				}
				tabw.Flush()
			},
		},
		{
			Name:  "deploy",
			Usage: "Deploy virtualmachine",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "zone, z",
					Value: "",
					Usage: "The zoneid or zonename of the virtualmachine",
				},
				cli.StringFlag{
					Name:  "serviceoffering, s",
					Value: "",
					Usage: "The serviceofferingid or serviceofferingname of the virtualmachine",
				},
				cli.StringFlag{
					Name:  "template, t",
					Value: "",
					Usage: "The templateid or templatename of the virtualmachine",
				},
				cli.StringFlag{
					Name:  "displayname",
					Value: "",
					Usage: "The displayname of the virtualmachine",
				},
			},
			Action: func(c *cli.Context) {
				setup(c)
				params := cloudstack.DeployVirtualMachineParameter{}
				if c.String("zone") != "" {
					params.SetZoneid(c.String("zone"))
				}
				if c.String("serviceoffering") != "" {
					params.SetServiceofferingid(c.String("serviceoffering"))
				}
				if c.String("template") != "" {
					params.SetTemplateid(c.String("template"))
				}
				if c.String("displayname") != "" {
					params.SetDisplayname(c.String("displayname"))
				}
				vm, err := client.DeployVirtualMachine(params)
				if err != nil {
					log.Fatal(err)
				}
				fmt.Fprintln(
					tabw,
					strings.Join(
						[]string{
							vm.Id.String,
							vm.Name.String,
							vm.Displayname.String,
							vm.State.String,
							vm.Zonename.String,
							vm.Templatename.String,
							vm.Serviceofferingname.String,
						}, string(sep)))
				tabw.Flush()
			},
		},
		{
			Name:  "start",
			Usage: "Start virtualmachine",
			Action: func(c *cli.Context) {
				setup(c)
				params := cloudstack.StartVirtualMachineParameter{}
				var ids []string
				if len(c.Args()) == 0 {
					scanner := bufio.NewScanner(os.Stdin)
					for scanner.Scan() {
						ids = append(ids, strings.Split(scanner.Text(), string(sep))[0])
					}
				} else if len(c.Args()) == 1 {
					ids = append(ids, c.Args()[0])
				}
				log.Println("ids:", ids)
				for _, id := range ids {
					params.SetId(id)
					vm, err := client.StartVirtualMachine(params)
					if err != nil {
						log.Fatal(err)
					}
					fmt.Fprintln(
						tabw,
						strings.Join(
							[]string{
								vm.Id.String,
								vm.Name.String,
								vm.Displayname.String,
								vm.State.String,
								vm.Zonename.String,
								vm.Templatename.String,
								vm.Serviceofferingname.String,
							}, string(sep)))
				}
				tabw.Flush()
			},
		},
		{
			Name:  "stop",
			Usage: "Stop virtualmachine",
			Action: func(c *cli.Context) {
				setup(c)
				params := cloudstack.StopVirtualMachineParameter{}
				var ids []string
				if len(c.Args()) == 0 {
					scanner := bufio.NewScanner(os.Stdin)
					for scanner.Scan() {
						ids = append(ids, strings.Split(scanner.Text(), string(sep))[0])
					}
				} else if len(c.Args()) == 1 {
					ids = append(ids, c.Args()[0])
				}
				log.Println("ids:", ids)
				for _, id := range ids {
					params.SetId(id)
					vm, err := client.StopVirtualMachine(params)
					if err != nil {
						log.Fatal(err)
					}
					fmt.Fprintln(
						tabw,
						strings.Join(
							[]string{
								vm.Id.String,
								vm.Name.String,
								vm.Displayname.String,
								vm.State.String,
								vm.Zonename.String,
								vm.Templatename.String,
								vm.Serviceofferingname.String,
							}, string(sep)))
				}
				tabw.Flush()
			},
		},
		{
			Name:  "destroy",
			Usage: "Destroy virtualmachine",
			Action: func(c *cli.Context) {
				setup(c)
				params := cloudstack.DestroyVirtualMachineParameter{}
				var ids []string
				if len(c.Args()) == 0 {
					scanner := bufio.NewScanner(os.Stdin)
					for scanner.Scan() {
						ids = append(ids, strings.Split(scanner.Text(), string(sep))[0])
					}
				} else if len(c.Args()) == 1 {
					ids = append(ids, c.Args()[0])
				}
				log.Println("ids:", ids)
				for _, id := range ids {
					params.SetId(id)
					vm, err := client.DestroyVirtualMachine(params)
					if err != nil {
						log.Fatal(err)
					}
					fmt.Fprintln(
						tabw,
						strings.Join(
							[]string{
								vm.Id.String,
								vm.Name.String,
								vm.Displayname.String,
								vm.State.String,
								vm.Zonename.String,
								vm.Templatename.String,
								vm.Serviceofferingname.String,
							}, string(sep)))
				}
				tabw.Flush()
			},
		},
		{
			Name:  "zones",
			Usage: "List zones",
			Action: func(c *cli.Context) {
				setup(c)
				params := cloudstack.ListZonesParameter{}
				zones, err := client.ListZones(params)
				if err != nil {
					log.Fatal(err)
				}
				for _, zone := range zones {
					fmt.Fprintln(
						tabw,
						strings.Join(
							[]string{
								zone.Id.String,
								zone.Name.String,
							}, string(sep)))
				}
				tabw.Flush()
			},
		},
		{
			Name:      "serviceofferings",
			ShortName: "sizes",
			Usage:     "List serviceofferings",
			Action: func(c *cli.Context) {
				setup(c)
				params := cloudstack.ListServiceOfferingsParameter{}
				serviceofferings, err := client.ListServiceOfferings(params)
				if err != nil {
					log.Fatal(err)
				}
				for _, serviceoffering := range serviceofferings {
					fmt.Fprintln(
						tabw,
						strings.Join(
							[]string{
								serviceoffering.Id.String,
								serviceoffering.Name.String,
								fmt.Sprint(serviceoffering.Cpunumber.Int64),
								fmt.Sprint(serviceoffering.Cpuspeed.Int64),
								fmt.Sprint(serviceoffering.Memory.Int64),
							}, string(sep)))
				}
				tabw.Flush()
			},
		},
		{
			Name:      "templates",
			ShortName: "images",
			Usage:     "list templates",
			Action: func(c *cli.Context) {
				setup(c)
				params := cloudstack.ListTemplatesParameter{}
				params.SetTemplatefilter("featured")
				templates, err := client.ListTemplates(params)
				if err != nil {
					log.Fatal(err)
				}
				for _, template := range templates {
					fmt.Fprintln(
						tabw,
						strings.Join(
							[]string{
								template.Id.String,
								template.Name.String,
								template.Displaytext.String,
							}, string(sep)))
				}
				tabw.Flush()
			},
		},
		{
			Name:      "networks",
			ShortName: "nws",
			Usage:     "list network",
			Action: func(c *cli.Context) {
				setup(c)
				params := cloudstack.ListNetworksParameter{}
				networks, err := client.ListNetworks(params)
				if err != nil {
					log.Fatal(err)
				}
				for _, network := range networks {
					fmt.Fprintln(
						tabw,
						strings.Join(
							[]string{
								network.Id.String,
								network.Name.String,
								network.Networkofferingname.String,
							}, string(sep)))
				}
				tabw.Flush()
			},
		},
		{
			Name:      "publicipaddresses",
			ShortName: "ips",
			Usage:     "list ipaddresses",
			Action: func(c *cli.Context) {
				setup(c)
				params := cloudstack.ListPublicIpAddressesParameter{}
				ips, err := client.ListPublicIpAddresses(params)
				if err != nil {
					log.Fatal(err)
				}
				for _, ip := range ips {
					fmt.Fprintln(
						tabw,
						strings.Join(
							[]string{
								ip.Id.String,
								ip.Zonename.String,
								ip.Associatednetworkname.String,
								fmt.Sprint(ip.Issourcenat.Bool),
								ip.Ipaddress.String,
								ip.Virtualmachinedisplayname.String,
							}, string(sep)))
				}
				tabw.Flush()
			},
		},
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package obj

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/aws/aws-sdk-go/service/storagegateway"
	"github.com/cenkalti/backoff"
)

type amazonClient struct {
	bucket       string
	distribution string
	s3           *s3.S3
	uploader     *s3manager.Uploader
}

func newAmazonClient(bucket string, distribution string, id string, secret string, token string, region string) (*amazonClient, error) {
	session := session.New(&aws.Config{
		Credentials: credentials.NewStaticCredentials(id, secret, token),
		Region:      aws.String(region),
	})
	return &amazonClient{
		bucket:   bucket,
		s3:       s3.New(session),
		uploader: s3manager.NewUploader(session),
	}, nil
}

func (c *amazonClient) usingCloudfront() bool {
	return c.distribution != ""
}

func (c *amazonClient) Writer(name string) (io.WriteCloser, error) {
	return newBackoffWriteCloser(c, newWriter(c, name)), nil
}

func (c *amazonClient) Walk(name string, fn func(name string) error) error {
	var fnErr error
	if err := c.s3.ListObjectsPages(
		&s3.ListObjectsInput{
			Bucket: aws.String(c.bucket),
			Prefix: aws.String(name),
		},
		func(listObjectsOutput *s3.ListObjectsOutput, lastPage bool) bool {
			for _, object := range listObjectsOutput.Contents {
				if err := fn(*object.Key); err != nil {
					fnErr = err
					return false
				}
			}
			return true
		},
	); err != nil {
		return err
	}
	return fnErr
}

func (c *amazonClient) Reader(name string, offset uint64, size uint64) (io.ReadCloser, error) {
	byteRange := byteRange(offset, size)
	if byteRange != "" {
		byteRange = fmt.Sprintf("bytes=%s", byteRange)
	}

	var reader io.ReadCloser
	if c.usingCloudfront {
		var resp *http.Response
		var connErr error
		url := fmt.Sprintf("http://%v.cloudfront.net/%v", c.distribution, name)

		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			return nil, err
		}
		req.Header.Add("Range", byteRange)

		backoff.RetryNotify(func() error {
			resp, connErr = http.DefaultClient.Do(req)
			if connErr != nil && isRetryableGetError(connErr) {
				fmt.Printf("this is a retryable error (%v)\n", connErr)
				return connErr
			}
			return nil
		}, backoff.NewExponentialBackOff(), func(err error, d time.Duration) {
			log.Infof("Error connecting to (%v); retrying in %s: %#v", url, d, err)
		})

		if resp.StatusCode != 200 {
			fmt.Printf("HTTP error code %v", resp.StatusCode)
			return nil, fmt.Errorf("cloudfront returned HTTP error code %v", resp.StatusCode)
		}
		reader = resp.Body
	} else {
		getObjectOutput, err := c.s3.GetObject(&s3.GetObjectInput{
			Bucket: aws.String(c.bucket),
			Key:    aws.String(name),
			Range:  aws.String(byteRange),
		})
		if err != nil {
			return nil, err
		}
		reader = getObjectOutput.Body
	}
	return newBackoffReadCloser(c, reader), nil
}

func (c *amazonClient) Delete(name string) error {
	_, err := c.s3.DeleteObject(&s3.DeleteObjectInput{
		Bucket: aws.String(c.bucket),
		Key:    aws.String(name),
	})
	return err
}

func (c *amazonClient) Exists(name string) bool {
	_, err := c.s3.HeadObject(&s3.HeadObjectInput{
		Bucket: aws.String(c.bucket),
		Key:    aws.String(name),
	})
	return err == nil
}

func (c *amazonClient) isRetryable(err error) (retVal bool) {
	fmt.Printf("is err (%v) retryable?\n", err)
	defer func() {
		fmt.Printf("err (%v) retryable? %v\n", retVal, err)
	}()
	if strings.Contains(err.Error(), "unexpected EOF") {
		return false
	}

	awsErr, ok := err.(awserr.Error)
	if !ok {
		return false
	}
	for _, c := range []string{
		storagegateway.ErrorCodeServiceUnavailable,
		storagegateway.ErrorCodeInternalError,
		storagegateway.ErrorCodeGatewayInternalError,
	} {
		if c == awsErr.Code() {
			return true
		}
	}
	return false
}

func (c *amazonClient) IsIgnorable(err error) bool {
	return false
}

func (c *amazonClient) IsNotExist(err error) bool {
	fmt.Printf("IsNotExist? error: %v\n", err)
	// cloudfront returns forbidden error for nonexisting data
	if strings.Contains(err.Error(), "error code 403") {
		fmt.Printf("its a 403, dne")
		return true
	}
	if strings.Contains(err.Error(), "error code 404") {
		fmt.Printf("its a 404, dne")
		return true
	}
	awsErr, ok := err.(awserr.Error)
	if !ok {
		return false
	}
	if awsErr.Code() == "NoSuchKey" {
		return true
	}
	return false
}

type amazonWriter struct {
	errChan chan error
	pipe    *io.PipeWriter
}

func newWriter(client *amazonClient, name string) *amazonWriter {
	reader, writer := io.Pipe()
	w := &amazonWriter{
		errChan: make(chan error),
		pipe:    writer,
	}
	go func() {
		_, err := client.uploader.Upload(&s3manager.UploadInput{
			Body:            reader,
			Bucket:          aws.String(client.bucket),
			Key:             aws.String(name),
			ContentEncoding: aws.String("application/octet-stream"),
		})
		w.errChan <- err
	}()
	return w
}

func (w *amazonWriter) Write(p []byte) (int, error) {
	return w.pipe.Write(p)
}

func (w *amazonWriter) Close() error {
	if err := w.pipe.Close(); err != nil {
		return err
	}
	return <-w.errChan
}

func isRetryableGetError(err error) bool {
	if strings.Contains(err.Error(), "dial tcp: i/o timeout") {
		fmt.Printf("SAW A DIAL TCP TIMEOUT ERROR\n")
		return true
	}
	return isNetRetryable(err)
}
<commit_msg>Check cloudfront DNE error only when hooked into cloudfront<commit_after>package obj

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/aws/aws-sdk-go/service/storagegateway"
	"github.com/cenkalti/backoff"
)

type amazonClient struct {
	bucket       string
	distribution string
	s3           *s3.S3
	uploader     *s3manager.Uploader
}

func newAmazonClient(bucket string, distribution string, id string, secret string, token string, region string) (*amazonClient, error) {
	session := session.New(&aws.Config{
		Credentials: credentials.NewStaticCredentials(id, secret, token),
		Region:      aws.String(region),
	})
	return &amazonClient{
		bucket:   bucket,
		s3:       s3.New(session),
		uploader: s3manager.NewUploader(session),
	}, nil
}

func (c *amazonClient) usingCloudfront() bool {
	return c.distribution != ""
}

func (c *amazonClient) Writer(name string) (io.WriteCloser, error) {
	return newBackoffWriteCloser(c, newWriter(c, name)), nil
}

func (c *amazonClient) Walk(name string, fn func(name string) error) error {
	var fnErr error
	if err := c.s3.ListObjectsPages(
		&s3.ListObjectsInput{
			Bucket: aws.String(c.bucket),
			Prefix: aws.String(name),
		},
		func(listObjectsOutput *s3.ListObjectsOutput, lastPage bool) bool {
			for _, object := range listObjectsOutput.Contents {
				if err := fn(*object.Key); err != nil {
					fnErr = err
					return false
				}
			}
			return true
		},
	); err != nil {
		return err
	}
	return fnErr
}

func (c *amazonClient) Reader(name string, offset uint64, size uint64) (io.ReadCloser, error) {
	byteRange := byteRange(offset, size)
	if byteRange != "" {
		byteRange = fmt.Sprintf("bytes=%s", byteRange)
	}

	var reader io.ReadCloser
	if c.usingCloudfront {
		var resp *http.Response
		var connErr error
		url := fmt.Sprintf("http://%v.cloudfront.net/%v", c.distribution, name)

		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			return nil, err
		}
		req.Header.Add("Range", byteRange)

		backoff.RetryNotify(func() error {
			resp, connErr = http.DefaultClient.Do(req)
			if connErr != nil && isRetryableGetError(connErr) {
				fmt.Printf("this is a retryable error (%v)\n", connErr)
				return connErr
			}
			return nil
		}, backoff.NewExponentialBackOff(), func(err error, d time.Duration) {
			log.Infof("Error connecting to (%v); retrying in %s: %#v", url, d, err)
		})

		if resp.StatusCode != 200 {
			fmt.Printf("HTTP error code %v", resp.StatusCode)
			return nil, fmt.Errorf("cloudfront returned HTTP error code %v", resp.StatusCode)
		}
		reader = resp.Body
	} else {
		getObjectOutput, err := c.s3.GetObject(&s3.GetObjectInput{
			Bucket: aws.String(c.bucket),
			Key:    aws.String(name),
			Range:  aws.String(byteRange),
		})
		if err != nil {
			return nil, err
		}
		reader = getObjectOutput.Body
	}
	return newBackoffReadCloser(c, reader), nil
}

func (c *amazonClient) Delete(name string) error {
	_, err := c.s3.DeleteObject(&s3.DeleteObjectInput{
		Bucket: aws.String(c.bucket),
		Key:    aws.String(name),
	})
	return err
}

func (c *amazonClient) Exists(name string) bool {
	_, err := c.s3.HeadObject(&s3.HeadObjectInput{
		Bucket: aws.String(c.bucket),
		Key:    aws.String(name),
	})
	return err == nil
}

func (c *amazonClient) isRetryable(err error) (retVal bool) {
	fmt.Printf("is err (%v) retryable?\n", err)
	defer func() {
		fmt.Printf("err (%v) retryable? %v\n", retVal, err)
	}()
	if strings.Contains(err.Error(), "unexpected EOF") {
		return false
	}

	awsErr, ok := err.(awserr.Error)
	if !ok {
		return false
	}
	for _, c := range []string{
		storagegateway.ErrorCodeServiceUnavailable,
		storagegateway.ErrorCodeInternalError,
		storagegateway.ErrorCodeGatewayInternalError,
	} {
		if c == awsErr.Code() {
			return true
		}
	}
	return false
}

func (c *amazonClient) IsIgnorable(err error) bool {
	return false
}

func (c *amazonClient) IsNotExist(err error) bool {
	if c.usingCloudfront() {
		// cloudfront returns forbidden error for nonexisting data
		if strings.Contains(err.Error(), "error code 403") {
			return true
		}
	}
	awsErr, ok := err.(awserr.Error)
	if !ok {
		return false
	}
	if awsErr.Code() == "NoSuchKey" {
		return true
	}
	return false
}

type amazonWriter struct {
	errChan chan error
	pipe    *io.PipeWriter
}

func newWriter(client *amazonClient, name string) *amazonWriter {
	reader, writer := io.Pipe()
	w := &amazonWriter{
		errChan: make(chan error),
		pipe:    writer,
	}
	go func() {
		_, err := client.uploader.Upload(&s3manager.UploadInput{
			Body:            reader,
			Bucket:          aws.String(client.bucket),
			Key:             aws.String(name),
			ContentEncoding: aws.String("application/octet-stream"),
		})
		w.errChan <- err
	}()
	return w
}

func (w *amazonWriter) Write(p []byte) (int, error) {
	return w.pipe.Write(p)
}

func (w *amazonWriter) Close() error {
	if err := w.pipe.Close(); err != nil {
		return err
	}
	return <-w.errChan
}

func isRetryableGetError(err error) bool {
	if strings.Contains(err.Error(), "dial tcp: i/o timeout") {
		fmt.Printf("SAW A DIAL TCP TIMEOUT ERROR\n")
		return true
	}
	return isNetRetryable(err)
}
<|endoftext|>


<commit_before>// Copyright 2016 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package opengl

type Context struct {
	Nearest            Filter
	Linear             Filter
	VertexShader       ShaderType
	FragmentShader     ShaderType
	ArrayBuffer        BufferType
	ElementArrayBuffer BufferType
	DynamicDraw        BufferUsage
	StaticDraw         BufferUsage
	Triangles          Mode
	Lines              Mode
	zero               operation
	one                operation
	srcAlpha           operation
	dstAlpha           operation
	oneMinusSrcAlpha   operation
	oneMinusDstAlpha   operation
	locationCache      *locationCache
	screenFramebuffer  Framebuffer // This might not be the default frame buffer '0' (e.g. iOS).
	lastFramebuffer    Framebuffer
	lastViewportWidth  int
	lastViewportHeight int
	lastCompositeMode  CompositeMode
	context
}

func (c *Context) bindFramebuffer(f Framebuffer) error {
	if c.lastFramebuffer == f {
		return nil
	}
	if err := c.bindFramebufferImpl(f); err != nil {
		return err
	}
	c.lastFramebuffer = f
	return nil
}

func (c *Context) SetViewport(f Framebuffer, width, height int) error {
	lf := c.lastFramebuffer
	c.bindFramebuffer(f)
	if lf != f || c.lastViewportWidth != width || c.lastViewportHeight != height {
		if err := c.setViewportImpl(width, height); err != nil {
			return nil
		}
		c.lastViewportWidth = width
		c.lastViewportHeight = height
	}
	return nil
}

func (c *Context) ScreenFramebuffer() Framebuffer {
	return c.screenFramebuffer
}

func (c *Context) ResetViewportSize() {
	c.lastViewportWidth = 0
	c.lastViewportHeight = 0
}
<commit_msg>opengl: Remove unneeded glViewport calls<commit_after>// Copyright 2016 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package opengl

type Context struct {
	Nearest            Filter
	Linear             Filter
	VertexShader       ShaderType
	FragmentShader     ShaderType
	ArrayBuffer        BufferType
	ElementArrayBuffer BufferType
	DynamicDraw        BufferUsage
	StaticDraw         BufferUsage
	Triangles          Mode
	Lines              Mode
	zero               operation
	one                operation
	srcAlpha           operation
	dstAlpha           operation
	oneMinusSrcAlpha   operation
	oneMinusDstAlpha   operation
	locationCache      *locationCache
	screenFramebuffer  Framebuffer // This might not be the default frame buffer '0' (e.g. iOS).
	lastFramebuffer    Framebuffer
	lastViewportWidth  int
	lastViewportHeight int
	lastCompositeMode  CompositeMode
	context
}

func (c *Context) bindFramebuffer(f Framebuffer) error {
	if c.lastFramebuffer == f {
		return nil
	}
	if err := c.bindFramebufferImpl(f); err != nil {
		return err
	}
	c.lastFramebuffer = f
	return nil
}

func (c *Context) SetViewport(f Framebuffer, width, height int) error {
	c.bindFramebuffer(f)
	if c.lastViewportWidth != width || c.lastViewportHeight != height {
		if err := c.setViewportImpl(width, height); err != nil {
			return nil
		}
		c.lastViewportWidth = width
		c.lastViewportHeight = height
	}
	return nil
}

func (c *Context) ScreenFramebuffer() Framebuffer {
	return c.screenFramebuffer
}

func (c *Context) ResetViewportSize() {
	c.lastViewportWidth = 0
	c.lastViewportHeight = 0
}
<|endoftext|>


<commit_before>// Copyright ©2013 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mat64

import "gopkg.in/check.v1"

func (s *S) TestCholesky(c *check.C) {
	for _, t := range []struct {
		a     *SymDense
		upper bool
		f     *TriDense

		want *TriDense
		pd   bool
	}{
		{
			a: NewSymDense(3, []float64{
				4, 1, 1,
				0, 2, 3,
				0, 0, 6,
			}),
			upper: false,
			f:     &TriDense{},

			want: NewTriDense(3, false, []float64{
				2, 0, 0,
				0.5, 1.3228756555322954, 0,
				0.5, 2.0788046015507495, 1.195228609334394,
			}),
			pd: true,
		},
		{
			a: NewSymDense(3, []float64{
				4, 1, 1,
				0, 2, 3,
				0, 0, 6,
			}),
			upper: true,
			f:     &TriDense{},

			want: NewTriDense(3, true, []float64{
				2, 0.5, 0.5,
				0, 1.3228756555322954, 2.0788046015507495,
				0, 0, 1.195228609334394,
			}),
			pd: true,
		},
		{
			a: NewSymDense(3, []float64{
				4, 1, 1,
				0, 2, 3,
				0, 0, 6,
			}),
			upper: false,
			f:     NewTriDense(3, false, nil),

			want: NewTriDense(3, false, []float64{
				2, 0, 0,
				0.5, 1.3228756555322954, 0,
				0.5, 2.0788046015507495, 1.195228609334394,
			}),
			pd: true,
		},
		{
			a: NewSymDense(3, []float64{
				4, 1, 1,
				0, 2, 3,
				0, 0, 6,
			}),
			upper: true,
			f:     NewTriDense(3, false, nil),

			want: NewTriDense(3, true, []float64{
				2, 0.5, 0.5,
				0, 1.3228756555322954, 2.0788046015507495,
				0, 0, 1.195228609334394,
			}),
			pd: true,
		},
	} {
		ok := t.f.Cholesky(t.a, t.upper)
		c.Check(ok, check.Equals, t.pd)
		fc := DenseCopyOf(t.f)
		c.Check(fc.Equals(t.want), check.Equals, true)

		ft := &Dense{}
		ft.TCopy(t.f)

		if t.upper {
			fc.Mul(ft, fc)
		} else {
			fc.Mul(fc, ft)
		}
		c.Check(fc.EqualsApprox(t.a, 1e-12), check.Equals, true)

		var x Dense
		x.SolveCholesky(t.f, eye())

		var res Dense
		res.Mul(t.a, &x)
		c.Check(res.EqualsApprox(eye(), 1e-12), check.Equals, true)

		x = Dense{}
		x.SolveTri(t.f, t.upper, eye())
		x.SolveTri(t.f, !t.upper, &x)

		res.Mul(t.a, &x)
		c.Check(res.EqualsApprox(eye(), 1e-12), check.Equals, true)
	}
}

func (s *S) TestCholeskySolve(c *check.C) {
	for _, t := range []struct {
		a   *SymDense
		b   *Dense
		ans *Dense
	}{
		{
			a: NewSymDense(2, []float64{
				1, 0,
				0, 1,
			}),
			b:   NewDense(2, 1, []float64{5, 6}),
			ans: NewDense(2, 1, []float64{5, 6}),
		},
	} {
		var f TriDense
		ok := f.Cholesky(t.a, false)
		c.Assert(ok, check.Equals, true)

		var x Dense
		x.SolveCholesky(&f, t.b)
		c.Check(x.EqualsApprox(t.ans, 1e-12), check.Equals, true)

		x = Dense{}
		x.SolveTri(&f, false, t.b)
		x.SolveTri(&f, true, &x)
		c.Check(x.EqualsApprox(t.ans, 1e-12), check.Equals, true)
	}
}
<commit_msg>Add test case for issue #119<commit_after>// Copyright ©2013 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mat64

import (
	"math"

	"gopkg.in/check.v1"
)

func (s *S) TestCholesky(c *check.C) {
	for _, t := range []struct {
		a     *SymDense
		upper bool
		f     *TriDense

		want *TriDense
		pd   bool
	}{
		{
			a: NewSymDense(3, []float64{
				4, 1, 1,
				0, 2, 3,
				0, 0, 6,
			}),
			upper: false,
			f:     &TriDense{},

			want: NewTriDense(3, false, []float64{
				2, 0, 0,
				0.5, 1.3228756555322954, 0,
				0.5, 2.0788046015507495, 1.195228609334394,
			}),
			pd: true,
		},
		{
			a: NewSymDense(3, []float64{
				4, 1, 1,
				0, 2, 3,
				0, 0, 6,
			}),
			upper: true,
			f:     &TriDense{},

			want: NewTriDense(3, true, []float64{
				2, 0.5, 0.5,
				0, 1.3228756555322954, 2.0788046015507495,
				0, 0, 1.195228609334394,
			}),
			pd: true,
		},
		{
			a: NewSymDense(3, []float64{
				4, 1, 1,
				0, 2, 3,
				0, 0, 6,
			}),
			upper: false,
			f:     NewTriDense(3, false, nil),

			want: NewTriDense(3, false, []float64{
				2, 0, 0,
				0.5, 1.3228756555322954, 0,
				0.5, 2.0788046015507495, 1.195228609334394,
			}),
			pd: true,
		},
		{
			a: NewSymDense(3, []float64{
				4, 1, 1,
				0, 2, 3,
				0, 0, 6,
			}),
			upper: true,
			f:     NewTriDense(3, false, nil),

			want: NewTriDense(3, true, []float64{
				2, 0.5, 0.5,
				0, 1.3228756555322954, 2.0788046015507495,
				0, 0, 1.195228609334394,
			}),
			pd: true,
		},
		{
			// Test case for issue #119.
			a: NewSymDense(3, []float64{
				4, 1, 1,
				0, 2, 3,
				0, 0, 6,
			}),
			upper: false,
			f: NewTriDense(3, false, []float64{
				math.NaN(), math.NaN(), math.NaN(),
				math.NaN(), math.NaN(), math.NaN(),
				math.NaN(), math.NaN(), math.NaN(),
			}),

			want: NewTriDense(3, false, []float64{
				2, 0, 0,
				0.5, 1.3228756555322954, 0,
				0.5, 2.0788046015507495, 1.195228609334394,
			}),
			pd: true,
		},
	} {
		ok := t.f.Cholesky(t.a, t.upper)
		c.Check(ok, check.Equals, t.pd)
		fc := DenseCopyOf(t.f)
		c.Check(fc.Equals(t.want), check.Equals, true)

		ft := &Dense{}
		ft.TCopy(t.f)

		if t.upper {
			fc.Mul(ft, fc)
		} else {
			fc.Mul(fc, ft)
		}
		c.Check(fc.EqualsApprox(t.a, 1e-12), check.Equals, true)

		var x Dense
		x.SolveCholesky(t.f, eye())

		var res Dense
		res.Mul(t.a, &x)
		c.Check(res.EqualsApprox(eye(), 1e-12), check.Equals, true)

		x = Dense{}
		x.SolveTri(t.f, t.upper, eye())
		x.SolveTri(t.f, !t.upper, &x)

		res.Mul(t.a, &x)
		c.Check(res.EqualsApprox(eye(), 1e-12), check.Equals, true)
	}
}

func (s *S) TestCholeskySolve(c *check.C) {
	for _, t := range []struct {
		a   *SymDense
		b   *Dense
		ans *Dense
	}{
		{
			a: NewSymDense(2, []float64{
				1, 0,
				0, 1,
			}),
			b:   NewDense(2, 1, []float64{5, 6}),
			ans: NewDense(2, 1, []float64{5, 6}),
		},
	} {
		var f TriDense
		ok := f.Cholesky(t.a, false)
		c.Assert(ok, check.Equals, true)

		var x Dense
		x.SolveCholesky(&f, t.b)
		c.Check(x.EqualsApprox(t.ans, 1e-12), check.Equals, true)

		x = Dense{}
		x.SolveTri(&f, false, t.b)
		x.SolveTri(&f, true, &x)
		c.Check(x.EqualsApprox(t.ans, 1e-12), check.Equals, true)
	}
}
<|endoftext|>


<commit_before>package connector

import "strings"

func normalizeURL(url *string) {
	v := *url
	v = strings.TrimRight(v, "/")

	if !strings.HasPrefix(v, "http://") || !strings.HasPrefix(v, "https://") {
		v = "http://" + v
	}

	*url = v
}
<commit_msg>Fix URL normalization in connector package<commit_after>package connector

import "strings"

func normalizeURL(url *string) {
	v := *url
	v = strings.TrimRight(v, "/")

	if !strings.HasPrefix(v, "http://") && !strings.HasPrefix(v, "https://") {
		v = "http://" + v
	}

	*url = v
}
<|endoftext|>


<commit_before>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package adb provides routines for an adb compatible CLI client.
package adb

import (
	"context"
	"errors"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"

	empty_pb "github.com/golang/protobuf/ptypes/empty"
	"github.com/google/waterfall/golang/client"
	waterfall_grpc "github.com/google/waterfall/proto/waterfall_go_grpc"
	"google.golang.org/grpc"
)

const (
	androidADBEnv = "ANDROID_ADB"
	androidSDKEnv = "ANDROID_SDK_HOME"
)

type ParseError struct{}

// Error returns the empty string. We use this to fallback to regular ADB.
// This is only used to do a type assertion on the error.
func (e ParseError) Error() string {
	return ""
}

// ClientFn allows the user to custumize the way the client connection is established.
type ClientFn func() (*grpc.ClientConn, error)

// Parsed args represents a parsed command line.
type ParsedArgs struct {
	Device  string
	Command string
	Args    []string
}

type cmdFn func(context.Context, ClientFn, []string) error

var (
	Commands = map[string]cmdFn{
		"shell":     shellFn,
		"push":      pushFn,
		"pull":      pullFn,
		"forward":   forwardFn,
		"bugreport": passthroughFn,
		"logcat":    passthroughFn,
		"install":   installFn,
		"uninstall": uninstallFn,
	}
)

func platformADB() (string, error) {
	p := os.Getenv("ANDROID_ADB")
	if p != "" {
		return p, nil
	}

	p = os.Getenv("ANDROID_SDK_HOME")
	if p != "" {
		return filepath.Join(p, "platform-tools/adb"), nil
	}
	return "", errors.New("unable to find platform ADB, neither ANDROID_ADB or ANDROID_SDK_HOME set")
}

// Fallback forks a new adb process and executes the provided command.
// Note that this method is terminal. We transfer control to adb.
// It will either fatal out or call Exit.
func Fallback(args []string) {
	adbBin, err := platformADB()
	if err != nil {
		log.Fatal(err)
	}

	cmd := exec.Command(adbBin, args...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGTERM}

	if err := cmd.Start(); err != nil {
		log.Fatal(err)
	}

	if err := cmd.Wait(); err != nil {
		if exitErr, ok := err.(*exec.ExitError); ok {
			if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
				os.Exit(status.ExitStatus())
			}
		}
		log.Fatal(err)
	}
	os.Exit(0)
}

func exe(ctx context.Context, c waterfall_grpc.WaterfallClient, cmd string, args ...string) (int, error) {
	// Pipe from stdin only if we pipes are being used
	return client.Exec(ctx, c, os.Stdout, os.Stderr, nil, cmd, args...)
}

func shell(ctx context.Context, c waterfall_grpc.WaterfallClient, cmd string, args ...string) error {
	// Ignore return code here. This is intended as a drop in replacement for adb, so we need to copy the behavior.
	_, err := exe(ctx, c, "/system/bin/sh", "-c", fmt.Sprintf("%s %s", cmd, strings.Join(args, " ")))
	return err
}

func shellFn(ctx context.Context, cfn ClientFn, args []string) error {
	if len(args) == 1 {
		// Not an actual error, but user is requesting an interactive shell session.
		// Return this in order to fallback to adb.
		return ParseError{}
	}
	conn, err := cfn()
	if err != nil {
		return err
	}
	defer conn.Close()
	return shell(ctx, waterfall_grpc.NewWaterfallClient(conn), args[1], args[2:]...)
}

func pushFn(ctx context.Context, cfn ClientFn, args []string) error {
	if len(args) != 3 {
		return ParseError{}
	}

	conn, err := cfn()
	if err != nil {
		return err
	}
	defer conn.Close()

	return client.Push(ctx, waterfall_grpc.NewWaterfallClient(conn), args[1], args[2])
}

func pullFn(ctx context.Context, cfn ClientFn, args []string) error {
	if len(args) != 3 {
		return ParseError{}
	}

	conn, err := cfn()
	if err != nil {
		return err
	}
	defer conn.Close()

	return client.Pull(ctx, waterfall_grpc.NewWaterfallClient(conn), args[1], args[2])
}

func installFn(ctx context.Context, cfn ClientFn, args []string) error {
	if len(args) < 2 {
		return ParseError{}
	}

	conn, err := cfn()
	if err != nil {
		return err
	}
	defer conn.Close()

	c := waterfall_grpc.NewWaterfallClient(conn)
	path := args[len(args)-1]
	tmp := filepath.Join("/data/local/tmp")
	if err := client.Push(ctx, c, path, tmp); err != nil {
		return err
	}
	defer exe(ctx, c, "rm", "-f", filepath.Join(tmp, filepath.Base(path)))

	return shell(ctx, c, "/system/bin/pm", append(args[:len(args)-1], filepath.Join(tmp, filepath.Base(path)))...)
}

func parseFwd(addr string, reverse bool) (string, error) {
	if reverse {
		if strings.HasPrefix(addr, "tcp:") {
			pts := strings.SplitN(addr, ":", 3)
			return "tcp:" + pts[2], nil
		}
		if strings.HasPrefix(addr, "unix:@") {
			return "localabstract:" + addr[6:], nil
		}
		if strings.HasPrefix(addr, "unix:") {
			return "localreserved:" + addr[5:], nil
		}
		return "", ParseError{}
	}

	pts := strings.Split(addr, ":")
	if len(pts) != 2 {
		return "", ParseError{}
	}

	switch pts[0] {
	case "tcp":
		return "tcp:localhost:" + pts[1], nil
	case "localabstract":
		return "unix:@" + pts[1], nil
	case "localreserved":
		fallthrough
	case "localfilesystem":
		return "unix:" + pts[1], nil
	default:
		return "", ParseError{}
	}
}

func forwardFn(ctx context.Context, cfn ClientFn, args []string) error {
	if len(args) < 2 || len(args) > 4 {
		return ParseError{}
	}

	conn, err := cfn()
	if err != nil {
		return err
	}
	defer conn.Close()
	c := waterfall_grpc.NewPortForwarderClient(conn)

	var src, dst string
	switch args[1] {
	case "--list":
		ss, err := c.List(ctx, &empty_pb.Empty{})
		if err != nil {
			return err
		}
		for _, s := range ss.Sessions {
			src, err := parseFwd(s.Src, true)
			if err != nil {
				return err
			}

			dst, err := parseFwd(s.Dst, true)
			if err != nil {
				return err
			}
			fmt.Printf("localhost:foo %s %s\n", src, dst)
		}
		return nil
	case "--remove":
		if len(args) != 3 {
			return ParseError{}
		}
		fwd, err := parseFwd(args[2], false)
		if err != nil {
			return err
		}
		_, err = c.Stop(ctx, &waterfall_grpc.PortForwardRequest{
			Session: &waterfall_grpc.ForwardSession{Src: fwd}})
		return err
	case "--remove-all":
		if len(args) != 2 {
			return ParseError{}
		}
		_, err = c.StopAll(ctx, &empty_pb.Empty{})
		return err
	case "--no-rebind":
		if src, err = parseFwd(args[2], false); err != nil {
			return err
		}
		if dst, err = parseFwd(args[3], false); err != nil {
			return err
		}
		_, err = c.ForwardPort(ctx, &waterfall_grpc.PortForwardRequest{
			Session: &waterfall_grpc.ForwardSession{Src: src, Dst: dst}, Rebind: false})
		return err
	default:
		if src, err = parseFwd(args[1], false); err != nil {
			return err
		}
		if dst, err = parseFwd(args[2], false); err != nil {
			return err
		}
		_, err = c.ForwardPort(ctx, &waterfall_grpc.PortForwardRequest{
			Session: &waterfall_grpc.ForwardSession{Src: src, Dst: dst}, Rebind: true})
		return err
	}
}

func passthroughFn(ctx context.Context, cfn ClientFn, args []string) error {
	conn, err := cfn()
	if err != nil {
		return err
	}
	defer conn.Close()

	_, err = exe(ctx, waterfall_grpc.NewWaterfallClient(conn), args[0], args[1:]...)
	return err
}

func uninstallFn(ctx context.Context, cfn ClientFn, args []string) error {
	if len(args) != 2 {
		return ParseError{}
	}

	conn, err := cfn()
	if err != nil {
		return err
	}
	defer conn.Close()

	return shell(ctx, waterfall_grpc.NewWaterfallClient(conn), "/system/bin/pm", args...)
}

// ParseCommand parses the comand line args
func ParseCommand(args []string) (ParsedArgs, error) {
	var dev string

	// Process any global options first. Break and fallback for any unsupported option.
	i := 0
	done := false

	for !done && i < len(args) {
		arg := args[i]
		switch arg {
		case "server":
			fallthrough
		case "nodaemon":
			fallthrough
		case "persist":
			fallthrough
		case "-p":
			fallthrough
		case "-a":
			fallthrough
		case "-e":
			fallthrough
		case "-d":
			fallthrough
		case "-t":
			return ParsedArgs{}, ParseError{}
		case "-s":
			if len(args) == i+1 {
				return ParsedArgs{}, ParseError{}
			}
			dev = args[i+1]
			i++
		case "wait-for-device": // ignore
		// H, P and L have no meaning for H2O.
		case "-H":
			fallthrough
		case "-P":
			fallthrough
		case "-L":
			i++
		default:
			done = true
			continue
		}
		i++
	}
	return ParsedArgs{Device: dev, Command: args[i], Args: args[i:]}, nil
}
<commit_msg>Fixed godocs<commit_after>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package adb provides routines for an adb compatible CLI client.
package adb

import (
	"context"
	"errors"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"

	empty_pb "github.com/golang/protobuf/ptypes/empty"
	"github.com/google/waterfall/golang/client"
	waterfall_grpc "github.com/google/waterfall/proto/waterfall_go_grpc"
	"google.golang.org/grpc"
)

const (
	androidADBEnv = "ANDROID_ADB"
	androidSDKEnv = "ANDROID_SDK_HOME"
)

// ParseError represents an command line parsing error.
type ParseError struct{}

// Error returns the empty string. We use this to fallback to regular ADB.
// This is only used to do a type assertion on the error.
func (e ParseError) Error() string {
	return ""
}

// ClientFn allows the user to custumize the way the client connection is established.
type ClientFn func() (*grpc.ClientConn, error)

// ParsedArgs args represents a parsed command line.
type ParsedArgs struct {
	Device  string
	Command string
	Args    []string
}

type cmdFn func(context.Context, ClientFn, []string) error

var (
	// Commands maps from a command name to the function to execute the command.
	Commands = map[string]cmdFn{
		"shell":     shellFn,
		"push":      pushFn,
		"pull":      pullFn,
		"forward":   forwardFn,
		"bugreport": passthroughFn,
		"logcat":    passthroughFn,
		"install":   installFn,
		"uninstall": uninstallFn,
	}
)

func platformADB() (string, error) {
	p := os.Getenv("ANDROID_ADB")
	if p != "" {
		return p, nil
	}

	p = os.Getenv("ANDROID_SDK_HOME")
	if p != "" {
		return filepath.Join(p, "platform-tools/adb"), nil
	}
	return "", errors.New("unable to find platform ADB, neither ANDROID_ADB or ANDROID_SDK_HOME set")
}

// Fallback forks a new adb process and executes the provided command.
// Note that this method is terminal. We transfer control to adb.
// It will either fatal out or call Exit.
func Fallback(args []string) {
	adbBin, err := platformADB()
	if err != nil {
		log.Fatal(err)
	}

	cmd := exec.Command(adbBin, args...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGTERM}

	if err := cmd.Start(); err != nil {
		log.Fatal(err)
	}

	if err := cmd.Wait(); err != nil {
		if exitErr, ok := err.(*exec.ExitError); ok {
			if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
				os.Exit(status.ExitStatus())
			}
		}
		log.Fatal(err)
	}
	os.Exit(0)
}

func exe(ctx context.Context, c waterfall_grpc.WaterfallClient, cmd string, args ...string) (int, error) {
	// Pipe from stdin only if we pipes are being used
	return client.Exec(ctx, c, os.Stdout, os.Stderr, nil, cmd, args...)
}

func shell(ctx context.Context, c waterfall_grpc.WaterfallClient, cmd string, args ...string) error {
	// Ignore return code here. This is intended as a drop in replacement for adb, so we need to copy the behavior.
	_, err := exe(ctx, c, "/system/bin/sh", "-c", fmt.Sprintf("%s %s", cmd, strings.Join(args, " ")))
	return err
}

func shellFn(ctx context.Context, cfn ClientFn, args []string) error {
	if len(args) == 1 {
		// Not an actual error, but user is requesting an interactive shell session.
		// Return this in order to fallback to adb.
		return ParseError{}
	}
	conn, err := cfn()
	if err != nil {
		return err
	}
	defer conn.Close()
	return shell(ctx, waterfall_grpc.NewWaterfallClient(conn), args[1], args[2:]...)
}

func pushFn(ctx context.Context, cfn ClientFn, args []string) error {
	if len(args) != 3 {
		return ParseError{}
	}

	conn, err := cfn()
	if err != nil {
		return err
	}
	defer conn.Close()

	return client.Push(ctx, waterfall_grpc.NewWaterfallClient(conn), args[1], args[2])
}

func pullFn(ctx context.Context, cfn ClientFn, args []string) error {
	if len(args) != 3 {
		return ParseError{}
	}

	conn, err := cfn()
	if err != nil {
		return err
	}
	defer conn.Close()

	return client.Pull(ctx, waterfall_grpc.NewWaterfallClient(conn), args[1], args[2])
}

func installFn(ctx context.Context, cfn ClientFn, args []string) error {
	if len(args) < 2 {
		return ParseError{}
	}

	conn, err := cfn()
	if err != nil {
		return err
	}
	defer conn.Close()

	c := waterfall_grpc.NewWaterfallClient(conn)
	path := args[len(args)-1]
	tmp := filepath.Join("/data/local/tmp")
	if err := client.Push(ctx, c, path, tmp); err != nil {
		return err
	}
	defer exe(ctx, c, "rm", "-f", filepath.Join(tmp, filepath.Base(path)))

	return shell(ctx, c, "/system/bin/pm", append(args[:len(args)-1], filepath.Join(tmp, filepath.Base(path)))...)
}

func parseFwd(addr string, reverse bool) (string, error) {
	if reverse {
		if strings.HasPrefix(addr, "tcp:") {
			pts := strings.SplitN(addr, ":", 3)
			return "tcp:" + pts[2], nil
		}
		if strings.HasPrefix(addr, "unix:@") {
			return "localabstract:" + addr[6:], nil
		}
		if strings.HasPrefix(addr, "unix:") {
			return "localreserved:" + addr[5:], nil
		}
		return "", ParseError{}
	}

	pts := strings.Split(addr, ":")
	if len(pts) != 2 {
		return "", ParseError{}
	}

	switch pts[0] {
	case "tcp":
		return "tcp:localhost:" + pts[1], nil
	case "localabstract":
		return "unix:@" + pts[1], nil
	case "localreserved":
		fallthrough
	case "localfilesystem":
		return "unix:" + pts[1], nil
	default:
		return "", ParseError{}
	}
}

func forwardFn(ctx context.Context, cfn ClientFn, args []string) error {
	if len(args) < 2 || len(args) > 4 {
		return ParseError{}
	}

	conn, err := cfn()
	if err != nil {
		return err
	}
	defer conn.Close()
	c := waterfall_grpc.NewPortForwarderClient(conn)

	var src, dst string
	switch args[1] {
	case "--list":
		ss, err := c.List(ctx, &empty_pb.Empty{})
		if err != nil {
			return err
		}
		for _, s := range ss.Sessions {
			src, err := parseFwd(s.Src, true)
			if err != nil {
				return err
			}

			dst, err := parseFwd(s.Dst, true)
			if err != nil {
				return err
			}
			fmt.Printf("localhost:foo %s %s\n", src, dst)
		}
		return nil
	case "--remove":
		if len(args) != 3 {
			return ParseError{}
		}
		fwd, err := parseFwd(args[2], false)
		if err != nil {
			return err
		}
		_, err = c.Stop(ctx, &waterfall_grpc.PortForwardRequest{
			Session: &waterfall_grpc.ForwardSession{Src: fwd}})
		return err
	case "--remove-all":
		if len(args) != 2 {
			return ParseError{}
		}
		_, err = c.StopAll(ctx, &empty_pb.Empty{})
		return err
	case "--no-rebind":
		if src, err = parseFwd(args[2], false); err != nil {
			return err
		}
		if dst, err = parseFwd(args[3], false); err != nil {
			return err
		}
		_, err = c.ForwardPort(ctx, &waterfall_grpc.PortForwardRequest{
			Session: &waterfall_grpc.ForwardSession{Src: src, Dst: dst}, Rebind: false})
		return err
	default:
		if src, err = parseFwd(args[1], false); err != nil {
			return err
		}
		if dst, err = parseFwd(args[2], false); err != nil {
			return err
		}
		_, err = c.ForwardPort(ctx, &waterfall_grpc.PortForwardRequest{
			Session: &waterfall_grpc.ForwardSession{Src: src, Dst: dst}, Rebind: true})
		return err
	}
}

func passthroughFn(ctx context.Context, cfn ClientFn, args []string) error {
	conn, err := cfn()
	if err != nil {
		return err
	}
	defer conn.Close()

	_, err = exe(ctx, waterfall_grpc.NewWaterfallClient(conn), args[0], args[1:]...)
	return err
}

func uninstallFn(ctx context.Context, cfn ClientFn, args []string) error {
	if len(args) != 2 {
		return ParseError{}
	}

	conn, err := cfn()
	if err != nil {
		return err
	}
	defer conn.Close()

	return shell(ctx, waterfall_grpc.NewWaterfallClient(conn), "/system/bin/pm", args...)
}

// ParseCommand parses the comand line args
func ParseCommand(args []string) (ParsedArgs, error) {
	var dev string

	// Process any global options first. Break and fallback for any unsupported option.
	i := 0
	done := false

	for !done && i < len(args) {
		arg := args[i]
		switch arg {
		case "server":
			fallthrough
		case "nodaemon":
			fallthrough
		case "persist":
			fallthrough
		case "-p":
			fallthrough
		case "-a":
			fallthrough
		case "-e":
			fallthrough
		case "-d":
			fallthrough
		case "-t":
			return ParsedArgs{}, ParseError{}
		case "-s":
			if len(args) == i+1 {
				return ParsedArgs{}, ParseError{}
			}
			dev = args[i+1]
			i++
		case "wait-for-device": // ignore
		// H, P and L have no meaning for H2O.
		case "-H":
			fallthrough
		case "-P":
			fallthrough
		case "-L":
			i++
		default:
			done = true
			continue
		}
		i++
	}
	return ParsedArgs{Device: dev, Command: args[i], Args: args[i:]}, nil
}
<|endoftext|>


<commit_before>package filer2

import (
	"context"
	"fmt"
	"os"
	"time"

	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func (f *Filer) appendToFile(targetFile string, data []byte) error {

	assignResult, uploadResult, err2 := f.assignAndUpload(data)
	if err2 != nil {
		return err2
	}

	// find out existing entry
	fullpath := util.FullPath(targetFile)
	entry, err := f.FindEntry(context.Background(), fullpath)
	var offset int64 = 0
	if err == filer_pb.ErrNotFound {
		entry = &Entry{
			FullPath: fullpath,
			Attr: Attr{
				Crtime: time.Now(),
				Mtime:  time.Now(),
				Mode:   os.FileMode(0644),
				Uid:    OS_UID,
				Gid:    OS_GID,
			},
		}
	} else {
		offset = int64(TotalSize(entry.Chunks))
	}

	// append to existing chunks
	chunk := &filer_pb.FileChunk{
		FileId:    assignResult.Fid,
		Offset:    offset,
		Size:      uint64(uploadResult.Size),
		Mtime:     time.Now().UnixNano(),
		ETag:      uploadResult.ETag,
		IsGzipped: uploadResult.Gzip > 0,
	}
	entry.Chunks = append(entry.Chunks, chunk)

	// update the entry
	err = f.CreateEntry(context.Background(), entry, false)

	return err
}

func (f *Filer) assignAndUpload(data []byte) (*operation.AssignResult, *operation.UploadResult, error) {
	// assign a volume location
	assignRequest := &operation.VolumeAssignRequest{
		Count:               1,
		Collection:          f.metaLogCollection,
		Replication:         f.metaLogReplication,
		WritableVolumeCount: 1,
	}
	assignResult, err := operation.Assign(f.GetMaster(), f.GrpcDialOption, assignRequest)
	if err != nil {
		return nil, nil, fmt.Errorf("AssignVolume: %v", err)
	}
	if assignResult.Error != "" {
		return nil, nil, fmt.Errorf("AssignVolume error: %v", assignResult.Error)
	}

	// upload data
	targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
	uploadResult, err := operation.UploadData(targetUrl, "", false, data, false, "", nil, assignResult.Auth)
	if err != nil {
		return nil, nil, fmt.Errorf("upload data %s: %v", targetUrl, err)
	}
	// println("uploaded to", targetUrl)
	return assignResult, uploadResult, nil
}
<commit_msg>add cipher option to meta data updates<commit_after>package filer2

import (
	"context"
	"fmt"
	"os"
	"time"

	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func (f *Filer) appendToFile(targetFile string, data []byte) error {

	assignResult, uploadResult, err2 := f.assignAndUpload(data)
	if err2 != nil {
		return err2
	}

	// find out existing entry
	fullpath := util.FullPath(targetFile)
	entry, err := f.FindEntry(context.Background(), fullpath)
	var offset int64 = 0
	if err == filer_pb.ErrNotFound {
		entry = &Entry{
			FullPath: fullpath,
			Attr: Attr{
				Crtime: time.Now(),
				Mtime:  time.Now(),
				Mode:   os.FileMode(0644),
				Uid:    OS_UID,
				Gid:    OS_GID,
			},
		}
	} else {
		offset = int64(TotalSize(entry.Chunks))
	}

	// append to existing chunks
	chunk := &filer_pb.FileChunk{
		FileId:    assignResult.Fid,
		Offset:    offset,
		Size:      uint64(uploadResult.Size),
		Mtime:     time.Now().UnixNano(),
		ETag:      uploadResult.ETag,
		IsGzipped: uploadResult.Gzip > 0,
	}
	entry.Chunks = append(entry.Chunks, chunk)

	// update the entry
	err = f.CreateEntry(context.Background(), entry, false)

	return err
}

func (f *Filer) assignAndUpload(data []byte) (*operation.AssignResult, *operation.UploadResult, error) {
	// assign a volume location
	assignRequest := &operation.VolumeAssignRequest{
		Count:               1,
		Collection:          f.metaLogCollection,
		Replication:         f.metaLogReplication,
		WritableVolumeCount: 1,
	}
	assignResult, err := operation.Assign(f.GetMaster(), f.GrpcDialOption, assignRequest)
	if err != nil {
		return nil, nil, fmt.Errorf("AssignVolume: %v", err)
	}
	if assignResult.Error != "" {
		return nil, nil, fmt.Errorf("AssignVolume error: %v", assignResult.Error)
	}

	// upload data
	targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
	uploadResult, err := operation.UploadData(targetUrl, "", f.Cipher, data, false, "", nil, assignResult.Auth)
	if err != nil {
		return nil, nil, fmt.Errorf("upload data %s: %v", targetUrl, err)
	}
	// println("uploaded to", targetUrl)
	return assignResult, uploadResult, nil
}
<|endoftext|>


<commit_before>package storage

import (
	"testing"
	"os"
	"io/ioutil"
	"math/rand"
)

/*
makediff test steps
1. launch weed server at your local/dev environment, (option
"garbageThreshold" for master and option "max" for volume should be set with specific value which would let
preparing test prerequisite easier )
   a) ./weed master -garbageThreshold=0.99 -mdir=./m
   b) ./weed volume -dir=./data -max=1 -mserver=localhost:9333 -port=8080
2. upload 4 different files, you could call dir/assign to get 4 different fids
   a)  upload file A with fid a
   b)  upload file B with fid b
   c)  upload file C with fid c
   d)  upload file D with fid d
3. update file A and C
   a)  modify file A and upload file A with fid a
   b)  modify file C and upload file C with fid c
   c)  record the current 1.idx's file size(lastCompactIndexOffset value)
4. Compacting the data file
   a)  run curl http://localhost:8080/admin/vacuum/compact?volumeId=1
   b)  verify the 1.cpd and 1.cpx is created under volume directory
5. update file B and delete file D
   a)  modify file B and upload file B with fid b
   d)  delete file B with fid b
6. Now you could run the following UT case, the case should be run successfully
7. Compact commit manually
   a)  mv 1.cpd 1.dat
   b)  mv 1.cpx 1.idx
8. Restart Volume Server
9. Now you should get updated file A,B,C
*/

func TestMakeDiff(t *testing.T) {

	v := new(Volume)
	//lastCompactIndexOffset value is the index file size before step 4
	v.lastCompactIndexOffset = 96
	v.SuperBlock.version = 0x2
	/*
		err := v.makeupDiff(
			"/yourpath/1.cpd",
			"/yourpath/1.cpx",
			"/yourpath/1.dat",
			"/yourpath/1.idx")
		if err != nil {
			t.Errorf("makeupDiff err is %v", err)
		} else {
			t.Log("makeupDiff Succeeded")
		}
	*/
}

func TestCompaction(t *testing.T) {
	dir, err := ioutil.TempDir("", "example")
	if err != nil {
		t.Fatalf("temp dir creation: %v", err)
	}
	defer os.RemoveAll(dir) // clean up

	v, err := NewVolume(dir, "", 1, NeedleMapInMemory, &ReplicaPlacement{}, &TTL{}, 0)
	if err != nil {
		t.Fatalf("volume creation: %v", err)
	}

	beforeCommitFileCount := 10
	afterCommitFileCount := 10

	infos := make([]*needleInfo, beforeCommitFileCount+afterCommitFileCount)

	for i := 1; i <= beforeCommitFileCount; i++ {
		doSomeWritesDeletes(i, v, t, infos)
	}

	v.Compact(0)

	for i := 1; i <= afterCommitFileCount; i++ {
		doSomeWritesDeletes(i+beforeCommitFileCount, v, t, infos)
	}

	v.commitCompact()

	v.Close()

	v, err = NewVolume(dir, "", 1, NeedleMapInMemory, nil, nil, 0)
	if err != nil {
		t.Fatalf("volume reloading: %v", err)
	}

	for i := 1; i <= beforeCommitFileCount+afterCommitFileCount; i++ {

		if infos[i-1] == nil {
			t.Fatal("not found file", i)
			continue
		}

		if infos[i-1].size == 0 {
			continue
		}

		n := newEmptyNeedle(uint64(i))
		size, err := v.readNeedle(n)
		if err != nil {
			t.Fatalf("read file %d: %v", i, err)
		}
		if infos[i-1].size != uint32(size) {
			t.Fatalf("read file %d size mismatch expected %d found %d", i, infos[i-1].size, size)
		}
		if infos[i-1].crc != n.Checksum {
			t.Fatalf("read file %d checksum mismatch expected %d found %d", i, infos[i-1].crc, n.Checksum)
		}

	}

}
func doSomeWritesDeletes(i int, v *Volume, t *testing.T, infos []*needleInfo) {
	n := newRandomNeedle(uint64(i))
	size, err := v.writeNeedle(n)
	if err != nil {
		t.Fatalf("write file %d: %v", i, err)
	}
	infos[i-1] = &needleInfo{
		size: size,
		crc:  n.Checksum,
	}
	println("written file", i, "checksum", n.Checksum.Value(), "size", size)
	if rand.Float64() < 0.5 {
		toBeDeleted := rand.Intn(i) + 1
		oldNeedle := newEmptyNeedle(uint64(toBeDeleted))
		v.deleteNeedle(oldNeedle)
		println("deleted file", toBeDeleted)
		infos[toBeDeleted-1] = &needleInfo{
			size: 0,
			crc:  n.Checksum,
		}
	}
}

type needleInfo struct {
	size uint32
	crc  CRC
}

func newRandomNeedle(id uint64) *Needle {
	n := new(Needle)
	n.Data = make([]byte, rand.Intn(1024))
	rand.Read(n.Data)

	n.Checksum = NewCRC(n.Data)
	n.Id = id
	return n
}

func newEmptyNeedle(id uint64) *Needle {
	n := new(Needle)
	n.Id = id
	return n
}
<commit_msg>increase test size<commit_after>package storage

import (
	"testing"
	"os"
	"io/ioutil"
	"math/rand"
)

/*
makediff test steps
1. launch weed server at your local/dev environment, (option
"garbageThreshold" for master and option "max" for volume should be set with specific value which would let
preparing test prerequisite easier )
   a) ./weed master -garbageThreshold=0.99 -mdir=./m
   b) ./weed volume -dir=./data -max=1 -mserver=localhost:9333 -port=8080
2. upload 4 different files, you could call dir/assign to get 4 different fids
   a)  upload file A with fid a
   b)  upload file B with fid b
   c)  upload file C with fid c
   d)  upload file D with fid d
3. update file A and C
   a)  modify file A and upload file A with fid a
   b)  modify file C and upload file C with fid c
   c)  record the current 1.idx's file size(lastCompactIndexOffset value)
4. Compacting the data file
   a)  run curl http://localhost:8080/admin/vacuum/compact?volumeId=1
   b)  verify the 1.cpd and 1.cpx is created under volume directory
5. update file B and delete file D
   a)  modify file B and upload file B with fid b
   d)  delete file B with fid b
6. Now you could run the following UT case, the case should be run successfully
7. Compact commit manually
   a)  mv 1.cpd 1.dat
   b)  mv 1.cpx 1.idx
8. Restart Volume Server
9. Now you should get updated file A,B,C
*/

func TestMakeDiff(t *testing.T) {

	v := new(Volume)
	//lastCompactIndexOffset value is the index file size before step 4
	v.lastCompactIndexOffset = 96
	v.SuperBlock.version = 0x2
	/*
		err := v.makeupDiff(
			"/yourpath/1.cpd",
			"/yourpath/1.cpx",
			"/yourpath/1.dat",
			"/yourpath/1.idx")
		if err != nil {
			t.Errorf("makeupDiff err is %v", err)
		} else {
			t.Log("makeupDiff Succeeded")
		}
	*/
}

func TestCompaction(t *testing.T) {
	dir, err := ioutil.TempDir("", "example")
	if err != nil {
		t.Fatalf("temp dir creation: %v", err)
	}
	defer os.RemoveAll(dir) // clean up

	v, err := NewVolume(dir, "", 1, NeedleMapInMemory, &ReplicaPlacement{}, &TTL{}, 0)
	if err != nil {
		t.Fatalf("volume creation: %v", err)
	}

	beforeCommitFileCount := 1000
	afterCommitFileCount := 1000

	infos := make([]*needleInfo, beforeCommitFileCount+afterCommitFileCount)

	for i := 1; i <= beforeCommitFileCount; i++ {
		doSomeWritesDeletes(i, v, t, infos)
	}

	v.Compact(0)

	for i := 1; i <= afterCommitFileCount; i++ {
		doSomeWritesDeletes(i+beforeCommitFileCount, v, t, infos)
	}

	v.commitCompact()

	v.Close()

	v, err = NewVolume(dir, "", 1, NeedleMapInMemory, nil, nil, 0)
	if err != nil {
		t.Fatalf("volume reloading: %v", err)
	}

	for i := 1; i <= beforeCommitFileCount+afterCommitFileCount; i++ {

		if infos[i-1] == nil {
			t.Fatal("not found file", i)
			continue
		}

		if infos[i-1].size == 0 {
			continue
		}

		n := newEmptyNeedle(uint64(i))
		size, err := v.readNeedle(n)
		if err != nil {
			t.Fatalf("read file %d: %v", i, err)
		}
		if infos[i-1].size != uint32(size) {
			t.Fatalf("read file %d size mismatch expected %d found %d", i, infos[i-1].size, size)
		}
		if infos[i-1].crc != n.Checksum {
			t.Fatalf("read file %d checksum mismatch expected %d found %d", i, infos[i-1].crc, n.Checksum)
		}

	}

}
func doSomeWritesDeletes(i int, v *Volume, t *testing.T, infos []*needleInfo) {
	n := newRandomNeedle(uint64(i))
	size, err := v.writeNeedle(n)
	if err != nil {
		t.Fatalf("write file %d: %v", i, err)
	}
	infos[i-1] = &needleInfo{
		size: size,
		crc:  n.Checksum,
	}
	println("written file", i, "checksum", n.Checksum.Value(), "size", size)
	if rand.Float64() < 0.5 {
		toBeDeleted := rand.Intn(i) + 1
		oldNeedle := newEmptyNeedle(uint64(toBeDeleted))
		v.deleteNeedle(oldNeedle)
		println("deleted file", toBeDeleted)
		infos[toBeDeleted-1] = &needleInfo{
			size: 0,
			crc:  n.Checksum,
		}
	}
}

type needleInfo struct {
	size uint32
	crc  CRC
}

func newRandomNeedle(id uint64) *Needle {
	n := new(Needle)
	n.Data = make([]byte, rand.Intn(1024))
	rand.Read(n.Data)

	n.Checksum = NewCRC(n.Data)
	n.Id = id
	return n
}

func newEmptyNeedle(id uint64) *Needle {
	n := new(Needle)
	n.Id = id
	return n
}
<|endoftext|>