<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"text/template"

	"github.com/cosiner/gohper/defval"
	"github.com/cosiner/gohper/errors"
	"github.com/cosiner/gohper/goutil"
	"github.com/cosiner/gohper/goutil/ast"
	"github.com/cosiner/gohper/os2/file"
	"github.com/cosiner/gohper/os2/path2"
	"github.com/cosiner/gohper/strings2"
)

var (
	infile       string
	outfile      string
	models       string
	tmplfile     string
	copyTmpl     bool
	useCamelCase bool
)

func cliArgs() {
	flag.StringVar(&infile, "i", "", "input file")
	flag.StringVar(&outfile, "o", "", "output file")
	flag.StringVar(&models, "m", "", "models to parse, seperate by comma")
	flag.StringVar(&tmplfile, "t", "", "template file, first find in current directory, else use default file")

	// make it true to enable default CamelCase
	flag.BoolVar(&useCamelCase, "cc", false, "use CamelCase of constants")
	flag.BoolVar(&copyTmpl, "cp", false, "copy tmpl file to default path")
	flag.Parse()
}

const TmplName = "model.tmpl"

// change this if need
var defTmplPath = filepath.Join(path2.Home(), ".config", "go", TmplName)

func main() {
	cliArgs()
	if copyTmpl {
		errors.Fatal(file.Copy(defTmplPath, TmplName))
		return
	}
	errors.CondDo(infile == "", errors.Err("No input file specified."), errors.FatalAnyln)

	mv := new(modelVisitor)
	mv.addModelNeedParse(strings2.TrimSplit(models, ","))

	errors.Fatalln(mv.parse(infile))

	if len(mv.models) == 0 {
		return
	}

	defval.String(&outfile, infile)

	if tmplfile == "" {
		tmplfile = TmplName
		if !file.IsExist(tmplfile) {
			tmplfile = defTmplPath
		}
	}
	errors.Fatal(
		file.OpenOrCreate(outfile, false, func(fd *os.File) error {
			modelFields := buildModelFields(mv)
			t, err := template.ParseFiles(tmplfile)
			if err == nil {
				err = t.Execute(fd, modelFields)
			}
			return err
		}),
	)
}

type StructName struct {
	Name           string // struct's normal name
	Self           string
	UnexportedName string
	UpperName      string // upper case name
	TableName      string
}

type FieldName struct {
	Name       string // field's normal name
	ColumnName string // field's column name, in snake_case
	ConstName  string // field's const name is in STRUCTNAME_FIELDNAME case
}

func NewFieldName(model *StructName, field string) *FieldName {
	f := &FieldName{
		Name: field,
	}
	if useCamelCase {
		f.ConstName = model.Name + field
	} else {
		f.ConstName = model.UpperName + "_" + strings.ToUpper(field)
	}
	return f
}

func NewStructName(name, table string) *StructName {
	s := &StructName{
		Name:           name,
		Self:           strings2.ToLowerAbridge(name),
		UnexportedName: goutil.ToUnexported(name),
		UpperName:      strings.ToUpper(name),
	}
	if table != "" {
		s.TableName = table
	} else {
		s.TableName = strings.ToLower(strings2.ToCamel(name))
	}
	return s
}

// buildModelFields build model map from parse result
func buildModelFields(mv *modelVisitor) map[*StructName][]*FieldName {
	names := make(map[*StructName][]*FieldName, len(models))
	for model, fields := range mv.models {
		modelStruct := NewStructName(model, mv.tables[model])
		for _, name := range fields {
			names[modelStruct] = append(names[modelStruct], NewFieldName(modelStruct, name))
		}
	}
	return names
}

type modelVisitor struct {
	models      map[string][]string
	tables      map[string]string
	modelsParse map[string]bool
}

func (mv *modelVisitor) addModelNeedParse(models []string) {
	mv.modelsParse = make(map[string]bool)
	for _, m := range models {
		if m != "" {
			mv.modelsParse[m] = true
		}
	}
}

// add add an model and it's field to parse result
func (mv *modelVisitor) add(model, table, field string) {
	if mv.models == nil {
		mv.models = make(map[string][]string, 10)
		mv.tables = make(map[string]string)
	}
	if table != "" {
		mv.tables[model] = table
	}
	mv.models[model] = append(mv.models[model], field)
}

// needParse check whether a model should be parsed
// unexporeted model don't parse
// if visitor's model list is not empty, only parse model exist in list
// otherwise parse all
func (mv *modelVisitor) needParse(model string) bool {
	return goutil.IsExported(model) && (len(mv.modelsParse) == 0 || mv.modelsParse[model])
}

// parse ast tree to find exported struct and it's fields
func (mv *modelVisitor) parse(file string) error {
	call := ast.Callback{
		Struct: func(a *ast.Attrs) error {
			needParse := mv.needParse(a.TypeName)
			fmt.Println(a.TypeName, needParse)
			if !needParse {
				return ast.TYPE_END
			}
			return nil
		},

		StructField: func(a *ast.Attrs) error {
			table := a.S.Tag.Get("table")
			if table == "-" {
				return ast.TYPE_END
			}
			if a.S.Tag.Get("column") != "-" {
				mv.add(a.TypeName, table, a.S.Field)
			}
			return nil
		},
	}
	return ast.ParseFile(file, call)
}
<commit_msg>update<commit_after>package main

import (
	"flag"
	"os"
	"path/filepath"
	"strings"
	"text/template"

	"github.com/cosiner/gohper/defval"
	"github.com/cosiner/gohper/errors"
	"github.com/cosiner/gohper/goutil"
	"github.com/cosiner/gohper/goutil/ast"
	"github.com/cosiner/gohper/os2/file"
	"github.com/cosiner/gohper/os2/path2"
	"github.com/cosiner/gohper/strings2"
)

var (
	infile       string
	outfile      string
	models       string
	tmplfile     string
	copyTmpl     bool
	useCamelCase bool
)

func cliArgs() {
	flag.StringVar(&infile, "i", "", "input file")
	flag.StringVar(&outfile, "o", "", "output file")
	flag.StringVar(&models, "m", "", "models to parse, seperate by comma")
	flag.StringVar(&tmplfile, "t", "", "template file, first find in current directory, else use default file")

	// make it true to enable default CamelCase
	flag.BoolVar(&useCamelCase, "cc", false, "use CamelCase of constants")
	flag.BoolVar(&copyTmpl, "cp", false, "copy tmpl file to default path")
	flag.Parse()
}

const TmplName = "model.tmpl"

// change this if need
var defTmplPath = filepath.Join(path2.Home(), ".config", "go", TmplName)

func main() {
	cliArgs()
	if copyTmpl {
		errors.Fatal(file.Copy(defTmplPath, TmplName))
		return
	}
	errors.CondDo(infile == "", errors.Err("No input file specified."), errors.FatalAnyln)

	mv := new(modelVisitor)
	mv.addModelNeedParse(strings2.TrimSplit(models, ","))

	errors.Fatalln(mv.parse(infile))

	if len(mv.models) == 0 {
		return
	}

	defval.String(&outfile, infile)

	if tmplfile == "" {
		tmplfile = TmplName
		if !file.IsExist(tmplfile) {
			tmplfile = defTmplPath
		}
	}
	errors.Fatal(
		file.OpenOrCreate(outfile, false, func(fd *os.File) error {
			modelFields := buildModelFields(mv)
			t, err := template.ParseFiles(tmplfile)
			if err == nil {
				err = t.Execute(fd, modelFields)
			}
			return err
		}),
	)
}

type StructName struct {
	Name           string // struct's normal name
	Self           string
	UnexportedName string
	UpperName      string // upper case name
	TableName      string
}

type FieldName struct {
	Name       string // field's normal name
	ColumnName string // field's column name, in snake_case
	ConstName  string // field's const name is in STRUCTNAME_FIELDNAME case
}

func NewFieldName(model *StructName, field string) *FieldName {
	f := &FieldName{
		Name: field,
	}
	if useCamelCase {
		f.ConstName = model.Name + field
	} else {
		f.ConstName = model.UpperName + "_" + strings.ToUpper(field)
	}
	return f
}

func NewStructName(name, table string) *StructName {
	s := &StructName{
		Name:           name,
		Self:           strings2.ToLowerAbridge(name),
		UnexportedName: goutil.ToUnexported(name),
		UpperName:      strings.ToUpper(name),
	}
	if table != "" {
		s.TableName = table
	} else {
		s.TableName = strings.ToLower(strings2.ToCamel(name))
	}
	return s
}

// buildModelFields build model map from parse result
func buildModelFields(mv *modelVisitor) map[*StructName][]*FieldName {
	names := make(map[*StructName][]*FieldName, len(models))
	for model, fields := range mv.models {
		modelStruct := NewStructName(model, mv.tables[model])
		for _, name := range fields {
			names[modelStruct] = append(names[modelStruct], NewFieldName(modelStruct, name))
		}
	}
	return names
}

type modelVisitor struct {
	models      map[string][]string
	tables      map[string]string
	modelsParse map[string]bool
}

func (mv *modelVisitor) addModelNeedParse(models []string) {
	mv.modelsParse = make(map[string]bool)
	for _, m := range models {
		if m != "" {
			mv.modelsParse[m] = true
		}
	}
}

// add add an model and it's field to parse result
func (mv *modelVisitor) add(model, table, field string) {
	if mv.models == nil {
		mv.models = make(map[string][]string, 10)
		mv.tables = make(map[string]string)
	}
	if table != "" {
		mv.tables[model] = table
	}
	mv.models[model] = append(mv.models[model], field)
}

// needParse check whether a model should be parsed
// unexporeted model don't parse
// if visitor's model list is not empty, only parse model exist in list
// otherwise parse all
func (mv *modelVisitor) needParse(model string) bool {
	return goutil.IsExported(model) && (len(mv.modelsParse) == 0 || mv.modelsParse[model])
}

// parse ast tree to find exported struct and it's fields
func (mv *modelVisitor) parse(file string) error {
	call := ast.Callback{
		Struct: func(a *ast.Attrs) (err error) {
			if !mv.needParse(a.TypeName) {
				err = ast.TYPE_END
			} else if table := a.S.Tag.Get("table"); table == "-" {
				err = ast.TYPE_END
			} else if a.S.Tag.Get("column") != "-" {
				mv.add(a.TypeName, table, a.S.Field)
			}
			return
		},
	}
	return ast.ParseFile(file, call)
}
<|endoftext|>


<commit_before>package slinga

import (
	"bufio"
	"github.com/golang/glog"
	"os/exec"
)

func runCmd(cmdName string, cmdArgs ...string) error {
	cmd := exec.Command(cmdName, cmdArgs...)
	glog.Infof("Running command '%s' with args: %s", cmdName, cmdArgs)

	cmdStdoutReader, err := cmd.StdoutPipe()
	if err != nil {
		glog.Errorf("Failed running command '%s' (with args: %s): %s", cmdName, cmdArgs, err)
		return err
	}

	cmdStdoutScanner := bufio.NewScanner(cmdStdoutReader)
	go func() {
		for cmdStdoutScanner.Scan() {
			glog.Infof("%s out | %s\n", cmdName, cmdStdoutScanner.Text())
		}
	}()

	cmdStderrReader, err := cmd.StderrPipe()
	if err != nil {
		glog.Errorf("Failed running command '%s' (with args: %s): %s", cmdName, cmdArgs, err)
		return err
	}

	cmdStderrScanner := bufio.NewScanner(cmdStderrReader)
	go func() {
		for cmdStderrScanner.Scan() {
			glog.Infof("%s err | %s\n", cmdName, cmdStderrScanner.Text())
		}
	}()

	err = cmd.Start()
	if err != nil {
		glog.Errorf("Failed running command '%s' (with args: %s): %s", cmdName, cmdArgs, err)
		return err
	}

	err = cmd.Wait()
	if err != nil {
		glog.Errorf("Failed running command '%s' (with args: %s): %s", cmdName, cmdArgs, err)
		return err
	}

	return nil
}
<commit_msg>Simplify and fix logging for fork.go<commit_after>package slinga

import (
	"os/exec"
	"fmt"
)

func runCmd(cmdName string, cmdArgs ...string) (string, error) {
	out, err := exec.Command(cmdName, cmdArgs...).Output()
	if err != nil {
		if exitErr, ok := err.(*exec.ExitError); ok {
			return "", fmt.Errorf("Exit err (%s): %s", exitErr.Error(), exitErr.Stderr)
		}
		return "", err
	}

	return string(out), nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"launchpad.net/gnuflag"
	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/juju"
	"launchpad.net/juju-core/state"
)

// GetConstraintsCommand shows the constraints for a service or environment.
type GetConstraintsCommand struct {
	EnvName     string
	ServiceName string
	out         cmd.Output
}

func (c *GetConstraintsCommand) Info() *cmd.Info {
	return &cmd.Info{"get-constraints", "", "view constraints", ""}
}

func formatConstraints(value interface{}) ([]byte, error) {
	return []byte(value.(state.Constraints).String()), nil
}

func (c *GetConstraintsCommand) SetFlags(f *gnuflag.FlagSet) {
	addEnvironFlags(&c.EnvName, f)
	c.out.AddFlags(f, "constraints", map[string]cmd.Formatter{
		"constraints": formatConstraints,
		"yaml":        cmd.FormatYaml,
		"json":        cmd.FormatJson,
	})
}

func (c *GetConstraintsCommand) Init(args []string) error {
	if len(args) > 0 {
		if !state.IsServiceName(args[0]) {
			return fmt.Errorf("invalid service name %q", args[0])
		}
		c.ServiceName, args = args[0], args[1:]
	}
	return cmd.CheckEmpty(args)
}

func (c *GetConstraintsCommand) Run(ctx *cmd.Context) (err error) {
	conn, err := juju.NewConnFromName(c.EnvName)
	if err != nil {
		return err
	}
	defer conn.Close()
	var cons state.Constraints
	if c.ServiceName == "" {
		cons, err = conn.State.EnvironConstraints()
	} else {
		var svc *state.Service
		if svc, err = conn.State.Service(c.ServiceName); err != nil {
			return err
		}
		cons, err = svc.Constraints()
	}
	if err != nil {
		return err
	}
	return c.out.Write(ctx, cons)
}

// SetConstraintsCommand shows the constraints for a service or environment.
type SetConstraintsCommand struct {
	EnvName     string
	ServiceName string
	Constraints state.Constraints
}

func (c *SetConstraintsCommand) Info() *cmd.Info {
	return &cmd.Info{"set-constraints", "[key=[value],...]", "replace constraints", ""}
}

func (c *SetConstraintsCommand) SetFlags(f *gnuflag.FlagSet) {
	addEnvironFlags(&c.EnvName, f)
	f.StringVar(&c.ServiceName, "s", "", "set service constraints")
	f.StringVar(&c.ServiceName, "service", "", "")
}

func (c *SetConstraintsCommand) Init(args []string) (err error) {
	if c.ServiceName != "" && !state.IsServiceName(c.ServiceName) {
		return fmt.Errorf("invalid service name %q", c.ServiceName)
	}
	c.Constraints, err = state.ParseConstraints(args...)
	return err
}

func (c *SetConstraintsCommand) Run(_ *cmd.Context) (err error) {
	conn, err := juju.NewConnFromName(c.EnvName)
	if err != nil {
		return err
	}
	defer conn.Close()
	if c.ServiceName == "" {
		return conn.State.SetEnvironConstraints(c.Constraints)
	}
	var svc *state.Service
	if svc, err = conn.State.Service(c.ServiceName); err != nil {
		return err
	}
	return svc.SetConstraints(c.Constraints)
}
<commit_msg>Fix up the Info construction for get and set constraints.<commit_after>package main

import (
	"fmt"
	"launchpad.net/gnuflag"
	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/juju"
	"launchpad.net/juju-core/state"
)

// GetConstraintsCommand shows the constraints for a service or environment.
type GetConstraintsCommand struct {
	EnvName     string
	ServiceName string
	out         cmd.Output
}

func (c *GetConstraintsCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "get-constraints",
		Args:    "[<service> [<constraints>...]",
		Purpose: "view constraints",
	}
}

func formatConstraints(value interface{}) ([]byte, error) {
	return []byte(value.(state.Constraints).String()), nil
}

func (c *GetConstraintsCommand) SetFlags(f *gnuflag.FlagSet) {
	addEnvironFlags(&c.EnvName, f)
	c.out.AddFlags(f, "constraints", map[string]cmd.Formatter{
		"constraints": formatConstraints,
		"yaml":        cmd.FormatYaml,
		"json":        cmd.FormatJson,
	})
}

func (c *GetConstraintsCommand) Init(args []string) error {
	if len(args) > 0 {
		if !state.IsServiceName(args[0]) {
			return fmt.Errorf("invalid service name %q", args[0])
		}
		c.ServiceName, args = args[0], args[1:]
	}
	return cmd.CheckEmpty(args)
}

func (c *GetConstraintsCommand) Run(ctx *cmd.Context) (err error) {
	conn, err := juju.NewConnFromName(c.EnvName)
	if err != nil {
		return err
	}
	defer conn.Close()
	var cons state.Constraints
	if c.ServiceName == "" {
		cons, err = conn.State.EnvironConstraints()
	} else {
		var svc *state.Service
		if svc, err = conn.State.Service(c.ServiceName); err != nil {
			return err
		}
		cons, err = svc.Constraints()
	}
	if err != nil {
		return err
	}
	return c.out.Write(ctx, cons)
}

// SetConstraintsCommand shows the constraints for a service or environment.
type SetConstraintsCommand struct {
	EnvName     string
	ServiceName string
	Constraints state.Constraints
}

func (c *SetConstraintsCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "set-constraints",
		Args:    "[key=[value],...]",
		Purpose: "replace constraints",
	}
}

func (c *SetConstraintsCommand) SetFlags(f *gnuflag.FlagSet) {
	addEnvironFlags(&c.EnvName, f)
	f.StringVar(&c.ServiceName, "s", "", "set service constraints")
	f.StringVar(&c.ServiceName, "service", "", "")
}

func (c *SetConstraintsCommand) Init(args []string) (err error) {
	if c.ServiceName != "" && !state.IsServiceName(c.ServiceName) {
		return fmt.Errorf("invalid service name %q", c.ServiceName)
	}
	c.Constraints, err = state.ParseConstraints(args...)
	return err
}

func (c *SetConstraintsCommand) Run(_ *cmd.Context) (err error) {
	conn, err := juju.NewConnFromName(c.EnvName)
	if err != nil {
		return err
	}
	defer conn.Close()
	if c.ServiceName == "" {
		return conn.State.SetEnvironConstraints(c.Constraints)
	}
	var svc *state.Service
	if svc, err = conn.State.Service(c.ServiceName); err != nil {
		return err
	}
	return svc.SetConstraints(c.Constraints)
}
<|endoftext|>


<commit_before>package store

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"

	"github.com/djherbis/buffer"
	"github.com/djherbis/nio"
	"github.com/pkg/errors"

	"github.com/oklog/prototype/pkg/fs"
	"github.com/oklog/ulid"
)

// QueryResult contains statistics about, and matching records for, a query.
type QueryResult struct {
	From     string `json:"from"`
	To       string `json:"to"`
	Q        string `json:"q"`
	Regex    bool   `json:"regex"`
	Duration string `json:"duration"`

	NodesQueried    int   `json:"nodes_queried"`
	SegmentsQueried int   `json:"segments_queried"`
	MaxDataSetSize  int64 `json:"max_data_set_size"`
	ErrorCount      int   `json:"error_count,omitempty"`

	Records io.ReadCloser // TODO(pb): audit to ensure closing is valid throughout
}

// EncodeTo encodes the QueryResult to the HTTP response writer.
// It also closes the records ReadCloser.
func (qr *QueryResult) EncodeTo(w http.ResponseWriter) {
	w.Header().Set(httpHeaderFrom, qr.From)
	w.Header().Set(httpHeaderTo, qr.To)
	w.Header().Set(httpHeaderQ, qr.Q)
	w.Header().Set(httpHeaderRegex, fmt.Sprint(qr.Regex))

	w.Header().Set(httpHeaderNodesQueried, strconv.Itoa(qr.NodesQueried))
	w.Header().Set(httpHeaderSegmentsQueried, strconv.Itoa(qr.SegmentsQueried))
	w.Header().Set(httpHeaderMaxDataSetSize, strconv.FormatInt(qr.MaxDataSetSize, 10))
	w.Header().Set(httpHeaderErrorCount, strconv.Itoa(qr.ErrorCount))
	w.Header().Set(httpHeaderDuration, qr.Duration)

	if qr.Records != nil {
		// CopyBuffer can be useful for complex query pipelines.
		// TODO(pb): validate the 1MB buffer size with profiling
		buf := make([]byte, 1024*1024)
		io.CopyBuffer(w, qr.Records, buf)
		qr.Records.Close()
	}
}

// DecodeFrom decodes the QueryResult from the HTTP response.
func (qr *QueryResult) DecodeFrom(resp *http.Response) error {
	qr.From = resp.Header.Get(httpHeaderFrom)
	qr.To = resp.Header.Get(httpHeaderTo)
	qr.Q = resp.Header.Get(httpHeaderQ)
	var err error
	if qr.Regex, err = strconv.ParseBool(resp.Header.Get(httpHeaderRegex)); err != nil {
		return errors.Wrap(err, "regex")
	}
	if qr.NodesQueried, err = strconv.Atoi(resp.Header.Get(httpHeaderNodesQueried)); err != nil {
		return errors.Wrap(err, "nodes queried")
	}
	if qr.SegmentsQueried, err = strconv.Atoi(resp.Header.Get(httpHeaderSegmentsQueried)); err != nil {
		return errors.Wrap(err, "segments queried")
	}
	if qr.MaxDataSetSize, err = strconv.ParseInt(resp.Header.Get(httpHeaderMaxDataSetSize), 10, 64); err != nil {
		return errors.Wrap(err, "max data set size")
	}
	if qr.ErrorCount, err = strconv.Atoi(resp.Header.Get(httpHeaderErrorCount)); err != nil {
		return errors.Wrap(err, "error count")
	}
	qr.Duration = resp.Header.Get(httpHeaderDuration)
	qr.Records = resp.Body
	return nil
}

// Merge the other QueryResult into this one.
func (qr *QueryResult) Merge(other QueryResult) error {
	// Union the simple integer types.
	qr.NodesQueried += other.NodesQueried
	qr.SegmentsQueried += other.SegmentsQueried
	if other.MaxDataSetSize > qr.MaxDataSetSize {
		qr.MaxDataSetSize = other.MaxDataSetSize
	}
	qr.ErrorCount += other.ErrorCount

	// Merge the record readers.
	// Both mergeRecords and multiCloser can handle nils.
	var buf bytes.Buffer
	_, _, _, err := mergeRecords(&buf, qr.Records, other.Records)
	multiCloser{qr.Records, other.Records}.Close()
	qr.Records = ioutil.NopCloser(&buf)

	// Done.
	return err
}

const (
	httpHeaderFrom            = "X-Oklog-From"
	httpHeaderTo              = "X-Oklog-To"
	httpHeaderQ               = "X-Oklog-Q"
	httpHeaderRegex           = "X-Oklog-Regex"
	httpHeaderNodesQueried    = "X-Oklog-Nodes-Queried"
	httpHeaderSegmentsQueried = "X-Oklog-Segments-Queried"
	httpHeaderMaxDataSetSize  = "X-Oklog-Max-Data-Set-Size"
	httpHeaderErrorCount      = "X-Oklog-Error-Count"
	httpHeaderDuration        = "X-Oklog-Duration"
)

type recordFilter func([]byte) bool

// newQueryReadCloser converts a batch of segments to a single io.ReadCloser.
// Records are yielded in time order, oldest first, hopefully efficiently!
// Only records passing the recordFilter are yielded.
// The sz of the segment files can be used as a proxy for read effort.
func newQueryReadCloser(fs fs.Filesystem, segments []string, pass recordFilter, bufsz int64) (rc io.ReadCloser, sz int64, err error) {
	// We will build successive ReadClosers for each batch.
	var rcs []io.ReadCloser

	// Don't leak FDs on error.
	defer func() {
		if err != nil {
			for _, rc := range rcs {
				rc.Close()
			}
		}
	}()

	// Batch the segments, and construct a ReadCloser for each batch.
	for _, batch := range batchSegments(segments) {
		switch len(batch) {
		case 0:
			continue // weird

		case 1:
			// A batch of one can be read straight thru.
			f, err := fs.Open(batch[0])
			if err != nil {
				return nil, sz, err // TODO(pb): don't leak FDs
			}
			rcs = append(rcs, newConcurrentFilteringReadCloser(f, pass, bufsz))
			sz += f.Size()

		default:
			// A batch of N requires a K-way merge.
			cfrcs, batchsz, err := makeConcurrentFilteringReadClosers(fs, batch, pass, bufsz)
			if err != nil {
				return nil, sz, err // TODO(pb): don't leak FDs
			}
			rc, err := newMergeReadCloser(cfrcs)
			if err != nil {
				return nil, sz, err // TODO(pb): don't leak FDs
			}
			rcs = append(rcs, rc)
			sz += batchsz
		}
	}

	// MultiReadCloser uses an io.MultiReader under the hood.
	// A MultiReader reads from each reader in sequence.
	rc = newMultiReadCloser(rcs...)
	rcs = nil // ownership of each ReadCloser is passed
	return rc, sz, nil
}

// batchSegments batches segments together if they overlap in time.
func batchSegments(segments []string) [][]string {
	// First, parse ranges from filename.
	// TODO(pb): handle weird filenames better, somehow
	type lexrange struct{ a, b string }
	ranges := make([]lexrange, len(segments))
	for i := range segments {
		f := strings.SplitN(basename(segments[i]), "-", 2)
		ranges[i] = lexrange{f[0], f[1]}
	}

	// Now, walk the segments.
	var (
		result = [][]string{}
		group  []string // current
		b      string   // of the group
	)
	for i := range segments {
		switch {
		case len(group) <= 0:
			// If the group is empty, it gets the segment.
			group = []string{segments[i]}
			b = ranges[i].b

		case ranges[i].a > b:
			// If the current segment doesn't overlap with the group,
			// the group is closed and we start a new group.
			result = append(result, group)
			group = []string{segments[i]}
			b = ranges[i].b

		default:
			// The current segment overlaps with the group,
			// so it is absorbed into the group.
			group = append(group, segments[i])
			b = max(b, ranges[i].b)
		}
	}
	if len(group) > 0 {
		result = append(result, group)
	}

	return result
}

func max(a, b string) string {
	if a > b {
		return a
	}
	return b
}

func makeConcurrentFilteringReadClosers(fs fs.Filesystem, segments []string, pass recordFilter, bufsz int64) (rcs []io.ReadCloser, sz int64, err error) {
	// Don't leak FDs on error.
	defer func() {
		if err != nil {
			for _, rc := range rcs {
				rc.Close()
			}
			rcs = nil
		}
	}()

	rcs = make([]io.ReadCloser, len(segments))
	for i := range segments {
		f, err := fs.Open(segments[i])
		if err != nil {
			return rcs, sz, err
		}
		rcs[i] = newConcurrentFilteringReadCloser(f, pass, bufsz)
		sz += f.Size()
	}

	return rcs, sz, nil
}

func newConcurrentFilteringReadCloser(src io.ReadCloser, pass recordFilter, bufsz int64) io.ReadCloser {
	r, w := nio.Pipe(buffer.New(bufsz))
	go func() {
		defer src.Close() // close the fs.File when we're done reading
		br := bufio.NewReader(src)
		for {
			line, err := br.ReadSlice('\n')
			if err != nil {
				w.CloseWithError(err)
				return
			}
			if !pass(line) {
				continue
			}
			switch n, err := w.Write(line); {
			case err == io.ErrClosedPipe:
				return // no need to close
			case err != nil:
				w.CloseWithError(err)
				return
			case n < len(line):
				w.CloseWithError(io.ErrShortWrite)
				return
			}
		}
	}()
	return r
}

// mergeReadCloser performs a K-way merge from multiple readers.
type mergeReadCloser struct {
	close   []io.Closer
	scanner []*bufio.Scanner
	ok      []bool
	record  [][]byte
	id      [][]byte
}

func newMergeReadCloser(rcs []io.ReadCloser) (io.ReadCloser, error) {
	// Initialize our state.
	rc := &mergeReadCloser{
		close:   make([]io.Closer, len(rcs)),
		scanner: make([]*bufio.Scanner, len(rcs)),
		ok:      make([]bool, len(rcs)),
		record:  make([][]byte, len(rcs)),
		id:      make([][]byte, len(rcs)),
	}

	// Initialize all of the scanners and their first record.
	const (
		scanBufferSize   = 64 * 1024      // 64KB
		scanMaxTokenSize = scanBufferSize // if equal, no allocs
	)
	for i := 0; i < len(rcs); i++ {
		rc.close[i] = rcs[i]
		rc.scanner[i] = bufio.NewScanner(rcs[i])
		rc.scanner[i].Buffer(make([]byte, scanBufferSize), scanMaxTokenSize)
		if err := rc.advance(i); err != nil {
			return nil, err
		}
	}

	// Ready to read.
	return rc, nil
}

func (rc *mergeReadCloser) Read(p []byte) (int, error) {
	// Pick the source with the smallest ID.
	// TODO(pb): could be improved with an e.g. tournament tree
	smallest := -1 // index
	for i := range rc.id {
		if !rc.ok[i] {
			continue // already drained
		}
		if smallest < 0 || bytes.Compare(rc.id[i], rc.id[smallest]) < 0 {
			smallest = i
		}
	}
	if smallest < 0 {
		return 0, io.EOF // everything is drained
	}

	// Copy the record over.
	src := append(rc.record[smallest], '\n')
	n := copy(p, src)
	if n < len(src) {
		panic("short read!") // TODO(pb): obviously needs fixing
	}

	// Advance the chosen source.
	if err := rc.advance(smallest); err != nil {
		return n, errors.Wrapf(err, "advancing reader %d", smallest)
	}

	// One read is complete.
	return n, nil
}

func (rc *mergeReadCloser) Close() error {
	return multiCloser(rc.close).Close()
}

func (rc *mergeReadCloser) advance(i int) error {
	if rc.ok[i] = rc.scanner[i].Scan(); rc.ok[i] {
		rc.record[i] = rc.scanner[i].Bytes()
		if len(rc.record[i]) < ulid.EncodedSize {
			panic("record is too short")
		}
		rc.id[i] = rc.record[i][:ulid.EncodedSize]
	} else if err := rc.scanner[i].Err(); err != nil && err != io.EOF {
		return err
	}
	return nil
}

type readCloser struct {
	io.Reader
	io.Closer
}

func newMultiReadCloser(rc ...io.ReadCloser) io.ReadCloser {
	var (
		r = make([]io.Reader, len(rc))
		c = make([]io.Closer, len(rc))
	)
	for i := range rc {
		r[i] = rc[i]
		c[i] = rc[i]
	}
	return readCloser{
		Reader: io.MultiReader(r...),
		Closer: multiCloser(c),
	}
}

// multiCloser closes all underlying io.Closers.
// If an error is encountered, closings continue.
type multiCloser []io.Closer

func (c multiCloser) Close() error {
	var errs []error
	for _, closer := range c {
		if closer == nil {
			continue
		}
		if err := closer.Close(); err != nil {
			errs = append(errs, err)
		}
	}
	if len(errs) > 0 {
		return multiCloseError(errs)
	}
	return nil
}

type multiCloseError []error

func (e multiCloseError) Error() string {
	a := make([]string, len(e))
	for i, err := range e {
		a[i] = err.Error()
	}
	return strings.Join(a, "; ")
}
<commit_msg>Comment cleanup<commit_after>package store

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"

	"github.com/djherbis/buffer"
	"github.com/djherbis/nio"
	"github.com/pkg/errors"

	"github.com/oklog/prototype/pkg/fs"
	"github.com/oklog/ulid"
)

// QueryResult contains statistics about, and matching records for, a query.
type QueryResult struct {
	From     string `json:"from"`
	To       string `json:"to"`
	Q        string `json:"q"`
	Regex    bool   `json:"regex"`
	Duration string `json:"duration"`

	NodesQueried    int   `json:"nodes_queried"`
	SegmentsQueried int   `json:"segments_queried"`
	MaxDataSetSize  int64 `json:"max_data_set_size"`
	ErrorCount      int   `json:"error_count,omitempty"`

	Records io.ReadCloser // TODO(pb): audit to ensure closing is valid throughout
}

// EncodeTo encodes the QueryResult to the HTTP response writer.
// It also closes the records ReadCloser.
func (qr *QueryResult) EncodeTo(w http.ResponseWriter) {
	w.Header().Set(httpHeaderFrom, qr.From)
	w.Header().Set(httpHeaderTo, qr.To)
	w.Header().Set(httpHeaderQ, qr.Q)
	w.Header().Set(httpHeaderRegex, fmt.Sprint(qr.Regex))

	w.Header().Set(httpHeaderNodesQueried, strconv.Itoa(qr.NodesQueried))
	w.Header().Set(httpHeaderSegmentsQueried, strconv.Itoa(qr.SegmentsQueried))
	w.Header().Set(httpHeaderMaxDataSetSize, strconv.FormatInt(qr.MaxDataSetSize, 10))
	w.Header().Set(httpHeaderErrorCount, strconv.Itoa(qr.ErrorCount))
	w.Header().Set(httpHeaderDuration, qr.Duration)

	if qr.Records != nil {
		// CopyBuffer can be useful for complex query pipelines.
		// TODO(pb): validate the 1MB buffer size with profiling
		buf := make([]byte, 1024*1024)
		io.CopyBuffer(w, qr.Records, buf)
		qr.Records.Close()
	}
}

// DecodeFrom decodes the QueryResult from the HTTP response.
func (qr *QueryResult) DecodeFrom(resp *http.Response) error {
	qr.From = resp.Header.Get(httpHeaderFrom)
	qr.To = resp.Header.Get(httpHeaderTo)
	qr.Q = resp.Header.Get(httpHeaderQ)
	var err error
	if qr.Regex, err = strconv.ParseBool(resp.Header.Get(httpHeaderRegex)); err != nil {
		return errors.Wrap(err, "regex")
	}
	if qr.NodesQueried, err = strconv.Atoi(resp.Header.Get(httpHeaderNodesQueried)); err != nil {
		return errors.Wrap(err, "nodes queried")
	}
	if qr.SegmentsQueried, err = strconv.Atoi(resp.Header.Get(httpHeaderSegmentsQueried)); err != nil {
		return errors.Wrap(err, "segments queried")
	}
	if qr.MaxDataSetSize, err = strconv.ParseInt(resp.Header.Get(httpHeaderMaxDataSetSize), 10, 64); err != nil {
		return errors.Wrap(err, "max data set size")
	}
	if qr.ErrorCount, err = strconv.Atoi(resp.Header.Get(httpHeaderErrorCount)); err != nil {
		return errors.Wrap(err, "error count")
	}
	qr.Duration = resp.Header.Get(httpHeaderDuration)
	qr.Records = resp.Body
	return nil
}

// Merge the other QueryResult into this one.
func (qr *QueryResult) Merge(other QueryResult) error {
	// Union the simple integer types.
	qr.NodesQueried += other.NodesQueried
	qr.SegmentsQueried += other.SegmentsQueried
	if other.MaxDataSetSize > qr.MaxDataSetSize {
		qr.MaxDataSetSize = other.MaxDataSetSize
	}
	qr.ErrorCount += other.ErrorCount

	// Merge the record readers.
	// Both mergeRecords and multiCloser can handle nils.
	var buf bytes.Buffer
	_, _, _, err := mergeRecords(&buf, qr.Records, other.Records)
	multiCloser{qr.Records, other.Records}.Close()
	qr.Records = ioutil.NopCloser(&buf)

	// Done.
	return err
}

const (
	httpHeaderFrom            = "X-Oklog-From"
	httpHeaderTo              = "X-Oklog-To"
	httpHeaderQ               = "X-Oklog-Q"
	httpHeaderRegex           = "X-Oklog-Regex"
	httpHeaderNodesQueried    = "X-Oklog-Nodes-Queried"
	httpHeaderSegmentsQueried = "X-Oklog-Segments-Queried"
	httpHeaderMaxDataSetSize  = "X-Oklog-Max-Data-Set-Size"
	httpHeaderErrorCount      = "X-Oklog-Error-Count"
	httpHeaderDuration        = "X-Oklog-Duration"
)

type recordFilter func([]byte) bool

// newQueryReadCloser converts a batch of segments to a single io.ReadCloser.
// Records are yielded in time order, oldest first, hopefully efficiently!
// Only records passing the recordFilter are yielded.
// The sz of the segment files can be used as a proxy for read effort.
func newQueryReadCloser(fs fs.Filesystem, segments []string, pass recordFilter, bufsz int64) (rc io.ReadCloser, sz int64, err error) {
	// We will build successive ReadClosers for each batch.
	var rcs []io.ReadCloser

	// Don't leak FDs on error.
	defer func() {
		if err != nil {
			for _, rc := range rcs {
				rc.Close()
			}
		}
	}()

	// Batch the segments, and construct a ReadCloser for each batch.
	for _, batch := range batchSegments(segments) {
		switch len(batch) {
		case 0:
			continue // weird

		case 1:
			// A batch of one can be read straight thru.
			f, err := fs.Open(batch[0])
			if err != nil {
				return nil, sz, err
			}
			rcs = append(rcs, newConcurrentFilteringReadCloser(f, pass, bufsz))
			sz += f.Size()

		default:
			// A batch of N requires a K-way merge.
			cfrcs, batchsz, err := makeConcurrentFilteringReadClosers(fs, batch, pass, bufsz)
			if err != nil {
				return nil, sz, err
			}
			rc, err := newMergeReadCloser(cfrcs)
			if err != nil {
				return nil, sz, err
			}
			rcs = append(rcs, rc)
			sz += batchsz
		}
	}

	// MultiReadCloser uses an io.MultiReader under the hood.
	// A MultiReader reads from each reader in sequence.
	rc = newMultiReadCloser(rcs...)
	rcs = nil // ownership of each ReadCloser is passed
	return rc, sz, nil
}

// batchSegments batches segments together if they overlap in time.
func batchSegments(segments []string) [][]string {
	// First, parse ranges from filename.
	// TODO(pb): handle weird filenames better, somehow
	type lexrange struct{ a, b string }
	ranges := make([]lexrange, len(segments))
	for i := range segments {
		f := strings.SplitN(basename(segments[i]), "-", 2)
		ranges[i] = lexrange{f[0], f[1]}
	}

	// Now, walk the segments.
	var (
		result = [][]string{}
		group  []string // current
		b      string   // of the group
	)
	for i := range segments {
		switch {
		case len(group) <= 0:
			// If the group is empty, it gets the segment.
			group = []string{segments[i]}
			b = ranges[i].b

		case ranges[i].a > b:
			// If the current segment doesn't overlap with the group,
			// the group is closed and we start a new group.
			result = append(result, group)
			group = []string{segments[i]}
			b = ranges[i].b

		default:
			// The current segment overlaps with the group,
			// so it is absorbed into the group.
			group = append(group, segments[i])
			b = max(b, ranges[i].b)
		}
	}
	if len(group) > 0 {
		result = append(result, group)
	}

	return result
}

func max(a, b string) string {
	if a > b {
		return a
	}
	return b
}

func makeConcurrentFilteringReadClosers(fs fs.Filesystem, segments []string, pass recordFilter, bufsz int64) (rcs []io.ReadCloser, sz int64, err error) {
	// Don't leak FDs on error.
	defer func() {
		if err != nil {
			for _, rc := range rcs {
				rc.Close()
			}
			rcs = nil
		}
	}()

	rcs = make([]io.ReadCloser, len(segments))
	for i := range segments {
		f, err := fs.Open(segments[i])
		if err != nil {
			return rcs, sz, err
		}
		rcs[i] = newConcurrentFilteringReadCloser(f, pass, bufsz)
		sz += f.Size()
	}

	return rcs, sz, nil
}

func newConcurrentFilteringReadCloser(src io.ReadCloser, pass recordFilter, bufsz int64) io.ReadCloser {
	r, w := nio.Pipe(buffer.New(bufsz))
	go func() {
		defer src.Close() // close the fs.File when we're done reading
		br := bufio.NewReader(src)
		for {
			line, err := br.ReadSlice('\n')
			if err != nil {
				w.CloseWithError(err)
				return
			}
			if !pass(line) {
				continue
			}
			switch n, err := w.Write(line); {
			case err == io.ErrClosedPipe:
				return // no need to close
			case err != nil:
				w.CloseWithError(err)
				return
			case n < len(line):
				w.CloseWithError(io.ErrShortWrite)
				return
			}
		}
	}()
	return r
}

// mergeReadCloser performs a K-way merge from multiple readers.
type mergeReadCloser struct {
	close   []io.Closer
	scanner []*bufio.Scanner
	ok      []bool
	record  [][]byte
	id      [][]byte
}

func newMergeReadCloser(rcs []io.ReadCloser) (io.ReadCloser, error) {
	// Initialize our state.
	rc := &mergeReadCloser{
		close:   make([]io.Closer, len(rcs)),
		scanner: make([]*bufio.Scanner, len(rcs)),
		ok:      make([]bool, len(rcs)),
		record:  make([][]byte, len(rcs)),
		id:      make([][]byte, len(rcs)),
	}

	// Initialize all of the scanners and their first record.
	const (
		scanBufferSize   = 64 * 1024      // 64KB
		scanMaxTokenSize = scanBufferSize // if equal, no allocs
	)
	for i := 0; i < len(rcs); i++ {
		rc.close[i] = rcs[i]
		rc.scanner[i] = bufio.NewScanner(rcs[i])
		rc.scanner[i].Buffer(make([]byte, scanBufferSize), scanMaxTokenSize)
		if err := rc.advance(i); err != nil {
			return nil, err
		}
	}

	// Ready to read.
	return rc, nil
}

func (rc *mergeReadCloser) Read(p []byte) (int, error) {
	// Pick the source with the smallest ID.
	// TODO(pb): could be improved with an e.g. tournament tree
	smallest := -1 // index
	for i := range rc.id {
		if !rc.ok[i] {
			continue // already drained
		}
		if smallest < 0 || bytes.Compare(rc.id[i], rc.id[smallest]) < 0 {
			smallest = i
		}
	}
	if smallest < 0 {
		return 0, io.EOF // everything is drained
	}

	// Copy the record over.
	src := append(rc.record[smallest], '\n')
	n := copy(p, src)
	if n < len(src) {
		panic("short read!") // TODO(pb): obviously needs fixing
	}

	// Advance the chosen source.
	if err := rc.advance(smallest); err != nil {
		return n, errors.Wrapf(err, "advancing reader %d", smallest)
	}

	// One read is complete.
	return n, nil
}

func (rc *mergeReadCloser) Close() error {
	return multiCloser(rc.close).Close()
}

func (rc *mergeReadCloser) advance(i int) error {
	if rc.ok[i] = rc.scanner[i].Scan(); rc.ok[i] {
		rc.record[i] = rc.scanner[i].Bytes()
		if len(rc.record[i]) < ulid.EncodedSize {
			panic("record is too short")
		}
		rc.id[i] = rc.record[i][:ulid.EncodedSize]
	} else if err := rc.scanner[i].Err(); err != nil && err != io.EOF {
		return err
	}
	return nil
}

type readCloser struct {
	io.Reader
	io.Closer
}

func newMultiReadCloser(rc ...io.ReadCloser) io.ReadCloser {
	var (
		r = make([]io.Reader, len(rc))
		c = make([]io.Closer, len(rc))
	)
	for i := range rc {
		r[i] = rc[i]
		c[i] = rc[i]
	}
	return readCloser{
		Reader: io.MultiReader(r...),
		Closer: multiCloser(c),
	}
}

// multiCloser closes all underlying io.Closers.
// If an error is encountered, closings continue.
type multiCloser []io.Closer

func (c multiCloser) Close() error {
	var errs []error
	for _, closer := range c {
		if closer == nil {
			continue
		}
		if err := closer.Close(); err != nil {
			errs = append(errs, err)
		}
	}
	if len(errs) > 0 {
		return multiCloseError(errs)
	}
	return nil
}

type multiCloseError []error

func (e multiCloseError) Error() string {
	a := make([]string, len(e))
	for i, err := range e {
		a[i] = err.Error()
	}
	return strings.Join(a, "; ")
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	. "launchpad.net/gocheck"
	"launchpad.net/goyaml"
	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/juju"
	"launchpad.net/juju-core/state"
	"os"
	"path/filepath"
)

type StatusSuite struct {
	envSuite
	repoPath, seriesPath string
	conn                 *juju.Conn
	st                   *state.State
}

var _ = Suite(&StatusSuite{})

func (s *StatusSuite) SetUpTest(c *C) {
	s.envSuite.SetUpTest(c, zkConfig)
	repoPath := c.MkDir()
	s.repoPath = os.Getenv("JUJU_REPOSITORY")
	os.Setenv("JUJU_REPOSITORY", repoPath)
	s.seriesPath = filepath.Join(repoPath, "precise")
	err := os.Mkdir(s.seriesPath, 0777)
	c.Assert(err, IsNil)
	s.conn, err = juju.NewConn("")
	c.Assert(err, IsNil)
	err = s.conn.Environ.Bootstrap(false)
	c.Assert(err, IsNil)
	s.st, err = s.conn.State()
	c.Assert(err, IsNil)
}

func (s *StatusSuite) TearDownTest(c *C) {
	s.envSuite.TearDownTest(c)
	os.Setenv("JUJU_REPOSITORY", s.repoPath)
	s.conn.Close()
}

var statusTests = []struct {
	title   string
	prepare func(*state.State, *C)
	output  map[string]string
}{
	{
		// unlikely, as you can't run juju status in real life without
		// machine/0 bootstrapped.
		"empty state",
		func(*state.State, *C) {},
		map[string]string{
			"yaml": "machines: {}\nservices: {}\n\n",
			"json": `{"machines":{},"services":{}}`,
		},
	},
}

func (s *StatusSuite) testStatus(format string, unmarshal func([]byte, interface{}) error, c *C) {
	for _, t := range statusTests {
		c.Logf("testing %s: %s", format, t.title)
		t.prepare(s.st, c)
		ctx := &cmd.Context{c.MkDir(), &bytes.Buffer{}, &bytes.Buffer{}}
		code := cmd.Main(&StatusCommand{}, ctx, []string{"--format", format})
		c.Check(code, Equals, 0)
		c.Assert(ctx.Stderr.(*bytes.Buffer).String(), Equals, "")

		expected := make(map[string]interface{})
		err := unmarshal([]byte(t.output[format]), &expected)
		c.Assert(err, IsNil)

		actual := make(map[string]interface{})
		err = unmarshal(ctx.Stdout.(*bytes.Buffer).Bytes(), &actual)
		c.Assert(err, IsNil)

		c.Assert(actual, DeepEquals, expected)
	}
}

func (s *StatusSuite) TestYamlStatus(c *C) {
	s.testStatus("yaml", goyaml.Unmarshal, c)
}

func (s *StatusSuite) TestJsonStatus(c *C) {
	s.testStatus("json", json.Unmarshal, c)
}
<commit_msg>format agnostic testing<commit_after>package main

import (
	"bytes"
	"encoding/json"
	. "launchpad.net/gocheck"
	"launchpad.net/goyaml"
	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/juju"
	"launchpad.net/juju-core/state"
	"os"
	"path/filepath"
)

type StatusSuite struct {
	envSuite
	repoPath, seriesPath string
	conn                 *juju.Conn
	st                   *state.State
}

var _ = Suite(&StatusSuite{})

func (s *StatusSuite) SetUpTest(c *C) {
	s.envSuite.SetUpTest(c, zkConfig)
	repoPath := c.MkDir()
	s.repoPath = os.Getenv("JUJU_REPOSITORY")
	os.Setenv("JUJU_REPOSITORY", repoPath)
	s.seriesPath = filepath.Join(repoPath, "precise")
	err := os.Mkdir(s.seriesPath, 0777)
	c.Assert(err, IsNil)
	s.conn, err = juju.NewConn("")
	c.Assert(err, IsNil)
	err = s.conn.Environ.Bootstrap(false)
	c.Assert(err, IsNil)
	s.st, err = s.conn.State()
	c.Assert(err, IsNil)
}

func (s *StatusSuite) TearDownTest(c *C) {
	s.envSuite.TearDownTest(c)
	os.Setenv("JUJU_REPOSITORY", s.repoPath)
	s.conn.Close()
}

var statusTests = []struct {
	title   string
	prepare func(*state.State, *C)
	output  map[string]interface{}
}{
	{
		// unlikely, as you can't run juju status in real life without
		// machine/0 bootstrapped.
		"empty state",
		func(*state.State, *C) {},
		map[string]interface{}{
			"machines": make(map[string]interface{}),
			"services": make(map[string]interface{}),
		},
	},
}

func (s *StatusSuite) testStatus(format string, marshal func(v interface{}) ([]byte, error), unmarshal func(data []byte, v interface{}) error, c *C) {
	for _, t := range statusTests {
		c.Logf("testing %T: %s", format, t.title)
		t.prepare(s.st, c)
		ctx := &cmd.Context{c.MkDir(), &bytes.Buffer{}, &bytes.Buffer{}}
		code := cmd.Main(&StatusCommand{}, ctx, []string{"--format", format})
		c.Check(code, Equals, 0)
		c.Assert(ctx.Stderr.(*bytes.Buffer).String(), Equals, "")

		buf, err := marshal(t.output)
		c.Assert(err, IsNil)
		expected := make(map[string]interface{})
		err = unmarshal(buf, &expected)
		c.Assert(err, IsNil)

		actual := make(map[string]interface{})
		err = unmarshal(ctx.Stdout.(*bytes.Buffer).Bytes(), &actual)
		c.Assert(err, IsNil)

		c.Assert(actual, DeepEquals, expected)
	}
}

func (s *StatusSuite) TestYamlStatus(c *C) {
	s.testStatus("yaml", goyaml.Marshal, goyaml.Unmarshal, c)
}

func (s *StatusSuite) TestJsonStatus(c *C) {
	s.testStatus("json", json.Marshal, json.Unmarshal, c)
}
<|endoftext|>


<commit_before>package store

import (
	"bytes"
	"container/list"
	"gob"
	"junta/util"
	"math"
	"os"
	"regexp"
	"strings"
)

const (
	Clobber = ""
	Missing = "0"
	Dir = "dir"
)

var (
	ErrBadPath = os.NewError("bad path")
	ErrBadMutation = os.NewError("bad mutation")
	ErrBadSnapshot = os.NewError("bad snapshot")
	ErrTooLate = os.NewError("too late")
	ErrCasMismatch = os.NewError("cas mismatch")
)

var waitRegexp = regexp.MustCompile(``)

type Store struct {
	applyCh chan apply
	lookupCh chan *lookup
	snapCh chan chan snap
	watchCh chan watch
	watches []watch
	todo map[uint64]apply
}

type apply struct {
	seqn uint64
	mutation string
}

type lookup struct {
	k string
	ch chan int
	v []string
	cas string
}

type snap struct {
	ver uint64
	root node
}

type watch struct {
	pat string
	in, out chan Event
	re *regexp.Regexp
	stop uint64
}

// Creates a new, empty data store. Mutations will be applied in order,
// starting at number 1 (number 0 can be thought of as the creation of the
// store).
func New() *Store {
	s := &Store{
		applyCh: make(chan apply),
		lookupCh: make(chan *lookup),
		snapCh: make(chan chan snap),
		watchCh: make(chan watch),
		todo: make(map[uint64]apply),
		watches: []watch{},
	}
	go s.process()
	return s
}

func split(path string) []string {
	if path == "/" {
		return []string{}
	}
	return strings.Split(path[1:], "/", -1)
}

func join(parts []string) string {
	return "/" + strings.Join(parts, "/")
}

func checkPath(k string) os.Error {
	switch {
	case len(k) < 1,
	     k[0] != '/',
	     len(k) > 1 && k[len(k) - 1] == '/',
	     strings.Count(k, "=") > 0,
	     strings.Count(k, " ") > 0:
		return ErrBadPath
	}
	return nil
}

func EncodeSet(path, body string, cas string) (mutation string, err os.Error) {
	if err = checkPath(path); err != nil {
		return
	}
	return cas + ":" + path + "=" + body, nil
}

func EncodeDel(path string, cas string) (mutation string, err os.Error) {
	if err := checkPath(path); err != nil {
		return
	}
	return cas + ":" + path, nil
}

// MustEncodeSet is like EncodeSet but panics if the mutation cannot be
// encoded. It simplifies safe initialization of global variables holding
// mutations.
func MustEncodeSet(path, body, cas string) (mutation string) {
	m, err := EncodeSet(path, body, cas)
	if err != nil {
		panic(err)
	}
	return m
}

// MustEncodeDel is like EncodeDel but panics if the mutation cannot be
// encoded. It simplifies safe initialization of global variables holding
// mutations.
func MustEncodeDel(path, cas string) (mutation string) {
	m, err := EncodeDel(path, cas)
	if err != nil {
		panic(err)
	}
	return m
}

func decode(mutation string) (path, v, cas string, keep bool, err os.Error) {
	cm := strings.Split(mutation, ":", 2)

	if len(cm) != 2 {
		err = ErrBadMutation
		return
	}

	kv := strings.Split(cm[1], "=", 2)

	if err = checkPath(kv[0]); err != nil {
		return
	}

	switch len(kv) {
	case 1:
		return kv[0], "", cm[0], false, nil
	case 2:
		return kv[0], kv[1], cm[0], true, nil
	}
	panic("can't happen")
}

func (s *Store) notify(e Event) {
	for _, w := range s.watches {
		if w.re.MatchString(e.Path) {
			w.in <- e
		}
	}
}

func append(ws *[]watch, w watch) {
	l := len(*ws)
	if l + 1 > cap(*ws) {
		ns := make([]watch, (l + 1)*2)
		copy(ns, *ws)
		*ws = ns
	}
	*ws = (*ws)[0:l + 1]
	(*ws)[l] = w
}

// Unbounded in-order buffering
func buffer(in, out chan Event) {
	list := list.New()
	for {
		f, e := list.Front(), Event{}
		var ch chan Event
		if f != nil {
			e = f.Value.(Event)
			ch = out
		}
		select {
		case x := <-in:
			if closed(in) {
				return
			}
			list.PushBack(x)
		case ch <- e:
			list.Remove(f)
		}
	}
}

func (s *Store) process() {
	logger := util.NewLogger("store")

	ver := uint64(0)
	values := root

	for {
		// Take any incoming requests and queue them up.
		select {
		case a := <-s.applyCh:
			if a.seqn > ver {
				s.todo[a.seqn] = a
			}
		case r := <-s.lookupCh:
			r.v, r.cas = values.getp(r.k)
			r.ch <- 1
		case ch := <-s.snapCh:
			ch <- snap{ver, values}
		case w := <-s.watchCh:
			if w.stop > ver {
				append(&s.watches, w)
			} else {
				w.in <- Event{Seqn:w.stop, Err:ErrTooLate}
				close(w.in)
			}
		}

		// If we have any mutations that can be applied, do them.
		for t, ok := s.todo[ver+1]; ok; t, ok = s.todo[ver+1] {
			var ev Event
			values, ev = values.apply(t.seqn, t.mutation)
			logger.Logf("apply %s %v %v %v %v %v", ev.Desc(), ev.Seqn, ev.Path, ev.Body, ev.Cas, ev.Err)
			s.notify(ev)
			for ver < ev.Seqn {
				ver++
				s.todo[ver] = apply{}, false
			}
		}
	}
}

// Applies `mutation` in sequence at position `seqn`. Any error that occurs
// will be written to `ErrorPath`. If a mutation has already been applied at
// this position, this one is sliently ignored.
//
// If `mutation` is a snapshot, notifications will not be sent.
//
// If `mutation` is `Nop`, no change will be made, but a dummy event will still
// be sent.
func (s *Store) Apply(seqn uint64, mutation string) {
	s.applyCh <- apply{seqn, mutation}
}

// Gets the value stored at `path`, if any.
//
// If no value is stored at `path`, `cas` will be `Missing` and `value` will be
// nil.
//
// if `path` is a directory, `cas` will be `Dir` and `value` will be a list of
// entries.
//
// Otherwise, `cas` is the cas token and `value[0]` is the body.
func (s *Store) Lookup(path string) (value []string, cas string) {
	l := lookup{k:path, ch:make(chan int)}
	s.lookupCh <- &l
	<-l.ch
	return l.v, l.cas
}

// Retrieves the body stored at `path` and returns it. If `path` is a directory
// or does not exist, returns an empty string.
//
// Note, with this function it is impossible to distinguish between an empty
// string stored at `path`, a missing entry, and a directory. If you need to
// tell the difference, use `Lookup`.
//
// Also note, this function does not return the CAS token for `path`. If you
// need the CAS token, use `Lookup`.
func (s *Store) LookupString(path string) (body string) {
	v, cas := s.Lookup(path)
	if cas == Missing || cas == Dir {
		return ""
	}
	return v[0]
}

// Returns a list of entries in the directory at `path`. If `path` is not a
// directory, returns an empty slice.
//
// Note, with this function it is impossible to distinguish between a string
// stored at `path` and a missing entry. If you need to tell the difference,
// use `Lookup`.
func (s *Store) LookupDir(path string) (entries []string) {
	v, cas := s.Lookup(path)
	if cas != Dir {
		return nil
	}
	return v
}

// Encodes the entire storage state, including the current sequence number, as
// a mutation. This mutation can be applied to an empty store to reproduce the
// state of `s`.
//
// Returns the sequence number of the snapshot and the mutation itself.
//
// A snapshot must be applied at sequence number 1. Once a snapshot has been
// applied, the store's sequence number will be set to `seqn`.
//
// Note that applying a snapshot does not send notifications.
func (s *Store) Snapshot() (seqn uint64, mutation string) {
	w := new(bytes.Buffer)
	ch := make(chan snap)
	s.snapCh <- ch
	ss := <-ch
	err := gob.NewEncoder(w).Encode(ss.ver)
	if err != nil {
		panic(err)
	}

	err = gob.NewEncoder(w).Encode(ss.root)
	if err != nil {
		panic(err)
	}

	return ss.ver, w.String()
}

// Subscribes `ch` to receive notifications when mutations are applied to paths
// in the store. One event will be sent for each mutation iff the event's path
// matches `pattern`, a Unix-style glob pattern.
//
// Glob notation:
//  - `?` matches a single char in a single path component
//  - `*` matches zero or more chars in a single path component
//  - `**` matches zero or more chars in zero or more components
//
// Notifications will not be sent for changes made as the result of applying a
// snapshot.
func (s *Store) Watch(pattern string, ch chan Event) {
	re, _ := compileGlob(pattern)
	in := make(chan Event)
	go buffer(in, ch)
	s.watchCh <- watch{pat:pattern, out:ch, in:in, re:re, stop:math.MaxUint64}
}

// Subscribes `ch` to receive a single event representing the change made at
// position `seqn`.
//
// If `seqn` was applied before the call to `Wait`, a dummy event will be
// sent with its `Err` set to `ErrTooLate`.
func (s *Store) Wait(seqn uint64, ch chan Event) {
	all := make(chan Event)
	s.watchCh <- watch{in:all, re:waitRegexp, stop:seqn}
	go func() {
		for e := range all {
			if e.Seqn == seqn {
				close(all)
				ch <- e
			}
		}
	}()
}

// Ensures that the application of mutation at `seqn` happens before the call
// to `Sync` returns.
func (st *Store) Sync(seqn uint64) {
	ch := make(chan Event)
	st.Wait(seqn, ch)
	<-ch
}
<commit_msg>proper capitalization<commit_after>package store

import (
	"bytes"
	"container/list"
	"gob"
	"junta/util"
	"math"
	"os"
	"regexp"
	"strings"
)

const (
	Clobber = ""
	Missing = "0"
	Dir = "dir"
)

var (
	ErrBadPath = os.NewError("bad path")
	ErrBadMutation = os.NewError("bad mutation")
	ErrBadSnapshot = os.NewError("bad snapshot")
	ErrTooLate = os.NewError("too late")
	ErrCasMismatch = os.NewError("cas mismatch")
)

var waitRegexp = regexp.MustCompile(``)

type Store struct {
	applyCh chan apply
	lookupCh chan *lookup
	snapCh chan chan snap
	watchCh chan watch
	watches []watch
	todo map[uint64]apply
}

type apply struct {
	seqn uint64
	mutation string
}

type lookup struct {
	k string
	ch chan int
	v []string
	cas string
}

type snap struct {
	ver uint64
	root node
}

type watch struct {
	pat string
	in, out chan Event
	re *regexp.Regexp
	stop uint64
}

// Creates a new, empty data store. Mutations will be applied in order,
// starting at number 1 (number 0 can be thought of as the creation of the
// store).
func New() *Store {
	s := &Store{
		applyCh: make(chan apply),
		lookupCh: make(chan *lookup),
		snapCh: make(chan chan snap),
		watchCh: make(chan watch),
		todo: make(map[uint64]apply),
		watches: []watch{},
	}
	go s.process()
	return s
}

func split(path string) []string {
	if path == "/" {
		return []string{}
	}
	return strings.Split(path[1:], "/", -1)
}

func join(parts []string) string {
	return "/" + strings.Join(parts, "/")
}

func checkPath(k string) os.Error {
	switch {
	case len(k) < 1,
	     k[0] != '/',
	     len(k) > 1 && k[len(k) - 1] == '/',
	     strings.Count(k, "=") > 0,
	     strings.Count(k, " ") > 0:
		return ErrBadPath
	}
	return nil
}

func EncodeSet(path, body string, cas string) (mutation string, err os.Error) {
	if err = checkPath(path); err != nil {
		return
	}
	return cas + ":" + path + "=" + body, nil
}

func EncodeDel(path string, cas string) (mutation string, err os.Error) {
	if err := checkPath(path); err != nil {
		return
	}
	return cas + ":" + path, nil
}

// MustEncodeSet is like EncodeSet but panics if the mutation cannot be
// encoded. It simplifies safe initialization of global variables holding
// mutations.
func MustEncodeSet(path, body, cas string) (mutation string) {
	m, err := EncodeSet(path, body, cas)
	if err != nil {
		panic(err)
	}
	return m
}

// MustEncodeDel is like EncodeDel but panics if the mutation cannot be
// encoded. It simplifies safe initialization of global variables holding
// mutations.
func MustEncodeDel(path, cas string) (mutation string) {
	m, err := EncodeDel(path, cas)
	if err != nil {
		panic(err)
	}
	return m
}

func decode(mutation string) (path, v, cas string, keep bool, err os.Error) {
	cm := strings.Split(mutation, ":", 2)

	if len(cm) != 2 {
		err = ErrBadMutation
		return
	}

	kv := strings.Split(cm[1], "=", 2)

	if err = checkPath(kv[0]); err != nil {
		return
	}

	switch len(kv) {
	case 1:
		return kv[0], "", cm[0], false, nil
	case 2:
		return kv[0], kv[1], cm[0], true, nil
	}
	panic("can't happen")
}

func (s *Store) notify(e Event) {
	for _, w := range s.watches {
		if w.re.MatchString(e.Path) {
			w.in <- e
		}
	}
}

func append(ws *[]watch, w watch) {
	l := len(*ws)
	if l + 1 > cap(*ws) {
		ns := make([]watch, (l + 1)*2)
		copy(ns, *ws)
		*ws = ns
	}
	*ws = (*ws)[0:l + 1]
	(*ws)[l] = w
}

// Unbounded in-order buffering
func buffer(in, out chan Event) {
	list := list.New()
	for {
		f, e := list.Front(), Event{}
		var ch chan Event
		if f != nil {
			e = f.Value.(Event)
			ch = out
		}
		select {
		case x := <-in:
			if closed(in) {
				return
			}
			list.PushBack(x)
		case ch <- e:
			list.Remove(f)
		}
	}
}

func (s *Store) process() {
	logger := util.NewLogger("store")

	ver := uint64(0)
	values := root

	for {
		// Take any incoming requests and queue them up.
		select {
		case a := <-s.applyCh:
			if a.seqn > ver {
				s.todo[a.seqn] = a
			}
		case r := <-s.lookupCh:
			r.v, r.cas = values.getp(r.k)
			r.ch <- 1
		case ch := <-s.snapCh:
			ch <- snap{ver, values}
		case w := <-s.watchCh:
			if w.stop > ver {
				append(&s.watches, w)
			} else {
				w.in <- Event{Seqn:w.stop, Err:ErrTooLate}
				close(w.in)
			}
		}

		// If we have any mutations that can be applied, do them.
		for t, ok := s.todo[ver+1]; ok; t, ok = s.todo[ver+1] {
			var ev Event
			values, ev = values.apply(t.seqn, t.mutation)
			logger.Logf("apply %s %v %v %v %v %v", ev.Desc(), ev.Seqn, ev.Path, ev.Body, ev.Cas, ev.Err)
			s.notify(ev)
			for ver < ev.Seqn {
				ver++
				s.todo[ver] = apply{}, false
			}
		}
	}
}

// Applies `mutation` in sequence at position `seqn`. Any error that occurs
// will be written to `ErrorPath`. If a mutation has already been applied at
// this position, this one is sliently ignored.
//
// If `mutation` is a snapshot, notifications will not be sent.
//
// If `mutation` is `Nop`, no change will be made, but a dummy event will still
// be sent.
func (s *Store) Apply(seqn uint64, mutation string) {
	s.applyCh <- apply{seqn, mutation}
}

// Gets the value stored at `path`, if any.
//
// If no value is stored at `path`, `cas` will be `Missing` and `value` will be
// nil.
//
// if `path` is a directory, `cas` will be `Dir` and `value` will be a list of
// entries.
//
// Otherwise, `cas` is the CAS token and `value[0]` is the body.
func (s *Store) Lookup(path string) (value []string, cas string) {
	l := lookup{k:path, ch:make(chan int)}
	s.lookupCh <- &l
	<-l.ch
	return l.v, l.cas
}

// Retrieves the body stored at `path` and returns it. If `path` is a directory
// or does not exist, returns an empty string.
//
// Note, with this function it is impossible to distinguish between an empty
// string stored at `path`, a missing entry, and a directory. If you need to
// tell the difference, use `Lookup`.
//
// Also note, this function does not return the CAS token for `path`. If you
// need the CAS token, use `Lookup`.
func (s *Store) LookupString(path string) (body string) {
	v, cas := s.Lookup(path)
	if cas == Missing || cas == Dir {
		return ""
	}
	return v[0]
}

// Returns a list of entries in the directory at `path`. If `path` is not a
// directory, returns an empty slice.
//
// Note, with this function it is impossible to distinguish between a string
// stored at `path` and a missing entry. If you need to tell the difference,
// use `Lookup`.
func (s *Store) LookupDir(path string) (entries []string) {
	v, cas := s.Lookup(path)
	if cas != Dir {
		return nil
	}
	return v
}

// Encodes the entire storage state, including the current sequence number, as
// a mutation. This mutation can be applied to an empty store to reproduce the
// state of `s`.
//
// Returns the sequence number of the snapshot and the mutation itself.
//
// A snapshot must be applied at sequence number 1. Once a snapshot has been
// applied, the store's sequence number will be set to `seqn`.
//
// Note that applying a snapshot does not send notifications.
func (s *Store) Snapshot() (seqn uint64, mutation string) {
	w := new(bytes.Buffer)
	ch := make(chan snap)
	s.snapCh <- ch
	ss := <-ch
	err := gob.NewEncoder(w).Encode(ss.ver)
	if err != nil {
		panic(err)
	}

	err = gob.NewEncoder(w).Encode(ss.root)
	if err != nil {
		panic(err)
	}

	return ss.ver, w.String()
}

// Subscribes `ch` to receive notifications when mutations are applied to paths
// in the store. One event will be sent for each mutation iff the event's path
// matches `pattern`, a Unix-style glob pattern.
//
// Glob notation:
//  - `?` matches a single char in a single path component
//  - `*` matches zero or more chars in a single path component
//  - `**` matches zero or more chars in zero or more components
//
// Notifications will not be sent for changes made as the result of applying a
// snapshot.
func (s *Store) Watch(pattern string, ch chan Event) {
	re, _ := compileGlob(pattern)
	in := make(chan Event)
	go buffer(in, ch)
	s.watchCh <- watch{pat:pattern, out:ch, in:in, re:re, stop:math.MaxUint64}
}

// Subscribes `ch` to receive a single event representing the change made at
// position `seqn`.
//
// If `seqn` was applied before the call to `Wait`, a dummy event will be
// sent with its `Err` set to `ErrTooLate`.
func (s *Store) Wait(seqn uint64, ch chan Event) {
	all := make(chan Event)
	s.watchCh <- watch{in:all, re:waitRegexp, stop:seqn}
	go func() {
		for e := range all {
			if e.Seqn == seqn {
				close(all)
				ch <- e
			}
		}
	}()
}

// Ensures that the application of mutation at `seqn` happens before the call
// to `Sync` returns.
func (st *Store) Sync(seqn uint64) {
	ch := make(chan Event)
	st.Wait(seqn, ch)
	<-ch
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"encoding/base64"
	"encoding/json"
	"fmt"

	"v.io/v23"
	"v.io/v23/context"
	"v.io/v23/naming"
	"v.io/v23/options"
	"v.io/v23/rpc"
	"v.io/v23/security"
	"v.io/v23/services/mgmt/application"
	"v.io/v23/services/mgmt/device"
	"v.io/x/lib/cmdline"
)

type configFlag device.Config

func (c *configFlag) String() string {
	jsonConfig, _ := json.Marshal(c)
	return string(jsonConfig)
}
func (c *configFlag) Set(s string) error {
	if err := json.Unmarshal([]byte(s), c); err != nil {
		return fmt.Errorf("Unmarshal(%v) failed: %v", s, err)
	}
	return nil
}

var configOverride configFlag = configFlag{}

type packagesFlag application.Packages

func (c *packagesFlag) String() string {
	jsonPackages, _ := json.Marshal(c)
	return string(jsonPackages)
}
func (c *packagesFlag) Set(s string) error {
	if err := json.Unmarshal([]byte(s), c); err != nil {
		return fmt.Errorf("Unmarshal(%v) failed: %v", s, err)
	}
	return nil
}

var packagesOverride packagesFlag = packagesFlag{}

func init() {
	cmdInstall.Flags.Var(&configOverride, "config", "JSON-encoded device.Config object, of the form: '{\"flag1\":\"value1\",\"flag2\":\"value2\"}'")
	cmdInstall.Flags.Var(&packagesOverride, "packages", "JSON-encoded application.Packages object, of the form: '{\"pkg1\":{\"File\":\"object name 1\"},\"pkg2\":{\"File\":\"object name 2\"}}'")
}

var cmdInstall = &cmdline.Command{
	Run:      runInstall,
	Name:     "install",
	Short:    "Install the given application.",
	Long:     "Install the given application.",
	ArgsName: "<device> <application>",
	ArgsLong: `
<device> is the vanadium object name of the device manager's app service.

<application> is the vanadium object name of the application.
`,
}

func runInstall(cmd *cmdline.Command, args []string) error {
	if expected, got := 2, len(args); expected != got {
		return cmd.UsageErrorf("install: incorrect number of arguments, expected %d, got %d", expected, got)
	}
	deviceName, appName := args[0], args[1]
	appID, err := device.ApplicationClient(deviceName).Install(gctx, appName, device.Config(configOverride), application.Packages(packagesOverride))
	// Reset the value for any future invocations of "install" or
	// "install-local" (we run more than one command per process in unit
	// tests).
	configOverride = configFlag{}
	packagesOverride = packagesFlag{}
	if err != nil {
		return fmt.Errorf("Install failed: %v", err)
	}
	fmt.Fprintf(cmd.Stdout(), "Successfully installed: %q\n", naming.Join(deviceName, appID))
	return nil
}

var cmdUninstall = &cmdline.Command{
	Run:      runUninstall,
	Name:     "uninstall",
	Short:    "Uninstall the given application installation.",
	Long:     "Uninstall the given application installation.",
	ArgsName: "<installation>",
	ArgsLong: `
<installation> is the vanadium object name of the application installation to
uninstall.
`,
}

func runUninstall(cmd *cmdline.Command, args []string) error {
	if expected, got := 1, len(args); expected != got {
		return cmd.UsageErrorf("uninstall: incorrect number of arguments, expected %d, got %d", expected, got)
	}
	installName := args[0]
	if err := device.ApplicationClient(installName).Uninstall(gctx); err != nil {
		return fmt.Errorf("Uninstall failed: %v", err)
	}
	fmt.Fprintf(cmd.Stdout(), "Successfully uninstalled: %q\n", installName)
	return nil
}

var cmdStart = &cmdline.Command{
	Run:      runStart,
	Name:     "start",
	Short:    "Start an instance of the given application.",
	Long:     "Start an instance of the given application.",
	ArgsName: "<application installation> <grant extension>",
	ArgsLong: `
<application installation> is the vanadium object name of the
application installation from which to start an instance.

<grant extension> is used to extend the default blessing of the
current principal when blessing the app instance.`,
}

type granter struct {
	rpc.CallOpt
	p         security.Principal
	extension string
}

func (g *granter) Grant(other security.Blessings) (security.Blessings, error) {
	return g.p.Bless(other.PublicKey(), g.p.BlessingStore().Default(), g.extension, security.UnconstrainedUse())
}

func runStart(cmd *cmdline.Command, args []string) error {
	if expected, got := 2, len(args); expected != got {
		return cmd.UsageErrorf("start: incorrect number of arguments, expected %d, got %d", expected, got)
	}
	appInstallation, grant := args[0], args[1]

	ctx, cancel := context.WithCancel(gctx)
	defer cancel()
	principal := v23.GetPrincipal(ctx)

	call, err := device.ApplicationClient(appInstallation).Start(ctx)
	if err != nil {
		return fmt.Errorf("Start failed: %v", err)
	}
	var appInstanceIDs []string
	for call.RecvStream().Advance() {
		switch msg := call.RecvStream().Value().(type) {
		case device.StartServerMessageInstanceName:
			appInstanceIDs = append(appInstanceIDs, msg.Value)
		case device.StartServerMessageInstancePublicKey:
			pubKey, err := security.UnmarshalPublicKey(msg.Value)
			if err != nil {
				return fmt.Errorf("Start failed: %v", err)
			}
			// TODO(caprita,rthellend): Get rid of security.UnconstrainedUse().
			blessings, err := principal.Bless(pubKey, principal.BlessingStore().Default(), grant, security.UnconstrainedUse())
			if err != nil {
				return fmt.Errorf("Start failed: %v", err)
			}
			call.SendStream().Send(device.StartClientMessageAppBlessings{blessings})
		default:
			fmt.Fprintf(cmd.Stderr(), "Received unexpected message: %#v\n", msg)
		}
	}
	if err := call.Finish(); err != nil {
		if len(appInstanceIDs) == 0 {
			return fmt.Errorf("Start failed: %v", err)
		} else {
			return fmt.Errorf(
				"Start failed: %v,\nView log with:\n debug logs read `debug glob %s/logs/STDERR-*`",
				err, naming.Join(appInstallation, appInstanceIDs[0]))
		}

	}
	for _, id := range appInstanceIDs {
		fmt.Fprintf(cmd.Stdout(), "Successfully started: %q\n", naming.Join(appInstallation, id))
	}
	return nil
}

var cmdClaim = &cmdline.Command{
	Run:      runClaim,
	Name:     "claim",
	Short:    "Claim the device.",
	Long:     "Claim the device.",
	ArgsName: "<device> <grant extension> <pairing token> <device publickey>",
	ArgsLong: `
<device> is the vanadium object name of the device manager's device service.

<grant extension> is used to extend the default blessing of the
current principal when blessing the app instance.

<pairing token> is a token that the device manager expects to be replayed
during a claim operation on the device.

<device publickey> is the marshalled public key of the device manager we
are claiming.`,
}

func runClaim(cmd *cmdline.Command, args []string) error {
	if expected, max, got := 2, 4, len(args); expected > got || got > max {
		return cmd.UsageErrorf("claim: incorrect number of arguments, expected atleast %d (max: %d), got %d", expected, max, got)
	}
	deviceName, grant := args[0], args[1]
	var pairingToken string
	if len(args) > 2 {
		pairingToken = args[2]
	}
	var serverKeyOpts rpc.CallOpt
	if len(args) > 3 {
		marshalledPublicKey, err := base64.URLEncoding.DecodeString(args[3])
		if err != nil {
			return fmt.Errorf("Failed to base64 decode publickey: %v", err)
		}
		if deviceKey, err := security.UnmarshalPublicKey(marshalledPublicKey); err != nil {
			return fmt.Errorf("Failed to unmarshal device public key:%v", err)
		} else {
			serverKeyOpts = options.ServerPublicKey{deviceKey}
		}
	}
	// Skip server endpoint authorization since an unclaimed device might have
	// roots that will not be recognized by the claimer.
	if err := device.ClaimableClient(deviceName).Claim(gctx, pairingToken, &granter{p: v23.GetPrincipal(gctx), extension: grant}, serverKeyOpts, options.SkipServerEndpointAuthorization{}); err != nil {
		return err
	}
	fmt.Fprintln(cmd.Stdout(), "Successfully claimed.")
	return nil
}

var cmdDescribe = &cmdline.Command{
	Run:      runDescribe,
	Name:     "describe",
	Short:    "Describe the device.",
	Long:     "Describe the device.",
	ArgsName: "<device>",
	ArgsLong: `
<device> is the vanadium object name of the device manager's device service.`,
}

func runDescribe(cmd *cmdline.Command, args []string) error {
	if expected, got := 1, len(args); expected != got {
		return cmd.UsageErrorf("describe: incorrect number of arguments, expected %d, got %d", expected, got)
	}
	deviceName := args[0]
	if description, err := device.DeviceClient(deviceName).Describe(gctx); err != nil {
		return fmt.Errorf("Describe failed: %v", err)
	} else {
		fmt.Fprintf(cmd.Stdout(), "%+v\n", description)
	}
	return nil
}

var cmdUpdate = &cmdline.Command{
	Run:      runUpdate,
	Name:     "update",
	Short:    "Update the device manager or application",
	Long:     "Update the device manager or application",
	ArgsName: "<object>",
	ArgsLong: `
<object> is the vanadium object name of the device manager or application
installation or instance to update.`,
}

func runUpdate(cmd *cmdline.Command, args []string) error {
	if expected, got := 1, len(args); expected != got {
		return cmd.UsageErrorf("update: incorrect number of arguments, expected %d, got %d", expected, got)
	}
	name := args[0]
	if err := device.ApplicationClient(name).Update(gctx); err != nil {
		return err
	}
	fmt.Fprintln(cmd.Stdout(), "Update successful.")
	return nil
}

var cmdRevert = &cmdline.Command{
	Run:      runRevert,
	Name:     "revert",
	Short:    "Revert the device manager or application",
	Long:     "Revert the device manager or application to its previous version",
	ArgsName: "<object>",
	ArgsLong: `
<object> is the vanadium object name of the device manager or application
installation to revert.`,
}

func runRevert(cmd *cmdline.Command, args []string) error {
	if expected, got := 1, len(args); expected != got {
		return cmd.UsageErrorf("revert: incorrect number of arguments, expected %d, got %d", expected, got)
	}
	deviceName := args[0]
	if err := device.ApplicationClient(deviceName).Revert(gctx); err != nil {
		return err
	}
	fmt.Fprintln(cmd.Stdout(), "Revert successful.")
	return nil
}

var cmdDebug = &cmdline.Command{
	Run:      runDebug,
	Name:     "debug",
	Short:    "Debug the device.",
	Long:     "Debug the device.",
	ArgsName: "<app name>",
	ArgsLong: `
<app name> is the vanadium object name of an app installation or instance.`,
}

func runDebug(cmd *cmdline.Command, args []string) error {
	if expected, got := 1, len(args); expected != got {
		return cmd.UsageErrorf("debug: incorrect number of arguments, expected %d, got %d", expected, got)
	}
	appName := args[0]
	if description, err := device.DeviceClient(appName).Debug(gctx); err != nil {
		return fmt.Errorf("Debug failed: %v", err)
	} else {
		fmt.Fprintf(cmd.Stdout(), "%v\n", description)
	}
	return nil
}

var cmdStatus = &cmdline.Command{
	Run:      runStatus,
	Name:     "status",
	Short:    "Get application status.",
	Long:     "Get the status of an application installation or instance.",
	ArgsName: "<app name>",
	ArgsLong: `
<app name> is the vanadium object name of an app installation or instance.`,
}

func runStatus(cmd *cmdline.Command, args []string) error {
	if expected, got := 1, len(args); expected != got {
		return cmd.UsageErrorf("status: incorrect number of arguments, expected %d, got %d", expected, got)
	}
	appName := args[0]
	status, err := device.DeviceClient(appName).Status(gctx)
	if err != nil {
		return fmt.Errorf("Status failed: %v", err)
	}
	switch s := status.(type) {
	case device.StatusInstance:
		fmt.Fprintf(cmd.Stdout(), "Instance [State:%v,Version:%v]", s.Value.State, s.Value.Version)
	case device.StatusInstallation:
		fmt.Fprintf(cmd.Stdout(), "Installation [State:%v,Version:%v]", s.Value.State, s.Value.Version)
	default:
		return fmt.Errorf("Status returned unknown type: %T", s)
	}
	return nil
}
<commit_msg>cmd/mgmt/device: add newline after printing status in device status command.<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"encoding/base64"
	"encoding/json"
	"fmt"

	"v.io/v23"
	"v.io/v23/context"
	"v.io/v23/naming"
	"v.io/v23/options"
	"v.io/v23/rpc"
	"v.io/v23/security"
	"v.io/v23/services/mgmt/application"
	"v.io/v23/services/mgmt/device"
	"v.io/x/lib/cmdline"
)

type configFlag device.Config

func (c *configFlag) String() string {
	jsonConfig, _ := json.Marshal(c)
	return string(jsonConfig)
}
func (c *configFlag) Set(s string) error {
	if err := json.Unmarshal([]byte(s), c); err != nil {
		return fmt.Errorf("Unmarshal(%v) failed: %v", s, err)
	}
	return nil
}

var configOverride configFlag = configFlag{}

type packagesFlag application.Packages

func (c *packagesFlag) String() string {
	jsonPackages, _ := json.Marshal(c)
	return string(jsonPackages)
}
func (c *packagesFlag) Set(s string) error {
	if err := json.Unmarshal([]byte(s), c); err != nil {
		return fmt.Errorf("Unmarshal(%v) failed: %v", s, err)
	}
	return nil
}

var packagesOverride packagesFlag = packagesFlag{}

func init() {
	cmdInstall.Flags.Var(&configOverride, "config", "JSON-encoded device.Config object, of the form: '{\"flag1\":\"value1\",\"flag2\":\"value2\"}'")
	cmdInstall.Flags.Var(&packagesOverride, "packages", "JSON-encoded application.Packages object, of the form: '{\"pkg1\":{\"File\":\"object name 1\"},\"pkg2\":{\"File\":\"object name 2\"}}'")
}

var cmdInstall = &cmdline.Command{
	Run:      runInstall,
	Name:     "install",
	Short:    "Install the given application.",
	Long:     "Install the given application.",
	ArgsName: "<device> <application>",
	ArgsLong: `
<device> is the vanadium object name of the device manager's app service.

<application> is the vanadium object name of the application.
`,
}

func runInstall(cmd *cmdline.Command, args []string) error {
	if expected, got := 2, len(args); expected != got {
		return cmd.UsageErrorf("install: incorrect number of arguments, expected %d, got %d", expected, got)
	}
	deviceName, appName := args[0], args[1]
	appID, err := device.ApplicationClient(deviceName).Install(gctx, appName, device.Config(configOverride), application.Packages(packagesOverride))
	// Reset the value for any future invocations of "install" or
	// "install-local" (we run more than one command per process in unit
	// tests).
	configOverride = configFlag{}
	packagesOverride = packagesFlag{}
	if err != nil {
		return fmt.Errorf("Install failed: %v", err)
	}
	fmt.Fprintf(cmd.Stdout(), "Successfully installed: %q\n", naming.Join(deviceName, appID))
	return nil
}

var cmdUninstall = &cmdline.Command{
	Run:      runUninstall,
	Name:     "uninstall",
	Short:    "Uninstall the given application installation.",
	Long:     "Uninstall the given application installation.",
	ArgsName: "<installation>",
	ArgsLong: `
<installation> is the vanadium object name of the application installation to
uninstall.
`,
}

func runUninstall(cmd *cmdline.Command, args []string) error {
	if expected, got := 1, len(args); expected != got {
		return cmd.UsageErrorf("uninstall: incorrect number of arguments, expected %d, got %d", expected, got)
	}
	installName := args[0]
	if err := device.ApplicationClient(installName).Uninstall(gctx); err != nil {
		return fmt.Errorf("Uninstall failed: %v", err)
	}
	fmt.Fprintf(cmd.Stdout(), "Successfully uninstalled: %q\n", installName)
	return nil
}

var cmdStart = &cmdline.Command{
	Run:      runStart,
	Name:     "start",
	Short:    "Start an instance of the given application.",
	Long:     "Start an instance of the given application.",
	ArgsName: "<application installation> <grant extension>",
	ArgsLong: `
<application installation> is the vanadium object name of the
application installation from which to start an instance.

<grant extension> is used to extend the default blessing of the
current principal when blessing the app instance.`,
}

type granter struct {
	rpc.CallOpt
	p         security.Principal
	extension string
}

func (g *granter) Grant(other security.Blessings) (security.Blessings, error) {
	return g.p.Bless(other.PublicKey(), g.p.BlessingStore().Default(), g.extension, security.UnconstrainedUse())
}

func runStart(cmd *cmdline.Command, args []string) error {
	if expected, got := 2, len(args); expected != got {
		return cmd.UsageErrorf("start: incorrect number of arguments, expected %d, got %d", expected, got)
	}
	appInstallation, grant := args[0], args[1]

	ctx, cancel := context.WithCancel(gctx)
	defer cancel()
	principal := v23.GetPrincipal(ctx)

	call, err := device.ApplicationClient(appInstallation).Start(ctx)
	if err != nil {
		return fmt.Errorf("Start failed: %v", err)
	}
	var appInstanceIDs []string
	for call.RecvStream().Advance() {
		switch msg := call.RecvStream().Value().(type) {
		case device.StartServerMessageInstanceName:
			appInstanceIDs = append(appInstanceIDs, msg.Value)
		case device.StartServerMessageInstancePublicKey:
			pubKey, err := security.UnmarshalPublicKey(msg.Value)
			if err != nil {
				return fmt.Errorf("Start failed: %v", err)
			}
			// TODO(caprita,rthellend): Get rid of security.UnconstrainedUse().
			blessings, err := principal.Bless(pubKey, principal.BlessingStore().Default(), grant, security.UnconstrainedUse())
			if err != nil {
				return fmt.Errorf("Start failed: %v", err)
			}
			call.SendStream().Send(device.StartClientMessageAppBlessings{blessings})
		default:
			fmt.Fprintf(cmd.Stderr(), "Received unexpected message: %#v\n", msg)
		}
	}
	if err := call.Finish(); err != nil {
		if len(appInstanceIDs) == 0 {
			return fmt.Errorf("Start failed: %v", err)
		} else {
			return fmt.Errorf(
				"Start failed: %v,\nView log with:\n debug logs read `debug glob %s/logs/STDERR-*`",
				err, naming.Join(appInstallation, appInstanceIDs[0]))
		}

	}
	for _, id := range appInstanceIDs {
		fmt.Fprintf(cmd.Stdout(), "Successfully started: %q\n", naming.Join(appInstallation, id))
	}
	return nil
}

var cmdClaim = &cmdline.Command{
	Run:      runClaim,
	Name:     "claim",
	Short:    "Claim the device.",
	Long:     "Claim the device.",
	ArgsName: "<device> <grant extension> <pairing token> <device publickey>",
	ArgsLong: `
<device> is the vanadium object name of the device manager's device service.

<grant extension> is used to extend the default blessing of the
current principal when blessing the app instance.

<pairing token> is a token that the device manager expects to be replayed
during a claim operation on the device.

<device publickey> is the marshalled public key of the device manager we
are claiming.`,
}

func runClaim(cmd *cmdline.Command, args []string) error {
	if expected, max, got := 2, 4, len(args); expected > got || got > max {
		return cmd.UsageErrorf("claim: incorrect number of arguments, expected atleast %d (max: %d), got %d", expected, max, got)
	}
	deviceName, grant := args[0], args[1]
	var pairingToken string
	if len(args) > 2 {
		pairingToken = args[2]
	}
	var serverKeyOpts rpc.CallOpt
	if len(args) > 3 {
		marshalledPublicKey, err := base64.URLEncoding.DecodeString(args[3])
		if err != nil {
			return fmt.Errorf("Failed to base64 decode publickey: %v", err)
		}
		if deviceKey, err := security.UnmarshalPublicKey(marshalledPublicKey); err != nil {
			return fmt.Errorf("Failed to unmarshal device public key:%v", err)
		} else {
			serverKeyOpts = options.ServerPublicKey{deviceKey}
		}
	}
	// Skip server endpoint authorization since an unclaimed device might have
	// roots that will not be recognized by the claimer.
	if err := device.ClaimableClient(deviceName).Claim(gctx, pairingToken, &granter{p: v23.GetPrincipal(gctx), extension: grant}, serverKeyOpts, options.SkipServerEndpointAuthorization{}); err != nil {
		return err
	}
	fmt.Fprintln(cmd.Stdout(), "Successfully claimed.")
	return nil
}

var cmdDescribe = &cmdline.Command{
	Run:      runDescribe,
	Name:     "describe",
	Short:    "Describe the device.",
	Long:     "Describe the device.",
	ArgsName: "<device>",
	ArgsLong: `
<device> is the vanadium object name of the device manager's device service.`,
}

func runDescribe(cmd *cmdline.Command, args []string) error {
	if expected, got := 1, len(args); expected != got {
		return cmd.UsageErrorf("describe: incorrect number of arguments, expected %d, got %d", expected, got)
	}
	deviceName := args[0]
	if description, err := device.DeviceClient(deviceName).Describe(gctx); err != nil {
		return fmt.Errorf("Describe failed: %v", err)
	} else {
		fmt.Fprintf(cmd.Stdout(), "%+v\n", description)
	}
	return nil
}

var cmdUpdate = &cmdline.Command{
	Run:      runUpdate,
	Name:     "update",
	Short:    "Update the device manager or application",
	Long:     "Update the device manager or application",
	ArgsName: "<object>",
	ArgsLong: `
<object> is the vanadium object name of the device manager or application
installation or instance to update.`,
}

func runUpdate(cmd *cmdline.Command, args []string) error {
	if expected, got := 1, len(args); expected != got {
		return cmd.UsageErrorf("update: incorrect number of arguments, expected %d, got %d", expected, got)
	}
	name := args[0]
	if err := device.ApplicationClient(name).Update(gctx); err != nil {
		return err
	}
	fmt.Fprintln(cmd.Stdout(), "Update successful.")
	return nil
}

var cmdRevert = &cmdline.Command{
	Run:      runRevert,
	Name:     "revert",
	Short:    "Revert the device manager or application",
	Long:     "Revert the device manager or application to its previous version",
	ArgsName: "<object>",
	ArgsLong: `
<object> is the vanadium object name of the device manager or application
installation to revert.`,
}

func runRevert(cmd *cmdline.Command, args []string) error {
	if expected, got := 1, len(args); expected != got {
		return cmd.UsageErrorf("revert: incorrect number of arguments, expected %d, got %d", expected, got)
	}
	deviceName := args[0]
	if err := device.ApplicationClient(deviceName).Revert(gctx); err != nil {
		return err
	}
	fmt.Fprintln(cmd.Stdout(), "Revert successful.")
	return nil
}

var cmdDebug = &cmdline.Command{
	Run:      runDebug,
	Name:     "debug",
	Short:    "Debug the device.",
	Long:     "Debug the device.",
	ArgsName: "<app name>",
	ArgsLong: `
<app name> is the vanadium object name of an app installation or instance.`,
}

func runDebug(cmd *cmdline.Command, args []string) error {
	if expected, got := 1, len(args); expected != got {
		return cmd.UsageErrorf("debug: incorrect number of arguments, expected %d, got %d", expected, got)
	}
	appName := args[0]
	if description, err := device.DeviceClient(appName).Debug(gctx); err != nil {
		return fmt.Errorf("Debug failed: %v", err)
	} else {
		fmt.Fprintf(cmd.Stdout(), "%v\n", description)
	}
	return nil
}

var cmdStatus = &cmdline.Command{
	Run:      runStatus,
	Name:     "status",
	Short:    "Get application status.",
	Long:     "Get the status of an application installation or instance.",
	ArgsName: "<app name>",
	ArgsLong: `
<app name> is the vanadium object name of an app installation or instance.`,
}

func runStatus(cmd *cmdline.Command, args []string) error {
	if expected, got := 1, len(args); expected != got {
		return cmd.UsageErrorf("status: incorrect number of arguments, expected %d, got %d", expected, got)
	}
	appName := args[0]
	status, err := device.DeviceClient(appName).Status(gctx)
	if err != nil {
		return fmt.Errorf("Status failed: %v", err)
	}
	switch s := status.(type) {
	case device.StatusInstance:
		fmt.Fprintf(cmd.Stdout(), "Instance [State:%v,Version:%v]\n", s.Value.State, s.Value.Version)
	case device.StatusInstallation:
		fmt.Fprintf(cmd.Stdout(), "Installation [State:%v,Version:%v]\n", s.Value.State, s.Value.Version)
	default:
		return fmt.Errorf("Status returned unknown type: %T", s)
	}
	return nil
}
<|endoftext|>


<commit_before>package tiles

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/Luqqk/wms-tiles-downloader/pkg/mercantile"
)

// Options struct stores all available flags
// and their values set by user.
type Options struct {
	URL     string
	Layer   string
	Format  string
	Service string
	Version string
	Width   string
	Height  string
	Srs     string
	Styles  string
	Zooms   Zooms
	Bbox    Bbox
	// If all options are correct,
	// build base URL for all tiles
	// requests.
	BaseURL string
}

// ValidateOptions validates options supplied by user.
// Downloading will start only, if all required options
// have been passed in correct format.
func (options *Options) ValidateOptions() error {
	switch {
	case options.URL == "":
		return errors.New("Wms server url is required")
	case options.Layer == "":
		return errors.New("Layer name is required")
	case options.Zooms == nil:
		return errors.New("Zooms are required")
	case options.Bbox == Bbox{}:
		return errors.New("Bbox is required")
	default:
		options.ParseBaseURL()
		return nil
	}
}

// ParseBaseURL builds base URL for all
// tiles requests based on passed arguments.
func (options *Options) ParseBaseURL() error {
	u, err := url.Parse(options.URL)
	if err != nil {
		return err
	}
	if u.Scheme == "" {
		u.Scheme = "https"
	}
	// Set query parameters.
	q := u.Query()
	q.Set("format", options.Format)
	q.Set("service", options.Service)
	q.Set("version", options.Version)
	q.Set("request", "WMS")
	q.Set("srs", options.Srs)
	q.Set("width", options.Width)
	q.Set("height", options.Height)
	q.Set("layers", options.Layer)
	q.Set("styles", options.Styles)
	// Encode and set BaseURL field of
	// the Options struct.
	u.RawQuery = q.Encode()
	options.BaseURL = u.String()
	return nil
}

// Zooms stores zoom levels, for which
// tiles should be downloaded.
type Zooms []int

// String is the method to format the flag's value, part of the flag.Value interface.
// The String method's output will be used in diagnostics.
func (zooms *Zooms) String() string {
	return fmt.Sprint(*zooms)
}

// Set is the method to set the flag value, part of the flag.Value interface.
// Converts comma-separated values (string in "int,int,int,(...)" format)
// to Zooms type.
func (zooms *Zooms) Set(value string) error {
	for _, val := range strings.Split(value, ",") {
		zoom, err := strconv.Atoi(val)
		if err != nil {
			return err
		}
		*zooms = append(*zooms, zoom)
	}
	return nil
}

// Bbox stores a web mercator bounding box, for which
// tiles should be downloaded.
type Bbox struct {
	Left   float64
	Bottom float64
	Right  float64
	Top    float64
}

// String is the method to format the flag's value, part of the flag.Value interface.
// The String method's output will be used in diagnostics.
func (bbox *Bbox) String() string {
	return fmt.Sprint(*bbox)
}

// Set is the method to set the flag value, part of the flag.Value interface.
// Converts comma-separated values (string in "left,bottom,right,top" format)
// to Bbox struct.
func (bbox *Bbox) Set(value string) error {
	bboxSlice := strings.Split(value, ",")
	left, _ := strconv.ParseFloat(bboxSlice[0], 64)
	bottom, _ := strconv.ParseFloat(bboxSlice[1], 64)
	right, _ := strconv.ParseFloat(bboxSlice[2], 64)
	top, _ := strconv.ParseFloat(bboxSlice[3], 64)
	*bbox = Bbox{Left: left, Bottom: bottom, Right: right, Top: top}
	return nil
}

// Create a Client for control over HTTP client settings.
// Client is safe for concurrent use by multiple goroutines
// and for efficiency should only be created once and re-used.
var client = &http.Client{
	Timeout: time.Second * 30,
}

// Tile contains content received from WMS server
// and other metadata about tile itself.
// For example tile's path in z/x/y tree.
type Tile struct {
	Content []byte
	Path    string
}

// Get sends http.Get request to WMS Server
// and returns response content.
func Get(tileID mercantile.TileID, options Options) (*Tile, error) {
	// Parse base url and format it
	// with the bbox of the tile.
	// Bbox is calculated by using
	// current tile's id (z/x/y).
	url, err := url.Parse(options.BaseURL)
	if err != nil {
		return &Tile{}, err
	}
	q := url.Query()
	q.Set("BBOX", FormatTileBbox(tileID))
	url.RawQuery = q.Encode()
	// Request tile using defined client,
	// read response body.
	resp, err := client.Get(url.String())
	if err != nil {
		return &Tile{}, err
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return &Tile{}, err
	}
	// Create Tile struct,
	// return pointer.
	tile := &Tile{
		Content: body,
		// TODO: File extension (".png" part) should be parsed
		// dynamically, based on --format parameter supplied by
		// the user. 'image/png' is default.
		Path: fmt.Sprintf("%v/%v/%v.png", tileID.Z, tileID.X, tileID.Y),
	}
	resp.Body.Close()
	return tile, nil
}

// FormatTileBbox converts tile (x, y, z) to bbox string (l,b,r,t)
func FormatTileBbox(tileID mercantile.TileID) string {
	bbox := mercantile.XyBounds(tileID)
	formattedBbox := fmt.Sprintf("%.9f,%.9f,%.9f,%.9f", bbox.Left, bbox.Bottom, bbox.Right, bbox.Top)
	return formattedBbox
}
<commit_msg>Add Save() method to save tiles on disk, new 'Name' field in Tile struct<commit_after>package tiles

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/Luqqk/wms-tiles-downloader/pkg/mercantile"
)

// Options struct stores all available flags
// and their values set by user.
type Options struct {
	URL     string
	Layer   string
	Format  string
	Service string
	Version string
	Width   string
	Height  string
	Srs     string
	Styles  string
	Zooms   Zooms
	Bbox    Bbox
	// If all options are correct,
	// build base URL for all tiles
	// requests.
	BaseURL string
}

// ValidateOptions validates options supplied by user.
// Downloading will start only, if all required options
// have been passed in correct format.
func (options *Options) ValidateOptions() error {
	switch {
	case options.URL == "":
		return errors.New("Wms server url is required")
	case options.Layer == "":
		return errors.New("Layer name is required")
	case options.Zooms == nil:
		return errors.New("Zooms are required")
	case options.Bbox == Bbox{}:
		return errors.New("Bbox is required")
	default:
		options.ParseBaseURL()
		return nil
	}
}

// ParseBaseURL builds base URL for all
// tiles requests based on passed arguments.
func (options *Options) ParseBaseURL() error {
	u, err := url.Parse(options.URL)
	if err != nil {
		return err
	}
	if u.Scheme == "" {
		u.Scheme = "https"
	}
	// Set query parameters.
	q := u.Query()
	q.Set("format", options.Format)
	q.Set("service", options.Service)
	q.Set("version", options.Version)
	q.Set("request", "WMS")
	q.Set("srs", options.Srs)
	q.Set("width", options.Width)
	q.Set("height", options.Height)
	q.Set("layers", options.Layer)
	q.Set("styles", options.Styles)
	// Encode and set BaseURL field of
	// the Options struct.
	u.RawQuery = q.Encode()
	options.BaseURL = u.String()
	return nil
}

// Zooms stores zoom levels, for which
// tiles should be downloaded.
type Zooms []int

// String is the method to format the flag's value, part of the flag.Value interface.
// The String method's output will be used in diagnostics.
func (zooms *Zooms) String() string {
	return fmt.Sprint(*zooms)
}

// Set is the method to set the flag value, part of the flag.Value interface.
// Converts comma-separated values (string in "int,int,int,(...)" format)
// to Zooms type.
func (zooms *Zooms) Set(value string) error {
	for _, val := range strings.Split(value, ",") {
		zoom, err := strconv.Atoi(val)
		if err != nil {
			return err
		}
		*zooms = append(*zooms, zoom)
	}
	return nil
}

// Bbox stores a web mercator bounding box, for which
// tiles should be downloaded.
type Bbox struct {
	Left   float64
	Bottom float64
	Right  float64
	Top    float64
}

// String is the method to format the flag's value, part of the flag.Value interface.
// The String method's output will be used in diagnostics.
func (bbox *Bbox) String() string {
	return fmt.Sprint(*bbox)
}

// Set is the method to set the flag value, part of the flag.Value interface.
// Converts comma-separated values (string in "left,bottom,right,top" format)
// to Bbox struct.
func (bbox *Bbox) Set(value string) error {
	bboxSlice := strings.Split(value, ",")
	left, _ := strconv.ParseFloat(bboxSlice[0], 64)
	bottom, _ := strconv.ParseFloat(bboxSlice[1], 64)
	right, _ := strconv.ParseFloat(bboxSlice[2], 64)
	top, _ := strconv.ParseFloat(bboxSlice[3], 64)
	*bbox = Bbox{Left: left, Bottom: bottom, Right: right, Top: top}
	return nil
}

// Create a Client for control over HTTP client settings.
// Client is safe for concurrent use by multiple goroutines
// and for efficiency should only be created once and re-used.
var client = &http.Client{
	Timeout: time.Second * 30,
}

// Tile contains content received from WMS server
// and other metadata about tile itself. For example
// tile's path in z/x tree, name under which the tile
// will be saved (y.png).
type Tile struct {
	Content []byte
	Path    string
	Name    string
}

// Get sends http.Get request to WMS Server
// and returns response content.
func Get(tileID mercantile.TileID, options Options) (*Tile, error) {
	// Parse base url and format it
	// with the bbox of the tile.
	// Bbox is calculated by using
	// current tile's id (z/x/y).
	url, err := url.Parse(options.BaseURL)
	if err != nil {
		return &Tile{}, err
	}
	q := url.Query()
	q.Set("BBOX", FormatTileBbox(tileID))
	url.RawQuery = q.Encode()
	// Request tile using defined client,
	// read response body.
	resp, err := client.Get(url.String())
	if err != nil {
		return &Tile{}, err
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return &Tile{}, err
	}
	// Create Tile struct,
	// return pointer.
	tile := &Tile{
		Content: body,
		Path:    fmt.Sprintf("%v/%v", tileID.Z, tileID.X),
		// TODO: File extension (".png" part) should be parsed
		// dynamically, based on --format parameter supplied by
		// the user. 'image/png' is default.
		Name: fmt.Sprintf("%v.png", tileID.Y),
	}
	resp.Body.Close()
	return tile, nil
}

// Save saves the tile passed in
// argument on hard drive.
func Save(tile Tile) error {
	err := os.MkdirAll(tile.Path, os.ModePerm)
	filepath := path.Join(tile.Path, tile.Name)
	err = ioutil.WriteFile(filepath, tile.Content, os.ModePerm)
	return err
}

// FormatTileBbox converts tile (x, y, z) to bbox string (l,b,r,t)
func FormatTileBbox(tileID mercantile.TileID) string {
	bbox := mercantile.XyBounds(tileID)
	formattedBbox := fmt.Sprintf("%.9f,%.9f,%.9f,%.9f", bbox.Left, bbox.Bottom, bbox.Right, bbox.Top)
	return formattedBbox
}
<|endoftext|>


<commit_before><commit_msg>Fixed the issue to  omit network address and broadcast address<commit_after><|endoftext|>


<commit_before>package cache

import (
    "time"
    "github.com/ridewindx/mel"
    "crypto/sha1"
    "io"
    "fmt"
    "net/http"
    "bytes"
)

type responseCache struct {
    status int
    header http.Header
    body []byte
}

type cachedWriter struct {
    mel.ResponseWriter

    key string
    expire time.Duration

    error

    bytes.Buffer
    *Cache
}

func (w *cachedWriter) Write(bytes []byte) (int, error) {
    size, err := w.ResponseWriter.Write(bytes)
    if err == nil {
        w.Buffer.Write(bytes)
    } else {
        w.error = err
    }
    return size, err
}

func (w *cachedWriter) cache() {
    if w.error == nil {
        // TODO
        return
    }
    rc := responseCache{
        status: w.Status(),
        header: w.Header(),
        body: w.Buffer.Bytes(),
    }
    err := w.Cache.Set(w.key, rc, w.expire)
    if err != nil {
        // TODO
    }
}

type Cache struct {
    Backend

    KeyPrefix string

    Store
}

func New(backend Backend) *Cache {
    return &Cache{
        Backend: backend,
    }
}

func (cache *Cache) CacheMiddleware(expire time.Duration) mel.Handler {
    return func(c *mel.Context) {

    }
}

func (cache *Cache) Cache(expire time.Duration, handler mel.Handler) mel.Handler {
    return func(c *mel.Context) {
        key := cache.makeKey(c.Request.URL.RequestURI())
        value, err := cache.Get(key)
        if err != nil {
            cw := &cachedWriter{
                ResponseWriter: c.Writer,
                key: key,
                expire: expire,
                Cache: cache,
            }
            c.Writer = cw
            handler(c)
            cw.cache()
        } else {
            r := value.(responseCache)
            c.Status(r.status)
            for k, v := range r.header {
                c.Writer.Header()[k] = v
            }
            c.Writer.Write(r.body)
        }
    }
}

func (cache *Cache) makeKey(url string) string {
    key := url
    if len(key) > 200 {
        h := sha1.New()
        io.WriteString(h, key)
        key = string(h.Sum(nil))
    }
    return fmt.Sprintf("%s:%s", cache.KeyPrefix, key)
}
<commit_msg>Update<commit_after>package cache

import (
    "time"
    "github.com/ridewindx/mel"
    "crypto/sha1"
    "io"
    "fmt"
    "net/http"
    "bytes"
)

type responseCache struct {
    status int
    header http.Header
    body []byte
}

type cachedWriter struct {
    mel.ResponseWriter

    key string
    expire time.Duration

    error

    bytes.Buffer
    *Cache
}

func (w *cachedWriter) Write(bytes []byte) (int, error) {
    size, err := w.ResponseWriter.Write(bytes)
    if err == nil {
        w.Buffer.Write(bytes)
    } else {
        w.error = err
    }
    return size, err
}

func (w *cachedWriter) cache() {
    if w.error == nil {
        // TODO
        return
    }
    rc := responseCache{
        status: w.Status(),
        header: w.Header(),
        body: w.Buffer.Bytes(),
    }
    err := w.Cache.Set(w.key, rc, w.expire)
    if err != nil {
        // TODO
    }
}

type Cache struct {
    Backend

    KeyPrefix string

    Store
}

func New(backend Backend) *Cache {
    return &Cache{
        Backend: backend,
    }
}

func (cache *Cache) CacheMiddleware(expire time.Duration) mel.Handler {
    return func(c *mel.Context) {

    }
}

func (cache *Cache) Cache(expire time.Duration, handler mel.Handler) mel.Handler {
    return func(c *mel.Context) {
        key := cache.makeKey(c.Request.URL.RequestURI())
        var r responseCache
        err := cache.Get(key, &r)
        if err != nil {
            cw := &cachedWriter{
                ResponseWriter: c.Writer,
                key: key,
                expire: expire,
                Cache: cache,
            }
            c.Writer = cw
            handler(c)
            cw.cache()
        } else {
            c.Status(r.status)
            for k, v := range r.header {
                c.Writer.Header()[k] = v
            }
            c.Writer.Write(r.body)
        }
    }
}

func (cache *Cache) makeKey(url string) string {
    key := url
    if len(key) > 200 {
        h := sha1.New()
        io.WriteString(h, key)
        key = string(h.Sum(nil))
    }
    return fmt.Sprintf("%s:%s", cache.KeyPrefix, key)
}
<|endoftext|>