<commit_before>package main

import (
	"fmt"
	"os"
	"strings"
	"text/template"

	is "github.com/containers/image/storage"
	"github.com/containers/storage"
	"github.com/pkg/errors"
	"github.com/urfave/cli"
)

type imageOutputParams struct {
	ID        string
	Name      string
	Digest    string
	CreatedAt string
	Size      string
}

type filterParams struct {
	dangling         string
	label            string
	beforeImage      string // Images are sorted by date, so we can just output until we see the image
	sinceImage       string // Images are sorted by date, so we can just output until we don't see the image
	seenImage        bool   // Hence this boolean
	referencePattern string
}

var (
	imagesFlags = []cli.Flag{
		cli.BoolFlag{
			Name:  "quiet, q",
			Usage: "display only image IDs",
		},
		cli.BoolFlag{
			Name:  "noheading, n",
			Usage: "do not print column headings",
		},
		cli.BoolFlag{
			Name:  "no-trunc, notruncate",
			Usage: "do not truncate output",
		},
		cli.BoolFlag{
			Name:  "digests",
			Usage: "show digests",
		},
		cli.StringFlag{
			Name:  "format",
			Usage: "pretty-print images using a Go template. will override --quiet",
		},
		cli.StringFlag{
			Name:  "filter, f",
			Usage: "filter output based on conditions provided (default [])",
		},
	}

	imagesDescription = "lists locally stored images."
	imagesCommand     = cli.Command{
		Name:        "images",
		Usage:       "list images in local storage",
		Description: imagesDescription,
		Flags:       imagesFlags,
		Action:      imagesCmd,
		ArgsUsage:   "",
	}
)

func imagesCmd(c *cli.Context) error {
	store, err := getStore(c)
	if err != nil {
		return err
	}

	quiet := false
	if c.IsSet("quiet") {
		quiet = c.Bool("quiet")
	}
	noheading := false
	if c.IsSet("noheading") {
		noheading = c.Bool("noheading")
	}
	truncate := true
	if c.IsSet("no-trunc") {
		truncate = !c.Bool("no-trunc")
	}
	digests := false
	if c.IsSet("digests") {
		digests = c.Bool("digests")
	}
	formatString := ""
	hasTemplate := false
	if c.IsSet("format") {
		formatString = c.String("format")
		hasTemplate = true
	}

	name := ""
	if len(c.Args()) == 1 {
		name = c.Args().Get(0)
	} else if len(c.Args()) > 1 {
		return errors.New("'buildah images' requires at most 1 argument")
	}

	images, err := store.Images()
	if err != nil {
		return errors.Wrapf(err, "error reading images")
	}

	var params *filterParams
	if c.IsSet("filter") {
		params, err = parseFilter(images, c.String("filter"))
		if err != nil {
			return errors.Wrapf(err, "error parsing filter")
		}
	} else {
		params = nil
	}

	if len(images) > 0 && !noheading && !quiet && !hasTemplate {
		outputHeader(truncate, digests)
	}

	return outputImages(images, formatString, store, params, name, hasTemplate, truncate, digests, quiet)
}

func parseFilter(images []storage.Image, filter string) (*filterParams, error) {
	params := new(filterParams)
	filterStrings := strings.Split(filter, ",")
	for _, param := range filterStrings {
		pair := strings.SplitN(param, "=", 2)
		switch strings.TrimSpace(pair[0]) {
		case "dangling":
			if pair[1] == "true" || pair[1] == "false" {
				params.dangling = pair[1]
			} else {
				return nil, fmt.Errorf("invalid filter: '%s=[%s]'", pair[0], pair[1])
			}
		case "label":
			params.label = pair[1]
		case "before":
			if imageExists(images, pair[1]) {
				params.beforeImage = pair[1]
			} else {
				return nil, fmt.Errorf("no such id: %s", pair[0])
			}
		case "since":
			if imageExists(images, pair[1]) {
				params.sinceImage = pair[1]
			} else {
				return nil, fmt.Errorf("no such id: %s``", pair[0])
			}
		case "reference":
			params.referencePattern = pair[1]
		default:
			return nil, fmt.Errorf("invalid filter: '%s'", pair[0])
		}
	}
	return params, nil
}

func imageExists(images []storage.Image, ref string) bool {
	for _, image := range images {
		if matchesID(image.ID, ref) {
			return true
		}
		for _, name := range image.Names {
			if matchesReference(name, ref) {
				return true
			}
		}
	}
	return false
}

func outputHeader(truncate, digests bool) {
	if truncate {
		fmt.Printf("%-20s %-56s ", "IMAGE ID", "IMAGE NAME")
	} else {
		fmt.Printf("%-64s %-56s ", "IMAGE ID", "IMAGE NAME")
	}

	if digests {
		fmt.Printf("%-64s ", "DIGEST")
	}

	fmt.Printf("%-22s %s\n", "CREATED AT", "SIZE")
}

func outputImages(images []storage.Image, format string, store storage.Store, filters *filterParams, argName string, hasTemplate, truncate, digests, quiet bool) error {
	for _, image := range images {
		imageMetadata, err := parseMetadata(image)
		if err != nil {
			fmt.Println(err)
		}
		createdTime := imageMetadata.CreatedTime.Format("Jan 2, 2006 15:04")
		digest := ""
		if len(imageMetadata.Blobs) > 0 {
			digest = string(imageMetadata.Blobs[0].Digest)
		}
		size, _ := getSize(image, store)

		names := []string{""}
		if len(image.Names) > 0 {
			names = image.Names
		} else {
			// images without names should be printed with "<none>" as the image name
			names = append(names, "<none>")
		}
		for _, name := range names {
			if !matchesFilter(image, store, name, filters) || !matchesReference(name, argName) {
				continue
			}
			if quiet {
				fmt.Printf("%-64s\n", image.ID)
				// We only want to print each id once
				break
			}

			params := imageOutputParams{
				ID:        image.ID,
				Name:      name,
				Digest:    digest,
				CreatedAt: createdTime,
				Size:      formattedSize(size),
			}
			if hasTemplate {
				err = outputUsingTemplate(format, params)
				if err != nil {
					return err
				}
				continue
			}

			outputUsingFormatString(truncate, digests, params)
		}
	}
	return nil
}

func matchesFilter(image storage.Image, store storage.Store, name string, params *filterParams) bool {
	if params == nil {
		return true
	}
	if params.dangling != "" && !matchesDangling(name, params.dangling) {
		return false
	} else if params.label != "" && !matchesLabel(image, store, params.label) {
		return false
	} else if params.beforeImage != "" && !matchesBeforeImage(image, name, params) {
		return false
	} else if params.sinceImage != "" && !matchesSinceImage(image, name, params) {
		return false
	} else if params.referencePattern != "" && !matchesReference(name, params.referencePattern) {
		return false
	}
	return true
}

func matchesDangling(name string, dangling string) bool {
	if dangling == "false" && name != "<none>" {
		return true
	} else if dangling == "true" && name == "<none>" {
		return true
	}
	return false
}

func matchesLabel(image storage.Image, store storage.Store, label string) bool {
	storeRef, err := is.Transport.ParseStoreReference(store, "@"+image.ID)
	if err != nil {

	}
	img, err := storeRef.NewImage(nil)
	if err != nil {
		return false
	}
	info, err := img.Inspect()
	if err != nil {
		return false
	}

	pair := strings.SplitN(label, "=", 2)
	for key, value := range info.Labels {
		if key == pair[0] {
			if len(pair) == 2 {
				if value == pair[1] {
					return true
				}
			} else {
				return false
			}
		}
	}
	return false
}

// Returns true if the image was created since the filter image.  Returns
// false otherwise
func matchesBeforeImage(image storage.Image, name string, params *filterParams) bool {
	if params.seenImage {
		return false
	}
	if matchesReference(name, params.beforeImage) || matchesID(image.ID, params.beforeImage) {
		params.seenImage = true
		return false
	}
	return true
}

// Returns true if the image was created since the filter image.  Returns
// false otherwise
func matchesSinceImage(image storage.Image, name string, params *filterParams) bool {
	if params.seenImage {
		return true
	}
	if matchesReference(name, params.sinceImage) || matchesID(image.ID, params.sinceImage) {
		params.seenImage = true
	}
	return false
}

func matchesID(id, argID string) bool {
	return strings.HasPrefix(argID, id)
}

func matchesReference(name, argName string) bool {
	if argName == "" {
		return true
	}
	splitName := strings.Split(name, ":")
	// If the arg contains a tag, we handle it differently than if it does not
	if strings.Contains(argName, ":") {
		splitArg := strings.Split(argName, ":")
		return strings.HasSuffix(splitName[0], splitArg[0]) && (splitName[1] == splitArg[1])
	}
	return strings.HasSuffix(splitName[0], argName)
}

func formattedSize(size int64) string {
	suffixes := [5]string{"B", "KB", "MB", "GB", "TB"}

	count := 0
	formattedSize := float64(size)
	for formattedSize >= 1024 && count < 4 {
		formattedSize /= 1024
		count++
	}
	return fmt.Sprintf("%.4g %s", formattedSize, suffixes[count])
}

func outputUsingTemplate(format string, params imageOutputParams) error {
	tmpl, err := template.New("image").Parse(format)
	if err != nil {
		return errors.Wrapf(err, "Template parsing error")
	}

	err = tmpl.Execute(os.Stdout, params)
	if err != nil {
		return err
	}
	fmt.Println()
	return nil
}

func outputUsingFormatString(truncate, digests bool, params imageOutputParams) {
	if truncate {
		fmt.Printf("%-20.12s %-56s", params.ID, params.Name)
	} else {
		fmt.Printf("%-64s %-56s", params.ID, params.Name)
	}

	if digests {
		fmt.Printf(" %-64s", params.Digest)
	}
	fmt.Printf(" %-22s %s\n", params.CreatedAt, params.Size)
}
<commit_msg>kpod images --digests output align<commit_after>package main

import (
	"fmt"
	"os"
	"strings"
	"text/template"

	is "github.com/containers/image/storage"
	"github.com/containers/storage"
	"github.com/pkg/errors"
	"github.com/urfave/cli"
)

type imageOutputParams struct {
	ID        string
	Name      string
	Digest    string
	CreatedAt string
	Size      string
}

type filterParams struct {
	dangling         string
	label            string
	beforeImage      string // Images are sorted by date, so we can just output until we see the image
	sinceImage       string // Images are sorted by date, so we can just output until we don't see the image
	seenImage        bool   // Hence this boolean
	referencePattern string
}

var (
	imagesFlags = []cli.Flag{
		cli.BoolFlag{
			Name:  "quiet, q",
			Usage: "display only image IDs",
		},
		cli.BoolFlag{
			Name:  "noheading, n",
			Usage: "do not print column headings",
		},
		cli.BoolFlag{
			Name:  "no-trunc, notruncate",
			Usage: "do not truncate output",
		},
		cli.BoolFlag{
			Name:  "digests",
			Usage: "show digests",
		},
		cli.StringFlag{
			Name:  "format",
			Usage: "pretty-print images using a Go template. will override --quiet",
		},
		cli.StringFlag{
			Name:  "filter, f",
			Usage: "filter output based on conditions provided (default [])",
		},
	}

	imagesDescription = "lists locally stored images."
	imagesCommand     = cli.Command{
		Name:        "images",
		Usage:       "list images in local storage",
		Description: imagesDescription,
		Flags:       imagesFlags,
		Action:      imagesCmd,
		ArgsUsage:   "",
	}
)

func imagesCmd(c *cli.Context) error {
	store, err := getStore(c)
	if err != nil {
		return err
	}

	quiet := false
	if c.IsSet("quiet") {
		quiet = c.Bool("quiet")
	}
	noheading := false
	if c.IsSet("noheading") {
		noheading = c.Bool("noheading")
	}
	truncate := true
	if c.IsSet("no-trunc") {
		truncate = !c.Bool("no-trunc")
	}
	digests := false
	if c.IsSet("digests") {
		digests = c.Bool("digests")
	}
	formatString := ""
	hasTemplate := false
	if c.IsSet("format") {
		formatString = c.String("format")
		hasTemplate = true
	}

	name := ""
	if len(c.Args()) == 1 {
		name = c.Args().Get(0)
	} else if len(c.Args()) > 1 {
		return errors.New("'buildah images' requires at most 1 argument")
	}

	images, err := store.Images()
	if err != nil {
		return errors.Wrapf(err, "error reading images")
	}

	var params *filterParams
	if c.IsSet("filter") {
		params, err = parseFilter(images, c.String("filter"))
		if err != nil {
			return errors.Wrapf(err, "error parsing filter")
		}
	} else {
		params = nil
	}

	if len(images) > 0 && !noheading && !quiet && !hasTemplate {
		outputHeader(truncate, digests)
	}

	return outputImages(images, formatString, store, params, name, hasTemplate, truncate, digests, quiet)
}

func parseFilter(images []storage.Image, filter string) (*filterParams, error) {
	params := new(filterParams)
	filterStrings := strings.Split(filter, ",")
	for _, param := range filterStrings {
		pair := strings.SplitN(param, "=", 2)
		switch strings.TrimSpace(pair[0]) {
		case "dangling":
			if pair[1] == "true" || pair[1] == "false" {
				params.dangling = pair[1]
			} else {
				return nil, fmt.Errorf("invalid filter: '%s=[%s]'", pair[0], pair[1])
			}
		case "label":
			params.label = pair[1]
		case "before":
			if imageExists(images, pair[1]) {
				params.beforeImage = pair[1]
			} else {
				return nil, fmt.Errorf("no such id: %s", pair[0])
			}
		case "since":
			if imageExists(images, pair[1]) {
				params.sinceImage = pair[1]
			} else {
				return nil, fmt.Errorf("no such id: %s``", pair[0])
			}
		case "reference":
			params.referencePattern = pair[1]
		default:
			return nil, fmt.Errorf("invalid filter: '%s'", pair[0])
		}
	}
	return params, nil
}

func imageExists(images []storage.Image, ref string) bool {
	for _, image := range images {
		if matchesID(image.ID, ref) {
			return true
		}
		for _, name := range image.Names {
			if matchesReference(name, ref) {
				return true
			}
		}
	}
	return false
}

func outputHeader(truncate, digests bool) {
	if truncate {
		fmt.Printf("%-20s %-56s ", "IMAGE ID", "IMAGE NAME")
	} else {
		fmt.Printf("%-64s %-56s ", "IMAGE ID", "IMAGE NAME")
	}

	if digests {
		fmt.Printf("%-71s ", "DIGEST")
	}

	fmt.Printf("%-22s %s\n", "CREATED AT", "SIZE")
}

func outputImages(images []storage.Image, format string, store storage.Store, filters *filterParams, argName string, hasTemplate, truncate, digests, quiet bool) error {
	for _, image := range images {
		imageMetadata, err := parseMetadata(image)
		if err != nil {
			fmt.Println(err)
		}
		createdTime := imageMetadata.CreatedTime.Format("Jan 2, 2006 15:04")
		digest := ""
		if len(imageMetadata.Blobs) > 0 {
			digest = string(imageMetadata.Blobs[0].Digest)
		}
		size, _ := getSize(image, store)

		names := []string{""}
		if len(image.Names) > 0 {
			names = image.Names
		} else {
			// images without names should be printed with "<none>" as the image name
			names = append(names, "<none>")
		}
		for _, name := range names {
			if !matchesFilter(image, store, name, filters) || !matchesReference(name, argName) {
				continue
			}
			if quiet {
				fmt.Printf("%-64s\n", image.ID)
				// We only want to print each id once
				break
			}

			params := imageOutputParams{
				ID:        image.ID,
				Name:      name,
				Digest:    digest,
				CreatedAt: createdTime,
				Size:      formattedSize(size),
			}
			if hasTemplate {
				err = outputUsingTemplate(format, params)
				if err != nil {
					return err
				}
				continue
			}

			outputUsingFormatString(truncate, digests, params)
		}
	}
	return nil
}

func matchesFilter(image storage.Image, store storage.Store, name string, params *filterParams) bool {
	if params == nil {
		return true
	}
	if params.dangling != "" && !matchesDangling(name, params.dangling) {
		return false
	} else if params.label != "" && !matchesLabel(image, store, params.label) {
		return false
	} else if params.beforeImage != "" && !matchesBeforeImage(image, name, params) {
		return false
	} else if params.sinceImage != "" && !matchesSinceImage(image, name, params) {
		return false
	} else if params.referencePattern != "" && !matchesReference(name, params.referencePattern) {
		return false
	}
	return true
}

func matchesDangling(name string, dangling string) bool {
	if dangling == "false" && name != "<none>" {
		return true
	} else if dangling == "true" && name == "<none>" {
		return true
	}
	return false
}

func matchesLabel(image storage.Image, store storage.Store, label string) bool {
	storeRef, err := is.Transport.ParseStoreReference(store, "@"+image.ID)
	if err != nil {

	}
	img, err := storeRef.NewImage(nil)
	if err != nil {
		return false
	}
	info, err := img.Inspect()
	if err != nil {
		return false
	}

	pair := strings.SplitN(label, "=", 2)
	for key, value := range info.Labels {
		if key == pair[0] {
			if len(pair) == 2 {
				if value == pair[1] {
					return true
				}
			} else {
				return false
			}
		}
	}
	return false
}

// Returns true if the image was created since the filter image.  Returns
// false otherwise
func matchesBeforeImage(image storage.Image, name string, params *filterParams) bool {
	if params.seenImage {
		return false
	}
	if matchesReference(name, params.beforeImage) || matchesID(image.ID, params.beforeImage) {
		params.seenImage = true
		return false
	}
	return true
}

// Returns true if the image was created since the filter image.  Returns
// false otherwise
func matchesSinceImage(image storage.Image, name string, params *filterParams) bool {
	if params.seenImage {
		return true
	}
	if matchesReference(name, params.sinceImage) || matchesID(image.ID, params.sinceImage) {
		params.seenImage = true
	}
	return false
}

func matchesID(id, argID string) bool {
	return strings.HasPrefix(argID, id)
}

func matchesReference(name, argName string) bool {
	if argName == "" {
		return true
	}
	splitName := strings.Split(name, ":")
	// If the arg contains a tag, we handle it differently than if it does not
	if strings.Contains(argName, ":") {
		splitArg := strings.Split(argName, ":")
		return strings.HasSuffix(splitName[0], splitArg[0]) && (splitName[1] == splitArg[1])
	}
	return strings.HasSuffix(splitName[0], argName)
}

func formattedSize(size int64) string {
	suffixes := [5]string{"B", "KB", "MB", "GB", "TB"}

	count := 0
	formattedSize := float64(size)
	for formattedSize >= 1024 && count < 4 {
		formattedSize /= 1024
		count++
	}
	return fmt.Sprintf("%.4g %s", formattedSize, suffixes[count])
}

func outputUsingTemplate(format string, params imageOutputParams) error {
	tmpl, err := template.New("image").Parse(format)
	if err != nil {
		return errors.Wrapf(err, "Template parsing error")
	}

	err = tmpl.Execute(os.Stdout, params)
	if err != nil {
		return err
	}
	fmt.Println()
	return nil
}

func outputUsingFormatString(truncate, digests bool, params imageOutputParams) {
	if truncate {
		fmt.Printf("%-20.12s %-56s", params.ID, params.Name)
	} else {
		fmt.Printf("%-64s %-56s", params.ID, params.Name)
	}

	if digests {
		fmt.Printf(" %-64s", params.Digest)
	}
	fmt.Printf(" %-22s %s\n", params.CreatedAt, params.Size)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"reflect"
	"strings"

	"github.com/containers/storage"
	"github.com/kubernetes-incubator/cri-o/cmd/kpod/formats"
	libkpodimage "github.com/kubernetes-incubator/cri-o/libkpod/image"
	digest "github.com/opencontainers/go-digest"
	"github.com/pkg/errors"
	"github.com/urfave/cli"
)

var (
	imagesFlags = []cli.Flag{
		cli.BoolFlag{
			Name:  "quiet, q",
			Usage: "display only image IDs",
		},
		cli.BoolFlag{
			Name:  "noheading, n",
			Usage: "do not print column headings",
		},
		cli.BoolFlag{
			Name:  "no-trunc, notruncate",
			Usage: "do not truncate output",
		},
		cli.BoolFlag{
			Name:  "digests",
			Usage: "show digests",
		},
		cli.StringFlag{
			Name:  "format",
			Usage: "Change the output format to JSON or a Go template",
		},
		cli.StringFlag{
			Name:  "filter, f",
			Usage: "filter output based on conditions provided (default [])",
		},
	}

	imagesDescription = "lists locally stored images."
	imagesCommand     = cli.Command{
		Name:        "images",
		Usage:       "list images in local storage",
		Description: imagesDescription,
		Flags:       imagesFlags,
		Action:      imagesCmd,
		ArgsUsage:   "",
	}
)

func imagesCmd(c *cli.Context) error {
	config, err := getConfig(c)
	if err != nil {
		return errors.Wrapf(err, "Could not get config")
	}
	store, err := getStore(config)
	if err != nil {
		return err
	}

	quiet := false
	if c.IsSet("quiet") {
		quiet = c.Bool("quiet")
	}
	noheading := false
	if c.IsSet("noheading") {
		noheading = c.Bool("noheading")
	}
	truncate := true
	if c.IsSet("no-trunc") {
		truncate = !c.Bool("no-trunc")
	}
	digests := false
	if c.IsSet("digests") {
		digests = c.Bool("digests")
	}
	outputFormat := genImagesFormat(quiet, truncate, digests)
	if c.IsSet("format") {
		outputFormat = c.String("format")
	}

	name := ""
	if len(c.Args()) == 1 {
		name = c.Args().Get(0)
	} else if len(c.Args()) > 1 {
		return errors.New("'kpod images' requires at most 1 argument")
	}

	var params *libkpodimage.FilterParams
	if c.IsSet("filter") {
		params, err = libkpodimage.ParseFilter(store, c.String("filter"))
		if err != nil {
			return errors.Wrapf(err, "error parsing filter")
		}
	} else {
		params = nil
	}

	imageList, err := libkpodimage.GetImagesMatchingFilter(store, params, name)
	if err != nil {
		return errors.Wrapf(err, "could not get list of images matching filter")
	}

	return outputImages(store, imageList, truncate, digests, quiet, outputFormat, noheading)
}

func genImagesFormat(quiet, truncate, digests bool) (format string) {
	if quiet {
		return "{{.ID}}"
	}
	if truncate {
		format = "table {{ .ID | printf \"%-20.12s\" }} "
	} else {
		format = "table {{ .ID | printf \"%-64s\" }} "
	}
	format += "{{ .Name | printf \"%-56s\" }} "

	if digests {
		format += "{{ .Digest | printf \"%-71s \"}} "
	}

	format += "{{ .CreatedAt | printf \"%-22s\" }} {{.Size}}"
	return
}

func outputImages(store storage.Store, images []storage.Image, truncate, digests, quiet bool, outputFormat string, noheading bool) error {
	imageOutput := []imageOutputParams{}

	lastID := ""
	for _, img := range images {
		if quiet && lastID == img.ID {
			continue // quiet should not show the same ID multiple times
		}
		createdTime := img.Created

		names := []string{""}
		if len(img.Names) > 0 {
			names = img.Names
		}

		info, imageDigest, size, _ := libkpodimage.InfoAndDigestAndSize(store, img)
		if info != nil {
			createdTime = info.Created
		}

		params := imageOutputParams{
			ID:        img.ID,
			Name:      names,
			Digest:    imageDigest,
			CreatedAt: createdTime.Format("Jan 2, 2006 15:04"),
			Size:      libkpodimage.FormattedSize(float64(size)),
		}
		imageOutput = append(imageOutput, params)
	}

	var out formats.Writer

	switch outputFormat {
	case formats.JSONString:
		out = formats.JSONStructArray{Output: toGeneric(imageOutput)}
	default:
		out = formats.StdoutTemplateArray{Output: toGeneric(imageOutput), Template: outputFormat, Fields: imageOutput[0].headerMap()}
	}

	formats.Writer(out).Out()

	return nil
}

type imageOutputParams struct {
	ID        string        `json:"id"`
	Name      []string      `json:"names"`
	Digest    digest.Digest `json:"digest"`
	CreatedAt string        `json:"created"`
	Size      string        `json:"size"`
}

func toGeneric(params []imageOutputParams) []interface{} {
	genericParams := make([]interface{}, len(params))
	for i, v := range params {
		genericParams[i] = interface{}(v)
	}
	return genericParams
}

func (i *imageOutputParams) headerMap() map[string]string {
	v := reflect.Indirect(reflect.ValueOf(i))
	values := make(map[string]string)

	for i := 0; i < v.NumField(); i++ {
		key := v.Type().Field(i).Name
		value := key
		if value == "ID" || value == "Name" {
			value = "Image" + value
		}
		values[key] = fmt.Sprintf("%s        ", strings.ToUpper(splitCamelCase(value)))
	}
	return values
}
<commit_msg>kpod-images: don't nil pointer on empty list<commit_after>package main

import (
	"fmt"
	"reflect"
	"strings"

	"github.com/containers/storage"
	"github.com/kubernetes-incubator/cri-o/cmd/kpod/formats"
	libkpodimage "github.com/kubernetes-incubator/cri-o/libkpod/image"
	digest "github.com/opencontainers/go-digest"
	"github.com/pkg/errors"
	"github.com/urfave/cli"
)

var (
	imagesFlags = []cli.Flag{
		cli.BoolFlag{
			Name:  "quiet, q",
			Usage: "display only image IDs",
		},
		cli.BoolFlag{
			Name:  "noheading, n",
			Usage: "do not print column headings",
		},
		cli.BoolFlag{
			Name:  "no-trunc, notruncate",
			Usage: "do not truncate output",
		},
		cli.BoolFlag{
			Name:  "digests",
			Usage: "show digests",
		},
		cli.StringFlag{
			Name:  "format",
			Usage: "Change the output format to JSON or a Go template",
		},
		cli.StringFlag{
			Name:  "filter, f",
			Usage: "filter output based on conditions provided (default [])",
		},
	}

	imagesDescription = "lists locally stored images."
	imagesCommand     = cli.Command{
		Name:        "images",
		Usage:       "list images in local storage",
		Description: imagesDescription,
		Flags:       imagesFlags,
		Action:      imagesCmd,
		ArgsUsage:   "",
	}
)

func imagesCmd(c *cli.Context) error {
	config, err := getConfig(c)
	if err != nil {
		return errors.Wrapf(err, "Could not get config")
	}
	store, err := getStore(config)
	if err != nil {
		return err
	}

	quiet := false
	if c.IsSet("quiet") {
		quiet = c.Bool("quiet")
	}
	noheading := false
	if c.IsSet("noheading") {
		noheading = c.Bool("noheading")
	}
	truncate := true
	if c.IsSet("no-trunc") {
		truncate = !c.Bool("no-trunc")
	}
	digests := false
	if c.IsSet("digests") {
		digests = c.Bool("digests")
	}
	outputFormat := genImagesFormat(quiet, truncate, digests)
	if c.IsSet("format") {
		outputFormat = c.String("format")
	}

	name := ""
	if len(c.Args()) == 1 {
		name = c.Args().Get(0)
	} else if len(c.Args()) > 1 {
		return errors.New("'kpod images' requires at most 1 argument")
	}

	var params *libkpodimage.FilterParams
	if c.IsSet("filter") {
		params, err = libkpodimage.ParseFilter(store, c.String("filter"))
		if err != nil {
			return errors.Wrapf(err, "error parsing filter")
		}
	} else {
		params = nil
	}

	imageList, err := libkpodimage.GetImagesMatchingFilter(store, params, name)
	if err != nil {
		return errors.Wrapf(err, "could not get list of images matching filter")
	}

	return outputImages(store, imageList, truncate, digests, quiet, outputFormat, noheading)
}

func genImagesFormat(quiet, truncate, digests bool) (format string) {
	if quiet {
		return "{{.ID}}"
	}
	if truncate {
		format = "table {{ .ID | printf \"%-20.12s\" }} "
	} else {
		format = "table {{ .ID | printf \"%-64s\" }} "
	}
	format += "{{ .Name | printf \"%-56s\" }} "

	if digests {
		format += "{{ .Digest | printf \"%-71s \"}} "
	}

	format += "{{ .CreatedAt | printf \"%-22s\" }} {{.Size}}"
	return
}

func outputImages(store storage.Store, images []storage.Image, truncate, digests, quiet bool, outputFormat string, noheading bool) error {
	imageOutput := []imageOutputParams{}

	lastID := ""
	for _, img := range images {
		if quiet && lastID == img.ID {
			continue // quiet should not show the same ID multiple times
		}
		createdTime := img.Created

		names := []string{""}
		if len(img.Names) > 0 {
			names = img.Names
		}

		info, imageDigest, size, _ := libkpodimage.InfoAndDigestAndSize(store, img)
		if info != nil {
			createdTime = info.Created
		}

		params := imageOutputParams{
			ID:        img.ID,
			Name:      names,
			Digest:    imageDigest,
			CreatedAt: createdTime.Format("Jan 2, 2006 15:04"),
			Size:      libkpodimage.FormattedSize(float64(size)),
		}
		imageOutput = append(imageOutput, params)
	}

	var out formats.Writer

	switch outputFormat {
	case formats.JSONString:
		out = formats.JSONStructArray{Output: toGeneric(imageOutput)}
	default:
		if len(imageOutput) == 0 {
			out = formats.StdoutTemplateArray{}
		} else {
			out = formats.StdoutTemplateArray{Output: toGeneric(imageOutput), Template: outputFormat, Fields: imageOutput[0].headerMap()}
		}
	}

	formats.Writer(out).Out()

	return nil
}

type imageOutputParams struct {
	ID        string        `json:"id"`
	Name      []string      `json:"names"`
	Digest    digest.Digest `json:"digest"`
	CreatedAt string        `json:"created"`
	Size      string        `json:"size"`
}

func toGeneric(params []imageOutputParams) []interface{} {
	genericParams := make([]interface{}, len(params))
	for i, v := range params {
		genericParams[i] = interface{}(v)
	}
	return genericParams
}

func (i *imageOutputParams) headerMap() map[string]string {
	v := reflect.Indirect(reflect.ValueOf(i))
	values := make(map[string]string)

	for i := 0; i < v.NumField(); i++ {
		key := v.Type().Field(i).Name
		value := key
		if value == "ID" || value == "Name" {
			value = "Image" + value
		}
		values[key] = fmt.Sprintf("%s        ", strings.ToUpper(splitCamelCase(value)))
	}
	return values
}
<|endoftext|>


<commit_before>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2017 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package cmd

import (
	"fmt"
	"os"

	"gopkg.in/guregu/null.v3"

	"github.com/loadimpact/k6/stats/cloud"
	"github.com/loadimpact/k6/ui"
	"github.com/pkg/errors"
	"github.com/spf13/afero"
	"github.com/spf13/cobra"
)

// loginCloudCommand represents the 'login cloud' command
var loginCloudCommand = &cobra.Command{
	Use:   "cloud",
	Short: "Authenticate with Load Impact",
	Long: `Authenticate with Load Impact.

This will set the default token used when just "k6 run -o cloud" is passed.`,
	Example: `
  # Show the stored token.
  k6 login cloud -s

  # Store a token.
  k6 login cloud -t YOUR_TOKEN

  # Log in with an email/password.
  k6 login cloud`[1:],
	Args: cobra.NoArgs,
	RunE: func(cmd *cobra.Command, args []string) error {
		fs := afero.NewOsFs()
		config, cdir, err := readDiskConfig(fs)
		if err != nil {
			return err
		}

		show := getNullBool(cmd.Flags(), "show")
		token := getNullString(cmd.Flags(), "token")

		conf := cloud.NewConfig().Apply(config.Collectors.Cloud)

		switch {
		case show.Bool:
		case token.Valid:
			conf.Token = token
		default:
			form := ui.Form{
				Fields: []ui.Field{
					ui.StringField{
						Key:   "Email",
						Label: "Email",
					},
					ui.StringField{
						Key:   "Password",
						Label: "Password",
					},
				},
			}
			vals, err := form.Run(os.Stdin, stdout)
			if err != nil {
				return err
			}
			email := vals["Email"].(string)
			password := vals["Password"].(string)

			client := cloud.NewClient("", conf.Host.String, Version)
			res, err := client.Login(email, password)
			if err != nil {
				return err
			}

			if res.Token == "" {
				return errors.New(`Your account has no API token, please generate one: "https://app.loadimpact.com/account/token".`)
			}

			conf.Token = null.StringFrom(res.Token)
		}

		config.Collectors.Cloud = conf
		if err := writeDiskConfig(fs, cdir, config); err != nil {
			return err
		}

		fmt.Fprintf(stdout, "  token: %s\n", ui.ValueColor.Sprint(conf.Token))
		return nil
	},
}

func init() {
	loginCmd.AddCommand(loginCloudCommand)
	loginCloudCommand.Flags().StringP("token", "t", "", "specify `token` to use")
	loginCloudCommand.Flags().BoolP("show", "s", false, "display saved token and exit")
}
<commit_msg>Fix wrong token output after cloud config refactoring<commit_after>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2017 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package cmd

import (
	"fmt"
	"os"

	"gopkg.in/guregu/null.v3"

	"github.com/loadimpact/k6/stats/cloud"
	"github.com/loadimpact/k6/ui"
	"github.com/pkg/errors"
	"github.com/spf13/afero"
	"github.com/spf13/cobra"
)

// loginCloudCommand represents the 'login cloud' command
var loginCloudCommand = &cobra.Command{
	Use:   "cloud",
	Short: "Authenticate with Load Impact",
	Long: `Authenticate with Load Impact.

This will set the default token used when just "k6 run -o cloud" is passed.`,
	Example: `
  # Show the stored token.
  k6 login cloud -s

  # Store a token.
  k6 login cloud -t YOUR_TOKEN

  # Log in with an email/password.
  k6 login cloud`[1:],
	Args: cobra.NoArgs,
	RunE: func(cmd *cobra.Command, args []string) error {
		fs := afero.NewOsFs()
		config, cdir, err := readDiskConfig(fs)
		if err != nil {
			return err
		}

		show := getNullBool(cmd.Flags(), "show")
		token := getNullString(cmd.Flags(), "token")

		conf := cloud.NewConfig().Apply(config.Collectors.Cloud)

		switch {
		case show.Bool:
		case token.Valid:
			conf.Token = token
		default:
			form := ui.Form{
				Fields: []ui.Field{
					ui.StringField{
						Key:   "Email",
						Label: "Email",
					},
					ui.StringField{
						Key:   "Password",
						Label: "Password",
					},
				},
			}
			vals, err := form.Run(os.Stdin, stdout)
			if err != nil {
				return err
			}
			email := vals["Email"].(string)
			password := vals["Password"].(string)

			client := cloud.NewClient("", conf.Host.String, Version)
			res, err := client.Login(email, password)
			if err != nil {
				return err
			}

			if res.Token == "" {
				return errors.New(`Your account has no API token, please generate one: "https://app.loadimpact.com/account/token".`)
			}

			conf.Token = null.StringFrom(res.Token)
		}

		config.Collectors.Cloud = conf
		if err := writeDiskConfig(fs, cdir, config); err != nil {
			return err
		}

		fmt.Fprintf(stdout, "  token: %s\n", ui.ValueColor.Sprint(conf.Token.String))
		return nil
	},
}

func init() {
	loginCmd.AddCommand(loginCloudCommand)
	loginCloudCommand.Flags().StringP("token", "t", "", "specify `token` to use")
	loginCloudCommand.Flags().BoolP("show", "s", false, "display saved token and exit")
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"

	"github.com/uli-go/xz/lzma"
)

type reader struct {
	file *os.File
	*bufio.Reader
	stdin  bool
	remove bool
}

func newReader(path string, opts *options) (r *reader, err error) {
	if path == "-" {
		r = &reader{
			file:   os.Stdin,
			Reader: bufio.NewReader(os.Stdin),
			stdin:  true,
		}
		return r, nil
	}
	fi, err := os.Lstat(path)
	if err != nil {
		return nil, err
	}
	if !fi.Mode().IsRegular() {
		return nil, fmt.Errorf("%s is not a reqular file", path)
	}
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	r = &reader{
		file:   file,
		Reader: bufio.NewReader(file),
	}
	if !opts.keep {
		r.remove = true
	}
	return r, nil
}

var errReaderClosed = errors.New("reader already closed")

func (r *reader) Close() error {
	if r.Reader == nil {
		return errReaderClosed
	}

	var err error
	if !r.stdin {
		if err = r.file.Close(); err != nil {
			return err
		}
		if r.remove {
			if err = os.Remove(r.file.Name()); err != nil {
				return err
			}
		}
	}

	*r = reader{}
	return nil
}

func (r *reader) Cancel() error {
	if r.Reader == nil {
		return errReaderClosed
	}

	if !r.stdin {
		if err := r.file.Close(); err != nil {
			return err
		}
	}

	*r = reader{}
	return nil
}

type writer struct {
	file *os.File
	*bufio.Writer
	stdout bool
	rename bool
	name   string
}

func newWriter(path string, opts *options) (w *writer, err error) {
	if path == "-" || opts.stdout {
		w = &writer{
			file:   os.Stdout,
			Writer: bufio.NewWriter(os.Stdout),
			stdout: true,
		}
		return w, nil
	}
	name := path + ".lzma"
	var dir string
	if dir, err = os.Getwd(); err != nil {
		return nil, err
	}
	file, err := ioutil.TempFile(dir, "lzma-")
	if err != nil {
		return nil, err
	}
	w = &writer{
		file:   file,
		Writer: bufio.NewWriter(file),
		rename: true,
		name:   name,
	}
	return w, nil
}

var errWriterClosed = errors.New("writer already closed")

func (w *writer) Close() error {
	if w.Writer == nil {
		return errWriterClosed
	}

	var err error
	if err = w.Writer.Flush(); err != nil {
		return err
	}

	if !w.stdout {
		if err = w.file.Close(); err != nil {
			return err
		}
		if w.rename {
			if err = os.Rename(w.file.Name(), w.name); err != nil {
				return err
			}
		} else {
			if err = os.Remove(w.file.Name()); err != nil {
				return err
			}
		}
	}

	*w = writer{}
	return nil
}

func (w *writer) Cancel() error {
	if w.Writer == nil {
		return errWriterClosed
	}

	var err error
	if !w.stdout {
		if err = w.file.Close(); err != nil {
			return err
		}
		if err = os.Remove(w.file.Name()); err != nil {
			return err
		}
	}

	*w = writer{}
	return nil
}

type decompressor struct {
	*lzma.Reader
	r *reader
}

func newDecompressor(path string, opts *options) (d *decompressor, err error) {
	r, err := newReader(path, opts)
	if err != nil {
		return nil, err
	}
	lr, err := lzma.NewReader(r)
	if err != nil {
		r.Cancel()
		return nil, err
	}
	d = &decompressor{Reader: lr, r: r}
	return d, nil
}

func (d *decompressor) Close() error {
	d.Reader = nil
	if err := d.r.Close(); err != nil {
		return err
	}
	d.r = nil
	return nil
}

func (d *decompressor) Cancel() error {
	if d.Reader == nil {
		return nil
	}
	if err := d.r.Cancel(); err != nil {
		return err
	}
	d.Reader = nil
	d.r = nil
	return nil
}

type compressor struct {
	*lzma.Writer
	w *writer
}

// parameters converts the lzmago executable flags to lzma parameters.
//
// I cannot use the preset config from the Tukaani project directly,
// because I don't have two algorithm modes and can't support parameters
// like nice_len or depth. So at this point in time I stay with the
// dictionary sizes the default combination of (LC,LP,LB) = (3,0,2).
// The default preset is 6.
// Following list provides exponents of two for the dictionary sizes:
// 18, 20, 21, 22, 22, 23, 23, 24, 25, 26.
func parameters(opts *options) lzma.Parameters {
	dictSizeExps := []uint{18, 20, 21, 22, 22, 23, 23, 24, 25, 26}
	dictSize := int64(1) << dictSizeExps[opts.preset]
	p := lzma.Parameters{
		LC:           3,
		LP:           0,
		PB:           2,
		DictSize:     dictSize,
		EOS:          true,
		ExtraBufSize: 16 * 1024,
	}
	return p
}

func newCompressor(path string, opts *options) (c *compressor, err error) {
	w, err := newWriter(path, opts)
	if err != nil {
		return nil, err
	}
	p := parameters(opts)
	lw, err := lzma.NewWriterParams(w, p)
	if err != nil {
		w.Cancel()
		return nil, err
	}
	c = &compressor{
		Writer: lw,
		w:      w,
	}
	return c, nil
}

func (c *compressor) Close() error {
	var err error
	if err = c.Writer.Close(); err != nil {
		return err
	}
	if err := c.w.Close(); err != nil {
		return err
	}
	c.w = nil
	c.Writer = nil
	return nil
}

func (c *compressor) Cancel() error {
	if c.Writer == nil {
		return nil
	}
	if err := c.w.Cancel(); err != nil {
		return err
	}
	c.w = nil
	c.Writer = nil
	return nil
}

type readCanceler interface {
	io.ReadCloser
	Cancel() error
}

func newReadCanceler(path string, opt *options) (r readCanceler, err error) {
	if opt.decompress {
		r, err = newDecompressor(path, opt)
	} else {
		r, err = newReader(path, opt)
	}
	return
}

type writeCanceler interface {
	io.WriteCloser
	Cancel() error
}

func newWriteCanceler(path string, opt *options) (w writeCanceler, err error) {
	if !opt.decompress {
		w, err = newCompressor(path, opt)
	} else {
		w, err = newWriter(path, opt)
	}
	return
}

func processLZMA(path string, opts *options) (err error) {
	r, err := newReadCanceler(path, opts)
	if err != nil {
		return err
	}
	defer func() {
		if err != nil {
			r.Cancel()
		} else {
			err = r.Close()
		}
	}()
	w, err := newWriteCanceler(path, opts)
	if err != nil {
		return err
	}
	defer func() {
		if err != nil {
			w.Cancel()
		} else {
			err = w.Close()
		}
	}()
	for {
		_, err = io.CopyN(w, r, 64*1024)
		if err != nil {
			if err == io.EOF {
				err = nil
			}
			return
		}
	}
}
<commit_msg>lzmago: fixed bug in writer<commit_after>package main

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strings"

	"github.com/uli-go/xz/lzma"
)

type reader struct {
	file *os.File
	*bufio.Reader
	stdin  bool
	remove bool
}

func newReader(path string, opts *options) (r *reader, err error) {
	if path == "-" {
		r = &reader{
			file:   os.Stdin,
			Reader: bufio.NewReader(os.Stdin),
			stdin:  true,
		}
		return r, nil
	}
	fi, err := os.Lstat(path)
	if err != nil {
		return nil, err
	}
	if !fi.Mode().IsRegular() {
		return nil, fmt.Errorf("%s is not a reqular file", path)
	}
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	r = &reader{
		file:   file,
		Reader: bufio.NewReader(file),
	}
	if !opts.keep {
		r.remove = true
	}
	return r, nil
}

var errReaderClosed = errors.New("reader already closed")

func (r *reader) Close() error {
	if r.Reader == nil {
		return errReaderClosed
	}

	var err error
	if !r.stdin {
		if err = r.file.Close(); err != nil {
			return err
		}
		if r.remove {
			if err = os.Remove(r.file.Name()); err != nil {
				return err
			}
		}
	}

	*r = reader{}
	return nil
}

func (r *reader) Cancel() error {
	if r.Reader == nil {
		return errReaderClosed
	}

	if !r.stdin {
		if err := r.file.Close(); err != nil {
			return err
		}
	}

	*r = reader{}
	return nil
}

type writer struct {
	file *os.File
	*bufio.Writer
	stdout bool
	rename bool
	name   string
}

func newWriter(path string, opts *options) (w *writer, err error) {
	if path == "-" || opts.stdout {
		w = &writer{
			file:   os.Stdout,
			Writer: bufio.NewWriter(os.Stdout),
			stdout: true,
		}
		return w, nil
	}
	const ext = ".lzma"
	var name string
	if opts.decompress {
		if !strings.HasSuffix(path, ext) {
			return nil, errors.New("unknown suffix -- file ignored")
		}
		name = path[:len(path)-len(ext)]
	} else {
		name = path + ext
	}
	var dir string
	if dir, err = os.Getwd(); err != nil {
		return nil, err
	}
	file, err := ioutil.TempFile(dir, "lzma-")
	if err != nil {
		return nil, err
	}
	w = &writer{
		file:   file,
		Writer: bufio.NewWriter(file),
		rename: true,
		name:   name,
	}
	return w, nil
}

var errWriterClosed = errors.New("writer already closed")

func (w *writer) Close() error {
	if w.Writer == nil {
		return errWriterClosed
	}

	var err error
	if err = w.Writer.Flush(); err != nil {
		return err
	}

	if !w.stdout {
		if err = w.file.Close(); err != nil {
			return err
		}
		if w.rename {
			if err = os.Rename(w.file.Name(), w.name); err != nil {
				return err
			}
		} else {
			if err = os.Remove(w.file.Name()); err != nil {
				return err
			}
		}
	}

	*w = writer{}
	return nil
}

func (w *writer) Cancel() error {
	if w.Writer == nil {
		return errWriterClosed
	}

	var err error
	if !w.stdout {
		if err = w.file.Close(); err != nil {
			return err
		}
		if err = os.Remove(w.file.Name()); err != nil {
			return err
		}
	}

	*w = writer{}
	return nil
}

type decompressor struct {
	*lzma.Reader
	r *reader
}

func newDecompressor(path string, opts *options) (d *decompressor, err error) {
	r, err := newReader(path, opts)
	if err != nil {
		return nil, err
	}
	lr, err := lzma.NewReader(r)
	if err != nil {
		r.Cancel()
		return nil, err
	}
	d = &decompressor{Reader: lr, r: r}
	return d, nil
}

func (d *decompressor) Close() error {
	d.Reader = nil
	if err := d.r.Close(); err != nil {
		return err
	}
	d.r = nil
	return nil
}

func (d *decompressor) Cancel() error {
	if d.Reader == nil {
		return nil
	}
	if err := d.r.Cancel(); err != nil {
		return err
	}
	d.Reader = nil
	d.r = nil
	return nil
}

type compressor struct {
	*lzma.Writer
	w *writer
}

// parameters converts the lzmago executable flags to lzma parameters.
//
// I cannot use the preset config from the Tukaani project directly,
// because I don't have two algorithm modes and can't support parameters
// like nice_len or depth. So at this point in time I stay with the
// dictionary sizes the default combination of (LC,LP,LB) = (3,0,2).
// The default preset is 6.
// Following list provides exponents of two for the dictionary sizes:
// 18, 20, 21, 22, 22, 23, 23, 24, 25, 26.
func parameters(opts *options) lzma.Parameters {
	dictSizeExps := []uint{18, 20, 21, 22, 22, 23, 23, 24, 25, 26}
	dictSize := int64(1) << dictSizeExps[opts.preset]
	p := lzma.Parameters{
		LC:           3,
		LP:           0,
		PB:           2,
		DictSize:     dictSize,
		EOS:          true,
		ExtraBufSize: 16 * 1024,
	}
	return p
}

func newCompressor(path string, opts *options) (c *compressor, err error) {
	w, err := newWriter(path, opts)
	if err != nil {
		return nil, err
	}
	p := parameters(opts)
	lw, err := lzma.NewWriterParams(w, p)
	if err != nil {
		w.Cancel()
		return nil, err
	}
	c = &compressor{
		Writer: lw,
		w:      w,
	}
	return c, nil
}

func (c *compressor) Close() error {
	var err error
	if err = c.Writer.Close(); err != nil {
		return err
	}
	if err := c.w.Close(); err != nil {
		return err
	}
	c.w = nil
	c.Writer = nil
	return nil
}

func (c *compressor) Cancel() error {
	if c.Writer == nil {
		return nil
	}
	if err := c.w.Cancel(); err != nil {
		return err
	}
	c.w = nil
	c.Writer = nil
	return nil
}

type readCanceler interface {
	io.ReadCloser
	Cancel() error
}

func newReadCanceler(path string, opt *options) (r readCanceler, err error) {
	if opt.decompress {
		r, err = newDecompressor(path, opt)
	} else {
		r, err = newReader(path, opt)
	}
	return
}

type writeCanceler interface {
	io.WriteCloser
	Cancel() error
}

func newWriteCanceler(path string, opt *options) (w writeCanceler, err error) {
	if !opt.decompress {
		w, err = newCompressor(path, opt)
	} else {
		w, err = newWriter(path, opt)
	}
	return
}

func processLZMA(path string, opts *options) (err error) {
	r, err := newReadCanceler(path, opts)
	if err != nil {
		return err
	}
	defer func() {
		if err != nil {
			r.Cancel()
		} else {
			err = r.Close()
		}
	}()
	w, err := newWriteCanceler(path, opts)
	if err != nil {
		return err
	}
	defer func() {
		if err != nil {
			w.Cancel()
		} else {
			err = w.Close()
		}
	}()
	for {
		_, err = io.CopyN(w, r, 64*1024)
		if err != nil {
			if err == io.EOF {
				err = nil
			}
			return
		}
	}
}
<|endoftext|>


<commit_before>// Ottily executes a javascript snippet on each line of an input file in parallel.
//
// Noop:
//
//     $ ottily datasets/simple.ldj
//     {"name": "ottily", "language": "Golang"}
//
// Inline script with -e:
//
//     $ ottily -e 'output=input.length' datasets/simple.ldj
//	   40
//
//     $ ottily -e 'o=JSON.parse(input); o["language"] = "Go"; output=JSON.stringify(o);' datasets/simple.ldj
//     {"language":"Go","name":"ottily"}
//
// Pass a script file:
//
//     $ ottily -s scripts/classified.js datasets/simple.ldj
//     CLASSIFIED
//
package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"runtime"
	"runtime/pprof"
	"strings"
	"sync"

	"github.com/robertkrimen/otto"
)

const NOOP_SCRIPT = "output = input"
const VERSION = "0.1.0"

func Worker(lines, out chan *string, script string, wg *sync.WaitGroup) {
	defer wg.Done()
	vm := otto.New()

	for line := range lines {
		vm.Set("input", *line)
		_, err := vm.Run(script)
		if err != nil {
			log.Fatal(err)
		}
		result, err := vm.Get("output")
		if result == otto.NullValue() {
			continue
		}
		if err != nil {
			log.Fatal(err)
		}
		r := result.String()
		out <- &r
	}
}

// FanInWriter writes the channel content to the writer
func FanInWriter(writer io.Writer, in chan *string, done chan bool) {
	for s := range in {
		writer.Write([]byte(*s))
		writer.Write([]byte("\n"))
	}
	done <- true
}

func main() {
	script := flag.String("s", "", "script to execute on each line of input")
	execute := flag.String("e", "", "execute argument on each line of input")
	numWorkers := flag.Int("w", runtime.NumCPU(), "number of workers")
	version := flag.Bool("v", false, "prints current program version")
	cpuprofile := flag.String("cpuprofile", "", "write cpu profile to file")

	flag.Parse()

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *version {
		fmt.Println(VERSION)
		os.Exit(0)
	}

	if flag.NArg() < 1 {
		log.Fatal("input file required")
	}

	content := NOOP_SCRIPT

	if *script != "" {
		ff, err := os.Open(*script)
		if err != nil {
			log.Fatal(err)
		}
		b, err := ioutil.ReadAll(ff)
		if err != nil {
			log.Fatal(err)
		}
		content = string(b)
	}

	if *execute != "" {
		content = *execute
	}

	ff, err := os.Open(flag.Arg(0))
	if err != nil {
		log.Fatal(err)
	}
	defer ff.Close()
	reader := bufio.NewReader(ff)

	if *numWorkers > 0 {
		runtime.GOMAXPROCS(*numWorkers)
	}

	queue := make(chan *string)
	out := make(chan *string)
	done := make(chan bool)
	var wg sync.WaitGroup

	go FanInWriter(os.Stdout, out, done)

	for i := 0; i < *numWorkers; i++ {
		wg.Add(1)
		go Worker(queue, out, content, &wg)
	}

	for {
		line, err := reader.ReadString('\n')
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatal(err)
		}
		line = strings.TrimSpace(line)
		queue <- &line
	}
	close(queue)
	wg.Wait()
	close(out)
	<-done
}
<commit_msg>precompile script<commit_after>// Ottily executes a javascript snippet on each line of an input file in parallel.
//
// Noop:
//
//     $ ottily datasets/simple.ldj
//     {"name": "ottily", "language": "Golang"}
//
// Inline script with -e:
//
//     $ ottily -e 'output=input.length' datasets/simple.ldj
//	   40
//
//     $ ottily -e 'o=JSON.parse(input); o["language"] = "Go"; output=JSON.stringify(o);' datasets/simple.ldj
//     {"language":"Go","name":"ottily"}
//
// Pass a script file:
//
//     $ ottily -s scripts/classified.js datasets/simple.ldj
//     CLASSIFIED
//
package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"runtime"
	"runtime/pprof"
	"strings"
	"sync"

	"github.com/robertkrimen/otto"
)

const NOOP_SCRIPT = "output = input"
const VERSION = "0.1.0"

func Worker(lines, out chan *string, script string, wg *sync.WaitGroup) {
	defer wg.Done()
	vm := otto.New()

	compiled, err := vm.Compile("", script)
	if err != nil {
		log.Fatal(err)
	}

	for line := range lines {
		vm.Set("input", *line)
		_, err := vm.Run(compiled)
		if err != nil {
			log.Fatal(err)
		}
		result, err := vm.Get("output")
		if result == otto.NullValue() {
			continue
		}
		if err != nil {
			log.Fatal(err)
		}
		r := result.String()
		out <- &r
	}
}

// FanInWriter writes the channel content to the writer
func FanInWriter(writer io.Writer, in chan *string, done chan bool) {
	for s := range in {
		writer.Write([]byte(*s))
		writer.Write([]byte("\n"))
	}
	done <- true
}

func main() {
	script := flag.String("s", "", "script to execute on each line of input")
	execute := flag.String("e", "", "execute argument on each line of input")
	numWorkers := flag.Int("w", runtime.NumCPU(), "number of workers")
	version := flag.Bool("v", false, "prints current program version")
	cpuprofile := flag.String("cpuprofile", "", "write cpu profile to file")

	flag.Parse()

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *version {
		fmt.Println(VERSION)
		os.Exit(0)
	}

	if flag.NArg() < 1 {
		log.Fatal("input file required")
	}

	content := NOOP_SCRIPT

	if *script != "" {
		ff, err := os.Open(*script)
		if err != nil {
			log.Fatal(err)
		}
		b, err := ioutil.ReadAll(ff)
		if err != nil {
			log.Fatal(err)
		}
		content = string(b)
	}

	if *execute != "" {
		content = *execute
	}

	ff, err := os.Open(flag.Arg(0))
	if err != nil {
		log.Fatal(err)
	}
	defer ff.Close()
	reader := bufio.NewReader(ff)

	if *numWorkers > 0 {
		runtime.GOMAXPROCS(*numWorkers)
	}

	queue := make(chan *string)
	out := make(chan *string)
	done := make(chan bool)
	var wg sync.WaitGroup

	go FanInWriter(os.Stdout, out, done)

	for i := 0; i < *numWorkers; i++ {
		wg.Add(1)
		go Worker(queue, out, content, &wg)
	}

	for {
		line, err := reader.ReadString('\n')
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatal(err)
		}
		line = strings.TrimSpace(line)
		queue <- &line
	}
	close(queue)
	wg.Wait()
	close(out)
	<-done
}
<|endoftext|>


<commit_before>package probe

import (
	"fmt"
	"path/filepath"
	"sort"
	"time"

	humanize "github.com/dustin/go-humanize"
	"github.com/go-errors/errors"
	"github.com/itchio/butler/comm"
	"github.com/itchio/butler/mansion"
	"github.com/itchio/wharf/bsdiff"
	"github.com/itchio/wharf/counter"
	"github.com/itchio/wharf/eos"
	"github.com/itchio/wharf/pwr"
	"github.com/itchio/wharf/tlc"
	"github.com/itchio/wharf/wire"
)

var args = struct {
	patch    *string
	fullpath *bool
}{}

func Register(ctx *mansion.Context) {
	cmd := ctx.App.Command("probe", "(Advanced) Show statistics about a patch file").Hidden()
	args.patch = cmd.Arg("patch", "Path of the patch to analyze").Required().String()
	args.fullpath = cmd.Flag("fullpath", "Display full path names").Bool()
	ctx.Register(cmd, do)
}

func do(ctx *mansion.Context) {
	ctx.Must(Do(ctx, *args.patch))
}

func Do(ctx *mansion.Context, patch string) error {
	patchReader, err := eos.Open(patch)
	if err != nil {
		return errors.Wrap(err, 0)
	}

	defer patchReader.Close()

	stats, err := patchReader.Stat()
	if err != nil {
		return errors.Wrap(err, 0)
	}

	comm.Opf("patch:  %s", humanize.IBytes(uint64(stats.Size())))

	cr := counter.NewReaderCallback(func(count int64) {
		comm.Progress(float64(count) / float64(stats.Size()))
	}, patchReader)

	rctx := wire.NewReadContext(cr)
	err = rctx.ExpectMagic(pwr.PatchMagic)
	if err != nil {
		return errors.Wrap(err, 0)
	}

	header := &pwr.PatchHeader{}
	err = rctx.ReadMessage(header)
	if err != nil {
		return errors.Wrap(err, 0)
	}

	rctx, err = pwr.DecompressWire(rctx, header.Compression)
	if err != nil {
		return errors.Wrap(err, 0)
	}

	target := &tlc.Container{}
	err = rctx.ReadMessage(target)
	if err != nil {
		return errors.Wrap(err, 0)
	}

	source := &tlc.Container{}
	err = rctx.ReadMessage(source)
	if err != nil {
		return errors.Wrap(err, 0)
	}

	comm.Logf("  before: %s in %s", humanize.IBytes(uint64(target.Size)), target.Stats())
	comm.Logf("   after: %s in %s", humanize.IBytes(uint64(target.Size)), source.Stats())

	startTime := time.Now()

	comm.StartProgressWithTotalBytes(stats.Size())

	var patchStats []patchStat

	sh := &pwr.SyncHeader{}
	rop := &pwr.SyncOp{}
	bc := &bsdiff.Control{}

	var numBsdiff = 0
	var numRsync = 0
	for fileIndex, f := range source.Files {
		sh.Reset()
		err = rctx.ReadMessage(sh)
		if err != nil {
			return errors.Wrap(err, 0)
		}

		stat := patchStat{
			fileIndex: int64(fileIndex),
			freshData: f.Size,
			algo:      sh.Type,
		}

		if sh.FileIndex != int64(fileIndex) {
			return fmt.Errorf("malformed patch: expected file %d, got %d", fileIndex, sh.FileIndex)
		}

		switch sh.Type {
		case pwr.SyncHeader_RSYNC:
			{
				numRsync++
				readingOps := true
				var pos int64

				for readingOps {
					rop.Reset()

					err = rctx.ReadMessage(rop)
					if err != nil {
						return errors.Wrap(err, 0)
					}

					switch rop.Type {
					case pwr.SyncOp_BLOCK_RANGE:
						fixedSize := (rop.BlockSpan - 1) * pwr.BlockSize
						lastIndex := rop.BlockIndex + (rop.BlockSpan - 1)
						lastSize := pwr.ComputeBlockSize(f.Size, lastIndex)
						totalSize := (fixedSize + lastSize)
						stat.freshData -= totalSize
						pos += totalSize
					case pwr.SyncOp_DATA:
						totalSize := int64(len(rop.Data))
						if ctx.Verbose {
							comm.Debugf("%s fresh data at %s (%d-%d)", humanize.IBytes(uint64(totalSize)), humanize.IBytes(uint64(pos)),
								pos, pos+totalSize)
						}
						pos += totalSize
					case pwr.SyncOp_HEY_YOU_DID_IT:
						readingOps = false
					}
				}
			}
		case pwr.SyncHeader_BSDIFF:
			{
				numBsdiff++
				readingOps := true

				bh := &pwr.BsdiffHeader{}
				err = rctx.ReadMessage(bh)
				if err != nil {
					return errors.Wrap(err, 0)
				}

				for readingOps {
					bc.Reset()

					err = rctx.ReadMessage(bc)
					if err != nil {
						return errors.Wrap(err, 0)
					}

					for _, b := range bc.Add {
						if b == 0 {
							stat.freshData--
						}
					}

					if bc.Eof {
						readingOps = false
					}
				}

				err = rctx.ReadMessage(rop)
				if err != nil {
					return errors.Wrap(err, 0)
				}

				if rop.Type != pwr.SyncOp_HEY_YOU_DID_IT {
					msg := fmt.Sprintf("expected HEY_YOU_DID_IT, got %s", rop.Type)
					return errors.New(msg)
				}
			}
		}

		patchStats = append(patchStats, stat)
	}

	comm.EndProgress()

	sort.Sort(byDecreasingFreshData(patchStats))

	var totalFresh int64
	for _, stat := range patchStats {
		totalFresh += stat.freshData
	}

	var freshThreshold = int64(0.9 * float64(totalFresh))
	var printedFresh int64

	duration := time.Since(startTime)

	perSec := humanize.IBytes(uint64(float64(stats.Size()) / duration.Seconds()))
	comm.Statf("Analyzed %s @ %s/s (%s total)", humanize.IBytes(uint64(stats.Size())), perSec, duration)
	comm.Statf("%d bsdiff series, %d rsync series", numBsdiff, numRsync)

	var numTouched = 0
	var numTotal = 0
	var naivePatchSize int64
	for _, stat := range patchStats {
		numTotal++
		if stat.freshData > 0 {
			numTouched++
			f := source.Files[stat.fileIndex]
			naivePatchSize += f.Size
		}
	}

	comm.Logf("")
	comm.Statf("Most of the fresh data is in the following files:")

	for i, stat := range patchStats {
		f := source.Files[stat.fileIndex]
		name := f.Path
		if !*args.fullpath {
			name = filepath.Base(name)
		}

		comm.Logf("  - %s / %s in %s (%.2f%% changed, %s)",
			humanize.IBytes(uint64(stat.freshData)),
			humanize.IBytes(uint64(f.Size)),
			name,
			float64(stat.freshData)/float64(f.Size)*100.0,
			stat.algo)

		printedFresh += stat.freshData
		if i >= 10 || printedFresh >= freshThreshold {
			break
		}
	}

	comm.Logf("")

	var kind = "simple"
	if numBsdiff > 0 {
		kind = "optimized"
	}
	comm.Statf("All in all, that's %s of fresh data in a %s %s patch",
		humanize.IBytes(uint64(totalFresh)),
		humanize.IBytes(uint64(stats.Size())),
		kind,
	)
	comm.Logf(" (%d/%d files are changed by this patch, they weigh a total of %s)", numTouched, numTotal, humanize.IBytes(uint64(naivePatchSize)))

	return nil
}

type patchStat struct {
	fileIndex int64
	freshData int64
	algo      pwr.SyncHeader_Type
}

type byDecreasingFreshData []patchStat

func (s byDecreasingFreshData) Len() int {
	return len(s)
}

func (s byDecreasingFreshData) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func (s byDecreasingFreshData) Less(i, j int) bool {
	return s[j].freshData < s[i].freshData
}
<commit_msg>--deep argument for probe, with little success<commit_after>package probe

import (
	"fmt"
	"path/filepath"
	"sort"
	"time"

	humanize "github.com/dustin/go-humanize"
	"github.com/go-errors/errors"
	"github.com/itchio/butler/comm"
	"github.com/itchio/butler/mansion"
	"github.com/itchio/wharf/bsdiff"
	"github.com/itchio/wharf/counter"
	"github.com/itchio/wharf/eos"
	"github.com/itchio/wharf/pwr"
	"github.com/itchio/wharf/tlc"
	"github.com/itchio/wharf/wire"
)

var args = struct {
	patch    *string
	fullpath *bool
	deep     *bool
}{}

func Register(ctx *mansion.Context) {
	cmd := ctx.App.Command("probe", "(Advanced) Show statistics about a patch file").Hidden()
	args.patch = cmd.Arg("patch", "Path of the patch to analyze").Required().String()
	args.fullpath = cmd.Flag("fullpath", "Display full path names").Bool()
	args.deep = cmd.Flag("deep", "Analyze the top N changed files further").Bool()
	ctx.Register(cmd, do)
}

func do(ctx *mansion.Context) {
	ctx.Must(Do(ctx, *args.patch))
}

func Do(ctx *mansion.Context, patch string) error {
	topFileIndices, err := doPrimaryAnalysis(ctx, patch)
	if err != nil {
		return errors.Wrap(err, 0)
	}

	if *args.deep {
		err = doDeepAnalysis(ctx, patch, topFileIndices)
		if err != nil {
			return errors.Wrap(err, 0)
		}
	}

	return nil
}

func doPrimaryAnalysis(ctx *mansion.Context, patch string) ([]int64, error) {
	patchReader, err := eos.Open(patch)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	defer patchReader.Close()

	stats, err := patchReader.Stat()
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	comm.Opf("patch:  %s", humanize.IBytes(uint64(stats.Size())))

	cr := counter.NewReaderCallback(func(count int64) {
		comm.Progress(float64(count) / float64(stats.Size()))
	}, patchReader)

	rctx := wire.NewReadContext(cr)
	err = rctx.ExpectMagic(pwr.PatchMagic)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	header := &pwr.PatchHeader{}
	err = rctx.ReadMessage(header)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	rctx, err = pwr.DecompressWire(rctx, header.Compression)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	target := &tlc.Container{}
	err = rctx.ReadMessage(target)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	source := &tlc.Container{}
	err = rctx.ReadMessage(source)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	comm.Logf("  before: %s in %s", humanize.IBytes(uint64(target.Size)), target.Stats())
	comm.Logf("   after: %s in %s", humanize.IBytes(uint64(target.Size)), source.Stats())

	startTime := time.Now()

	comm.StartProgressWithTotalBytes(stats.Size())

	var patchStats []patchStat

	sh := &pwr.SyncHeader{}
	rop := &pwr.SyncOp{}
	bc := &bsdiff.Control{}

	var numBsdiff = 0
	var numRsync = 0
	for fileIndex, f := range source.Files {
		sh.Reset()
		err = rctx.ReadMessage(sh)
		if err != nil {
			return nil, errors.Wrap(err, 0)
		}

		stat := patchStat{
			fileIndex: int64(fileIndex),
			freshData: f.Size,
			algo:      sh.Type,
		}

		if sh.FileIndex != int64(fileIndex) {
			return nil, fmt.Errorf("malformed patch: expected file %d, got %d", fileIndex, sh.FileIndex)
		}

		switch sh.Type {
		case pwr.SyncHeader_RSYNC:
			{
				numRsync++
				readingOps := true
				var pos int64

				for readingOps {
					rop.Reset()

					err = rctx.ReadMessage(rop)
					if err != nil {
						return nil, errors.Wrap(err, 0)
					}

					switch rop.Type {
					case pwr.SyncOp_BLOCK_RANGE:
						fixedSize := (rop.BlockSpan - 1) * pwr.BlockSize
						lastIndex := rop.BlockIndex + (rop.BlockSpan - 1)
						lastSize := pwr.ComputeBlockSize(f.Size, lastIndex)
						totalSize := (fixedSize + lastSize)
						stat.freshData -= totalSize
						pos += totalSize
					case pwr.SyncOp_DATA:
						totalSize := int64(len(rop.Data))
						if ctx.Verbose {
							comm.Debugf("%s fresh data at %s (%d-%d)", humanize.IBytes(uint64(totalSize)), humanize.IBytes(uint64(pos)),
								pos, pos+totalSize)
						}
						pos += totalSize
					case pwr.SyncOp_HEY_YOU_DID_IT:
						readingOps = false
					}
				}
			}
		case pwr.SyncHeader_BSDIFF:
			{
				numBsdiff++
				readingOps := true

				bh := &pwr.BsdiffHeader{}
				err = rctx.ReadMessage(bh)
				if err != nil {
					return nil, errors.Wrap(err, 0)
				}

				for readingOps {
					bc.Reset()

					err = rctx.ReadMessage(bc)
					if err != nil {
						return nil, errors.Wrap(err, 0)
					}

					for _, b := range bc.Add {
						if b == 0 {
							stat.freshData--
						}
					}

					if bc.Eof {
						readingOps = false
					}
				}

				err = rctx.ReadMessage(rop)
				if err != nil {
					return nil, errors.Wrap(err, 0)
				}

				if rop.Type != pwr.SyncOp_HEY_YOU_DID_IT {
					msg := fmt.Sprintf("expected HEY_YOU_DID_IT, got %s", rop.Type)
					return nil, errors.New(msg)
				}
			}
		}

		patchStats = append(patchStats, stat)
	}

	comm.EndProgress()

	sort.Sort(byDecreasingFreshData(patchStats))

	var totalFresh int64
	for _, stat := range patchStats {
		totalFresh += stat.freshData
	}

	var freshThreshold = int64(0.9 * float64(totalFresh))
	var printedFresh int64

	duration := time.Since(startTime)

	perSec := humanize.IBytes(uint64(float64(stats.Size()) / duration.Seconds()))
	comm.Statf("Analyzed %s @ %s/s (%s total)", humanize.IBytes(uint64(stats.Size())), perSec, duration)
	comm.Statf("%d bsdiff series, %d rsync series", numBsdiff, numRsync)

	var numTouched = 0
	var numTotal = 0
	var naivePatchSize int64
	for _, stat := range patchStats {
		numTotal++
		if stat.freshData > 0 {
			numTouched++
			f := source.Files[stat.fileIndex]
			naivePatchSize += f.Size
		}
	}

	comm.Logf("")
	comm.Statf("Most of the fresh data is in the following files:")

	var topFileIndices []int64
	for i, stat := range patchStats {
		f := source.Files[stat.fileIndex]
		name := f.Path
		if !*args.fullpath {
			name = filepath.Base(name)
		}

		comm.Logf("  - %s / %s in %s (%.2f%% changed, %s)",
			humanize.IBytes(uint64(stat.freshData)),
			humanize.IBytes(uint64(f.Size)),
			name,
			float64(stat.freshData)/float64(f.Size)*100.0,
			stat.algo)

		printedFresh += stat.freshData
		topFileIndices = append(topFileIndices, stat.fileIndex)

		if i >= 10 || printedFresh >= freshThreshold {
			break
		}
	}

	comm.Logf("")

	var kind = "simple"
	if numBsdiff > 0 {
		kind = "optimized"
	}
	comm.Statf("All in all, that's %s of fresh data in a %s %s patch",
		humanize.IBytes(uint64(totalFresh)),
		humanize.IBytes(uint64(stats.Size())),
		kind,
	)
	comm.Logf(" (%d/%d files are changed by this patch, they weigh a total of %s)", numTouched, numTotal, humanize.IBytes(uint64(naivePatchSize)))

	return topFileIndices, nil
}

type deepDiveContext struct {
	target *tlc.Container
	source *tlc.Container
	rctx   *wire.ReadContext
}

func doDeepAnalysis(ctx *mansion.Context, patch string, topFileIndices []int64) error {
	comm.Logf("")
	comm.Statf("Now deep-diving into top %d files", len(topFileIndices))

	topIndexMap := make(map[int64]bool)
	for _, i := range topFileIndices {
		topIndexMap[i] = true
	}

	patchReader, err := eos.Open(patch)
	if err != nil {
		return errors.Wrap(err, 0)
	}

	defer patchReader.Close()

	stats, err := patchReader.Stat()
	if err != nil {
		return errors.Wrap(err, 0)
	}

	comm.Opf("patch:  %s", humanize.IBytes(uint64(stats.Size())))

	cr := counter.NewReaderCallback(func(count int64) {
		comm.Progress(float64(count) / float64(stats.Size()))
	}, patchReader)

	rctx := wire.NewReadContext(cr)
	err = rctx.ExpectMagic(pwr.PatchMagic)
	if err != nil {
		return errors.Wrap(err, 0)
	}

	header := &pwr.PatchHeader{}
	err = rctx.ReadMessage(header)
	if err != nil {
		return errors.Wrap(err, 0)
	}

	rctx, err = pwr.DecompressWire(rctx, header.Compression)
	if err != nil {
		return errors.Wrap(err, 0)
	}

	target := &tlc.Container{}
	err = rctx.ReadMessage(target)
	if err != nil {
		return errors.Wrap(err, 0)
	}

	source := &tlc.Container{}
	err = rctx.ReadMessage(source)
	if err != nil {
		return errors.Wrap(err, 0)
	}

	ddc := &deepDiveContext{
		target: target,
		source: source,
		rctx:   rctx,
	}

	sh := &pwr.SyncHeader{}

	for fileIndex := range source.Files {
		sh.Reset()
		err = rctx.ReadMessage(sh)
		if err != nil {
			return errors.Wrap(err, 0)
		}

		if sh.FileIndex != int64(fileIndex) {
			return fmt.Errorf("malformed patch: expected file %d, got %d", fileIndex, sh.FileIndex)
		}

		if topIndexMap[sh.FileIndex] {
			err = ddc.analyzeSeries(sh)
		} else {
			comm.Debugf("Skipping %d...", sh.FileIndex)
			err = ddc.skipSeries(sh)
			if err != nil {
				return errors.Wrap(err, 0)
			}
		}
	}

	return nil
}

func (ddc *deepDiveContext) analyzeSeries(sh *pwr.SyncHeader) error {
	comm.Logf("")
	comm.Logf("=============================================")
	comm.Logf("")

	switch sh.Type {
	case pwr.SyncHeader_RSYNC:
		return ddc.analyzeRsync(sh)
	case pwr.SyncHeader_BSDIFF:
		return ddc.analyzeBsdiff(sh)
	default:
		return fmt.Errorf("don't know how to analyze series of type %d", sh.Type)
	}
}

func (ddc *deepDiveContext) analyzeRsync(sh *pwr.SyncHeader) error {
	f := ddc.source.Files[sh.FileIndex]
	comm.Logf("Analyzing rsync series for '%s'", f.Path)

	rctx := ddc.rctx
	readingOps := true

	rop := &pwr.SyncOp{}

	targetBlocks := make(map[int64]int64)

	for readingOps {
		rop.Reset()

		err := rctx.ReadMessage(rop)
		if err != nil {
			return errors.Wrap(err, 0)
		}

		switch rop.Type {
		case pwr.SyncOp_BLOCK_RANGE:
			i := rop.FileIndex
			targetBlocks[i] = targetBlocks[i] + rop.BlockSpan
		case pwr.SyncOp_DATA:
			// TODO: something
		case pwr.SyncOp_HEY_YOU_DID_IT:
			readingOps = false
		}
	}

	if len(targetBlocks) > 0 {
		comm.Statf("Sourcing from '%d' blocks total: ", len(targetBlocks))
		for i, numBlocks := range targetBlocks {
			tf := ddc.target.Files[i]
			comm.Statf("Taking %d blocks from '%s'", numBlocks, tf.Path)
		}
	} else {
		comm.Statf("Entirely fresh data!")
	}

	return nil
}

func (ddc *deepDiveContext) analyzeBsdiff(sh *pwr.SyncHeader) error {
	f := ddc.source.Files[sh.FileIndex]
	comm.Logf("Analyzing bsdiff series for '%s'", f.Path)

	rctx := ddc.rctx
	readingOps := true

	bh := &pwr.BsdiffHeader{}
	err := rctx.ReadMessage(bh)
	if err != nil {
		return errors.Wrap(err, 0)
	}

	tf := ddc.target.Files[bh.TargetIndex]
	comm.Logf("Diffed against target file '%s'", tf.Path)
	if tf.Path == f.Path {
		comm.Logf("Same path, can do in-place!")
	}

	bc := &bsdiff.Control{}

	var oldpos int64
	var newpos int64

	var pristine int64

	for readingOps {
		bc.Reset()

		err = rctx.ReadMessage(bc)
		if err != nil {
			return errors.Wrap(err, 0)
		}

		if bc.Eof {
			readingOps = false
			break
		}

		if len(bc.Add) > 0 {
			if oldpos == newpos {
				var unchanged int64
				for _, b := range bc.Add {
					if b == 0 {
						unchanged++
					}
				}
				pristine += unchanged
			} else {
				// comm.Logf("at %d, applying %d range from %d", newpos, len(bc.Add), oldpos)
			}

			oldpos += int64(len(bc.Add))
			newpos += int64(len(bc.Add))
		}

		if len(bc.Copy) > 0 {
			newpos += int64(len(bc.Copy))
		}

		oldpos += bc.Seek
	}

	rop := &pwr.SyncOp{}

	err = rctx.ReadMessage(rop)
	if err != nil {
		return errors.Wrap(err, 0)
	}

	if rop.Type != pwr.SyncOp_HEY_YOU_DID_IT {
		msg := fmt.Sprintf("expected HEY_YOU_DID_IT, got %s", rop.Type)
		return errors.New(msg)
	}

	comm.Statf("%s / %s pristine after patch application", humanize.IBytes(uint64(pristine)), humanize.IBytes(uint64(tf.Size)))
	comm.Statf("File went from %s to %s", humanize.IBytes(uint64(tf.Size)), humanize.IBytes(uint64(f.Size)))

	return nil
}

func (ddc *deepDiveContext) skipSeries(sh *pwr.SyncHeader) error {
	rctx := ddc.rctx
	rop := &pwr.SyncOp{}
	bc := &bsdiff.Control{}

	switch sh.Type {
	case pwr.SyncHeader_RSYNC:
		{
			readingOps := true
			for readingOps {
				rop.Reset()

				err := rctx.ReadMessage(rop)
				if err != nil {
					return errors.Wrap(err, 0)
				}

				if rop.Type == pwr.SyncOp_HEY_YOU_DID_IT {
					// yay, we did it!
					readingOps = false
				}
			}
		}
	case pwr.SyncHeader_BSDIFF:
		{
			bh := &pwr.BsdiffHeader{}
			err := rctx.ReadMessage(bh)
			if err != nil {
				return errors.Wrap(err, 0)
			}

			readingOps := true
			for readingOps {
				bc.Reset()

				err := rctx.ReadMessage(bc)
				if err != nil {
					return errors.Wrap(err, 0)
				}

				if bc.Eof {
					readingOps = false
				}
			}

			rop.Reset()
			err = rctx.ReadMessage(rop)
			if err != nil {
				return errors.Wrap(err, 0)
			}

			if rop.Type != pwr.SyncOp_HEY_YOU_DID_IT {
				// oh noes, we didn't do it
				return errors.New("missing HEY_YOU_DID_IT after bsdiff series")
			}
		}
	default:
		return fmt.Errorf("dunno how to skip series of type %d", sh.Type)
	}

	return nil
}

type patchStat struct {
	fileIndex int64
	freshData int64
	algo      pwr.SyncHeader_Type
}

type byDecreasingFreshData []patchStat

func (s byDecreasingFreshData) Len() int {
	return len(s)
}

func (s byDecreasingFreshData) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func (s byDecreasingFreshData) Less(i, j int) bool {
	return s[j].freshData < s[i].freshData
}
<|endoftext|>


<commit_before>// Copyright 2015 CodeIgnition. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package main

import (
	"errors"
	"flag"
	"log"
	"sync"

	"golang.org/x/net/context"

	"github.com/codeignition/recon/cmd/recond/config"
	"github.com/codeignition/recon/policy"
	_ "github.com/codeignition/recon/policy/handlers"
	"github.com/nats-io/nats"
)

const agentsAPIPath = "/api/agents" // agents path in the marksman server

// natsEncConn is the opened with the URL obtained from marksman.
// It is populated if the agent registers successfully.
var natsEncConn *nats.EncodedConn

// ctxCancelFunc stores the map of policy name to
// the context cancel function.
var ctxCancelFunc = struct {
	sync.Mutex
	m map[string]context.CancelFunc
}{
	m: make(map[string]context.CancelFunc),
}

func main() {
	log.SetPrefix("recond: ")

	var marksmanAddr = flag.String("marksman", "http://localhost:3000", "address of the marksman server")
	flag.Parse()

	conf, err := config.Init()
	if err != nil {
		log.Fatalln(err)
	}

	// agent represents a single agent on which the recond
	// is running.
	var agent = &Agent{
		UID: conf.UID,
	}

	err = agent.register(*marksmanAddr)
	if err != nil {
		log.Fatalln(err)
	}

	defer natsEncConn.Close()

	if err := addSystemDataPolicy(conf); err != nil {
		log.Fatal(err)
	}

	go runStoredPolicies(conf)

	natsEncConn.Subscribe(agent.UID+"_policy_add", AddPolicyHandler(conf))
	natsEncConn.Subscribe(agent.UID+"_policy_delete", DeletePolicyHandler(conf))

	// this is just to block the main function from exiting
	c := make(chan struct{})
	<-c
}

func runStoredPolicies(c *config.Config) {
	for _, p := range c.PolicyConfig {
		log.Printf("adding the policy %s...", p.Name)
		go func(p policy.Policy) {
			ctx, cancel := context.WithCancel(context.Background())
			events, err := p.Execute(ctx)
			if err != nil {
				log.Print(err) // TODO: send to a nats errors channel
			}
			ctxCancelFunc.Lock()
			ctxCancelFunc.m[p.Name] = cancel
			ctxCancelFunc.Unlock()

			for e := range events {
				natsEncConn.Publish("policy_events", e)
			}
		}(p)
	}
}

func addSystemDataPolicy(c *config.Config) error {
	// if the policy already exists, return silently
	for _, p := range c.PolicyConfig {
		if p.Name == "default_system_data" {
			return nil
		}
	}

	p := policy.Policy{
		Name:     "default_system_data",
		AgentUID: c.UID,
		Type:     "system_data",
		M: map[string]string{
			"interval": "5s",
		},
	}
	if err := c.AddPolicy(p); err != nil {
		return err

	}
	if err := c.Save(); err != nil {
		return err
	}
	return nil
}

func deletePolicy(c *config.Config, p policy.Policy) error {
	defer ctxCancelFunc.Unlock()
	ctxCancelFunc.Lock()

	if _, ok := ctxCancelFunc.m[p.Name]; !ok {
		return errors.New("policy not found")
	}

	log.Printf("deleting the policy %s...", p.Name)

	delete(ctxCancelFunc.m, p.Name)
	for i, q := range c.PolicyConfig {
		if q.Name == p.Name {
			c.PolicyConfig = append(c.PolicyConfig[:i], c.PolicyConfig[i+1:]...)
		}
	}
	if err := c.Save(); err != nil {
		return err
	}
	return nil
}
<commit_msg>safely delete policies<commit_after>// Copyright 2015 CodeIgnition. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package main

import (
	"errors"
	"flag"
	"log"
	"sync"

	"golang.org/x/net/context"

	"github.com/codeignition/recon/cmd/recond/config"
	"github.com/codeignition/recon/policy"
	_ "github.com/codeignition/recon/policy/handlers"
	"github.com/nats-io/nats"
)

const agentsAPIPath = "/api/agents" // agents path in the marksman server

// natsEncConn is the opened with the URL obtained from marksman.
// It is populated if the agent registers successfully.
var natsEncConn *nats.EncodedConn

// ctxCancelFunc stores the map of policy name to
// the context cancel function.
var ctxCancelFunc = struct {
	sync.Mutex
	m map[string]context.CancelFunc
}{
	m: make(map[string]context.CancelFunc),
}

func main() {
	log.SetPrefix("recond: ")

	var marksmanAddr = flag.String("marksman", "http://localhost:3000", "address of the marksman server")
	flag.Parse()

	conf, err := config.Init()
	if err != nil {
		log.Fatalln(err)
	}

	// agent represents a single agent on which the recond
	// is running.
	var agent = &Agent{
		UID: conf.UID,
	}

	err = agent.register(*marksmanAddr)
	if err != nil {
		log.Fatalln(err)
	}

	defer natsEncConn.Close()

	if err := addSystemDataPolicy(conf); err != nil {
		log.Fatal(err)
	}

	go runStoredPolicies(conf)

	natsEncConn.Subscribe(agent.UID+"_policy_add", AddPolicyHandler(conf))
	natsEncConn.Subscribe(agent.UID+"_policy_delete", DeletePolicyHandler(conf))

	// this is just to block the main function from exiting
	c := make(chan struct{})
	<-c
}

func runStoredPolicies(c *config.Config) {
	for _, p := range c.PolicyConfig {
		log.Printf("adding the policy %s...", p.Name)
		go func(p policy.Policy) {
			ctx, cancel := context.WithCancel(context.Background())
			events, err := p.Execute(ctx)
			if err != nil {
				log.Print(err) // TODO: send to a nats errors channel
			}
			ctxCancelFunc.Lock()
			ctxCancelFunc.m[p.Name] = cancel
			ctxCancelFunc.Unlock()

			for e := range events {
				natsEncConn.Publish("policy_events", e)
			}
		}(p)
	}
}

func addSystemDataPolicy(c *config.Config) error {
	// if the policy already exists, return silently
	for _, p := range c.PolicyConfig {
		if p.Name == "default_system_data" {
			return nil
		}
	}

	p := policy.Policy{
		Name:     "default_system_data",
		AgentUID: c.UID,
		Type:     "system_data",
		M: map[string]string{
			"interval": "5s",
		},
	}
	if err := c.AddPolicy(p); err != nil {
		return err

	}
	if err := c.Save(); err != nil {
		return err
	}
	return nil
}

func deletePolicy(c *config.Config, p policy.Policy) error {
	defer ctxCancelFunc.Unlock()
	ctxCancelFunc.Lock()

	if _, ok := ctxCancelFunc.m[p.Name]; !ok {
		return errors.New("policy not found")
	}

	log.Printf("deleting the policy %s...", p.Name)

	delete(ctxCancelFunc.m, p.Name)
	defer c.Unlock()
	c.Lock()
	for i, q := range c.PolicyConfig {
		if q.Name == p.Name {
			c.PolicyConfig = append(c.PolicyConfig[:i], c.PolicyConfig[i+1:]...)
		}
	}
	if err := c.Save(); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package helpers

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"strings"

	"encoding/csv"

	"github.com/containerum/solutions"
	"github.com/olekukonko/tablewriter"
	"gopkg.in/src-d/go-git.v4"
	"gopkg.in/src-d/go-git.v4/plumbing"
)

func gitClone(repoUrl, branch, destDir string) error {
	_, err := git.PlainClone(destDir, false, &git.CloneOptions{
		URL:           repoUrl,
		Progress:      os.Stdout,
		ReferenceName: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branch)),
		SingleBranch:  true,
		Depth:         1,
	})
	return err
}

func githubDownload(user, repo, branch, destDir string, files []string) error {
	for _, file := range files {
		resp, err := http.Get(fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", user, repo, branch, file))
		if err != nil {
			return err
		}

		if resp.StatusCode != http.StatusOK {
			return fmt.Errorf(resp.Status)
		}

		content, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}

		if err := ioutil.WriteFile(path.Join(destDir, file), content, os.ModePerm); err != nil {
			return err
		}
	}
	return nil
}

func fetchFiles(name, branch, destDir string, files []string) error {
	nameItems := strings.Split(name, "/")
	switch len(nameItems) {
	case 1: //containerum solution
		return githubDownload("containerum", name, branch, destDir, files)
	case 2: //3rd party solution on github
		return githubDownload(nameItems[0], nameItems[1], branch, destDir, files)
	default: //3rd party solution on any git hosting
		return gitClone(name, branch, destDir)
	}
	return nil
}

func DownloadSolution(name, solutionPath, branch string) error {
	if err := os.MkdirAll(solutionPath, os.ModePerm); err != nil {
		return err
	}

	// download config
	if err := fetchFiles(name, branch, solutionPath, []string{solutions.SolutionConfigFile}); err != nil {
		return err
	}

	// parse and download template files
	cfgFile, err := ioutil.ReadFile(path.Join(solutionPath, solutions.SolutionConfigFile))
	if err != nil {
		return err
	}
	var cfgObj solutions.SolutionConfig
	if err := json.Unmarshal(cfgFile, &cfgObj); err != nil {
		return err
	}

	var files []string
	for _, v := range cfgObj.Run {
		files = append(files, v.ConfigFile)
	}

	return fetchFiles(name, branch, solutionPath, files)
}

var (
	SolutionListUrl string // csv file with solutions description
)

func ShowSolutionList() error {
	resp, err := http.Get(SolutionListUrl)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf(resp.Status)
	}

	tbl, err := tablewriter.NewCSVReader(os.Stdout, csv.NewReader(resp.Body), true)
	if err != nil {
		return err
	}

	tbl.SetAlignment(tablewriter.ALIGN_CENTER)
	tbl.Render()

	return nil
}
<commit_msg>fix download solution function<commit_after>package helpers

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"strings"

	"encoding/csv"

	"github.com/containerum/solutions"
	"github.com/olekukonko/tablewriter"
	"gopkg.in/src-d/go-git.v4"
	"gopkg.in/src-d/go-git.v4/plumbing"
)

func gitClone(repoUrl, branch, destDir string) error {
	_, err := git.PlainClone(destDir, false, &git.CloneOptions{
		URL:           repoUrl,
		Progress:      os.Stdout,
		ReferenceName: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branch)),
		SingleBranch:  true,
		Depth:         1,
	})
	return err
}

func githubDownload(user, repo, branch, destDir string, files []string) error {
	for _, file := range files {
		resp, err := http.Get(fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", user, repo, branch, file))
		if err != nil {
			return err
		}

		if resp.StatusCode != http.StatusOK {
			return fmt.Errorf(resp.Status)
		}

		content, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}

		if err := ioutil.WriteFile(path.Join(destDir, file), content, os.ModePerm); err != nil {
			return err
		}
	}
	return nil
}

func fetchFiles(name, branch, destDir string, files []string) error {
	nameItems := strings.Split(name, "/")
	switch len(nameItems) {
	case 1: //containerum solution
		return githubDownload("containerum", name, branch, destDir, files)
	case 2: //3rd party solution on github
		return githubDownload(nameItems[0], nameItems[1], branch, destDir, files)
	default: //3rd party solution on any git hosting
		return gitClone(name, branch, destDir)
	}
	return nil
}

func DownloadSolution(name, solutionPath, branch, solutionConfigFile string) error {
	if err := os.MkdirAll(solutionPath, os.ModePerm); err != nil {
		return err
	}

	// download config
	if err := fetchFiles(name, branch, solutionPath, []string{solutionConfigFile}); err != nil {
		return err
	}

	// parse and download template files
	cfgFile, err := ioutil.ReadFile(path.Join(solutionPath, solutionConfigFile))
	if err != nil {
		return err
	}
	var cfgObj solutions.SolutionConfig
	if err := json.Unmarshal(cfgFile, &cfgObj); err != nil {
		return err
	}

	var files []string
	for _, v := range cfgObj.Run {
		files = append(files, v.ConfigFile)
	}

	return fetchFiles(name, branch, solutionPath, files)
}

var (
	SolutionListUrl string // csv file with solutions description
)

func ShowSolutionList() error {
	resp, err := http.Get(SolutionListUrl)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf(resp.Status)
	}

	tbl, err := tablewriter.NewCSVReader(os.Stdout, csv.NewReader(resp.Body), true)
	if err != nil {
		return err
	}

	tbl.SetAlignment(tablewriter.ALIGN_CENTER)
	tbl.Render()

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"os"

	"github.com/pborman/getopt"
	"github.com/starkandwayne/goutils/ansi"
	"github.com/starkandwayne/shield/api"
	"github.com/starkandwayne/shield/cmd/shield/commands"
	_ "github.com/starkandwayne/shield/cmd/shield/commands/archives"
	"github.com/starkandwayne/shield/cmd/shield/commands/backends"
	_ "github.com/starkandwayne/shield/cmd/shield/commands/info"
	_ "github.com/starkandwayne/shield/cmd/shield/commands/jobs"
	_ "github.com/starkandwayne/shield/cmd/shield/commands/policies"
	_ "github.com/starkandwayne/shield/cmd/shield/commands/schedules"
	_ "github.com/starkandwayne/shield/cmd/shield/commands/stores"
	_ "github.com/starkandwayne/shield/cmd/shield/commands/targets"
	_ "github.com/starkandwayne/shield/cmd/shield/commands/tasks"
	"github.com/starkandwayne/shield/cmd/shield/log"
)

var (
	debug = false
	//Version gets overridden by lflags when building
	Version = ""
)

func main() {
	commands.Opts = &commands.Options{
		Shield:   getopt.StringLong("shield", 'H', "", "DEPRECATED - Previously required to point to a SHIELD backend to talk to. Now used to auto-vivify ~/.shield_config if necessary"),
		Used:     getopt.BoolLong("used", 0, "Only show things that are in-use by something else"),
		Unused:   getopt.BoolLong("unused", 0, "Only show things that are not used by something else"),
		Paused:   getopt.BoolLong("paused", 0, "Only show jobs that are paused"),
		Unpaused: getopt.BoolLong("unpaused", 0, "Only show jobs that are unpaused"),
		All:      getopt.BoolLong("all", 'a', "Show all the things"),

		Debug:             getopt.BoolLong("debug", 'D', "Enable debugging"),
		Trace:             getopt.BoolLong("trace", 'T', "Enable trace mode"),
		Raw:               getopt.BoolLong("raw", 0, "Operate in RAW mode, reading and writing only JSON"),
		ShowUUID:          getopt.BoolLong("uuid", 0, "Return UUID"),
		UpdateIfExists:    getopt.BoolLong("update-if-exists", 0, "Create will update record if another exists with same name"),
		Fuzzy:             getopt.BoolLong("fuzzy", 0, "In RAW mode, perform fuzzy (inexact) searching"),
		SkipSSLValidation: getopt.BoolLong("skip-ssl-validation", 'k', "Disable SSL Certificate Validation"),

		Status:    getopt.StringLong("status", 'S', "", "Only show archives/tasks with the given status"),
		Target:    getopt.StringLong("target", 't', "", "Only show things for the target with this UUID"),
		Store:     getopt.StringLong("store", 's', "", "Only show things for the store with this UUID"),
		Schedule:  getopt.StringLong("schedule", 'w', "", "Only show things for the schedule with this UUID"),
		Retention: getopt.StringLong("policy", 'p', "", "Only show things for the retention policy with this UUID"),
		Plugin:    getopt.StringLong("plugin", 'P', "", "Only show things for the given target or store plugin"),
		After:     getopt.StringLong("after", 'A', "", "Only show archives that were taken after the given date, in YYYYMMDD format."),
		Before:    getopt.StringLong("before", 'B', "", "Only show archives that were taken before the given date, in YYYYMMDD format."),
		To:        getopt.StringLong("to", 0, "", "Restore the archive in question to a different target, specified by UUID"),
		Limit:     getopt.StringLong("limit", 0, "", "Display only the X most recent tasks or archives"),

		Config:  getopt.StringLong("config", 'c', os.Getenv("HOME")+"/.shield_config", "Overrides ~/.shield_config as the SHIELD config file"),
		Version: getopt.BoolLong("version", 'v', "Display the SHIELD version"),
	}

	var command []string
	var cmdLine = getopt.CommandLine
	args := os.Args
	for {
		cmdLine.Parse(args)
		if cmdLine.NArgs() == 0 {
			break
		}
		command = append(command, cmdLine.Arg(0))
		args = cmdLine.Args()
	}

	log.ToggleDebug(*commands.Opts.Debug)
	log.ToggleTrace(*commands.Opts.Trace)

	log.DEBUG("shield cli starting up")

	if *commands.Opts.SkipSSLValidation {
		os.Setenv("SHIELD_SKIP_SSL_VERIFY", "true")
	}

	if *commands.Opts.Version {
		if Version == "" {
			fmt.Println("shield cli (development)")
		} else {
			fmt.Printf("shield cli v%s\n", Version)
		}
		os.Exit(0)
	}

	commands.AddGlobalFlag(commands.FlagInfo{
		Name: "debug", Short: 'D',
		Desc: "Enable the output of debug output",
	})
	commands.AddGlobalFlag(commands.FlagInfo{
		Name: "trace", Short: 'T',
		Desc: "Enable the output of verbose trace output",
	})
	commands.AddGlobalFlag(commands.FlagInfo{
		Name: "skip-ssl-validation", Short: 'k',
		Desc: "Disable SSL certificate validation",
	})
	commands.AddGlobalFlag(commands.FlagInfo{
		Name: "raw",
		Desc: "Takes any input and gives any output as a JSON object",
	})

	err := api.LoadConfig(*commands.Opts.Config)
	if err != nil {
		ansi.Fprintf(os.Stderr, "\n@R{ERROR:} Could not parse %s: %s\n", *commands.Opts.Config, err)
		os.Exit(1)
	}

	cmd, cmdname, args := commands.ParseCommand(command...)
	log.DEBUG("Command: '%s'", cmdname)
	//Check if user gave a valid command
	if cmd == nil {
		ansi.Fprintf(os.Stderr, "@R{unrecognized command `%s'}\n", cmdname)
		os.Exit(1)
	}
	commands.MaybeWarnDeprecation(cmdname, cmd)

	// only check for backends + creds if we aren't manipulating backends/help
	helpCmd, _, _ := commands.ParseCommand("help")
	backendsCmd, _, _ := commands.ParseCommand("backends")
	backendCmd, _, _ := commands.ParseCommand("backend")
	cBackendCmd, _, _ := commands.ParseCommand("create-backend")
	if cmd != helpCmd && cmd != backendsCmd && cmd != backendCmd && cmd != cBackendCmd {
		if *commands.Opts.Shield != "" || os.Getenv("SHIELD_API") != "" {
			ansi.Fprintf(os.Stderr, "@Y{WARNING: -H, --host, and the SHIELD_API environment variable have been deprecated and will be removed in a later release.} Use `shield backend` instead\n")
		}

		backends.Load()
	}

	if err := cmd.Run(args...); err != nil {
		if *commands.Opts.Raw {
			j, err := json.Marshal(map[string]string{"error": err.Error()})
			if err != nil {
				panic("Couldn't parse error json")
			}
			fmt.Println(j)
		} else {
			ansi.Fprintf(os.Stderr, "@R{%s}\n", err)
		}
		os.Exit(1)
	} else {
		os.Exit(0)
	}
}
<commit_msg>Raw errors print as strings, not byte arrays<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"os"

	"github.com/pborman/getopt"
	"github.com/starkandwayne/goutils/ansi"
	"github.com/starkandwayne/shield/api"
	"github.com/starkandwayne/shield/cmd/shield/commands"
	_ "github.com/starkandwayne/shield/cmd/shield/commands/archives"
	"github.com/starkandwayne/shield/cmd/shield/commands/backends"
	_ "github.com/starkandwayne/shield/cmd/shield/commands/info"
	_ "github.com/starkandwayne/shield/cmd/shield/commands/jobs"
	_ "github.com/starkandwayne/shield/cmd/shield/commands/policies"
	_ "github.com/starkandwayne/shield/cmd/shield/commands/schedules"
	_ "github.com/starkandwayne/shield/cmd/shield/commands/stores"
	_ "github.com/starkandwayne/shield/cmd/shield/commands/targets"
	_ "github.com/starkandwayne/shield/cmd/shield/commands/tasks"
	"github.com/starkandwayne/shield/cmd/shield/log"
)

var (
	debug = false
	//Version gets overridden by lflags when building
	Version = ""
)

func main() {
	commands.Opts = &commands.Options{
		Shield:   getopt.StringLong("shield", 'H', "", "DEPRECATED - Previously required to point to a SHIELD backend to talk to. Now used to auto-vivify ~/.shield_config if necessary"),
		Used:     getopt.BoolLong("used", 0, "Only show things that are in-use by something else"),
		Unused:   getopt.BoolLong("unused", 0, "Only show things that are not used by something else"),
		Paused:   getopt.BoolLong("paused", 0, "Only show jobs that are paused"),
		Unpaused: getopt.BoolLong("unpaused", 0, "Only show jobs that are unpaused"),
		All:      getopt.BoolLong("all", 'a', "Show all the things"),

		Debug:             getopt.BoolLong("debug", 'D', "Enable debugging"),
		Trace:             getopt.BoolLong("trace", 'T', "Enable trace mode"),
		Raw:               getopt.BoolLong("raw", 0, "Operate in RAW mode, reading and writing only JSON"),
		ShowUUID:          getopt.BoolLong("uuid", 0, "Return UUID"),
		UpdateIfExists:    getopt.BoolLong("update-if-exists", 0, "Create will update record if another exists with same name"),
		Fuzzy:             getopt.BoolLong("fuzzy", 0, "In RAW mode, perform fuzzy (inexact) searching"),
		SkipSSLValidation: getopt.BoolLong("skip-ssl-validation", 'k', "Disable SSL Certificate Validation"),

		Status:    getopt.StringLong("status", 'S', "", "Only show archives/tasks with the given status"),
		Target:    getopt.StringLong("target", 't', "", "Only show things for the target with this UUID"),
		Store:     getopt.StringLong("store", 's', "", "Only show things for the store with this UUID"),
		Schedule:  getopt.StringLong("schedule", 'w', "", "Only show things for the schedule with this UUID"),
		Retention: getopt.StringLong("policy", 'p', "", "Only show things for the retention policy with this UUID"),
		Plugin:    getopt.StringLong("plugin", 'P', "", "Only show things for the given target or store plugin"),
		After:     getopt.StringLong("after", 'A', "", "Only show archives that were taken after the given date, in YYYYMMDD format."),
		Before:    getopt.StringLong("before", 'B', "", "Only show archives that were taken before the given date, in YYYYMMDD format."),
		To:        getopt.StringLong("to", 0, "", "Restore the archive in question to a different target, specified by UUID"),
		Limit:     getopt.StringLong("limit", 0, "", "Display only the X most recent tasks or archives"),

		Config:  getopt.StringLong("config", 'c', os.Getenv("HOME")+"/.shield_config", "Overrides ~/.shield_config as the SHIELD config file"),
		Version: getopt.BoolLong("version", 'v', "Display the SHIELD version"),
	}

	var command []string
	var cmdLine = getopt.CommandLine
	args := os.Args
	for {
		cmdLine.Parse(args)
		if cmdLine.NArgs() == 0 {
			break
		}
		command = append(command, cmdLine.Arg(0))
		args = cmdLine.Args()
	}

	log.ToggleDebug(*commands.Opts.Debug)
	log.ToggleTrace(*commands.Opts.Trace)

	log.DEBUG("shield cli starting up")

	if *commands.Opts.SkipSSLValidation {
		os.Setenv("SHIELD_SKIP_SSL_VERIFY", "true")
	}

	if *commands.Opts.Version {
		if Version == "" {
			fmt.Println("shield cli (development)")
		} else {
			fmt.Printf("shield cli v%s\n", Version)
		}
		os.Exit(0)
	}

	commands.AddGlobalFlag(commands.FlagInfo{
		Name: "debug", Short: 'D',
		Desc: "Enable the output of debug output",
	})
	commands.AddGlobalFlag(commands.FlagInfo{
		Name: "trace", Short: 'T',
		Desc: "Enable the output of verbose trace output",
	})
	commands.AddGlobalFlag(commands.FlagInfo{
		Name: "skip-ssl-validation", Short: 'k',
		Desc: "Disable SSL certificate validation",
	})
	commands.AddGlobalFlag(commands.FlagInfo{
		Name: "raw",
		Desc: "Takes any input and gives any output as a JSON object",
	})

	err := api.LoadConfig(*commands.Opts.Config)
	if err != nil {
		ansi.Fprintf(os.Stderr, "\n@R{ERROR:} Could not parse %s: %s\n", *commands.Opts.Config, err)
		os.Exit(1)
	}

	cmd, cmdname, args := commands.ParseCommand(command...)
	log.DEBUG("Command: '%s'", cmdname)
	//Check if user gave a valid command
	if cmd == nil {
		ansi.Fprintf(os.Stderr, "@R{unrecognized command `%s'}\n", cmdname)
		os.Exit(1)
	}
	commands.MaybeWarnDeprecation(cmdname, cmd)

	// only check for backends + creds if we aren't manipulating backends/help
	helpCmd, _, _ := commands.ParseCommand("help")
	backendsCmd, _, _ := commands.ParseCommand("backends")
	backendCmd, _, _ := commands.ParseCommand("backend")
	cBackendCmd, _, _ := commands.ParseCommand("create-backend")
	if cmd != helpCmd && cmd != backendsCmd && cmd != backendCmd && cmd != cBackendCmd {
		if *commands.Opts.Shield != "" || os.Getenv("SHIELD_API") != "" {
			ansi.Fprintf(os.Stderr, "@Y{WARNING: -H, --host, and the SHIELD_API environment variable have been deprecated and will be removed in a later release.} Use `shield backend` instead\n")
		}

		backends.Load()
	}

	if err := cmd.Run(args...); err != nil {
		if *commands.Opts.Raw {
			j, err := json.Marshal(map[string]string{"error": err.Error()})
			if err != nil {
				panic("Couldn't parse error json")
			}
			fmt.Println(string(j))
		} else {
			ansi.Fprintf(os.Stderr, "@R{%s}\n", err)
		}
		os.Exit(1)
	} else {
		os.Exit(0)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"os"

	"github.com/NebulousLabs/Sia/types"

	"github.com/spf13/cobra"
)

var (
	renterExportCmd = &cobra.Command{
		Use:   "export",
		Short: "export renter data to various formats",
		Long:  "Export renter data in various formats.",
		// Run field not provided; export requires a subcommand.
	}

	renterExportContractTxnsCmd = &cobra.Command{
		Use:   "contract-txns [destination]",
		Short: "export the renter's contracts for import to `https://rankings.sia.tech/`",
		Long: "Export the renter's current contract set in JSON format to the specified " +
			"file. Intended for upload to `https://rankings.sia.tech/`.",
		Run: wrap(renterexportcontracttxnscmd),
	}
)

// renterexportcontracttxnscmd is the handler for the command `siac renter export contract-txns`.
// Exports the current contract set to JSON.
func renterexportcontracttxnscmd(destination string) {
	cs, err := httpClient.RenterContractsGet()
	if err != nil {
		die("Could not retrieve contracts:", err)
	}
	var contractTxns []types.Transaction
	contracts := append(cs.Contracts, cs.OldContracts...)
	for _, c := range contracts {
		contractTxns = append(contractTxns, c.LastTransaction)
	}
	destination = abs(destination)
	file, err := os.Create(destination)
	if err != nil {
		die("Could not export to file:", err)
	}
	err = json.NewEncoder(file).Encode(contractTxns)
	if err != nil {
		die("Could not export to file:", err)
	}
	fmt.Println("Exported contract data to", destination)
}
<commit_msg>Remove old contracts from export<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"os"

	"github.com/NebulousLabs/Sia/types"

	"github.com/spf13/cobra"
)

var (
	renterExportCmd = &cobra.Command{
		Use:   "export",
		Short: "export renter data to various formats",
		Long:  "Export renter data in various formats.",
		// Run field not provided; export requires a subcommand.
	}

	renterExportContractTxnsCmd = &cobra.Command{
		Use:   "contract-txns [destination]",
		Short: "export the renter's contracts for import to `https://rankings.sia.tech/`",
		Long: "Export the renter's current contract set in JSON format to the specified " +
			"file. Intended for upload to `https://rankings.sia.tech/`.",
		Run: wrap(renterexportcontracttxnscmd),
	}
)

// renterexportcontracttxnscmd is the handler for the command `siac renter export contract-txns`.
// Exports the current contract set to JSON.
func renterexportcontracttxnscmd(destination string) {
	cs, err := httpClient.RenterContractsGet()
	if err != nil {
		die("Could not retrieve contracts:", err)
	}
	var contractTxns []types.Transaction
	for _, c := range cs.Contracts {
		contractTxns = append(contractTxns, c.LastTransaction)
	}
	destination = abs(destination)
	file, err := os.Create(destination)
	if err != nil {
		die("Could not export to file:", err)
	}
	err = json.NewEncoder(file).Encode(contractTxns)
	if err != nil {
		die("Could not export to file:", err)
	}
	fmt.Println("Exported contract data to", destination)
}
<|endoftext|>