<commit_before>package halo

import (
	"encoding/binary"
	"io/ioutil"
	"os"
	"sort"

	"github.com/phil-mansfield/shellfish/cmd/catalog"
	"github.com/phil-mansfield/shellfish/io"
)

// halos allows for arrays of halo properties to be sorted simultaneously.
type halos struct {
	rids               []int
	xs, ys, zs, ms, rs []float64
}

// This type is a look into the heart of darkness.
type VarColumns struct {
	ColumnLookup map[string]int
	Names []string
	Columns []int
	Generator []string
	NBinary int
	RadiusUnits string
}

func NewVarColumns(
	names []string, columns []int64, radiusUnits string,
) *VarColumns {
	vc :=&VarColumns{}
	vc.ColumnLookup = make(map[string]int)

	for i := range names {
		vc.ColumnLookup[names[i]] = i
		vc.Names = append(vc.Names, names[i])
		vc.Columns = append(vc.Columns, int(columns[i]))
	}
	vc.Generator = make([]string, len(vc.Names))
	vc.NBinary = len(vc.Names)

	mNames := []string{"M200m", "M200c", "M500c", "M2500c"}
	rNames := []string{"R200m", "R200c", "R500c", "R2500c"}
	for i, _ := range mNames {
		_, rOk := vc.ColumnLookup[rNames[i]]
		_, mOk := vc.ColumnLookup[mNames[i]]
		if mOk && !rOk {
			vc.ColumnLookup[rNames[i]] = len(vc.Names)
			vc.Names = append(vc.Names, rNames[i])
			vc.Columns = append(vc.Columns, -1)
			vc.Generator = append(vc.Generator, mNames[i])
		}
	}
	vc.RadiusUnits = radiusUnits

	return vc
}

func (vc *VarColumns) GetColumn(
	cols [][]float64, name string, cosmo *io.CosmologyHeader,
) []float64 {
	col, _ := vc.ColumnLookup[name]
	if vc.Generator[col] == "" { return cols[col] }

	gCol, _ := vc.ColumnLookup[vc.Generator[col]]
	ms := cols[gCol]
	rs := make([]float64, len(ms))

	rad, _ := RadiusFromString(name)

	rad.Radius(cosmo, ms, rs)
	// Convert from Mpc/h to the units of the halo catalog.
	ucf := UnitConversionFactor(vc.RadiusUnits, cosmo)
	for i := range rs {
		rs[i] /= ucf
	}

	return rs
}

func (vc *VarColumns) GetIDs(cols [][]float64) []int {
	col, _ := vc.ColumnLookup["ID"]

	out := make([]int, len(cols[col]))
	for i := range out { out[i] = int(cols[col][i]) }
	return out
}

func (hs *halos) Len() int           { return len(hs.rs) }
func (hs *halos) Less(i, j int) bool { return hs.rs[i] < hs.rs[j] }
func (hs *halos) Swap(i, j int) {
	hs.rs[i], hs.rs[j] = hs.rs[j], hs.rs[i]
	hs.ms[i], hs.ms[j] = hs.ms[j], hs.ms[i]
	hs.xs[i], hs.xs[j] = hs.xs[j], hs.xs[i]
	hs.ys[i], hs.ys[j] = hs.ys[j], hs.ys[i]
	hs.zs[i], hs.zs[j] = hs.zs[j], hs.zs[i]
	hs.rids[i], hs.rids[j] = hs.rids[j], hs.rids[i]
}

func RockstarConvert(
	inFile, outFile string, vars *VarColumns, cosmo *io.CosmologyHeader,
) error {
	valIdxs := vars.Columns
	for i := range valIdxs {
		if valIdxs[i] == -1 {
			valIdxs = valIdxs[:i]
			break
		}
	}

	cols, err := readTable(inFile, valIdxs)
	if err != nil {
		return err
	}

	f, err := os.Create(outFile)
	if err != nil {
		return err
	}
	defer f.Close()

	err = binary.Write(f, binary.LittleEndian, int64(len(cols[0])))
	if err != nil {
		return err
	}
	for _, col := range cols {
		err := binary.Write(f, binary.LittleEndian, col)
		if err != nil {
			return err
		}
	}

	return nil
}

type idxSet struct {
	xs   []float64
	idxs []int
}

func (set idxSet) Less(i, j int) bool { return set.xs[i] < set.xs[j] }
func (set idxSet) Len() int           { return len(set.xs) }
func (set idxSet) Swap(i, j int) {
	set.xs[i], set.xs[j] = set.xs[j], set.xs[i]
	set.idxs[i], set.idxs[j] = set.idxs[j], set.idxs[i]
}

func idxSort(xs []float64) []int {
	xsCopy := make([]float64, len(xs))
	copy(xsCopy, xs)
	idxs := make([]int, len(xs))
	for i := range idxs {
		idxs[i] = i
	}

	set := idxSet{}
	set.idxs = idxs
	set.xs = xsCopy
	sort.Sort(set)
	return idxs
}

func RockstarConvertTopN(
	inFile, outFile string, n int, vars *VarColumns, cosmo *io.CosmologyHeader,
) error {
	valIdxs := vars.Columns
	for i := range valIdxs {
		if valIdxs[i] == -1 {
			valIdxs = valIdxs[:i]
			break
		}
	}

	cols, err := readTable(inFile, valIdxs)
	if err != nil {
		return err
	}

	if n > len(cols[0]) {
		n = len(cols[0])
	}

	idxs := idxSort(vars.GetColumn(cols, "M200m", cosmo))[len(cols[0])-n:]

	outCols := make([][]float64, len(cols))
	for i := range cols {
		outCols[i] = make([]float64, len(idxs))
	}

	for j := range cols {
		for i, idx := range idxs {
			outCols[j][i] = cols[j][idx]
		}
	}

	f, err := os.Create(outFile)
	if err != nil {
		return err
	}
	defer f.Close()

	err = binary.Write(f, binary.LittleEndian, int64(n))
	if err != nil {
		return err
	}
	for _, col := range outCols {
		err := binary.Write(f, binary.LittleEndian, col)
		if err != nil {
			return err
		}
	}

	return nil
}

func ReadBinaryRockstar(
	file string, vc *VarColumns,
) (ids []int, rawCols [][]float64, err error) {
	return readRockstarVals(file, binaryColGetter, vc)
}

func readRockstarVals(
	file string, getter colGetter, vc *VarColumns,
) (ids []int, rawCols [][]float64, err error) {

	colIdxs := make([]int, vc.NBinary)
	for i := range colIdxs { colIdxs[i] = i }

	rawCols, err = getter(file, colIdxs)
	if err != nil { return nil, nil, err }

	ids = vc.GetIDs(rawCols)
	return ids, rawCols, err
}

type colGetter func(file string, colIdxs []int) ([][]float64, error)

func binaryColGetter(file string, colIdxs []int) ([][]float64, error) {
	// TODO: make this not slow as molasses. Just read everything in one go.
	f, err := os.Open(file)
	if err != nil {
		return nil, err
	}

	n := int64(0)
	err = binary.Read(f, binary.LittleEndian, &n)
	if err != nil {
		return nil, err
	}

	jump := n * 8
	cols := make([][]float64, len(colIdxs))
	for i := range cols {
		cols[i] = make([]float64, n)
	}
	for i, colIdx := range colIdxs {
		_, err = f.Seek(8+jump*int64(colIdx), 0)
		if err != nil {
			return nil, err
		}
		err = binary.Read(f, binary.LittleEndian, cols[i])
		if err != nil {
			return nil, err
		}
	}
	return cols, nil
}

func readTable(file string, colIdxs []int) ([][]float64, error) {
	// TODO: Heavily optimize this.

	f, err := os.Open(file)
	if err != nil {
		return nil, err
	}

	data, err := ioutil.ReadAll(f)
	if err != nil {
		return nil, err
	}

	_, floats, err := catalog.Parse(data, nil, colIdxs)
	return floats, err
}
<commit_msg>Replaced old catalog.Parse() call with catalog.ReadFile()<commit_after>package halo

import (
	"encoding/binary"
	"os"
	"sort"

	"github.com/phil-mansfield/shellfish/cmd/catalog"
	"github.com/phil-mansfield/shellfish/io"
)

// halos allows for arrays of halo properties to be sorted simultaneously.
type halos struct {
	rids               []int
	xs, ys, zs, ms, rs []float64
}

// This type is a look into the heart of darkness.
type VarColumns struct {
	ColumnLookup map[string]int
	Names []string
	Columns []int
	Generator []string
	NBinary int
	RadiusUnits string
}

func NewVarColumns(
	names []string, columns []int64, radiusUnits string,
) *VarColumns {
	vc :=&VarColumns{}
	vc.ColumnLookup = make(map[string]int)

	for i := range names {
		vc.ColumnLookup[names[i]] = i
		vc.Names = append(vc.Names, names[i])
		vc.Columns = append(vc.Columns, int(columns[i]))
	}
	vc.Generator = make([]string, len(vc.Names))
	vc.NBinary = len(vc.Names)

	mNames := []string{"M200m", "M200c", "M500c", "M2500c"}
	rNames := []string{"R200m", "R200c", "R500c", "R2500c"}
	for i, _ := range mNames {
		_, rOk := vc.ColumnLookup[rNames[i]]
		_, mOk := vc.ColumnLookup[mNames[i]]
		if mOk && !rOk {
			vc.ColumnLookup[rNames[i]] = len(vc.Names)
			vc.Names = append(vc.Names, rNames[i])
			vc.Columns = append(vc.Columns, -1)
			vc.Generator = append(vc.Generator, mNames[i])
		}
	}
	vc.RadiusUnits = radiusUnits

	return vc
}

func (vc *VarColumns) GetColumn(
	cols [][]float64, name string, cosmo *io.CosmologyHeader,
) []float64 {
	col, _ := vc.ColumnLookup[name]
	if vc.Generator[col] == "" { return cols[col] }

	gCol, _ := vc.ColumnLookup[vc.Generator[col]]
	ms := cols[gCol]
	rs := make([]float64, len(ms))

	rad, _ := RadiusFromString(name)

	rad.Radius(cosmo, ms, rs)
	// Convert from Mpc/h to the units of the halo catalog.
	ucf := UnitConversionFactor(vc.RadiusUnits, cosmo)
	for i := range rs {
		rs[i] /= ucf
	}

	return rs
}

func (vc *VarColumns) GetIDs(cols [][]float64) []int {
	col, _ := vc.ColumnLookup["ID"]

	out := make([]int, len(cols[col]))
	for i := range out { out[i] = int(cols[col][i]) }
	return out
}

func (hs *halos) Len() int           { return len(hs.rs) }
func (hs *halos) Less(i, j int) bool { return hs.rs[i] < hs.rs[j] }
func (hs *halos) Swap(i, j int) {
	hs.rs[i], hs.rs[j] = hs.rs[j], hs.rs[i]
	hs.ms[i], hs.ms[j] = hs.ms[j], hs.ms[i]
	hs.xs[i], hs.xs[j] = hs.xs[j], hs.xs[i]
	hs.ys[i], hs.ys[j] = hs.ys[j], hs.ys[i]
	hs.zs[i], hs.zs[j] = hs.zs[j], hs.zs[i]
	hs.rids[i], hs.rids[j] = hs.rids[j], hs.rids[i]
}

func RockstarConvert(
	inFile, outFile string, vars *VarColumns, cosmo *io.CosmologyHeader,
) error {
	valIdxs := vars.Columns
	for i := range valIdxs {
		if valIdxs[i] == -1 {
			valIdxs = valIdxs[:i]
			break
		}
	}

	cols, err := readTable(inFile, valIdxs)
	if err != nil {
		return err
	}

	f, err := os.Create(outFile)
	if err != nil {
		return err
	}
	defer f.Close()

	err = binary.Write(f, binary.LittleEndian, int64(len(cols[0])))
	if err != nil {
		return err
	}
	for _, col := range cols {
		err := binary.Write(f, binary.LittleEndian, col)
		if err != nil {
			return err
		}
	}

	return nil
}

type idxSet struct {
	xs   []float64
	idxs []int
}

func (set idxSet) Less(i, j int) bool { return set.xs[i] < set.xs[j] }
func (set idxSet) Len() int           { return len(set.xs) }
func (set idxSet) Swap(i, j int) {
	set.xs[i], set.xs[j] = set.xs[j], set.xs[i]
	set.idxs[i], set.idxs[j] = set.idxs[j], set.idxs[i]
}

func idxSort(xs []float64) []int {
	xsCopy := make([]float64, len(xs))
	copy(xsCopy, xs)
	idxs := make([]int, len(xs))
	for i := range idxs {
		idxs[i] = i
	}

	set := idxSet{}
	set.idxs = idxs
	set.xs = xsCopy
	sort.Sort(set)
	return idxs
}

func RockstarConvertTopN(
	inFile, outFile string, n int, vars *VarColumns, cosmo *io.CosmologyHeader,
) error {
	valIdxs := vars.Columns
	for i := range valIdxs {
		if valIdxs[i] == -1 {
			valIdxs = valIdxs[:i]
			break
		}
	}

	cols, err := readTable(inFile, valIdxs)
	if err != nil {
		return err
	}

	if n > len(cols[0]) {
		n = len(cols[0])
	}

	idxs := idxSort(vars.GetColumn(cols, "M200m", cosmo))[len(cols[0])-n:]

	outCols := make([][]float64, len(cols))
	for i := range cols {
		outCols[i] = make([]float64, len(idxs))
	}

	for j := range cols {
		for i, idx := range idxs {
			outCols[j][i] = cols[j][idx]
		}
	}

	f, err := os.Create(outFile)
	if err != nil {
		return err
	}
	defer f.Close()

	err = binary.Write(f, binary.LittleEndian, int64(n))
	if err != nil {
		return err
	}
	for _, col := range outCols {
		err := binary.Write(f, binary.LittleEndian, col)
		if err != nil {
			return err
		}
	}

	return nil
}

func ReadBinaryRockstar(
	file string, vc *VarColumns,
) (ids []int, rawCols [][]float64, err error) {
	return readRockstarVals(file, binaryColGetter, vc)
}

func readRockstarVals(
	file string, getter colGetter, vc *VarColumns,
) (ids []int, rawCols [][]float64, err error) {

	colIdxs := make([]int, vc.NBinary)
	for i := range colIdxs { colIdxs[i] = i }

	rawCols, err = getter(file, colIdxs)
	if err != nil { return nil, nil, err }

	ids = vc.GetIDs(rawCols)
	return ids, rawCols, err
}

type colGetter func(file string, colIdxs []int) ([][]float64, error)

func binaryColGetter(file string, colIdxs []int) ([][]float64, error) {
	// TODO: make this not slow as molasses. Just read everything in one go.
	f, err := os.Open(file)
	if err != nil {
		return nil, err
	}

	n := int64(0)
	err = binary.Read(f, binary.LittleEndian, &n)
	if err != nil {
		return nil, err
	}

	jump := n * 8
	cols := make([][]float64, len(colIdxs))
	for i := range cols {
		cols[i] = make([]float64, n)
	}
	for i, colIdx := range colIdxs {
		_, err = f.Seek(8+jump*int64(colIdx), 0)
		if err != nil {
			return nil, err
		}
		err = binary.Read(f, binary.LittleEndian, cols[i])
		if err != nil {
			return nil, err
		}
	}
	return cols, nil
}

func readTable(file string, colIdxs []int) ([][]float64, error) {
	// TODO: Heavily optimize this.

	_, floats, err := catalog.ReadFile(file, nil, colIdxs)
	return floats, err
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"

	"github.com/dtan4/k8ship/kubernetes"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

// historyCmd represents the history command
var historyCmd = &cobra.Command{
	Use:   "history",
	Short: "View deployment history",
	RunE:  doHistory,
}

var historyOpts = struct {
	namespace string
}{}

func doHistory(cmd *cobra.Command, args []string) error {
	client, err := kubernetes.NewClient(rootOpts.annotationPrefix, rootOpts.kubeconfig, rootOpts.context)
	if err != nil {
		return errors.Wrap(err, "failed to create Kubernetes client")
	}

	ds, err := client.ListDeployments(historyOpts.namespace)
	if err != nil {
		return errors.Wrap(err, "failed to retrieve Deployments")
	}

	if len(ds) == 0 {
		return errors.Errorf("no Deployment found in namespace %s", historyOpts.namespace)
	}

	for _, d := range ds {
		fmt.Println("===== " + d.Name())

		rs, err := client.ListReplicaSets(d)
		if err != nil {
			return errors.Wrap(err, "failed to retrieve ReplicaSets")
		}

		for _, r := range rs {
			fmt.Printf("%s %s %s %s\n", r.Revision(), r.Name(), r.CreatedAt(), r.Images())
		}
	}

	return nil
}

func init() {
	RootCmd.AddCommand(historyCmd)

	historyCmd.Flags().StringVarP(&historyOpts.namespace, "namespace", "n", kubernetes.DefaultNamespace(), "Kubernetes namespace")
}
<commit_msg>Sort history by createdAt<commit_after>package cmd

import (
	"fmt"
	"sort"

	"github.com/dtan4/k8ship/kubernetes"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

// historyCmd represents the history command
var historyCmd = &cobra.Command{
	Use:   "history",
	Short: "View deployment history",
	RunE:  doHistory,
}

var historyOpts = struct {
	namespace string
}{}

func doHistory(cmd *cobra.Command, args []string) error {
	client, err := kubernetes.NewClient(rootOpts.annotationPrefix, rootOpts.kubeconfig, rootOpts.context)
	if err != nil {
		return errors.Wrap(err, "failed to create Kubernetes client")
	}

	ds, err := client.ListDeployments(historyOpts.namespace)
	if err != nil {
		return errors.Wrap(err, "failed to retrieve Deployments")
	}

	if len(ds) == 0 {
		return errors.Errorf("no Deployment found in namespace %s", historyOpts.namespace)
	}

	for _, d := range ds {
		fmt.Println("===== " + d.Name())

		rs, err := client.ListReplicaSets(d)
		if err != nil {
			return errors.Wrap(err, "failed to retrieve ReplicaSets")
		}

		lines := formatHistory(rs)
		sort.Sort(sort.Reverse(sort.StringSlice(lines)))

		for _, l := range lines {
			fmt.Println(l)
		}
	}

	return nil
}

func formatHistory(rs []*kubernetes.ReplicaSet) []string {
	lines := make([]string, 0, len(rs))

	for _, r := range rs {
		lines = append(lines, fmt.Sprintf("%s %s %s", r.CreatedAt(), r.Revision(), r.Images()))
	}

	return lines
}

func init() {
	RootCmd.AddCommand(historyCmd)

	historyCmd.Flags().StringVarP(&historyOpts.namespace, "namespace", "n", kubernetes.DefaultNamespace(), "Kubernetes namespace")
}
<|endoftext|>


<commit_before>// Copyright 2014 Volker Dobler.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"path"
	"sort"
	"strings"
	"time"

	"github.com/vdobler/ht/cookiejar"
	"github.com/vdobler/ht/ht"
	"github.com/vdobler/ht/sanitize"
	"github.com/vdobler/ht/scope"
	"github.com/vdobler/ht/suite"
)

var cmdExec = &Command{
	RunSuites:   runExecute,
	Usage:       "exec [options] <suite>...",
	Description: "generate request and test response",
	Flag:        flag.NewFlagSet("run", flag.ContinueOnError),
	Help: `
Exec loads the given suites and executes them.
Variables set with the -D flag overwrite variables read from file with -Dfile.
The current variable assignment at the end of a suite carries over to the next
suite. All suites (which keep cookies) share a common jar if cookies are
loaded via -cookie flag; otherwise each suite has its own cookiejar.

As a convenience exec recognises the /... syntax of the go tool to load all
*.ssuite files below dir: 'ht exec dir/...' is just syntactical suggar for
'ht exec $(find dir -type f -name \*.suite | sort)'.

The exit code is 3 if bogus tests or checks are found, 2 if test errors
are present, 1 if only check failures occurred and 0 if everything passed,
nothing was executed or everything was skipped. Note that the status of
Teardown test are ignored while determining the exit code.

A suite and the used tests may be given as an archive file like this:
<entrypoint>@<archivefile>. Here <entrypoint> is the formal suite filename
in the filesytem file <archivefile>. Archivefiles are collection of HJSON
objects as described in the main help (run '$ ht help').
`,
}

var carryVars bool

func init() {
	addOnlyFlag(cmdExec.Flag)
	addSkipFlag(cmdExec.Flag)

	addTestFlags(cmdExec.Flag)
	addOutputFlag(cmdExec.Flag)

	cmdExec.Flag.BoolVar(&carryVars, "carry", false,
		"carry variables from finished suite to next suite")

}

func runExecute(cmd *Command, suites []*suite.RawSuite) {
	if ssilent {
		silent = true
	}
	prepareHT()
	jar := loadCookies()

	prepareOutputDir()
	var errors ht.ErrorList

	outcome, err := executeSuites(suites, variablesFlag, jar)
	errors = errors.Append(err)
	err = reportOverall(outcome)
	errors = errors.Append(err)

	terminate(outcome, errors)
}

// PrepareHT and print diagnostics.
func prepareHT() {
	// Set several parameters of package ht.
	if randomSeed == 0 {
		randomSeed = time.Now().UnixNano()
	}
	scope.ResetCounter <- counterSeed
	scope.Random = rand.New(rand.NewSource(randomSeed))
	ht.PhantomJSExecutable = phantomjs
	if !silent {
		fmt.Printf("Seeding random number generator with %d.\n", randomSeed)
		fmt.Printf("Resetting global counter to %d.\n", counterSeed)
		fmt.Printf("Using %q as PhantomJS executable.\n", phantomjs)
	}
	if skipTLSVerify {
		if !silent {
			fmt.Println("Skipping verification of TLS certificates presented by any server.")
		}
		ht.Transport.TLSClientConfig.InsecureSkipVerify = true
	}

	if !silent {
		// Log variables and values sorted by variable name.
		varnames := make([]string, 0, len(variablesFlag))
		for v := range variablesFlag {
			varnames = append(varnames, v)
		}
		sort.Strings(varnames)
		for _, v := range varnames {
			fmt.Printf("Variable %s = %q\n", v, variablesFlag[v])
		}
	}
}

// Panics if selected output dir cannot be created.
func prepareOutputDir() {
	if outputDir == "/dev/null" {
		mute = true
	} else if outputDir == "" {
		outputDir = time.Now().Format("2006-01-02_15h04m05s")
	}

	if !mute {
		err := os.MkdirAll(outputDir, 0766)
		if err != nil {
			log.Panic(err)
		}
	}
}

// execute suites one by one saving each suite to disk once finished.
// Returns the accumulated overall result.
func executeSuites(suites []*suite.RawSuite, variables map[string]string, jar *cookiejar.Jar) (*accumulator, error) {
	bufferedStdout := bufio.NewWriterSize(os.Stdout, 256)
	defer bufferedStdout.Flush()
	logger := log.New(bufferedStdout, "", 0)
	errors := ht.ErrorList{}
	var err error

	if !mute {
		if outputDir == "" {
			outputDir = time.Now().Format("2006-01-02_15h04m05s")
		}
		err = os.MkdirAll(outputDir, 0766)
		errors = errors.Append(err)
	}

	accum := newAccumulator()
	for i, s := range suites {
		if !ssilent {
			logger.Println("Starting Suite", i+1, s.Name, s.File.Name)
		}
		outcome := s.Execute(variables, jar, logger)
		bufferedStdout.Flush()

		accum.update(outcome)

		if carryVars {
			variables = outcome.FinalVariables // carry over variables ???
		}
		if !silent {
			err = outcome.PrintReport(os.Stdout)
		} else if !ssilent {
			err = outcome.PrintShortReport(os.Stdout)
			fmt.Println()
		}
		errors = errors.Append(err)

		err = saveSingle(outputDir, outcome)
		errors = errors.Append(err)
		if len(suites) > 1 {
			err := saveOverallReport(outputDir, accum)
			errors = errors.Append(err)
		}

	}
	return accum, errors.AsError()
}

// ----------------------------------------------------------------------------
// Reporting functions

// saveSingle takes care of dumping the suite s into a subfolder of
// outputdir. It will produce:
//     _Report_.html  with accomaning files for the response bodies
//     junit-report.xml
//     result.txt
//     variables.json
//     cookies.json
func saveSingle(outputDir string, s *suite.Suite) error {
	if mute || outputDir == "/dev/null" {
		return nil
	}

	dirname := path.Join(outputDir, sanitize.Filename(s.Name))
	fmt.Printf("Saving result of suite %q to folder %q.\n", s.Name, dirname)
	err := os.MkdirAll(dirname, 0766)
	if err != nil {
		return err
	}

	errors := ht.ErrorList{}
	err = suite.HTMLReport(dirname, s)
	errors = errors.Append(err)

	file, err := os.Create(path.Join(dirname, "result.txt"))
	errors = errors.Append(err)
	if err == nil {
		err = s.PrintReport(file)
		errors = errors.Append(err)
		errors = errors.Append(file.Close())
	}

	cwd, err := os.Getwd()
	errors = errors.Append(err)
	reportURL := "file://" + path.Join(cwd, dirname, "_Report_.html")
	fmt.Printf("See %s\n", reportURL)

	junit, err := s.JUnit4XML()
	errors = errors.Append(err)
	if err == nil {
		err = ioutil.WriteFile(path.Join(dirname, "junit-report.xml"),
			[]byte(junit), 0666)
		errors = errors.Append(err)
	}

	// TODO: handle errors
	err = saveVariables(s.FinalVariables, path.Join(dirname, "variables.json"))
	errors = errors.Append(err)
	err = saveCookiesFromJar(s.Jar, path.Join(dirname, "cookies.json"))
	errors = errors.Append(err)

	return errors.AsError()
}

// reportOverall summary data to stdout and save final variables and cookies.
func reportOverall(a *accumulator) error {
	var errors ht.ErrorList
	var err error
	// Save consolidated variables if required.
	if vardump != "" && !mute {
		err = saveVariables(a.vars, vardump)
		errors = errors.Append(err)
	}

	// Save consolidated cookies if required.
	if cookiedump != "" && !mute {
		err = saveCookies(a.cookies, cookiedump)
		errors = errors.Append(err)
	}

	if !ssilent {
		fmt.Println()
		fmt.Printf("Total %d,  Passed %d,  Skipped %d,  Errored %d,  Failed %d,  Bogus %d\n",
			a.total, a.pass, a.skip, a.err, a.fail, a.bogus)
		fmt.Println(strings.ToUpper(a.status.String()))
	}

	return errors.AsError()
}

// TODO: handle errors, preparse template, cleanup mess, learn to program
func saveOverallReport(dirname string, accum *accumulator) error {
	if mute {
		return nil
	}

	tmplSrc := `<!DOCTYPE html>

  <meta http-equiv="content-type" content="text/html; charset=UTF-8" />

  <title>Overall Results</title>

Overall Result

    {{range .}}

          {{.Status}} {{.Name}}

    {{end}}

`
	templ := template.Must(template.New("report").Parse(tmplSrc))
	buf := &bytes.Buffer{}
	err := templ.Execute(buf, accum.suites)
	if err != nil {
		return err
	}

	file := dirname + "/_Report_.html"
	if err := ioutil.WriteFile(file, buf.Bytes(), 0666); err != nil {
		log.Panicf("Failed to write file: %q with error %s", file, err)
	}
	cwd, err := os.Getwd()
	if err == nil {
		reportURL := "file://" + path.Join(cwd, file)
		fmt.Printf("Overall: %s\n", reportURL)
	}
	return err
}

// terminate cmd/ht with proper exit status depending in present errors and
// the outcome of the executed suites.
func terminate(outcome *accumulator, errors ht.ErrorList) {
	if errors.AsError() != nil {
		fmt.Fprintln(os.Stderr, "Error encountered during execution:")
		for _, msg := range errors.AsStrings() {
			fmt.Fprintln(os.Stderr, msg)
		}
		os.Exit(7)
	}

	switch outcome.status {
	case ht.NotRun, ht.Skipped, ht.Pass:
		os.Exit(0)
	case ht.Fail:
		os.Exit(1)
	case ht.Error:
		os.Exit(2)
	case ht.Bogus:
		os.Exit(3)
	}

}

// ----------------------------------------------------------------------------
// Helpers for saving and loading variables and cookies.

func saveVariables(vars map[string]string, filename string) error {
	if mute {
		return nil
	}
	b, err := json.MarshalIndent(vars, "    ", "")
	if err != nil {
		return nil
	}
	return ioutil.WriteFile(filename, b, 0666)
}

func saveCookiesFromJar(jar *cookiejar.Jar, filename string) error {
	if jar == nil {
		return nil
	}

	cookies := make(map[string]cookiejar.Entry)
	for _, tld := range jar.ETLDsPlus1(nil) {
		for _, cookie := range jar.Entries(tld, nil) {
			id := cookie.ID()
			cookies[id] = cookie
		}
	}
	return saveCookies(cookies, filename)
}

func saveCookies(cookies map[string]cookiejar.Entry, filename string) error {
	if mute {
		return nil
	}
	b, err := json.MarshalIndent(cookies, "    ", "")
	if err != nil {
		return nil
	}
	return ioutil.WriteFile(filename, b, 0666)
}

func loadCookies() *cookiejar.Jar {
	if cookie == "" {
		return nil
	}
	buf, err := ioutil.ReadFile(cookie)
	if err != nil {
		log.Panicf("Cannot read cookie file: %s", err)
	}

	cookies := make(map[string]cookiejar.Entry)
	err = json.Unmarshal(buf, &cookies)
	if err != nil {
		log.Panicf("Cannot decode cookie file: %s", err)
	}
	cs := make([]cookiejar.Entry, 0, len(cookies))
	for _, c := range cookies {
		cs = append(cs, c)
	}

	jar, _ := cookiejar.New(nil)
	jar.LoadEntries(cs)
	return jar
}

// ----------------------------------------------------------------------------
// Accumulator of several suites to collect statistical summaries.

type accumulator struct {
	status  ht.Status
	vars    map[string]string
	cookies map[string]cookiejar.Entry

	total, notrun, skip, pass, err, fail, bogus int

	suites []struct {
		Name, Path, Status string
	}
}

func newAccumulator() *accumulator {
	return &accumulator{
		vars:    make(map[string]string),
		cookies: make(map[string]cookiejar.Entry),
	}
}

func (a *accumulator) update(s *suite.Suite) {
	// Reporting
	a.suites = append(a.suites, struct {
		Name, Path, Status string
	}{
		s.Name,
		sanitize.Filename(s.Name),
		strings.ToUpper(s.Status.String()),
	})

	// Status
	if s.Status > a.status {
		a.status = s.Status
	}

	// Variables
	for name, value := range s.FinalVariables {
		a.vars[name] = value
	}

	// Cookies
	if s.Jar != nil {
		for _, tld := range s.Jar.ETLDsPlus1(nil) {
			for _, cookie := range s.Jar.Entries(tld, nil) {
				id := cookie.ID()
				a.cookies[id] = cookie
			}
		}
	}

	for _, t := range s.Tests {
		switch t.Status {
		case ht.NotRun:
			a.notrun++
		case ht.Skipped:
			a.skip++
		case ht.Pass:
			a.pass++
		case ht.Error:
			a.err++
		case ht.Fail:
			a.fail++
		case ht.Bogus:
			a.bogus++
		}
		a.total++
	}
}
<commit_msg>cmd/ht: display aggregated test result in overall report<commit_after>// Copyright 2014 Volker Dobler.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"path"
	"sort"
	"strings"
	"time"

	"github.com/vdobler/ht/cookiejar"
	"github.com/vdobler/ht/ht"
	"github.com/vdobler/ht/sanitize"
	"github.com/vdobler/ht/scope"
	"github.com/vdobler/ht/suite"
)

var cmdExec = &Command{
	RunSuites:   runExecute,
	Usage:       "exec [options] <suite>...",
	Description: "generate request and test response",
	Flag:        flag.NewFlagSet("run", flag.ContinueOnError),
	Help: `
Exec loads the given suites and executes them.
Variables set with the -D flag overwrite variables read from file with -Dfile.
The current variable assignment at the end of a suite carries over to the next
suite. All suites (which keep cookies) share a common jar if cookies are
loaded via -cookie flag; otherwise each suite has its own cookiejar.

As a convenience exec recognises the /... syntax of the go tool to load all
*.ssuite files below dir: 'ht exec dir/...' is just syntactical suggar for
'ht exec $(find dir -type f -name \*.suite | sort)'.

The exit code is 3 if bogus tests or checks are found, 2 if test errors
are present, 1 if only check failures occurred and 0 if everything passed,
nothing was executed or everything was skipped. Note that the status of
Teardown test are ignored while determining the exit code.

A suite and the used tests may be given as an archive file like this:
<entrypoint>@<archivefile>. Here <entrypoint> is the formal suite filename
in the filesytem file <archivefile>. Archivefiles are collection of HJSON
objects as described in the main help (run '$ ht help').
`,
}

var carryVars bool

func init() {
	addOnlyFlag(cmdExec.Flag)
	addSkipFlag(cmdExec.Flag)

	addTestFlags(cmdExec.Flag)
	addOutputFlag(cmdExec.Flag)

	cmdExec.Flag.BoolVar(&carryVars, "carry", false,
		"carry variables from finished suite to next suite")

}

func runExecute(cmd *Command, suites []*suite.RawSuite) {
	if ssilent {
		silent = true
	}
	prepareHT()
	jar := loadCookies()

	prepareOutputDir()
	var errors ht.ErrorList

	outcome, err := executeSuites(suites, variablesFlag, jar)
	errors = errors.Append(err)
	err = reportOverall(outcome)
	errors = errors.Append(err)

	terminate(outcome, errors)
}

// PrepareHT and print diagnostics.
func prepareHT() {
	// Set several parameters of package ht.
	if randomSeed == 0 {
		randomSeed = time.Now().UnixNano()
	}
	scope.ResetCounter <- counterSeed
	scope.Random = rand.New(rand.NewSource(randomSeed))
	ht.PhantomJSExecutable = phantomjs
	if !silent {
		fmt.Printf("Seeding random number generator with %d.\n", randomSeed)
		fmt.Printf("Resetting global counter to %d.\n", counterSeed)
		fmt.Printf("Using %q as PhantomJS executable.\n", phantomjs)
	}
	if skipTLSVerify {
		if !silent {
			fmt.Println("Skipping verification of TLS certificates presented by any server.")
		}
		ht.Transport.TLSClientConfig.InsecureSkipVerify = true
	}

	if !silent {
		// Log variables and values sorted by variable name.
		varnames := make([]string, 0, len(variablesFlag))
		for v := range variablesFlag {
			varnames = append(varnames, v)
		}
		sort.Strings(varnames)
		for _, v := range varnames {
			fmt.Printf("Variable %s = %q\n", v, variablesFlag[v])
		}
	}
}

// Panics if selected output dir cannot be created.
func prepareOutputDir() {
	if outputDir == "/dev/null" {
		mute = true
	} else if outputDir == "" {
		outputDir = time.Now().Format("2006-01-02_15h04m05s")
	}

	if !mute {
		err := os.MkdirAll(outputDir, 0766)
		if err != nil {
			log.Panic(err)
		}
	}
}

// execute suites one by one saving each suite to disk once finished.
// Returns the accumulated overall result.
func executeSuites(suites []*suite.RawSuite, variables map[string]string, jar *cookiejar.Jar) (*accumulator, error) {
	bufferedStdout := bufio.NewWriterSize(os.Stdout, 256)
	defer bufferedStdout.Flush()
	logger := log.New(bufferedStdout, "", 0)
	errors := ht.ErrorList{}
	var err error

	if !mute {
		if outputDir == "" {
			outputDir = time.Now().Format("2006-01-02_15h04m05s")
		}
		err = os.MkdirAll(outputDir, 0766)
		errors = errors.Append(err)
	}

	accum := newAccumulator()
	for i, s := range suites {
		if !ssilent {
			logger.Println("Starting Suite", i+1, s.Name, s.File.Name)
		}
		outcome := s.Execute(variables, jar, logger)
		bufferedStdout.Flush()

		accum.update(outcome)

		if carryVars {
			variables = outcome.FinalVariables // carry over variables ???
		}
		if !silent {
			err = outcome.PrintReport(os.Stdout)
		} else if !ssilent {
			err = outcome.PrintShortReport(os.Stdout)
			fmt.Println()
		}
		errors = errors.Append(err)

		err = saveSingle(outputDir, outcome)
		errors = errors.Append(err)
		if len(suites) > 1 {
			err := saveOverallReport(outputDir, accum)
			errors = errors.Append(err)
		}

	}
	return accum, errors.AsError()
}

// ----------------------------------------------------------------------------
// Reporting functions

// saveSingle takes care of dumping the suite s into a subfolder of
// outputdir. It will produce:
//     _Report_.html  with accomaning files for the response bodies
//     junit-report.xml
//     result.txt
//     variables.json
//     cookies.json
func saveSingle(outputDir string, s *suite.Suite) error {
	if mute || outputDir == "/dev/null" {
		return nil
	}

	dirname := path.Join(outputDir, sanitize.Filename(s.Name))
	fmt.Printf("Saving result of suite %q to folder %q.\n", s.Name, dirname)
	err := os.MkdirAll(dirname, 0766)
	if err != nil {
		return err
	}

	errors := ht.ErrorList{}
	err = suite.HTMLReport(dirname, s)
	errors = errors.Append(err)

	file, err := os.Create(path.Join(dirname, "result.txt"))
	errors = errors.Append(err)
	if err == nil {
		err = s.PrintReport(file)
		errors = errors.Append(err)
		errors = errors.Append(file.Close())
	}

	cwd, err := os.Getwd()
	errors = errors.Append(err)
	reportURL := "file://" + path.Join(cwd, dirname, "_Report_.html")
	fmt.Printf("See %s\n", reportURL)

	junit, err := s.JUnit4XML()
	errors = errors.Append(err)
	if err == nil {
		err = ioutil.WriteFile(path.Join(dirname, "junit-report.xml"),
			[]byte(junit), 0666)
		errors = errors.Append(err)
	}

	// TODO: handle errors
	err = saveVariables(s.FinalVariables, path.Join(dirname, "variables.json"))
	errors = errors.Append(err)
	err = saveCookiesFromJar(s.Jar, path.Join(dirname, "cookies.json"))
	errors = errors.Append(err)

	return errors.AsError()
}

// reportOverall summary data to stdout and save final variables and cookies.
func reportOverall(a *accumulator) error {
	var errors ht.ErrorList
	var err error
	// Save consolidated variables if required.
	if vardump != "" && !mute {
		err = saveVariables(a.vars, vardump)
		errors = errors.Append(err)
	}

	// Save consolidated cookies if required.
	if cookiedump != "" && !mute {
		err = saveCookies(a.cookies, cookiedump)
		errors = errors.Append(err)
	}

	if !ssilent {
		fmt.Println()
		fmt.Printf("Total %d,  Passed %d,  Skipped %d,  Errored %d,  Failed %d,  Bogus %d\n",
			a.Total, a.Pass, a.Skip, a.Err, a.Fail, a.Bogus)
		fmt.Println(strings.ToUpper(a.Status.String()))
	}

	return errors.AsError()
}

var overallReportTemplate = template.Must(template.New("report").Parse(`<!DOCTYPE html>

  <meta http-equiv="content-type" content="text/html; charset=UTF-8" />

  <title>Overall Results</title>

Overall Result: {{.Status}}

    {{range .Suites}}

          {{.Status}} {{.Name}}

    {{end}}

        Total {{.Total}},
        Passed {{.Pass}},
        Skipped {{.Skip}},
        Errored {{.Err}},
        Failed {{.Fail}},
        Bogus {{.Bogus}}

`))

func saveOverallReport(dirname string, accum *accumulator) error {
	if mute {
		return nil
	}

	filename := path.Join(dirname, "_Report_.html")
	file, err := os.Create(filename)
	if err != nil {
		return err
	}
	err = overallReportTemplate.Execute(file, accum)
	if err != nil {
		return err
	}
	err = file.Close()
	if err != nil {
		return err
	}

	cwd, err := os.Getwd()
	if err == nil {
		reportURL := "file://" + path.Join(cwd, filename)
		fmt.Printf("Overall: %s\n", reportURL)
	}
	return err
}

// terminate cmd/ht with proper exit status depending in present errors and
// the outcome of the executed suites.
func terminate(outcome *accumulator, errors ht.ErrorList) {
	if errors.AsError() != nil {
		fmt.Fprintln(os.Stderr, "Error encountered during execution:")
		for _, msg := range errors.AsStrings() {
			fmt.Fprintln(os.Stderr, msg)
		}
		os.Exit(7)
	}

	switch outcome.Status {
	case ht.NotRun, ht.Skipped, ht.Pass:
		os.Exit(0)
	case ht.Fail:
		os.Exit(1)
	case ht.Error:
		os.Exit(2)
	case ht.Bogus:
		os.Exit(3)
	}

}

// ----------------------------------------------------------------------------
// Helpers for saving and loading variables and cookies.

func saveVariables(vars map[string]string, filename string) error {
	if mute {
		return nil
	}
	b, err := json.MarshalIndent(vars, "    ", "")
	if err != nil {
		return nil
	}
	return ioutil.WriteFile(filename, b, 0666)
}

func saveCookiesFromJar(jar *cookiejar.Jar, filename string) error {
	if jar == nil {
		return nil
	}

	cookies := make(map[string]cookiejar.Entry)
	for _, tld := range jar.ETLDsPlus1(nil) {
		for _, cookie := range jar.Entries(tld, nil) {
			id := cookie.ID()
			cookies[id] = cookie
		}
	}
	return saveCookies(cookies, filename)
}

func saveCookies(cookies map[string]cookiejar.Entry, filename string) error {
	if mute {
		return nil
	}
	b, err := json.MarshalIndent(cookies, "    ", "")
	if err != nil {
		return nil
	}
	return ioutil.WriteFile(filename, b, 0666)
}

func loadCookies() *cookiejar.Jar {
	if cookie == "" {
		return nil
	}
	buf, err := ioutil.ReadFile(cookie)
	if err != nil {
		log.Panicf("Cannot read cookie file: %s", err)
	}

	cookies := make(map[string]cookiejar.Entry)
	err = json.Unmarshal(buf, &cookies)
	if err != nil {
		log.Panicf("Cannot decode cookie file: %s", err)
	}
	cs := make([]cookiejar.Entry, 0, len(cookies))
	for _, c := range cookies {
		cs = append(cs, c)
	}

	jar, _ := cookiejar.New(nil)
	jar.LoadEntries(cs)
	return jar
}

// ----------------------------------------------------------------------------
// Accumulator of several suites to collect statistical summaries.

type accumulator struct {
	Status  ht.Status
	vars    map[string]string
	cookies map[string]cookiejar.Entry

	Total, Notrun, Skip, Pass, Err, Fail, Bogus int

	Suites []struct {
		Name, Path, Status string
	}
}

func newAccumulator() *accumulator {
	return &accumulator{
		vars:    make(map[string]string),
		cookies: make(map[string]cookiejar.Entry),
	}
}

func (a *accumulator) update(s *suite.Suite) {
	// Reporting
	a.Suites = append(a.Suites, struct {
		Name, Path, Status string
	}{
		s.Name,
		sanitize.Filename(s.Name),
		strings.ToUpper(s.Status.String()),
	})

	// Status
	if s.Status > a.Status {
		a.Status = s.Status
	}

	// Variables
	for name, value := range s.FinalVariables {
		a.vars[name] = value
	}

	// Cookies
	if s.Jar != nil {
		for _, tld := range s.Jar.ETLDsPlus1(nil) {
			for _, cookie := range s.Jar.Entries(tld, nil) {
				id := cookie.ID()
				a.cookies[id] = cookie
			}
		}
	}

	for _, t := range s.Tests {
		switch t.Status {
		case ht.NotRun:
			a.Notrun++
		case ht.Skipped:
			a.Skip++
		case ht.Pass:
			a.Pass++
		case ht.Error:
			a.Err++
		case ht.Fail:
			a.Fail++
		case ht.Bogus:
			a.Bogus++
		}
		a.Total++
	}
}
<|endoftext|>


<commit_before>// +build !windows

package cmd

import (
	"github.com/genzj/goTApaper/install"
	"github.com/spf13/cobra"
)

var installCmd = &cobra.Command{
	Use:   "install",
	Short: "Make goTApaper run at system startup",
	Long:  `Make goTApaper run at system startup`,
	Run: func(cmd *cobra.Command, args []string) {
		installEntry()
	},
}

func init() {
	RootCmd.AddCommand(installCmd)
}

func installEntry() {
	install.InstallStartUp()
}
<commit_msg>enable install command on windows<commit_after>package cmd

import (
	"github.com/genzj/goTApaper/install"
	"github.com/spf13/cobra"
)

var installCmd = &cobra.Command{
	Use:   "install",
	Short: "Make goTApaper run at system startup",
	Long:  `Make goTApaper run at system startup`,
	Run: func(cmd *cobra.Command, args []string) {
		installEntry()
	},
}

func init() {
	RootCmd.AddCommand(installCmd)
}

func installEntry() {
	install.InstallStartUp()
}
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package cmd

import (
	"fmt"
	"io"
	"os"
	"time"

	"launchpad.net/gnuflag"
	"launchpad.net/loggo"

	"launchpad.net/juju-core/juju/osenv"
)

// Log supplies the necessary functionality for Commands that wish to set up
// logging.
type Log struct {
	Path    string
	Verbose bool
	Debug   bool
	ShowLog bool
	Config  string
}

// AddFlags adds appropriate flags to f.
func (l *Log) AddFlags(f *gnuflag.FlagSet) {
	f.StringVar(&l.Path, "log-file", "", "path to write log to")
	// TODO(thumper): rename verbose to --show-log
	f.BoolVar(&l.Verbose, "v", false, "if set, log additional messages")
	f.BoolVar(&l.Verbose, "verbose", false, "if set, log additional messages")
	f.BoolVar(&l.Debug, "debug", false, "if set, log debugging messages")
	defaultLogConfig := os.Getenv(osenv.JujuLoggingConfig)
	f.StringVar(&l.Config, "log-config", defaultLogConfig, "specify log levels for modules")
	f.BoolVar(&l.ShowLog, "show-log", false, "if set, write the log file to stderr")
}

// Start starts logging using the given Context.
func (l *Log) Start(ctx *Context) error {
	if l.Path != "" {
		path := ctx.AbsPath(l.Path)
		target, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
		if err != nil {
			return err
		}
		writer := loggo.NewSimpleWriter(target, &loggo.DefaultFormatter{})
		err = loggo.RegisterWriter("logfile", writer, loggo.TRACE)
		if err != nil {
			return err
		}
	}
	level := loggo.WARNING
	if l.Verbose {
		ctx.Stdout.Write([]byte("verbose is deprecated with the current meaning, use show-log\n"))
		l.ShowLog = true
	}
	if l.ShowLog {
		level = loggo.INFO
	}
	if l.Debug {
		l.ShowLog = true
		level = loggo.DEBUG
	}

	if l.ShowLog {
		// We replace the default writer to use ctx.Stderr rather than os.Stderr.
		writer := loggo.NewSimpleWriter(ctx.Stderr, &loggo.DefaultFormatter{})
		_, err := loggo.ReplaceDefaultWriter(writer)
		if err != nil {
			return err
		}
	} else {
		loggo.RemoveWriter("default")
	}
	// Set the level on the root logger.
	loggo.GetLogger("").SetLogLevel(level)
	// Override the logging config with specified logging config.
	loggo.ConfigureLoggers(l.Config)
	return nil
}

// NewCommandLogWriter creates a loggo writer for registration
// by the callers of a command. This way the logged output can also
// be displayed otherwise, e.g. on the screen.
func NewCommandLogWriter(name string, out, err io.Writer) loggo.Writer {
	return &commandLogWriter{name, out, err}
}

// commandLogWriter filters the log messages for name.
type commandLogWriter struct {
	name string
	out  io.Writer
	err  io.Writer
}

// Write implements loggo's Writer interface.
func (s *commandLogWriter) Write(level loggo.Level, name, filename string, line int, timestamp time.Time, message string) {
	if name == s.name {
		if level <= loggo.INFO {
			fmt.Fprintf(s.out, "%s\n", message)
		} else {
			fmt.Fprintf(s.err, "%s\n", message)
		}
	}
}
<commit_msg>Make a simple formatter if the user isn't showing the log.<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package cmd

import (
	"fmt"
	"io"
	"os"
	"time"

	"launchpad.net/gnuflag"
	"launchpad.net/loggo"

	"launchpad.net/juju-core/juju/osenv"
)

// Log supplies the necessary functionality for Commands that wish to set up
// logging.
type Log struct {
	Path    string
	Verbose bool
	Debug   bool
	ShowLog bool
	Config  string
}

// AddFlags adds appropriate flags to f.
func (l *Log) AddFlags(f *gnuflag.FlagSet) {
	f.StringVar(&l.Path, "log-file", "", "path to write log to")
	// TODO(thumper): rename verbose to --show-log
	f.BoolVar(&l.Verbose, "v", false, "if set, log additional messages")
	f.BoolVar(&l.Verbose, "verbose", false, "if set, log additional messages")
	f.BoolVar(&l.Debug, "debug", false, "if set, log debugging messages")
	defaultLogConfig := os.Getenv(osenv.JujuLoggingConfig)
	f.StringVar(&l.Config, "log-config", defaultLogConfig, "specify log levels for modules")
	f.BoolVar(&l.ShowLog, "show-log", false, "if set, write the log file to stderr")
}

// Start starts logging using the given Context.
func (l *Log) Start(ctx *Context) error {
	if l.Path != "" {
		path := ctx.AbsPath(l.Path)
		target, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
		if err != nil {
			return err
		}
		writer := loggo.NewSimpleWriter(target, &loggo.DefaultFormatter{})
		err = loggo.RegisterWriter("logfile", writer, loggo.TRACE)
		if err != nil {
			return err
		}
	}
	level := loggo.WARNING
	if l.Verbose {
		ctx.Stdout.Write([]byte("verbose is deprecated with the current meaning, use show-log\n"))
		l.ShowLog = true
	}
	if l.ShowLog {
		level = loggo.INFO
	}
	if l.Debug {
		l.ShowLog = true
		level = loggo.DEBUG
	}

	if l.ShowLog {
		// We replace the default writer to use ctx.Stderr rather than os.Stderr.
		writer := loggo.NewSimpleWriter(ctx.Stderr, &loggo.DefaultFormatter{})
		_, err := loggo.ReplaceDefaultWriter(writer)
		if err != nil {
			return err
		}
	} else {
		loggo.RemoveWriter("default")
		// Create a simple writer that doesn't show filenames, or timestamps,
		// and only shows warning or above.
		writer := loggo.NewSimpleWriter(ctx.Stderr, &warningFormatter{})
		err = loggo.RegisterWriter("warning", writer, loggo.WARNING)
		if err != nil {
			return err
		}
	}
	// Set the level on the root logger.
	loggo.GetLogger("").SetLogLevel(level)
	// Override the logging config with specified logging config.
	loggo.ConfigureLoggers(l.Config)
	return nil
}

// warningFormatter is a simple loggo formatter that produces something like:
//   WARNING The message...
type warningFormatter struct{}

func (*warningFormatter) Format(level Level, module, filename string, line int, timestamp time.Time, message string) string {
	return fmt.Sprintf("%s %s", level, message)
}

// NewCommandLogWriter creates a loggo writer for registration
// by the callers of a command. This way the logged output can also
// be displayed otherwise, e.g. on the screen.
func NewCommandLogWriter(name string, out, err io.Writer) loggo.Writer {
	return &commandLogWriter{name, out, err}
}

// commandLogWriter filters the log messages for name.
type commandLogWriter struct {
	name string
	out  io.Writer
	err  io.Writer
}

// Write implements loggo's Writer interface.
func (s *commandLogWriter) Write(level loggo.Level, name, filename string, line int, timestamp time.Time, message string) {
	if name == s.name {
		if level <= loggo.INFO {
			fmt.Fprintf(s.out, "%s\n", message)
		} else {
			fmt.Fprintf(s.err, "%s\n", message)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/edmonds/golang-mtbl"
	"github.com/hdm/inetdata-parsers/utils"
	"io/ioutil"
	"os"
	"runtime"
)

func usage() {
	fmt.Println("Usage: " + os.Args[0] + " [options] <mtbl> ... <mtbl>")
	fmt.Println("")
	fmt.Println("Queries one or more MTBL databases")
	fmt.Println("")
	fmt.Println("Options:")
	flag.PrintDefaults()
}

func findPaths(args []string) []string {
	var paths []string
	for i := range args {
		path := args[i]
		info, e := os.Stat(path)
		if e != nil {
			fmt.Fprintf(os.Stderr, "Error: Path %s : %v\n", path, e)
			os.Exit(1)
		}

		if info.Mode().IsRegular() {
			paths = append(paths, path)
			continue
		}

		if info.Mode().IsDir() {
			if files, e := ioutil.ReadDir(path); e == nil {
				for _, f := range files {
					if f.Mode().IsRegular() {
						npath := path + string(os.PathSeparator) + f.Name()
						paths = append(paths, npath)
					}
				}
			}
		}
	}
	return paths
}

func main() {

	runtime.GOMAXPROCS(runtime.NumCPU())
	os.Setenv("LC_ALL", "C")

	flag.Usage = func() { usage() }

	key_only := flag.Bool("k", false, "Display key names only")
	val_only := flag.Bool("v", false, "Display values only")
	prefix := flag.String("p", "", "Only return keys with this prefix")
	rev_prefix := flag.String("r", "", "Only return keys with this prefix in reverse form")
	rev_key := flag.Bool("R", false, "Display matches with the key in reverse form")
	no_quotes := flag.Bool("n", false, "Print raw values, not quoted values")
	as_json := flag.Bool("j", false, "Print each record as a single line of JSON")
	version := flag.Bool("version", false, "Show the version and build timestamp")

	flag.Parse()

	if *version {
		utils.PrintVersion()
		os.Exit(0)
	}

	if len(flag.Args()) == 0 {
		usage()
		os.Exit(1)
	}

	if *key_only && *val_only {
		fmt.Fprintf(os.Stderr, "Error: Only one of -k or -v can be specified\n")
		usage()
		os.Exit(1)
	}

	if len(*prefix) > 0 && len(*rev_prefix) > 0 {
		fmt.Fprintf(os.Stderr, "Error: Only one of -p or -r can be specified\n")
		usage()
		os.Exit(1)
	}

	paths := findPaths(flag.Args())

	for i := range paths {
		path := paths[i]
		r, e := mtbl.ReaderInit(path, &mtbl.ReaderOptions{VerifyChecksums: true})
		defer r.Destroy()

		if e != nil {
			fmt.Fprintf(os.Stderr, "Error reading %s: %s\n", path, e)
			os.Exit(1)
		}

		var it *mtbl.Iter

		if len(*prefix) > 0 {
			p := *prefix
			it = mtbl.IterPrefix(r, []byte(p))
		} else if len(*rev_prefix) > 0 {
			p := utils.ReverseKey(*rev_prefix)
			it = mtbl.IterPrefix(r, []byte(p))
		} else {
			it = mtbl.IterAll(r)
		}

		for {
			key_bytes, val_bytes, ok := it.Next()
			if !ok {
				break
			}

			key := string(key_bytes)
			val := string(val_bytes)

			if *rev_key {
				key = utils.ReverseKey(key)
			}

			if *as_json {
				o := make(map[string]interface{})
				v := make([][]string, 1)

				if de := json.Unmarshal([]byte(val), &v); de != nil {
					fmt.Fprintf(os.Stderr, "Could not unmarshal %s -> %s as json: %s\n", key, val, e)
					continue
				}

				o["key"] = string(key)
				o["val"] = v

				b, e := json.Marshal(o)
				if e != nil {
					fmt.Fprintf(os.Stderr, "Could not marshal %s -> %s as json: %s\n", key, val, e)
					continue
				}
				fmt.Println(string(b))

			} else if *key_only {
				fmt.Printf("%s\n", key)
			} else if *val_only {
				if *no_quotes {
					fmt.Printf("%s\n", val)
				} else {
					fmt.Printf("%q\n", val)
				}
			} else {
				if *no_quotes {
					fmt.Printf("%s\t%s\n", key, val)
				} else {
					fmt.Printf("%s\t%q\n", key, val)
				}
			}
		}

	}

}
<commit_msg>Add -cidr and -domain mode to mq<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/edmonds/golang-mtbl"
	"github.com/hdm/inetdata-parsers/utils"
	"github.com/peterbourgon/mergemap"
	"io/ioutil"
	"math"
	"net"
	"os"
	"runtime"
	"strings"
)

var key_only *bool
var val_only *bool
var prefix *string
var rev_prefix *string
var rev_key *bool
var no_quotes *bool
var as_json *bool
var version *bool
var domain *string
var cidr *string

func usage() {
	fmt.Println("Usage: " + os.Args[0] + " [options] <mtbl> ... <mtbl>")
	fmt.Println("")
	fmt.Println("Queries one or more MTBL databases")
	fmt.Println("")
	fmt.Println("Options:")
	flag.PrintDefaults()
}

func findPaths(args []string) []string {
	var paths []string
	for i := range args {
		path := args[i]
		info, e := os.Stat(path)
		if e != nil {
			fmt.Fprintf(os.Stderr, "Error: Path %s : %v\n", path, e)
			os.Exit(1)
		}

		if info.Mode().IsRegular() {
			paths = append(paths, path)
			continue
		}

		if info.Mode().IsDir() {
			if files, e := ioutil.ReadDir(path); e == nil {
				for _, f := range files {
					if f.Mode().IsRegular() {
						npath := path + string(os.PathSeparator) + f.Name()
						paths = append(paths, npath)
					}
				}
			}
		}
	}
	return paths
}

// TODO: Rework to handle [][]string merges
func mergeFunc(key []byte, val0 []byte, val1 []byte) (mergedVal []byte) {
	var v0, v1 map[string]interface{}

	if e := json.Unmarshal(val0, &v0); e != nil {
		return val1
	}

	if e := json.Unmarshal(val1, &v1); e != nil {
		return val0
	}

	m := mergemap.Merge(v0, v1)
	d, e := json.Marshal(m)
	if e != nil {
		fmt.Fprintf(os.Stderr, "JSON merge error: %v -> %v + %v\n", e, val0, val1)
		return val0
	}

	return d
}

func writeOutput(key_bytes []byte, val_bytes []byte) {

	key := string(key_bytes)
	val := string(val_bytes)

	if *rev_key {
		key = utils.ReverseKey(key)
	}

	if *as_json {
		o := make(map[string]interface{})
		v := make([][]string, 1)

		if de := json.Unmarshal([]byte(val), &v); de != nil {
			fmt.Fprintf(os.Stderr, "Could not unmarshal %s -> %s as json: %s\n", key, val, de)
			return
		}

		o["key"] = string(key)
		o["val"] = v

		b, je := json.Marshal(o)
		if je != nil {
			fmt.Fprintf(os.Stderr, "Could not marshal %s -> %s as json: %s\n", key, val, je)
			return
		}
		fmt.Println(string(b))

	} else if *key_only {
		fmt.Printf("%s\n", key)
	} else if *val_only {
		if *no_quotes {
			fmt.Printf("%s\n", val)
		} else {
			fmt.Printf("%q\n", val)
		}
	} else {
		if *no_quotes {
			fmt.Printf("%s\t%s\n", key, val)
		} else {
			fmt.Printf("%s\t%q\n", key, val)
		}
	}
}

func searchPrefix(m *mtbl.Merger, prefix string) {
	it := mtbl.IterPrefix(m, []byte(prefix))
	for {
		key_bytes, val_bytes, ok := it.Next()
		if !ok {
			break
		}
		writeOutput(key_bytes, val_bytes)
	}
}

func searchAll(m *mtbl.Merger) {
	it := mtbl.IterAll(m)
	for {
		key_bytes, val_bytes, ok := it.Next()
		if !ok {
			break
		}
		writeOutput(key_bytes, val_bytes)
	}
}

func searchDomain(m *mtbl.Merger, domain string) {
	rdomain := []byte(utils.ReverseKey(domain))

	// Domain searches always use reversed keys
	*rev_key = true

	// Exact match: "example.com"
	exact, found := mtbl.Get(m, rdomain)
	if found {
		writeOutput([]byte(rdomain), exact)
	}

	// Subdomain matches: ".example.com"
	dot_domain := append(rdomain, '.')
	it := mtbl.IterPrefix(m, dot_domain)
	for {
		key_bytes, val_bytes, ok := it.Next()
		if !ok {
			break
		}
		writeOutput(key_bytes, val_bytes)
	}
}

func searchCIDR(m *mtbl.Merger, cidr string) {

	// Parse CIDR into base address + mask
	ip, net, err := net.ParseCIDR(cidr)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Invalid CIDR %s: %s", cidr, err.Error())
		return
	}

	// Verify IPv4 for now
	ip4 := net.IP.To4()
	if ip4 == nil {
		fmt.Fprintf(os.Stderr, "Invalid IPv4 CIDR %s: %s", cidr, err.Error())
		return
	}

	net_base, ip_err := utils.IPv4_to_UInt(net.IP.String())
	if ip_err != nil {
		fmt.Fprintf(os.Stderr, "Invalid IPv4 Address %s: %s", ip.String(), err.Error())
		return
	}

	mask_ones, mask_total := net.Mask.Size()

	// Does not work for IPv6 due to cast to uint32
	net_size := uint32(math.Pow(2, float64(mask_total-mask_ones)))

	cur_base := net_base
	end_base := net_base + net_size

	// TODO: Special case /16s to speed up
	if mask_ones == 16 {
		ip_prefix := strings.Join(strings.SplitN(utils.UInt_to_IPv4(cur_base), ".", 4)[0:2], ".") + "."
		searchPrefix(m, ip_prefix)
		return
	}

	for ; end_base-cur_base >= 256; cur_base += 256 {
		ip_prefix := strings.Join(strings.SplitN(utils.UInt_to_IPv4(cur_base), ".", 4)[0:3], ".") + "."
		searchPrefix(m, ip_prefix)
	}

	if end_base-cur_base == 0 {
		return
	}

	// One final prefix search
	ip_prefix := strings.Join(strings.SplitN(utils.UInt_to_IPv4(cur_base), ".", 4)[0:3], ".") + "."

	it := mtbl.IterPrefix(m, []byte(ip_prefix))
	for {
		key_bytes, val_bytes, ok := it.Next()
		if !ok {
			break
		}

		// Only print values in our CIDR
		cur_val, _ := utils.IPv4_to_UInt(string(key_bytes))
		if cur_val >= cur_base && cur_val <= end_base {
			writeOutput(key_bytes, val_bytes)
		}
	}

	fmt.Fprintf(os.Stderr, "Final: [%s] %s\n", cidr, ip_prefix)
}

func main() {

	runtime.GOMAXPROCS(runtime.NumCPU())
	os.Setenv("LC_ALL", "C")

	flag.Usage = func() { usage() }

	key_only = flag.Bool("k", false, "Display key names only")
	val_only = flag.Bool("v", false, "Display values only")
	prefix = flag.String("p", "", "Only return keys with this prefix")
	rev_prefix = flag.String("r", "", "Only return keys with this prefix in reverse form")
	rev_key = flag.Bool("R", false, "Display matches with the key in reverse form")
	no_quotes = flag.Bool("n", false, "Print raw values, not quoted values")
	as_json = flag.Bool("j", false, "Print each record as a single line of JSON")
	version = flag.Bool("version", false, "Show the version and build timestamp")
	domain = flag.String("domain", "", "Search for all matches for a specified domain")
	cidr = flag.String("cidr", "", "Search for all matches for the specified CIDR")

	flag.Parse()

	if *version {
		utils.PrintVersion()
		os.Exit(0)
	}

	if len(flag.Args()) == 0 {
		usage()
		os.Exit(1)
	}

	if *key_only && *val_only {
		fmt.Fprintf(os.Stderr, "Error: Only one of -k or -v can be specified\n")
		usage()
		os.Exit(1)
	}

	if len(*prefix) > 0 && len(*rev_prefix) > 0 {
		fmt.Fprintf(os.Stderr, "Error: Only one of -p or -r can be specified\n")
		usage()
		os.Exit(1)
	}

	if len(*domain) > 0 && (len(*prefix) > 0 || len(*rev_prefix) > 0 || len(*cidr) > 0) {
		fmt.Fprintf(os.Stderr, "Error: Only one of -p, -r, -domain, or -cidr can be specified\n")
		usage()
		os.Exit(1)
	}

	if len(*cidr) > 0 && (len(*prefix) > 0 || len(*rev_prefix) > 0 || len(*domain) > 0) {
		fmt.Fprintf(os.Stderr, "Error: Only one of -p, -r, -domain, or -cidr can be specified\n")
		usage()
		os.Exit(1)
	}

	paths := findPaths(flag.Args())

	m := mtbl.MergerInit(&mtbl.MergerOptions{Merge: mergeFunc})
	defer m.Destroy()

	for i := range paths {
		path := paths[i]

		r, e := mtbl.ReaderInit(path, &mtbl.ReaderOptions{VerifyChecksums: true})
		defer r.Destroy()
		if e != nil {
			fmt.Fprintf(os.Stderr, "Error reading %s: %s\n", path, e)
			os.Exit(1)
		}
		m.AddSource(r)
	}

	if len(*domain) > 0 {
		searchDomain(m, *domain)
		return
	}

	if len(*cidr) > 0 {
		searchCIDR(m, *cidr)
		return
	}

	if len(*prefix) > 0 {
		searchPrefix(m, *prefix)
		return
	}

	if len(*rev_prefix) > 0 {
		p := utils.ReverseKey(*rev_prefix)
		searchPrefix(m, p)
		return
	}

	searchAll(m)
}
<|endoftext|>


<commit_before>/*
 * MinIO Client (C) 2020 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"fmt"
	"os"
	"sync"
	"sync/atomic"

	"github.com/fatih/color"
	"github.com/minio/cli"
	"github.com/minio/mc/pkg/probe"
	"github.com/minio/minio/pkg/console"
)

// mv command flags.
var (
	mvFlags = []cli.Flag{
		cli.BoolFlag{
			Name:  "recursive, r",
			Usage: "move recursively",
		},
		cli.StringFlag{
			Name:  "older-than",
			Usage: "move objects older than L days, M hours and N minutes",
		},
		cli.StringFlag{
			Name:  "newer-than",
			Usage: "move objects newer than L days, M hours and N minutes",
		},
		cli.StringFlag{
			Name:  "storage-class, sc",
			Usage: "set storage class for new object(s) on target",
		},
		cli.StringFlag{
			Name:  "encrypt",
			Usage: "encrypt/decrypt objects (using server-side encryption with server managed keys)",
		},
		cli.StringFlag{
			Name:  "attr",
			Usage: "add custom metadata for the object",
		},
		cli.BoolFlag{
			Name:  "continue, c",
			Usage: "create or resume move session",
		},
		cli.BoolFlag{
			Name:  "preserve, a",
			Usage: "preserve filesystem attributes (mode, ownership, timestamps)",
		},
		cli.BoolFlag{
			Name:  "disable-multipart",
			Usage: "disable multipart upload feature",
		},
	}
)

// Move command.
var mvCmd = cli.Command{
	Name:   "mv",
	Usage:  "move objects",
	Action: mainMove,
	Before: setGlobalsFromContext,
	Flags:  append(append(mvFlags, ioFlags...), globalFlags...),
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} [FLAGS] SOURCE [SOURCE...] TARGET

FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
ENVIRONMENT VARIABLES:
  MC_ENCRYPT:      list of comma delimited prefixes
  MC_ENCRYPT_KEY:  list of comma delimited prefix=secret values

EXAMPLES:
  01. Move a list of objects from local file system to Amazon S3 cloud storage.
      {{.Prompt}} {{.HelpName}} Music/*.ogg s3/jukebox/

  02. Move a folder recursively from MinIO cloud storage to Amazon S3 cloud storage.
      {{.Prompt}} {{.HelpName}} --recursive play/mybucket/burningman2011/ s3/mybucket/

  03. Move multiple local folders recursively to MinIO cloud storage.
      {{.Prompt}} {{.HelpName}} --recursive backup/2014/ backup/2015/ play/archive/

  04. Move a bucket recursively from aliased Amazon S3 cloud storage to local filesystem on Windows.
      {{.Prompt}} {{.HelpName}} --recursive s3\documents\2014\ C:\Backups\2014

  05. Move files older than 7 days and 10 hours from MinIO cloud storage to Amazon S3 cloud storage.
      {{.Prompt}} {{.HelpName}} --older-than 7d10h play/mybucket/burningman2011/ s3/mybucket/

  06. Move files newer than 7 days and 10 hours from MinIO cloud storage to a local path.
      {{.Prompt}} {{.HelpName}} --newer-than 7d10h play/mybucket/burningman2011/ ~/latest/

  07. Move an object with name containing unicode characters to Amazon S3 cloud storage.
      {{.Prompt}} {{.HelpName}} 本語 s3/andoria/

  08. Move a local folder with space separated characters to Amazon S3 cloud storage.
      {{.Prompt}} {{.HelpName}} --recursive 'workdir/documents/May 2014/' s3/miniocloud

  09. Move a folder with encrypted objects recursively from Amazon S3 to MinIO cloud storage.
      {{.Prompt}} {{.HelpName}} --recursive --encrypt-key "s3/documents/=32byteslongsecretkeymustbegiven1,myminio/documents/=32byteslongsecretkeymustbegiven2" s3/documents/ myminio/documents/

  10. Move a folder with encrypted objects recursively from Amazon S3 to MinIO cloud storage. In case the encryption key contains non-printable character like tab, pass the
      base64 encoded string as key.
      {{.Prompt}} {{.HelpName}} --recursive --encrypt-key "s3/documents/=MzJieXRlc2xvbmdzZWNyZWFiY2RlZmcJZ2l2ZW5uMjE=,myminio/documents/=MzJieXRlc2xvbmdzZWNyZWFiY2RlZmcJZ2l2ZW5uMjE=" s3/documents/ myminio/documents/

  11. Move a list of objects from local file system to MinIO cloud storage with specified metadata, separated by ";"
      {{.Prompt}} {{.HelpName}} --attr "key1=value1;key2=value2" Music/*.mp4 play/mybucket/

  12. Move a folder recursively from MinIO cloud storage to Amazon S3 cloud storage with Cache-Control and custom metadata, separated by ";".
      {{.Prompt}} {{.HelpName}} --attr "Cache-Control=max-age=90000,min-fresh=9000;key1=value1;key2=value2" --recursive play/mybucket/burningman2011/ s3/mybucket/

  13. Move a text file to an object storage and assign REDUCED_REDUNDANCY storage-class to the uploaded object.
      {{.Prompt}} {{.HelpName}} --storage-class REDUCED_REDUNDANCY myobject.txt play/mybucket

  14. Move a text file to an object storage and create or resume copy session.
      {{.Prompt}} {{.HelpName}} --recursive --continue dir/ play/mybucket

  15. Move a text file to an object storage and preserve the file system attribute as metadata.
      {{.Prompt}} {{.HelpName}} -a myobject.txt play/mybucket

  16. Move a text file to an object storage with object lock mode set to 'GOVERNANCE' with retention date.
      {{.Prompt}} {{.HelpName}} --attr "x-amz-object-lock-mode=GOVERNANCE;x-amz-object-lock-retain-until-date=2020-01-11T01:57:02Z" locked.txt play/locked-bucket/

  17. Move a text file to an object storage and disable multipart upload feature.
      {{.Prompt}} {{.HelpName}} --disable-multipart myobject.txt play/mybucket
`,
}

type removeClientInfo struct {
	client    Client
	contentCh chan *ClientContent
	errorCh   <-chan *probe.Error
}

type removeManager struct {
	removeMap      map[string]*removeClientInfo
	removeMapMutex sync.RWMutex
	doneCh         chan struct{}
	isClosed       int32
	wg             sync.WaitGroup
}

func (rm *removeManager) readErrors(errorCh <-chan *probe.Error, targetURL string) {
	rm.wg.Add(1)
	go func() {
		defer rm.wg.Done()
		var stop bool
		for !stop {
			select {
			case pErr, ok := <-errorCh:
				if ok {
					errorIf(pErr.Trace(targetURL), "Failed to remove in`"+targetURL+"`.")
				}
			case <-rm.doneCh:
				stop = true
			}
		}

		for pErr := range errorCh {
			if pErr != nil {
				errorIf(pErr.Trace(targetURL), "Failed to remove in `"+targetURL+"`.")
			}
		}
	}()
}

func (rm *removeManager) add(targetAlias, targetURL string) {
	if atomic.LoadInt32(&rm.isClosed) != 0 {
		return
	}

	rm.removeMapMutex.RLock()
	clientInfo := rm.removeMap[targetAlias]
	rm.removeMapMutex.RUnlock()

	if clientInfo == nil {
		client, pErr := newClientFromAlias(targetAlias, targetURL)
		if pErr != nil {
			errorIf(pErr.Trace(targetURL), "Invalid argument `"+targetURL+"`.")
			return
		}

		contentCh := make(chan *ClientContent)
		errorCh := client.Remove(false, false, false, contentCh)
		rm.readErrors(errorCh, targetURL)

		clientInfo = &removeClientInfo{
			client:    client,
			contentCh: contentCh,
			errorCh:   errorCh,
		}

		rm.removeMapMutex.Lock()
		rm.removeMap[targetAlias] = clientInfo
		rm.removeMapMutex.Unlock()
	}

	go func() {
		clientInfo.contentCh <- &ClientContent{URL: *newClientURL(targetURL)}
	}()
}

func (rm *removeManager) close() {
	atomic.StoreInt32(&rm.isClosed, 1)
	rm.removeMapMutex.Lock()
	defer rm.removeMapMutex.Unlock()

	for _, clientInfo := range rm.removeMap {
		close(clientInfo.contentCh)
	}

	close(rm.doneCh)

	rm.wg.Wait()
}

var rmManager = &removeManager{
	removeMap: make(map[string]*removeClientInfo),
	doneCh:    make(chan struct{}),
}

func bgRemove(url string) {
	remove := func(targetAlias, targetURL string) {
		clnt, pErr := newClientFromAlias(targetAlias, targetURL)
		if pErr != nil {
			errorIf(pErr.Trace(url), "Invalid argument `"+url+"`.")
		}

		contentCh := make(chan *ClientContent, 1)
		contentCh <- &ClientContent{URL: *newClientURL(targetURL)}
		close(contentCh)
		errorCh := clnt.Remove(false, false, false, contentCh)
		for pErr := range errorCh {
			if pErr != nil {
				errorIf(pErr.Trace(url), "Failed to remove `"+url+"`.")
				switch pErr.ToGoError().(type) {
				case PathInsufficientPermission:
					// Ignore Permission error.
					continue
				}
			}
		}
	}

	targetAlias, targetURL, _ := mustExpandAlias(url)
	if targetAlias == "" {
		// File system does not support batch deletion hence use individual deletion.
		go remove(targetAlias, targetURL)
		return
	}

	rmManager.add(targetAlias, targetURL)
}

// mainMove is the entry point for mv command.
func mainMove(ctx *cli.Context) error {
	// Parse encryption keys per command.
	encKeyDB, err := getEncKeys(ctx)
	fatalIf(err, "Unable to parse encryption keys.")

	// Parse metadata.
	userMetaMap := make(map[string]string)
	if ctx.String("attr") != "" {
		userMetaMap, err = getMetaDataEntry(ctx.String("attr"))
		fatalIf(err, "Unable to parse attribute %v", ctx.String("attr"))
	}

	// check 'copy' cli arguments.
	checkCopySyntax(ctx, encKeyDB, true)

	for _, urlStr := range ctx.Args() {
		client, err := newClient(urlStr)
		if err != nil {
			fatalIf(err.Trace(), "Cannot parse the provided url.")
		}

		if s3Client, ok := client.(*S3Client); ok {
			if _, _, _, err = s3Client.GetObjectLockConfig(); err == nil {
				fatalIf(err, fmt.Sprintf("object lock configuration is enabled on the specified bucket in alias %v.", urlStr))
			}
		}
	}

	// Additional command speific theme customization.
	console.SetColor("Copy", color.New(color.FgGreen, color.Bold))

	recursive := ctx.Bool("recursive")
	olderThan := ctx.String("older-than")
	newerThan := ctx.String("newer-than")
	storageClass := ctx.String("storage-class")
	sseKeys := os.Getenv("MC_ENCRYPT_KEY")
	if key := ctx.String("encrypt-key"); key != "" {
		sseKeys = key
	}

	if sseKeys != "" {
		sseKeys, err = getDecodedKey(sseKeys)
		fatalIf(err, "Unable to parse encryption keys.")
	}
	sse := ctx.String("encrypt")

	var session *sessionV8

	if ctx.Bool("continue") {
		sessionID := getHash("mv", ctx.Args())
		if isSessionExists(sessionID) {
			session, err = loadSessionV8(sessionID)
			fatalIf(err.Trace(sessionID), "Unable to load session.")
		} else {
			session = newSessionV8(sessionID)
			session.Header.CommandType = "mv"
			session.Header.CommandBoolFlags["recursive"] = recursive
			session.Header.CommandStringFlags["older-than"] = olderThan
			session.Header.CommandStringFlags["newer-than"] = newerThan
			session.Header.CommandStringFlags["storage-class"] = storageClass
			session.Header.CommandStringFlags["encrypt-key"] = sseKeys
			session.Header.CommandStringFlags["encrypt"] = sse
			session.Header.CommandBoolFlags["session"] = ctx.Bool("continue")

			if ctx.Bool("preserve") {
				session.Header.CommandBoolFlags["preserve"] = ctx.Bool("preserve")
			}
			session.Header.UserMetaData = userMetaMap
			session.Header.CommandBoolFlags["disable-multipart"] = ctx.Bool("disable-multipart")

			var e error
			if session.Header.RootPath, e = os.Getwd(); e != nil {
				session.Delete()
				fatalIf(probe.NewError(e), "Unable to get current working folder.")
			}

			// extract URLs.
			session.Header.CommandArgs = ctx.Args()
		}
	}

	e := doCopySession(ctx, session, encKeyDB, true)
	if session != nil {
		session.Delete()
	}

	console.Colorize("Copy", "Waiting for move operations to complete")
	rmManager.close()

	return e
}
<commit_msg>fix: create empty probe error to fatalIf() (#3199)<commit_after>/*
 * MinIO Client (C) 2020 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"errors"
	"fmt"
	"os"
	"sync"
	"sync/atomic"

	"github.com/fatih/color"
	"github.com/minio/cli"
	"github.com/minio/mc/pkg/probe"
	"github.com/minio/minio/pkg/console"
)

// mv command flags.
var (
	mvFlags = []cli.Flag{
		cli.BoolFlag{
			Name:  "recursive, r",
			Usage: "move recursively",
		},
		cli.StringFlag{
			Name:  "older-than",
			Usage: "move objects older than L days, M hours and N minutes",
		},
		cli.StringFlag{
			Name:  "newer-than",
			Usage: "move objects newer than L days, M hours and N minutes",
		},
		cli.StringFlag{
			Name:  "storage-class, sc",
			Usage: "set storage class for new object(s) on target",
		},
		cli.StringFlag{
			Name:  "encrypt",
			Usage: "encrypt/decrypt objects (using server-side encryption with server managed keys)",
		},
		cli.StringFlag{
			Name:  "attr",
			Usage: "add custom metadata for the object",
		},
		cli.BoolFlag{
			Name:  "continue, c",
			Usage: "create or resume move session",
		},
		cli.BoolFlag{
			Name:  "preserve, a",
			Usage: "preserve filesystem attributes (mode, ownership, timestamps)",
		},
		cli.BoolFlag{
			Name:  "disable-multipart",
			Usage: "disable multipart upload feature",
		},
	}
)

// Move command.
var mvCmd = cli.Command{
	Name:   "mv",
	Usage:  "move objects",
	Action: mainMove,
	Before: setGlobalsFromContext,
	Flags:  append(append(mvFlags, ioFlags...), globalFlags...),
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} [FLAGS] SOURCE [SOURCE...] TARGET

FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
ENVIRONMENT VARIABLES:
  MC_ENCRYPT:      list of comma delimited prefixes
  MC_ENCRYPT_KEY:  list of comma delimited prefix=secret values

EXAMPLES:
  01. Move a list of objects from local file system to Amazon S3 cloud storage.
      {{.Prompt}} {{.HelpName}} Music/*.ogg s3/jukebox/

  02. Move a folder recursively from MinIO cloud storage to Amazon S3 cloud storage.
      {{.Prompt}} {{.HelpName}} --recursive play/mybucket/burningman2011/ s3/mybucket/

  03. Move multiple local folders recursively to MinIO cloud storage.
      {{.Prompt}} {{.HelpName}} --recursive backup/2014/ backup/2015/ play/archive/

  04. Move a bucket recursively from aliased Amazon S3 cloud storage to local filesystem on Windows.
      {{.Prompt}} {{.HelpName}} --recursive s3\documents\2014\ C:\Backups\2014

  05. Move files older than 7 days and 10 hours from MinIO cloud storage to Amazon S3 cloud storage.
      {{.Prompt}} {{.HelpName}} --older-than 7d10h play/mybucket/burningman2011/ s3/mybucket/

  06. Move files newer than 7 days and 10 hours from MinIO cloud storage to a local path.
      {{.Prompt}} {{.HelpName}} --newer-than 7d10h play/mybucket/burningman2011/ ~/latest/

  07. Move an object with name containing unicode characters to Amazon S3 cloud storage.
      {{.Prompt}} {{.HelpName}} 本語 s3/andoria/

  08. Move a local folder with space separated characters to Amazon S3 cloud storage.
      {{.Prompt}} {{.HelpName}} --recursive 'workdir/documents/May 2014/' s3/miniocloud

  09. Move a folder with encrypted objects recursively from Amazon S3 to MinIO cloud storage.
      {{.Prompt}} {{.HelpName}} --recursive --encrypt-key "s3/documents/=32byteslongsecretkeymustbegiven1,myminio/documents/=32byteslongsecretkeymustbegiven2" s3/documents/ myminio/documents/

  10. Move a folder with encrypted objects recursively from Amazon S3 to MinIO cloud storage. In case the encryption key contains non-printable character like tab, pass the
      base64 encoded string as key.
      {{.Prompt}} {{.HelpName}} --recursive --encrypt-key "s3/documents/=MzJieXRlc2xvbmdzZWNyZWFiY2RlZmcJZ2l2ZW5uMjE=,myminio/documents/=MzJieXRlc2xvbmdzZWNyZWFiY2RlZmcJZ2l2ZW5uMjE=" s3/documents/ myminio/documents/

  11. Move a list of objects from local file system to MinIO cloud storage with specified metadata, separated by ";"
      {{.Prompt}} {{.HelpName}} --attr "key1=value1;key2=value2" Music/*.mp4 play/mybucket/

  12. Move a folder recursively from MinIO cloud storage to Amazon S3 cloud storage with Cache-Control and custom metadata, separated by ";".
      {{.Prompt}} {{.HelpName}} --attr "Cache-Control=max-age=90000,min-fresh=9000;key1=value1;key2=value2" --recursive play/mybucket/burningman2011/ s3/mybucket/

  13. Move a text file to an object storage and assign REDUCED_REDUNDANCY storage-class to the uploaded object.
      {{.Prompt}} {{.HelpName}} --storage-class REDUCED_REDUNDANCY myobject.txt play/mybucket

  14. Move a text file to an object storage and create or resume copy session.
      {{.Prompt}} {{.HelpName}} --recursive --continue dir/ play/mybucket

  15. Move a text file to an object storage and preserve the file system attribute as metadata.
      {{.Prompt}} {{.HelpName}} -a myobject.txt play/mybucket

  16. Move a text file to an object storage with object lock mode set to 'GOVERNANCE' with retention date.
      {{.Prompt}} {{.HelpName}} --attr "x-amz-object-lock-mode=GOVERNANCE;x-amz-object-lock-retain-until-date=2020-01-11T01:57:02Z" locked.txt play/locked-bucket/

  17. Move a text file to an object storage and disable multipart upload feature.
      {{.Prompt}} {{.HelpName}} --disable-multipart myobject.txt play/mybucket
`,
}

type removeClientInfo struct {
	client    Client
	contentCh chan *ClientContent
	errorCh   <-chan *probe.Error
}

type removeManager struct {
	removeMap      map[string]*removeClientInfo
	removeMapMutex sync.RWMutex
	doneCh         chan struct{}
	isClosed       int32
	wg             sync.WaitGroup
}

func (rm *removeManager) readErrors(errorCh <-chan *probe.Error, targetURL string) {
	rm.wg.Add(1)
	go func() {
		defer rm.wg.Done()
		var stop bool
		for !stop {
			select {
			case pErr, ok := <-errorCh:
				if ok {
					errorIf(pErr.Trace(targetURL), "Failed to remove in`"+targetURL+"`.")
				}
			case <-rm.doneCh:
				stop = true
			}
		}

		for pErr := range errorCh {
			if pErr != nil {
				errorIf(pErr.Trace(targetURL), "Failed to remove in `"+targetURL+"`.")
			}
		}
	}()
}

func (rm *removeManager) add(targetAlias, targetURL string) {
	if atomic.LoadInt32(&rm.isClosed) != 0 {
		return
	}

	rm.removeMapMutex.RLock()
	clientInfo := rm.removeMap[targetAlias]
	rm.removeMapMutex.RUnlock()

	if clientInfo == nil {
		client, pErr := newClientFromAlias(targetAlias, targetURL)
		if pErr != nil {
			errorIf(pErr.Trace(targetURL), "Invalid argument `"+targetURL+"`.")
			return
		}

		contentCh := make(chan *ClientContent)
		errorCh := client.Remove(false, false, false, contentCh)
		rm.readErrors(errorCh, targetURL)

		clientInfo = &removeClientInfo{
			client:    client,
			contentCh: contentCh,
			errorCh:   errorCh,
		}

		rm.removeMapMutex.Lock()
		rm.removeMap[targetAlias] = clientInfo
		rm.removeMapMutex.Unlock()
	}

	go func() {
		clientInfo.contentCh <- &ClientContent{URL: *newClientURL(targetURL)}
	}()
}

func (rm *removeManager) close() {
	atomic.StoreInt32(&rm.isClosed, 1)
	rm.removeMapMutex.Lock()
	defer rm.removeMapMutex.Unlock()

	for _, clientInfo := range rm.removeMap {
		close(clientInfo.contentCh)
	}

	close(rm.doneCh)

	rm.wg.Wait()
}

var rmManager = &removeManager{
	removeMap: make(map[string]*removeClientInfo),
	doneCh:    make(chan struct{}),
}

func bgRemove(url string) {
	remove := func(targetAlias, targetURL string) {
		clnt, pErr := newClientFromAlias(targetAlias, targetURL)
		if pErr != nil {
			errorIf(pErr.Trace(url), "Invalid argument `"+url+"`.")
		}

		contentCh := make(chan *ClientContent, 1)
		contentCh <- &ClientContent{URL: *newClientURL(targetURL)}
		close(contentCh)
		errorCh := clnt.Remove(false, false, false, contentCh)
		for pErr := range errorCh {
			if pErr != nil {
				errorIf(pErr.Trace(url), "Failed to remove `"+url+"`.")
				switch pErr.ToGoError().(type) {
				case PathInsufficientPermission:
					// Ignore Permission error.
					continue
				}
			}
		}
	}

	targetAlias, targetURL, _ := mustExpandAlias(url)
	if targetAlias == "" {
		// File system does not support batch deletion hence use individual deletion.
		go remove(targetAlias, targetURL)
		return
	}

	rmManager.add(targetAlias, targetURL)
}

// mainMove is the entry point for mv command.
func mainMove(ctx *cli.Context) error {
	// Parse encryption keys per command.
	encKeyDB, err := getEncKeys(ctx)
	fatalIf(err, "Unable to parse encryption keys.")

	// Parse metadata.
	userMetaMap := make(map[string]string)
	if ctx.String("attr") != "" {
		userMetaMap, err = getMetaDataEntry(ctx.String("attr"))
		fatalIf(err, "Unable to parse attribute %v", ctx.String("attr"))
	}

	// check 'copy' cli arguments.
	checkCopySyntax(ctx, encKeyDB, true)

	for _, urlStr := range ctx.Args() {
		client, err := newClient(urlStr)
		if err != nil {
			fatalIf(err.Trace(), "Cannot parse the provided url.")
		}

		if s3Client, ok := client.(*S3Client); ok {
			if _, _, _, err = s3Client.GetObjectLockConfig(); err == nil {
				fatalIf(probe.NewError(errors.New("")), fmt.Sprintf("Object lock configuration is enabled on the specified bucket in alias %v.", urlStr))
			}
		}
	}

	// Additional command speific theme customization.
	console.SetColor("Copy", color.New(color.FgGreen, color.Bold))

	recursive := ctx.Bool("recursive")
	olderThan := ctx.String("older-than")
	newerThan := ctx.String("newer-than")
	storageClass := ctx.String("storage-class")
	sseKeys := os.Getenv("MC_ENCRYPT_KEY")
	if key := ctx.String("encrypt-key"); key != "" {
		sseKeys = key
	}

	if sseKeys != "" {
		sseKeys, err = getDecodedKey(sseKeys)
		fatalIf(err, "Unable to parse encryption keys.")
	}
	sse := ctx.String("encrypt")

	var session *sessionV8

	if ctx.Bool("continue") {
		sessionID := getHash("mv", ctx.Args())
		if isSessionExists(sessionID) {
			session, err = loadSessionV8(sessionID)
			fatalIf(err.Trace(sessionID), "Unable to load session.")
		} else {
			session = newSessionV8(sessionID)
			session.Header.CommandType = "mv"
			session.Header.CommandBoolFlags["recursive"] = recursive
			session.Header.CommandStringFlags["older-than"] = olderThan
			session.Header.CommandStringFlags["newer-than"] = newerThan
			session.Header.CommandStringFlags["storage-class"] = storageClass
			session.Header.CommandStringFlags["encrypt-key"] = sseKeys
			session.Header.CommandStringFlags["encrypt"] = sse
			session.Header.CommandBoolFlags["session"] = ctx.Bool("continue")

			if ctx.Bool("preserve") {
				session.Header.CommandBoolFlags["preserve"] = ctx.Bool("preserve")
			}
			session.Header.UserMetaData = userMetaMap
			session.Header.CommandBoolFlags["disable-multipart"] = ctx.Bool("disable-multipart")

			var e error
			if session.Header.RootPath, e = os.Getwd(); e != nil {
				session.Delete()
				fatalIf(probe.NewError(e), "Unable to get current working folder.")
			}

			// extract URLs.
			session.Header.CommandArgs = ctx.Args()
		}
	}

	e := doCopySession(ctx, session, encKeyDB, true)
	if session != nil {
		session.Delete()
	}

	console.Colorize("Copy", "Waiting for move operations to complete")
	rmManager.close()

	return e
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"

	"github.com/spf13/cobra"
)

const Version = "0.2.0"

// versionCmd represents the version command
var versionCmd = &cobra.Command{
	Use:     "version",
	Aliases: []string{"v"},
	Short:   "Print the version of this tool",
	Run: func(cmd *cobra.Command, args []string) {
		fmt.Printf("gitlab-cli %v\n", Version)
	},
}

func init() {
	RootCmd.AddCommand(versionCmd)
}
<commit_msg>Update version<commit_after>package cmd

import (
	"fmt"

	"github.com/spf13/cobra"
)

const Version = "0.2.1"

// versionCmd represents the version command
var versionCmd = &cobra.Command{
	Use:     "version",
	Aliases: []string{"v"},
	Short:   "Print the version of this tool",
	Run: func(cmd *cobra.Command, args []string) {
		fmt.Printf("gitlab-cli %v\n", Version)
	},
}

func init() {
	RootCmd.AddCommand(versionCmd)
}
<|endoftext|>


<commit_before>package cmd

import "github.com/spf13/cobra"

func init() {
	RootCmd.AddCommand(versionCmd)
}

var versionCmd = &cobra.Command{
	Use:   "version",
	Short: "Display the version of mbt",
	Run: func(cmd *cobra.Command, args []string) {
		println("0.14.5")
	},
}
<commit_msg>Bump version - v0.15.0<commit_after>package cmd

import "github.com/spf13/cobra"

func init() {
	RootCmd.AddCommand(versionCmd)
}

var versionCmd = &cobra.Command{
	Use:   "version",
	Short: "Display the version of mbt",
	Run: func(cmd *cobra.Command, args []string) {
		println("0.15.0")
	},
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"

	"github.com/mrspock/godocsis"
)

var (
	community = flag.String("community", "public", "RW community to use when sending restart request")
	csvmode   = flag.Bool("csv", false, "CSV mode for easy import to spreadsheet")
)

func printVerbose(cmd godocsis.CM) {
	fmt.Printf("%s ", cmd.IPaddr)
	fmt.Printf("US(dBmV):%.01f ", float32(cmd.RF.USLevel[0])/10)
	separator := ","
	fmt.Printf("DS(dBmV):")
	for no, ds := range cmd.RF.DSLevel {
		if no == cmd.RF.DsBondingSize()-1 {
			separator = ""
		}
		fmt.Printf("%.01f%v", float32(ds)/10, separator)
	}
	fmt.Println("")

}
func printCSV(cmd godocsis.CM) {
	fmt.Printf("%s:", cmd.IPaddr)
	fmt.Printf("%.01f:", float32(cmd.RF.USLevel[0])/10)
	separator := ","
	for no, ds := range cmd.RF.DSLevel {
		if no == cmd.RF.DsBondingSize()-1 {
			separator = ""
		}
		fmt.Printf("%.01f%v", float32(ds)/10, separator)
	}
	fmt.Println("")

}
func main() {
	flag.Parse()
	if len(flag.Args()) < 1 {
		fmt.Println("Usage: cmparams [--csv] [--community <community>] <ip> <ip>")
		return
	}

	s := godocsis.Session
	s.Community = *community
	for _, ip := range flag.Args() {
		s.Target = ip
		rs, err := godocsis.RFLevel(s)
		if err != nil {
			//fmt.Fprintf(os.Stderr, "Problem: %v", err)
			fmt.Fprintf(os.Stderr, "%s:OFFLINE\n", ip)
			//panic(err)
		} else {
			if *csvmode {
				printCSV(rs)
			} else {
				printVerbose(rs)
			}
			//			fmt.Printf("%s:", ip)
			//			fmt.Printf("%.01f:", float32(rs.RF.USLevel[0])/10)
			//			separator := ","
			//			for no, ds := range rs.RF.DSLevel {
			//				if no == rs.RF.DsBondingSize()-1 {
			//					separator = ""
			//				}
			//				fmt.Printf("%.01f%v", float32(ds)/10, separator)
			//			}
			//			fmt.Println("")
			//
		}
	}

}
<commit_msg>Removed not used code<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"strings"

	"github.com/mrspock/godocsis"
)

var (
	community = flag.String("community", "public", "RW community to use when sending restart request")
	csvmode   = flag.Bool("json", false, "JSON mode for easy import")
)

func printJSON(data []string) {
	var rs []string
	for i, v := range data {
		rs = append(rs, fmt.Sprintf("\n{\n\t\"id\": \"%d\",\n\t\"value\": \"%s\"\n}", i, v))
	}
	fmt.Println("[")
	fmt.Println(strings.Join(rs, ","))
	fmt.Println("]")
}
func main() {
	flag.Parse()
	if len(flag.Args()) < 1 {
		fmt.Println("Usage: cmlog [--json] [--community <community>] ip")
		return
	}

	s := godocsis.Session
	s.Community = *community
	for _, ip := range flag.Args() {
		s.Target = ip
		rs, err := godocsis.GetLogs(s)
		if err != nil {
			//fmt.Fprintf(os.Stderr, "Problem: %v", err)
			fmt.Fprintf(os.Stderr, "%s:OFFLINE\n", ip)
			//panic(err)
		} else {
			if *csvmode {
				printJSON(rs)

			} else {
				//printVerbose(rs)
				for i, l := range rs {
					fmt.Printf("%d: %s\n", i, l)

				}
			}

		}
	}

}
<|endoftext|>


<commit_before>package tui

import (
	"image"
	"math"
)

var _ Widget = &Box{}

// Alignment is used to set the direction in which widgets are laid out.
type Alignment int

// Available alignment options.
const (
	Horizontal Alignment = iota
	Vertical
)

// Box is a layout for placing widgets either horizontally or vertically. If
// horizontally, all widgets with have the same height. If vertically, they
// will all have the same width.
type Box struct {
	WidgetBase

	children []Widget

	border bool
	title  string

	alignment Alignment
}

// NewVBox returns a new vertically aligned Box.
func NewVBox(c ...Widget) *Box {
	return &Box{
		children:  c,
		alignment: Vertical,
	}
}

// NewHBox returns a new horizontally aligned Box.
func NewHBox(c ...Widget) *Box {
	return &Box{
		children:  c,
		alignment: Horizontal,
	}
}

// Append adds the given widget at the end of the Box.
func (b *Box) Append(w Widget) {
	b.children = append(b.children, w)
}

// SetBorder sets whether the border is visible or not.
func (b *Box) SetBorder(enabled bool) {
	b.border = enabled
}

// SetTitle sets the title of the box.
func (b *Box) SetTitle(title string) {
	b.title = title
}

// Alignment returns the current alignment of the Box.
func (b *Box) Alignment() Alignment {
	return b.alignment
}

// IsFocused return true if one of the children is focused.
func (b *Box) IsFocused() bool {
	for _, w := range b.children {
		if w.IsFocused() {
			return true
		}
	}
	return false
}

// Draw recursively draws the widgets it contains.
func (b *Box) Draw(p *Painter) {
	style := "box"
	if b.IsFocused() {
		style += ".focused"
	}

	sz := b.Size()

	if b.border {
		p.WithStyle(style, func(p *Painter) {
			p.DrawRect(0, 0, sz.X, sz.Y)
			p.WithMask(image.Rect(2, 0, sz.X-3, 0)).DrawText(2, 0, b.title)
		})

		p.Translate(1, 1)
		defer p.Restore()
	}

	var off image.Point
	for _, child := range b.children {
		switch b.Alignment() {
		case Horizontal:
			p.Translate(off.X, 0)
		case Vertical:
			p.Translate(0, off.Y)
		}

		child.Draw(p.WithMask(image.Rectangle{
			Min: image.ZP,
			Max: child.Size().Sub(image.Point{1, 1}),
		}))

		p.Restore()

		off = off.Add(child.Size())
	}
}

// MinSizeHint returns the minimum size hint for the layout.
func (b *Box) MinSizeHint() image.Point {
	var minSize image.Point

	for _, child := range b.children {
		size := child.MinSizeHint()
		if b.Alignment() == Horizontal {
			minSize.X += size.X
			if size.Y > minSize.Y {
				minSize.Y = size.Y
			}
		} else {
			minSize.Y += size.Y
			if size.X > minSize.X {
				minSize.X = size.X
			}
		}
	}

	if b.border {
		minSize = minSize.Add(image.Point{2, 2})
	}

	return minSize
}

// SizeHint returns the recommended size hint for the layout.
func (b *Box) SizeHint() image.Point {
	var sizeHint image.Point

	for _, child := range b.children {
		size := child.SizeHint()
		if b.Alignment() == Horizontal {
			sizeHint.X += size.X
			if size.Y > sizeHint.Y {
				sizeHint.Y = size.Y
			}
		} else {
			sizeHint.Y += size.Y
			if size.X > sizeHint.X {
				sizeHint.X = size.X
			}
		}
	}

	if b.border {
		sizeHint = sizeHint.Add(image.Point{2, 2})
	}

	return sizeHint
}

// OnKeyEvent handles an event and propagates it to all children.
func (b *Box) OnKeyEvent(ev KeyEvent) {
	for _, child := range b.children {
		child.OnKeyEvent(ev)
	}
}

// Resize recursively updates the size of the Box and all the widgets it
// contains. This is a potentially expensive operation and should be invoked
// with restraint.
//
// Resize is called by the layout engine and is not intended to be used by end
// users.
func (b *Box) Resize(size image.Point) {
	b.size = size
	inner := b.size
	if b.border {
		inner = b.size.Sub(image.Point{2, 2})
	}
	b.layoutChildren(inner)
}

func (b *Box) layoutChildren(size image.Point) {
	space := doLayout(b.children, dim(b.Alignment(), size), b.Alignment())

	for i, s := range space {
		switch b.Alignment() {
		case Horizontal:
			b.children[i].Resize(image.Point{s, size.Y})
		case Vertical:
			b.children[i].Resize(image.Point{size.X, s})
		}
	}
}

func doLayout(ws []Widget, space int, a Alignment) []int {
	sizes := make([]int, len(ws))

	if len(sizes) == 0 {
		return sizes
	}

	remaining := space

	// Distribute MinSizeHint
	for {
		var changed bool
		for i, sz := range sizes {
			if sz < dim(a, ws[i].MinSizeHint()) {
				sizes[i] = sz + 1
				remaining--
				if remaining <= 0 {
					goto Resize
				}
				changed = true
			}
		}
		if !changed {
			break
		}
	}

	// Distribute Minimum
	for {
		var changed bool
		for i, sz := range sizes {
			p := alignedSizePolicy(a, ws[i])
			if p == Minimum && sz < dim(a, ws[i].SizeHint()) {
				sizes[i] = sz + 1
				remaining--
				if remaining <= 0 {
					goto Resize
				}
				changed = true
			}
		}
		if !changed {
			break
		}
	}

	// Distribute Preferred
	for {
		var changed bool
		for i, sz := range sizes {
			p := alignedSizePolicy(a, ws[i])
			if (p == Preferred || p == Maximum) && sz < dim(a, ws[i].SizeHint()) {
				sizes[i] = sz + 1
				remaining--
				if remaining <= 0 {
					goto Resize
				}
				changed = true
			}
		}
		if !changed {
			break
		}
	}

	// Distribute Expanding
	for {
		var changed bool
		for i, sz := range sizes {
			p := alignedSizePolicy(a, ws[i])
			if p == Expanding {
				sizes[i] = sz + 1
				remaining--
				if remaining <= 0 {
					goto Resize
				}
				changed = true
			}
		}
		if !changed {
			break
		}
	}

	// Distribute remaining space
	for {
		min := math.MaxInt8
		for i, s := range sizes {
			p := alignedSizePolicy(a, ws[i])
			if (p == Preferred || p == Minimum) && s <= min {
				min = s
			}
		}
		var changed bool
		for i, sz := range sizes {
			if sz != min {
				continue
			}
			p := alignedSizePolicy(a, ws[i])
			if p == Preferred || p == Minimum {
				sizes[i] = sz + 1
				remaining--
				if remaining <= 0 {
					goto Resize
				}
				changed = true
			}
		}
		if !changed {
			break
		}
	}

Resize:

	return sizes
}

func dim(a Alignment, pt image.Point) int {
	if a == Horizontal {
		return pt.X
	}
	return pt.Y
}

func alignedSizePolicy(a Alignment, w Widget) SizePolicy {
	hpol, vpol := w.SizePolicy()
	if a == Horizontal {
		return hpol
	}
	return vpol
}
<commit_msg>Fix typo in docs<commit_after>package tui

import (
	"image"
	"math"
)

var _ Widget = &Box{}

// Alignment is used to set the direction in which widgets are laid out.
type Alignment int

// Available alignment options.
const (
	Horizontal Alignment = iota
	Vertical
)

// Box is a layout for placing widgets either horizontally or vertically. If
// horizontally, all widgets will have the same height. If vertically, they
// will all have the same width.
type Box struct {
	WidgetBase

	children []Widget

	border bool
	title  string

	alignment Alignment
}

// NewVBox returns a new vertically aligned Box.
func NewVBox(c ...Widget) *Box {
	return &Box{
		children:  c,
		alignment: Vertical,
	}
}

// NewHBox returns a new horizontally aligned Box.
func NewHBox(c ...Widget) *Box {
	return &Box{
		children:  c,
		alignment: Horizontal,
	}
}

// Append adds the given widget at the end of the Box.
func (b *Box) Append(w Widget) {
	b.children = append(b.children, w)
}

// SetBorder sets whether the border is visible or not.
func (b *Box) SetBorder(enabled bool) {
	b.border = enabled
}

// SetTitle sets the title of the box.
func (b *Box) SetTitle(title string) {
	b.title = title
}

// Alignment returns the current alignment of the Box.
func (b *Box) Alignment() Alignment {
	return b.alignment
}

// IsFocused return true if one of the children is focused.
func (b *Box) IsFocused() bool {
	for _, w := range b.children {
		if w.IsFocused() {
			return true
		}
	}
	return false
}

// Draw recursively draws the widgets it contains.
func (b *Box) Draw(p *Painter) {
	style := "box"
	if b.IsFocused() {
		style += ".focused"
	}

	sz := b.Size()

	if b.border {
		p.WithStyle(style, func(p *Painter) {
			p.DrawRect(0, 0, sz.X, sz.Y)
			p.WithMask(image.Rect(2, 0, sz.X-3, 0)).DrawText(2, 0, b.title)
		})

		p.Translate(1, 1)
		defer p.Restore()
	}

	var off image.Point
	for _, child := range b.children {
		switch b.Alignment() {
		case Horizontal:
			p.Translate(off.X, 0)
		case Vertical:
			p.Translate(0, off.Y)
		}

		child.Draw(p.WithMask(image.Rectangle{
			Min: image.ZP,
			Max: child.Size().Sub(image.Point{1, 1}),
		}))

		p.Restore()

		off = off.Add(child.Size())
	}
}

// MinSizeHint returns the minimum size hint for the layout.
func (b *Box) MinSizeHint() image.Point {
	var minSize image.Point

	for _, child := range b.children {
		size := child.MinSizeHint()
		if b.Alignment() == Horizontal {
			minSize.X += size.X
			if size.Y > minSize.Y {
				minSize.Y = size.Y
			}
		} else {
			minSize.Y += size.Y
			if size.X > minSize.X {
				minSize.X = size.X
			}
		}
	}

	if b.border {
		minSize = minSize.Add(image.Point{2, 2})
	}

	return minSize
}

// SizeHint returns the recommended size hint for the layout.
func (b *Box) SizeHint() image.Point {
	var sizeHint image.Point

	for _, child := range b.children {
		size := child.SizeHint()
		if b.Alignment() == Horizontal {
			sizeHint.X += size.X
			if size.Y > sizeHint.Y {
				sizeHint.Y = size.Y
			}
		} else {
			sizeHint.Y += size.Y
			if size.X > sizeHint.X {
				sizeHint.X = size.X
			}
		}
	}

	if b.border {
		sizeHint = sizeHint.Add(image.Point{2, 2})
	}

	return sizeHint
}

// OnKeyEvent handles an event and propagates it to all children.
func (b *Box) OnKeyEvent(ev KeyEvent) {
	for _, child := range b.children {
		child.OnKeyEvent(ev)
	}
}

// Resize recursively updates the size of the Box and all the widgets it
// contains. This is a potentially expensive operation and should be invoked
// with restraint.
//
// Resize is called by the layout engine and is not intended to be used by end
// users.
func (b *Box) Resize(size image.Point) {
	b.size = size
	inner := b.size
	if b.border {
		inner = b.size.Sub(image.Point{2, 2})
	}
	b.layoutChildren(inner)
}

func (b *Box) layoutChildren(size image.Point) {
	space := doLayout(b.children, dim(b.Alignment(), size), b.Alignment())

	for i, s := range space {
		switch b.Alignment() {
		case Horizontal:
			b.children[i].Resize(image.Point{s, size.Y})
		case Vertical:
			b.children[i].Resize(image.Point{size.X, s})
		}
	}
}

func doLayout(ws []Widget, space int, a Alignment) []int {
	sizes := make([]int, len(ws))

	if len(sizes) == 0 {
		return sizes
	}

	remaining := space

	// Distribute MinSizeHint
	for {
		var changed bool
		for i, sz := range sizes {
			if sz < dim(a, ws[i].MinSizeHint()) {
				sizes[i] = sz + 1
				remaining--
				if remaining <= 0 {
					goto Resize
				}
				changed = true
			}
		}
		if !changed {
			break
		}
	}

	// Distribute Minimum
	for {
		var changed bool
		for i, sz := range sizes {
			p := alignedSizePolicy(a, ws[i])
			if p == Minimum && sz < dim(a, ws[i].SizeHint()) {
				sizes[i] = sz + 1
				remaining--
				if remaining <= 0 {
					goto Resize
				}
				changed = true
			}
		}
		if !changed {
			break
		}
	}

	// Distribute Preferred
	for {
		var changed bool
		for i, sz := range sizes {
			p := alignedSizePolicy(a, ws[i])
			if (p == Preferred || p == Maximum) && sz < dim(a, ws[i].SizeHint()) {
				sizes[i] = sz + 1
				remaining--
				if remaining <= 0 {
					goto Resize
				}
				changed = true
			}
		}
		if !changed {
			break
		}
	}

	// Distribute Expanding
	for {
		var changed bool
		for i, sz := range sizes {
			p := alignedSizePolicy(a, ws[i])
			if p == Expanding {
				sizes[i] = sz + 1
				remaining--
				if remaining <= 0 {
					goto Resize
				}
				changed = true
			}
		}
		if !changed {
			break
		}
	}

	// Distribute remaining space
	for {
		min := math.MaxInt8
		for i, s := range sizes {
			p := alignedSizePolicy(a, ws[i])
			if (p == Preferred || p == Minimum) && s <= min {
				min = s
			}
		}
		var changed bool
		for i, sz := range sizes {
			if sz != min {
				continue
			}
			p := alignedSizePolicy(a, ws[i])
			if p == Preferred || p == Minimum {
				sizes[i] = sz + 1
				remaining--
				if remaining <= 0 {
					goto Resize
				}
				changed = true
			}
		}
		if !changed {
			break
		}
	}

Resize:

	return sizes
}

func dim(a Alignment, pt image.Point) int {
	if a == Horizontal {
		return pt.X
	}
	return pt.Y
}

func alignedSizePolicy(a Alignment, w Widget) SizePolicy {
	hpol, vpol := w.SizePolicy()
	if a == Horizontal {
		return hpol
	}
	return vpol
}
<|endoftext|>