<commit_before>package streamer

import (
	"io"
	"math"
	"os"
	"sync"

	"github.com/spacemonkeygo/errors"
	"github.com/ugorji/go/codec"
)

var _ Mux = &CborMux{}

type CborMux struct {
	// the fd tip handles append; reads all use ReadAt and track their offsets individually
	file  *os.File
	codec *codec.Encoder
	wmu   sync.Mutex
}

type cborMuxRow struct {
	Label int    `json:"l"`
	Msg   []byte `json:"m,omitempty"`
	Sig   int    `json:"x,omitempty"` // 1->closed
}

func CborFileMux(filePath string) Mux {
	file, err := os.OpenFile(filePath, os.O_CREATE|os.O_RDWR|os.O_EXCL, 0644)
	if err != nil {
		panic(errors.IOError.Wrap(err))
	}
	file.Write([]byte{codec.CborStreamArray})
	return &CborMux{
		file:  file,
		codec: codec.NewEncoder(file, new(codec.CborHandle)),
	}
	// consider using runtime.`SetFinalizer to close?  currently unhandled.
}

func (m *CborMux) write(label int, msg []byte) {
	m.wmu.Lock()
	defer m.wmu.Unlock()
	const magic_RAW = 0
	const magic_UTF8 = 1
	//	m.codec.MustEncode(cborMuxRow{
	//		Label: label,
	//		Msg:   msg,
	//	})
	_, enc := codec.GenHelperEncoder(m.codec)
	enc.EncodeMapStart(2)
	enc.EncodeString(magic_UTF8, "l")
	enc.EncodeInt(int64(label))
	enc.EncodeString(magic_UTF8, "m")
	enc.EncodeStringBytes(magic_RAW, msg)
}

func (m *CborMux) Close() {
	m.wmu.Lock()
	defer m.wmu.Unlock()
	m.file.Write([]byte{0xff}) // should be `codec.CborStreamBreak`, plz update upstream for vis
	// don't *actually* close, because readers can still be active on the same fd.
	// TODO further writes should be forced into an error state
}

func (m *CborMux) Appender(label int) io.WriteCloser {
	return &cborMuxAppender{m, label}
}

type cborMuxAppender struct {
	m     *CborMux
	label int
}

func (a *cborMuxAppender) Write(msg []byte) (int, error) {
	a.m.write(a.label, msg)
	return len(msg), nil
}

func (a *cborMuxAppender) Close() error {
	a.m.wmu.Lock()
	defer a.m.wmu.Unlock()
	const magic_UTF8 = 1
	_, enc := codec.GenHelperEncoder(a.m.codec)
	enc.EncodeMapStart(2)
	enc.EncodeString(magic_UTF8, "l")
	enc.EncodeInt(int64(a.label))
	enc.EncodeString(magic_UTF8, "x")
	enc.EncodeInt(int64(1))
	return nil
}

func (m *CborMux) Reader(labels ...int) io.Reader {
	// asking for a reader for a label that was never used will never
	// hit a close flag, so... don't do that?
	r := io.NewSectionReader(m.file, 1, math.MaxInt64/2)
	// TODO offset of one because that's the array open!
	// do something much more sane than skip it, please
	return &cborMuxReader{
		labels: &intset{labels},
		codec:  codec.NewDecoder(NewTailReader(r), new(codec.CborHandle)),
	}
}

type cborMuxReader struct {
	labels *intset // remove them as we hit their close
	codec  *codec.Decoder
	buf    []byte // any remaining bytes from the last incomplete read
}

func (r *cborMuxReader) Read(msg []byte) (n int, err error) {
	// loop over read attempts to pump uninteresting messages
	for n == 0 && err == nil {
		n, err = r.read(msg)
	}
	return
}

/*
	Internal read method; may return `(0,nil)` in a number of occations,
	all of which the public `Read` method will translate into a retry
	so that higher level consumers of the Reader interface don't get stuck
	spin-looping.

	Specifically, these situations cause empty reads:
	  - absorbing a message that isn't selected by this reader's filters
	  - absorbing a message that's a signal and has no body

	Hitting EOF on the backing file is handled by a `tailReader` before
	data gets to this method, because the cbor decoder needs to be insulated
	from seeing EOFs.
*/
func (r *cborMuxReader) read(msg []byte) (int, error) {
	// first, finish yielding any buffered bytes from prior incomplete reads.
	if len(r.buf) > 0 {
		n := copy(msg, r.buf)
		r.buf = r.buf[n:]
		return n, nil
	}
	// scan the file for more rows and work with any that match our labels.
	var row cborMuxRow
	err := r.codec.Decode(&row)
	if err == io.EOF {
		// we don't pass EOF up unless our cbor says we're closed.
		// this could be a "temporary" EOF and appends will still be incoming.
		return 0, nil
	} else if err != nil {
		panic(err)
	}
	switch row.Sig {
	case 1:
		r.labels.Remove(row.Label)
		if r.labels.Empty() {
			return 0, io.EOF
		} else {
			// still more labels must be closed before we're EOF
			return 0, nil
		}
	default:
		if r.labels.Contains(row.Label) {
			n := copy(msg, row.Msg)
			r.buf = row.Msg[n:]
			return n, nil
		} else {
			// consuming an uninteresting label
			return 0, nil
		}
	}
}

type intset struct {
	s []int
}

func (s *intset) Contains(i int) bool {
	for _, v := range s.s {
		if i == v {
			return true
		}
	}
	return false
}

func (s *intset) Remove(i int) {
	old := s.s
	s.s = make([]int, 0, len(old))
	for _, v := range old {
		if v != i {
			s.s = append(s.s, v)
		}
	}
}

func (s *intset) Empty() bool {
	return len(s.s) == 0
}
<commit_msg>Put a bufferer in the mux reader flow.<commit_after>package streamer

import (
	"bufio"
	"io"
	"math"
	"os"
	"sync"

	"github.com/spacemonkeygo/errors"
	"github.com/ugorji/go/codec"
)

var _ Mux = &CborMux{}

type CborMux struct {
	// the fd tip handles append; reads all use ReadAt and track their offsets individually
	file  *os.File
	codec *codec.Encoder
	wmu   sync.Mutex
}

type cborMuxRow struct {
	Label int    `json:"l"`
	Msg   []byte `json:"m,omitempty"`
	Sig   int    `json:"x,omitempty"` // 1->closed
}

func CborFileMux(filePath string) Mux {
	file, err := os.OpenFile(filePath, os.O_CREATE|os.O_RDWR|os.O_EXCL, 0644)
	if err != nil {
		panic(errors.IOError.Wrap(err))
	}
	file.Write([]byte{codec.CborStreamArray})
	return &CborMux{
		file:  file,
		codec: codec.NewEncoder(file, new(codec.CborHandle)),
	}
	// consider using runtime.`SetFinalizer to close?  currently unhandled.
}

func (m *CborMux) write(label int, msg []byte) {
	m.wmu.Lock()
	defer m.wmu.Unlock()
	const magic_RAW = 0
	const magic_UTF8 = 1
	//	m.codec.MustEncode(cborMuxRow{
	//		Label: label,
	//		Msg:   msg,
	//	})
	_, enc := codec.GenHelperEncoder(m.codec)
	enc.EncodeMapStart(2)
	enc.EncodeString(magic_UTF8, "l")
	enc.EncodeInt(int64(label))
	enc.EncodeString(magic_UTF8, "m")
	enc.EncodeStringBytes(magic_RAW, msg)
}

func (m *CborMux) Close() {
	m.wmu.Lock()
	defer m.wmu.Unlock()
	m.file.Write([]byte{0xff}) // should be `codec.CborStreamBreak`, plz update upstream for vis
	// don't *actually* close, because readers can still be active on the same fd.
	// TODO further writes should be forced into an error state
}

func (m *CborMux) Appender(label int) io.WriteCloser {
	return &cborMuxAppender{m, label}
}

type cborMuxAppender struct {
	m     *CborMux
	label int
}

func (a *cborMuxAppender) Write(msg []byte) (int, error) {
	a.m.write(a.label, msg)
	return len(msg), nil
}

func (a *cborMuxAppender) Close() error {
	a.m.wmu.Lock()
	defer a.m.wmu.Unlock()
	const magic_UTF8 = 1
	_, enc := codec.GenHelperEncoder(a.m.codec)
	enc.EncodeMapStart(2)
	enc.EncodeString(magic_UTF8, "l")
	enc.EncodeInt(int64(a.label))
	enc.EncodeString(magic_UTF8, "x")
	enc.EncodeInt(int64(1))
	return nil
}

func (m *CborMux) Reader(labels ...int) io.Reader {
	// asking for a reader for a label that was never used will never
	// hit a close flag, so... don't do that?
	var r io.Reader
	r = io.NewSectionReader(m.file, 1, math.MaxInt64/2)
	// TODO offset of one because that's the array open!
	// do something much more sane than skip it, please
	r = NewTailReader(r)
	r = bufio.NewReader(r)
	return &cborMuxReader{
		labels: &intset{labels},
		codec:  codec.NewDecoder(r, new(codec.CborHandle)),
	}
}

type cborMuxReader struct {
	labels *intset // remove them as we hit their close
	codec  *codec.Decoder
	buf    []byte // any remaining bytes from the last incomplete read
}

func (r *cborMuxReader) Read(msg []byte) (n int, err error) {
	// loop over read attempts to pump uninteresting messages
	for n == 0 && err == nil {
		n, err = r.read(msg)
	}
	return
}

/*
	Internal read method; may return `(0,nil)` in a number of occations,
	all of which the public `Read` method will translate into a retry
	so that higher level consumers of the Reader interface don't get stuck
	spin-looping.

	Specifically, these situations cause empty reads:
	  - absorbing a message that isn't selected by this reader's filters
	  - absorbing a message that's a signal and has no body

	Hitting EOF on the backing file is handled by a `tailReader` before
	data gets to this method, because the cbor decoder needs to be insulated
	from seeing EOFs.
*/
func (r *cborMuxReader) read(msg []byte) (int, error) {
	// first, finish yielding any buffered bytes from prior incomplete reads.
	if len(r.buf) > 0 {
		n := copy(msg, r.buf)
		r.buf = r.buf[n:]
		return n, nil
	}
	// scan the file for more rows and work with any that match our labels.
	var row cborMuxRow
	err := r.codec.Decode(&row)
	if err == io.EOF {
		// we don't pass EOF up unless our cbor says we're closed.
		// this could be a "temporary" EOF and appends will still be incoming.
		return 0, nil
	} else if err != nil {
		panic(err)
	}
	switch row.Sig {
	case 1:
		r.labels.Remove(row.Label)
		if r.labels.Empty() {
			return 0, io.EOF
		} else {
			// still more labels must be closed before we're EOF
			return 0, nil
		}
	default:
		if r.labels.Contains(row.Label) {
			n := copy(msg, row.Msg)
			r.buf = row.Msg[n:]
			return n, nil
		} else {
			// consuming an uninteresting label
			return 0, nil
		}
	}
}

type intset struct {
	s []int
}

func (s *intset) Contains(i int) bool {
	for _, v := range s.s {
		if i == v {
			return true
		}
	}
	return false
}

func (s *intset) Remove(i int) {
	old := s.s
	s.s = make([]int, 0, len(old))
	for _, v := range old {
		if v != i {
			s.s = append(s.s, v)
		}
	}
}

func (s *intset) Empty() bool {
	return len(s.s) == 0
}
<|endoftext|>


<commit_before>package solver

// This file deals with an attempt for an efficient binary/ternary clause allocator/deallocator.
// Since lots of binary/ternary clauses are created then (sometimes) destroyed, we allocate
// and reuse lots of them here, to relax the GC's work.

const (
	nbLitsAlloc = 5000000 // How many literals are initialized at first?
)

type allocator struct {
	lits    []Lit // A list of lits, that will be sliced to make []Lit
	ptrFree int   // Index of the first free item in lits
}

var alloc allocator

// newLits returns a slice of lits containing the given literals.
// It is taken from the preinitialized pool if possible,
// or is created from scratch.
func (a *allocator) newLits(lits ...Lit) []Lit {
	if a.ptrFree+len(lits) > len(a.lits) {
		a.lits = make([]Lit, nbLitsAlloc)
		copy(a.lits, lits)
		a.ptrFree = len(lits)
		return a.lits[:len(lits)]
	}
	copy(a.lits[a.ptrFree:], lits)
	a.ptrFree += len(lits)
	return a.lits[a.ptrFree-len(lits) : a.ptrFree]
}
<commit_msg>updated clause_alloc's comments<commit_after>package solver

// This file deals with an attempt for an efficient clause allocator/deallocator, to relax GC's work.

const (
	nbLitsAlloc = 5000000 // How many literals are initialized at first?
)

type allocator struct {
	lits    []Lit // A list of lits, that will be sliced to make []Lit
	ptrFree int   // Index of the first free item in lits
}

var alloc allocator

// newLits returns a slice of lits containing the given literals.
// It is taken from the preinitialized pool if possible,
// or is created from scratch.
func (a *allocator) newLits(lits ...Lit) []Lit {
	if a.ptrFree+len(lits) > len(a.lits) {
		a.lits = make([]Lit, nbLitsAlloc)
		copy(a.lits, lits)
		a.ptrFree = len(lits)
		return a.lits[:len(lits)]
	}
	copy(a.lits[a.ptrFree:], lits)
	a.ptrFree += len(lits)
	return a.lits[a.ptrFree-len(lits) : a.ptrFree]
}
<|endoftext|>


<commit_before>// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"context"
	"fmt"
	"os"
	"time"

	tea "github.com/charmbracelet/bubbletea"
	humanize "github.com/dustin/go-humanize"
	"github.com/minio/cli"
	json "github.com/minio/colorjson"
	"github.com/minio/madmin-go"
	"github.com/minio/mc/pkg/probe"
)

var adminSpeedtestFlags = []cli.Flag{
	cli.StringFlag{
		Name:  "duration",
		Usage: "duration the entire speedtests are run",
		Value: "10s",
	},
	cli.StringFlag{
		Name:  "size",
		Usage: "size of the object used for uploads/downloads",
		Value: "64MiB",
	},
	cli.IntFlag{
		Name:  "concurrent",
		Usage: "number of concurrent requests per server",
		Value: 32,
	},
	cli.BoolFlag{
		Name:  "verbose, v",
		Usage: "Show per-server stats",
	},
}

var adminSpeedtestCmd = cli.Command{
	Name:            "speedtest",
	Usage:           "Run server side speed test",
	Action:          mainAdminSpeedtest,
	OnUsageError:    onUsageError,
	Before:          setGlobalsFromContext,
	Flags:           append(adminSpeedtestFlags, globalFlags...),
	HideHelpCommand: true,
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} [FLAGS] TARGET

FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
EXAMPLES:
  1. Run speedtest with autotuning the concurrency to figure out the maximum throughput and iops values:
     {{.Prompt}} {{.HelpName}} myminio/

  2. Run speedtest for 20 seconds with object size of 128MiB, 32 concurrent requests per server:
     {{.Prompt}} {{.HelpName}} myminio/ --duration 20s --size 128MiB --concurrent 32
`,
}

func (s speedTestResult) StringVerbose() (msg string) {
	result := s.result
	if globalSpeedTestVerbose {
		msg += "\n\n"
		for _, node := range result.PUTStats.Servers {
			msg += fmt.Sprintf("PUT:\n   * %s: %s/s %s objs/s", node.Endpoint, humanize.IBytes(node.ThroughputPerSec), humanize.Comma(int64(node.ObjectsPerSec)))
			if node.Err != "" {
				msg += " error: " + node.Err
			}
		}

		msg += "\n"

		for _, node := range result.GETStats.Servers {
			msg += fmt.Sprintf("GET:\n   * %s: %s/s %s objs/s", node.Endpoint, humanize.IBytes(node.ThroughputPerSec), humanize.Comma(int64(node.ObjectsPerSec)))
			if node.Err != "" {
				msg += " error: " + node.Err
			}
		}

		msg += "\n"
	}
	return msg
}

func (s speedTestResult) String() (msg string) {
	result := s.result
	msg += fmt.Sprintf("MinIO %s, %d servers, %d drives, %s objects, %d threads",
		result.Version, result.Servers, result.Disks,
		humanize.IBytes(uint64(result.Size)), result.Concurrent)

	return msg
}

func (s speedTestResult) JSON() string {
	JSONBytes, e := json.MarshalIndent(s.result, "", "    ")
	fatalIf(probe.NewError(e), "Unable to marshal into JSON.")
	return string(JSONBytes)
}

var globalSpeedTestVerbose bool

func mainAdminSpeedtest(ctx *cli.Context) error {
	if len(ctx.Args()) != 1 {
		cli.ShowCommandHelpAndExit(ctx, "speedtest", 1) // last argument is exit code
	}

	// Get the alias parameter from cli
	args := ctx.Args()
	aliasedURL := args.Get(0)

	client, perr := newAdminClient(aliasedURL)
	if perr != nil {
		fatalIf(perr.Trace(aliasedURL), "Unable to initialize admin client.")
		return nil
	}

	ctxt, cancel := context.WithCancel(globalContext)
	defer cancel()

	duration, e := time.ParseDuration(ctx.String("duration"))
	if e != nil {
		fatalIf(probe.NewError(e), "Unable to parse duration")
		return nil
	}
	if duration <= 0 {
		fatalIf(errInvalidArgument(), "duration cannot be 0 or negative")
		return nil
	}
	size, e := humanize.ParseBytes(ctx.String("size"))
	if e != nil {
		fatalIf(probe.NewError(e), "Unable to parse object size")
		return nil
	}
	if size < 0 {
		fatalIf(errInvalidArgument(), "size is expected to be atleast 0 bytes")
		return nil
	}
	concurrent := ctx.Int("concurrent")
	if concurrent <= 0 {
		fatalIf(errInvalidArgument(), "concurrency cannot be '0' or negative")
		return nil
	}
	globalSpeedTestVerbose = ctx.Bool("verbose")

	// Turn-off autotuning only when "concurrent" is specified
	// in all other scenarios keep auto-tuning on.
	autotune := !ctx.IsSet("concurrent")

	resultCh, err := client.Speedtest(ctxt, madmin.SpeedtestOpts{
		Size:        int(size),
		Duration:    duration,
		Concurrency: concurrent,
		Autotune:    autotune,
	})
	fatalIf(probe.NewError(err), "Failed to execute speedtest")

	if globalJSON {
		for result := range resultCh {
			if result.Version == "" {
				continue
			}
			printMsg(speedTestResult{
				result: result,
			})
		}
		return nil
	}

	done := make(chan struct{})

	p := tea.NewProgram(initSpeedTestUI())
	go func() {
		if e := p.Start(); e != nil {
			os.Exit(1)
		}
		close(done)
	}()

	go func() {
		var result madmin.SpeedTestResult
		for result = range resultCh {
			if result.Version == "" {
				continue
			}
			p.Send(speedTestResult{
				result: result,
			})
		}
		p.Send(speedTestResult{
			result: result,
			final:  true,
		})
	}()

	<-done

	return nil
}
<commit_msg>fix: admin speedtest verbose output \n (#3943)<commit_after>// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"context"
	"fmt"
	"os"
	"time"

	tea "github.com/charmbracelet/bubbletea"
	humanize "github.com/dustin/go-humanize"
	"github.com/minio/cli"
	json "github.com/minio/colorjson"
	"github.com/minio/madmin-go"
	"github.com/minio/mc/pkg/probe"
)

var adminSpeedtestFlags = []cli.Flag{
	cli.StringFlag{
		Name:  "duration",
		Usage: "duration the entire speedtests are run",
		Value: "10s",
	},
	cli.StringFlag{
		Name:  "size",
		Usage: "size of the object used for uploads/downloads",
		Value: "64MiB",
	},
	cli.IntFlag{
		Name:  "concurrent",
		Usage: "number of concurrent requests per server",
		Value: 32,
	},
	cli.BoolFlag{
		Name:  "verbose, v",
		Usage: "Show per-server stats",
	},
}

var adminSpeedtestCmd = cli.Command{
	Name:            "speedtest",
	Usage:           "Run server side speed test",
	Action:          mainAdminSpeedtest,
	OnUsageError:    onUsageError,
	Before:          setGlobalsFromContext,
	Flags:           append(adminSpeedtestFlags, globalFlags...),
	HideHelpCommand: true,
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} [FLAGS] TARGET

FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
EXAMPLES:
  1. Run speedtest with autotuning the concurrency to figure out the maximum throughput and iops values:
     {{.Prompt}} {{.HelpName}} myminio/

  2. Run speedtest for 20 seconds with object size of 128MiB, 32 concurrent requests per server:
     {{.Prompt}} {{.HelpName}} myminio/ --duration 20s --size 128MiB --concurrent 32
`,
}

func (s speedTestResult) StringVerbose() (msg string) {
	result := s.result
	if globalSpeedTestVerbose {
		msg += "\n\n"
		msg += "PUT:\n"
		for _, node := range result.PUTStats.Servers {
			msg += fmt.Sprintf("   * %s: %s/s %s objs/s\n", node.Endpoint, humanize.IBytes(node.ThroughputPerSec), humanize.Comma(int64(node.ObjectsPerSec)))
			if node.Err != "" {
				msg += " error: " + node.Err
			}
		}

		msg += "GET:\n"
		for _, node := range result.GETStats.Servers {
			msg += fmt.Sprintf("   * %s: %s/s %s objs/s\n", node.Endpoint, humanize.IBytes(node.ThroughputPerSec), humanize.Comma(int64(node.ObjectsPerSec)))
			if node.Err != "" {
				msg += " error: " + node.Err
			}
		}

	}
	return msg
}

func (s speedTestResult) String() (msg string) {
	result := s.result
	msg += fmt.Sprintf("MinIO %s, %d servers, %d drives, %s objects, %d threads",
		result.Version, result.Servers, result.Disks,
		humanize.IBytes(uint64(result.Size)), result.Concurrent)

	return msg
}

func (s speedTestResult) JSON() string {
	JSONBytes, e := json.MarshalIndent(s.result, "", "    ")
	fatalIf(probe.NewError(e), "Unable to marshal into JSON.")
	return string(JSONBytes)
}

var globalSpeedTestVerbose bool

func mainAdminSpeedtest(ctx *cli.Context) error {
	if len(ctx.Args()) != 1 {
		cli.ShowCommandHelpAndExit(ctx, "speedtest", 1) // last argument is exit code
	}

	// Get the alias parameter from cli
	args := ctx.Args()
	aliasedURL := args.Get(0)

	client, perr := newAdminClient(aliasedURL)
	if perr != nil {
		fatalIf(perr.Trace(aliasedURL), "Unable to initialize admin client.")
		return nil
	}

	ctxt, cancel := context.WithCancel(globalContext)
	defer cancel()

	duration, e := time.ParseDuration(ctx.String("duration"))
	if e != nil {
		fatalIf(probe.NewError(e), "Unable to parse duration")
		return nil
	}
	if duration <= 0 {
		fatalIf(errInvalidArgument(), "duration cannot be 0 or negative")
		return nil
	}
	size, e := humanize.ParseBytes(ctx.String("size"))
	if e != nil {
		fatalIf(probe.NewError(e), "Unable to parse object size")
		return nil
	}
	if size < 0 {
		fatalIf(errInvalidArgument(), "size is expected to be atleast 0 bytes")
		return nil
	}
	concurrent := ctx.Int("concurrent")
	if concurrent <= 0 {
		fatalIf(errInvalidArgument(), "concurrency cannot be '0' or negative")
		return nil
	}
	globalSpeedTestVerbose = ctx.Bool("verbose")

	// Turn-off autotuning only when "concurrent" is specified
	// in all other scenarios keep auto-tuning on.
	autotune := !ctx.IsSet("concurrent")

	resultCh, err := client.Speedtest(ctxt, madmin.SpeedtestOpts{
		Size:        int(size),
		Duration:    duration,
		Concurrency: concurrent,
		Autotune:    autotune,
	})
	fatalIf(probe.NewError(err), "Failed to execute speedtest")

	if globalJSON {
		for result := range resultCh {
			if result.Version == "" {
				continue
			}
			printMsg(speedTestResult{
				result: result,
			})
		}
		return nil
	}

	done := make(chan struct{})

	p := tea.NewProgram(initSpeedTestUI())
	go func() {
		if e := p.Start(); e != nil {
			os.Exit(1)
		}
		close(done)
	}()

	go func() {
		var result madmin.SpeedTestResult
		for result = range resultCh {
			if result.Version == "" {
				continue
			}
			p.Send(speedTestResult{
				result: result,
			})
		}
		p.Send(speedTestResult{
			result: result,
			final:  true,
		})
	}()

	<-done

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"context"
	"flag"

	"os"

	ctrl "sigs.k8s.io/controller-runtime"

	"github.com/jetstack/cert-manager/cmd/cainjector/app"
	logf "github.com/jetstack/cert-manager/pkg/logs"
	utilcmd "github.com/jetstack/cert-manager/pkg/util/cmd"
)

func main() {
	logf.InitLogs(flag.CommandLine)
	defer logf.FlushLogs()
	ctrl.SetLogger(logf.Log)

	stopCh := utilcmd.SetupSignalHandler()
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	go func() {
		<-stopCh
		logf.Log.Info("stop received")
		cancel()
	}()
	cmd := app.NewCommandStartInjectorController(ctx, os.Stdout, os.Stderr)
	cmd.Flags().AddGoFlagSet(flag.CommandLine)

	flag.CommandLine.Parse([]string{})
	if err := cmd.Execute(); err != nil {
		cmd.PrintErrln(err)
	}
}
<commit_msg>Exit(1) and explain what causes the context to be cancelled<commit_after>/*
Copyright 2020 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"context"
	"flag"

	"os"

	ctrl "sigs.k8s.io/controller-runtime"

	"github.com/jetstack/cert-manager/cmd/cainjector/app"
	logf "github.com/jetstack/cert-manager/pkg/logs"
	utilcmd "github.com/jetstack/cert-manager/pkg/util/cmd"
)

func main() {
	logf.InitLogs(flag.CommandLine)
	defer logf.FlushLogs()
	ctrl.SetLogger(logf.Log)

	// Set up signal handlers and a cancellable context which gets cancelled on
	// when either SIGINT or SIGTERM are received.
	stopCh := utilcmd.SetupSignalHandler()
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	go func() {
		<-stopCh
		cancel()
	}()

	cmd := app.NewCommandStartInjectorController(ctx, os.Stdout, os.Stderr)
	cmd.Flags().AddGoFlagSet(flag.CommandLine)

	flag.CommandLine.Parse([]string{})
	if err := cmd.Execute(); err != nil {
		cmd.PrintErrln(err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/golang/glog"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/api/v1"
	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/release_1_4"
	"k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
	clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api"
)

const (
	kubeletAPIPodsURL = "http://127.0.0.1:10255/pods"
	ignorePath        = "/srv/kubernetes/manifests"
	activePath        = "/etc/kubernetes/manifests"
	kubeconfigPath    = "/etc/kubernetes/kubeconfig"
	secretsPath       = "/etc/kubernetes/checkpoint-secrets"

	tempAPIServer = "temp-apiserver"
	kubeAPIServer = "kube-apiserver"
)

var podAPIServerMeta = unversioned.TypeMeta{
	APIVersion: "v1",
	Kind:       "Pod",
}

var (
	secureAPIAddr = fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS"))
)

func main() {
	glog.Info("begin pods checkpointing...")
	run(kubeAPIServer, tempAPIServer, api.NamespaceSystem)
}

func run(actualPodName, tempPodName, namespace string) {
	client := newAPIClient()
	for {
		var podList v1.PodList
		if err := json.Unmarshal(getPodsFromKubeletAPI(), &podList); err != nil {
			glog.Fatal(err)
		}
		switch {
		case bothRunning(podList, actualPodName, tempPodName, namespace):
			glog.Infof("both temp %v and actual %v pods running, removing temp pod", actualPodName, tempPodName)
			// Both the temp and actual pods are running.
			// Remove the temp manifest from the config dir so that the
			// kubelet will stop it.
			if err := os.Remove(activeManifest(tempPodName)); err != nil {
				glog.Error(err)
			}
		case isPodRunning(podList, client, actualPodName, namespace):
			glog.Infof("actual pod %v found, creating temp pod manifest", actualPodName)
			// The actual is running. Let's snapshot the pod,
			// clean it up a bit, and then save it to the ignore path for
			// later use.
			checkpointPod := createCheckpointPod(podList, actualPodName, namespace)
			convertSecretsToVolumeMounts(client, &checkpointPod)
			writeManifest(checkpointPod, tempPodName)
			glog.Infof("finished creating temp pod %v manifest at %s\n", tempPodName, checkpointManifest(tempPodName))

		default:
			glog.Info("no actual pod running, installing temp pod static manifest")
			b, err := ioutil.ReadFile(checkpointManifest(tempPodName))
			if err != nil {
				glog.Error(err)
			} else {
				if err := ioutil.WriteFile(activeManifest(tempPodName), b, 0644); err != nil {
					glog.Error(err)
				}
			}
		}
		time.Sleep(1 * time.Second)
	}
}

func stripNonessentialInfo(p *v1.Pod) {
	p.Spec.ServiceAccountName = ""
	p.Spec.DeprecatedServiceAccount = ""
	p.Status.Reset()
}

func getPodsFromKubeletAPI() []byte {
	var pods []byte
	res, err := http.Get(kubeletAPIPodsURL)
	if err != nil {
		glog.Error(err)
		return pods
	}
	pods, err = ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		glog.Error(err)
	}
	return pods
}

func bothRunning(pods v1.PodList, an, tn, ns string) bool {
	var actualPodSeen, tempPodSeen bool
	for _, p := range pods.Items {
		actualPodSeen = actualPodSeen || isPod(p, an, ns)
		tempPodSeen = tempPodSeen || isPod(p, tn, ns)
		if actualPodSeen && tempPodSeen {
			return true
		}
	}
	return false
}

func isPodRunning(pods v1.PodList, client clientset.Interface, n, ns string) bool {
	for _, p := range pods.Items {
		if isPod(p, n, ns) {
			if n == kubeAPIServer {
				// Make sure it's actually running. Sometimes we get that
				// pod manifest back, but the server is not actually running.
				_, err := client.Discovery().ServerVersion()
				return err == nil
			}
			return true
		}
	}
	return false
}

func isPod(pod v1.Pod, n, ns string) bool {
	return strings.Contains(pod.Name, n) && pod.Namespace == ns
}

// cleanVolumes will sanitize the list of volumes and volume mounts
// to remove the default service account token.
func cleanVolumes(p *v1.Pod) {
	volumes := make([]v1.Volume, 0, len(p.Spec.Volumes))
	for _, v := range p.Spec.Volumes {
		if !strings.HasPrefix(v.Name, "default-token") {
			volumes = append(volumes, v)
		}
	}
	p.Spec.Volumes = volumes
	for i := range p.Spec.Containers {
		c := &p.Spec.Containers[i]
		volumeMounts := make([]v1.VolumeMount, 0, len(c.VolumeMounts))
		for _, vm := range c.VolumeMounts {
			if !strings.HasPrefix(vm.Name, "default-token") {
				volumeMounts = append(volumeMounts, vm)
			}
		}
		c.VolumeMounts = volumeMounts
	}
}

// writeManifest will write the manifest to the ignore path.
// It first writes the file to a temp file, and then atomically moves it into
// the actual ignore path and correct file name.
func writeManifest(manifest v1.Pod, name string) {
	m, err := json.Marshal(manifest)
	if err != nil {
		glog.Fatal(err)
	}
	writeAndAtomicCopy(m, checkpointManifest(name))
}

func createCheckpointPod(podList v1.PodList, n, ns string) v1.Pod {
	var checkpointPod v1.Pod
	for _, p := range podList.Items {
		if isPod(p, n, ns) {
			checkpointPod = p
			break
		}
	}
	// the pod we manifest we got from kubelet does not have TypeMeta.
	// Add it now.
	checkpointPod.TypeMeta = podAPIServerMeta
	cleanVolumes(&checkpointPod)
	stripNonessentialInfo(&checkpointPod)
	return checkpointPod
}

func newAPIClient() clientset.Interface {
	kubeConfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
		&clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath},
		&clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: secureAPIAddr}}).ClientConfig()
	if err != nil {
		glog.Fatal(err)
	}
	return clientset.NewForConfigOrDie(kubeConfig)
}

func convertSecretsToVolumeMounts(client clientset.Interface, pod *v1.Pod) {
	glog.Info("converting secrets to volume mounts")
	spec := pod.Spec
	for i := range spec.Volumes {
		v := &spec.Volumes[i]
		if v.Secret != nil {
			secretName := v.Secret.SecretName
			basePath := filepath.Join(secretsPath, pod.Name, v.Secret.SecretName)
			v.HostPath = &v1.HostPathVolumeSource{
				Path: basePath,
			}
			copySecretsToDisk(client, secretName, basePath)
			v.Secret = nil
		}
	}
}

func copySecretsToDisk(client clientset.Interface, secretName, basePath string) {
	glog.Info("copying secrets to disk")
	if err := os.MkdirAll(basePath, 0755); err != nil {
		glog.Fatal(err)
	}
	glog.Infof("created directory %s", basePath)
	s, err := client.Core().Secrets(api.NamespaceSystem).Get(secretName)
	if err != nil {
		glog.Fatal(err)
	}
	for name, value := range s.Data {
		path := filepath.Join(basePath, name)
		writeAndAtomicCopy(value, path)
	}
}

func writeAndAtomicCopy(data []byte, path string) {
	// First write a "temp" file.
	tmpfile := filepath.Join(filepath.Dir(path), "."+filepath.Base(path))
	if err := ioutil.WriteFile(tmpfile, data, 0644); err != nil {
		glog.Fatal(err)
	}
	// Finally, copy that file to the correct location.
	if err := os.Rename(tmpfile, path); err != nil {
		glog.Fatal(err)
	}
}

func activeManifest(name string) string {
	return filepath.Join(activePath, name+".json")
}

func checkpointManifest(name string) string {
	return filepath.Join(ignorePath, name+".json")
}
<commit_msg>checkpoint: explicitly load checkpoint manifests<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/golang/glog"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/api/v1"
	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/release_1_4"
	"k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
	clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api"
)

const (
	kubeletAPIPodsURL = "http://127.0.0.1:10255/pods"
	ignorePath        = "/srv/kubernetes/manifests"
	activePath        = "/etc/kubernetes/manifests"
	kubeconfigPath    = "/etc/kubernetes/kubeconfig"
	secretsPath       = "/etc/kubernetes/checkpoint-secrets"

	tempAPIServer = "temp-apiserver"
	kubeAPIServer = "kube-apiserver"
)

var podAPIServerMeta = unversioned.TypeMeta{
	APIVersion: "v1",
	Kind:       "Pod",
}

var (
	secureAPIAddr = fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS"))
)

func main() {
	checkpoints, err := getCheckpointManifests()
	if err != nil {
		glog.Fatalf("failed to load existing checkpoint manifests: %v", err)
	}
	glog.Info("begin pods checkpointing...")
	run(kubeAPIServer, tempAPIServer, api.NamespaceSystem, checkpoints)
}

func run(actualPodName, tempPodName, namespace string, checkpoints map[string]struct{}) {
	client := newAPIClient()
	for {
		_, checkpointed := checkpoints[checkpointManifest(tempPodName)]

		var podList v1.PodList
		if err := json.Unmarshal(getPodsFromKubeletAPI(), &podList); err != nil {
			glog.Fatal(err)
		}
		switch {
		case bothRunning(podList, actualPodName, tempPodName, namespace):
			glog.Infof("both temp %v and actual %v pods running, removing temp pod", actualPodName, tempPodName)
			// Both the temp and actual pods are running.
			// Remove the temp manifest from the config dir so that the
			// kubelet will stop it.
			if err := os.Remove(activeManifest(tempPodName)); err != nil {
				glog.Error(err)
			}
		case isPodRunning(podList, client, actualPodName, namespace):
			glog.Infof("actual pod %v found, creating checkpoint pod manifest", actualPodName)
			// The actual is running. Let's snapshot the pod,
			// clean it up a bit, and then save it to the ignore path for
			// later use.
			checkpointPod := createCheckpointPod(podList, actualPodName, namespace)
			convertSecretsToVolumeMounts(client, &checkpointPod)
			writeManifest(checkpointPod, tempPodName)
			checkpoints[checkpointManifest(tempPodName)] = struct{}{}
			glog.Infof("finished creating checkpoint pod %v manifest at %s\n", tempPodName, checkpointManifest(tempPodName))

		case checkpointed:
			glog.Info("no actual pod running, installing checkpoint pod static manifest")
			b, err := ioutil.ReadFile(checkpointManifest(tempPodName))
			if err != nil {
				glog.Error(err)
			} else {
				if err := ioutil.WriteFile(activeManifest(tempPodName), b, 0644); err != nil {
					glog.Error(err)
				}
			}
		}
		time.Sleep(1 * time.Second)
	}
}

func stripNonessentialInfo(p *v1.Pod) {
	p.Spec.ServiceAccountName = ""
	p.Spec.DeprecatedServiceAccount = ""
	p.Status.Reset()
}

func getPodsFromKubeletAPI() []byte {
	var pods []byte
	res, err := http.Get(kubeletAPIPodsURL)
	if err != nil {
		glog.Error(err)
		return pods
	}
	pods, err = ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		glog.Error(err)
	}
	return pods
}

func bothRunning(pods v1.PodList, an, tn, ns string) bool {
	var actualPodSeen, tempPodSeen bool
	for _, p := range pods.Items {
		actualPodSeen = actualPodSeen || isPod(p, an, ns)
		tempPodSeen = tempPodSeen || isPod(p, tn, ns)
		if actualPodSeen && tempPodSeen {
			return true
		}
	}
	return false
}

func isPodRunning(pods v1.PodList, client clientset.Interface, n, ns string) bool {
	for _, p := range pods.Items {
		if isPod(p, n, ns) {
			if n == kubeAPIServer {
				// Make sure it's actually running. Sometimes we get that
				// pod manifest back, but the server is not actually running.
				_, err := client.Discovery().ServerVersion()
				return err == nil
			}
			return true
		}
	}
	return false
}

func isPod(pod v1.Pod, n, ns string) bool {
	return strings.Contains(pod.Name, n) && pod.Namespace == ns
}

// cleanVolumes will sanitize the list of volumes and volume mounts
// to remove the default service account token.
func cleanVolumes(p *v1.Pod) {
	volumes := make([]v1.Volume, 0, len(p.Spec.Volumes))
	for _, v := range p.Spec.Volumes {
		if !strings.HasPrefix(v.Name, "default-token") {
			volumes = append(volumes, v)
		}
	}
	p.Spec.Volumes = volumes
	for i := range p.Spec.Containers {
		c := &p.Spec.Containers[i]
		volumeMounts := make([]v1.VolumeMount, 0, len(c.VolumeMounts))
		for _, vm := range c.VolumeMounts {
			if !strings.HasPrefix(vm.Name, "default-token") {
				volumeMounts = append(volumeMounts, vm)
			}
		}
		c.VolumeMounts = volumeMounts
	}
}

// writeManifest will write the manifest to the ignore path.
// It first writes the file to a temp file, and then atomically moves it into
// the actual ignore path and correct file name.
func writeManifest(manifest v1.Pod, name string) {
	m, err := json.Marshal(manifest)
	if err != nil {
		glog.Fatal(err)
	}
	writeAndAtomicCopy(m, checkpointManifest(name))
}

func createCheckpointPod(podList v1.PodList, n, ns string) v1.Pod {
	var checkpointPod v1.Pod
	for _, p := range podList.Items {
		if isPod(p, n, ns) {
			checkpointPod = p
			break
		}
	}
	// the pod we manifest we got from kubelet does not have TypeMeta.
	// Add it now.
	checkpointPod.TypeMeta = podAPIServerMeta
	cleanVolumes(&checkpointPod)
	stripNonessentialInfo(&checkpointPod)
	return checkpointPod
}

func newAPIClient() clientset.Interface {
	kubeConfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
		&clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath},
		&clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: secureAPIAddr}}).ClientConfig()
	if err != nil {
		glog.Fatal(err)
	}
	return clientset.NewForConfigOrDie(kubeConfig)
}

func convertSecretsToVolumeMounts(client clientset.Interface, pod *v1.Pod) {
	glog.Info("converting secrets to volume mounts")
	spec := pod.Spec
	for i := range spec.Volumes {
		v := &spec.Volumes[i]
		if v.Secret != nil {
			secretName := v.Secret.SecretName
			basePath := filepath.Join(secretsPath, pod.Name, v.Secret.SecretName)
			v.HostPath = &v1.HostPathVolumeSource{
				Path: basePath,
			}
			copySecretsToDisk(client, secretName, basePath)
			v.Secret = nil
		}
	}
}

func copySecretsToDisk(client clientset.Interface, secretName, basePath string) {
	glog.Info("copying secrets to disk")
	if err := os.MkdirAll(basePath, 0755); err != nil {
		glog.Fatal(err)
	}
	glog.Infof("created directory %s", basePath)
	s, err := client.Core().Secrets(api.NamespaceSystem).Get(secretName)
	if err != nil {
		glog.Fatal(err)
	}
	for name, value := range s.Data {
		path := filepath.Join(basePath, name)
		writeAndAtomicCopy(value, path)
	}
}

func writeAndAtomicCopy(data []byte, path string) {
	// First write a "temp" file.
	tmpfile := filepath.Join(filepath.Dir(path), "."+filepath.Base(path))
	if err := ioutil.WriteFile(tmpfile, data, 0644); err != nil {
		glog.Fatal(err)
	}
	// Finally, copy that file to the correct location.
	if err := os.Rename(tmpfile, path); err != nil {
		glog.Fatal(err)
	}
}

func activeManifest(name string) string {
	return filepath.Join(activePath, name+".json")
}

func checkpointManifest(name string) string {
	return filepath.Join(ignorePath, name+".json")
}

func getCheckpointManifests() (map[string]struct{}, error) {
	checkpoints := make(map[string]struct{})

	fs, err := ioutil.ReadDir(ignorePath)
	if err != nil {
		if os.IsNotExist(err) {
			return checkpoints, nil
		}
		return nil, err
	}
	for _, f := range fs {
		glog.Infof("found checkpoint pod manifests %v", f.Name())
		checkpoints[path.Join(ignorePath, f.Name())] = struct{}{}
	}
	return checkpoints, nil
}
<|endoftext|>


<commit_before>/*
Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"fmt"
	"io"

	"github.com/spf13/cobra"
)

const completionDesc = `
Generate autocompletions script for Helm for the specified shell (bash or zsh).

This command can generate shell autocompletions. e.g.

	$ helm completion bash

Can be sourced as such

	$ source <(helm completion bash)
`

var (
	completionShells = map[string]func(out io.Writer, cmd *cobra.Command) error{
		"bash": runCompletionBash,
		"zsh":  runCompletionZsh,
	}
)

func newCompletionCmd(out io.Writer) *cobra.Command {
	shells := []string{}
	for s := range completionShells {
		shells = append(shells, s)
	}

	cmd := &cobra.Command{
		Use:   "completion SHELL",
		Short: "Generate autocompletions script for the specified shell (bash or zsh)",
		Long:  completionDesc,
		RunE: func(cmd *cobra.Command, args []string) error {
			return runCompletion(out, cmd, args)
		},
		ValidArgs: shells,
	}

	return cmd
}

func runCompletion(out io.Writer, cmd *cobra.Command, args []string) error {
	if len(args) == 0 {
		return fmt.Errorf("shell not specified")
	}
	if len(args) > 1 {
		return fmt.Errorf("too many arguments, expected only the shell type")
	}
	run, found := completionShells[args[0]]
	if !found {
		return fmt.Errorf("unsupported shell type %q", args[0])
	}

	return run(out, cmd)
}

func runCompletionBash(out io.Writer, cmd *cobra.Command) error {
	return cmd.Root().GenBashCompletion(out)
}

func runCompletionZsh(out io.Writer, cmd *cobra.Command) error {
	zshInitialization := `#compdef helm

__helm_bash_source() {
	alias shopt=':'
	alias _expand=_bash_expand
	alias _complete=_bash_comp
	emulate -L sh
	setopt kshglob noshglob braceexpand
	source "$@"
}
__helm_type() {
	# -t is not supported by zsh
	if [ "$1" == "-t" ]; then
		shift
		# fake Bash 4 to disable "complete -o nospace". Instead
		# "compopt +-o nospace" is used in the code to toggle trailing
		# spaces. We don't support that, but leave trailing spaces on
		# all the time
		if [ "$1" = "__helm_compopt" ]; then
			echo builtin
			return 0
		fi
	fi
	type "$@"
}
__helm_compgen() {
	local completions w
	completions=( $(compgen "$@") ) || return $?
	# filter by given word as prefix
	while [[ "$1" = -* && "$1" != -- ]]; do
		shift
		shift
	done
	if [[ "$1" == -- ]]; then
		shift
	fi
	for w in "${completions[@]}"; do
		if [[ "${w}" = "$1"* ]]; then
			echo "${w}"
		fi
	done
}
__helm_compopt() {
	true # don't do anything. Not supported by bashcompinit in zsh
}
__helm_declare() {
	if [ "$1" == "-F" ]; then
		whence -w "$@"
	else
		builtin declare "$@"
	fi
}
__helm_ltrim_colon_completions()
{
	if [[ "$1" == *:* && "$COMP_WORDBREAKS" == *:* ]]; then
		# Remove colon-word prefix from COMPREPLY items
		local colon_word=${1%${1##*:}}
		local i=${#COMPREPLY[*]}
		while [[ $((--i)) -ge 0 ]]; do
			COMPREPLY[$i]=${COMPREPLY[$i]#"$colon_word"}
		done
	fi
}
__helm_get_comp_words_by_ref() {
	cur="${COMP_WORDS[COMP_CWORD]}"
	prev="${COMP_WORDS[${COMP_CWORD}-1]}"
	words=("${COMP_WORDS[@]}")
	cword=("${COMP_CWORD[@]}")
}
__helm_filedir() {
	local RET OLD_IFS w qw
	__debug "_filedir $@ cur=$cur"
	if [[ "$1" = \~* ]]; then
		# somehow does not work. Maybe, zsh does not call this at all
		eval echo "$1"
		return 0
	fi
	OLD_IFS="$IFS"
	IFS=$'\n'
	if [ "$1" = "-d" ]; then
		shift
		RET=( $(compgen -d) )
	else
		RET=( $(compgen -f) )
	fi
	IFS="$OLD_IFS"
	IFS="," __debug "RET=${RET[@]} len=${#RET[@]}"
	for w in ${RET[@]}; do
		if [[ ! "${w}" = "${cur}"* ]]; then
			continue
		fi
		if eval "[[ \"\${w}\" = *.$1 || -d \"\${w}\" ]]"; then
			qw="$(__helm_quote "${w}")"
			if [ -d "${w}" ]; then
				COMPREPLY+=("${qw}/")
			else
				COMPREPLY+=("${qw}")
			fi
		fi
	done
}
__helm_quote() {
	if [[ $1 == \'* || $1 == \"* ]]; then
		# Leave out first character
		printf %q "${1:1}"
	else
		printf %q "$1"
	fi
}
autoload -U +X bashcompinit && bashcompinit
# use word boundary patterns for BSD or GNU sed
LWORD='[[:<:]]'
RWORD='[[:>:]]'
if sed --help 2>&1 | grep -q GNU; then
	LWORD='\<'
	RWORD='\>'
fi
__helm_convert_bash_to_zsh() {
	sed \
	-e 's/declare -F/whence -w/' \
	-e 's/_get_comp_words_by_ref "\$@"/_get_comp_words_by_ref "\$*"/' \
	-e 's/local \([a-zA-Z0-9_]*\)=/local \1; \1=/' \
	-e 's/flags+=("\(--.*\)=")/flags+=("\1"); two_word_flags+=("\1")/' \
	-e 's/must_have_one_flag+=("\(--.*\)=")/must_have_one_flag+=("\1")/' \
	-e "s/${LWORD}_filedir${RWORD}/__helm_filedir/g" \
	-e "s/${LWORD}_get_comp_words_by_ref${RWORD}/__helm_get_comp_words_by_ref/g" \
	-e "s/${LWORD}__ltrim_colon_completions${RWORD}/__helm_ltrim_colon_completions/g" \
	-e "s/${LWORD}compgen${RWORD}/__helm_compgen/g" \
	-e "s/${LWORD}compopt${RWORD}/__helm_compopt/g" \
	-e "s/${LWORD}declare${RWORD}/__helm_declare/g" \
	-e "s/\\\$(type${RWORD}/\$(__helm_type/g" \
	-e 's/aliashash\["\(.\{1,\}\)"\]/aliashash[\1]/g' \
	-e 's/flaghash\[${flagname/flaghash[${flagname%=/' \
	-e 's/FUNCNAME/funcstack/g' \
	<<'BASH_COMPLETION_EOF'
`
	out.Write([]byte(zshInitialization))

	buf := new(bytes.Buffer)
	cmd.Root().GenBashCompletion(buf)
	out.Write(buf.Bytes())

	zshTail := `
BASH_COMPLETION_EOF
}
__helm_bash_source <(__helm_convert_bash_to_zsh)
`
	out.Write([]byte(zshTail))
	return nil
}
<commit_msg>Revert "Cannot have an = sign as an index"<commit_after>/*
Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"fmt"
	"io"

	"github.com/spf13/cobra"
)

const completionDesc = `
Generate autocompletions script for Helm for the specified shell (bash or zsh).

This command can generate shell autocompletions. e.g.

	$ helm completion bash

Can be sourced as such

	$ source <(helm completion bash)
`

var (
	completionShells = map[string]func(out io.Writer, cmd *cobra.Command) error{
		"bash": runCompletionBash,
		"zsh":  runCompletionZsh,
	}
)

func newCompletionCmd(out io.Writer) *cobra.Command {
	shells := []string{}
	for s := range completionShells {
		shells = append(shells, s)
	}

	cmd := &cobra.Command{
		Use:   "completion SHELL",
		Short: "Generate autocompletions script for the specified shell (bash or zsh)",
		Long:  completionDesc,
		RunE: func(cmd *cobra.Command, args []string) error {
			return runCompletion(out, cmd, args)
		},
		ValidArgs: shells,
	}

	return cmd
}

func runCompletion(out io.Writer, cmd *cobra.Command, args []string) error {
	if len(args) == 0 {
		return fmt.Errorf("shell not specified")
	}
	if len(args) > 1 {
		return fmt.Errorf("too many arguments, expected only the shell type")
	}
	run, found := completionShells[args[0]]
	if !found {
		return fmt.Errorf("unsupported shell type %q", args[0])
	}

	return run(out, cmd)
}

func runCompletionBash(out io.Writer, cmd *cobra.Command) error {
	return cmd.Root().GenBashCompletion(out)
}

func runCompletionZsh(out io.Writer, cmd *cobra.Command) error {
	zshInitialization := `#compdef helm

__helm_bash_source() {
	alias shopt=':'
	alias _expand=_bash_expand
	alias _complete=_bash_comp
	emulate -L sh
	setopt kshglob noshglob braceexpand
	source "$@"
}
__helm_type() {
	# -t is not supported by zsh
	if [ "$1" == "-t" ]; then
		shift
		# fake Bash 4 to disable "complete -o nospace". Instead
		# "compopt +-o nospace" is used in the code to toggle trailing
		# spaces. We don't support that, but leave trailing spaces on
		# all the time
		if [ "$1" = "__helm_compopt" ]; then
			echo builtin
			return 0
		fi
	fi
	type "$@"
}
__helm_compgen() {
	local completions w
	completions=( $(compgen "$@") ) || return $?
	# filter by given word as prefix
	while [[ "$1" = -* && "$1" != -- ]]; do
		shift
		shift
	done
	if [[ "$1" == -- ]]; then
		shift
	fi
	for w in "${completions[@]}"; do
		if [[ "${w}" = "$1"* ]]; then
			echo "${w}"
		fi
	done
}
__helm_compopt() {
	true # don't do anything. Not supported by bashcompinit in zsh
}
__helm_declare() {
	if [ "$1" == "-F" ]; then
		whence -w "$@"
	else
		builtin declare "$@"
	fi
}
__helm_ltrim_colon_completions()
{
	if [[ "$1" == *:* && "$COMP_WORDBREAKS" == *:* ]]; then
		# Remove colon-word prefix from COMPREPLY items
		local colon_word=${1%${1##*:}}
		local i=${#COMPREPLY[*]}
		while [[ $((--i)) -ge 0 ]]; do
			COMPREPLY[$i]=${COMPREPLY[$i]#"$colon_word"}
		done
	fi
}
__helm_get_comp_words_by_ref() {
	cur="${COMP_WORDS[COMP_CWORD]}"
	prev="${COMP_WORDS[${COMP_CWORD}-1]}"
	words=("${COMP_WORDS[@]}")
	cword=("${COMP_CWORD[@]}")
}
__helm_filedir() {
	local RET OLD_IFS w qw
	__debug "_filedir $@ cur=$cur"
	if [[ "$1" = \~* ]]; then
		# somehow does not work. Maybe, zsh does not call this at all
		eval echo "$1"
		return 0
	fi
	OLD_IFS="$IFS"
	IFS=$'\n'
	if [ "$1" = "-d" ]; then
		shift
		RET=( $(compgen -d) )
	else
		RET=( $(compgen -f) )
	fi
	IFS="$OLD_IFS"
	IFS="," __debug "RET=${RET[@]} len=${#RET[@]}"
	for w in ${RET[@]}; do
		if [[ ! "${w}" = "${cur}"* ]]; then
			continue
		fi
		if eval "[[ \"\${w}\" = *.$1 || -d \"\${w}\" ]]"; then
			qw="$(__helm_quote "${w}")"
			if [ -d "${w}" ]; then
				COMPREPLY+=("${qw}/")
			else
				COMPREPLY+=("${qw}")
			fi
		fi
	done
}
__helm_quote() {
	if [[ $1 == \'* || $1 == \"* ]]; then
		# Leave out first character
		printf %q "${1:1}"
	else
		printf %q "$1"
	fi
}
autoload -U +X bashcompinit && bashcompinit
# use word boundary patterns for BSD or GNU sed
LWORD='[[:<:]]'
RWORD='[[:>:]]'
if sed --help 2>&1 | grep -q GNU; then
	LWORD='\<'
	RWORD='\>'
fi
__helm_convert_bash_to_zsh() {
	sed \
	-e 's/declare -F/whence -w/' \
	-e 's/_get_comp_words_by_ref "\$@"/_get_comp_words_by_ref "\$*"/' \
	-e 's/local \([a-zA-Z0-9_]*\)=/local \1; \1=/' \
	-e 's/flags+=("\(--.*\)=")/flags+=("\1"); two_word_flags+=("\1")/' \
	-e 's/must_have_one_flag+=("\(--.*\)=")/must_have_one_flag+=("\1")/' \
	-e "s/${LWORD}_filedir${RWORD}/__helm_filedir/g" \
	-e "s/${LWORD}_get_comp_words_by_ref${RWORD}/__helm_get_comp_words_by_ref/g" \
	-e "s/${LWORD}__ltrim_colon_completions${RWORD}/__helm_ltrim_colon_completions/g" \
	-e "s/${LWORD}compgen${RWORD}/__helm_compgen/g" \
	-e "s/${LWORD}compopt${RWORD}/__helm_compopt/g" \
	-e "s/${LWORD}declare${RWORD}/__helm_declare/g" \
	-e "s/\\\$(type${RWORD}/\$(__helm_type/g" \
	-e 's/aliashash\["\(.\{1,\}\)"\]/aliashash[\1]/g' \
	-e 's/FUNCNAME/funcstack/g' \
	<<'BASH_COMPLETION_EOF'
`
	out.Write([]byte(zshInitialization))

	buf := new(bytes.Buffer)
	cmd.Root().GenBashCompletion(buf)
	out.Write(buf.Bytes())

	zshTail := `
BASH_COMPLETION_EOF
}
__helm_bash_source <(__helm_convert_bash_to_zsh)
`
	out.Write([]byte(zshTail))
	return nil
}
<|endoftext|>


<commit_before>package arm

import (
	"context"
	"fmt"
	"time"

	"github.com/hashicorp/packer/builder/azure/common/constants"
	"github.com/hashicorp/packer/common/retry"
	"github.com/hashicorp/packer/helper/multistep"
	"github.com/hashicorp/packer/packer"
)

const (
	maxResourcesToDelete = 50
)

type StepDeleteResourceGroup struct {
	client *AzureClient
	delete func(ctx context.Context, state multistep.StateBag, resourceGroupName string) error
	say    func(message string)
	error  func(e error)
}

func NewStepDeleteResourceGroup(client *AzureClient, ui packer.Ui) *StepDeleteResourceGroup {
	var step = &StepDeleteResourceGroup{
		client: client,
		say:    func(message string) { ui.Say(message) },
		error:  func(e error) { ui.Error(e.Error()) },
	}

	step.delete = step.deleteResourceGroup
	return step
}

func (s *StepDeleteResourceGroup) deleteResourceGroup(ctx context.Context, state multistep.StateBag, resourceGroupName string) error {
	var err error
	if state.Get(constants.ArmIsExistingResourceGroup).(bool) {
		s.say("\nThe resource group was not created by Packer, only deleting individual resources ...")
		var deploymentName = state.Get(constants.ArmDeploymentName).(string)
		err = s.deleteDeploymentResources(ctx, deploymentName, resourceGroupName)
		if err != nil {
			return err
		}

		if keyVaultDeploymentName, ok := state.GetOk(constants.ArmKeyVaultDeploymentName); ok {
			err = s.deleteDeploymentResources(ctx, keyVaultDeploymentName.(string), resourceGroupName)
			if err != nil {
				return err
			}
		}

		return nil
	} else {
		s.say("\nThe resource group was created by Packer, deleting ...")
		f, err := s.client.GroupsClient.Delete(ctx, resourceGroupName)
		if err == nil {
			if state.Get(constants.ArmAsyncResourceGroupDelete).(bool) {
				// No need to wait for the completion for delete if request is Accepted
				s.say(fmt.Sprintf("\nResource Group is being deleted, not waiting for deletion due to config. Resource Group Name '%s'", resourceGroupName))
			} else {
				f.WaitForCompletionRef(ctx, s.client.GroupsClient.Client)
			}

		}

		if err != nil {
			s.say(s.client.LastError.Error())
		}
		return err
	}
}

func (s *StepDeleteResourceGroup) deleteDeploymentResources(ctx context.Context, deploymentName, resourceGroupName string) error {
	maxResources := int32(maxResourcesToDelete)

	deploymentOperations, err := s.client.DeploymentOperationsClient.ListComplete(ctx, resourceGroupName, deploymentName, &maxResources)
	if err != nil {
		s.reportIfError(err, resourceGroupName)
		return err
	}

	for deploymentOperations.NotDone() {
		deploymentOperation := deploymentOperations.Value()
		// Sometimes an empty operation is added to the list by Azure
		if deploymentOperation.Properties.TargetResource == nil {
			deploymentOperations.Next()
			continue
		}

		resourceName := *deploymentOperation.Properties.TargetResource.ResourceName
		resourceType := *deploymentOperation.Properties.TargetResource.ResourceType

		s.say(fmt.Sprintf(" -> %s : '%s'",
			resourceType,
			resourceName))

		err := retry.Config{
			Tries:      10,
			RetryDelay: (&retry.Backoff{InitialBackoff: 10 * time.Second, MaxBackoff: 600 * time.Second, Multiplier: 2}).Linear,
		}.Run(ctx, func(ctx context.Context) error {
			err := deleteResource(ctx, s.client,
				resourceType,
				resourceName,
				resourceGroupName)
			if err != nil {
				s.reportIfError(err, resourceName)
			}
			return err
		})

		if err = deploymentOperations.Next(); err != nil {
			return err
		}
	}

	return nil
}

func (s *StepDeleteResourceGroup) reportIfError(err error, resourceName string) {
	if err != nil {
		s.say(fmt.Sprintf("Error deleting resource. Please delete manually.\n\n"+
			"Name: %s\n"+
			"Error: %s", resourceName, err.Error()))
		s.error(err)
	}
}

func (s *StepDeleteResourceGroup) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
	s.say("Deleting resource group ...")

	var resourceGroupName = state.Get(constants.ArmResourceGroupName).(string)
	s.say(fmt.Sprintf(" -> ResourceGroupName : '%s'", resourceGroupName))

	err := s.delete(ctx, state, resourceGroupName)
	if err != nil {
		state.Put(constants.Error, err)
		s.error(err)

		return multistep.ActionHalt
	}

	state.Put(constants.ArmIsResourceGroupCreated, false)

	return multistep.ActionContinue
}

func (*StepDeleteResourceGroup) Cleanup(multistep.StateBag) {
}
<commit_msg>Add error check<commit_after>package arm

import (
	"context"
	"fmt"
	"time"

	"github.com/hashicorp/packer/builder/azure/common/constants"
	"github.com/hashicorp/packer/common/retry"
	"github.com/hashicorp/packer/helper/multistep"
	"github.com/hashicorp/packer/packer"
)

const (
	maxResourcesToDelete = 50
)

type StepDeleteResourceGroup struct {
	client *AzureClient
	delete func(ctx context.Context, state multistep.StateBag, resourceGroupName string) error
	say    func(message string)
	error  func(e error)
}

func NewStepDeleteResourceGroup(client *AzureClient, ui packer.Ui) *StepDeleteResourceGroup {
	var step = &StepDeleteResourceGroup{
		client: client,
		say:    func(message string) { ui.Say(message) },
		error:  func(e error) { ui.Error(e.Error()) },
	}

	step.delete = step.deleteResourceGroup
	return step
}

func (s *StepDeleteResourceGroup) deleteResourceGroup(ctx context.Context, state multistep.StateBag, resourceGroupName string) error {
	var err error
	if state.Get(constants.ArmIsExistingResourceGroup).(bool) {
		s.say("\nThe resource group was not created by Packer, only deleting individual resources ...")
		var deploymentName = state.Get(constants.ArmDeploymentName).(string)
		err = s.deleteDeploymentResources(ctx, deploymentName, resourceGroupName)
		if err != nil {
			return err
		}

		if keyVaultDeploymentName, ok := state.GetOk(constants.ArmKeyVaultDeploymentName); ok {
			err = s.deleteDeploymentResources(ctx, keyVaultDeploymentName.(string), resourceGroupName)
			if err != nil {
				return err
			}
		}

		return nil
	} else {
		s.say("\nThe resource group was created by Packer, deleting ...")
		f, err := s.client.GroupsClient.Delete(ctx, resourceGroupName)
		if err == nil {
			if state.Get(constants.ArmAsyncResourceGroupDelete).(bool) {
				// No need to wait for the completion for delete if request is Accepted
				s.say(fmt.Sprintf("\nResource Group is being deleted, not waiting for deletion due to config. Resource Group Name '%s'", resourceGroupName))
			} else {
				f.WaitForCompletionRef(ctx, s.client.GroupsClient.Client)
			}

		}

		if err != nil {
			s.say(s.client.LastError.Error())
		}
		return err
	}
}

func (s *StepDeleteResourceGroup) deleteDeploymentResources(ctx context.Context, deploymentName, resourceGroupName string) error {
	maxResources := int32(maxResourcesToDelete)

	deploymentOperations, err := s.client.DeploymentOperationsClient.ListComplete(ctx, resourceGroupName, deploymentName, &maxResources)
	if err != nil {
		s.reportIfError(err, resourceGroupName)
		return err
	}

	for deploymentOperations.NotDone() {
		deploymentOperation := deploymentOperations.Value()
		// Sometimes an empty operation is added to the list by Azure
		if deploymentOperation.Properties.TargetResource == nil {
			deploymentOperations.Next()
			continue
		}

		resourceName := *deploymentOperation.Properties.TargetResource.ResourceName
		resourceType := *deploymentOperation.Properties.TargetResource.ResourceType

		s.say(fmt.Sprintf(" -> %s : '%s'",
			resourceType,
			resourceName))

		err := retry.Config{
			Tries:      10,
			RetryDelay: (&retry.Backoff{InitialBackoff: 10 * time.Second, MaxBackoff: 600 * time.Second, Multiplier: 2}).Linear,
		}.Run(ctx, func(ctx context.Context) error {
			err := deleteResource(ctx, s.client,
				resourceType,
				resourceName,
				resourceGroupName)
			if err != nil {
				s.reportIfError(err, resourceName)
			}
			return err
		})
		if err != nil {
			return err
		}

		if err = deploymentOperations.Next(); err != nil {
			return err
		}
	}

	return nil
}

func (s *StepDeleteResourceGroup) reportIfError(err error, resourceName string) {
	if err != nil {
		s.say(fmt.Sprintf("Error deleting resource. Please delete manually.\n\n"+
			"Name: %s\n"+
			"Error: %s", resourceName, err.Error()))
		s.error(err)
	}
}

func (s *StepDeleteResourceGroup) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
	s.say("Deleting resource group ...")

	var resourceGroupName = state.Get(constants.ArmResourceGroupName).(string)
	s.say(fmt.Sprintf(" -> ResourceGroupName : '%s'", resourceGroupName))

	err := s.delete(ctx, state, resourceGroupName)
	if err != nil {
		state.Put(constants.Error, err)
		s.error(err)

		return multistep.ActionHalt
	}

	state.Put(constants.ArmIsResourceGroupCreated, false)

	return multistep.ActionContinue
}

func (*StepDeleteResourceGroup) Cleanup(multistep.StateBag) {
}
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package main

import (
	"encoding/base64"
	"fmt"
	"time"
	"net"

	"launchpad.net/gnuflag"
	"launchpad.net/goyaml"

	"launchpad.net/juju-core/agent"
	"launchpad.net/juju-core/agent/mongo"
	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/constraints"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/config"
	"launchpad.net/juju-core/instance"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/state/api/params"
	"launchpad.net/juju-core/worker/peergrouper"
)

type BootstrapCommand struct {
	cmd.CommandBase
	AgentConf
	EnvConfig   map[string]interface{}
	Constraints constraints.Value
	Hardware    instance.HardwareCharacteristics
	InstanceId  string
}

// Info returns a decription of the command.
func (c *BootstrapCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "bootstrap-state",
		Purpose: "initialize juju state",
	}
}

func (c *BootstrapCommand) SetFlags(f *gnuflag.FlagSet) {
	c.AgentConf.AddFlags(f)
	yamlBase64Var(f, &c.EnvConfig, "env-config", "", "initial environment configuration (yaml, base64 encoded)")
	f.Var(constraints.ConstraintsValue{Target: &c.Constraints}, "constraints", "initial environment constraints (space-separated strings)")
	f.Var(&c.Hardware, "hardware", "hardware characteristics (space-separated strings)")
	f.StringVar(&c.InstanceId, "instance-id", "", "unique instance-id for bootstrap machine")
}

// Init initializes the command for running.
func (c *BootstrapCommand) Init(args []string) error {
	if len(c.EnvConfig) == 0 {
		return requiredError("env-config")
	}
	if c.InstanceId == "" {
		return requiredError("instance-id")
	}
	return c.AgentConf.CheckArgs(args)
}

// Run initializes state for an environment.
func (c *BootstrapCommand) Run(_ *cmd.Context) error {
	envCfg, err := config.New(config.NoDefaults, c.EnvConfig)
	if err != nil {
		return err
	}
	err = c.ReadConfig("machine-0")
	if err != nil {
		return err
	}
	agentConfig := c.CurrentConfig()

	// agent.Jobs is an optional field in the agent config, and was
	// introduced after 1.17.2. We default to allowing units on
	// machine-0 if missing.
	jobs := agentConfig.Jobs()
	if len(jobs) == 0 {
		jobs = []params.MachineJob{
			params.JobManageEnviron,
			params.JobHostUnits,
		}
	}

	// Get the bootstrap machine's addresses from the provider.
	env, err := environs.New(envCfg)
	if err != nil {
		return err
	}
	instanceId := instance.Id(c.InstanceId)
	instances, err := env.Instances([]instance.Id{instanceId})
	if err != nil {
		return err
	}
	addrs, err := instances[0].Addresses()
	if err != nil {
		return err
	}

	// Create system-identity file
	if err := agent.WriteSystemIdentityFile(agentConfig); err != nil {
		return err
	}

	// Generate a shared secret for the Mongo replica set, and write it out.
	sharedSecret, err := mongo.GenerateSharedSecret()
	if err != nil {
		return err
	}
	info, ok := agentConfig.StateServingInfo()
	if !ok {
		return fmt.Errorf("bootstrap machine config has no state serving info")
	}
	info.SharedSecret = sharedSecret
	err = c.ChangeConfig(func(agentConfig agent.ConfigSetter) {
		agentConfig.SetStateServingInfo(info)
	})
	if err != nil {
		return fmt.Errorf("cannot write agent config: %v", err)
	}
	agentConfig = c.CurrentConfig()

	if err := c.startMongo(addrs, agentConfig); err != nil {
		return err
	}

	logger.Infof("started mongo")
	// Initialise state, and store any agent config (e.g. password) changes.
	var st *state.State
	var m *state.Machine
	err = nil
	writeErr := c.ChangeConfig(func(agentConfig agent.ConfigSetter) {
		st, m, err = agent.InitializeState(
			agentConfig,
			envCfg,
			agent.BootstrapMachineConfig{
				Addresses:       addrs,
				Constraints:     c.Constraints,
				Jobs:            jobs,
				InstanceId:      instanceId,
				Characteristics: c.Hardware,
				SharedSecret:    sharedSecret,
			},
			state.DefaultDialOpts(),
			environs.NewStatePolicy(),
		)
	})
	if writeErr != nil {
		return fmt.Errorf("cannot write initial configuration: %v", err)
	}
	if err != nil {
		return err
	}
	defer st.Close()

	// bootstrap machine always gets the vote
	return m.SetHasVote(true)
}

func (c *BootstrapCommand) startMongo(addrs []instance.Address, agentConfig agent.Config) error {
	logger.Debugf("starting mongo")

	info, ok := agentConfig.StateInfo()
	if !ok {
		return fmt.Errorf("no state info available")
	}
	// When bootstrapping, we need to allow enough time for mongo
	// to start as there's no retry loop in place.
	// 5 minutes should suffice.
	bootstrapDialOpts := DialOpts{Timeout:  5 * time.Minute}
	dialInfo, err := state.DialInfo(info, bootstrapDialOpts)
	if err != nil {
		return err
	}
	servingInfo, ok := agentConfig.StateServingInfo()
	if !ok {
		return fmt.Errorf("agent config has no state serving info")
	}
	// Use localhost to dial the mongo server, because it's running in
	// auth mode and will refuse to perform any operations unless
	// we dial that address.
	dialInfo.Addrs = []string{
		net.JoinHostPort("127.0.0.1", fmt.Sprint(servingInfo.StatePort)),
	}

	logger.Debugf("calling ensureMongoServer")
	withHA := shouldEnableHA(agentConfig)
	err = ensureMongoServer(
		agentConfig.DataDir(),
		agentConfig.Value(agent.Namespace),
		servingInfo,
		withHA,
	)
	if err != nil {
		return err
	}
	// If we are not doing HA, there is no need to set up replica set.
	if !withHA {
		return nil
	}

	peerAddr := mongo.SelectPeerAddress(addrs)
	if peerAddr == "" {
		return fmt.Errorf("no appropriate peer address found in %q", addrs)
	}
	peerHostPort := net.JoinHostPort(peerAddr, fmt.Sprint(servingInfo.StatePort))

	return maybeInitiateMongoServer(peergrouper.InitiateMongoParams{
		DialInfo:       dialInfo,
		MemberHostPort: peerHostPort,
	})
}

// yamlBase64Value implements gnuflag.Value on a map[string]interface{}.
type yamlBase64Value map[string]interface{}

// Set decodes the base64 value into yaml then expands that into a map.
func (v *yamlBase64Value) Set(value string) error {
	decoded, err := base64.StdEncoding.DecodeString(value)
	if err != nil {
		return err
	}
	return goyaml.Unmarshal(decoded, v)
}

func (v *yamlBase64Value) String() string {
	return fmt.Sprintf("%v", *v)
}

// yamlBase64Var sets up a gnuflag flag analogous to the FlagSet.*Var methods.
func yamlBase64Var(fs *gnuflag.FlagSet, target *map[string]interface{}, name string, value string, usage string) {
	fs.Var((*yamlBase64Value)(target), name, usage)
}
<commit_msg>go gmt<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package main

import (
	"encoding/base64"
	"fmt"
	"net"
	"time"

	"launchpad.net/gnuflag"
	"launchpad.net/goyaml"

	"launchpad.net/juju-core/agent"
	"launchpad.net/juju-core/agent/mongo"
	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/constraints"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/config"
	"launchpad.net/juju-core/instance"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/state/api/params"
	"launchpad.net/juju-core/worker/peergrouper"
)

type BootstrapCommand struct {
	cmd.CommandBase
	AgentConf
	EnvConfig   map[string]interface{}
	Constraints constraints.Value
	Hardware    instance.HardwareCharacteristics
	InstanceId  string
}

// Info returns a decription of the command.
func (c *BootstrapCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "bootstrap-state",
		Purpose: "initialize juju state",
	}
}

func (c *BootstrapCommand) SetFlags(f *gnuflag.FlagSet) {
	c.AgentConf.AddFlags(f)
	yamlBase64Var(f, &c.EnvConfig, "env-config", "", "initial environment configuration (yaml, base64 encoded)")
	f.Var(constraints.ConstraintsValue{Target: &c.Constraints}, "constraints", "initial environment constraints (space-separated strings)")
	f.Var(&c.Hardware, "hardware", "hardware characteristics (space-separated strings)")
	f.StringVar(&c.InstanceId, "instance-id", "", "unique instance-id for bootstrap machine")
}

// Init initializes the command for running.
func (c *BootstrapCommand) Init(args []string) error {
	if len(c.EnvConfig) == 0 {
		return requiredError("env-config")
	}
	if c.InstanceId == "" {
		return requiredError("instance-id")
	}
	return c.AgentConf.CheckArgs(args)
}

// Run initializes state for an environment.
func (c *BootstrapCommand) Run(_ *cmd.Context) error {
	envCfg, err := config.New(config.NoDefaults, c.EnvConfig)
	if err != nil {
		return err
	}
	err = c.ReadConfig("machine-0")
	if err != nil {
		return err
	}
	agentConfig := c.CurrentConfig()

	// agent.Jobs is an optional field in the agent config, and was
	// introduced after 1.17.2. We default to allowing units on
	// machine-0 if missing.
	jobs := agentConfig.Jobs()
	if len(jobs) == 0 {
		jobs = []params.MachineJob{
			params.JobManageEnviron,
			params.JobHostUnits,
		}
	}

	// Get the bootstrap machine's addresses from the provider.
	env, err := environs.New(envCfg)
	if err != nil {
		return err
	}
	instanceId := instance.Id(c.InstanceId)
	instances, err := env.Instances([]instance.Id{instanceId})
	if err != nil {
		return err
	}
	addrs, err := instances[0].Addresses()
	if err != nil {
		return err
	}

	// Create system-identity file
	if err := agent.WriteSystemIdentityFile(agentConfig); err != nil {
		return err
	}

	// Generate a shared secret for the Mongo replica set, and write it out.
	sharedSecret, err := mongo.GenerateSharedSecret()
	if err != nil {
		return err
	}
	info, ok := agentConfig.StateServingInfo()
	if !ok {
		return fmt.Errorf("bootstrap machine config has no state serving info")
	}
	info.SharedSecret = sharedSecret
	err = c.ChangeConfig(func(agentConfig agent.ConfigSetter) {
		agentConfig.SetStateServingInfo(info)
	})
	if err != nil {
		return fmt.Errorf("cannot write agent config: %v", err)
	}
	agentConfig = c.CurrentConfig()

	if err := c.startMongo(addrs, agentConfig); err != nil {
		return err
	}

	logger.Infof("started mongo")
	// Initialise state, and store any agent config (e.g. password) changes.
	var st *state.State
	var m *state.Machine
	err = nil
	writeErr := c.ChangeConfig(func(agentConfig agent.ConfigSetter) {
		st, m, err = agent.InitializeState(
			agentConfig,
			envCfg,
			agent.BootstrapMachineConfig{
				Addresses:       addrs,
				Constraints:     c.Constraints,
				Jobs:            jobs,
				InstanceId:      instanceId,
				Characteristics: c.Hardware,
				SharedSecret:    sharedSecret,
			},
			state.DefaultDialOpts(),
			environs.NewStatePolicy(),
		)
	})
	if writeErr != nil {
		return fmt.Errorf("cannot write initial configuration: %v", err)
	}
	if err != nil {
		return err
	}
	defer st.Close()

	// bootstrap machine always gets the vote
	return m.SetHasVote(true)
}

func (c *BootstrapCommand) startMongo(addrs []instance.Address, agentConfig agent.Config) error {
	logger.Debugf("starting mongo")

	info, ok := agentConfig.StateInfo()
	if !ok {
		return fmt.Errorf("no state info available")
	}
	// When bootstrapping, we need to allow enough time for mongo
	// to start as there's no retry loop in place.
	// 5 minutes should suffice.
	bootstrapDialOpts := DialOpts{Timeout: 5 * time.Minute}
	dialInfo, err := state.DialInfo(info, bootstrapDialOpts)
	if err != nil {
		return err
	}
	servingInfo, ok := agentConfig.StateServingInfo()
	if !ok {
		return fmt.Errorf("agent config has no state serving info")
	}
	// Use localhost to dial the mongo server, because it's running in
	// auth mode and will refuse to perform any operations unless
	// we dial that address.
	dialInfo.Addrs = []string{
		net.JoinHostPort("127.0.0.1", fmt.Sprint(servingInfo.StatePort)),
	}

	logger.Debugf("calling ensureMongoServer")
	withHA := shouldEnableHA(agentConfig)
	err = ensureMongoServer(
		agentConfig.DataDir(),
		agentConfig.Value(agent.Namespace),
		servingInfo,
		withHA,
	)
	if err != nil {
		return err
	}
	// If we are not doing HA, there is no need to set up replica set.
	if !withHA {
		return nil
	}

	peerAddr := mongo.SelectPeerAddress(addrs)
	if peerAddr == "" {
		return fmt.Errorf("no appropriate peer address found in %q", addrs)
	}
	peerHostPort := net.JoinHostPort(peerAddr, fmt.Sprint(servingInfo.StatePort))

	return maybeInitiateMongoServer(peergrouper.InitiateMongoParams{
		DialInfo:       dialInfo,
		MemberHostPort: peerHostPort,
	})
}

// yamlBase64Value implements gnuflag.Value on a map[string]interface{}.
type yamlBase64Value map[string]interface{}

// Set decodes the base64 value into yaml then expands that into a map.
func (v *yamlBase64Value) Set(value string) error {
	decoded, err := base64.StdEncoding.DecodeString(value)
	if err != nil {
		return err
	}
	return goyaml.Unmarshal(decoded, v)
}

func (v *yamlBase64Value) String() string {
	return fmt.Sprintf("%v", *v)
}

// yamlBase64Var sets up a gnuflag flag analogous to the FlagSet.*Var methods.
func yamlBase64Var(fs *gnuflag.FlagSet, target *map[string]interface{}, name string, value string, usage string) {
	fs.Var((*yamlBase64Value)(target), name, usage)
}
<|endoftext|>


<commit_before>package manager

import (
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/spf13/cobra"

	bivacCmd "github.com/camptocamp/bivac/cmd"
	"github.com/camptocamp/bivac/internal/manager"
	"github.com/camptocamp/bivac/pkg/volume"
)

var (
	server       manager.Server
	orchestrator string

	// Orchestrators is a copy of manager.Orchestrators which allows orchestrator
	// configuration from Cobra variables
	Orchestrators manager.Orchestrators

	dbPath           string
	resticForgetArgs string

	providersFile string
	targetURL     string
	retryCount    int
	logServer     string
	agentImage    string
)
var envs = make(map[string]string)

var managerCmd = &cobra.Command{
	Use:   "manager",
	Short: "Start Bivac backup manager",
	Run: func(cmd *cobra.Command, args []string) {
		// Global variables
		whitelistVolumes, _ := cmd.Flags().GetString("whitelist")
		blacklistVolumes, _ := cmd.Flags().GetString("blacklist")

		volumesFilters := volume.Filters{
			Whitelist: strings.Split(whitelistVolumes, ","),
			Blacklist: strings.Split(blacklistVolumes, ","),
		}

		o, err := manager.GetOrchestrator(orchestrator, Orchestrators)
		if err != nil {
			log.Errorf("failed to retrieve orchestrator: %s", err)
			return
		}

		err = manager.Start(bivacCmd.BuildInfo, o, server, volumesFilters, providersFile, targetURL, logServer, agentImage, retryCount)
		if err != nil {
			log.Errorf("failed to start manager: %s", err)
			return
		}
	},
}

func init() {
	managerCmd.Flags().StringVarP(&server.Address, "server.address", "", "0.0.0.0:8182", "Address to bind on.")
	envs["BIVAC_SERVER_ADDRESS"] = "server.address"
	managerCmd.Flags().StringVarP(&server.PSK, "server.psk", "", "", "Pre-shared key.")
	envs["BIVAC_SERVER_PSK"] = "server.psk"

	managerCmd.Flags().StringVarP(&orchestrator, "orchestrator", "o", "", "Orchestrator on which Bivac should connect to.")
	envs["BIVAC_ORCHESTRATOR"] = "orchestrator"

	managerCmd.Flags().StringVarP(&Orchestrators.Docker.Endpoint, "docker.endpoint", "", "unix:///var/run/docker.sock", "Docker endpoint.")
	envs["BIVAC_DOCKER_ENDPOINT"] = "docker.endpoint"

	managerCmd.Flags().StringVarP(&Orchestrators.Cattle.URL, "cattle.url", "", "", "The Cattle URL.")
	envs["CATTLE_URL"] = "cattle.url"
	managerCmd.Flags().StringVarP(&Orchestrators.Cattle.AccessKey, "cattle.accesskey", "", "", "The Cattle access key.")
	envs["CATTLE_ACCESS_KEY"] = "cattle.accesskey"
	managerCmd.Flags().StringVarP(&Orchestrators.Cattle.SecretKey, "cattle.secretkey", "", "", "The Cattle secret key.")
	envs["CATTLE_SECRET_KEY"] = "cattle.secretkey"

	managerCmd.Flags().StringVarP(&Orchestrators.Kubernetes.Namespace, "kubernetes.namespace", "", "", "Namespace where you want to run Bivac.")
	envs["KUBERNETES_NAMESPACE"] = "kubernetes.namespace"
	managerCmd.Flags().BoolVarP(&Orchestrators.Kubernetes.AllNamespaces, "kubernetes.all-namespaces", "", false, "Backup volumes of all namespaces.")
	envs["KUBERNETES_ALL_NAMESPACES"] = "kubernetes.all-namespaces"
	managerCmd.Flags().StringVarP(&Orchestrators.Kubernetes.KubeConfig, "kubernetes.kubeconfig", "", "", "Path to your kuberconfig file.")
	envs["KUBERNETES_KUBECONFIG"] = "kubernetes.kubeconfig"
	managerCmd.Flags().StringVarP(&Orchestrators.Kubernetes.AgentServiceAccount, "kubernetes.agent-service-account", "", "", "Specify service account for agents.")
	envs["KUBERNETES_AGENT_SERVICE_ACCOUNT"] = "kubernetes.agent-service-account"

	managerCmd.Flags().StringVarP(&resticForgetArgs, "restic.forget.args", "", "--keep-daily 15 --prune", "Restic forget arguments.")
	envs["RESTIC_FORGET_ARGS"] = "restic.forget.args"

	managerCmd.Flags().StringVarP(&providersFile, "providers.config", "", "/providers-config.default.toml", "Configuration file for providers.")
	envs["BIVAC_PROVIDERS_CONFIG"] = "providers.config"

	managerCmd.Flags().StringVarP(&targetURL, "target.url", "r", "", "The target URL to push the backups to.")
	envs["BIVAC_TARGET_URL"] = "target.url"

	managerCmd.Flags().IntVarP(&retryCount, "retry.count", "", 0, "Retry to backup the volume if something goes wrong with Bivac.")
	envs["BIVAC_RETRY_COUNT"] = "retry.count"

	managerCmd.Flags().StringVarP(&logServer, "log.server", "", "", "Manager's API address that will receive logs from agents.")
	envs["BIVAC_LOG_SERVER"] = "log.server"

	managerCmd.Flags().StringVarP(&agentImage, "agent.image", "", "camptocamp/bivac:2.0.0", "Agent's Docker image.")
	envs["BIVAC_AGENT_IMAGE"] = "agent.image"

	bivacCmd.SetValuesFromEnv(envs, managerCmd.Flags())
	bivacCmd.RootCmd.AddCommand(managerCmd)
}
<commit_msg>cmd/manager: forget group by host as default<commit_after>package manager

import (
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/spf13/cobra"

	bivacCmd "github.com/camptocamp/bivac/cmd"
	"github.com/camptocamp/bivac/internal/manager"
	"github.com/camptocamp/bivac/pkg/volume"
)

var (
	server       manager.Server
	orchestrator string

	// Orchestrators is a copy of manager.Orchestrators which allows orchestrator
	// configuration from Cobra variables
	Orchestrators manager.Orchestrators

	dbPath           string
	resticForgetArgs string

	providersFile string
	targetURL     string
	retryCount    int
	logServer     string
	agentImage    string
)
var envs = make(map[string]string)

var managerCmd = &cobra.Command{
	Use:   "manager",
	Short: "Start Bivac backup manager",
	Run: func(cmd *cobra.Command, args []string) {
		// Global variables
		whitelistVolumes, _ := cmd.Flags().GetString("whitelist")
		blacklistVolumes, _ := cmd.Flags().GetString("blacklist")

		volumesFilters := volume.Filters{
			Whitelist: strings.Split(whitelistVolumes, ","),
			Blacklist: strings.Split(blacklistVolumes, ","),
		}

		o, err := manager.GetOrchestrator(orchestrator, Orchestrators)
		if err != nil {
			log.Errorf("failed to retrieve orchestrator: %s", err)
			return
		}

		err = manager.Start(bivacCmd.BuildInfo, o, server, volumesFilters, providersFile, targetURL, logServer, agentImage, retryCount)
		if err != nil {
			log.Errorf("failed to start manager: %s", err)
			return
		}
	},
}

func init() {
	managerCmd.Flags().StringVarP(&server.Address, "server.address", "", "0.0.0.0:8182", "Address to bind on.")
	envs["BIVAC_SERVER_ADDRESS"] = "server.address"
	managerCmd.Flags().StringVarP(&server.PSK, "server.psk", "", "", "Pre-shared key.")
	envs["BIVAC_SERVER_PSK"] = "server.psk"

	managerCmd.Flags().StringVarP(&orchestrator, "orchestrator", "o", "", "Orchestrator on which Bivac should connect to.")
	envs["BIVAC_ORCHESTRATOR"] = "orchestrator"

	managerCmd.Flags().StringVarP(&Orchestrators.Docker.Endpoint, "docker.endpoint", "", "unix:///var/run/docker.sock", "Docker endpoint.")
	envs["BIVAC_DOCKER_ENDPOINT"] = "docker.endpoint"

	managerCmd.Flags().StringVarP(&Orchestrators.Cattle.URL, "cattle.url", "", "", "The Cattle URL.")
	envs["CATTLE_URL"] = "cattle.url"
	managerCmd.Flags().StringVarP(&Orchestrators.Cattle.AccessKey, "cattle.accesskey", "", "", "The Cattle access key.")
	envs["CATTLE_ACCESS_KEY"] = "cattle.accesskey"
	managerCmd.Flags().StringVarP(&Orchestrators.Cattle.SecretKey, "cattle.secretkey", "", "", "The Cattle secret key.")
	envs["CATTLE_SECRET_KEY"] = "cattle.secretkey"

	managerCmd.Flags().StringVarP(&Orchestrators.Kubernetes.Namespace, "kubernetes.namespace", "", "", "Namespace where you want to run Bivac.")
	envs["KUBERNETES_NAMESPACE"] = "kubernetes.namespace"
	managerCmd.Flags().BoolVarP(&Orchestrators.Kubernetes.AllNamespaces, "kubernetes.all-namespaces", "", false, "Backup volumes of all namespaces.")
	envs["KUBERNETES_ALL_NAMESPACES"] = "kubernetes.all-namespaces"
	managerCmd.Flags().StringVarP(&Orchestrators.Kubernetes.KubeConfig, "kubernetes.kubeconfig", "", "", "Path to your kuberconfig file.")
	envs["KUBERNETES_KUBECONFIG"] = "kubernetes.kubeconfig"
	managerCmd.Flags().StringVarP(&Orchestrators.Kubernetes.AgentServiceAccount, "kubernetes.agent-service-account", "", "", "Specify service account for agents.")
	envs["KUBERNETES_AGENT_SERVICE_ACCOUNT"] = "kubernetes.agent-service-account"

	managerCmd.Flags().StringVarP(&resticForgetArgs, "restic.forget.args", "", "--group-by host --keep-daily 15 --prune", "Restic forget arguments.")
	envs["RESTIC_FORGET_ARGS"] = "restic.forget.args"

	managerCmd.Flags().StringVarP(&providersFile, "providers.config", "", "/providers-config.default.toml", "Configuration file for providers.")
	envs["BIVAC_PROVIDERS_CONFIG"] = "providers.config"

	managerCmd.Flags().StringVarP(&targetURL, "target.url", "r", "", "The target URL to push the backups to.")
	envs["BIVAC_TARGET_URL"] = "target.url"

	managerCmd.Flags().IntVarP(&retryCount, "retry.count", "", 0, "Retry to backup the volume if something goes wrong with Bivac.")
	envs["BIVAC_RETRY_COUNT"] = "retry.count"

	managerCmd.Flags().StringVarP(&logServer, "log.server", "", "", "Manager's API address that will receive logs from agents.")
	envs["BIVAC_LOG_SERVER"] = "log.server"

	managerCmd.Flags().StringVarP(&agentImage, "agent.image", "", "camptocamp/bivac:2.0.0", "Agent's Docker image.")
	envs["BIVAC_AGENT_IMAGE"] = "agent.image"

	bivacCmd.SetValuesFromEnv(envs, managerCmd.Flags())
	bivacCmd.RootCmd.AddCommand(managerCmd)
}
<|endoftext|>


<commit_before>package mccli

import (
	"fmt"
	"os"
	"time"

	"github.com/codegangsta/cli"
	"github.com/materials-commons/gohandy/fs"
	"github.com/materials-commons/mcstore/cmd/pkg/mc"
)

var (
	WatchCommand = cli.Command{
		Name:    "watch",
		Aliases: []string{"w"},
		Usage:   "Watch commands",
		Subcommands: []cli.Command{
			watchProjectCommand,
		},
	}

	watchProjectCommand = cli.Command{
		Name:    "project",
		Aliases: []string{"proj", "p"},
		Usage:   "Watch a project for file changes",
		Action:  watchProjectCLI,
	}
)

type projectWatcher struct {
	*mc.ClientAPI
	projectName string
}

func watchProjectCLI(c *cli.Context) {
	if len(c.Args()) != 1 {
		fmt.Println("You must specify a project.")
		os.Exit(1)
	}

	projectName := c.Args()[0]

	if db, err := mc.ProjectOpener.OpenProjectDB(projectName); err != nil {
		fmt.Println("Unknown project:", projectName)
		os.Exit(1)
	} else {
		p := &projectWatcher{
			ClientAPI:   mc.NewClientAPI(),
			projectName: projectName,
		}
		path := db.Project().Path
		fmt.Printf("Watching project %s located at %s for changes...\n", projectName, path)
		p.watchProject(path, db)
	}
}

func (w *projectWatcher) watchProject(path string, db mc.ProjectDB) {
	for {
		watcher, err := fs.NewRecursiveWatcher(path)
		if err != nil {
			time.Sleep(time.Minute)
			continue
		}
		watcher.Start()

	FsEventsLoop:
		for {
			select {
			case event := <-watcher.Events:
				w.handleFileChangeEvent(event, db)
			case err := <-watcher.ErrorEvents:
				fmt.Println("file events error:", err)
				break FsEventsLoop
			}
		}
		watcher.Close()
	}
}

func (w *projectWatcher) handleFileChangeEvent(event fs.Event, db mc.ProjectDB) {
	switch {
	case event.IsCreate():
		w.handleCreate(event.Name)
	case event.IsDelete():
		// ignore
	case event.IsModify():
		w.handleModify(event.Name)
	case event.IsRename():
		// ignore
	case event.IsAttrib():
		// ignore
	default:
		// ignore
	}
}

func (w *projectWatcher) handleCreate(path string) {
	switch finfo, err := os.Stat(path); {
	case err != nil:
		fmt.Printf("Error stating %s: %s\n", path, err)
	case finfo.IsDir():
		w.dirCreate(path)
	case finfo.Mode().IsRegular():
		w.fileUpload(path)
	}
}

func (w *projectWatcher) dirCreate(path string) {
	if err := w.CreateDirectory(w.projectName, path); err != nil {
		fmt.Printf("Failed to create new directory %s: %s\n", path, err)
	} else {
		fmt.Println("Created new directory: ", path)
	}
}

func (w *projectWatcher) fileUpload(path string) {
	if err := w.UploadFile(w.projectName, path); err != nil {
		fmt.Printf("Failed to upload file %s: %s\n", path, err)
	}
}

func (w *projectWatcher) handleModify(path string) {
	if finfo, err := os.Stat(path); err != nil {
		fmt.Printf("Error getting file info for %s: %s\n", path, err)
	} else if finfo.Mode().IsRegular() {
		w.fileUpload(path)
	}
}
<commit_msg>Add in ignoring certain files.<commit_after>package mccli

import (
	"fmt"
	"os"
	"time"

	"github.com/codegangsta/cli"
	"github.com/materials-commons/gohandy/fs"
	"github.com/materials-commons/mcstore/cmd/pkg/mc"
	"github.com/materials-commons/mcstore/pkg/files"
)

var (
	WatchCommand = cli.Command{
		Name:    "watch",
		Aliases: []string{"w"},
		Usage:   "Watch commands",
		Subcommands: []cli.Command{
			watchProjectCommand,
		},
	}

	watchProjectCommand = cli.Command{
		Name:    "project",
		Aliases: []string{"proj", "p"},
		Usage:   "Watch a project for file changes",
		Action:  watchProjectCLI,
	}
)

type projectWatcher struct {
	*mc.ClientAPI
	projectName string
}

func watchProjectCLI(c *cli.Context) {
	if len(c.Args()) != 1 {
		fmt.Println("You must specify a project.")
		os.Exit(1)
	}

	projectName := c.Args()[0]

	if db, err := mc.ProjectOpener.OpenProjectDB(projectName); err != nil {
		fmt.Println("Unknown project:", projectName)
		os.Exit(1)
	} else {
		p := &projectWatcher{
			ClientAPI:   mc.NewClientAPI(),
			projectName: projectName,
		}
		path := db.Project().Path
		fmt.Printf("Watching project %s located at %s for changes...\n", projectName, path)
		p.watchProject(path, db)
	}
}

func (w *projectWatcher) watchProject(path string, db mc.ProjectDB) {
	for {
		watcher, err := fs.NewRecursiveWatcher(path)
		if err != nil {
			time.Sleep(time.Minute)
			continue
		}
		watcher.Start()

	FsEventsLoop:
		for {
			select {
			case event := <-watcher.Events:
				w.handleFileChangeEvent(event, db)
			case err := <-watcher.ErrorEvents:
				fmt.Println("file events error:", err)
				break FsEventsLoop
			}
		}
		watcher.Close()
	}
}

func (w *projectWatcher) handleFileChangeEvent(event fs.Event, db mc.ProjectDB) {
	switch {
	case event.IsCreate():
		w.handleCreate(event.Name)
	case event.IsDelete():
		// ignore
	case event.IsModify():
		w.handleModify(event.Name)
	case event.IsRename():
		// ignore
	case event.IsAttrib():
		// ignore
	default:
		// ignore
	}
}

func (w *projectWatcher) handleCreate(path string) {
	switch finfo, err := os.Stat(path); {
	case err != nil:
		fmt.Printf("Error stating %s: %s\n", path, err)
	case files.IgnoreDotAndTempFiles(path, finfo):
		// ignore
	case finfo.IsDir():
		w.dirCreate(path)
	case finfo.Mode().IsRegular():
		w.fileUpload(path)
	}
}

func (w *projectWatcher) dirCreate(path string) {
	if err := w.CreateDirectory(w.projectName, path); err != nil {
		fmt.Printf("Failed to create new directory %s: %s\n", path, err)
	} else {
		fmt.Println("Created new directory: ", path)
	}
}

func (w *projectWatcher) fileUpload(path string) {
	if err := w.UploadFile(w.projectName, path); err != nil {
		fmt.Printf("Failed to upload file %s: %s\n", path, err)
	}
}

func (w *projectWatcher) handleModify(path string) {
	if finfo, err := os.Stat(path); err != nil {
		fmt.Printf("Error getting file info for %s: %s\n", path, err)
	} else if finfo.Mode().IsRegular() {
		if !files.IgnoreDotAndTempFiles(path, finfo) {
			w.fileUpload(path)
		}
	}
}
<|endoftext|>


<commit_before>package taglib

//#include <taglib/tag_c.h>
//#include <stdlib.h>
// #cgo LDFLAGS: -ltag_c
import "C"
import "unsafe"

type Tags struct {
	Title, Artist, Album, Comment, Genre string
	Year, Track int
}

type Properties struct {
	Length, Bitrate, Samplerate, Channels int
}

type File C.TagLib_File

func Open(filename string) *File {
	fp := C.CString(filename)
	defer C.free(unsafe.Pointer(fp))
	return (*File)(C.taglib_file_new(fp))
}

func (f *File)Close() {
	C.taglib_file_free((*C.TagLib_File)(f))
}

func (f *File)GetTags() *Tags {
	ts := C.taglib_file_tag((*C.TagLib_File)(f))

	a := Tags{}

	if ts!=nil {
		a.Title   = C.GoString(C.taglib_tag_title(ts))
		a.Artist  = C.GoString(C.taglib_tag_artist(ts))
		a.Album   = C.GoString(C.taglib_tag_album(ts))
		a.Comment = C.GoString(C.taglib_tag_comment(ts))
		a.Genre   = C.GoString(C.taglib_tag_genre(ts))
		a.Year    = int(C.taglib_tag_year(ts))
		a.Track   = int(C.taglib_tag_track(ts))
	}
	return &a
}

func (f *File)GetProperties() *Properties {
	ap := C.taglib_file_audioproperties((*C.TagLib_File)(f))
	if ap==nil { return nil }

	p := Properties{}
	p.Length     = int(C.taglib_audioproperties_length(ap))
	p.Bitrate    = int(C.taglib_audioproperties_bitrate(ap))
	p.Samplerate = int(C.taglib_audioproperties_samplerate(ap))
	p.Channels   = int(C.taglib_audioproperties_channels(ap))

	return &p
}

func GetTags(filename string) *Tags {
	tf := Open(filename)
	if tf==nil {
		return nil
	}
	defer tf.Close()
	return tf.GetTags()
}

func GetProperties(filename string) *Properties {
	tf := Open(filename)
	if tf==nil {
		return nil
	}
	defer tf.Close()
	return tf.GetProperties()
}
<commit_msg>Adding comments and fixing a memory leak on libtag_c freeing char*<commit_after>package taglib

//#include <taglib/tag_c.h>
//#include <stdlib.h>
// #cgo LDFLAGS: -ltag_c
import "C"
import "unsafe"

type Tags struct {
	Title, Artist, Album, Comment, Genre string
	Year, Track                          int
}

type Properties struct {
	Length, Bitrate, Samplerate, Channels int
}

type File C.TagLib_File

func Open(filename string) *File {
	fp := C.CString(filename)
	defer C.free(unsafe.Pointer(fp))
	return (*File)(C.taglib_file_new(fp))
}

func (f *File) Close() {
	C.taglib_file_free((*C.TagLib_File)(f))
}

/*
Get the ID3 taglib.Tags from this taglib.File
*/
func (f *File) GetTags() *Tags {
	ts := C.taglib_file_tag((*C.TagLib_File)(f))

	a := Tags{}

	if ts != nil {
		a.Title = C.GoString(C.taglib_tag_title(ts))
		a.Artist = C.GoString(C.taglib_tag_artist(ts))
		a.Album = C.GoString(C.taglib_tag_album(ts))
		a.Comment = C.GoString(C.taglib_tag_comment(ts))
		a.Genre = C.GoString(C.taglib_tag_genre(ts))
		a.Year = int(C.taglib_tag_year(ts))
		a.Track = int(C.taglib_tag_track(ts))
	}

	defer C.taglib_tag_free_strings()
	return &a
}

/*
Get the taglib.Properties from this taglib.File
*/
func (f *File) GetProperties() *Properties {
	ap := C.taglib_file_audioproperties((*C.TagLib_File)(f))
	if ap == nil {
		return nil
	}

	p := Properties{}
	p.Length = int(C.taglib_audioproperties_length(ap))
	p.Bitrate = int(C.taglib_audioproperties_bitrate(ap))
	p.Samplerate = int(C.taglib_audioproperties_samplerate(ap))
	p.Channels = int(C.taglib_audioproperties_channels(ap))

	defer C.taglib_tag_free_strings()
	return &p
}

/*
Get the ID3 taglib.Tags from filename
*/
func GetTags(filename string) *Tags {
	tf := Open(filename)
	if tf == nil {
		return nil
	}
	defer tf.Close()
	return tf.GetTags()
}

/*
Get the taglib.Properties from filename
*/
func GetProperties(filename string) *Properties {
	tf := Open(filename)
	if tf == nil {
		return nil
	}
	defer tf.Close()
	return tf.GetProperties()
}
<|endoftext|>


<commit_before>package config

import (
	"fmt"

	"gopkg.in/go-playground/validator.v8"
)

var validate *validator.Validate

func init() {
	config := &validator.Config{TagName: "validate"}
	validate = validator.New(config)
}

//Validater validates a config
type Validater interface {
	Validate() error
}

func (config BalancerConfig) Validate() error {
	/* Validate BGP config */
	if config.ClusterMode == "anycast" {
		if err := config.Bgp.Validate(); err != nil {
			return err
		}
	}

	/* Validate IPAM config */
	if len(config.Ipam.Ranges) > 0 {
		if err := config.Ipam.Validate(); err != nil {
			return err
		}
	}

	/* Validate Join nodes param */
	if !config.Bootstrap {
		if len(config.Join) == 0 {
			return fmt.Errorf("You need to specify join nodes")
		}

		for _, v := range config.Join {
			if err := validate.Field(v, "ip"); err != nil {
				return fmt.Errorf("join parameter needs to be a valid IP v4")
			}
		}
	}

	return nil
}

func (bgp Bgp) Validate() error {
	return validate.Struct(bgp)
}

func (ipam Ipam) Validate() error {
	return validate.Struct(ipam)
}
<commit_msg>Better validation message<commit_after>package config

import (
	"fmt"

	"gopkg.in/go-playground/validator.v8"
)

var validate *validator.Validate

func init() {
	config := &validator.Config{TagName: "validate"}
	validate = validator.New(config)
}

//Validater validates a config
type Validater interface {
	Validate() error
}

func (config BalancerConfig) Validate() error {
	/* Validate BGP config */
	if config.ClusterMode == "anycast" {
		if err := config.Bgp.Validate(); err != nil {
			return err
		}
	}

	/* Validate IPAM config */
	if len(config.Ipam.Ranges) > 0 {
		if err := config.Ipam.Validate(); err != nil {
			return err
		}
	}

	/* Validate Join nodes param */
	if !config.Bootstrap {
		if len(config.Join) == 0 {
			return fmt.Errorf("You need to specify join nodes or start in Bootstrap mode.")
		}

		for _, v := range config.Join {
			if err := validate.Field(v, "ip"); err != nil {
				return fmt.Errorf("join parameter needs to be a valid IP v4")
			}
		}
	}

	return nil
}

func (bgp Bgp) Validate() error {
	return validate.Struct(bgp)
}

func (ipam Ipam) Validate() error {
	return validate.Struct(ipam)
}
<|endoftext|>