<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"runtime/pprof"
	"strings"
	"time"

	"golang.org/x/net/context"

	"github.com/nelhage/taktician/ai"
	"github.com/nelhage/taktician/cli"
	"github.com/nelhage/taktician/ptn"
	"github.com/nelhage/taktician/tak"
)

var (
	all     = flag.Bool("all", false, "show all possible moves")
	tps     = flag.Bool("tps", false, "render position in tps")
	quiet   = flag.Bool("quiet", false, "don't print board diagrams")
	explain = flag.Bool("explain", false, "explain scoring")
	eval    = flag.Bool("evaluate", false, "only show static evaluation")

	move    = flag.Int("move", 0, "PTN move number to analyze")
	final   = flag.Bool("final", false, "analyze final position only")
	black   = flag.Bool("black", false, "only analyze black's move")
	white   = flag.Bool("white", false, "only analyze white's move")
	variant = flag.String("variant", "", "apply the listed moves after the given position")

	debug     = flag.Int("debug", 1, "debug level")
	depth     = flag.Int("depth", 0, "minimax depth")
	timeLimit = flag.Duration("limit", time.Minute, "limit of how much time to use")

	seed         = flag.Int64("seed", 0, "specify a seed")
	sort         = flag.Bool("sort", true, "sort moves via history heuristic")
	table        = flag.Bool("table", true, "use the transposition table")
	nullMove     = flag.Bool("null-move", true, "use null-move pruning")
	extendForces = flag.Bool("extend-forces", true, "extend forced moves")
	reduceSlides = flag.Bool("reduce-slides", true, "reduce trivial slides")
	multiCut     = flag.Bool("multi-cut", true, "use multi-cut pruning")

	precise = flag.Bool("precise", false, "Limit to optimizations that provably preserve the game-theoretic value")

	cpuProfile = flag.String("cpuprofile", "", "write CPU profile")

	weights = flag.String("weights", "", "JSON-encoded evaluation weights")
)

func main() {
	flag.Parse()

	parsed, e := ptn.ParseFile(flag.Arg(0))
	if e != nil {
		log.Fatal("parse:", e)
	}
	color := tak.NoColor
	switch {
	case *white && *black:
		log.Fatal("-white and -black are exclusive")
	case *white:
		color = tak.White
	case *black:
		color = tak.Black
	case *move != 0:
		color = tak.White
	}

	if *cpuProfile != "" {
		f, e := os.OpenFile(*cpuProfile, os.O_WRONLY|os.O_CREATE, 0644)
		if e != nil {
			log.Fatalf("open cpu-profile: %s: %v", *cpuProfile, e)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *move != 0 || *final {
		p, e := parsed.PositionAtMove(*move, color)
		if e != nil {
			log.Fatal("find move:", e)
		}

		if *variant != "" {
			p, e = applyVariant(p, *variant)
			if e != nil {
				log.Fatal("-variant:", e)
			}
		}

		analyze(p)
	} else {
		p, e := parsed.InitialPosition()
		if e != nil {
			log.Fatal("initial:", e)
		}
		w, b := makeAI(p), makeAI(p)
		it := parsed.Iterator()
		for it.Next() {
			p := it.Position()
			m := it.PeekMove()
			switch {
			case p.ToMove() == tak.White && color != tak.Black:
				fmt.Printf("%d. %s\n", p.MoveNumber()/2+1, ptn.FormatMove(&m))
				analyzeWith(w, p)
			case p.ToMove() == tak.Black && color != tak.White:
				fmt.Printf("%d. ... %s\n", p.MoveNumber()/2+1, ptn.FormatMove(&m))
				analyzeWith(b, p)
			}
		}
		if e := it.Err(); e != nil {
			log.Fatalf("%d: %v", it.PTNMove(), e)
		}
	}
}

func applyVariant(p *tak.Position, variant string) (*tak.Position, error) {
	ms := strings.Split(variant, " ")
	for _, moveStr := range ms {
		m, e := ptn.ParseMove(moveStr)
		if e != nil {
			return nil, e
		}
		p, e = p.Move(&m)
		if e != nil {
			return nil, fmt.Errorf("bad move `%s': %v", moveStr, e)
		}
	}
	return p, nil
}

func makeAI(p *tak.Position) *ai.MinimaxAI {
	var w ai.Weights
	if *weights == "" {
		w = ai.DefaultWeights[p.Size()]
	} else {
		e := json.Unmarshal([]byte(*weights), &w)
		if e != nil {
			log.Fatalf("parse weights: %v", e)
		}
	}
	cfg := ai.MinimaxConfig{
		Size:  p.Size(),
		Depth: *depth,
		Seed:  *seed,
		Debug: *debug,

		NoSort:         !*sort,
		NoTable:        !*table,
		NoNullMove:     !*nullMove,
		NoExtendForces: !*extendForces,
		NoReduceSlides: !*reduceSlides,
		NoMultiCut:     !*multiCut,

		Evaluate: ai.MakeEvaluator(p.Size(), &w),
	}
	if *precise {
		cfg.MakePrecise()
	}
	return ai.NewMinimax(cfg)
}

func analyze(p *tak.Position) {
	analyzeWith(makeAI(p), p)
}

func analyzeWith(player *ai.MinimaxAI, p *tak.Position) {
	if *eval {
		val := player.Evaluate(p)
		if p.ToMove() == tak.Black {
			val = -val
		}
		fmt.Printf(" Val=%d\n", val)
		if *explain {
			ai.ExplainScore(player, os.Stdout, p)
		}
		return
	}
	ctx := context.Background()
	if *timeLimit != 0 {
		var cancel func()
		ctx, cancel = context.WithTimeout(ctx, *timeLimit)
		defer cancel()
	}
	pvs, val, _ := player.AnalyzeAll(ctx, p)
	if !*quiet {
		cli.RenderBoard(nil, os.Stdout, p)
		if *explain {
			ai.ExplainScore(player, os.Stdout, p)
		}
	}
	fmt.Printf("AI analysis:\n")
	for _, pv := range pvs {
		fmt.Printf(" pv=")
		for _, m := range pv {
			fmt.Printf("%s ", ptn.FormatMove(&m))
		}
		fmt.Printf("\n")
	}
	fmt.Printf(" value=%d\n", val)
	if *tps {
		fmt.Printf("[TPS \"%s\"]\n", ptn.FormatTPS(p))
	}
	if *all {
		fmt.Printf(" all moves:")
		for _, m := range p.AllMoves(nil) {
			fmt.Printf(" %s", ptn.FormatMove(&m))
		}
		fmt.Printf("\n")
	}
	fmt.Println()

	if len(pvs) == 0 || *quiet {
		return
	}

	for _, m := range pvs[0] {
		n, e := p.Move(&m)
		if e != nil {
			log.Printf("illegal move in pv: %s: %v", ptn.FormatMove(&m), e)
			if val < ai.WinThreshold && val > -ai.WinThreshold {
				log.Fatal("illegal move in non-terminal pv!")
			}
			return
		}
		p = n
	}

	fmt.Println("Resulting position:")
	cli.RenderBoard(nil, os.Stdout, p)
	if *explain {
		ai.ExplainScore(player, os.Stdout, p)
	}
	fmt.Println()
	fmt.Println()
}
<commit_msg>variant -> variation<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"runtime/pprof"
	"strings"
	"time"

	"golang.org/x/net/context"

	"github.com/nelhage/taktician/ai"
	"github.com/nelhage/taktician/cli"
	"github.com/nelhage/taktician/ptn"
	"github.com/nelhage/taktician/tak"
)

var (
	all     = flag.Bool("all", false, "show all possible moves")
	tps     = flag.Bool("tps", false, "render position in tps")
	quiet   = flag.Bool("quiet", false, "don't print board diagrams")
	explain = flag.Bool("explain", false, "explain scoring")
	eval    = flag.Bool("evaluate", false, "only show static evaluation")

	move      = flag.Int("move", 0, "PTN move number to analyze")
	final     = flag.Bool("final", false, "analyze final position only")
	black     = flag.Bool("black", false, "only analyze black's move")
	white     = flag.Bool("white", false, "only analyze white's move")
	variation = flag.String("variation", "", "apply the listed moves after the given position")

	debug     = flag.Int("debug", 1, "debug level")
	depth     = flag.Int("depth", 0, "minimax depth")
	timeLimit = flag.Duration("limit", time.Minute, "limit of how much time to use")

	seed         = flag.Int64("seed", 0, "specify a seed")
	sort         = flag.Bool("sort", true, "sort moves via history heuristic")
	table        = flag.Bool("table", true, "use the transposition table")
	nullMove     = flag.Bool("null-move", true, "use null-move pruning")
	extendForces = flag.Bool("extend-forces", true, "extend forced moves")
	reduceSlides = flag.Bool("reduce-slides", true, "reduce trivial slides")
	multiCut     = flag.Bool("multi-cut", true, "use multi-cut pruning")

	precise = flag.Bool("precise", false, "Limit to optimizations that provably preserve the game-theoretic value")

	cpuProfile = flag.String("cpuprofile", "", "write CPU profile")

	weights = flag.String("weights", "", "JSON-encoded evaluation weights")
)

func main() {
	flag.Parse()

	parsed, e := ptn.ParseFile(flag.Arg(0))
	if e != nil {
		log.Fatal("parse:", e)
	}
	color := tak.NoColor
	switch {
	case *white && *black:
		log.Fatal("-white and -black are exclusive")
	case *white:
		color = tak.White
	case *black:
		color = tak.Black
	case *move != 0:
		color = tak.White
	}

	if *cpuProfile != "" {
		f, e := os.OpenFile(*cpuProfile, os.O_WRONLY|os.O_CREATE, 0644)
		if e != nil {
			log.Fatalf("open cpu-profile: %s: %v", *cpuProfile, e)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *move != 0 || *final {
		p, e := parsed.PositionAtMove(*move, color)
		if e != nil {
			log.Fatal("find move:", e)
		}

		if *variation != "" {
			p, e = applyVariation(p, *variation)
			if e != nil {
				log.Fatal("-variation:", e)
			}
		}

		analyze(p)
	} else {
		p, e := parsed.InitialPosition()
		if e != nil {
			log.Fatal("initial:", e)
		}
		w, b := makeAI(p), makeAI(p)
		it := parsed.Iterator()
		for it.Next() {
			p := it.Position()
			m := it.PeekMove()
			switch {
			case p.ToMove() == tak.White && color != tak.Black:
				fmt.Printf("%d. %s\n", p.MoveNumber()/2+1, ptn.FormatMove(&m))
				analyzeWith(w, p)
			case p.ToMove() == tak.Black && color != tak.White:
				fmt.Printf("%d. ... %s\n", p.MoveNumber()/2+1, ptn.FormatMove(&m))
				analyzeWith(b, p)
			}
		}
		if e := it.Err(); e != nil {
			log.Fatalf("%d: %v", it.PTNMove(), e)
		}
	}
}

func applyVariation(p *tak.Position, variant string) (*tak.Position, error) {
	ms := strings.Split(variant, " ")
	for _, moveStr := range ms {
		m, e := ptn.ParseMove(moveStr)
		if e != nil {
			return nil, e
		}
		p, e = p.Move(&m)
		if e != nil {
			return nil, fmt.Errorf("bad move `%s': %v", moveStr, e)
		}
	}
	return p, nil
}

func makeAI(p *tak.Position) *ai.MinimaxAI {
	var w ai.Weights
	if *weights == "" {
		w = ai.DefaultWeights[p.Size()]
	} else {
		e := json.Unmarshal([]byte(*weights), &w)
		if e != nil {
			log.Fatalf("parse weights: %v", e)
		}
	}
	cfg := ai.MinimaxConfig{
		Size:  p.Size(),
		Depth: *depth,
		Seed:  *seed,
		Debug: *debug,

		NoSort:         !*sort,
		NoTable:        !*table,
		NoNullMove:     !*nullMove,
		NoExtendForces: !*extendForces,
		NoReduceSlides: !*reduceSlides,
		NoMultiCut:     !*multiCut,

		Evaluate: ai.MakeEvaluator(p.Size(), &w),
	}
	if *precise {
		cfg.MakePrecise()
	}
	return ai.NewMinimax(cfg)
}

func analyze(p *tak.Position) {
	analyzeWith(makeAI(p), p)
}

func analyzeWith(player *ai.MinimaxAI, p *tak.Position) {
	if *eval {
		val := player.Evaluate(p)
		if p.ToMove() == tak.Black {
			val = -val
		}
		fmt.Printf(" Val=%d\n", val)
		if *explain {
			ai.ExplainScore(player, os.Stdout, p)
		}
		return
	}
	ctx := context.Background()
	if *timeLimit != 0 {
		var cancel func()
		ctx, cancel = context.WithTimeout(ctx, *timeLimit)
		defer cancel()
	}
	pvs, val, _ := player.AnalyzeAll(ctx, p)
	if !*quiet {
		cli.RenderBoard(nil, os.Stdout, p)
		if *explain {
			ai.ExplainScore(player, os.Stdout, p)
		}
	}
	fmt.Printf("AI analysis:\n")
	for _, pv := range pvs {
		fmt.Printf(" pv=")
		for _, m := range pv {
			fmt.Printf("%s ", ptn.FormatMove(&m))
		}
		fmt.Printf("\n")
	}
	fmt.Printf(" value=%d\n", val)
	if *tps {
		fmt.Printf("[TPS \"%s\"]\n", ptn.FormatTPS(p))
	}
	if *all {
		fmt.Printf(" all moves:")
		for _, m := range p.AllMoves(nil) {
			fmt.Printf(" %s", ptn.FormatMove(&m))
		}
		fmt.Printf("\n")
	}
	fmt.Println()

	if len(pvs) == 0 || *quiet {
		return
	}

	for _, m := range pvs[0] {
		n, e := p.Move(&m)
		if e != nil {
			log.Printf("illegal move in pv: %s: %v", ptn.FormatMove(&m), e)
			if val < ai.WinThreshold && val > -ai.WinThreshold {
				log.Fatal("illegal move in non-terminal pv!")
			}
			return
		}
		p = n
	}

	fmt.Println("Resulting position:")
	cli.RenderBoard(nil, os.Stdout, p)
	if *explain {
		ai.ExplainScore(player, os.Stdout, p)
	}
	fmt.Println()
	fmt.Println()
}
<|endoftext|>


<commit_before>/*

	Autoreader is a simple program, designed to be run from go:generate, that
	helps generate the annoying boilerplate to implement
	boardgame.PropertyReader and boardgame.PropertyReadSetter.

*/
package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"github.com/MarcGrol/golangAnnotations/parser"
	"io"
	"log"
	"os"
	"strings"
	"text/template"
)

var headerTemplate *template.Template
var readerTemplate *template.Template
var readSetterTemplate *template.Template

const magicDocLinePrefix = "+autoreader"

type appOptions struct {
	OutputFile       string
	PackageDirectory string
	Help             bool
	flagSet          *flag.FlagSet
}

type templateConfig struct {
	FirstLetter string
	StructName  string
}

func init() {
	headerTemplate = template.Must(template.New("header").Parse(headerTemplateText))
	readerTemplate = template.Must(template.New("reader").Parse(readerTemplateText))
	readSetterTemplate = template.Must(template.New("readsetter").Parse(readSetterTemplateText))
}

func defineFlags(options *appOptions) {
	options.flagSet.StringVar(&options.OutputFile, "out", "auto_reader.go", "Defines which file to render output to. WARNING: it will be overwritten!")
	options.flagSet.StringVar(&options.PackageDirectory, "pkg", "examplepkg/", "Which package to process")
	options.flagSet.BoolVar(&options.Help, "h", false, "If set, print help message and quit.")
}

func getOptions(flagSet *flag.FlagSet, flagArguments []string) *appOptions {
	options := &appOptions{flagSet: flagSet}
	defineFlags(options)
	flagSet.Parse(flagArguments)
	return options
}

func main() {
	flagSet := flag.CommandLine
	process(getOptions(flagSet, os.Args[1:]), os.Stderr)
}

func process(options *appOptions, errOut io.ReadWriter) {

	if options.Help {
		options.flagSet.SetOutput(errOut)
		options.flagSet.PrintDefaults()
		return
	}

	output, err := processPackage(options.PackageDirectory)

	if err != nil {
		fmt.Fprintln(errOut, "ERROR", err)
		return
	}

	fmt.Fprintln(errOut, output)
}

func processPackage(location string) (output string, err error) {
	sources, err := parser.ParseSourceDir(location, ".*")

	if err != nil {
		return "", errors.New("Couldn't parse sources: " + err.Error())
	}

	haveOutputHeader := false

	for _, theStruct := range sources.Structs {

		if !haveOutputHeader {
			output += headerForPackage(theStruct.PackageName)
			haveOutputHeader = true
		}

		enableAutoReader := false

		for _, docLine := range theStruct.DocLines {
			docLine = strings.TrimPrefix(docLine, "//")
			docLine = strings.TrimSpace(docLine)
			if strings.HasPrefix(docLine, magicDocLinePrefix) {
				enableAutoReader = true
				break
			}
		}

		if !enableAutoReader {
			continue
		}

		output += readerForStruct(theStruct.Name)
		output += readSetterForStruct(theStruct.Name)
	}

	return output, nil
}

func templateOutput(template *template.Template, values interface{}) string {
	buf := new(bytes.Buffer)

	err := template.Execute(buf, values)

	if err != nil {
		log.Println(err)
	}

	return buf.String()
}

func headerForPackage(packageName string) string {
	return templateOutput(headerTemplate, map[string]string{
		"packageName": packageName,
	})
}

func readerForStruct(structName string) string {

	return templateOutput(readerTemplate, templateConfig{
		FirstLetter: structName[:1],
		StructName:  structName,
	})

}

func readSetterForStruct(structName string) string {
	return templateOutput(readSetterTemplate, templateConfig{
		FirstLetter: structName[:1],
		StructName:  structName,
	})
}

const headerTemplateText = `/************************************
 *
 * This file was auto-generated by autoreader. DO NOT EDIT.
 *
 ************************************/
package {{.packageName}}

import (
	"github.com/jkomoros/boardgame"
)

`

const readerTemplateText = `func ({{.FirstLetter}} *{{.StructName}}) Reader() boardgame.PropertyReader {
	return boardgame.DefaultReader({{.FirstLetter}})
}

`

const readSetterTemplateText = `func ({{.FirstLetter}} *{{.StructName}}) ReadSetter() boardgame.PropertyReadSetter {
	return boardgame.DefaultReadSetter({{.FirstLetter}})
}

`
<commit_msg>Actually output to the specified file. Part of #301.<commit_after>/*

	Autoreader is a simple program, designed to be run from go:generate, that
	helps generate the annoying boilerplate to implement
	boardgame.PropertyReader and boardgame.PropertyReadSetter.

*/
package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"github.com/MarcGrol/golangAnnotations/parser"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"text/template"
)

var headerTemplate *template.Template
var readerTemplate *template.Template
var readSetterTemplate *template.Template

const magicDocLinePrefix = "+autoreader"

type appOptions struct {
	OutputFile       string
	PackageDirectory string
	Help             bool
	flagSet          *flag.FlagSet
}

type templateConfig struct {
	FirstLetter string
	StructName  string
}

func init() {
	headerTemplate = template.Must(template.New("header").Parse(headerTemplateText))
	readerTemplate = template.Must(template.New("reader").Parse(readerTemplateText))
	readSetterTemplate = template.Must(template.New("readsetter").Parse(readSetterTemplateText))
}

func defineFlags(options *appOptions) {
	options.flagSet.StringVar(&options.OutputFile, "out", "auto_reader.go", "Defines which file to render output to. WARNING: it will be overwritten!")
	options.flagSet.StringVar(&options.PackageDirectory, "pkg", "examplepkg/", "Which package to process")
	options.flagSet.BoolVar(&options.Help, "h", false, "If set, print help message and quit.")
}

func getOptions(flagSet *flag.FlagSet, flagArguments []string) *appOptions {
	options := &appOptions{flagSet: flagSet}
	defineFlags(options)
	flagSet.Parse(flagArguments)
	return options
}

func main() {
	flagSet := flag.CommandLine
	process(getOptions(flagSet, os.Args[1:]), os.Stderr)
}

func process(options *appOptions, errOut io.ReadWriter) {

	if options.Help {
		options.flagSet.SetOutput(errOut)
		options.flagSet.PrintDefaults()
		return
	}

	output, err := processPackage(options.PackageDirectory)

	if err != nil {
		fmt.Fprintln(errOut, "ERROR", err)
		return
	}

	ioutil.WriteFile(options.OutputFile, []byte(output), 0644)
}

func processPackage(location string) (output string, err error) {
	sources, err := parser.ParseSourceDir(location, ".*")

	if err != nil {
		return "", errors.New("Couldn't parse sources: " + err.Error())
	}

	haveOutputHeader := false

	for _, theStruct := range sources.Structs {

		if !haveOutputHeader {
			output += headerForPackage(theStruct.PackageName)
			haveOutputHeader = true
		}

		enableAutoReader := false

		for _, docLine := range theStruct.DocLines {
			docLine = strings.TrimPrefix(docLine, "//")
			docLine = strings.TrimSpace(docLine)
			if strings.HasPrefix(docLine, magicDocLinePrefix) {
				enableAutoReader = true
				break
			}
		}

		if !enableAutoReader {
			continue
		}

		output += readerForStruct(theStruct.Name)
		output += readSetterForStruct(theStruct.Name)
	}

	return output, nil
}

func templateOutput(template *template.Template, values interface{}) string {
	buf := new(bytes.Buffer)

	err := template.Execute(buf, values)

	if err != nil {
		log.Println(err)
	}

	return buf.String()
}

func headerForPackage(packageName string) string {
	return templateOutput(headerTemplate, map[string]string{
		"packageName": packageName,
	})
}

func readerForStruct(structName string) string {

	return templateOutput(readerTemplate, templateConfig{
		FirstLetter: structName[:1],
		StructName:  structName,
	})

}

func readSetterForStruct(structName string) string {
	return templateOutput(readSetterTemplate, templateConfig{
		FirstLetter: structName[:1],
		StructName:  structName,
	})
}

const headerTemplateText = `/************************************
 *
 * This file was auto-generated by autoreader. DO NOT EDIT.
 *
 ************************************/
package {{.packageName}}

import (
	"github.com/jkomoros/boardgame"
)

`

const readerTemplateText = `func ({{.FirstLetter}} *{{.StructName}}) Reader() boardgame.PropertyReader {
	return boardgame.DefaultReader({{.FirstLetter}})
}

`

const readSetterTemplateText = `func ({{.FirstLetter}} *{{.StructName}}) ReadSetter() boardgame.PropertyReadSetter {
	return boardgame.DefaultReadSetter({{.FirstLetter}})
}

`
<|endoftext|>


<commit_before>/*

	Autoreader is a simple program, designed to be run from go:generate, that
	helps generate the annoying boilerplate to implement
	boardgame.PropertyReader and boardgame.PropertyReadSetter, as well as
	generating the boilerplate for enums.

	Autoreader processes a package of go files, searching for structs that
	have a comment immediately above their declaration that begins with
	"+autoreader". For each such struct, it creates a Reader(), ReadSetter(),
	and ReadSetConfigurer() method that implement boardgame.Reader,
	boardgame.ReadSetter, and boardgame.ReadSetConfigurer, respectively. If it
	finds a const() block at the top-level decorated with the magic comment it
	will also generate enum boilerplate. See the package doc of enum for more
	on what you need to include.

	auto-generated enums will automatically have values like
	PrefixVeryLongName set to have a string value of "Very Long Name"; that is
	title-case will be taken to mean word boundaries. If you want to transform
	the created values to lowercase or uppercase, include a line of
	`transform:lower` or `transform:upper`, respectively, in the comment lines
	immediately before the constant. `transform:none` means default behavior,
	leave as title case. If you want to change the default transform for an
	entire const group, have the transform line in the comment block above the
	constant block.  If you want to override a specific item in the enum's
	name, include a comment immediately above that matches that pattern
	`display:"myVal"`, where myVal is the exact string to use. myVal may be
	zero-length, and may include quoted quotes. If enums are autogenerated,
	and the struct in your package that appears to be your gameDelegate
	doesn't already have a ConfigureEnums(), one will be generated for you.

	Producing a ReadSetConfigurator requires a ReadSetter, and producing a
	ReadSetter requires a Reader. By default if you have the magic comment of
	`+autoreader` it with produce all three. However, if you want only some of
	the methods, include an argument for the highest one you want, e.g.
	`+autoreader readsetter` to generate a Reader() and ReadSetter().

	This package will automatically create additional type transform methods
	to handle fields whose literal type is boardgame.ImmutableSizedStack,
	boardgame.SizedStack, boardgame.MergedStack, enum.RangeValue, and
	enum.TreeValue.

	You can configure which package to process and where to write output via
	command-line flags. By default it processes the current package and writes
	its output to auto_reader.go, overwriting whatever file was there before.
	See command-line options by passing -h. Structs with an +autoreader
	comment that are in a _test.go file will be outputin auto_reader_test.go.

	The outputted readers, readsetters, and readsetconfigurers use a hard-
	coded list of fields for performance (reflection would be about 30% slower
	under normal usage). You should re-run go generate every time you add a
	struct or modify the fields on a struct.

	The defaults are set reasonably so that you can use go:generate very
	easily. See examplepkg/ for a very simple example.

*/
package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"github.com/MarcGrol/golangAnnotations/parser"
	"go/format"
	"io"
	"io/ioutil"
	"log"
	"os"
	"text/template"
)

//debugSaveBadCode, if true, will save even code that is not legal go if it
//can't be formatted. Useful for debugging bad template output temporarily.
//Should never be set to true for real uses.
const debugSaveBadCode = false

type appOptions struct {
	OutputFile       string
	OutputFileTest   string
	EnumOutputFile   string
	PackageDirectory string
	PrintToConsole   bool
	OutputEnum       bool
	OutputReader     bool
	OutputReaderTest bool
	Help             bool
	flagSet          *flag.FlagSet
}

type templateConfig struct {
	FirstLetter string
	StructName  string
}

func defineFlags(options *appOptions) {
	options.flagSet.StringVar(&options.OutputFile, "out", "auto_reader.go", "Defines which file to render output to. WARNING: it will be overwritten!")
	options.flagSet.StringVar(&options.OutputFileTest, "outtest", "auto_reader_test.go", "For structs in files that end in _test.go, what is the filename they should be exported to?")
	options.flagSet.StringVar(&options.EnumOutputFile, "enumout", "auto_enum.go", "Where to output the auto-enum file. WARNING: it will be overwritten!")
	options.flagSet.StringVar(&options.PackageDirectory, "pkg", ".", "Which package to process")
	options.flagSet.BoolVar(&options.OutputEnum, "enum", true, "Whether or not to output auto_enum.go")
	options.flagSet.BoolVar(&options.OutputReader, "reader", true, "Whether or not to output auto_reader.go")
	options.flagSet.BoolVar(&options.OutputReaderTest, "readertest", true, "Whether or not to output auto_reader_test.go")
	options.flagSet.BoolVar(&options.Help, "h", false, "If set, print help message and quit.")
	options.flagSet.BoolVar(&options.PrintToConsole, "print", false, "If true, will print result to console instead of writing to out.")
}

func getOptions(flagSet *flag.FlagSet, flagArguments []string) *appOptions {
	options := &appOptions{flagSet: flagSet}
	defineFlags(options)
	flagSet.Parse(flagArguments)
	return options
}

func main() {
	flagSet := flag.CommandLine
	process(getOptions(flagSet, os.Args[1:]), os.Stdout, os.Stderr)
}

func process(options *appOptions, out io.ReadWriter, errOut io.ReadWriter) {

	if options.Help {
		options.flagSet.SetOutput(out)
		options.flagSet.PrintDefaults()
		return
	}

	output, testOutput, enumOutput, err := processPackage(options.PackageDirectory)

	if err != nil {
		fmt.Fprintln(errOut, "ERROR", err)
		return
	}

	if options.PrintToConsole {
		if options.OutputReader {
			fmt.Fprintln(out, output)

		}
		if options.OutputReaderTest {
			fmt.Fprintln(out, testOutput)
		}
		if options.OutputEnum {
			fmt.Fprintln(out, enumOutput)
		}

		return
	}

	if output != "" && options.OutputReader {
		ioutil.WriteFile(options.OutputFile, []byte(output), 0644)
	}

	if testOutput != "" && options.OutputReaderTest {
		ioutil.WriteFile(options.OutputFileTest, []byte(testOutput), 0644)
	}

	if enumOutput != "" && options.OutputEnum {
		ioutil.WriteFile(options.EnumOutputFile, []byte(enumOutput), 0644)
	}

}

func processPackage(location string) (output string, testOutput string, enumOutput string, err error) {

	sources, err := parser.ParseSourceDir(location, ".*")

	if err != nil {
		return "", "", "", errors.New("Couldn't parse sources: " + err.Error())
	}

	output, testOutput, err = processStructs(sources, location)

	if err != nil {
		return "", "", "", errors.New("Couldn't process structs: " + err.Error())
	}

	enumOutput, err = processEnums(location)

	if err != nil {
		return "", "", "", errors.New("Couldn't process enums: " + err.Error())
	}

	formattedBytes, err := format.Source([]byte(output))

	if err != nil {
		if debugSaveBadCode {
			formattedBytes = []byte(output)
		} else {
			return "", "", "", errors.New("Couldn't go fmt code for reader: " + err.Error())
		}
	}

	formattedTestBytes, err := format.Source([]byte(testOutput))

	if err != nil {
		if debugSaveBadCode {
			formattedTestBytes = []byte(testOutput)
		} else {
			return "", "", "", errors.New("Couldn't go fmt code for reader: " + err.Error())
		}
	}

	formattedEnumBytes, err := format.Source([]byte(enumOutput))

	if err != nil {
		if debugSaveBadCode {
			formattedEnumBytes = []byte(enumOutput)
		} else {
			return "", "", "", errors.New("Couldn't go fmt code for enums: " + err.Error())
		}
	}

	return string(formattedBytes), string(formattedTestBytes), string(formattedEnumBytes), nil

}

func templateOutput(template *template.Template, values interface{}) string {
	buf := new(bytes.Buffer)

	err := template.Execute(buf, values)

	if err != nil {
		log.Println(err)
	}

	return buf.String()
}
<commit_msg>Update autoreader documentation about tree enums. Part of #628.<commit_after>/*

	Autoreader is a simple program, designed to be run from go:generate, that
	helps generate the annoying boilerplate to implement
	boardgame.PropertyReader and boardgame.PropertyReadSetter, as well as
	generating the boilerplate for enums.

	Autoreader processes a package of go files, searching for structs that
	have a comment immediately above their declaration that begins with
	"+autoreader". For each such struct, it creates a Reader(), ReadSetter(),
	and ReadSetConfigurer() method that implement boardgame.Reader,
	boardgame.ReadSetter, and boardgame.ReadSetConfigurer, respectively. If it
	finds a const() block at the top-level decorated with the magic comment it
	will also generate enum boilerplate. See the package doc of enum for more
	on what you need to include.

	auto-generated enums will automatically have values like
	PrefixVeryLongName set to have a string value of "Very Long Name"; that is
	title-case will be taken to mean word boundaries. If you want to transform
	the created values to lowercase or uppercase, include a line of
	`transform:lower` or `transform:upper`, respectively, in the comment lines
	immediately before the constant. `transform:none` means default behavior,
	leave as title case. If you want to change the default transform for an
	entire const group, have the transform line in the comment block above the
	constant block.  If you want to override a specific item in the enum's
	name, include a comment immediately above that matches that pattern
	`display:"myVal"`, where myVal is the exact string to use. myVal may be
	zero-length, and may include quoted quotes. If your enum has a key that is
	named with the prefix of the rest of the enum values, and evaluates to 0,
	then a TreeEnum will be created. See the documentation in the enum package
	for how to control nesting in a TreeEnum. If enums are autogenerated, and
	the struct in your package that appears to be your gameDelegate doesn't
	already have a ConfigureEnums(), one will be generated for you.

	Producing a ReadSetConfigurator requires a ReadSetter, and producing a
	ReadSetter requires a Reader. By default if you have the magic comment of
	`+autoreader` it with produce all three. However, if you want only some of
	the methods, include an argument for the highest one you want, e.g.
	`+autoreader readsetter` to generate a Reader() and ReadSetter().

	This package will automatically create additional type transform methods
	to handle fields whose literal type is boardgame.ImmutableSizedStack,
	boardgame.SizedStack, boardgame.MergedStack, enum.RangeValue, and
	enum.TreeValue.

	You can configure which package to process and where to write output via
	command-line flags. By default it processes the current package and writes
	its output to auto_reader.go, overwriting whatever file was there before.
	See command-line options by passing -h. Structs with an +autoreader
	comment that are in a _test.go file will be outputin auto_reader_test.go.

	The outputted readers, readsetters, and readsetconfigurers use a hard-
	coded list of fields for performance (reflection would be about 30% slower
	under normal usage). You should re-run go generate every time you add a
	struct or modify the fields on a struct.

	The defaults are set reasonably so that you can use go:generate very
	easily. See examplepkg/ for a very simple example.

*/
package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"github.com/MarcGrol/golangAnnotations/parser"
	"go/format"
	"io"
	"io/ioutil"
	"log"
	"os"
	"text/template"
)

//debugSaveBadCode, if true, will save even code that is not legal go if it
//can't be formatted. Useful for debugging bad template output temporarily.
//Should never be set to true for real uses.
const debugSaveBadCode = false

type appOptions struct {
	OutputFile       string
	OutputFileTest   string
	EnumOutputFile   string
	PackageDirectory string
	PrintToConsole   bool
	OutputEnum       bool
	OutputReader     bool
	OutputReaderTest bool
	Help             bool
	flagSet          *flag.FlagSet
}

type templateConfig struct {
	FirstLetter string
	StructName  string
}

func defineFlags(options *appOptions) {
	options.flagSet.StringVar(&options.OutputFile, "out", "auto_reader.go", "Defines which file to render output to. WARNING: it will be overwritten!")
	options.flagSet.StringVar(&options.OutputFileTest, "outtest", "auto_reader_test.go", "For structs in files that end in _test.go, what is the filename they should be exported to?")
	options.flagSet.StringVar(&options.EnumOutputFile, "enumout", "auto_enum.go", "Where to output the auto-enum file. WARNING: it will be overwritten!")
	options.flagSet.StringVar(&options.PackageDirectory, "pkg", ".", "Which package to process")
	options.flagSet.BoolVar(&options.OutputEnum, "enum", true, "Whether or not to output auto_enum.go")
	options.flagSet.BoolVar(&options.OutputReader, "reader", true, "Whether or not to output auto_reader.go")
	options.flagSet.BoolVar(&options.OutputReaderTest, "readertest", true, "Whether or not to output auto_reader_test.go")
	options.flagSet.BoolVar(&options.Help, "h", false, "If set, print help message and quit.")
	options.flagSet.BoolVar(&options.PrintToConsole, "print", false, "If true, will print result to console instead of writing to out.")
}

func getOptions(flagSet *flag.FlagSet, flagArguments []string) *appOptions {
	options := &appOptions{flagSet: flagSet}
	defineFlags(options)
	flagSet.Parse(flagArguments)
	return options
}

func main() {
	flagSet := flag.CommandLine
	process(getOptions(flagSet, os.Args[1:]), os.Stdout, os.Stderr)
}

func process(options *appOptions, out io.ReadWriter, errOut io.ReadWriter) {

	if options.Help {
		options.flagSet.SetOutput(out)
		options.flagSet.PrintDefaults()
		return
	}

	output, testOutput, enumOutput, err := processPackage(options.PackageDirectory)

	if err != nil {
		fmt.Fprintln(errOut, "ERROR", err)
		return
	}

	if options.PrintToConsole {
		if options.OutputReader {
			fmt.Fprintln(out, output)

		}
		if options.OutputReaderTest {
			fmt.Fprintln(out, testOutput)
		}
		if options.OutputEnum {
			fmt.Fprintln(out, enumOutput)
		}

		return
	}

	if output != "" && options.OutputReader {
		ioutil.WriteFile(options.OutputFile, []byte(output), 0644)
	}

	if testOutput != "" && options.OutputReaderTest {
		ioutil.WriteFile(options.OutputFileTest, []byte(testOutput), 0644)
	}

	if enumOutput != "" && options.OutputEnum {
		ioutil.WriteFile(options.EnumOutputFile, []byte(enumOutput), 0644)
	}

}

func processPackage(location string) (output string, testOutput string, enumOutput string, err error) {

	sources, err := parser.ParseSourceDir(location, ".*")

	if err != nil {
		return "", "", "", errors.New("Couldn't parse sources: " + err.Error())
	}

	output, testOutput, err = processStructs(sources, location)

	if err != nil {
		return "", "", "", errors.New("Couldn't process structs: " + err.Error())
	}

	enumOutput, err = processEnums(location)

	if err != nil {
		return "", "", "", errors.New("Couldn't process enums: " + err.Error())
	}

	formattedBytes, err := format.Source([]byte(output))

	if err != nil {
		if debugSaveBadCode {
			formattedBytes = []byte(output)
		} else {
			return "", "", "", errors.New("Couldn't go fmt code for reader: " + err.Error())
		}
	}

	formattedTestBytes, err := format.Source([]byte(testOutput))

	if err != nil {
		if debugSaveBadCode {
			formattedTestBytes = []byte(testOutput)
		} else {
			return "", "", "", errors.New("Couldn't go fmt code for reader: " + err.Error())
		}
	}

	formattedEnumBytes, err := format.Source([]byte(enumOutput))

	if err != nil {
		if debugSaveBadCode {
			formattedEnumBytes = []byte(enumOutput)
		} else {
			return "", "", "", errors.New("Couldn't go fmt code for enums: " + err.Error())
		}
	}

	return string(formattedBytes), string(formattedTestBytes), string(formattedEnumBytes), nil

}

func templateOutput(template *template.Template, values interface{}) string {
	buf := new(bytes.Buffer)

	err := template.Execute(buf, values)

	if err != nil {
		log.Println(err)
	}

	return buf.String()
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/golang/glog"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/api/v1"
	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/release_1_2"
	"k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
	clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api"
)

const (
	kubeletAPIPodsURL = "http://localhost:10255/pods"
	ignorePath        = "/srv/kubernetes/manifests"
	activePath        = "/etc/kubernetes/manifests"
	manifestFilename  = "apiserver.json"
	kubeconfigPath    = "/etc/kubernetes/kubeconfig"
	secretsPath       = "/etc/kubernetes/checkpoint-secrets"
)

var (
	tempAPIServer      = []byte("temp-apiserver")
	kubeAPIServer      = []byte("kube-apiserver")
	activeManifest     = filepath.Join(activePath, manifestFilename)
	checkpointManifest = filepath.Join(ignorePath, manifestFilename)
	secureAPIAddr      = fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS"))
)

var tempAPIServerManifest = v1.Pod{
	TypeMeta: unversioned.TypeMeta{
		APIVersion: "v1",
		Kind:       "Pod",
	},
	ObjectMeta: v1.ObjectMeta{
		Name:      "temp-apiserver",
		Namespace: "kube-system",
	},
}

func main() {
	glog.Info("begin apiserver checkpointing...")
	run()
}

func run() {
	client := newAPIClient()
	for {
		var podList v1.PodList
		if err := json.Unmarshal(getPodsFromKubeletAPI(), &podList); err != nil {
			glog.Fatal(err)
		}
		switch {
		case bothAPIServersRunning(podList):
			glog.Info("both temp and kube apiserver running, removing temp apiserver")
			// Both the self-hosted API Server and the temp API Server are running.
			// Remove the temp API Server manifest from the config dir so that the
			// kubelet will stop it.
			if err := os.Remove(activeManifest); err != nil {
				glog.Error(err)
			}
		case kubeSystemAPIServerRunning(podList):
			glog.Info("kube-apiserver found, creating temp-apiserver manifest")
			// The self-hosted API Server is running. Let's snapshot the pod,
			// clean it up a bit, and then save it to the ignore path for
			// later use.
			tempAPIServerManifest.Spec = parseAPIPodSpec(podList)
			convertSecretsToVolumeMounts(client, &tempAPIServerManifest)
			writeManifest(tempAPIServerManifest)
			glog.Infof("finished creating temp-apiserver manifest at %s\n", checkpointManifest)

		default:
			glog.Info("no apiserver running, installing temp apiserver static manifest")
			b, err := ioutil.ReadFile(checkpointManifest)
			if err != nil {
				glog.Error(err)
			} else {
				if err := ioutil.WriteFile(activeManifest, b, 0644); err != nil {
					glog.Error(err)
				}
			}
		}
		time.Sleep(60 * time.Second)
	}
}

func getPodsFromKubeletAPI() []byte {
	var pods []byte
	res, err := http.Get(kubeletAPIPodsURL)
	if err != nil {
		glog.Error(err)
		return pods
	}
	pods, err = ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		glog.Error(err)
	}
	return pods
}

func bothAPIServersRunning(pods v1.PodList) bool {
	var kubeAPISeen, tempAPISeen bool
	for _, p := range pods.Items {
		kubeAPISeen = kubeAPISeen || isKubeAPI(p)
		tempAPISeen = tempAPISeen || isTempAPI(p)
		if kubeAPISeen && tempAPISeen {
			return true
		}
	}
	return false
}

func kubeSystemAPIServerRunning(pods v1.PodList) bool {
	for _, p := range pods.Items {
		if isKubeAPI(p) {
			return true
		}
	}
	return false
}

func isKubeAPI(pod v1.Pod) bool {
	return strings.Contains(pod.Name, "kube-apiserver") && pod.Namespace == api.NamespaceSystem
}

func isTempAPI(pod v1.Pod) bool {
	return strings.Contains(pod.Name, "temp-apiserver") && pod.Namespace == api.NamespaceSystem
}

// cleanVolumes will sanitize the list of volumes and volume mounts
// to remove the default service account token.
func cleanVolumes(p *v1.Pod) {
	volumes := make([]v1.Volume, 0, len(p.Spec.Volumes))
	for _, v := range p.Spec.Volumes {
		if !strings.HasPrefix(v.Name, "default-token") {
			volumes = append(volumes, v)
		}
	}
	p.Spec.Volumes = volumes
	for i := range p.Spec.Containers {
		c := &p.Spec.Containers[i]
		volumeMounts := make([]v1.VolumeMount, 0, len(c.VolumeMounts))
		for _, vm := range c.VolumeMounts {
			if !strings.HasPrefix(vm.Name, "default-token") {
				volumeMounts = append(volumeMounts, vm)
			}
		}
		c.VolumeMounts = volumeMounts
	}
}

// We need to ensure that the temp apiserver is not binding
// on the same insecure port as our self-hosted apiserver, otherwise
// it will exit immediately instead of waiting to bind.
func modifyInsecurePort(p *v1.Pod) {
	for i := range p.Spec.Containers {
		c := &p.Spec.Containers[i]
		cmds := c.Command
		for i, cmd := range cmds {
			if strings.Contains(cmd, "insecure-port") {
				cmds[i] = strings.Replace(cmd, "8080", "8081", 1)
				break
			}
		}
	}
}

// writeManifest will write the manifest to the ignore path.
// It first writes the file to a temp file, and then atomically moves it into
// the actual ignore path and correct file name.
func writeManifest(manifest v1.Pod) {
	m, err := json.Marshal(manifest)
	if err != nil {
		glog.Fatal(err)
	}
	writeAndAtomicCopy(m, checkpointManifest)
}

func parseAPIPodSpec(podList v1.PodList) v1.PodSpec {
	var apiPod v1.Pod
	for _, p := range podList.Items {
		if isKubeAPI(p) {
			apiPod = p
			break
		}
	}
	cleanVolumes(&apiPod)
	modifyInsecurePort(&apiPod)
	return apiPod.Spec
}

func newAPIClient() clientset.Interface {
	kubeConfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
		&clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath},
		&clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: secureAPIAddr}}).ClientConfig()
	if err != nil {
		glog.Fatal(err)
	}
	return clientset.NewForConfigOrDie(kubeConfig)
}

func convertSecretsToVolumeMounts(client clientset.Interface, pod *v1.Pod) {
	glog.Info("converting secrets to volume mounts")
	spec := pod.Spec
	for i := range spec.Volumes {
		v := &spec.Volumes[i]
		if v.Secret != nil {
			secretName := v.Secret.SecretName
			basePath := filepath.Join(secretsPath, pod.Name, v.Secret.SecretName)
			v.HostPath = &v1.HostPathVolumeSource{
				Path: basePath,
			}
			copySecretsToDisk(client, secretName, basePath)
			v.Secret = nil
			break
		}
	}
}

func copySecretsToDisk(client clientset.Interface, secretName, basePath string) {
	glog.Info("copying secrets to disk")
	if err := os.MkdirAll(basePath, 0755); err != nil {
		glog.Fatal(err)
	}
	glog.Infof("created directory %s", basePath)
	s, err := client.Core().Secrets(api.NamespaceSystem).Get(secretName)
	if err != nil {
		glog.Fatal(err)
	}
	for name, value := range s.Data {
		path := filepath.Join(basePath, name)
		writeAndAtomicCopy(value, path)
	}
}

func writeAndAtomicCopy(data []byte, path string) {
	// First write a "temp" file.
	tmpfile := filepath.Join(filepath.Dir(path), "."+filepath.Base(path))
	if err := ioutil.WriteFile(tmpfile, data, 0644); err != nil {
		glog.Fatal(err)
	}
	// Finally, copy that file to the correct location.
	if err := os.Rename(tmpfile, path); err != nil {
		glog.Fatal(err)
	}
}
<commit_msg>cmd/checkpoint: Verify API server running (#65)<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/golang/glog"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/api/v1"
	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/release_1_2"
	"k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
	clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api"
)

const (
	kubeletAPIPodsURL = "http://localhost:10255/pods"
	ignorePath        = "/srv/kubernetes/manifests"
	activePath        = "/etc/kubernetes/manifests"
	manifestFilename  = "apiserver.json"
	kubeconfigPath    = "/etc/kubernetes/kubeconfig"
	secretsPath       = "/etc/kubernetes/checkpoint-secrets"
)

var (
	tempAPIServer      = []byte("temp-apiserver")
	kubeAPIServer      = []byte("kube-apiserver")
	activeManifest     = filepath.Join(activePath, manifestFilename)
	checkpointManifest = filepath.Join(ignorePath, manifestFilename)
	secureAPIAddr      = fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS"))
)

var tempAPIServerManifest = v1.Pod{
	TypeMeta: unversioned.TypeMeta{
		APIVersion: "v1",
		Kind:       "Pod",
	},
	ObjectMeta: v1.ObjectMeta{
		Name:      "temp-apiserver",
		Namespace: "kube-system",
	},
}

func main() {
	glog.Info("begin apiserver checkpointing...")
	run()
}

func run() {
	client := newAPIClient()
	for {
		var podList v1.PodList
		if err := json.Unmarshal(getPodsFromKubeletAPI(), &podList); err != nil {
			glog.Fatal(err)
		}
		switch {
		case bothAPIServersRunning(podList):
			glog.Info("both temp and kube apiserver running, removing temp apiserver")
			// Both the self-hosted API Server and the temp API Server are running.
			// Remove the temp API Server manifest from the config dir so that the
			// kubelet will stop it.
			if err := os.Remove(activeManifest); err != nil {
				glog.Error(err)
			}
		case kubeSystemAPIServerRunning(podList, client):
			glog.Info("kube-apiserver found, creating temp-apiserver manifest")
			// The self-hosted API Server is running. Let's snapshot the pod,
			// clean it up a bit, and then save it to the ignore path for
			// later use.
			tempAPIServerManifest.Spec = parseAPIPodSpec(podList)
			convertSecretsToVolumeMounts(client, &tempAPIServerManifest)
			writeManifest(tempAPIServerManifest)
			glog.Infof("finished creating temp-apiserver manifest at %s\n", checkpointManifest)

		default:
			glog.Info("no apiserver running, installing temp apiserver static manifest")
			b, err := ioutil.ReadFile(checkpointManifest)
			if err != nil {
				glog.Error(err)
			} else {
				if err := ioutil.WriteFile(activeManifest, b, 0644); err != nil {
					glog.Error(err)
				}
			}
		}
		time.Sleep(60 * time.Second)
	}
}

func getPodsFromKubeletAPI() []byte {
	var pods []byte
	res, err := http.Get(kubeletAPIPodsURL)
	if err != nil {
		glog.Error(err)
		return pods
	}
	pods, err = ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		glog.Error(err)
	}
	return pods
}

func bothAPIServersRunning(pods v1.PodList) bool {
	var kubeAPISeen, tempAPISeen bool
	for _, p := range pods.Items {
		kubeAPISeen = kubeAPISeen || isKubeAPI(p)
		tempAPISeen = tempAPISeen || isTempAPI(p)
		if kubeAPISeen && tempAPISeen {
			return true
		}
	}
	return false
}

func kubeSystemAPIServerRunning(pods v1.PodList, client clientset.Interface) bool {
	for _, p := range pods.Items {
		if isKubeAPI(p) {
			// Make sure it's actually running. Sometimes we get that
			// pod manifest back, but the server is not actually running.
			_, err := client.Discovery().ServerVersion()
			return err == nil
		}
	}
	return false
}

func isKubeAPI(pod v1.Pod) bool {
	return strings.Contains(pod.Name, "kube-apiserver") && pod.Namespace == api.NamespaceSystem
}

func isTempAPI(pod v1.Pod) bool {
	return strings.Contains(pod.Name, "temp-apiserver") && pod.Namespace == api.NamespaceSystem
}

// cleanVolumes will sanitize the list of volumes and volume mounts
// to remove the default service account token.
func cleanVolumes(p *v1.Pod) {
	volumes := make([]v1.Volume, 0, len(p.Spec.Volumes))
	for _, v := range p.Spec.Volumes {
		if !strings.HasPrefix(v.Name, "default-token") {
			volumes = append(volumes, v)
		}
	}
	p.Spec.Volumes = volumes
	for i := range p.Spec.Containers {
		c := &p.Spec.Containers[i]
		volumeMounts := make([]v1.VolumeMount, 0, len(c.VolumeMounts))
		for _, vm := range c.VolumeMounts {
			if !strings.HasPrefix(vm.Name, "default-token") {
				volumeMounts = append(volumeMounts, vm)
			}
		}
		c.VolumeMounts = volumeMounts
	}
}

// We need to ensure that the temp apiserver is not binding
// on the same insecure port as our self-hosted apiserver, otherwise
// it will exit immediately instead of waiting to bind.
func modifyInsecurePort(p *v1.Pod) {
	for i := range p.Spec.Containers {
		c := &p.Spec.Containers[i]
		cmds := c.Command
		for i, cmd := range cmds {
			if strings.Contains(cmd, "insecure-port") {
				cmds[i] = strings.Replace(cmd, "8080", "8081", 1)
				break
			}
		}
	}
}

// writeManifest will write the manifest to the ignore path.
// It first writes the file to a temp file, and then atomically moves it into
// the actual ignore path and correct file name.
func writeManifest(manifest v1.Pod) {
	m, err := json.Marshal(manifest)
	if err != nil {
		glog.Fatal(err)
	}
	writeAndAtomicCopy(m, checkpointManifest)
}

func parseAPIPodSpec(podList v1.PodList) v1.PodSpec {
	var apiPod v1.Pod
	for _, p := range podList.Items {
		if isKubeAPI(p) {
			apiPod = p
			break
		}
	}
	cleanVolumes(&apiPod)
	modifyInsecurePort(&apiPod)
	return apiPod.Spec
}

func newAPIClient() clientset.Interface {
	kubeConfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
		&clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath},
		&clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: secureAPIAddr}}).ClientConfig()
	if err != nil {
		glog.Fatal(err)
	}
	return clientset.NewForConfigOrDie(kubeConfig)
}

func convertSecretsToVolumeMounts(client clientset.Interface, pod *v1.Pod) {
	glog.Info("converting secrets to volume mounts")
	spec := pod.Spec
	for i := range spec.Volumes {
		v := &spec.Volumes[i]
		if v.Secret != nil {
			secretName := v.Secret.SecretName
			basePath := filepath.Join(secretsPath, pod.Name, v.Secret.SecretName)
			v.HostPath = &v1.HostPathVolumeSource{
				Path: basePath,
			}
			copySecretsToDisk(client, secretName, basePath)
			v.Secret = nil
			break
		}
	}
}

func copySecretsToDisk(client clientset.Interface, secretName, basePath string) {
	glog.Info("copying secrets to disk")
	if err := os.MkdirAll(basePath, 0755); err != nil {
		glog.Fatal(err)
	}
	glog.Infof("created directory %s", basePath)
	s, err := client.Core().Secrets(api.NamespaceSystem).Get(secretName)
	if err != nil {
		glog.Fatal(err)
	}
	for name, value := range s.Data {
		path := filepath.Join(basePath, name)
		writeAndAtomicCopy(value, path)
	}
}

func writeAndAtomicCopy(data []byte, path string) {
	// First write a "temp" file.
	tmpfile := filepath.Join(filepath.Dir(path), "."+filepath.Base(path))
	if err := ioutil.WriteFile(tmpfile, data, 0644); err != nil {
		glog.Fatal(err)
	}
	// Finally, copy that file to the correct location.
	if err := os.Rename(tmpfile, path); err != nil {
		glog.Fatal(err)
	}
}
<|endoftext|>


<commit_before>/*
 * Minio Client (C) 2017 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"github.com/fatih/color"
	"github.com/minio/cli"
	"github.com/minio/mc/pkg/console"
	"github.com/minio/mc/pkg/probe"
)

var configHostAddCmd = cli.Command{
	Name:            "add",
	ShortName:       "a",
	Usage:           "Add a new host to configuration file.",
	Action:          mainConfigHostAdd,
	Before:          setGlobalsFromContext,
	Flags:           globalFlags,
	HideHelpCommand: true,
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} ALIAS URL ACCESS-KEY SECRET-KEY [API]

FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
EXAMPLES:
  1. Add Amazon S3 storage service under "mys3" alias. For security reasons turn off bash history momentarily.
     $ set +o history
     $ {{.HelpName}} mys3 https://s3.amazonaws.com \
                 BKIKJAA5BMMU2RHO6IBB V8f1CwQqAcwo80UEIJEjc5gVQUSSx5ohQ9GSrr12
     $ set -o history

  2. Add Amazon S3 accelerated storage service under "mys3-accel" alias. For security reasons turn off bash history momentarily.
     $ set +o history
     $ {{.HelpName}} mys3-accel https://s3-accelerate.amazonaws.com \
                 BKIKJAA5BMMU2RHO6IBB V8f1CwQqAcwo80UEIJEjc5gVQUSSx5ohQ9GSrr12
     $ set -o history

  3. Add Amazon S3 IAM temporary credentials with limited access, please make sure to override the signature probe by explicitly
     providing the signature type.
     $ set +o history
     $ {{.HelpName}} mys3-iam https://s3.amazonaws.com \
                 BKIKJAA5BMMU2RHO6IBB V8f1CwQqAcwo80UEIJEjc5gVQUSSx5ohQ9GSrr12 s3v4
     $ set -o history

`,
}

// checkConfigHostAddSyntax - verifies input arguments to 'config host add'.
func checkConfigHostAddSyntax(ctx *cli.Context) {
	args := ctx.Args()
	argsNr := len(args)
	if argsNr < 4 || argsNr > 5 {
		fatalIf(errInvalidArgument().Trace(ctx.Args().Tail()...),
			"Incorrect number of arguments for host add command.")
	}

	alias := args.Get(0)
	url := args.Get(1)
	accessKey := args.Get(2)
	secretKey := args.Get(3)
	api := args.Get(4)

	if !isValidAlias(alias) {
		fatalIf(errInvalidAlias(alias), "Invalid alias")
	}

	if !isValidHostURL(url) {
		fatalIf(errInvalidURL(url), "Invalid URL.")
	}

	if !isValidAccessKey(accessKey) {
		fatalIf(errInvalidArgument().Trace(accessKey),
			"Invalid access key `"+accessKey+"`.")
	}

	if !isValidSecretKey(secretKey) {
		fatalIf(errInvalidArgument().Trace(secretKey),
			"Invalid secret key `"+secretKey+"`.")
	}

	if api != "" && !isValidAPI(api) { // Empty value set to default "S3v4".
		fatalIf(errInvalidArgument().Trace(api),
			"Unrecognized API signature. Valid options are `[S3v4, S3v2]`.")
	}
}

// addHost - add a host config.
func addHost(alias string, hostCfgV8 hostConfigV8) {
	mcCfgV8, err := loadMcConfig()
	fatalIf(err.Trace(globalMCConfigVersion), "Unable to load config `"+mustGetMcConfigPath()+"`.")

	// Add new host.
	mcCfgV8.Hosts[alias] = hostCfgV8

	err = saveMcConfig(mcCfgV8)
	fatalIf(err.Trace(alias), "Unable to update hosts in config version `"+mustGetMcConfigPath()+"`.")

	printMsg(hostMessage{
		op:        "add",
		Alias:     alias,
		URL:       hostCfgV8.URL,
		AccessKey: hostCfgV8.AccessKey,
		SecretKey: hostCfgV8.SecretKey,
		API:       hostCfgV8.API,
	})
}

func newS3Config(args cli.Args) (*Config, *probe.Error) {
	var (
		url       = args.Get(0)
		accessKey = args.Get(1)
		secretKey = args.Get(2)
		api       = args.Get(3)
	)

	// If api is provided we do not auto probe signature, this is
	// required in situations when signature type is provided by the user.
	if api != "" {
		return &Config{
			AccessKey: accessKey,
			SecretKey: secretKey,
			Signature: api,
			HostURL:   url,
		}, nil
	}

	s3Config := &Config{
		AccessKey: accessKey,
		SecretKey: secretKey,
		Signature: "s3v4",
		HostURL:   urlJoinPath(url, "probe-bucket-sign"),
	}

	s3Client, err := s3New(s3Config)
	if err != nil {
		return nil, err.Trace(args...)
	}

	if _, err = s3Client.Stat(false, false); err != nil {
		switch err.ToGoError().(type) {
		case BucketDoesNotExist:
			// Bucket doesn't exist, means signature probing worked V4.
		default:
			// Attempt with signature v2, since v4 seem to have failed.
			s3Config.Signature = "s3v2"
			s3Client, err = s3New(s3Config)
			if err != nil {
				return nil, err.Trace(args...)
			}
			if _, err = s3Client.Stat(false, false); err != nil {
				switch err.ToGoError().(type) {
				case BucketDoesNotExist:
					// Bucket doesn't exist, means signature probing worked with V2.
				default:
					return nil, err.Trace(args...)
				}
			}
		}
	}

	// Set the host url with original URL.
	s3Config.HostURL = url

	// Success.
	return s3Config, nil
}

func mainConfigHostAdd(ctx *cli.Context) error {
	checkConfigHostAddSyntax(ctx)

	console.SetColor("HostMessage", color.New(color.FgGreen))

	s3Config, err := newS3Config(ctx.Args().Tail())
	fatalIf(err.Trace(ctx.Args()...), "Unable to initialize new config from the provided credentials")

	addHost(ctx.Args().Get(0), hostConfigV8{
		URL:       s3Config.HostURL,
		AccessKey: s3Config.AccessKey,
		SecretKey: s3Config.SecretKey,
		API:       s3Config.Signature,
	}) // Add a host with specified credentials.
	return nil
}
<commit_msg>fix: Consider insecure flag in host add cmd (#2331)<commit_after>/*
 * Minio Client (C) 2017 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"github.com/fatih/color"
	"github.com/minio/cli"
	"github.com/minio/mc/pkg/console"
	"github.com/minio/mc/pkg/probe"
)

var configHostAddCmd = cli.Command{
	Name:            "add",
	ShortName:       "a",
	Usage:           "Add a new host to configuration file.",
	Action:          mainConfigHostAdd,
	Before:          setGlobalsFromContext,
	Flags:           globalFlags,
	HideHelpCommand: true,
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} ALIAS URL ACCESS-KEY SECRET-KEY [API]

FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
EXAMPLES:
  1. Add Amazon S3 storage service under "mys3" alias. For security reasons turn off bash history momentarily.
     $ set +o history
     $ {{.HelpName}} mys3 https://s3.amazonaws.com \
                 BKIKJAA5BMMU2RHO6IBB V8f1CwQqAcwo80UEIJEjc5gVQUSSx5ohQ9GSrr12
     $ set -o history

  2. Add Amazon S3 accelerated storage service under "mys3-accel" alias. For security reasons turn off bash history momentarily.
     $ set +o history
     $ {{.HelpName}} mys3-accel https://s3-accelerate.amazonaws.com \
                 BKIKJAA5BMMU2RHO6IBB V8f1CwQqAcwo80UEIJEjc5gVQUSSx5ohQ9GSrr12
     $ set -o history

  3. Add Amazon S3 IAM temporary credentials with limited access, please make sure to override the signature probe by explicitly
     providing the signature type.
     $ set +o history
     $ {{.HelpName}} mys3-iam https://s3.amazonaws.com \
                 BKIKJAA5BMMU2RHO6IBB V8f1CwQqAcwo80UEIJEjc5gVQUSSx5ohQ9GSrr12 s3v4
     $ set -o history

`,
}

// checkConfigHostAddSyntax - verifies input arguments to 'config host add'.
func checkConfigHostAddSyntax(ctx *cli.Context) {
	args := ctx.Args()
	argsNr := len(args)
	if argsNr < 4 || argsNr > 5 {
		fatalIf(errInvalidArgument().Trace(ctx.Args().Tail()...),
			"Incorrect number of arguments for host add command.")
	}

	alias := args.Get(0)
	url := args.Get(1)
	accessKey := args.Get(2)
	secretKey := args.Get(3)
	api := args.Get(4)

	if !isValidAlias(alias) {
		fatalIf(errInvalidAlias(alias), "Invalid alias")
	}

	if !isValidHostURL(url) {
		fatalIf(errInvalidURL(url), "Invalid URL.")
	}

	if !isValidAccessKey(accessKey) {
		fatalIf(errInvalidArgument().Trace(accessKey),
			"Invalid access key `"+accessKey+"`.")
	}

	if !isValidSecretKey(secretKey) {
		fatalIf(errInvalidArgument().Trace(secretKey),
			"Invalid secret key `"+secretKey+"`.")
	}

	if api != "" && !isValidAPI(api) { // Empty value set to default "S3v4".
		fatalIf(errInvalidArgument().Trace(api),
			"Unrecognized API signature. Valid options are `[S3v4, S3v2]`.")
	}
}

// addHost - add a host config.
func addHost(alias string, hostCfgV8 hostConfigV8) {
	mcCfgV8, err := loadMcConfig()
	fatalIf(err.Trace(globalMCConfigVersion), "Unable to load config `"+mustGetMcConfigPath()+"`.")

	// Add new host.
	mcCfgV8.Hosts[alias] = hostCfgV8

	err = saveMcConfig(mcCfgV8)
	fatalIf(err.Trace(alias), "Unable to update hosts in config version `"+mustGetMcConfigPath()+"`.")

	printMsg(hostMessage{
		op:        "add",
		Alias:     alias,
		URL:       hostCfgV8.URL,
		AccessKey: hostCfgV8.AccessKey,
		SecretKey: hostCfgV8.SecretKey,
		API:       hostCfgV8.API,
	})
}

func newS3Config(args cli.Args) (*Config, *probe.Error) {
	var (
		url       = args.Get(0)
		accessKey = args.Get(1)
		secretKey = args.Get(2)
		api       = args.Get(3)
	)

	// If api is provided we do not auto probe signature, this is
	// required in situations when signature type is provided by the user.
	if api != "" {
		return &Config{
			Insecure:  globalInsecure,
			AccessKey: accessKey,
			SecretKey: secretKey,
			Signature: api,
			HostURL:   url,
		}, nil
	}

	s3Config := &Config{
		Insecure:  globalInsecure,
		AccessKey: accessKey,
		SecretKey: secretKey,
		Signature: "s3v4",
		HostURL:   urlJoinPath(url, "probe-bucket-sign"),
	}

	s3Client, err := s3New(s3Config)
	if err != nil {
		return nil, err.Trace(args...)
	}

	if _, err = s3Client.Stat(false, false); err != nil {
		switch err.ToGoError().(type) {
		case BucketDoesNotExist:
			// Bucket doesn't exist, means signature probing worked V4.
		default:
			// Attempt with signature v2, since v4 seem to have failed.
			s3Config.Signature = "s3v2"
			s3Client, err = s3New(s3Config)
			if err != nil {
				return nil, err.Trace(args...)
			}
			if _, err = s3Client.Stat(false, false); err != nil {
				switch err.ToGoError().(type) {
				case BucketDoesNotExist:
					// Bucket doesn't exist, means signature probing worked with V2.
				default:
					return nil, err.Trace(args...)
				}
			}
		}
	}

	// Set the host url with original URL.
	s3Config.HostURL = url

	// Success.
	return s3Config, nil
}

func mainConfigHostAdd(ctx *cli.Context) error {
	checkConfigHostAddSyntax(ctx)

	console.SetColor("HostMessage", color.New(color.FgGreen))

	s3Config, err := newS3Config(ctx.Args().Tail())
	fatalIf(err.Trace(ctx.Args()...), "Unable to initialize new config from the provided credentials")

	addHost(ctx.Args().Get(0), hostConfigV8{
		URL:       s3Config.HostURL,
		AccessKey: s3Config.AccessKey,
		SecretKey: s3Config.SecretKey,
		API:       s3Config.Signature,
	}) // Add a host with specified credentials.
	return nil
}
<|endoftext|>


<commit_before>package command

import (
	"flag"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
	"github.com/funkygao/golib/gofmt"
	"github.com/funkygao/golib/progress"
)

type Lags struct {
	Ui  cli.Ui
	Cmd string

	onlineOnly      bool
	groupPattern    string
	topicPattern    string
	watchMode       bool
	problematicMode bool
	lagThreshold    int
}

func (this *Lags) Run(args []string) (exitCode int) {
	const secondsInMinute = 60
	var (
		cluster string
		zone    string
	)
	cmdFlags := flag.NewFlagSet("lags", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.StringVar(&cluster, "c", "", "")
	cmdFlags.BoolVar(&this.onlineOnly, "l", false, "")
	cmdFlags.BoolVar(&this.problematicMode, "p", false, "")
	cmdFlags.StringVar(&this.groupPattern, "g", "", "")
	cmdFlags.StringVar(&this.topicPattern, "t", "", "")
	cmdFlags.BoolVar(&this.watchMode, "w", false, "")
	cmdFlags.IntVar(&this.lagThreshold, "lag", 5000, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if validateArgs(this, this.Ui).
		require("-c", "-t").
		invalid(args) {
		return 2
	}

	if this.watchMode {
		refreshScreen()
	}

	if this.problematicMode {
		this.onlineOnly = true
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
	bar := progress.New(secondsInMinute)
	if cluster == "" {
		for {
			zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
				this.Ui.Output(zkcluster.Name())
				this.printConsumersLag(zkcluster)
			})

			if this.watchMode {
				for i := 1; i <= secondsInMinute; i++ {
					bar.ShowProgress(i)
					time.Sleep(time.Second)
				}
			} else {
				break
			}

			printSwallowedErrors(this.Ui, zkzone)
		}

		return
	}

	for {
		this.Ui.Output(cluster)
		zkcluster := zkzone.NewCluster(cluster) // panic if invalid cluster
		this.printConsumersLag(zkcluster)

		if this.watchMode {
			for i := 1; i <= secondsInMinute; i++ {
				bar.ShowProgress(i)
				time.Sleep(time.Second)
			}
			//refreshScreen()
		} else {
			break
		}

		printSwallowedErrors(this.Ui, zkzone)
	}

	return
}

func (this *Lags) printConsumersLag(zkcluster *zk.ZkCluster) {
	// sort by group name
	consumersByGroup := zkcluster.ConsumersByGroup(this.groupPattern)
	sortedGroups := make([]string, 0, len(consumersByGroup))
	for group, _ := range consumersByGroup {
		sortedGroups = append(sortedGroups, group)
	}
	sort.Strings(sortedGroups)

	for _, group := range sortedGroups {
		this.Ui.Output(strings.Repeat(" ", 4) + group)

		sortedTopicAndPartitionIds := make([]string, 0)
		consumers := make(map[string]zk.ConsumerMeta)
		for _, t := range consumersByGroup[group] {
			key := fmt.Sprintf("%s:%s", t.Topic, t.PartitionId)
			sortedTopicAndPartitionIds = append(sortedTopicAndPartitionIds, key)

			consumers[key] = t
		}
		sort.Strings(sortedTopicAndPartitionIds)

		for _, topicAndPartitionId := range sortedTopicAndPartitionIds {
			consumer := consumers[topicAndPartitionId]

			if !patternMatched(consumer.Topic, this.topicPattern) {
				continue
			}

			var (
				lagOutput string
				symbol    string
			)
			if consumer.Lag > int64(this.lagThreshold) {
				lagOutput = color.Red("%15s", gofmt.Comma(consumer.Lag))
				if consumer.Online {
					symbol = color.Yellow("⚠")
				} else {
					symbol = color.Yellow("☔")
				}
			} else {
				lagOutput = color.Blue("%15s", gofmt.Comma(consumer.Lag))
				if consumer.Online {
					symbol = color.Green("☀")
				} else {
					symbol = color.Yellow("☔")
				}
			}

			if consumer.Online {
				if this.problematicMode && consumer.Lag <= int64(this.lagThreshold) {
					continue
				}

				var (
					host   string
					uptime string
				)
				if consumer.ConsumerZnode == nil {
					host = "unrecognized"
					uptime = "-"
				} else {
					host = color.Green("%90s", consumer.ConsumerZnode.Host())
					uptime = gofmt.PrettySince(consumer.ConsumerZnode.Uptime())
				}

				this.Ui.Output(fmt.Sprintf("\t%s %35s/%-2s %12s -> %-12s %s %s\n%s %s",
					symbol,
					consumer.Topic, consumer.PartitionId,
					gofmt.Comma(consumer.ProducerOffset),
					gofmt.Comma(consumer.ConsumerOffset),
					lagOutput,
					gofmt.PrettySince(consumer.Mtime.Time()),
					host, uptime))
			} else if !this.onlineOnly {
				this.Ui.Output(fmt.Sprintf("\t%s %35s/%-2s %12s -> %-12s %s %s",
					symbol,
					consumer.Topic, consumer.PartitionId,
					gofmt.Comma(consumer.ProducerOffset),
					gofmt.Comma(consumer.ConsumerOffset),
					lagOutput,
					gofmt.PrettySince(consumer.Mtime.Time())))
			}
		}
	}
}

func (*Lags) Synopsis() string {
	return "Display high level consumer group lags for a topic"
}

func (this *Lags) Help() string {
	help := fmt.Sprintf(`
Usage: %s lags [options]

    Display high level consumer group lags for a topic

    e,g.
    %s -z prod -c trade -t OrderStatusMsg

Options:

    -z zone
      Default %s

    -c cluster

    -g consumer group name pattern

    -t topic name pattern

`, this.Cmd, this.Cmd, ctx.ZkDefaultZone())
	return strings.TrimSpace(help)
}
<commit_msg>make 'fk lags' output more readable<commit_after>package command

import (
	"flag"
	"fmt"
	"sort"
	"strings"

	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/gofmt"
	"github.com/ryanuber/columnize"
)

type Lags struct {
	Ui  cli.Ui
	Cmd string

	onlineOnly      bool
	groupPattern    string
	topicPattern    string
	problematicMode bool
	lagThreshold    int
}

func (this *Lags) Run(args []string) (exitCode int) {
	const secondsInMinute = 60
	var (
		cluster string
		zone    string
	)
	cmdFlags := flag.NewFlagSet("lags", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.StringVar(&cluster, "c", "", "")
	cmdFlags.BoolVar(&this.onlineOnly, "l", false, "")
	cmdFlags.BoolVar(&this.problematicMode, "p", false, "")
	cmdFlags.StringVar(&this.groupPattern, "g", "", "")
	cmdFlags.StringVar(&this.topicPattern, "t", "", "")
	cmdFlags.IntVar(&this.lagThreshold, "lag", 5000, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if validateArgs(this, this.Ui).
		require("-c", "-t").
		invalid(args) {
		return 2
	}

	if this.problematicMode {
		this.onlineOnly = true
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
	zkcluster := zkzone.NewCluster(cluster) // panic if invalid cluster
	this.printConsumersLag(zkcluster)
	printSwallowedErrors(this.Ui, zkzone)

	return
}

func (this *Lags) printConsumersLag(zkcluster *zk.ZkCluster) {
	lines := make([]string, 0)
	header := "Group|Topic/Partition|Offet|UpTo|Lag|Refreshed"
	lines = append(lines, header)

	// sort by group name
	consumersByGroup := zkcluster.ConsumersByGroup(this.groupPattern)
	sortedGroups := make([]string, 0, len(consumersByGroup))
	for group, _ := range consumersByGroup {
		sortedGroups = append(sortedGroups, group)
	}
	sort.Strings(sortedGroups)

	for _, group := range sortedGroups {
		sortedTopicAndPartitionIds := make([]string, 0)
		consumers := make(map[string]zk.ConsumerMeta)
		for _, t := range consumersByGroup[group] {
			key := fmt.Sprintf("%s:%s", t.Topic, t.PartitionId)
			sortedTopicAndPartitionIds = append(sortedTopicAndPartitionIds, key)

			consumers[key] = t
		}
		sort.Strings(sortedTopicAndPartitionIds)

		for _, topicAndPartitionId := range sortedTopicAndPartitionIds {
			consumer := consumers[topicAndPartitionId]

			if !patternMatched(consumer.Topic, this.topicPattern) {
				continue
			}
			if !consumer.Online {
				continue
			}

			if consumer.Online {
				lines = append(lines,
					fmt.Sprintf("%s|%s/%s|%s|%s|%s|%s",
						group,
						consumer.Topic, consumer.PartitionId,
						gofmt.Comma(consumer.ProducerOffset),
						gofmt.Comma(consumer.ConsumerOffset),
						gofmt.Comma(consumer.Lag),
						gofmt.PrettySince(consumer.Mtime.Time()),
					))
			}
		}
	}

	this.Ui.Output(columnize.SimpleFormat(lines))
}

func (*Lags) Synopsis() string {
	return "Display online high level consumer group lags for a topic"
}

func (this *Lags) Help() string {
	help := fmt.Sprintf(`
Usage: %s lags [options]

    Display online high level consumer group lags for a topic

    e,g.
    %s -z prod -c trade -t OrderStatusMsg

Options:

    -z zone
      Default %s

    -c cluster

    -g consumer group name pattern

    -t topic name pattern

`, this.Cmd, this.Cmd, ctx.ZkDefaultZone())
	return strings.TrimSpace(help)
}
<|endoftext|>


<commit_before>package cmd

import (
	"os/exec"
	"testing"

	"github.com/acarl005/stripansi"
	"github.com/stretchr/testify/require"
)

func Test_issueShow(t *testing.T) {
	t.Parallel()
	repo := copyTestRepo(t)
	cmd := exec.Command(labBinaryPath, "issue", "show", "1", "--comments")
	cmd.Dir = repo

	b, err := cmd.CombinedOutput()
	if err != nil {
		t.Log(string(b))
		t.Error(err)
	}

	out := string(b)
	out = stripansi.Strip(out) // This is required because glamour adds a lot of ansi chars

	require.Contains(t, out, `
#1 test issue for lab list
===================================

-----------------------------------
Project: zaquestion/test
Status: Open
Assignees: zaquestion, lab-testing
Author: lab-testing
Milestone: 1.0
Due Date: 2018-01-01 00:00:00 +0000 UTC
Time Stats: Estimated 1w, Spent 1d
Labels: bug
WebURL: https://gitlab.com/zaquestion/test/-/issues/1
`)

	require.Contains(t, string(b), `commented at`)
}

func Test_issueShow_updated_comments(t *testing.T) {
	t.Parallel()
	repo := copyTestRepo(t)
	cmd := exec.Command(labBinaryPath, "issue", "show", "8", "--comments")
	cmd.Dir = repo

	b, err := cmd.CombinedOutput()
	if err != nil {
		t.Log(string(b))
		t.Error(err)
	}

	out := string(b)
	out = stripansi.Strip(out) // This is required because glamour adds a lot of ansi chars

	require.Contains(t, string(b), `updated comment at`)
}
<commit_msg>issue_show: fix test case<commit_after>package cmd

import (
	"os/exec"
	"testing"

	"github.com/acarl005/stripansi"
	"github.com/stretchr/testify/require"
)

func Test_issueShow(t *testing.T) {
	t.Parallel()
	repo := copyTestRepo(t)
	cmd := exec.Command(labBinaryPath, "issue", "show", "1", "--comments")
	cmd.Dir = repo

	b, err := cmd.CombinedOutput()
	if err != nil {
		t.Log(string(b))
		t.Error(err)
	}

	out := string(b)
	out = stripansi.Strip(out) // This is required because glamour adds a lot of ansi chars

	require.Contains(t, out, `
#1 test issue for lab list
===================================

-----------------------------------
Project: zaquestion/test
Status: Open
Assignees: zaquestion, lab-testing
Author: lab-testing
Milestone: 1.0
Due Date: 2018-01-01 00:00:00 +0000 UTC
Time Stats: Estimated 40h, Spent 8h
Labels: bug
WebURL: https://gitlab.com/zaquestion/test/-/issues/1
`)

	require.Contains(t, string(b), `commented at`)
}

func Test_issueShow_updated_comments(t *testing.T) {
	t.Parallel()
	repo := copyTestRepo(t)
	cmd := exec.Command(labBinaryPath, "issue", "show", "8", "--comments")
	cmd.Dir = repo

	b, err := cmd.CombinedOutput()
	if err != nil {
		t.Log(string(b))
		t.Error(err)
	}

	out := string(b)
	out = stripansi.Strip(out) // This is required because glamour adds a lot of ansi chars

	require.Contains(t, string(b), `updated comment at`)
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"github.com/spf13/cobra"

	"k8s.io/release/pkg/gcp/auth"
	"k8s.io/release/pkg/gcp/build"
	"k8s.io/release/pkg/git"
	"k8s.io/release/pkg/release"
	"k8s.io/release/pkg/util"
)

type gcbmgrOptions struct {
	stage        bool
	release      bool
	stream       bool
	branch       string
	releaseType  string
	buildVersion string
	gcpUser      string
}

var (
	gcbmgrOpts = &gcbmgrOptions{}
	buildOpts  = &build.Options{}

	// TODO: Commenting these packages/commands out since they fail in CI.
	//       These can be fixed by changing the CI test image to one that includes the packages.
	//nolint:gocritic
	/*
		requiredPackages = []string{
			"jq",
			"git",
			"bsdmainutils",
		}

		// TODO: Do we really need this if we use the Google Cloud SDK instead?
		requiredCommands = []string{
			"gsutil",
			"gcloud",
		}
	*/
)

// gcbmgrCmd is a krel subcommand which invokes runGcbmgr()
var gcbmgrCmd = &cobra.Command{
	Use:           "gcbmgr",
	Short:         "Run gcbmgr",
	SilenceUsage:  true,
	SilenceErrors: true,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runGcbmgr()
	},
}

func init() {
	// Submit types
	gcbmgrCmd.PersistentFlags().BoolVar(
		&gcbmgrOpts.stage,
		"stage",
		false,
		"Submit a stage run to GCB",
	)
	gcbmgrCmd.PersistentFlags().BoolVar(
		&gcbmgrOpts.release,
		"release",
		false,
		"Submit a release run to GCB",
	)

	gcbmgrCmd.PersistentFlags().StringVar(
		&gcbmgrOpts.branch,
		"branch",
		"",
		"Branch to run the specified GCB run against",
	)

	// Release types
	gcbmgrCmd.PersistentFlags().StringVar(
		&gcbmgrOpts.releaseType,
		"type",
		"prerelease",
		"Release type (must be one of: 'prerelease', 'rc', 'official')",
	)

	// TODO: Remove default once find_green_build logic exists
	gcbmgrCmd.PersistentFlags().StringVar(
		&gcbmgrOpts.buildVersion,
		"build-version",
		"",
		"Build version",
	)

	// gcloud options
	gcbmgrCmd.PersistentFlags().StringVar(
		&buildOpts.Project,
		"project",
		release.DefaultProject,
		"Branch to run the specified GCB run against",
	)
	gcbmgrCmd.PersistentFlags().BoolVar(
		&gcbmgrOpts.stream,
		"stream",
		false,
		"If specified, GCB will run synchronously, tailing its' logs to stdout",
	)
	gcbmgrCmd.PersistentFlags().StringVar(
		&buildOpts.CloudbuildFile,
		"gcb-config",
		build.DefaultCloudbuildFile,
		"If provided, this will be used as the name of the Google Cloud Build config file.",
	)
	gcbmgrCmd.PersistentFlags().StringVar(
		&gcbmgrOpts.gcpUser,
		"gcp-user",
		"",
		"If provided, this will be used as the GCP_USER_TAG.",
	)

	rootCmd.AddCommand(gcbmgrCmd)
}

// runGcbmgr is the function invoked by 'krel gcbmgr', responsible for submitting release jobs to GCB
func runGcbmgr() error {
	// TODO: Commenting these checks out since they fail in CI.
	//       These can be fixed by changing the CI test image to one that includes the packages.
	//nolint:gocritic
	/*
		logrus.Info("Checking for required packages...")
		pkgAvailable, pkgAvailableErr := util.PackagesAvailable(requiredPackages...)
		if pkgAvailableErr != nil {
			return pkgAvailableErr
		}
		if !pkgAvailable {
			return errors.New("packages required to run gcbmgr are not present; cannot continue")
		}

		logrus.Info("Checking for required commands...")
		if cmdAvailable := command.Available(requiredCommands...); !cmdAvailable {
			return errors.New("binaries required to run gcbmgr are not present; cannot continue")
		}
	*/

	// TODO: Add gitlib::repo_state check

	logrus.Infof("Running gcbmgr with the following options: %v", *gcbmgrOpts)
	logrus.Infof("Build options: %v", *buildOpts)

	if gcbmgrOpts.stage && gcbmgrOpts.release {
		return errors.New("cannot specify both the 'stage' and 'release' flag; resubmit with only one build type selected")
	}

	buildOpts.NoSource = true
	buildOpts.DiskSize = release.DefaultDiskSize

	buildOpts.Async = true

	if gcbmgrOpts.stream {
		buildOpts.Async = false
	}

	gcbSubs, gcbSubsErr := setGCBSubstitutions(gcbmgrOpts)
	if gcbSubs == nil || gcbSubsErr != nil {
		return gcbSubsErr
	}

	if rootOpts.nomock {
		// TODO: Consider a '--yes' flag so we can mock this
		_, nomockSubmit, askErr := util.Ask(
			"Really submit a --nomock release job against the $RELEASE_BRANCH branch?",
			"yes",
			3,
		)
		if askErr != nil {
			return askErr
		}

		if nomockSubmit {
			gcbSubs["NOMOCK_TAG"] = "nomock"
			gcbSubs["NOMOCK"] = fmt.Sprintf("--%s", gcbSubs["NOMOCK_TAG"])
		}
	} else {
		// TODO: Remove once cloudbuild.yaml doesn't strictly require vars to be set.
		gcbSubs["NOMOCK_TAG"] = ""
		gcbSubs["NOMOCK"] = ""

		bucketPrefix := release.BucketPrefix

		userBucket := fmt.Sprintf("%s%s", bucketPrefix, gcbSubs["GCP_USER_TAG"])
		userBucketSetErr := os.Setenv("USER_BUCKET", userBucket)
		if userBucketSetErr != nil {
			return userBucketSetErr
		}

		testBucket := fmt.Sprintf("%s%s", bucketPrefix, "gcb")
		testBucketSetErr := os.Setenv("BUCKET", testBucket)
		if testBucketSetErr != nil {
			return testBucketSetErr
		}
	}

	toolRoot, err := os.Getwd()
	if err != nil {
		return err
	}

	logrus.Info("Listing GCB substitutions prior to build submission...")
	for k, v := range gcbSubs {
		logrus.Infof("%s: %s", k, v)
	}

	var jobType string
	switch {
	// TODO: Consider a '--validate' flag to validate the GCB config without submitting
	case gcbmgrOpts.stage:
		jobType = "stage"
	case gcbmgrOpts.release:
		jobType = "release"
		buildOpts.DiskSize = "100"
	default:
		return listJobs()
	}

	buildOpts.ConfigDir = filepath.Join(toolRoot, "gcb", jobType)
	prepareBuildErr := build.PrepareBuilds(buildOpts)
	if prepareBuildErr != nil {
		return prepareBuildErr
	}

	// TODO: Need actual values
	var jobName, uploaded string

	version, err := git.GetTag()
	if err != nil {
		return err
	}

	return build.RunSingleJob(buildOpts, jobName, uploaded, version, gcbSubs)
}

// setGCBSubstitutions takes a set of gcbmgrOptions and returns a map of GCB substitutions
func setGCBSubstitutions(o *gcbmgrOptions) (map[string]string, error) {
	gcbSubs := map[string]string{}

	toolOrg := release.GetToolOrg()
	gcbSubs["TOOL_ORG"] = toolOrg

	toolRepo := release.GetToolRepo()
	gcbSubs["TOOL_REPO"] = toolRepo

	toolBranch := release.GetToolBranch()
	gcbSubs["TOOL_BRANCH"] = toolBranch

	gcpUser := o.gcpUser
	if gcpUser == "" {
		var gcpUserErr error
		gcpUser, gcpUserErr = auth.GetCurrentGCPUser()
		if gcpUserErr != nil {
			return gcbSubs, gcpUserErr
		}
	} else {
		// TODO: Consider removing this once the 'gcloud auth' is testable in CI
		gcpUser = auth.NormalizeGCPUser(gcpUser)
	}

	gcbSubs["GCP_USER_TAG"] = gcpUser

	// TODO: The naming for these env vars is clumsy/confusing, but we're bound by anago right now.
	releaseType := o.releaseType
	switch releaseType {
	case "official":
		gcbSubs["OFFICIAL_TAG"] = releaseType
		gcbSubs["OFFICIAL"] = fmt.Sprintf("--%s", releaseType)

		// TODO: Remove once cloudbuild.yaml doesn't strictly require vars to be set.
		gcbSubs["RC_TAG"] = ""
		gcbSubs["RC"] = ""
	case "rc":
		gcbSubs["RC_TAG"] = releaseType
		gcbSubs["RC"] = fmt.Sprintf("--%s", releaseType)

		// TODO: Remove once cloudbuild.yaml doesn't strictly require vars to be set.
		gcbSubs["OFFICIAL_TAG"] = ""
		gcbSubs["OFFICIAL"] = ""
	case "prerelease":
		// TODO: Remove once cloudbuild.yaml doesn't strictly require vars to be set.
		gcbSubs["OFFICIAL_TAG"] = ""
		gcbSubs["OFFICIAL"] = ""
		gcbSubs["RC_TAG"] = ""
		gcbSubs["RC"] = ""
	}

	branch := o.branch
	if branch == "" {
		return gcbSubs, errors.New("Release branch must be set to continue")
	}

	gcbSubs["RELEASE_BRANCH"] = branch

	if o.stage {
		// TODO: Remove once we remove support for --built-at-head.
		gcbSubs["BUILD_AT_HEAD"] = ""
	}

	buildVersion := o.buildVersion
	if buildVersion == "" {
		if o.release {
			return gcbSubs, errors.New("Build version must be specified when sending a release GCB run")
		}

		var versionErr error
		buildVersion, versionErr = release.GetKubeVersionForBranch(
			release.VersionTypeCILatest, o.branch,
		)
		if versionErr != nil {
			return gcbSubs, versionErr
		}
	}

	buildpoint := buildVersion
	buildpoint = strings.ReplaceAll(buildpoint, "+", "-")
	gcbSubs["BUILD_POINT"] = buildpoint

	// TODO: Add conditionals for find_green_build
	buildVersion = fmt.Sprintf("--buildversion=%s", buildVersion)
	gcbSubs["BUILDVERSION"] = buildVersion

	kubecrossBranches := []string{
		branch,
		"master",
	}

	kubecrossVersion, kubecrossVersionErr := release.GetKubecrossVersion(kubecrossBranches...)
	if kubecrossVersionErr != nil {
		return gcbSubs, kubecrossVersionErr
	}
	gcbSubs["KUBE_CROSS_VERSION"] = kubecrossVersion

	return gcbSubs, nil
}

// listJobs lists recent GCB jobs run in the specified project
func listJobs() error {
	logrus.Info("Listing GCB jobs is not currently supported.")

	// TODO: Add job listing logic
	// logrus.Info("Listing recent GCB jobs...")
	return nil
}
<commit_msg>gcbmr: missing replace the release branch in the confirmation msg<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"github.com/spf13/cobra"

	"k8s.io/release/pkg/gcp/auth"
	"k8s.io/release/pkg/gcp/build"
	"k8s.io/release/pkg/git"
	"k8s.io/release/pkg/release"
	"k8s.io/release/pkg/util"
)

type gcbmgrOptions struct {
	stage        bool
	release      bool
	stream       bool
	branch       string
	releaseType  string
	buildVersion string
	gcpUser      string
}

var (
	gcbmgrOpts = &gcbmgrOptions{}
	buildOpts  = &build.Options{}

	// TODO: Commenting these packages/commands out since they fail in CI.
	//       These can be fixed by changing the CI test image to one that includes the packages.
	//nolint:gocritic
	/*
		requiredPackages = []string{
			"jq",
			"git",
			"bsdmainutils",
		}

		// TODO: Do we really need this if we use the Google Cloud SDK instead?
		requiredCommands = []string{
			"gsutil",
			"gcloud",
		}
	*/
)

// gcbmgrCmd is a krel subcommand which invokes runGcbmgr()
var gcbmgrCmd = &cobra.Command{
	Use:           "gcbmgr",
	Short:         "Run gcbmgr",
	SilenceUsage:  true,
	SilenceErrors: true,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runGcbmgr()
	},
}

func init() {
	// Submit types
	gcbmgrCmd.PersistentFlags().BoolVar(
		&gcbmgrOpts.stage,
		"stage",
		false,
		"Submit a stage run to GCB",
	)
	gcbmgrCmd.PersistentFlags().BoolVar(
		&gcbmgrOpts.release,
		"release",
		false,
		"Submit a release run to GCB",
	)

	gcbmgrCmd.PersistentFlags().StringVar(
		&gcbmgrOpts.branch,
		"branch",
		"",
		"Branch to run the specified GCB run against",
	)

	// Release types
	gcbmgrCmd.PersistentFlags().StringVar(
		&gcbmgrOpts.releaseType,
		"type",
		"prerelease",
		"Release type (must be one of: 'prerelease', 'rc', 'official')",
	)

	// TODO: Remove default once find_green_build logic exists
	gcbmgrCmd.PersistentFlags().StringVar(
		&gcbmgrOpts.buildVersion,
		"build-version",
		"",
		"Build version",
	)

	// gcloud options
	gcbmgrCmd.PersistentFlags().StringVar(
		&buildOpts.Project,
		"project",
		release.DefaultProject,
		"Branch to run the specified GCB run against",
	)
	gcbmgrCmd.PersistentFlags().BoolVar(
		&gcbmgrOpts.stream,
		"stream",
		false,
		"If specified, GCB will run synchronously, tailing its' logs to stdout",
	)
	gcbmgrCmd.PersistentFlags().StringVar(
		&buildOpts.CloudbuildFile,
		"gcb-config",
		build.DefaultCloudbuildFile,
		"If provided, this will be used as the name of the Google Cloud Build config file.",
	)
	gcbmgrCmd.PersistentFlags().StringVar(
		&gcbmgrOpts.gcpUser,
		"gcp-user",
		"",
		"If provided, this will be used as the GCP_USER_TAG.",
	)

	rootCmd.AddCommand(gcbmgrCmd)
}

// runGcbmgr is the function invoked by 'krel gcbmgr', responsible for submitting release jobs to GCB
func runGcbmgr() error {
	// TODO: Commenting these checks out since they fail in CI.
	//       These can be fixed by changing the CI test image to one that includes the packages.
	//nolint:gocritic
	/*
		logrus.Info("Checking for required packages...")
		pkgAvailable, pkgAvailableErr := util.PackagesAvailable(requiredPackages...)
		if pkgAvailableErr != nil {
			return pkgAvailableErr
		}
		if !pkgAvailable {
			return errors.New("packages required to run gcbmgr are not present; cannot continue")
		}

		logrus.Info("Checking for required commands...")
		if cmdAvailable := command.Available(requiredCommands...); !cmdAvailable {
			return errors.New("binaries required to run gcbmgr are not present; cannot continue")
		}
	*/

	// TODO: Add gitlib::repo_state check

	logrus.Infof("Running gcbmgr with the following options: %v", *gcbmgrOpts)
	logrus.Infof("Build options: %v", *buildOpts)

	if gcbmgrOpts.stage && gcbmgrOpts.release {
		return errors.New("cannot specify both the 'stage' and 'release' flag; resubmit with only one build type selected")
	}

	buildOpts.NoSource = true
	buildOpts.DiskSize = release.DefaultDiskSize

	buildOpts.Async = true

	if gcbmgrOpts.stream {
		buildOpts.Async = false
	}

	gcbSubs, gcbSubsErr := setGCBSubstitutions(gcbmgrOpts)
	if gcbSubs == nil || gcbSubsErr != nil {
		return gcbSubsErr
	}

	if rootOpts.nomock {
		// TODO: Consider a '--yes' flag so we can mock this
		_, nomockSubmit, askErr := util.Ask(
			fmt.Sprintf("Really submit a --nomock release job against the %s branch?", gcbmgrOpts.branch),
			"yes",
			3,
		)
		if askErr != nil {
			return askErr
		}

		if nomockSubmit {
			gcbSubs["NOMOCK_TAG"] = "nomock"
			gcbSubs["NOMOCK"] = fmt.Sprintf("--%s", gcbSubs["NOMOCK_TAG"])
		}
	} else {
		// TODO: Remove once cloudbuild.yaml doesn't strictly require vars to be set.
		gcbSubs["NOMOCK_TAG"] = ""
		gcbSubs["NOMOCK"] = ""

		bucketPrefix := release.BucketPrefix

		userBucket := fmt.Sprintf("%s%s", bucketPrefix, gcbSubs["GCP_USER_TAG"])
		userBucketSetErr := os.Setenv("USER_BUCKET", userBucket)
		if userBucketSetErr != nil {
			return userBucketSetErr
		}

		testBucket := fmt.Sprintf("%s%s", bucketPrefix, "gcb")
		testBucketSetErr := os.Setenv("BUCKET", testBucket)
		if testBucketSetErr != nil {
			return testBucketSetErr
		}
	}

	toolRoot, err := os.Getwd()
	if err != nil {
		return err
	}

	logrus.Info("Listing GCB substitutions prior to build submission...")
	for k, v := range gcbSubs {
		logrus.Infof("%s: %s", k, v)
	}

	var jobType string
	switch {
	// TODO: Consider a '--validate' flag to validate the GCB config without submitting
	case gcbmgrOpts.stage:
		jobType = "stage"
	case gcbmgrOpts.release:
		jobType = "release"
		buildOpts.DiskSize = "100"
	default:
		return listJobs()
	}

	buildOpts.ConfigDir = filepath.Join(toolRoot, "gcb", jobType)
	prepareBuildErr := build.PrepareBuilds(buildOpts)
	if prepareBuildErr != nil {
		return prepareBuildErr
	}

	// TODO: Need actual values
	var jobName, uploaded string

	version, err := git.GetTag()
	if err != nil {
		return err
	}

	return build.RunSingleJob(buildOpts, jobName, uploaded, version, gcbSubs)
}

// setGCBSubstitutions takes a set of gcbmgrOptions and returns a map of GCB substitutions
func setGCBSubstitutions(o *gcbmgrOptions) (map[string]string, error) {
	gcbSubs := map[string]string{}

	toolOrg := release.GetToolOrg()
	gcbSubs["TOOL_ORG"] = toolOrg

	toolRepo := release.GetToolRepo()
	gcbSubs["TOOL_REPO"] = toolRepo

	toolBranch := release.GetToolBranch()
	gcbSubs["TOOL_BRANCH"] = toolBranch

	gcpUser := o.gcpUser
	if gcpUser == "" {
		var gcpUserErr error
		gcpUser, gcpUserErr = auth.GetCurrentGCPUser()
		if gcpUserErr != nil {
			return gcbSubs, gcpUserErr
		}
	} else {
		// TODO: Consider removing this once the 'gcloud auth' is testable in CI
		gcpUser = auth.NormalizeGCPUser(gcpUser)
	}

	gcbSubs["GCP_USER_TAG"] = gcpUser

	// TODO: The naming for these env vars is clumsy/confusing, but we're bound by anago right now.
	releaseType := o.releaseType
	switch releaseType {
	case "official":
		gcbSubs["OFFICIAL_TAG"] = releaseType
		gcbSubs["OFFICIAL"] = fmt.Sprintf("--%s", releaseType)

		// TODO: Remove once cloudbuild.yaml doesn't strictly require vars to be set.
		gcbSubs["RC_TAG"] = ""
		gcbSubs["RC"] = ""
	case "rc":
		gcbSubs["RC_TAG"] = releaseType
		gcbSubs["RC"] = fmt.Sprintf("--%s", releaseType)

		// TODO: Remove once cloudbuild.yaml doesn't strictly require vars to be set.
		gcbSubs["OFFICIAL_TAG"] = ""
		gcbSubs["OFFICIAL"] = ""
	case "prerelease":
		// TODO: Remove once cloudbuild.yaml doesn't strictly require vars to be set.
		gcbSubs["OFFICIAL_TAG"] = ""
		gcbSubs["OFFICIAL"] = ""
		gcbSubs["RC_TAG"] = ""
		gcbSubs["RC"] = ""
	}

	branch := o.branch
	if branch == "" {
		return gcbSubs, errors.New("Release branch must be set to continue")
	}

	gcbSubs["RELEASE_BRANCH"] = branch

	if o.stage {
		// TODO: Remove once we remove support for --built-at-head.
		gcbSubs["BUILD_AT_HEAD"] = ""
	}

	buildVersion := o.buildVersion
	if buildVersion == "" {
		if o.release {
			return gcbSubs, errors.New("Build version must be specified when sending a release GCB run")
		}

		var versionErr error
		buildVersion, versionErr = release.GetKubeVersionForBranch(
			release.VersionTypeCILatest, o.branch,
		)
		if versionErr != nil {
			return gcbSubs, versionErr
		}
	}

	buildpoint := buildVersion
	buildpoint = strings.ReplaceAll(buildpoint, "+", "-")
	gcbSubs["BUILD_POINT"] = buildpoint

	// TODO: Add conditionals for find_green_build
	buildVersion = fmt.Sprintf("--buildversion=%s", buildVersion)
	gcbSubs["BUILDVERSION"] = buildVersion

	kubecrossBranches := []string{
		branch,
		"master",
	}

	kubecrossVersion, kubecrossVersionErr := release.GetKubecrossVersion(kubecrossBranches...)
	if kubecrossVersionErr != nil {
		return gcbSubs, kubecrossVersionErr
	}
	gcbSubs["KUBE_CROSS_VERSION"] = kubecrossVersion

	return gcbSubs, nil
}

// listJobs lists recent GCB jobs run in the specified project
func listJobs() error {
	logrus.Info("Listing GCB jobs is not currently supported.")

	// TODO: Add job listing logic
	// logrus.Info("Listing recent GCB jobs...")
	return nil
}
<|endoftext|>


<commit_before>package runfile

import (
	"fmt"
	"github.com/codegangsta/cli"
	"os"
	"pfi/sensorbee/sensorbee/server/config"
)

// SetUp sets up a command for running single BQL file.
func SetUp() cli.Command {
	cmd := cli.Command{
		Name:        "runfile",
		Usage:       "run a BQL file",
		Description: "runfile command runs a BQL file",
		Action:      Run,
	}

	cmd.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "config, c",
			Value:  "",
			Usage:  "file path of a config file in YAML format (only logging and storage sections are used)",
			EnvVar: "SENSORBEE_CONFIG",
		},
	}
	return cmd
}

func Run(c *cli.Context) {
	// TODO: Merge this implementation with cmd/run

	conf, err := func() (*config.Config, error) {
		if c.IsSet("config") {
			p := c.String("config")
			in, err := ioutil.ReadFile(p)
			if err != nil {
				return nil, fmt.Errorf("Cannot read the config file %v: %v\n", p, err)
			}

			var yml map[string]interface{}
			if err := yaml.Unmarshal(in, &yml); err != nil {
				return nil, fmt.Errorf("Cannot parse the config file %v: %v\n", p, err)
			}
			m, err := data.NewMap(yml)
			if err != nil {
				return nil, fmt.Errorf("The config file %v has invalid values: %v\n", p, err)
			}
			return config.New(m)

		} else {
			// Currently there's no required parameters. However, when a required
			// parameter is added to Config, remove this else block and add a
			// default value like "/etc/sensorbee/config.yaml" to the config option.
			return config.New(data.Map{})
		}
	}()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
	}

	// TODO: set up logger
	// TODO: set up storage
	// TODO: parse statements
	// TODO: bql.TopologyBuilder
	// TODO: AddStmt: Pause all sources, and resume them after build the topology
	// TODO: Wait until all sources stops
	// TODO: call core.Topology.Stop
}
<commit_msg>add UDS strage set up and building topology<commit_after>package runfile

import (
	"fmt"
	"github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"io/ioutil"
	"os"
	"pfi/sensorbee/sensorbee/bql"
	"pfi/sensorbee/sensorbee/bql/parser"
	"pfi/sensorbee/sensorbee/core"
	"pfi/sensorbee/sensorbee/server/config"
)

// SetUp sets up a command for running single BQL file.
func SetUp() cli.Command {
	cmd := cli.Command{
		Name:        "runfile",
		Usage:       "run a BQL file",
		Description: "runfile command runs a BQL file",
		Action:      Run,
	}

	cmd.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "config, c",
			Value:  "",
			Usage:  "file path of a config file in YAML format (only logging and storage sections are used)",
			EnvVar: "SENSORBEE_CONFIG",
		},
	}
	return cmd
}

func Run(c *cli.Context) {
	// TODO: Merge this implementation with cmd/run

	if len(c.Args()) != 1 {
		cli.ShowSubcommandHelp(c)
		os.Exit(1)
	}

	conf, err := func() (*config.Config, error) {
		if c.IsSet("config") {
			p := c.String("config")
			in, err := ioutil.ReadFile(p)
			if err != nil {
				return nil, fmt.Errorf("Cannot read the config file %v: %v\n", p, err)
			}

			var yml map[string]interface{}
			if err := yaml.Unmarshal(in, &yml); err != nil {
				return nil, fmt.Errorf("Cannot parse the config file %v: %v\n", p, err)
			}
			m, err := data.NewMap(yml)
			if err != nil {
				return nil, fmt.Errorf("The config file %v has invalid values: %v\n", p, err)
			}
			return config.New(m)

		} else {
			// Currently there's no required parameters. However, when a required
			// parameter is added to Config, remove this else block and add a
			// default value like "/etc/sensorbee/config.yaml" to the config option.
			return config.New(data.Map{})
		}
	}()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	w, err := conf.Logging.CreateWriter()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	logger := logrus.New()
	logger.Out = w

	udsStorage, err := setUpUDSStorage(conf.Storage.UDS)
	if err != nil {
		logger.WithField("err", err).Error("Cannot set up a storage for UDSs")
		os.Exit(1)
	}

	logrus.Info("Setting up a topology")
	bqlFile := c.Args()[0]
	tb, err := setUpTopology(bqlFile)
	if err != nil {
		logger.WithFields(logrus.Fields{
			"err":      err,
			"bql_file": bqlFile,
		}).Error("Cannot set up the topology")
	}
	defer func() {
		logrus.Info("Stopping the topology")
		if err := tb.Topology().Stop(); err != nil {
			logrus.WithField("err", err).Error("Cannot stop the topology")
		}
	}()

	logrus.Info("Starting the topology")
	for name, s := range tb.Topology().Sources() {
		if err := s.Resume(); err != nil {
			logrus.WithFields(logrus.Fields{
				"err", err,
				"source": name,
			}).Error("Cannot resume the source")
			return
		}
	}

	for _, s := range tb.Topology().Sources() {
		// TODO: error check if necessary
		s.State().Wait(core.TSStopped)
	}
	logrus.Info("All sources has been stopped.")

}

func setUpUDSStorage(conf *config.UDSStorage) (udf.UDSStorage, error) {
	// TODO: merge this implementation with server/context.go
	// Parameters are already validated in conf
	switch conf.Type {
	case "in_memory":
		return udf.NewInMemoryUDSStorage(), nil
	case "fs":
		dir, _ := data.AsString(conf.Params["dir"])
		var tempDir string
		if v, ok := conf.Params["temp_dir"]; ok {
			tempDir, _ = data.AsString(v)
		}
		return udsstorage.NewFS(dir, tempDir)
	default:
		return nil, fmt.Errorf("unsupported uds storage type: %v", conf.Type)
	}
}

func setUpTopology(logger *logrus.Logger, conf *config.Config, bqlFile string) (*bql.TopologyBuilder, error) {
	queries, err := func() (string, error) {
		f, err := os.Open(bqlFile)
		if err != nil {
			return err
		}
		defer f.Close()
		b, err := ioutil.ReadAll(f)
		if err != nil {
			return "", err
		}
		return string(b), nil
	}()
	if err != nil {
		return err
	}

	bp := parser.New()
	// TODO: provide better parse error reporting using ParseStmt instead of ParseStmts
	stmts, err := bp.ParseStmts(string(queries))
	if err != nil {
		return nil, err
	}

	// TODO: create context using conf and setting the logger
	// TODO: create core.Topology
	// TODO: create bql.TopologyBuilder

	for _, stmt := range stmts {
		// TODO: if stmt is CREATE SOURCE, create it with PAUSED
		if _, err := tb.AddStmt(stmt); err != nil {
			logger.WithFields(logrus.Fields{
				"err":      err,
				"topology": name,
				"stmt":     stmt,
			}).Error("Cannot add a statement to the topology")
			return nil, err
		}
	}
	// TODO: return bql.TopologyBuilder, nil
}
<|endoftext|>