<commit_before>package snapshot

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/desal/cmd"
	"github.com/desal/dsutil"
	"github.com/desal/git"
	"github.com/desal/gocmd"
)

type empty struct{}
type set map[string]empty

type Context struct {
	startPkg string
	doneDirs set
	output   cmd.Output
	goPath   []string
	goCtx    *gocmd.Context
	gitCtx   *git.Context
}

type DepsFile struct {
	Deps     []PkgDep
	TestDeps []PkgDep
}

type PkgDep struct {
	ImportPath string
	GitRemote  string //Blank for standard packages
	SHA        string //Blank for standard packages
}

func NewContext(output cmd.Output, goPath []string) *Context {
	return &Context{
		doneDirs: set{},
		output:   output,
		goPath:   goPath,
		goCtx:    gocmd.New(output, goPath),
		gitCtx:   git.New(output),
	}
}

func pkgContains(parent, child string) bool {
	if parent == child || strings.HasPrefix(child, parent+"/") {
		return true
	}
	return false
}

func (c *Context) scanDeps(startingList map[string]map[string]interface{}, workingDir string, deps set) []PkgDep {
	result := []PkgDep{}
outer:
	for importPath, _ := range deps {
		if c.goCtx.IsStdLib(importPath) {
			continue
		}

		if _, isStartingPkg := startingList[importPath]; isStartingPkg {
			continue
		}

		list, _ := c.goCtx.List(workingDir, importPath)

		cmdCtx := cmd.NewContext(".", c.output, cmd.Must)
		dir, _ := dsutil.SanitisePath(cmdCtx, list[importPath]["Dir"].(string))

		if !strings.HasSuffix(dir, importPath) {
			c.output.Error("Package import path directory mismatch (path sanity failure) %s should end in %s", dir, importPath)
			os.Exit(1)
		}

		if _, done := c.doneDirs[dir]; done {
			continue
		}

		for doneDir, _ := range c.doneDirs {
			if strings.HasPrefix(dir, doneDir+"/") {
				continue outer
			}
		}

		if !c.gitCtx.IsGit(dir) {
			c.output.Error("Import %s (%s) is not a git repository", importPath, dir)
			os.Exit(1)
		}

		status, _ := c.gitCtx.Status(dir, true)
		if status != git.Clean {
			c.output.Error("Import %s (%s) has git status %s", importPath, dir, status.String())
			os.Exit(1)
		}

		topLevel, _ := c.gitCtx.TopLevel(dir, true)
		//		charsToChop := len(dir) - len(topLevel)
		rootImportPath := importPath[:len(importPath)+len(topLevel)-len(dir)]

		c.doneDirs[topLevel] = empty{}

		remoteOriginUrl, _ := c.gitCtx.RemoteOriginUrl(dir, true)
		SHA, _ := c.gitCtx.SHA(dir, true)

		result = append(result, PkgDep{rootImportPath, remoteOriginUrl, SHA})
	}
	return result
}

//pkg string should be a space delimited list of packages including all subfolders
//typically ./...
func (c *Context) Snapshot(workingDir, pkgString string) DepsFile {
	list, _ := c.goCtx.List(workingDir, pkgString)

	allTestImports := set{}
	regDeps := set{}

	for _, e := range list {

		if testImportsInt, ok := e["TestImports"]; ok {
			testImports := testImportsInt.([]interface{})
			for _, testImport := range testImports {
				allTestImports[testImport.(string)] = empty{}
			}
		}

		if depsInt, ok := e["Deps"]; ok {
			deps := depsInt.([]interface{})
			for _, dep := range deps {
				regDeps[dep.(string)] = empty{}
			}
		}
	}

	allTestImportList := []string{}
	for i, _ := range allTestImports {
		allTestImportList = append(allTestImportList, i)
	}

	testDeps := set{}
	if len(allTestImportList) > 0 {
		testList, _ := c.goCtx.List(workingDir, strings.Join(allTestImportList, " "))
		for _, e := range testList {
			if depsInt, ok := e["Deps"]; ok {
				deps := depsInt.([]interface{})
				for _, dep := range deps {
					if _, isRegDep := regDeps[dep.(string)]; isRegDep {
						continue
					}
					testDeps[dep.(string)] = empty{}
				}
			}
		}
	}

	depsScan := c.scanDeps(list, workingDir, regDeps)
	testDepsScan := c.scanDeps(list, workingDir, testDeps)

	return DepsFile{Deps: depsScan, TestDeps: testDepsScan}

}

func (c *Context) reproduceDep(pkgDep PkgDep) {
	dir := filepath.Join(c.goPath[0], "src", pkgDep.ImportPath)
	if dsutil.CheckPath(dir) {
		c.output.Error("Package '%s' can't be cloned to '%s', path already exists.", pkgDep.ImportPath, dir)
		os.Exit(1)
	}

	c.gitCtx.Clone(dir, pkgDep.GitRemote, true)
	sha, _ := c.gitCtx.SHA(dir, true)
	if sha != pkgDep.SHA {
		c.gitCtx.Checkout(dir, pkgDep.SHA, true)
	}
}

func (c *Context) Reproduce(workingDir string, depsFile DepsFile, doTests bool) {
	for _, pkgDep := range depsFile.Deps {
		c.reproduceDep(pkgDep)
	}
	if doTests {
		for _, pkgDep := range depsFile.TestDeps {
			c.reproduceDep(pkgDep)
		}
	}
}

func ReadJson(filename string) (DepsFile, error) {
	var result DepsFile

	file, err := ioutil.ReadFile(filename)
	if err != nil {
		return result, err
	}

	json.Unmarshal(file, &result)
	return result, nil
}

func WriteJson(filename string, depsFile DepsFile) error {
	jsonOutput, err := json.MarshalIndent(&depsFile, "", "  ")
	if err != nil {
		return err
	}

	return ioutil.WriteFile(filename, jsonOutput, 0644)
}
<commit_msg>Added XTestImports<commit_after>package snapshot

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/desal/cmd"
	"github.com/desal/dsutil"
	"github.com/desal/git"
	"github.com/desal/gocmd"
)

type empty struct{}
type set map[string]empty

type Context struct {
	startPkg string
	doneDirs set
	output   cmd.Output
	goPath   []string
	goCtx    *gocmd.Context
	gitCtx   *git.Context
}

type DepsFile struct {
	Deps     []PkgDep
	TestDeps []PkgDep
}

type PkgDep struct {
	ImportPath string
	GitRemote  string //Blank for standard packages
	SHA        string //Blank for standard packages
}

func NewContext(output cmd.Output, goPath []string) *Context {
	return &Context{
		doneDirs: set{},
		output:   output,
		goPath:   goPath,
		goCtx:    gocmd.New(output, goPath),
		gitCtx:   git.New(output),
	}
}

func pkgContains(parent, child string) bool {
	if parent == child || strings.HasPrefix(child, parent+"/") {
		return true
	}
	return false
}

func (c *Context) scanDeps(startingList map[string]map[string]interface{}, workingDir string, deps set) []PkgDep {
	result := []PkgDep{}
outer:
	for importPath, _ := range deps {
		if c.goCtx.IsStdLib(importPath) {
			continue
		}

		if _, isStartingPkg := startingList[importPath]; isStartingPkg {
			continue
		}

		list, _ := c.goCtx.List(workingDir, importPath)

		cmdCtx := cmd.NewContext(".", c.output, cmd.Must)
		dir, _ := dsutil.SanitisePath(cmdCtx, list[importPath]["Dir"].(string))

		if !strings.HasSuffix(dir, importPath) {
			c.output.Error("Package import path directory mismatch (path sanity failure) %s should end in %s", dir, importPath)
			os.Exit(1)
		}

		if _, done := c.doneDirs[dir]; done {
			continue
		}

		for doneDir, _ := range c.doneDirs {
			if strings.HasPrefix(dir, doneDir+"/") {
				continue outer
			}
		}

		if !c.gitCtx.IsGit(dir) {
			c.output.Error("Import %s (%s) is not a git repository", importPath, dir)
			os.Exit(1)
		}

		status, _ := c.gitCtx.Status(dir, true)
		if status != git.Clean {
			c.output.Error("Import %s (%s) has git status %s", importPath, dir, status.String())
			os.Exit(1)
		}

		topLevel, _ := c.gitCtx.TopLevel(dir, true)
		//		charsToChop := len(dir) - len(topLevel)
		rootImportPath := importPath[:len(importPath)+len(topLevel)-len(dir)]

		c.doneDirs[topLevel] = empty{}

		remoteOriginUrl, _ := c.gitCtx.RemoteOriginUrl(dir, true)
		SHA, _ := c.gitCtx.SHA(dir, true)

		result = append(result, PkgDep{rootImportPath, remoteOriginUrl, SHA})
	}
	return result
}

//pkg string should be a space delimited list of packages including all subfolders
//typically ./...
func (c *Context) Snapshot(workingDir, pkgString string) DepsFile {
	list, _ := c.goCtx.List(workingDir, pkgString)

	allTestImports := set{}
	regDeps := set{}

	for _, e := range list {

		if testImportsInt, ok := e["TestImports"]; ok {
			testImports := testImportsInt.([]interface{})
			for _, testImport := range testImports {
				allTestImports[testImport.(string)] = empty{}
			}
		}

		if xTestImportsInt, ok := e["XTestImports"]; ok {
			xTestImports := xTestImportsInt.([]interface{})
			for _, testImport := range xTestImports {
				allTestImports[testImport.(string)] = empty{}
			}
		}

		if depsInt, ok := e["Deps"]; ok {
			deps := depsInt.([]interface{})
			for _, dep := range deps {
				regDeps[dep.(string)] = empty{}
			}
		}
	}

	allTestImportList := []string{}
	for i, _ := range allTestImports {
		allTestImportList = append(allTestImportList, i)
	}

	testDeps := set{}
	if len(allTestImportList) > 0 {
		testList, _ := c.goCtx.List(workingDir, strings.Join(allTestImportList, " "))
		for _, e := range testList {
			if depsInt, ok := e["Deps"]; ok {
				deps := depsInt.([]interface{})
				for _, dep := range deps {
					if _, isRegDep := regDeps[dep.(string)]; isRegDep {
						continue
					}
					testDeps[dep.(string)] = empty{}
				}
			}
		}
	}

	depsScan := c.scanDeps(list, workingDir, regDeps)
	testDepsScan := c.scanDeps(list, workingDir, testDeps)

	return DepsFile{Deps: depsScan, TestDeps: testDepsScan}

}

func (c *Context) reproduceDep(pkgDep PkgDep) {
	dir := filepath.Join(c.goPath[0], "src", pkgDep.ImportPath)
	if dsutil.CheckPath(dir) {
		c.output.Error("Package '%s' can't be cloned to '%s', path already exists.", pkgDep.ImportPath, dir)
		os.Exit(1)
	}

	c.gitCtx.Clone(dir, pkgDep.GitRemote, true)
	sha, _ := c.gitCtx.SHA(dir, true)
	if sha != pkgDep.SHA {
		c.gitCtx.Checkout(dir, pkgDep.SHA, true)
	}
}

func (c *Context) Reproduce(workingDir string, depsFile DepsFile, doTests bool) {
	for _, pkgDep := range depsFile.Deps {
		c.reproduceDep(pkgDep)
	}
	if doTests {
		for _, pkgDep := range depsFile.TestDeps {
			c.reproduceDep(pkgDep)
		}
	}
}

func ReadJson(filename string) (DepsFile, error) {
	var result DepsFile

	file, err := ioutil.ReadFile(filename)
	if err != nil {
		return result, err
	}

	json.Unmarshal(file, &result)
	return result, nil
}

func WriteJson(filename string, depsFile DepsFile) error {
	jsonOutput, err := json.MarshalIndent(&depsFile, "", "  ")
	if err != nil {
		return err
	}

	return ioutil.WriteFile(filename, jsonOutput, 0644)
}
<|endoftext|>


<commit_before>package source

import (
	"errors"
	"io"
	"os"
	"path"
	"path/filepath"
	"strings"
)

type Input interface {
	Files() []*File
}

type File struct {
	name        string
	LogicalName string
	Contents    io.Reader
	Section     string
	Dir         string
}

type Filesystem struct {
	files      []*File
	Base       string
	AvoidPaths []string
}

func (f *Filesystem) Files() []*File {
	if len(f.files) < 1 {
		f.captureFiles()
	}
	return f.files
}

var errMissingBaseDir = errors.New("source: missing base directory")

func (f *Filesystem) add(name string, reader io.Reader) (err error) {

	if name, err = f.getRelativePath(name); err != nil {
		return err
	}

	// section should be the first part of the path
	dir, logical := path.Split(name)
	parts := strings.Split(dir, "/")
	section := parts[0]

	if section == "." {
		section = ""
	}

	f.files = append(f.files, &File{
		name:        name,
		LogicalName: logical,
		Contents:    reader,
		Section:     section,
		Dir:         dir,
	})

	return
}

func (f *Filesystem) getRelativePath(name string) (final string, err error) {
	if filepath.IsAbs(name) && f.Base == "" {
		return "", errMissingBaseDir
	}
	name = filepath.Clean(name)
	base := filepath.Clean(f.Base)

	name, err = filepath.Rel(base, name)
	if err != nil {
		return "", err
	}
	name = filepath.ToSlash(name)
	return name, nil
}

func (f *Filesystem) captureFiles() {

	walker := func(filePath string, fi os.FileInfo, err error) error {
		if err != nil {
			return nil
		}

		if fi.IsDir() {
			if f.avoid(filePath) {
				return filepath.SkipDir
			}
			return nil
		} else {
			if ignoreDotFile(filePath) {
				return nil
			}
			file, err := os.Open(filePath)
			if err != nil {
				return err
			}
			f.add(filePath, file)
			return nil
		}
	}

	filepath.Walk(f.Base, walker)
}

func (f *Filesystem) avoid(filePath string) bool {
	for _, avoid := range f.AvoidPaths {
		if avoid == filePath {
			return true
		}
	}
	return false
}

func ignoreDotFile(filePath string) bool {
	return filepath.Base(filePath)[0] == '.'
}
<commit_msg>Avoid locking the files for an extended amount of time. Sublime Text doesn't like this and shows an error when modifying a file in rapid succession.<commit_after>package source

import (
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
)

type Input interface {
	Files() []*File
}

type File struct {
	name        string
	LogicalName string
	Contents    io.Reader
	Section     string
	Dir         string
}

type Filesystem struct {
	files      []*File
	Base       string
	AvoidPaths []string
}

func (f *Filesystem) Files() []*File {
	if len(f.files) < 1 {
		f.captureFiles()
	}
	return f.files
}

var errMissingBaseDir = errors.New("source: missing base directory")

func (f *Filesystem) add(name string, reader io.Reader) (err error) {

	if name, err = f.getRelativePath(name); err != nil {
		return err
	}

	// section should be the first part of the path
	dir, logical := path.Split(name)
	parts := strings.Split(dir, "/")
	section := parts[0]

	if section == "." {
		section = ""
	}

	f.files = append(f.files, &File{
		name:        name,
		LogicalName: logical,
		Contents:    reader,
		Section:     section,
		Dir:         dir,
	})

	return
}

func (f *Filesystem) getRelativePath(name string) (final string, err error) {
	if filepath.IsAbs(name) && f.Base == "" {
		return "", errMissingBaseDir
	}
	name = filepath.Clean(name)
	base := filepath.Clean(f.Base)

	name, err = filepath.Rel(base, name)
	if err != nil {
		return "", err
	}
	name = filepath.ToSlash(name)
	return name, nil
}

func (f *Filesystem) captureFiles() {

	walker := func(filePath string, fi os.FileInfo, err error) error {
		if err != nil {
			return nil
		}

		if fi.IsDir() {
			if f.avoid(filePath) {
				return filepath.SkipDir
			}
			return nil
		} else {
			if ignoreDotFile(filePath) {
				return nil
			}
			data, err := ioutil.ReadFile(filePath)
			if err != nil {
				return err
			}
			f.add(filePath, bytes.NewBuffer(data))
			return nil
		}
	}

	filepath.Walk(f.Base, walker)
}

func (f *Filesystem) avoid(filePath string) bool {
	for _, avoid := range f.AvoidPaths {
		if avoid == filePath {
			return true
		}
	}
	return false
}

func ignoreDotFile(filePath string) bool {
	return filepath.Base(filePath)[0] == '.'
}
<|endoftext|>


<commit_before>package volume

import (
	"errors"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Volume Promise", func() {
	var (
		promise Promise
	)

	BeforeEach(func() {
		promise = NewPromise()
	})

	Context("newly created", func() {
		It("is pending", func() {
			Expect(promise.IsPending()).To(BeTrue())
		})

		It("can not return a value yet", func() {
			_, _, err := promise.GetValue()

			Expect(err).To(Equal(ErrPromiseStillPending))
		})
	})

	Context("when fulfilled", func() {
		It("is not pending", func() {
			promise.Fulfill(Volume{})

			Expect(promise.IsPending()).To(BeFalse())
		})

		It("returns a non-empty volume in value", func() {
			volume := Volume{
				Handle:     "test-handle",
				Path:       "test-path",
				Properties: make(Properties),
			}

			promise.Fulfill(volume)

			val, _, _ := promise.GetValue()

			Expect(val).To(Equal(volume))
		})

		It("returns a nil error in value", func() {
			volume := Volume{
				Handle:     "test-handle",
				Path:       "test-path",
				Properties: make(Properties),
			}

			promise.Fulfill(volume)

			_, val, _ := promise.GetValue()

			Expect(val).To(BeNil())
		})

		It("can return a value", func() {
			volume := Volume{
				Handle:     "test-handle",
				Path:       "test-path",
				Properties: make(Properties),
			}

			promise.Fulfill(volume)

			_, _, err := promise.GetValue()

			Expect(err).To(BeNil())
		})
	})

	Context("when rejected", func() {
		var (
			testErr = errors.New("test-error")
		)

		It("is not pending", func() {
			promise.Reject(testErr)

			Expect(promise.IsPending()).To(BeFalse())
		})

		It("returns an empty volume in value", func() {
			promise.Reject(testErr)

			val, _, _ := promise.GetValue()

			Expect(val).To(Equal(Volume{}))
		})

		It("returns a non-nil error in value", func() {
			promise.Reject(testErr)

			_, val, _ := promise.GetValue()

			Expect(val).To(Equal(testErr))
		})

		It("can return a value", func() {
			promise.Reject(testErr)

			_, _, err := promise.GetValue()

			Expect(err).To(BeNil())
		})

		Context("when rejecting again", func() {
			Context("when canceled", func() {
				It("returns ErrPromiseCanceled", func() {
					promise.Reject(ErrPromiseCanceled)

					err := promise.Reject(testErr)

					Expect(err).To(Equal(ErrPromiseCanceled))
				})
			})

			Context("when not canceled", func() {
				It("returns ErrPromiseNotPending", func() {
					promise.Reject(testErr)

					err := promise.Reject(testErr)

					Expect(err).To(Equal(ErrPromiseNotPending))
				})
			})
		})
	})
})
<commit_msg>Fix handling of canceled VolumePromises<commit_after>package volume

import (
	"errors"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Volume Promise", func() {
	var (
		promise    Promise
		testErr    = errors.New("some-error")
		testVolume = Volume{
			Handle:     "some-handle",
			Path:       "some-path",
			Properties: make(Properties),
		}
	)

	BeforeEach(func() {
		promise = NewPromise()
	})

	Context("newly created", func() {
		It("is pending", func() {
			Expect(promise.IsPending()).To(BeTrue())
		})

		It("can not return a value yet", func() {
			_, _, err := promise.GetValue()

			Expect(err).To(Equal(ErrPromiseStillPending))
		})
	})

	Context("when fulfilled", func() {
		It("is not pending", func() {
			promise.Fulfill(testVolume)

			Expect(promise.IsPending()).To(BeFalse())
		})

		It("returns a non-empty volume in value", func() {
			promise.Fulfill(testVolume)

			val, _, _ := promise.GetValue()

			Expect(val).To(Equal(testVolume))
		})

		It("returns a nil error in value", func() {
			promise.Fulfill(testVolume)

			_, val, _ := promise.GetValue()

			Expect(val).To(BeNil())
		})

		It("can return a value", func() {
			promise.Fulfill(testVolume)

			_, _, err := promise.GetValue()

			Expect(err).To(BeNil())
		})

		Context("when not pending", func() {
			Context("when canceled", func() {
				It("returns ErrPromiseCanceled", func() {
					promise.Reject(ErrPromiseCanceled)

					err := promise.Fulfill(testVolume)

					Expect(err).To(Equal(ErrPromiseCanceled))
				})
			})

			Context("when fulfilled", func() {
				It("returns ErrPromiseNotPending", func() {
					promise.Fulfill(testVolume)

					err := promise.Fulfill(testVolume)

					Expect(err).To(Equal(ErrPromiseNotPending))
				})
			})

			Context("when rejected", func() {
				It("returns ErrPromiseNotPending", func() {
					promise.Reject(testErr)

					err := promise.Fulfill(testVolume)

					Expect(err).To(Equal(ErrPromiseNotPending))
				})
			})
		})
	})

	Context("when rejected", func() {
		It("is not pending", func() {
			promise.Reject(testErr)

			Expect(promise.IsPending()).To(BeFalse())
		})

		It("returns an empty volume in value", func() {
			promise.Reject(testErr)

			val, _, _ := promise.GetValue()

			Expect(val).To(Equal(Volume{}))
		})

		It("returns a non-nil error in value", func() {
			promise.Reject(testErr)

			_, val, _ := promise.GetValue()

			Expect(val).To(Equal(testErr))
		})

		It("can return a value", func() {
			promise.Reject(testErr)

			_, _, err := promise.GetValue()

			Expect(err).To(BeNil())
		})

		Context("when rejecting again", func() {
			Context("when canceled", func() {
				It("returns ErrPromiseNotPending", func() {
					promise.Reject(ErrPromiseCanceled)

					err := promise.Reject(testErr)

					Expect(err).To(Equal(ErrPromiseNotPending))
				})
			})

			Context("when not canceled", func() {
				It("returns ErrPromiseNotPending", func() {
					promise.Reject(testErr)

					err := promise.Reject(testErr)

					Expect(err).To(Equal(ErrPromiseNotPending))
				})
			})
		})
	})
})
<|endoftext|>


<commit_before>package ethui

import (
	"bitbucket.org/kardianos/osext"
	"fmt"
	"github.com/ethereum/eth-go"
	"github.com/ethereum/eth-go/ethchain"
	"github.com/ethereum/eth-go/ethutil"
	"github.com/niemeyer/qml"
	"github.com/obscuren/mutan"
	"math/big"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"
)

// UI Library that has some basic functionality exposed
type UiLib struct {
	engine    *qml.Engine
	eth       *eth.Ethereum
	connected bool
	assetPath string
	// The main application window
	win *qml.Window
}

func NewUiLib(engine *qml.Engine, eth *eth.Ethereum, assetPath string) *UiLib {
	if assetPath == "" {
		assetPath = DefaultAssetPath()
	}
	return &UiLib{engine: engine, eth: eth, assetPath: assetPath}
}

// Opens a QML file (external application)
func (ui *UiLib) Open(path string) {
	component, err := ui.engine.LoadFile(path[7:])
	if err != nil {
		ethutil.Config.Log.Debugln(err)
	}
	win := component.CreateWindow(nil)

	go func() {
		win.Show()
		win.Wait()
	}()
}

func (ui *UiLib) Connect(button qml.Object) {
	if !ui.connected {
		ui.eth.Start()
		ui.connected = true
		button.Set("enabled", false)
	}
}

func (ui *UiLib) ConnectToPeer(addr string) {
	ui.eth.ConnectToPeer(addr)
}

func (ui *UiLib) AssetPath(p string) string {
	return path.Join(ui.assetPath, p)
}

func DefaultAssetPath() string {
	var base string
	// If the current working directory is the go-ethereum dir
	// assume a debug build and use the source directory as
	// asset directory.
	pwd, _ := os.Getwd()
	if pwd == path.Join(os.Getenv("GOPATH"), "src", "github.com", "ethereum", "go-ethereum", "ethereal") {
		base = path.Join(pwd, "assets")
	} else {
		switch runtime.GOOS {
		case "darwin":
			// Get Binary Directory
			exedir, _ := osext.ExecutableFolder()
			base = filepath.Join(exedir, "../Resources")
		case "linux":
			base = "/usr/share/ethereal"
		case "window":
			fallthrough
		default:
			base = "."
		}
	}

	return base
}

type memAddr struct {
	Num   string
	Value string
}

func (ui *UiLib) DebugTx(recipient, valueStr, gasStr, gasPriceStr, data string) (string, error) {
	state := ui.eth.BlockChain().CurrentBlock.State()

	asm, err := mutan.Compile(strings.NewReader(data), false)
	if err != nil {
		fmt.Println(err)
	}

	callerScript := ethutil.Assemble(asm...)
	dis := ethchain.Disassemble(callerScript)
	ui.win.Root().Call("clearAsm")
	for _, str := range dis {
		ui.win.Root().Call("setAsm", str)
	}
	callerTx := ethchain.NewContractCreationTx(ethutil.Big(valueStr), ethutil.Big(gasPriceStr), callerScript)

	// Contract addr as test address
	keyPair := ethutil.Config.Db.GetKeys()[0]
	account := ui.eth.StateManager().GetAddrState(keyPair.Address()).Account
	c := ethchain.MakeContract(callerTx, state)
	callerClosure := ethchain.NewClosure(account, c, c.Script(), state, ethutil.Big(gasStr), new(big.Int))

	block := ui.eth.BlockChain().CurrentBlock
	vm := ethchain.NewVm(state, ethchain.RuntimeVars{
		Origin:      account.Address(),
		BlockNumber: block.BlockInfo().Number,
		PrevHash:    block.PrevHash,
		Coinbase:    block.Coinbase,
		Time:        block.Time,
		Diff:        block.Difficulty,
		TxData:      nil,
	})
	callerClosure.Call(vm, nil, func(op ethchain.OpCode, mem *ethchain.Memory, stack *ethchain.Stack) {
		ui.win.Root().Call("clearMem")
		ui.win.Root().Call("clearStack")

		addr := 0
		for i := 0; i+32 <= mem.Len(); i += 32 {
			ui.win.Root().Call("setMem", memAddr{fmt.Sprintf("%03d", addr), fmt.Sprintf("% x", mem.Data()[i:i+32])})
			addr++
		}

		for _, val := range stack.Data() {
			ui.win.Root().Call("setStack", val.String())
		}
	})
	state.Reset()

	return "", nil
}
<commit_msg>Debugger<commit_after>package ethui

import (
	"bitbucket.org/kardianos/osext"
	"fmt"
	"github.com/ethereum/eth-go"
	"github.com/ethereum/eth-go/ethchain"
	"github.com/ethereum/eth-go/ethutil"
	"github.com/niemeyer/qml"
	"github.com/obscuren/mutan"
	"math/big"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"
)

// UI Library that has some basic functionality exposed
type UiLib struct {
	engine    *qml.Engine
	eth       *eth.Ethereum
	connected bool
	assetPath string
	// The main application window
	win *qml.Window
}

func NewUiLib(engine *qml.Engine, eth *eth.Ethereum, assetPath string) *UiLib {
	if assetPath == "" {
		assetPath = DefaultAssetPath()
	}
	return &UiLib{engine: engine, eth: eth, assetPath: assetPath}
}

// Opens a QML file (external application)
func (ui *UiLib) Open(path string) {
	component, err := ui.engine.LoadFile(path[7:])
	if err != nil {
		ethutil.Config.Log.Debugln(err)
	}
	win := component.CreateWindow(nil)

	go func() {
		win.Show()
		win.Wait()
	}()
}

func (ui *UiLib) Connect(button qml.Object) {
	if !ui.connected {
		ui.eth.Start()
		ui.connected = true
		button.Set("enabled", false)
	}
}

func (ui *UiLib) ConnectToPeer(addr string) {
	ui.eth.ConnectToPeer(addr)
}

func (ui *UiLib) AssetPath(p string) string {
	return path.Join(ui.assetPath, p)
}

func DefaultAssetPath() string {
	var base string
	// If the current working directory is the go-ethereum dir
	// assume a debug build and use the source directory as
	// asset directory.
	pwd, _ := os.Getwd()
	if pwd == path.Join(os.Getenv("GOPATH"), "src", "github.com", "ethereum", "go-ethereum", "ethereal") {
		base = path.Join(pwd, "assets")
	} else {
		switch runtime.GOOS {
		case "darwin":
			// Get Binary Directory
			exedir, _ := osext.ExecutableFolder()
			base = filepath.Join(exedir, "../Resources")
		case "linux":
			base = "/usr/share/ethereal"
		case "window":
			fallthrough
		default:
			base = "."
		}
	}

	return base
}

type memAddr struct {
	Num   string
	Value string
}

type Debugger struct {
	ui   *UiLib
	next chan bool
}

func (d *Debugger) halting(op ethchain.OpCode, mem *ethchain.Memory, stack *ethchain.Stack) {
	d.ui.win.Root().Call("clearMem")
	d.ui.win.Root().Call("clearStack")

	addr := 0
	for i := 0; i+32 <= mem.Len(); i += 32 {
		d.ui.win.Root().Call("setMem", memAddr{fmt.Sprintf("%03d", addr), fmt.Sprintf("% x", mem.Data()[i:i+32])})
		addr++
	}

	for _, val := range stack.Data() {
		d.ui.win.Root().Call("setStack", val.String())
	}

out:
	for {
		select {
		case <-d.next:
			break out
		default:
		}
	}
}

func (d *Debugger) Next() {
	d.next <- true
}

func (ui *UiLib) DebugTx(recipient, valueStr, gasStr, gasPriceStr, data string) {
	state := ui.eth.BlockChain().CurrentBlock.State()

	asm, err := mutan.Compile(strings.NewReader(data), false)
	if err != nil {
		fmt.Println(err)
	}

	callerScript := ethutil.Assemble(asm...)
	dis := ethchain.Disassemble(callerScript)
	ui.win.Root().Call("clearAsm")
	for _, str := range dis {
		ui.win.Root().Call("setAsm", str)
	}
	callerTx := ethchain.NewContractCreationTx(ethutil.Big(valueStr), ethutil.Big(gasPriceStr), callerScript)

	// Contract addr as test address
	keyPair := ethutil.Config.Db.GetKeys()[0]
	account := ui.eth.StateManager().GetAddrState(keyPair.Address()).Account
	c := ethchain.MakeContract(callerTx, state)
	callerClosure := ethchain.NewClosure(account, c, c.Script(), state, ethutil.Big(gasStr), new(big.Int))

	block := ui.eth.BlockChain().CurrentBlock
	vm := ethchain.NewVm(state, ethchain.RuntimeVars{
		Origin:      account.Address(),
		BlockNumber: block.BlockInfo().Number,
		PrevHash:    block.PrevHash,
		Coinbase:    block.Coinbase,
		Time:        block.Time,
		Diff:        block.Difficulty,
		TxData:      nil,
	})

	db := &Debugger{ui, make(chan bool)}
	ui.engine.Context().SetVar("db", db)
	go func() {
		callerClosure.Call(vm, nil, db.halting)

		state.Reset()
	}()
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package master

import (
	"fmt"
	"runtime"
	"time"

	apierrs "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	kuberuntime "k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/tools/clientcmd"
	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/v1"
	extensions "k8s.io/kubernetes/pkg/apis/extensions/v1beta1"
	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
)

func CreateClientFromFile(path string) (*clientset.Clientset, error) {
	adminKubeconfig, err := clientcmd.LoadFromFile(path)
	if err != nil {
		return nil, fmt.Errorf("failed to load admin kubeconfig [%v]", err)
	}
	adminClientConfig, err := clientcmd.NewDefaultClientConfig(*adminKubeconfig, &clientcmd.ConfigOverrides{}).ClientConfig()
	if err != nil {
		return nil, fmt.Errorf("failed to create API client configuration [%v]", err)
	}

	client, err := clientset.NewForConfig(adminClientConfig)
	if err != nil {
		return nil, fmt.Errorf("failed to create API client [%v]", err)
	}
	return client, nil
}

func CreateClientAndWaitForAPI(file string) (*clientset.Clientset, error) {
	client, err := CreateClientFromFile(file)
	if err != nil {
		return nil, err
	}

	fmt.Println("[apiclient] Created API client, waiting for the control plane to become ready")
	WaitForAPI(client)

	fmt.Println("[apiclient] Waiting for at least one node to register and become ready")
	start := time.Now()
	wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
		nodeList, err := client.Nodes().List(metav1.ListOptions{})
		if err != nil {
			fmt.Println("[apiclient] Temporarily unable to list nodes (will retry)")
			return false, nil
		}
		if len(nodeList.Items) < 1 {
			return false, nil
		}
		n := &nodeList.Items[0]
		if !v1.IsNodeReady(n) {
			fmt.Println("[apiclient] First node has registered, but is not ready yet")
			return false, nil
		}

		fmt.Printf("[apiclient] First node is ready after %f seconds\n", time.Since(start).Seconds())
		return true, nil
	})

	if err := createAndWaitForADummyDeployment(client); err != nil {
		return nil, err
	}

	return client, nil
}

func WaitForAPI(client *clientset.Clientset) {
	start := time.Now()
	wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
		// TODO: use /healthz API instead of this
		cs, err := client.ComponentStatuses().List(metav1.ListOptions{})
		if err != nil {
			if apierrs.IsForbidden(err) {
				fmt.Println("[apiclient] Waiting for API server authorization")
			}
			return false, nil
		}

		// TODO(phase2) must revisit this when we implement HA
		if len(cs.Items) < 3 {
			return false, nil
		}
		for _, item := range cs.Items {
			for _, condition := range item.Conditions {
				if condition.Type != v1.ComponentHealthy {
					fmt.Printf("[apiclient] Control plane component %q is still unhealthy: %#v\n", item.ObjectMeta.Name, item.Conditions)
					return false, nil
				}
			}
		}

		fmt.Printf("[apiclient] All control plane components are healthy after %f seconds\n", time.Since(start).Seconds())
		return true, nil
	})
}

func createAndWaitForADummyDeployment(client *clientset.Clientset) error {
	dummyDeploymentBytes, err := kubeadmutil.ParseTemplate(DummyDeployment, struct{ ImageRepository, Arch string }{
		ImageRepository: kubeadmapi.GlobalEnvParams.RepositoryPrefix,
		Arch:            runtime.GOARCH,
	})
	if err != nil {
		return fmt.Errorf("error when parsing dummy deployment template: %v", err)
	}

	dummyDeployment := &extensions.Deployment{}
	if err := kuberuntime.DecodeInto(api.Codecs.UniversalDecoder(), dummyDeploymentBytes, dummyDeployment); err != nil {
		return fmt.Errorf("unable to decode dummy deployment %v", err)
	}

	wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
		// TODO: we should check the error, as some cases may be fatal
		if _, err := client.ExtensionsV1beta1().Deployments(metav1.NamespaceSystem).Create(dummyDeployment); err != nil {
			fmt.Printf("[apiclient] Failed to create test deployment [%v] (will retry)\n", err)
			return false, nil
		}
		return true, nil
	})

	wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
		d, err := client.ExtensionsV1beta1().Deployments(metav1.NamespaceSystem).Get("dummy", metav1.GetOptions{})
		if err != nil {
			fmt.Printf("[apiclient] Failed to get test deployment [%v] (will retry)\n", err)
			return false, nil
		}
		if d.Status.AvailableReplicas < 1 {
			return false, nil
		}
		return true, nil
	})

	fmt.Println("[apiclient] Test deployment succeeded")

	// TODO: In the future, make sure the ReplicaSet and Pod are garbage collected
	if err := client.ExtensionsV1beta1().Deployments(metav1.NamespaceSystem).Delete("dummy", &metav1.DeleteOptions{}); err != nil {
		fmt.Printf("[apiclient] Failed to delete test deployment [%v] (will ignore)\n", err)
	}
	return nil
}
<commit_msg>Set OrphanDependents=&falseVar so the GC will (or should remove the dummy Pod<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package master

import (
	"fmt"
	"runtime"
	"time"

	apierrs "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	kuberuntime "k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/tools/clientcmd"
	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/v1"
	extensions "k8s.io/kubernetes/pkg/apis/extensions/v1beta1"
	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
)

func CreateClientFromFile(path string) (*clientset.Clientset, error) {
	adminKubeconfig, err := clientcmd.LoadFromFile(path)
	if err != nil {
		return nil, fmt.Errorf("failed to load admin kubeconfig [%v]", err)
	}
	adminClientConfig, err := clientcmd.NewDefaultClientConfig(*adminKubeconfig, &clientcmd.ConfigOverrides{}).ClientConfig()
	if err != nil {
		return nil, fmt.Errorf("failed to create API client configuration [%v]", err)
	}

	client, err := clientset.NewForConfig(adminClientConfig)
	if err != nil {
		return nil, fmt.Errorf("failed to create API client [%v]", err)
	}
	return client, nil
}

func CreateClientAndWaitForAPI(file string) (*clientset.Clientset, error) {
	client, err := CreateClientFromFile(file)
	if err != nil {
		return nil, err
	}

	fmt.Println("[apiclient] Created API client, waiting for the control plane to become ready")
	WaitForAPI(client)

	fmt.Println("[apiclient] Waiting for at least one node to register and become ready")
	start := time.Now()
	wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
		nodeList, err := client.Nodes().List(metav1.ListOptions{})
		if err != nil {
			fmt.Println("[apiclient] Temporarily unable to list nodes (will retry)")
			return false, nil
		}
		if len(nodeList.Items) < 1 {
			return false, nil
		}
		n := &nodeList.Items[0]
		if !v1.IsNodeReady(n) {
			fmt.Println("[apiclient] First node has registered, but is not ready yet")
			return false, nil
		}

		fmt.Printf("[apiclient] First node is ready after %f seconds\n", time.Since(start).Seconds())
		return true, nil
	})

	if err := createAndWaitForADummyDeployment(client); err != nil {
		return nil, err
	}

	return client, nil
}

func WaitForAPI(client *clientset.Clientset) {
	start := time.Now()
	wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
		// TODO: use /healthz API instead of this
		cs, err := client.ComponentStatuses().List(metav1.ListOptions{})
		if err != nil {
			if apierrs.IsForbidden(err) {
				fmt.Println("[apiclient] Waiting for API server authorization")
			}
			return false, nil
		}

		// TODO(phase2) must revisit this when we implement HA
		if len(cs.Items) < 3 {
			return false, nil
		}
		for _, item := range cs.Items {
			for _, condition := range item.Conditions {
				if condition.Type != v1.ComponentHealthy {
					fmt.Printf("[apiclient] Control plane component %q is still unhealthy: %#v\n", item.ObjectMeta.Name, item.Conditions)
					return false, nil
				}
			}
		}

		fmt.Printf("[apiclient] All control plane components are healthy after %f seconds\n", time.Since(start).Seconds())
		return true, nil
	})
}

func createAndWaitForADummyDeployment(client *clientset.Clientset) error {
	dummyDeploymentBytes, err := kubeadmutil.ParseTemplate(DummyDeployment, struct{ ImageRepository, Arch string }{
		ImageRepository: kubeadmapi.GlobalEnvParams.RepositoryPrefix,
		Arch:            runtime.GOARCH,
	})
	if err != nil {
		return fmt.Errorf("error when parsing dummy deployment template: %v", err)
	}

	dummyDeployment := &extensions.Deployment{}
	if err := kuberuntime.DecodeInto(api.Codecs.UniversalDecoder(), dummyDeploymentBytes, dummyDeployment); err != nil {
		return fmt.Errorf("unable to decode dummy deployment %v", err)
	}

	wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
		// TODO: we should check the error, as some cases may be fatal
		if _, err := client.ExtensionsV1beta1().Deployments(metav1.NamespaceSystem).Create(dummyDeployment); err != nil {
			fmt.Printf("[apiclient] Failed to create test deployment [%v] (will retry)\n", err)
			return false, nil
		}
		return true, nil
	})

	wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
		d, err := client.ExtensionsV1beta1().Deployments(metav1.NamespaceSystem).Get("dummy", metav1.GetOptions{})
		if err != nil {
			fmt.Printf("[apiclient] Failed to get test deployment [%v] (will retry)\n", err)
			return false, nil
		}
		if d.Status.AvailableReplicas < 1 {
			return false, nil
		}
		return true, nil
	})

	fmt.Println("[apiclient] Test deployment succeeded")

	falseVar := false
	if err := client.ExtensionsV1beta1().Deployments(metav1.NamespaceSystem).Delete("dummy", &metav1.DeleteOptions{
		OrphanDependents: &falseVar,
	}); err != nil {
		fmt.Printf("[apiclient] Failed to delete test deployment [%v] (will ignore)\n", err)
	}
	return nil
}
<|endoftext|>


<commit_before>package exml

import (
	"bytes"
	"encoding/xml"
	"errors"
	"io"
	"strings"
)

type Callback interface{}
type ElemCallback func(Attrs)
type TextCallback func(CharData)
type ErrorCallback func(error)

type handler struct {
	callback    Callback
	subHandlers map[string]*handler
	parent      *handler
}

type Decoder struct {
	decoder        *xml.Decoder
	topHandler     *handler
	currentHandler *handler
	text           *bytes.Buffer
	errorCallback  ErrorCallback
}

func NewDecoder(r io.Reader) *Decoder {
	topHandler := &handler{nil, nil, nil}
	return &Decoder{
		decoder:        xml.NewDecoder(r),
		text:           bytes.NewBuffer(nil),
		topHandler:     topHandler,
		currentHandler: topHandler,
	}
}

func (d *Decoder) On(event string, callback Callback) {
	h := d.currentHandler
	events := strings.Split(event, "/")
	for i, event := range events {
		sub := h.subHandlers[event]
		if i < len(events)-1 {
			if sub == nil {
				sub = &handler{nil, nil, h}
			}
		} else {
			sub = &handler{callback, nil, h}
		}

		if h.subHandlers == nil {
			h.subHandlers = make(map[string]*handler)
		}

		h.subHandlers[event] = sub
		h = sub
	}
}

func (d *Decoder) OnError(handler ErrorCallback) {
	d.errorCallback = handler
}

func (d *Decoder) Run() {
	for {
		token, err := d.decoder.Token()
		if token == nil {
			if d.errorCallback != nil {
				d.errorCallback(err)
			}
			break
		}

		switch t := token.(type) {
		case xml.StartElement:
			h := d.topHandler.subHandlers[t.Name.Local]
			if h == nil && d.currentHandler != d.topHandler {
				h = d.currentHandler.subHandlers[t.Name.Local]
			}
			if h != nil {
				h.parent = d.currentHandler
				d.currentHandler = h
				if h != nil && h.callback != nil {
					h.callback.(func(Attrs))(t.Attr)
				}
			}
			break
		case xml.CharData:
			d.text.Write(t)
			break
		case xml.EndElement:
			text := bytes.TrimSpace(d.text.Bytes())
			d.text.Reset()
			if len(text) > 0 {
				h := d.topHandler.subHandlers["$text"]
				if h == nil {
					h = d.currentHandler.subHandlers["$text"]
				}
				if h != nil && h.callback != nil {
					h.callback.(func(CharData))(text)
				}
			}
			if d.currentHandler != d.topHandler {
				d.currentHandler = d.currentHandler.parent
			}
			break
		}
	}
}

func Assign(slot *string) func(CharData) {
	return func(c CharData) {
		*slot = string(c)
	}
}

func Append(a *[]string) func(CharData) {
	return func(c CharData) {
		*a = append(*a, string(c))
	}
}

type Attrs []xml.Attr
type CharData xml.CharData

func (a Attrs) Get(name string) (string, error) {
	for _, attr := range a {
		if attr.Name.Local == name {
			return attr.Value, nil
		}
	}

	return "", errors.New("attribute not found")
}
<commit_msg>Tweaked event registration.<commit_after>package exml

import (
	"bytes"
	"encoding/xml"
	"errors"
	"io"
	"strings"
)

type Callback interface{}
type ElemCallback func(Attrs)
type TextCallback func(CharData)
type ErrorCallback func(error)

type handler struct {
	callback    Callback
	subHandlers map[string]*handler
	parent      *handler
}

type Decoder struct {
	decoder        *xml.Decoder
	topHandler     *handler
	currentHandler *handler
	text           *bytes.Buffer
	errorCallback  ErrorCallback
}

func NewDecoder(r io.Reader) *Decoder {
	topHandler := &handler{nil, nil, nil}
	return &Decoder{
		decoder:        xml.NewDecoder(r),
		text:           bytes.NewBuffer(nil),
		topHandler:     topHandler,
		currentHandler: topHandler,
	}
}

func (d *Decoder) On(event string, callback Callback) {
	events := strings.Split(event, "/")
	depth := len(events) - 1
	h := d.currentHandler

	for i, ev := range events {
		var sub *handler
		if i < depth {
			sub = h.subHandlers[ev]
			if sub == nil {
				sub = &handler{nil, nil, h}
			}
		} else {
			sub = &handler{callback, nil, h}
		}

		if h.subHandlers == nil {
			h.subHandlers = make(map[string]*handler)
		}

		h.subHandlers[ev] = sub
		h = sub
	}
}

func (d *Decoder) OnError(handler ErrorCallback) {
	d.errorCallback = handler
}

func (d *Decoder) Run() {
	for {
		token, err := d.decoder.Token()
		if token == nil {
			if d.errorCallback != nil {
				d.errorCallback(err)
			}
			break
		}

		switch t := token.(type) {
		case xml.StartElement:
			h := d.topHandler.subHandlers[t.Name.Local]
			if h == nil && d.currentHandler != d.topHandler {
				h = d.currentHandler.subHandlers[t.Name.Local]
			}
			if h != nil {
				h.parent = d.currentHandler
				d.currentHandler = h
				if h != nil && h.callback != nil {
					h.callback.(func(Attrs))(t.Attr)
				}
			}
			break
		case xml.CharData:
			d.text.Write(t)
			break
		case xml.EndElement:
			text := bytes.TrimSpace(d.text.Bytes())
			d.text.Reset()
			if len(text) > 0 {
				h := d.topHandler.subHandlers["$text"]
				if h == nil {
					h = d.currentHandler.subHandlers["$text"]
				}
				if h != nil && h.callback != nil {
					h.callback.(func(CharData))(text)
				}
			}
			if d.currentHandler != d.topHandler {
				d.currentHandler = d.currentHandler.parent
			}
			break
		}
	}
}

func Assign(slot *string) func(CharData) {
	return func(c CharData) {
		*slot = string(c)
	}
}

func Append(a *[]string) func(CharData) {
	return func(c CharData) {
		*a = append(*a, string(c))
	}
}

type Attrs []xml.Attr
type CharData xml.CharData

func (a Attrs) Get(name string) (string, error) {
	for _, attr := range a {
		if attr.Name.Local == name {
			return attr.Value, nil
		}
	}

	return "", errors.New("attribute not found")
}
<|endoftext|>


<commit_before>// Copyright 2020 Netflix Inc
// Author: Colin McIntosh (colin@netflix.com)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package prometheus

import (
	"errors"
	"fmt"
	"github.com/openconfig/gnmi-gateway/gateway/configuration"
	"github.com/openconfig/gnmi-gateway/gateway/openconfig"
	"github.com/openconfig/gnmi/cache"
	"github.com/openconfig/gnmi/ctree"
	gnmipb "github.com/openconfig/gnmi/proto/gnmi"
	prom "github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"net/http"
	"strings"
)

func NewPrometheusExporter(config *configuration.GatewayConfig) *PrometheusExporter {
	return &PrometheusExporter{
		config:     config,
		deltaCalc:  NewDeltaCalculator(),
		metrics:    make(map[Hash]prom.Metric),
		typeLookup: new(openconfig.TypeLookup),
	}
}

type PrometheusExporter struct {
	config     *configuration.GatewayConfig
	cache      *cache.Cache
	deltaCalc  *DeltaCalculator
	metrics    map[Hash]prom.Metric
	typeLookup *openconfig.TypeLookup
}

func (e *PrometheusExporter) Export(leaf *ctree.Leaf) {
	notification := leaf.Value().(*gnmipb.Notification)
	for _, update := range notification.Update {
		value, isNumber := GetNumberValues(update.Val)
		if !isNumber {
			continue
		}
		metricName, labels := UpdateToMetricNameAndLabels(notification.GetPrefix(), update)
		metricHash := NewStringMapHash(metricName, labels)

		metric, exists := e.metrics[metricHash]
		if !exists {
			var path []string
			for _, elem := range update.Path.Elem {
				path = append(path, elem.Name)
			}
			metricType := e.typeLookup.GetTypeByPath(path)

			switch metricType {
			case "counter64":
				metric = promauto.NewCounter(prom.CounterOpts{
					Name:        metricName,
					ConstLabels: labels,
				})
			case "gauge32":
			default:
				metric = promauto.NewGauge(prom.GaugeOpts{
					Name:        metricName,
					ConstLabels: labels,
				})
			}
			e.metrics[metricHash] = metric
		}

		switch m := metric.(type) {
		case prom.Counter:
			delta, _ := e.deltaCalc.Calc(metricHash, value)
			m.Add(delta)
		case prom.Gauge:
			m.Set(value)
		}
	}
}

func (e *PrometheusExporter) Start(cache *cache.Cache) error {
	e.config.Log.Info().Msg("Starting Prometheus exporter.")
	if e.config.OpenConfigDirectory == "" {
		return errors.New("value is not set for OpenConfigDirectory configuration")
	}
	e.cache = cache
	err := e.typeLookup.LoadAllModules(e.config.OpenConfigDirectory)
	if err != nil {
		e.config.Log.Error().Err(err).Msgf("Unable to load OpenConfig modules in %s: %v", e.config.OpenConfigDirectory, err)
		return err
	}

	e.config.Log.Info().Msg("Starting Prometheus exporter.")
	e.cache.SetClient(e.Export)
	go e.runHttpServer()
	return nil
}

func (e *PrometheusExporter) runHttpServer() {
	var errCount = 0
	var lastError error
	for {
		e.config.Log.Info().Msg("Starting Prometheus HTTP server.")
		http.Handle("/metrics", promhttp.Handler())
		err := http.ListenAndServe(":59100", nil)
		if err != nil {
			e.config.Log.Error().Err(err).Msgf("Prometheus HTTP server stopped with an error: %v", err)
			if err.Error() == lastError.Error() {
				errCount = errCount + 1
				if errCount >= 3 {
					panic(fmt.Errorf("too many errors returned by Prometheus HTTP server: %s", err.Error()))
				}
			} else {
				errCount = 0
				lastError = err
			}
		}
	}
}

func GetNumberValues(tv *gnmipb.TypedValue) (float64, bool) {
	if tv != nil && tv.Value != nil {
		switch tv.Value.(type) {
		case *gnmipb.TypedValue_StringVal:
			return 0, false
		case *gnmipb.TypedValue_IntVal:
			return float64(tv.GetIntVal()), true
		case *gnmipb.TypedValue_UintVal:
			return float64(tv.GetUintVal()), true
		case *gnmipb.TypedValue_BoolVal:
			if tv.GetBoolVal() {
				return 1, true
			} else {
				return 0, false
			}
		case *gnmipb.TypedValue_FloatVal:
			return float64(tv.GetFloatVal()), true
		case *gnmipb.TypedValue_LeaflistVal:
			return 0, false
		case *gnmipb.TypedValue_BytesVal:
			return 0, false
		default:
			return 0, false
		}
	}
	return 0, false
}

func UpdateToMetricNameAndLabels(prefix *gnmipb.Path, update *gnmipb.Update) (string, map[string]string) {
	metricName := ""
	labels := make(map[string]string)

	if prefix != nil {
		target := prefix.GetTarget()
		if target != "" {
			labels["target"] = target
		}
	}

	for _, elem := range update.Path.Elem {
		elemName := strings.ReplaceAll(elem.Name, "-", "_")
		if metricName == "" {
			metricName = elemName
		} else {
			metricName = metricName + "_" + elemName
		}

		for key, value := range elem.Key {
			labelKey := metricName + "_" + strings.ReplaceAll(key, "-", "_")
			labels[labelKey] = value
		}
	}
	return metricName, labels
}
<commit_msg>Fix sending bad/duplicate counter metrics to Prometheus.<commit_after>// Copyright 2020 Netflix Inc
// Author: Colin McIntosh (colin@netflix.com)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package prometheus

import (
	"errors"
	"fmt"
	"github.com/openconfig/gnmi-gateway/gateway/configuration"
	"github.com/openconfig/gnmi-gateway/gateway/openconfig"
	"github.com/openconfig/gnmi/cache"
	"github.com/openconfig/gnmi/ctree"
	gnmipb "github.com/openconfig/gnmi/proto/gnmi"
	prom "github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"net/http"
	"strings"
)

func NewPrometheusExporter(config *configuration.GatewayConfig) *PrometheusExporter {
	return &PrometheusExporter{
		config:     config,
		deltaCalc:  NewDeltaCalculator(),
		metrics:    make(map[Hash]prom.Metric),
		typeLookup: new(openconfig.TypeLookup),
	}
}

type PrometheusExporter struct {
	config     *configuration.GatewayConfig
	cache      *cache.Cache
	deltaCalc  *DeltaCalculator
	metrics    map[Hash]prom.Metric
	typeLookup *openconfig.TypeLookup
}

func (e *PrometheusExporter) Export(leaf *ctree.Leaf) {
	notification := leaf.Value().(*gnmipb.Notification)
	for _, update := range notification.Update {
		value, isNumber := GetNumberValues(update.Val)
		if !isNumber {
			continue
		}
		metricName, labels := UpdateToMetricNameAndLabels(notification.GetPrefix(), update)
		metricHash := NewStringMapHash(metricName, labels)

		metric, exists := e.metrics[metricHash]
		if !exists {
			var path []string
			for _, elem := range update.Path.Elem {
				path = append(path, elem.Name)
			}
			metricType := e.typeLookup.GetTypeByPath(path)

			switch metricType {
			case "counter64":
				metric = promauto.NewCounter(prom.CounterOpts{
					Name:        metricName,
					ConstLabels: labels,
				})
			case "gauge32":
			default:
				metric = promauto.NewGauge(prom.GaugeOpts{
					Name:        metricName,
					ConstLabels: labels,
				})
			}
			e.metrics[metricHash] = metric
		}

		switch m := metric.(type) {
		case prom.Counter:
			delta, exists := e.deltaCalc.Calc(metricHash, value)
			if exists {
				m.Add(delta)
			}
		case prom.Gauge:
			m.Set(value)
		}
	}
}

func (e *PrometheusExporter) Start(cache *cache.Cache) error {
	e.config.Log.Info().Msg("Starting Prometheus exporter.")
	if e.config.OpenConfigDirectory == "" {
		return errors.New("value is not set for OpenConfigDirectory configuration")
	}
	e.cache = cache
	err := e.typeLookup.LoadAllModules(e.config.OpenConfigDirectory)
	if err != nil {
		e.config.Log.Error().Err(err).Msgf("Unable to load OpenConfig modules in %s: %v", e.config.OpenConfigDirectory, err)
		return err
	}

	e.config.Log.Info().Msg("Starting Prometheus exporter.")
	e.cache.SetClient(e.Export)
	go e.runHttpServer()
	return nil
}

func (e *PrometheusExporter) runHttpServer() {
	var errCount = 0
	var lastError error
	for {
		e.config.Log.Info().Msg("Starting Prometheus HTTP server.")
		http.Handle("/metrics", promhttp.Handler())
		err := http.ListenAndServe(":59100", nil)
		if err != nil {
			e.config.Log.Error().Err(err).Msgf("Prometheus HTTP server stopped with an error: %v", err)
			if err.Error() == lastError.Error() {
				errCount = errCount + 1
				if errCount >= 3 {
					panic(fmt.Errorf("too many errors returned by Prometheus HTTP server: %s", err.Error()))
				}
			} else {
				errCount = 0
				lastError = err
			}
		}
	}
}

func GetNumberValues(tv *gnmipb.TypedValue) (float64, bool) {
	if tv != nil && tv.Value != nil {
		switch tv.Value.(type) {
		case *gnmipb.TypedValue_StringVal:
			return 0, false
		case *gnmipb.TypedValue_IntVal:
			return float64(tv.GetIntVal()), true
		case *gnmipb.TypedValue_UintVal:
			return float64(tv.GetUintVal()), true
		case *gnmipb.TypedValue_BoolVal:
			if tv.GetBoolVal() {
				return 1, true
			} else {
				return 0, false
			}
		case *gnmipb.TypedValue_FloatVal:
			return float64(tv.GetFloatVal()), true
		case *gnmipb.TypedValue_LeaflistVal:
			return 0, false
		case *gnmipb.TypedValue_BytesVal:
			return 0, false
		default:
			return 0, false
		}
	}
	return 0, false
}

func UpdateToMetricNameAndLabels(prefix *gnmipb.Path, update *gnmipb.Update) (string, map[string]string) {
	metricName := ""
	labels := make(map[string]string)

	if prefix != nil {
		target := prefix.GetTarget()
		if target != "" {
			labels["target"] = target
		}
	}

	for _, elem := range update.Path.Elem {
		elemName := strings.ReplaceAll(elem.Name, "-", "_")
		if metricName == "" {
			metricName = elemName
		} else {
			metricName = metricName + "_" + elemName
		}

		for key, value := range elem.Key {
			labelKey := metricName + "_" + strings.ReplaceAll(key, "-", "_")
			labels[labelKey] = value
		}
	}
	return metricName, labels
}
<|endoftext|>


<commit_before>package admin

import (
	"fmt"

	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/app"
	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/app/args"
	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/app/with"
	"github.com/BytemarkHosting/bytemark-client/lib/output"
	brainRequests "github.com/BytemarkHosting/bytemark-client/lib/requests/brain"
	"github.com/urfave/cli"
)

func init() {
	Commands = append(Commands, cli.Command{
		Name:   "show",
		Action: cli.ShowSubcommandHelp,
		Subcommands: []cli.Command{
			{
				Name:      "vlans",
				Usage:     "shows available VLANs",
				UsageText: "--admin show vlans [--json]",
				Flags:     app.OutputFlags("VLANs", "array"),
				Action: app.Action(with.Auth, func(c *app.Context) error {
					vlans, err := c.Client().GetVLANs()
					if err != nil {
						return err
					}
					return c.OutputInDesiredForm(vlans, output.Table)
				}),
			},
			{
				Name:        "disc by id",
				Usage:       "displays details about a disc",
				UsageText:   "show disc by id [--json] <id>",
				Description: `Displays a collection of details about the disc.`,
				Flags: append(app.OutputFlags("disc details", "object"),
					cli.IntFlag{
						Name:  "disc",
						Usage: "the disc to display",
					},
				),
				Action: app.Action(with.Auth, args.Optional("disc"), with.RequiredFlags("disc"), func(c *app.Context) error {
					disc, err := c.Client().GetDiscByID(c.Int("disc"))
					if err != nil {
						return err
					}

					return c.OutputInDesiredForm(disc)
				}),
			},
			{
				Name:      "vlan",
				Usage:     "shows the details of a VLAN",
				UsageText: "--admin show vlan [--json] <num>",
				Flags: append(app.OutputFlags("VLAN", "object"),
					cli.IntFlag{
						Name:  "num",
						Usage: "the num of the VLAN to display",
					},
				),
				Action: app.Action(args.Optional("num"), with.RequiredFlags("num"), with.Auth, func(c *app.Context) error {
					vlan, err := c.Client().GetVLAN(c.Int("num"))
					if err != nil {
						return err
					}
					return c.OutputInDesiredForm(vlan, output.Table)
				}),
			},
			{
				Name:      "ip ranges",
				Usage:     "shows all IP ranges",
				UsageText: "--admin show ip ranges [--json]",
				Flags:     app.OutputFlags("ip ranges", "array"),
				Action: app.Action(with.Auth, func(c *app.Context) error {
					ipRanges, err := c.Client().GetIPRanges()
					if err != nil {
						return err
					}
					return c.OutputInDesiredForm(ipRanges, output.Table)
				}),
			},
			{
				Name:      "ip range",
				Usage:     "shows the details of an IP range",
				UsageText: "--admin show ip range [--json] <ip-range>",
				Flags: append(app.OutputFlags("ip range details", "object"),
					cli.StringFlag{
						Name:  "ip-range",
						Usage: "the ID or CIDR representation of the IP range to display",
					},
				),
				Action: app.Action(args.Optional("ip-range"), with.RequiredFlags("ip-range"), with.Auth, func(c *app.Context) error {
					ipRange, err := c.Client().GetIPRange(c.String("ip-range"))
					if err != nil {
						return err
					}
					return c.OutputInDesiredForm(ipRange, output.Table)
				}),
			},
			{
				Name:      "heads",
				Usage:     "shows the details of all heads",
				UsageText: "--admin show heads [--json]",
				Flags:     app.OutputFlags("heads", "array"),
				Action: app.Action(with.Auth, func(c *app.Context) error {
					heads, err := c.Client().GetHeads()
					if err != nil {
						return err
					}
					return c.OutputInDesiredForm(heads, output.Table)
				}),
			},
			{
				Name:      "head",
				Usage:     "shows the details of the specified head",
				UsageText: "--admin show head <head> [--json]",
				Flags: append(app.OutputFlags("head details", "object"),
					cli.StringFlag{
						Name:  "head",
						Usage: "the ID of the head to display",
					},
				),
				Action: app.Action(args.Optional("head"), with.RequiredFlags("head"), with.Auth, func(c *app.Context) error {
					head, err := c.Client().GetHead(c.String("head"))
					if err != nil {
						return err
					}
					return c.OutputInDesiredForm(head, output.Table)
				}),
			},
			{
				Name:      "tails",
				Usage:     "shows the details of all tails",
				UsageText: "--admin show tails [--json]",
				Flags:     app.OutputFlags("tails", "array"),
				Action: app.Action(with.Auth, func(c *app.Context) error {
					tails, err := c.Client().GetTails()
					if err != nil {
						return err
					}
					return c.OutputInDesiredForm(tails, output.Table)
				}),
			},
			{
				Name:      "tail",
				Usage:     "shows the details of the specified tail",
				UsageText: "--admin show tail <tail> [--json]",
				Flags: append(app.OutputFlags("tail details", "object"),
					cli.StringFlag{
						Name:  "tail",
						Usage: "the ID of the tail to display",
					},
				),
				Action: app.Action(args.Optional("tail"), with.RequiredFlags("tail"), with.Auth, func(c *app.Context) error {
					tail, err := c.Client().GetTail(c.String("tail"))
					if err != nil {
						return err
					}
					return c.OutputInDesiredForm(tail, output.Table)
				}),
			},
			{
				Name:      "storage pools",
				Usage:     "shows the details of all storage pools",
				UsageText: "--admin show storage pools [--json]",
				Flags:     app.OutputFlags("storage pools", "array"),
				Action: app.Action(with.Auth, func(c *app.Context) error {
					storagePools, err := c.Client().GetStoragePools()
					if err != nil {
						return err
					}
					return c.OutputInDesiredForm(storagePools, output.Table)
				}),
			},
			{
				Name:      "storage pool",
				Usage:     "shows the details of the specified storage pool",
				UsageText: "--admin show storage pool [--json] <storage-pool>",
				Flags: append(app.OutputFlags("storage pool", "object"),
					cli.StringFlag{
						Name:  "storage-pool",
						Usage: "The ID or label of the storage pool to display",
					},
				),
				Action: app.Action(args.Optional("storage-pool"), with.RequiredFlags("storage-pool"), with.Auth, func(c *app.Context) error {
					storagePool, err := c.Client().GetStoragePool(c.String("storage-pool"))
					if err != nil {
						return err
					}
					return c.OutputInDesiredForm(storagePool, output.Table)
				}),
			},
			{
				Name:      "migrating discs",
				Usage:     "shows a list of migrating discs",
				UsageText: "--admin show migrating_discs [--json]",
				Flags:     app.OutputFlags("migrating discs", "array"),
				Action: app.Action(with.Auth, func(ctx *app.Context) error {
					discs, err := ctx.Client().GetMigratingDiscs()
					if err != nil {
						return err
					}
					// this is super horrid :|
					if ctx.String("table-fields") == "" {
						err := ctx.Context.Set("table-fields", "ID, StoragePool, NewStoragePool, StorageGrade, NewStorageGrade, Size, MigrationProgress, MigrationEta, MigrationSpeed")
						if err != nil {
							return err
						}
					}
					fmt.Fprintln(ctx.App().Writer, "Storage sizes are in MB, speeds in MB/s, and times in seconds.")
					return ctx.OutputInDesiredForm(discs, output.Table)

				}),
			},
			{
				Name:      "migrating vms",
				Usage:     "shows a list of migrating servers",
				UsageText: "--admin show migrating_vms [--json]",
				Flags:     app.OutputFlags("migrating servers", "array"),
				Action: app.Action(with.Auth, func(c *app.Context) error {
					vms, err := c.Client().GetMigratingVMs()
					if err != nil {
						return err
					}
					return c.OutputInDesiredForm(vms, output.Table)
				}),
			},
			{
				Name:      "migration",
				Usage:     "shows a migration job",
				UsageText: "--admin show migration [--json] <id>",
				Flags: append(app.OutputFlags("migration job", "object"),
					cli.IntFlag{
						Name:  "id",
						Usage: "the ID of the migration job",
					},
				),
				Action: app.Action(with.Auth, args.Optional("id"), with.RequiredFlags("id"), func(c *app.Context) error {
					mj, err := brainRequests.GetMigrationJob(c.Client(), c.Int("id"))
					if err != nil {
						return err
					}

					mj.Active, err = brainRequests.GetMigrationJobActiveMigrations(c.Client(), c.Int("id"))
					if err != nil {
						return err
					}

					return c.OutputInDesiredForm(mj)
				}),
			},
			{
				Name:      "migrations",
				Usage:     "shows all unfinished migration jobs",
				UsageText: "--admin show migrations",
				Action: app.Action(with.Auth, func(c *app.Context) error {
					mjs, err := brainRequests.GetMigrationJobs(c.Client())
					if err != nil {
						return err
					}

					return c.OutputInDesiredForm(mjs)
				}),
			},
			{
				Name:      "stopped eligible vms",
				Usage:     "shows a list of stopped VMs that should be running",
				UsageText: "--admin show stopped_eligible_vms [--json]",
				Flags:     app.OutputFlags("servers", "array"),
				Action: app.Action(with.Auth, func(c *app.Context) error {
					vms, err := c.Client().GetStoppedEligibleVMs()
					if err != nil {
						return err
					}
					return c.OutputInDesiredForm(vms, output.Table)
				}),
			},
			{
				Name:      "recent vms",
				Usage:     "shows a list of stopped VMs that should be running",
				UsageText: "--admin show recent_vms [--json | --table] [--table-fields <fields> | --table-fields help]",
				Flags:     app.OutputFlags("servers", "array"),
				Action: app.Action(with.Auth, func(c *app.Context) error {
					vms, err := c.Client().GetRecentVMs()
					if err != nil {
						return err
					}
					return c.OutputInDesiredForm(vms, output.Table)
				}),
			},
		},
	})
}
<commit_msg>append the new show commands subfolder<commit_after>package admin

import (
	"fmt"

	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/app"
	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/app/args"
	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/app/with"
	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/commands/admin/show"
	"github.com/BytemarkHosting/bytemark-client/lib/output"
	brainRequests "github.com/BytemarkHosting/bytemark-client/lib/requests/brain"
	"github.com/urfave/cli"
)

func init() {
	Commands = append(Commands, cli.Command{
		Name:        "show",
		Action:      cli.ShowSubcommandHelp,
		Subcommands: append(showCommands, show.Commands...),
	})
}

var showCommands = []cli.Command{
	{
		Name:      "vlans",
		Usage:     "shows available VLANs",
		UsageText: "--admin show vlans [--json]",
		Flags:     app.OutputFlags("VLANs", "array"),
		Action: app.Action(with.Auth, func(c *app.Context) error {
			vlans, err := c.Client().GetVLANs()
			if err != nil {
				return err
			}
			return c.OutputInDesiredForm(vlans, output.Table)
		}),
	},
	{
		Name:        "disc by id",
		Usage:       "displays details about a disc",
		UsageText:   "show disc by id [--json] <id>",
		Description: `Displays a collection of details about the disc.`,
		Flags: append(app.OutputFlags("disc details", "object"),
			cli.IntFlag{
				Name:  "disc",
				Usage: "the disc to display",
			},
		),
		Action: app.Action(with.Auth, args.Optional("disc"), with.RequiredFlags("disc"), func(c *app.Context) error {
			disc, err := c.Client().GetDiscByID(c.Int("disc"))
			if err != nil {
				return err
			}

			return c.OutputInDesiredForm(disc)
		}),
	},
	{
		Name:      "vlan",
		Usage:     "shows the details of a VLAN",
		UsageText: "--admin show vlan [--json] <num>",
		Flags: append(app.OutputFlags("VLAN", "object"),
			cli.IntFlag{
				Name:  "num",
				Usage: "the num of the VLAN to display",
			},
		),
		Action: app.Action(args.Optional("num"), with.RequiredFlags("num"), with.Auth, func(c *app.Context) error {
			vlan, err := c.Client().GetVLAN(c.Int("num"))
			if err != nil {
				return err
			}
			return c.OutputInDesiredForm(vlan, output.Table)
		}),
	},
	{
		Name:      "ip ranges",
		Usage:     "shows all IP ranges",
		UsageText: "--admin show ip ranges [--json]",
		Flags:     app.OutputFlags("ip ranges", "array"),
		Action: app.Action(with.Auth, func(c *app.Context) error {
			ipRanges, err := c.Client().GetIPRanges()
			if err != nil {
				return err
			}
			return c.OutputInDesiredForm(ipRanges, output.Table)
		}),
	},
	{
		Name:      "ip range",
		Usage:     "shows the details of an IP range",
		UsageText: "--admin show ip range [--json] <ip-range>",
		Flags: append(app.OutputFlags("ip range details", "object"),
			cli.StringFlag{
				Name:  "ip-range",
				Usage: "the ID or CIDR representation of the IP range to display",
			},
		),
		Action: app.Action(args.Optional("ip-range"), with.RequiredFlags("ip-range"), with.Auth, func(c *app.Context) error {
			ipRange, err := c.Client().GetIPRange(c.String("ip-range"))
			if err != nil {
				return err
			}
			return c.OutputInDesiredForm(ipRange, output.Table)
		}),
	},
	{
		Name:      "heads",
		Usage:     "shows the details of all heads",
		UsageText: "--admin show heads [--json]",
		Flags:     app.OutputFlags("heads", "array"),
		Action: app.Action(with.Auth, func(c *app.Context) error {
			heads, err := c.Client().GetHeads()
			if err != nil {
				return err
			}
			return c.OutputInDesiredForm(heads, output.Table)
		}),
	},
	{
		Name:      "head",
		Usage:     "shows the details of the specified head",
		UsageText: "--admin show head <head> [--json]",
		Flags: append(app.OutputFlags("head details", "object"),
			cli.StringFlag{
				Name:  "head",
				Usage: "the ID of the head to display",
			},
		),
		Action: app.Action(args.Optional("head"), with.RequiredFlags("head"), with.Auth, func(c *app.Context) error {
			head, err := c.Client().GetHead(c.String("head"))
			if err != nil {
				return err
			}
			return c.OutputInDesiredForm(head, output.Table)
		}),
	},
	{
		Name:      "tails",
		Usage:     "shows the details of all tails",
		UsageText: "--admin show tails [--json]",
		Flags:     app.OutputFlags("tails", "array"),
		Action: app.Action(with.Auth, func(c *app.Context) error {
			tails, err := c.Client().GetTails()
			if err != nil {
				return err
			}
			return c.OutputInDesiredForm(tails, output.Table)
		}),
	},
	{
		Name:      "tail",
		Usage:     "shows the details of the specified tail",
		UsageText: "--admin show tail <tail> [--json]",
		Flags: append(app.OutputFlags("tail details", "object"),
			cli.StringFlag{
				Name:  "tail",
				Usage: "the ID of the tail to display",
			},
		),
		Action: app.Action(args.Optional("tail"), with.RequiredFlags("tail"), with.Auth, func(c *app.Context) error {
			tail, err := c.Client().GetTail(c.String("tail"))
			if err != nil {
				return err
			}
			return c.OutputInDesiredForm(tail, output.Table)
		}),
	},
	{
		Name:      "storage pools",
		Usage:     "shows the details of all storage pools",
		UsageText: "--admin show storage pools [--json]",
		Flags:     app.OutputFlags("storage pools", "array"),
		Action: app.Action(with.Auth, func(c *app.Context) error {
			storagePools, err := c.Client().GetStoragePools()
			if err != nil {
				return err
			}
			return c.OutputInDesiredForm(storagePools, output.Table)
		}),
	},
	{
		Name:      "storage pool",
		Usage:     "shows the details of the specified storage pool",
		UsageText: "--admin show storage pool [--json] <storage-pool>",
		Flags: append(app.OutputFlags("storage pool", "object"),
			cli.StringFlag{
				Name:  "storage-pool",
				Usage: "The ID or label of the storage pool to display",
			},
		),
		Action: app.Action(args.Optional("storage-pool"), with.RequiredFlags("storage-pool"), with.Auth, func(c *app.Context) error {
			storagePool, err := c.Client().GetStoragePool(c.String("storage-pool"))
			if err != nil {
				return err
			}
			return c.OutputInDesiredForm(storagePool, output.Table)
		}),
	},
	{
		Name:      "migrating discs",
		Usage:     "shows a list of migrating discs",
		UsageText: "--admin show migrating_discs [--json]",
		Flags:     app.OutputFlags("migrating discs", "array"),
		Action: app.Action(with.Auth, func(ctx *app.Context) error {
			discs, err := ctx.Client().GetMigratingDiscs()
			if err != nil {
				return err
			}
			// this is super horrid :|
			if ctx.String("table-fields") == "" {
				err := ctx.Context.Set("table-fields", "ID, StoragePool, NewStoragePool, StorageGrade, NewStorageGrade, Size, MigrationProgress, MigrationEta, MigrationSpeed")
				if err != nil {
					return err
				}
			}
			fmt.Fprintln(ctx.App().Writer, "Storage sizes are in MB, speeds in MB/s, and times in seconds.")
			return ctx.OutputInDesiredForm(discs, output.Table)

		}),
	},
	{
		Name:      "migrating vms",
		Usage:     "shows a list of migrating servers",
		UsageText: "--admin show migrating_vms [--json]",
		Flags:     app.OutputFlags("migrating servers", "array"),
		Action: app.Action(with.Auth, func(c *app.Context) error {
			vms, err := c.Client().GetMigratingVMs()
			if err != nil {
				return err
			}
			return c.OutputInDesiredForm(vms, output.Table)
		}),
	},
	{
		Name:      "migration",
		Usage:     "shows a migration job",
		UsageText: "--admin show migration [--json] <id>",
		Flags: append(app.OutputFlags("migration job", "object"),
			cli.IntFlag{
				Name:  "id",
				Usage: "the ID of the migration job",
			},
		),
		Action: app.Action(with.Auth, args.Optional("id"), with.RequiredFlags("id"), func(c *app.Context) error {
			mj, err := brainRequests.GetMigrationJob(c.Client(), c.Int("id"))
			if err != nil {
				return err
			}

			mj.Active, err = brainRequests.GetMigrationJobActiveMigrations(c.Client(), c.Int("id"))
			if err != nil {
				return err
			}

			return c.OutputInDesiredForm(mj)
		}),
	},
	{
		Name:      "migrations",
		Usage:     "shows all unfinished migration jobs",
		UsageText: "--admin show migrations",
		Action: app.Action(with.Auth, func(c *app.Context) error {
			mjs, err := brainRequests.GetMigrationJobs(c.Client())
			if err != nil {
				return err
			}

			return c.OutputInDesiredForm(mjs)
		}),
	},
	{
		Name:      "stopped eligible vms",
		Usage:     "shows a list of stopped VMs that should be running",
		UsageText: "--admin show stopped_eligible_vms [--json]",
		Flags:     app.OutputFlags("servers", "array"),
		Action: app.Action(with.Auth, func(c *app.Context) error {
			vms, err := c.Client().GetStoppedEligibleVMs()
			if err != nil {
				return err
			}
			return c.OutputInDesiredForm(vms, output.Table)
		}),
	},
}
<|endoftext|>


<commit_before>package gencorpus

import (
	"context"
	"encoding/csv"
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"runtime"
	"strings"
	"sync/atomic"
	"time"

	"github.com/google/subcommands"
	"github.com/nelhage/taktician/ai"
	"github.com/nelhage/taktician/prove"
	"github.com/nelhage/taktician/ptn"
	"github.com/nelhage/taktician/tak"
	"golang.org/x/sync/errgroup"
)

type Command struct {
	seed int64
	size int

	games int

	epsilon float64
	depth   int
	threads int

	limit    time.Duration
	analysis string

	stats  bool
	output string
}

func (*Command) Name() string     { return "gencorpus" }
func (*Command) Synopsis() string { return "Generate a corpus of 3x3 positions" }
func (*Command) Usage() string {
	return `gencorpus [flags]
`
}

func (c *Command) SetFlags(flags *flag.FlagSet) {
	flags.IntVar(&c.size, "size", 3, "what size to analyze")
	flags.IntVar(&c.games, "games", 100, "games to generate")
	flags.Int64Var(&c.seed, "seed", 0, "Random seed")
	flags.IntVar(&c.threads, "threads", runtime.NumCPU(), "Number of threads")

	flags.StringVar(&c.analysis, "analysis", "minimax", "Analysis engine to run: minimax,dfpn,winning,none")
	flags.DurationVar(&c.limit, "limit", 5*time.Second, "Minimax time limit when scoring")

	flags.BoolVar(&c.stats, "stats", false, "compute and print stats")
	flags.IntVar(&c.depth, "depth", 2, "minimax depth")
	flags.Float64Var(&c.epsilon, "epsilon", 0.95, "epsilon for epsilon-greedy generation")

	flags.StringVar(&c.output, "output", "positions.txt", "output file")

}

type game struct {
	positions []*tak.Position
	moves     []tak.Move
}

func growslice[T any](sl []T, newlen int) []T {
	if len(sl) >= newlen {
		return sl
	}
	newsl := make([]T, newlen)
	copy(newsl, sl)
	return newsl
}

type entry struct {
	pos        *tak.Position
	move       tak.Move
	value      float64
	otherMoves []tak.Move
}

func fmtMoves(moves []tak.Move) string {
	var sb strings.Builder
	for i, m := range moves {
		if i != 0 {
			sb.WriteString(",")
		}
		sb.WriteString(ptn.FormatMove(m))
	}
	return sb.String()
}

func (c *Command) Execute(ctx context.Context, flag *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
	var byLength []int
	var posCount []map[uint64]int

	grp, ctx := errgroup.WithContext(ctx)

	games := make(chan *game)
	positions := make(chan *tak.Position)

	grp.Go(func() error {
		c.generateGames(ctx, games)
		return nil
	})
	grp.Go(func() error {
		defer close(positions)
		rng := rand.New(rand.NewSource(c.seed))

		seen := make(map[uint64]struct{})

		for g := range games {
			if c.stats {
				moves := len(g.positions)
				byLength = growslice(byLength, moves)
				byLength[moves-1] += 1
				posCount = growslice(posCount, moves+1)
				for i, p := range g.positions {
					if posCount[i] == nil {
						posCount[i] = make(map[uint64]int)
					}
					posCount[i][p.Hash()] += 1
				}
			}
			// select position
			var idx int
			r := rng.Float64()
			if r < 0.01 {
				idx = int(rng.Int31n(4))
			} else if r < 0.25 {
				idx = 4 + int(rng.Int31n(5))
			} else if r < 0.95 {
				npos := len(g.positions)
				if npos <= 9 {
					continue
				}
				idx = 9 + int(rng.Int31n(int32(npos)-9))
			}
			if idx >= len(g.positions)-1 {
				continue
			}
			pos := g.positions[idx]
			if _, ok := seen[pos.Hash()]; ok {
				continue
			}
			seen[pos.Hash()] = struct{}{}
			select {
			case positions <- pos:
			case <-ctx.Done():
				return ctx.Err()
			}
		}

		if c.stats {
			for i := range byLength {
				log.Printf("ply=%3d games=%3d uniq=%4d", i, byLength[i], len(posCount[i]))
			}
		}
		return nil
	})

	results := make(chan entry)
	grp.Go(func() error {
		c.evaluate(ctx, positions, results)
		return nil
	})
	grp.Go(func() error {
		fh, err := os.Create(c.output)
		if err != nil {
			return fmt.Errorf("open %q: %w", c.output, err)
		}
		defer fh.Close()
		wr := csv.NewWriter(fh)
		defer wr.Flush()

		for e := range results {
			wr.Write([]string{
				ptn.FormatTPS(e.pos),
				ptn.FormatMove(e.move),
				fmt.Sprintf("%+f", e.value),
				fmtMoves(e.otherMoves),
			})
		}
		return nil
	})

	if err := grp.Wait(); err != nil {
		log.Println(err.Error())
		return subcommands.ExitFailure
	}

	return subcommands.ExitSuccess
}

func (c *Command) evaluate(ctx context.Context, positions <-chan *tak.Position, results chan<- entry) {
	defer close(results)
	grp, ctx := errgroup.WithContext(ctx)
	for i := 0; i < c.threads; i++ {
		grp.Go(func() error {
			var analyze func(p *entry)
			if c.analysis == "dfpn" {
				prover := prove.NewDFPN(&prove.DFPNConfig{
					// Attacker: tak.White,
					TableMem: 100 * 1 << 20,
				})
				analyze = func(e *entry) {
					res, _ := prover.Prove(e.pos)
					e.move = res.Move

					if res.Result == prove.EvalUnknown {
						log.Printf("unprovable! %q bounds=%d,%d",
							ptn.FormatTPS(e.pos),
							res.Proof,
							res.Disproof,
						)
					}
					if res.Result == prove.EvalTrue {
						e.value = 1.0
					} else {
						e.value = -1.0
					}
				}
			} else if c.analysis == "minimax" {
				mm := ai.NewMinimax(ai.MinimaxConfig{
					Size:     c.size,
					TableMem: 100 * 1 << 20,
				})

				analyze = func(e *entry) {
					ctx, cancel := context.WithTimeout(context.Background(), c.limit)
					defer cancel()
					pv, val, _ := mm.Analyze(
						ctx,
						e.pos,
					)
					e.move = pv[0]
					if val > ai.WinThreshold {
						e.value = 1.0
					} else if val < -ai.WinThreshold {
						e.value = -1.0
					} else if val > 0 {
						e.value = 0.5
					} else if val < 0 {
						e.value = 0.5
					}
				}
			} else if c.analysis == "winning" {
				weights := ai.DefaultWeights[c.size]
				weights[ai.Terminal_Flats] = 0
				weights[ai.Terminal_Plies] = 0
				weights[ai.Terminal_OpponentReserves] = 0
				weights[ai.Terminal_Reserves] = 0
				mm := ai.NewMinimax(ai.MinimaxConfig{
					Size:     c.size,
					TableMem: 100 * 1 << 20,
					Evaluate: ai.MakeEvaluator(c.size, &weights),
				})

				analyze = func(e *entry) {
					ctx, cancel := context.WithTimeout(context.Background(), c.limit)
					defer cancel()
					_, val, _ := mm.Analyze(
						ctx,
						e.pos,
					)
					if val <= ai.WinThreshold {
						return
					}
					pvs, _, _ := mm.AnalyzeAll(
						context.Background(),
						e.pos,
					)
					e.value = 1.0

					e.move = pvs[0][0]
					for _, pv := range pvs[1:] {
						e.otherMoves = append(e.otherMoves, pv[0])
					}
				}
			} else if c.analysis == "none" {
				analyze = func(e *entry) {}
			} else {
				log.Fatalf("unknown analysis: %q", c.analysis)
			}

			for p := range positions {
				ent := entry{pos: p}
				analyze(&ent)
				if c.analysis == "winning" && ent.value != 1.0 {
					continue
				}

				select {
				case results <- ent:
				case <-ctx.Done():
					return ctx.Err()
				}

			}
			return nil
		})
	}
	grp.Wait()
}

func (c *Command) generateGames(ctx context.Context, games chan<- *game) {
	defer close(games)
	todo := int64(c.games)

	grp, ctx := errgroup.WithContext(ctx)
	for i := 0; i < c.threads; i++ {
		grp.Go(func() error {
			c.generateWorker(ctx, games, &todo, i)
			return nil
		})
	}
	grp.Wait()
}

const prime = 1099511628211

func (c *Command) generateWorker(ctx context.Context, games chan<- *game, todo *int64, id int) {
	rng := rand.New(rand.NewSource(prime*c.seed + int64(id)))
	mm := ai.NewMinimax(ai.MinimaxConfig{
		Size:     c.size,
		Seed:     rng.Int63(),
		Depth:    c.depth,
		TableMem: -1,
	})
	rnd := ai.NewRandom(rng.Int63())
	for {
		gid := atomic.AddInt64(todo, -1)
		if gid < 0 {
			return
		}
		pos := tak.New(tak.Config{Size: c.size})
		g := game{positions: []*tak.Position{pos}}
		for {
			if done, _ := pos.GameOver(); done {
				break
			}
			var player ai.TakPlayer
			if rng.Float64() < c.epsilon {
				player = rnd
			} else {
				player = mm
			}

			for {
				m := player.GetMove(ctx, pos)
				child, err := pos.Move(m)
				if err != nil {
					continue
				}
				g.positions = append(g.positions, child)
				g.moves = append(g.moves, m)
				pos = child
				break
			}
		}
		select {
		case games <- &g:
		case <-ctx.Done():
			return
		}
	}
}
<commit_msg>Better analysis of all winning moves<commit_after>package gencorpus

import (
	"context"
	"encoding/csv"
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"runtime"
	"strings"
	"sync/atomic"
	"time"

	"github.com/google/subcommands"
	"github.com/nelhage/taktician/ai"
	"github.com/nelhage/taktician/prove"
	"github.com/nelhage/taktician/ptn"
	"github.com/nelhage/taktician/tak"
	"golang.org/x/sync/errgroup"
)

type Command struct {
	seed int64
	size int

	games int

	epsilon float64
	depth   int
	threads int

	limit    time.Duration
	analysis string

	stats  bool
	output string
}

func (*Command) Name() string     { return "gencorpus" }
func (*Command) Synopsis() string { return "Generate a corpus of 3x3 positions" }
func (*Command) Usage() string {
	return `gencorpus [flags]
`
}

func (c *Command) SetFlags(flags *flag.FlagSet) {
	flags.IntVar(&c.size, "size", 3, "what size to analyze")
	flags.IntVar(&c.games, "games", 100, "games to generate")
	flags.Int64Var(&c.seed, "seed", 0, "Random seed")
	flags.IntVar(&c.threads, "threads", runtime.NumCPU(), "Number of threads")

	flags.StringVar(&c.analysis, "analysis", "minimax", "Analysis engine to run: minimax,dfpn,winning,none")
	flags.DurationVar(&c.limit, "limit", 5*time.Second, "Minimax time limit when scoring")

	flags.BoolVar(&c.stats, "stats", false, "compute and print stats")
	flags.IntVar(&c.depth, "depth", 2, "minimax depth")
	flags.Float64Var(&c.epsilon, "epsilon", 0.95, "epsilon for epsilon-greedy generation")

	flags.StringVar(&c.output, "output", "positions.txt", "output file")

}

type game struct {
	positions []*tak.Position
	moves     []tak.Move
}

func growslice[T any](sl []T, newlen int) []T {
	if len(sl) >= newlen {
		return sl
	}
	newsl := make([]T, newlen)
	copy(newsl, sl)
	return newsl
}

type entry struct {
	pos        *tak.Position
	move       tak.Move
	value      float64
	otherMoves []tak.Move
}

func fmtMoves(moves []tak.Move) string {
	var sb strings.Builder
	for i, m := range moves {
		if i != 0 {
			sb.WriteString(",")
		}
		sb.WriteString(ptn.FormatMove(m))
	}
	return sb.String()
}

func (c *Command) Execute(ctx context.Context, flag *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
	var byLength []int
	var posCount []map[uint64]int

	grp, ctx := errgroup.WithContext(ctx)

	games := make(chan *game)
	positions := make(chan *tak.Position)

	grp.Go(func() error {
		c.generateGames(ctx, games)
		return nil
	})
	grp.Go(func() error {
		defer close(positions)
		rng := rand.New(rand.NewSource(c.seed))

		seen := make(map[uint64]struct{})

		for g := range games {
			if c.stats {
				moves := len(g.positions)
				byLength = growslice(byLength, moves)
				byLength[moves-1] += 1
				posCount = growslice(posCount, moves+1)
				for i, p := range g.positions {
					if posCount[i] == nil {
						posCount[i] = make(map[uint64]int)
					}
					posCount[i][p.Hash()] += 1
				}
			}
			// select position
			var idx int
			r := rng.Float64()
			if r < 0.01 {
				idx = int(rng.Int31n(4))
			} else if r < 0.25 {
				idx = 4 + int(rng.Int31n(5))
			} else if r < 0.95 {
				npos := len(g.positions)
				if npos <= 9 {
					continue
				}
				idx = 9 + int(rng.Int31n(int32(npos)-9))
			}
			if idx >= len(g.positions)-1 {
				continue
			}
			pos := g.positions[idx]
			if _, ok := seen[pos.Hash()]; ok {
				continue
			}
			seen[pos.Hash()] = struct{}{}
			select {
			case positions <- pos:
			case <-ctx.Done():
				return ctx.Err()
			}
		}

		if c.stats {
			for i := range byLength {
				log.Printf("ply=%3d games=%3d uniq=%4d", i, byLength[i], len(posCount[i]))
			}
		}
		return nil
	})

	results := make(chan entry)
	grp.Go(func() error {
		c.evaluate(ctx, positions, results)
		return nil
	})
	grp.Go(func() error {
		fh, err := os.Create(c.output)
		if err != nil {
			return fmt.Errorf("open %q: %w", c.output, err)
		}
		defer fh.Close()
		wr := csv.NewWriter(fh)
		defer wr.Flush()

		for e := range results {
			wr.Write([]string{
				ptn.FormatTPS(e.pos),
				ptn.FormatMove(e.move),
				fmt.Sprintf("%+f", e.value),
				fmtMoves(e.otherMoves),
			})
		}
		return nil
	})

	if err := grp.Wait(); err != nil {
		log.Println(err.Error())
		return subcommands.ExitFailure
	}

	return subcommands.ExitSuccess
}

func (c *Command) analyzeWinning(mm *ai.MinimaxAI, e *entry) {
	ctx, cancel := context.WithTimeout(context.Background(), c.limit)
	defer cancel()
	pv, val, _ := mm.Analyze(
		ctx,
		e.pos,
	)
	if val <= ai.WinThreshold {
		return
	}

	e.move = pv[0]

	var buf [100]tak.Move
	moves := e.pos.AllMoves(buf[:0])
	var ch *tak.Position
	var err error
	for _, m := range moves {
		ch, err = e.pos.MovePreallocated(m, ch)
		if err != nil {
			continue
		}
		if ok, winner := ch.GameOver(); ok {
			if winner == e.pos.ToMove() {
				e.otherMoves = append(e.otherMoves, m)
			}
			continue
		}
		_, val, _ = mm.Analyze(ctx, ch)
		if val < -ai.WinThreshold {
			e.otherMoves = append(e.otherMoves, m)
		}

		if ctx.Err() != nil {
			return
		}
	}

	e.value = 1.0
}

func (c *Command) evaluate(ctx context.Context, positions <-chan *tak.Position, results chan<- entry) {
	defer close(results)
	grp, ctx := errgroup.WithContext(ctx)
	for i := 0; i < c.threads; i++ {
		grp.Go(func() error {
			var analyze func(p *entry)
			if c.analysis == "dfpn" {
				prover := prove.NewDFPN(&prove.DFPNConfig{
					// Attacker: tak.White,
					TableMem: 100 * 1 << 20,
				})
				analyze = func(e *entry) {
					res, _ := prover.Prove(e.pos)
					e.move = res.Move

					if res.Result == prove.EvalUnknown {
						log.Printf("unprovable! %q bounds=%d,%d",
							ptn.FormatTPS(e.pos),
							res.Proof,
							res.Disproof,
						)
					}
					if res.Result == prove.EvalTrue {
						e.value = 1.0
					} else {
						e.value = -1.0
					}
				}
			} else if c.analysis == "minimax" {
				mm := ai.NewMinimax(ai.MinimaxConfig{
					Size:     c.size,
					TableMem: 100 * 1 << 20,
				})

				analyze = func(e *entry) {
					ctx, cancel := context.WithTimeout(context.Background(), c.limit)
					defer cancel()
					pv, val, _ := mm.Analyze(
						ctx,
						e.pos,
					)
					e.move = pv[0]
					if val > ai.WinThreshold {
						e.value = 1.0
					} else if val < -ai.WinThreshold {
						e.value = -1.0
					} else if val > 0 {
						e.value = 0.5
					} else if val < 0 {
						e.value = 0.5
					}
				}
			} else if c.analysis == "winning" {
				weights := ai.DefaultWeights[c.size]
				weights[ai.Terminal_Flats] = 0
				weights[ai.Terminal_Plies] = 0
				weights[ai.Terminal_OpponentReserves] = 0
				weights[ai.Terminal_Reserves] = 0
				mm := ai.NewMinimax(ai.MinimaxConfig{
					Size:     c.size,
					TableMem: 100 * 1 << 20,
					Evaluate: ai.MakeEvaluator(c.size, &weights),
				})

				analyze = func(e *entry) {
					c.analyzeWinning(mm, e)
				}
			} else if c.analysis == "none" {
				analyze = func(e *entry) {}
			} else {
				log.Fatalf("unknown analysis: %q", c.analysis)
			}

			for p := range positions {
				ent := entry{pos: p}
				analyze(&ent)
				if c.analysis == "winning" && ent.value != 1.0 {
					continue
				}

				select {
				case results <- ent:
				case <-ctx.Done():
					return ctx.Err()
				}

			}
			return nil
		})
	}
	grp.Wait()
}

func (c *Command) generateGames(ctx context.Context, games chan<- *game) {
	defer close(games)
	todo := int64(c.games)

	grp, ctx := errgroup.WithContext(ctx)
	for i := 0; i < c.threads; i++ {
		grp.Go(func() error {
			c.generateWorker(ctx, games, &todo, i)
			return nil
		})
	}
	grp.Wait()
}

const prime = 1099511628211

func (c *Command) generateWorker(ctx context.Context, games chan<- *game, todo *int64, id int) {
	rng := rand.New(rand.NewSource(prime*c.seed + int64(id)))
	mm := ai.NewMinimax(ai.MinimaxConfig{
		Size:     c.size,
		Seed:     rng.Int63(),
		Depth:    c.depth,
		TableMem: -1,
	})
	rnd := ai.NewRandom(rng.Int63())
	for {
		gid := atomic.AddInt64(todo, -1)
		if gid < 0 {
			return
		}
		pos := tak.New(tak.Config{Size: c.size})
		g := game{positions: []*tak.Position{pos}}
		for {
			if done, _ := pos.GameOver(); done {
				break
			}
			var player ai.TakPlayer
			if rng.Float64() < c.epsilon {
				player = rnd
			} else {
				player = mm
			}

			for {
				m := player.GetMove(ctx, pos)
				child, err := pos.Move(m)
				if err != nil {
					continue
				}
				g.positions = append(g.positions, child)
				g.moves = append(g.moves, m)
				pos = child
				break
			}
		}
		select {
		case games <- &g:
		case <-ctx.Done():
			return
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package phases

import (
	"fmt"
	"net"

	"github.com/spf13/cobra"

	netutil "k8s.io/apimachinery/pkg/util/net"
	"k8s.io/apimachinery/pkg/util/validation/field"
	"k8s.io/client-go/pkg/api"
	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
	kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
	certphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
)

func NewCmdCerts() *cobra.Command {
	cmd := &cobra.Command{
		Use:     "certs",
		Aliases: []string{"certificates"},
		Short:   "Generate certificates for a Kubernetes cluster.",
		RunE:    subCmdRunE("certs"),
	}

	cmd.AddCommand(NewCmdSelfSign())
	return cmd
}

func NewCmdSelfSign() *cobra.Command {
	// TODO: Move this into a dedicated Certificates Phase API object
	cfg := &kubeadmapiext.MasterConfiguration{}
	// Default values for the cobra help text
	api.Scheme.Default(cfg)

	cmd := &cobra.Command{
		Use:   "selfsign",
		Short: "Generate the CA, APIServer signing/client cert, the ServiceAccount public/private keys and a CA and client cert for the front proxy",
		Run: func(cmd *cobra.Command, args []string) {

			// Run the defaulting once again to take passed flags into account
			api.Scheme.Default(cfg)
			internalcfg := &kubeadmapi.MasterConfiguration{}
			api.Scheme.Convert(cfg, internalcfg, nil)

			err := RunSelfSign(internalcfg)
			kubeadmutil.CheckErr(err)
		},
	}
	cmd.Flags().StringVar(&cfg.Networking.DNSDomain, "dns-domain", cfg.Networking.DNSDomain, "The DNS Domain for the Kubernetes cluster.")
	cmd.Flags().StringVar(&cfg.CertificatesDir, "cert-dir", cfg.CertificatesDir, "The path where to save and store the certificates")
	cmd.Flags().StringVar(&cfg.Networking.ServiceSubnet, "service-cidr", cfg.Networking.ServiceSubnet, "The subnet for the Services in the cluster.")
	cmd.Flags().StringSliceVar(&cfg.APIServerCertSANs, "cert-altnames", []string{}, "Optional extra altnames to use for the API Server serving cert. Can be both IP addresses and dns names.")
	return cmd
}

// RunSelfSign generates certificate assets in the specified directory
func RunSelfSign(config *kubeadmapi.MasterConfiguration) error {
	if err := validateArgs(config); err != nil {
		return fmt.Errorf("The argument validation failed: %v", err)
	}

	// If it's possible to detect the default IP, add it to the SANs as well. Otherwise, just go with the provided ones
	ip, err := netutil.ChooseBindAddress(net.ParseIP(config.API.AdvertiseAddress))
	if err == nil {
		config.API.AdvertiseAddress = ip.String()
	}

	err = certphase.CreatePKIAssets(config)
	if err != nil {
		return err
	}
	return nil
}

func validateArgs(config *kubeadmapi.MasterConfiguration) error {
	allErrs := field.ErrorList{}
	allErrs = append(allErrs, validation.ValidateNetworking(&config.Networking, field.NewPath("networking"))...)
	allErrs = append(allErrs, validation.ValidateAbsolutePath(config.CertificatesDir, field.NewPath("cert-dir"))...)
	allErrs = append(allErrs, validation.ValidateAPIServerCertSANs(config.APIServerCertSANs, field.NewPath("cert-altnames"))...)
	return allErrs.ToAggregate()
}
<commit_msg>Add --apiserver-advertise-address option to kubeadm alpha phase certs selfsign<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package phases

import (
	"fmt"
	"net"

	"github.com/spf13/cobra"

	netutil "k8s.io/apimachinery/pkg/util/net"
	"k8s.io/apimachinery/pkg/util/validation/field"
	"k8s.io/client-go/pkg/api"
	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
	kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
	certphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
)

func NewCmdCerts() *cobra.Command {
	cmd := &cobra.Command{
		Use:     "certs",
		Aliases: []string{"certificates"},
		Short:   "Generate certificates for a Kubernetes cluster.",
		RunE:    subCmdRunE("certs"),
	}

	cmd.AddCommand(NewCmdSelfSign())
	return cmd
}

func NewCmdSelfSign() *cobra.Command {
	// TODO: Move this into a dedicated Certificates Phase API object
	cfg := &kubeadmapiext.MasterConfiguration{}
	// Default values for the cobra help text
	api.Scheme.Default(cfg)

	cmd := &cobra.Command{
		Use:   "selfsign",
		Short: "Generate the CA, APIServer signing/client cert, the ServiceAccount public/private keys and a CA and client cert for the front proxy",
		Run: func(cmd *cobra.Command, args []string) {

			// Run the defaulting once again to take passed flags into account
			api.Scheme.Default(cfg)
			internalcfg := &kubeadmapi.MasterConfiguration{}
			api.Scheme.Convert(cfg, internalcfg, nil)

			err := RunSelfSign(internalcfg)
			kubeadmutil.CheckErr(err)
		},
	}
	cmd.Flags().StringVar(&cfg.Networking.DNSDomain, "dns-domain", cfg.Networking.DNSDomain, "The DNS Domain for the Kubernetes cluster.")
	cmd.Flags().StringVar(&cfg.CertificatesDir, "cert-dir", cfg.CertificatesDir, "The path where to save and store the certificates")
	cmd.Flags().StringVar(&cfg.Networking.ServiceSubnet, "service-cidr", cfg.Networking.ServiceSubnet, "The subnet for the Services in the cluster.")
	cmd.Flags().StringSliceVar(&cfg.APIServerCertSANs, "cert-altnames", []string{}, "Optional extra altnames to use for the API Server serving cert. Can be both IP addresses and dns names.")
	cmd.Flags().StringVar(&cfg.API.AdvertiseAddress, "apiserver-advertise-address", cfg.API.AdvertiseAddress, "The IP address the API Server will advertise it's listening on. 0.0.0.0 means the default network interface's address.")

	return cmd
}

// RunSelfSign generates certificate assets in the specified directory
func RunSelfSign(config *kubeadmapi.MasterConfiguration) error {
	if err := validateArgs(config); err != nil {
		return fmt.Errorf("The argument validation failed: %v", err)
	}

	// If it's possible to detect the default IP, add it to the SANs as well. Otherwise, just go with the provided ones
	ip, err := netutil.ChooseBindAddress(net.ParseIP(config.API.AdvertiseAddress))
	if err == nil {
		config.API.AdvertiseAddress = ip.String()
	}

	err = certphase.CreatePKIAssets(config)
	if err != nil {
		return err
	}
	return nil
}

func validateArgs(config *kubeadmapi.MasterConfiguration) error {
	allErrs := field.ErrorList{}
	allErrs = append(allErrs, validation.ValidateNetworking(&config.Networking, field.NewPath("networking"))...)
	allErrs = append(allErrs, validation.ValidateAbsolutePath(config.CertificatesDir, field.NewPath("cert-dir"))...)
	allErrs = append(allErrs, validation.ValidateAPIServerCertSANs(config.APIServerCertSANs, field.NewPath("cert-altnames"))...)
	allErrs = append(allErrs, validation.ValidateIPFromString(config.API.AdvertiseAddress, field.NewPath("apiserver-advertise-address"))...)

	return allErrs.ToAggregate()
}
<|endoftext|>