<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build go1.8 // TODO(adonovan) determine which versions we need to test here
// +build !windows

package loader_test

import (
	"fmt"
	"go/token"
	"log"
	"path/filepath"
	"runtime"
	"sort"
	"strings"

	"golang.org/x/tools/go/loader"
)

func printProgram(prog *loader.Program) {
	// Created packages are the initial packages specified by a call
	// to CreateFromFilenames or CreateFromFiles.
	var names []string
	for _, info := range prog.Created {
		names = append(names, info.Pkg.Path())
	}
	fmt.Printf("created: %s\n", names)

	// Imported packages are the initial packages specified by a
	// call to Import or ImportWithTests.
	names = nil
	for _, info := range prog.Imported {
		if strings.Contains(info.Pkg.Path(), "internal") {
			continue // skip, to reduce fragility
		}
		names = append(names, info.Pkg.Path())
	}
	sort.Strings(names)
	fmt.Printf("imported: %s\n", names)

	// InitialPackages contains the union of created and imported.
	names = nil
	for _, info := range prog.InitialPackages() {
		names = append(names, info.Pkg.Path())
	}
	sort.Strings(names)
	fmt.Printf("initial: %s\n", names)

	// AllPackages contains all initial packages and their dependencies.
	names = nil
	for pkg := range prog.AllPackages {
		names = append(names, pkg.Path())
	}
	sort.Strings(names)
	fmt.Printf("all: %s\n", names)
}

func printFilenames(fset *token.FileSet, info *loader.PackageInfo) {
	var names []string
	for _, f := range info.Files {
		names = append(names, filepath.Base(fset.File(f.Pos()).Name()))
	}
	fmt.Printf("%s.Files: %s\n", info.Pkg.Path(), names)
}

// This example loads a set of packages and all of their dependencies
// from a typical command-line.  FromArgs parses a command line and
// makes calls to the other methods of Config shown in the examples that
// follow.
func ExampleConfig_FromArgs() {
	args := []string{"mytool", "unicode/utf8", "errors", "runtime", "--", "foo", "bar"}
	const wantTests = false

	var conf loader.Config
	rest, err := conf.FromArgs(args[1:], wantTests)
	prog, err := conf.Load()
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("rest: %s\n", rest)
	printProgram(prog)
	// Output:
	// rest: [foo bar]
	// created: []
	// imported: [errors runtime unicode/utf8]
	// initial: [errors runtime unicode/utf8]
	// all: [errors runtime runtime/internal/atomic runtime/internal/sys unicode/utf8 unsafe]
}

// This example creates and type-checks a single package (without tests)
// from a list of filenames, and loads all of its dependencies.
// (The input files are actually only a small part of the math/cmplx package.)
func ExampleConfig_CreateFromFilenames() {
	var conf loader.Config
	conf.CreateFromFilenames("math/cmplx",
		filepath.Join(runtime.GOROOT(), "src/math/cmplx/abs.go"),
		filepath.Join(runtime.GOROOT(), "src/math/cmplx/sin.go"))
	prog, err := conf.Load()
	if err != nil {
		log.Fatal(err)
	}

	printProgram(prog)
	// Output:
	// created: [math/cmplx]
	// imported: []
	// initial: [math/cmplx]
	// all: [math math/cmplx unsafe]
}

// In the examples below, for stability, the chosen packages are
// relatively small, platform-independent, and low-level (and thus
// infrequently changing).
// The strconv package has internal and external tests.

const hello = `package main

import "fmt"

func main() {
	fmt.Println("Hello, world.")
}
`

// This example creates and type-checks a package from a list of
// already-parsed files, and loads all its dependencies.
func ExampleConfig_CreateFromFiles() {
	var conf loader.Config
	f, err := conf.ParseFile("hello.go", hello)
	if err != nil {
		log.Fatal(err)
	}
	conf.CreateFromFiles("hello", f)
	prog, err := conf.Load()
	if err != nil {
		log.Fatal(err)
	}

	printProgram(prog)
	printFilenames(prog.Fset, prog.Package("strconv"))
	// Output:
	// created: [hello]
	// imported: []
	// initial: [hello]
	// all: [errors fmt hello internal/race io math os reflect runtime runtime/internal/atomic runtime/internal/sys strconv sync sync/atomic syscall time unicode/utf8 unsafe]
	// strconv.Files: [atob.go atof.go atoi.go decimal.go doc.go extfloat.go ftoa.go isprint.go itoa.go quote.go]
}

// This example imports three packages, including the tests for one of
// them, and loads all their dependencies.
func ExampleConfig_Import() {
	// ImportWithTest("strconv") causes strconv to include
	// internal_test.go, and creates an external test package,
	// strconv_test.
	// (Compare with the example of CreateFromFiles.)

	var conf loader.Config
	conf.Import("unicode/utf8")
	conf.Import("errors")
	conf.ImportWithTests("strconv")
	prog, err := conf.Load()
	if err != nil {
		log.Fatal(err)
	}

	printProgram(prog)
	printFilenames(prog.Fset, prog.Package("strconv"))
	printFilenames(prog.Fset, prog.Package("strconv_test"))
	// Output:
	// created: [strconv_test]
	// imported: [errors strconv unicode/utf8]
	// initial: [errors strconv strconv_test unicode/utf8]
	// all: [bufio bytes errors flag fmt internal/race io log math math/rand os reflect runtime runtime/debug runtime/internal/atomic runtime/internal/sys runtime/trace sort strconv strconv_test strings sync sync/atomic syscall testing time unicode unicode/utf8 unsafe]
	// strconv.Files: [atob.go atof.go atoi.go decimal.go doc.go extfloat.go ftoa.go isprint.go itoa.go quote.go internal_test.go]
	// strconv_test.Files: [atob_test.go atof_test.go atoi_test.go decimal_test.go example_test.go fp_test.go ftoa_test.go itoa_test.go quote_test.go strconv_test.go]
}
<commit_msg>go/loader: fix fragile example test<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build go1.8 // TODO(adonovan) determine which versions we need to test here
// +build !windows

package loader_test

import (
	"fmt"
	"go/token"
	"log"
	"path/filepath"
	"runtime"
	"sort"
	"strings"

	"golang.org/x/tools/go/loader"
)

func printProgram(prog *loader.Program) {
	// Created packages are the initial packages specified by a call
	// to CreateFromFilenames or CreateFromFiles.
	var names []string
	for _, info := range prog.Created {
		names = append(names, info.Pkg.Path())
	}
	fmt.Printf("created: %s\n", names)

	// Imported packages are the initial packages specified by a
	// call to Import or ImportWithTests.
	names = nil
	for _, info := range prog.Imported {
		if strings.Contains(info.Pkg.Path(), "internal") {
			continue // skip, to reduce fragility
		}
		names = append(names, info.Pkg.Path())
	}
	sort.Strings(names)
	fmt.Printf("imported: %s\n", names)

	// InitialPackages contains the union of created and imported.
	names = nil
	for _, info := range prog.InitialPackages() {
		names = append(names, info.Pkg.Path())
	}
	sort.Strings(names)
	fmt.Printf("initial: %s\n", names)

	// AllPackages contains all initial packages and their dependencies.
	names = nil
	for pkg := range prog.AllPackages {
		names = append(names, pkg.Path())
	}
	sort.Strings(names)
	fmt.Printf("all: %s\n", names)
}

func printFilenames(fset *token.FileSet, info *loader.PackageInfo) {
	var names []string
	for _, f := range info.Files {
		names = append(names, filepath.Base(fset.File(f.Pos()).Name()))
	}
	fmt.Printf("%s.Files: %s\n", info.Pkg.Path(), names)
}

// This example loads a set of packages and all of their dependencies
// from a typical command-line.  FromArgs parses a command line and
// makes calls to the other methods of Config shown in the examples that
// follow.
func ExampleConfig_FromArgs() {
	args := []string{"mytool", "unicode/utf8", "errors", "runtime", "--", "foo", "bar"}
	const wantTests = false

	var conf loader.Config
	rest, err := conf.FromArgs(args[1:], wantTests)
	prog, err := conf.Load()
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("rest: %s\n", rest)
	printProgram(prog)
	// Output:
	// rest: [foo bar]
	// created: []
	// imported: [errors runtime unicode/utf8]
	// initial: [errors runtime unicode/utf8]
	// all: [errors runtime runtime/internal/atomic runtime/internal/sys unicode/utf8 unsafe]
}

// This example creates and type-checks a single package (without tests)
// from a list of filenames, and loads all of its dependencies.
// (The input files are actually only a small part of the math/cmplx package.)
func ExampleConfig_CreateFromFilenames() {
	var conf loader.Config
	conf.CreateFromFilenames("math/cmplx",
		filepath.Join(runtime.GOROOT(), "src/math/cmplx/abs.go"),
		filepath.Join(runtime.GOROOT(), "src/math/cmplx/sin.go"))
	prog, err := conf.Load()
	if err != nil {
		log.Fatal(err)
	}

	printProgram(prog)
	// Output:
	// created: [math/cmplx]
	// imported: []
	// initial: [math/cmplx]
	// all: [math math/cmplx unsafe]
}

// In the examples below, for stability, the chosen packages are
// relatively small, platform-independent, and low-level (and thus
// infrequently changing).
// The strconv package has internal and external tests.

const hello = `package main

import "fmt"

func main() {
	fmt.Println("Hello, world.")
}
`

// This example creates and type-checks a package from a list of
// already-parsed files, and loads all its dependencies.
func ExampleConfig_CreateFromFiles() {
	var conf loader.Config
	f, err := conf.ParseFile("hello.go", hello)
	if err != nil {
		log.Fatal(err)
	}
	conf.CreateFromFiles("hello", f)
	prog, err := conf.Load()
	if err != nil {
		log.Fatal(err)
	}

	printProgram(prog)
	printFilenames(prog.Fset, prog.Package("strconv"))
	// Output:
	// created: [hello]
	// imported: []
	// initial: [hello]
	// all: [errors fmt hello internal/race io math os reflect runtime runtime/internal/atomic runtime/internal/sys strconv sync sync/atomic syscall time unicode/utf8 unsafe]
	// strconv.Files: [atob.go atof.go atoi.go decimal.go doc.go extfloat.go ftoa.go isprint.go itoa.go quote.go]
}

// This example imports three packages, including the tests for one of
// them, and loads all their dependencies.
func ExampleConfig_Import() {
	// ImportWithTest("strconv") causes strconv to include
	// internal_test.go, and creates an external test package,
	// strconv_test.
	// (Compare with the example of CreateFromFiles.)

	var conf loader.Config
	conf.Import("unicode/utf8")
	conf.Import("errors")
	conf.ImportWithTests("strconv")
	prog, err := conf.Load()
	if err != nil {
		log.Fatal(err)
	}

	printProgram(prog)
	printFilenames(prog.Fset, prog.Package("strconv"))
	printFilenames(prog.Fset, prog.Package("strconv_test"))
	// Output:
	// created: [strconv_test]
	// imported: [errors strconv unicode/utf8]
	// initial: [errors strconv strconv_test unicode/utf8]
	// all: [bufio bytes context errors flag fmt internal/race io log math math/rand os reflect runtime runtime/debug runtime/internal/atomic runtime/internal/sys runtime/trace sort strconv strconv_test strings sync sync/atomic syscall testing time unicode unicode/utf8 unsafe]
	// strconv.Files: [atob.go atof.go atoi.go decimal.go doc.go extfloat.go ftoa.go isprint.go itoa.go quote.go internal_test.go]
	// strconv_test.Files: [atob_test.go atof_test.go atoi_test.go decimal_test.go example_test.go fp_test.go ftoa_test.go itoa_test.go quote_test.go strconv_test.go]
}
<|endoftext|>


<commit_before>package logs

// Message extracts the message from the provided log line.
func Message(line string) string {
	panic("Please implement the Message() function")
}

// MessageLen counts the amount of characters (runes) in the message of the log line.
func MessageLen(line string) int {
	panic("Please implement the MessageLen() function")
}

// LogLevel extracts the log level string from the provided log line.
func LogLevel(line string) string {
	panic("Please implement the LogLevel() function")
}

// Reformat reformats the log line in the format `message (logLevel)`.
func Reformat(line string) string {
	panic("Please implement the Reformat() function")
}
<commit_msg>Implement Message<commit_after>package logs

import "strings"

// Message extracts the message from the provided log line.
func Message(line string) string {
	parts := strings.Split(line, ":")
	return strings.TrimSpace(parts[1])
}

// MessageLen counts the amount of characters (runes) in the message of the log line.
func MessageLen(line string) int {
	panic("Please implement the MessageLen() function")
}

// LogLevel extracts the log level string from the provided log line.
func LogLevel(line string) string {
	panic("Please implement the LogLevel() function")
}

// Reformat reformats the log line in the format `message (logLevel)`.
func Reformat(line string) string {
	panic("Please implement the Reformat() function")
}
<|endoftext|>


<commit_before>package gomap

import (
	"reflect"
)

// Options is a configuration container to pass to the
// new instance methods for GoMap
type Options struct {
	MappingConfig []Mapping
}

// GoMap holds all configuration for any mappings
// registered at the startup
type GoMap struct {
	mappingConfig []Mapping
}

// Mapping between two different structs
type Mapping struct {
	Key        string
	FieldLinks map[string]MapConfig
}

// MapConfig describes rules for the destination field
type MapConfig struct {
	Ignore bool
	Source string
}

// New returns a new gomapme Confguration struct
func New(options Options) *GoMap {
	return &GoMap{
		mappingConfig: options.MappingConfig,
	}
}

// NewDefault returns a plain GoMap func with default configuration
func NewDefault() *GoMap {
	gomap := GoMap{
		mappingConfig: make([]Mapping, 0),
	}
	return &gomap
}

// Map transforms the input struct to the output struct. Always pass the
// destination by reference and the source by value
func (g *GoMap) Map(s interface{}, d interface{}) {
	dstPtrVal := reflect.ValueOf(d)
	dstPtrType := dstPtrVal.Type()
	dstType := dstPtrType.Elem()
	dstVal := reflect.Indirect(dstPtrVal)
	srcVal := reflect.ValueOf(s)
	srcType := reflect.TypeOf(s)

	hasConfig, config := g.getConfig(srcType.Name() + dstType.Name())

	// loop the desintation VM fields
	for i := 0; i < dstType.NumField(); i++ {
		var src string
		ft := dstType.Field(i)

		if hasConfig {
			if _, ok := config.FieldLinks[ft.Name]; ok {
				if config.FieldLinks[ft.Name].Ignore {
					continue
				}
				src = config.FieldLinks[ft.Name].Source
			}
		}

		if src == "" {
			src = ft.Name
		}

		if sv := srcVal.FieldByName(src); sv.IsValid() {
			fv := dstVal.FieldByName(ft.Name)
			//add logic here to cast
			fv.Set(sv)
		}
	}
}

func (g *GoMap) getConfig(key string) (bool, Mapping) {
	var m Mapping
	for i := range g.mappingConfig {
		if g.mappingConfig[i].Key == key {
			return true, g.mappingConfig[i]
		}
	}
	return false, m
}

// Add applys a new mapping between two structs to the
// global configuration
func (g *GoMap) Add(source interface{}, destination interface{}, config map[string]MapConfig) {
	key := reflect.TypeOf(source).Name() + reflect.TypeOf(destination).Name()
	g.mappingConfig = append(g.mappingConfig, Mapping{key, config})
}
<commit_msg>Cleaned up map logic<commit_after>package gomap

import (
	"reflect"
)

// Options is a configuration container to pass to the
// new instance methods for GoMap
type Options struct {
	MappingConfig []Mapping
}

// GoMap holds all configuration for any mappings
// registered at the startup
type GoMap struct {
	mappingConfig []Mapping
}

// Mapping between two different structs
type Mapping struct {
	Key        string
	FieldLinks map[string]MapConfig
}

// MapConfig describes rules for the destination field
type MapConfig struct {
	Ignore bool
	Source string
}

// New returns a new gomapme Confguration struct
func New(options Options) *GoMap {
	return &GoMap{
		mappingConfig: options.MappingConfig,
	}
}

// NewDefault returns a plain GoMap func with default configuration
func NewDefault() *GoMap {
	gomap := GoMap{
		mappingConfig: make([]Mapping, 0),
	}
	return &gomap
}

// Map transforms the input struct to the output struct. Always pass the
// destination by reference and the source by value
func (g *GoMap) Map(s interface{}, d interface{}) {
	dstPtrVal := reflect.ValueOf(d)
	dstPtrType := dstPtrVal.Type()
	dstType := dstPtrType.Elem()
	dstVal := reflect.Indirect(dstPtrVal)
	srcVal := reflect.ValueOf(s)
	srcType := reflect.TypeOf(s)

	hasConfig, config := g.getConfig(srcType.Name() + dstType.Name())

	// loop the desintation VM fields
	for i := 0; i < dstType.NumField(); i++ {
		ft := dstType.Field(i)
		src := ft.Name

		if hasConfig {
			if _, ok := config.FieldLinks[ft.Name]; ok {
				if config.FieldLinks[ft.Name].Ignore {
					continue
				}
				src = config.FieldLinks[ft.Name].Source
			}
		}

		if sv := srcVal.FieldByName(src); sv.IsValid() {
			fv := dstVal.FieldByName(ft.Name)
			//add logic here to cast
			fv.Set(sv)
		}
	}
}

func (g *GoMap) getConfig(key string) (bool, Mapping) {
	var m Mapping
	for i := range g.mappingConfig {
		if g.mappingConfig[i].Key == key {
			return true, g.mappingConfig[i]
		}
	}
	return false, m
}

// Add applys a new mapping between two structs to the
// global configuration
func (g *GoMap) Add(source interface{}, destination interface{}, config map[string]MapConfig) {
	key := reflect.TypeOf(source).Name() + reflect.TypeOf(destination).Name()
	g.mappingConfig = append(g.mappingConfig, Mapping{key, config})
}
<|endoftext|>


<commit_before>package storage

type NeedleValue struct {
	Key    Key
	Offset uint32 `comment:"Volume offset"` //since aligned to 8 bytes, range is 4G*8=32G
	Size   uint32 `comment:"Size of the data portion"`
}

const (
	batch = 100000
)

type Key uint64

type CompactSection struct {
	values   []NeedleValue
	overflow map[Key]NeedleValue
	start    Key
	end      Key
	counter  int
}

func NewCompactSection(start Key) *CompactSection {
	return &CompactSection{
		values:   make([]NeedleValue, batch),
		overflow: make(map[Key]NeedleValue),
		start:    start,
	}
}

//return old entry size
func (cs *CompactSection) Set(key Key, offset uint32, size uint32) uint32 {
	ret := uint32(0)
	if key > cs.end {
		cs.end = key
	}
	if i := cs.binarySearchValues(key); i >= 0 {
		ret = cs.values[i].Size
		//println("key", key, "old size", ret)
		cs.values[i].Offset, cs.values[i].Size = offset, size
	} else {
		needOverflow := cs.counter >= batch
		needOverflow = needOverflow || cs.counter > 0 && cs.values[cs.counter-1].Key > key
		if needOverflow {
			//println("start", cs.start, "counter", cs.counter, "key", key)
			if oldValue, found := cs.overflow[key]; found {
				ret = oldValue.Size
			}
			cs.overflow[key] = NeedleValue{Key: key, Offset: offset, Size: size}
		} else {
			p := &cs.values[cs.counter]
			p.Key, p.Offset, p.Size = key, offset, size
			//println("added index", cs.counter, "key", key, cs.values[cs.counter].Key)
			cs.counter++
		}
	}
	return ret
}

//return old entry size
func (cs *CompactSection) Delete(key Key) uint32 {
	ret := uint32(0)
	if i := cs.binarySearchValues(key); i >= 0 {
		if cs.values[i].Size > 0 {
			ret = cs.values[i].Size
			cs.values[i].Size = 0
		}
	}
	if v, found := cs.overflow[key]; found {
		delete(cs.overflow, key)
		ret = v.Size
	}
	return ret
}
func (cs *CompactSection) Get(key Key) (*NeedleValue, bool) {
	if v, ok := cs.overflow[key]; ok {
		return &v, true
	}
	if i := cs.binarySearchValues(key); i >= 0 {
		return &cs.values[i], true
	}
	return nil, false
}
func (cs *CompactSection) binarySearchValues(key Key) int {
	l, h := 0, cs.counter-1
	if h >= 0 && cs.values[h].Key < key {
		return -2
	}
	//println("looking for key", key)
	for l <= h {
		m := (l + h) / 2
		//println("mid", m, "key", cs.values[m].Key, cs.values[m].Offset, cs.values[m].Size)
		if cs.values[m].Key < key {
			l = m + 1
		} else if key < cs.values[m].Key {
			h = m - 1
		} else {
			//println("found", m)
			return m
		}
	}
	return -1
}

//This map assumes mostly inserting increasing keys
type CompactMap struct {
	list []*CompactSection
}

func NewCompactMap() CompactMap {
	return CompactMap{}
}

func (cm *CompactMap) Set(key Key, offset uint32, size uint32) uint32 {
	x := cm.binarySearchCompactSection(key)
	if x < 0 {
		//println(x, "creating", len(cm.list), "section, starting", key)
		cm.list = append(cm.list, NewCompactSection(key))
		x = len(cm.list) - 1
		//keep compact section sorted by start
		for x > 0 {
			if cm.list[x-1].start > cm.list[x].start {
				cm.list[x-1], cm.list[x] = cm.list[x], cm.list[x-1]
				x = x - 1
			} else {
				break
			}
		}
	}
	return cm.list[x].Set(key, offset, size)
}
func (cm *CompactMap) Delete(key Key) uint32 {
	x := cm.binarySearchCompactSection(key)
	if x < 0 {
		return uint32(0)
	}
	return cm.list[x].Delete(key)
}
func (cm *CompactMap) Get(key Key) (*NeedleValue, bool) {
	x := cm.binarySearchCompactSection(key)
	if x < 0 {
		return nil, false
	}
	return cm.list[x].Get(key)
}
func (cm *CompactMap) binarySearchCompactSection(key Key) int {
	l, h := 0, len(cm.list)-1
	if h < 0 {
		return -5
	}
	if cm.list[h].start <= key {
		if cm.list[h].counter < batch || key <= cm.list[h].end {
			return h
		}
		return -4
	}
	for l <= h {
		m := (l + h) / 2
		if key < cm.list[m].start {
			h = m - 1
		} else { // cm.list[m].start <= key
			if cm.list[m+1].start <= key {
				l = m + 1
			} else {
				return m
			}
		}
	}
	return -3
}

func (cm *CompactMap) Visit(visit func(NeedleValue) error) error {
	for _, cs := range cm.list {
		for _, v := range cs.overflow {
			if err := visit(v); err != nil {
				return err
			}
		}
		for _, v := range cs.values {
			if _, found := cs.overflow[v.Key]; !found {
				if err := visit(v); err != nil {
					return err
				}
			}
		}
	}
	return nil
}
<commit_msg>Add comments<commit_after>package storage

type NeedleValue struct {
	Key    Key
	Offset uint32 `comment:"Volume offset"` //since aligned to 8 bytes, range is 4G*8=32G
	Size   uint32 `comment:"Size of the data portion"`
}

const (
	batch = 100000
)

type Key uint64

type CompactSection struct {
	values   []NeedleValue
	overflow map[Key]NeedleValue
	start    Key
	end      Key
	counter  int
}

func NewCompactSection(start Key) *CompactSection {
	return &CompactSection{
		values:   make([]NeedleValue, batch),
		overflow: make(map[Key]NeedleValue),
		start:    start,
	}
}

//return old entry size
func (cs *CompactSection) Set(key Key, offset uint32, size uint32) uint32 {
	ret := uint32(0)
	if key > cs.end {
		cs.end = key
	}
	if i := cs.binarySearchValues(key); i >= 0 {
		ret = cs.values[i].Size
		//println("key", key, "old size", ret)
		cs.values[i].Offset, cs.values[i].Size = offset, size
	} else {
		needOverflow := cs.counter >= batch
		needOverflow = needOverflow || cs.counter > 0 && cs.values[cs.counter-1].Key > key
		if needOverflow {
			//println("start", cs.start, "counter", cs.counter, "key", key)
			if oldValue, found := cs.overflow[key]; found {
				ret = oldValue.Size
			}
			cs.overflow[key] = NeedleValue{Key: key, Offset: offset, Size: size}
		} else {
			p := &cs.values[cs.counter]
			p.Key, p.Offset, p.Size = key, offset, size
			//println("added index", cs.counter, "key", key, cs.values[cs.counter].Key)
			cs.counter++
		}
	}
	return ret
}

//return old entry size
func (cs *CompactSection) Delete(key Key) uint32 {
	ret := uint32(0)
	if i := cs.binarySearchValues(key); i >= 0 {
		if cs.values[i].Size > 0 {
			ret = cs.values[i].Size
			cs.values[i].Size = 0
		}
	}
	if v, found := cs.overflow[key]; found {
		delete(cs.overflow, key)
		ret = v.Size
	}
	return ret
}
func (cs *CompactSection) Get(key Key) (*NeedleValue, bool) {
	if v, ok := cs.overflow[key]; ok {
		return &v, true
	}
	if i := cs.binarySearchValues(key); i >= 0 {
		return &cs.values[i], true
	}
	return nil, false
}
func (cs *CompactSection) binarySearchValues(key Key) int {
	l, h := 0, cs.counter-1
	if h >= 0 && cs.values[h].Key < key {
		return -2
	}
	//println("looking for key", key)
	for l <= h {
		m := (l + h) / 2
		//println("mid", m, "key", cs.values[m].Key, cs.values[m].Offset, cs.values[m].Size)
		if cs.values[m].Key < key {
			l = m + 1
		} else if key < cs.values[m].Key {
			h = m - 1
		} else {
			//println("found", m)
			return m
		}
	}
	return -1
}

//This map assumes mostly inserting increasing keys
type CompactMap struct {
	list []*CompactSection
}

func NewCompactMap() CompactMap {
	return CompactMap{}
}

func (cm *CompactMap) Set(key Key, offset uint32, size uint32) uint32 {
	x := cm.binarySearchCompactSection(key)
	if x < 0 {
		//println(x, "creating", len(cm.list), "section, starting", key)
		cm.list = append(cm.list, NewCompactSection(key))
		x = len(cm.list) - 1
		//keep compact section sorted by start
		for x > 0 {
			if cm.list[x-1].start > cm.list[x].start {
				cm.list[x-1], cm.list[x] = cm.list[x], cm.list[x-1]
				x = x - 1
			} else {
				break
			}
		}
	}
	return cm.list[x].Set(key, offset, size)
}
func (cm *CompactMap) Delete(key Key) uint32 {
	x := cm.binarySearchCompactSection(key)
	if x < 0 {
		return uint32(0)
	}
	return cm.list[x].Delete(key)
}
func (cm *CompactMap) Get(key Key) (*NeedleValue, bool) {
	x := cm.binarySearchCompactSection(key)
	if x < 0 {
		return nil, false
	}
	return cm.list[x].Get(key)
}
func (cm *CompactMap) binarySearchCompactSection(key Key) int {
	l, h := 0, len(cm.list)-1
	if h < 0 {
		return -5
	}
	if cm.list[h].start <= key {
		if cm.list[h].counter < batch || key <= cm.list[h].end {
			return h
		}
		return -4
	}
	for l <= h {
		m := (l + h) / 2
		if key < cm.list[m].start {
			h = m - 1
		} else { // cm.list[m].start <= key
			if cm.list[m+1].start <= key {
				l = m + 1
			} else {
				return m
			}
		}
	}
	return -3
}

// Visit visits all entries or stop if any error when visiting
func (cm *CompactMap) Visit(visit func(NeedleValue) error) error {
	for _, cs := range cm.list {
		for _, v := range cs.overflow {
			if err := visit(v); err != nil {
				return err
			}
		}
		for _, v := range cs.values {
			if _, found := cs.overflow[v.Key]; !found {
				if err := visit(v); err != nil {
					return err
				}
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package steam

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"
)

const (
	CurrencyUSD = "1"
	CurrencyGBP = "2"
	CurrencyEUR = "3"
	CurrencyCHF = "4"
	CurrencyRUB = "5"
	CurrencyPLN = "6"
	CurrencyBRL = "7"
	CurrencyJPY = "8"
	CurrencyNOK = "9"
	CurrencyIDR = "10"
	CurrencyMYR = "11"
	CurrencyPHP = "12"
	CurrencySGD = "13"
	CurrencyTHB = "14"
	CurrencyVND = "15"
	CurrencyKRW = "16"
	CurrencyTRY = "17"
	CurrencyUAH = "18"
	CurrencyMXN = "19"
	CurrencyCAD = "20"
	CurrencyAUD = "21"
	CurrencyNZD = "22"
	CurrencyCNY = "23"
	CurrencyINR = "24"
	CurrencyCLP = "25"
	CurrencyPEN = "26"
	CurrencyCOP = "27"
	CurrencyZAR = "28"
	CurrencyHKD = "29"
	CurrencyTWD = "30"
	CurrencySAR = "31"
	CurrencyAED = "32"
)

type MarketItemPriceOverview struct {
	Success     bool   `json:"success"`
	LowestPrice string `json:"lowest_price"`
	MedianPrice string `json:"median_price"`
	Volume      string `json:"volume"`
}

type MarketItemPrice struct {
	Date  string
	Price float64
	Count string
}

type MarketItemResponse struct {
	Success     bool        `json:"success"`
	PricePrefix string      `json:"price_prefix"`
	PriceSuffix string      `json:"price_suffix"`
	Prices      interface{} `json:"prices"`
}

type MarketSellResponse struct {
	Success                    bool   `json:"success"`
	RequiresConfirmation       uint32 `json:"requires_confirmation"`
	MobileConfirmationRequired bool   `json:"needs_mobile_confirmation"`
	EmailConfirmationRequired  bool   `json:"needs_email_confirmation"`
	EmailDomain                string `json:"email_domain"`
}

type MarketBuyOrderResponse struct {
	ErrCode int    `json:"success"`
	ErrMsg  string `json:"message"` // Set if ErrCode != 1
	OrderID uint64 `json:"buy_orderid,string"`
}

var (
	ErrCannotLoadPrices     = errors.New("unable to load prices at this time")
	ErrInvalidPriceResponse = errors.New("invalid market pricehistory response")
)

func (session *Session) GetMarketItemPriceHistory(appID uint64, marketHashName string) ([]*MarketItemPrice, error) {
	resp, err := session.client.Get("https://steamcommunity.com/market/pricehistory/?" + url.Values{
		"appid":            {strconv.FormatUint(appID, 10)},
		"market_hash_name": {marketHashName},
	}.Encode())
	if resp != nil {
		defer resp.Body.Close()
	}

	if err != nil {
		return nil, err
	}

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("http error: %d", resp.StatusCode)
	}

	response := MarketItemResponse{}
	if err = json.NewDecoder(resp.Body).Decode(&response); err != nil {
		return nil, err
	}

	if !response.Success {
		return nil, ErrCannotLoadPrices
	}

	var prices []interface{}
	var ok bool
	if prices, ok = response.Prices.([]interface{}); !ok {
		return nil, ErrCannotLoadPrices
	}

	items := []*MarketItemPrice{}
	for _, v := range prices {
		if v, ok := v.([]interface{}); ok {
			item := &MarketItemPrice{}
			for _, val := range v {
				switch val := val.(type) {
				case string:
					if len(item.Date) != 0 {
						item.Count = val
					} else {
						item.Date = val
					}
				case float64:
					item.Price = val
				}
			}
			items = append(items, item)
		}
	}
	return items, nil
}

func (session *Session) GetMarketItemPriceOverview(appID uint64, country, currencyID, marketHashName string) (*MarketItemPriceOverview, error) {
	resp, err := session.client.Get("https://steamcommunity.com/market/priceoverview/?" + url.Values{
		"appid":            {strconv.FormatUint(appID, 10)},
		"country":          {country},
		"currencyID":       {currencyID},
		"market_hash_name": {marketHashName},
	}.Encode())
	if resp != nil {
		defer resp.Body.Close()
	}

	if err != nil {
		return nil, err
	}

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("http error: %d", resp.StatusCode)
	}

	overview := &MarketItemPriceOverview{}
	if err = json.NewDecoder(resp.Body).Decode(overview); err != nil {
		return nil, err
	}

	return overview, nil
}

func (session *Session) SellItem(item *InventoryItem, amount, price uint64) (*MarketSellResponse, error) {
	resp, err := session.client.PostForm("https://steamcommunity.com/market/sellitem/", url.Values{
		"amount":    {strconv.FormatUint(amount, 10)},
		"appid":     {strconv.FormatUint(uint64(item.AppID), 10)},
		"assetid":   {strconv.FormatUint(item.AssetID, 10)},
		"contextid": {strconv.FormatUint(item.ContextID, 10)},
		"price":     {strconv.FormatUint(price, 10)},
		"sessionid": {session.sessionID},
	})
	if resp != nil {
		defer resp.Body.Close()
	}

	if err != nil {
		return nil, err
	}

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("http error: %d", resp.StatusCode)
	}

	response := &MarketSellResponse{}
	if err = json.NewDecoder(resp.Body).Decode(response); err != nil {
		return nil, err
	}

	return response, nil
}

func (session *Session) PlaceBuyOrder(appid uint64, priceTotal float64, quantity uint64, currencyID, marketHashName string) (*MarketBuyOrderResponse, error) {
	req, err := http.NewRequest(
		http.MethodPost,
		"https://steamcommunity.com/market/createbuyorder/",
		strings.NewReader(url.Values{
			"appid":            {strconv.FormatUint(appid, 10)},
			"currency":         {currencyID},
			"market_hash_name": {marketHashName},
			"price_total":      {strconv.FormatUint(uint64(priceTotal*100), 10)},
			"quantity":         {strconv.FormatUint(quantity, 10)},
			"sessionid":        {session.sessionID},
		}.Encode()),
	)
	if err != nil {
		return nil, err
	}

	req.Header.Add(
		"Referer",
		fmt.Sprintf("https://steamcommunity.com/market/listings/%d/%s", appid, strings.Replace(marketHashName, " ", "%20", -1)),
	)
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")

	resp, err := session.client.Do(req)
	if resp != nil {
		defer resp.Body.Close()
	}

	if err != nil {
		return nil, err
	}

	response := &MarketBuyOrderResponse{}
	if err = json.NewDecoder(resp.Body).Decode(response); err != nil {
		return nil, err
	}

	return response, nil
}

func (session *Session) CancelBuyOrder(orderid uint64) error {
	req, err := http.NewRequest(
		http.MethodPost,
		"https://steamcommunity.com/market/cancelbuyorder/",
		strings.NewReader(url.Values{
			"sessionid":   {session.sessionID},
			"buy_orderid": {strconv.FormatUint(orderid, 10)},
		}.Encode()),
	)
	if err != nil {
		return err
	}

	req.Header.Add("Referer", "https://steamcommunity.com/market")
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")

	resp, err := session.client.Do(req)
	if resp != nil {
		resp.Body.Close()
	}

	if err != nil {
		return err
	}

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("cannot cancel %d: %d", orderid, resp.StatusCode)
	}

	return nil
}
<commit_msg>Fixed making referer from marketHashName<commit_after>package steam

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"
)

const (
	CurrencyUSD = "1"
	CurrencyGBP = "2"
	CurrencyEUR = "3"
	CurrencyCHF = "4"
	CurrencyRUB = "5"
	CurrencyPLN = "6"
	CurrencyBRL = "7"
	CurrencyJPY = "8"
	CurrencyNOK = "9"
	CurrencyIDR = "10"
	CurrencyMYR = "11"
	CurrencyPHP = "12"
	CurrencySGD = "13"
	CurrencyTHB = "14"
	CurrencyVND = "15"
	CurrencyKRW = "16"
	CurrencyTRY = "17"
	CurrencyUAH = "18"
	CurrencyMXN = "19"
	CurrencyCAD = "20"
	CurrencyAUD = "21"
	CurrencyNZD = "22"
	CurrencyCNY = "23"
	CurrencyINR = "24"
	CurrencyCLP = "25"
	CurrencyPEN = "26"
	CurrencyCOP = "27"
	CurrencyZAR = "28"
	CurrencyHKD = "29"
	CurrencyTWD = "30"
	CurrencySAR = "31"
	CurrencyAED = "32"
)

type MarketItemPriceOverview struct {
	Success     bool   `json:"success"`
	LowestPrice string `json:"lowest_price"`
	MedianPrice string `json:"median_price"`
	Volume      string `json:"volume"`
}

type MarketItemPrice struct {
	Date  string
	Price float64
	Count string
}

type MarketItemResponse struct {
	Success     bool        `json:"success"`
	PricePrefix string      `json:"price_prefix"`
	PriceSuffix string      `json:"price_suffix"`
	Prices      interface{} `json:"prices"`
}

type MarketSellResponse struct {
	Success                    bool   `json:"success"`
	RequiresConfirmation       uint32 `json:"requires_confirmation"`
	MobileConfirmationRequired bool   `json:"needs_mobile_confirmation"`
	EmailConfirmationRequired  bool   `json:"needs_email_confirmation"`
	EmailDomain                string `json:"email_domain"`
}

type MarketBuyOrderResponse struct {
	ErrCode int    `json:"success"`
	ErrMsg  string `json:"message"` // Set if ErrCode != 1
	OrderID uint64 `json:"buy_orderid,string"`
}

var (
	ErrCannotLoadPrices     = errors.New("unable to load prices at this time")
	ErrInvalidPriceResponse = errors.New("invalid market pricehistory response")
)

func (session *Session) GetMarketItemPriceHistory(appID uint64, marketHashName string) ([]*MarketItemPrice, error) {
	resp, err := session.client.Get("https://steamcommunity.com/market/pricehistory/?" + url.Values{
		"appid":            {strconv.FormatUint(appID, 10)},
		"market_hash_name": {marketHashName},
	}.Encode())
	if resp != nil {
		defer resp.Body.Close()
	}

	if err != nil {
		return nil, err
	}

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("http error: %d", resp.StatusCode)
	}

	response := MarketItemResponse{}
	if err = json.NewDecoder(resp.Body).Decode(&response); err != nil {
		return nil, err
	}

	if !response.Success {
		return nil, ErrCannotLoadPrices
	}

	var prices []interface{}
	var ok bool
	if prices, ok = response.Prices.([]interface{}); !ok {
		return nil, ErrCannotLoadPrices
	}

	items := []*MarketItemPrice{}
	for _, v := range prices {
		if v, ok := v.([]interface{}); ok {
			item := &MarketItemPrice{}
			for _, val := range v {
				switch val := val.(type) {
				case string:
					if len(item.Date) != 0 {
						item.Count = val
					} else {
						item.Date = val
					}
				case float64:
					item.Price = val
				}
			}
			items = append(items, item)
		}
	}
	return items, nil
}

func (session *Session) GetMarketItemPriceOverview(appID uint64, country, currencyID, marketHashName string) (*MarketItemPriceOverview, error) {
	resp, err := session.client.Get("https://steamcommunity.com/market/priceoverview/?" + url.Values{
		"appid":            {strconv.FormatUint(appID, 10)},
		"country":          {country},
		"currencyID":       {currencyID},
		"market_hash_name": {marketHashName},
	}.Encode())
	if resp != nil {
		defer resp.Body.Close()
	}

	if err != nil {
		return nil, err
	}

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("http error: %d", resp.StatusCode)
	}

	overview := &MarketItemPriceOverview{}
	if err = json.NewDecoder(resp.Body).Decode(overview); err != nil {
		return nil, err
	}

	return overview, nil
}

func (session *Session) SellItem(item *InventoryItem, amount, price uint64) (*MarketSellResponse, error) {
	resp, err := session.client.PostForm("https://steamcommunity.com/market/sellitem/", url.Values{
		"amount":    {strconv.FormatUint(amount, 10)},
		"appid":     {strconv.FormatUint(uint64(item.AppID), 10)},
		"assetid":   {strconv.FormatUint(item.AssetID, 10)},
		"contextid": {strconv.FormatUint(item.ContextID, 10)},
		"price":     {strconv.FormatUint(price, 10)},
		"sessionid": {session.sessionID},
	})
	if resp != nil {
		defer resp.Body.Close()
	}

	if err != nil {
		return nil, err
	}

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("http error: %d", resp.StatusCode)
	}

	response := &MarketSellResponse{}
	if err = json.NewDecoder(resp.Body).Decode(response); err != nil {
		return nil, err
	}

	return response, nil
}

func (session *Session) PlaceBuyOrder(appid uint64, priceTotal float64, quantity uint64, currencyID, marketHashName string) (*MarketBuyOrderResponse, error) {
	req, err := http.NewRequest(
		http.MethodPost,
		"https://steamcommunity.com/market/createbuyorder/",
		strings.NewReader(url.Values{
			"appid":            {strconv.FormatUint(appid, 10)},
			"currency":         {currencyID},
			"market_hash_name": {marketHashName},
			"price_total":      {strconv.FormatUint(uint64(priceTotal*100), 10)},
			"quantity":         {strconv.FormatUint(quantity, 10)},
			"sessionid":        {session.sessionID},
		}.Encode()),
	)
	if err != nil {
		return nil, err
	}

	var referer string
	referer = strings.Replace(marketHashName, " ", "%20", -1)
	referer = strings.Replace(referer, "#", "%23", -1)

	req.Header.Add(
		"Referer",
		fmt.Sprintf("https://steamcommunity.com/market/listings/%d/%s", appid, referer ),
	)
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")

	resp, err := session.client.Do(req)
	if resp != nil {
		defer resp.Body.Close()
	}

	if err != nil {
		return nil, err
	}

	response := &MarketBuyOrderResponse{}
	if err = json.NewDecoder(resp.Body).Decode(response); err != nil {
		return nil, err
	}

	return response, nil
}

func (session *Session) CancelBuyOrder(orderid uint64) error {
	req, err := http.NewRequest(
		http.MethodPost,
		"https://steamcommunity.com/market/cancelbuyorder/",
		strings.NewReader(url.Values{
			"sessionid":   {session.sessionID},
			"buy_orderid": {strconv.FormatUint(orderid, 10)},
		}.Encode()),
	)
	if err != nil {
		return err
	}

	req.Header.Add("Referer", "https://steamcommunity.com/market")
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")

	resp, err := session.client.Do(req)
	if resp != nil {
		resp.Body.Close()
	}

	if err != nil {
		return err
	}

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("cannot cancel %d: %d", orderid, resp.StatusCode)
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Package "matrix" provides types and operations for matrix manipulation.
*/
package matrix
<commit_msg>"Matrix" is an interface for all implementation of matrix in this library.<commit_after>/*
Package "matrix" provides types and operations for matrix manipulation.
*/
package matrix

type Matrix interface {
	// TODO: Define.
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"html/template"
	"io"
	"log"
	"path/filepath"
)

// defaultTitle is the site pages default title.
const defaultTitle = "Google I/O 2015"

// meta is a page meta info.
type meta map[string]interface{}

// tmplFunc is a map of functions available to all templates.
var tmplFunc = template.FuncMap{
	"safeHTML": func(v string) template.HTML { return template.HTML(v) },
}

// renderTemplate executes a template found in name.html file
// using either layout_full.html or layout_partial.html as the root template.
func renderTemplate(w io.Writer, name string, partial bool) error {
	var layout string
	if partial {
		layout = "layout_partial.html"
	} else {
		layout = "layout_full.html"
	}

	t, err := template.New(layout).Delims("{%", "%}").Funcs(tmplFunc).ParseFiles(
		filepath.Join(rootDir, "templates", layout),
		filepath.Join(rootDir, "templates", name+".html"),
	)
	if err != nil {
		return err
	}

	m := pageMeta(t)
	data := struct {
		Title string
		Slug  string
		Meta  meta
	}{
		Title: pageTitle(m),
		Slug:  name,
		Meta:  m,
	}
	return t.Execute(w, data)
}

// pageTitle extracts "title" property of the page meta and appends defaultTitle to it.
// It returns defaultTitle if meta does not contain "title" or it is zero.
func pageTitle(m meta) string {
	title, ok := m["title"].(string)
	if !ok || title == "" {
		return defaultTitle
	}
	return title + " - " + defaultTitle
}

// pageMeta extracts page meta info by executing "meta" template.
// It returns empty meta if template execution fails.
func pageMeta(t *template.Template) (m meta) {
	m = make(meta)

	b := new(bytes.Buffer)
	b.WriteRune('{')
	if err := t.ExecuteTemplate(b, "meta", nil); err != nil {
		return
	}
	b.WriteRune('}')

	if err := json.Unmarshal(b.Bytes(), &m); err != nil {
		log.Printf("pageMeta: %v", err)
	}
	return
}
<commit_msg>Clarify pageMeta func assumptions<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"html/template"
	"io"
	"log"
	"path/filepath"
)

// defaultTitle is the site pages default title.
const defaultTitle = "Google I/O 2015"

// meta is a page meta info.
type meta map[string]interface{}

// tmplFunc is a map of functions available to all templates.
var tmplFunc = template.FuncMap{
	"safeHTML": func(v string) template.HTML { return template.HTML(v) },
}

// renderTemplate executes a template found in name.html file
// using either layout_full.html or layout_partial.html as the root template.
func renderTemplate(w io.Writer, name string, partial bool) error {
	var layout string
	if partial {
		layout = "layout_partial.html"
	} else {
		layout = "layout_full.html"
	}

	t, err := template.New(layout).Delims("{%", "%}").Funcs(tmplFunc).ParseFiles(
		filepath.Join(rootDir, "templates", layout),
		filepath.Join(rootDir, "templates", name+".html"),
	)
	if err != nil {
		return err
	}

	m := pageMeta(t)
	data := struct {
		Title string
		Slug  string
		Meta  meta
	}{
		Title: pageTitle(m),
		Slug:  name,
		Meta:  m,
	}
	return t.Execute(w, data)
}

// pageTitle extracts "title" property of the page meta and appends defaultTitle to it.
// It returns defaultTitle if meta does not contain "title" or it is zero.
func pageTitle(m meta) string {
	title, ok := m["title"].(string)
	if !ok || title == "" {
		return defaultTitle
	}
	return title + " - " + defaultTitle
}

// pageMeta extracts page meta info by executing "meta" template.
// The template is assumed to be a JSON object body (w/o {}).
// Returns empty meta if template execution fails.
func pageMeta(t *template.Template) (m meta) {
	m = make(meta)

	b := new(bytes.Buffer)
	b.WriteRune('{')
	if err := t.ExecuteTemplate(b, "meta", nil); err != nil {
		return
	}
	b.WriteRune('}')

	if err := json.Unmarshal(b.Bytes(), &m); err != nil {
		log.Printf("pageMeta: %v", err)
	}
	return
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 The btcsuite developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package cfgutil

// ExplicitString is a string value implementing the flags.Marshaler and
// flags.Unmarshaler interfaces so it may be used as a config struct field.  It
// records whether the value was explicitly set by the flags package.  This is
// useful when behavior must be modified depending on whether a flag was set by
// the user or left as a default.  Without recording this, it would be
// impossible to determine whether flag with a default value was unmodified or
// explicitly set to the default.
type ExplicitString struct {
	Value         string
	explicitlySet bool
}

// NewExplicitString creates a string flag with the provided default value.
func NewExplicitString(defaultValue string) *ExplicitString {
	return &ExplicitString{Value: defaultValue, explicitlySet: false}
}

// ExplicitlySet returns whether the flag was explicitly set through the
// flags.Unmarshaler interface.
func (e *ExplicitString) ExplicitlySet() bool { return e.explicitlySet }

// MarshalFlag implements the flags.Marshaler interface.
func (e *ExplicitString) MarshalFlag() (string, error) { return e.Value, nil }

// UnmarshalFlag implements the flags.Unmarshaler interface.
func (e *ExplicitString) UnmarshalFlag(value string) error {
	e.Value = value
	e.explicitlySet = true
	return nil
}
<commit_msg>Make cfgutil.ExplicitString a fmt.Stringer.<commit_after>// Copyright (c) 2016 The btcsuite developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package cfgutil

// ExplicitString is a string value implementing the flags.Marshaler and
// flags.Unmarshaler interfaces so it may be used as a config struct field.  It
// records whether the value was explicitly set by the flags package.  This is
// useful when behavior must be modified depending on whether a flag was set by
// the user or left as a default.  Without recording this, it would be
// impossible to determine whether flag with a default value was unmodified or
// explicitly set to the default.
type ExplicitString struct {
	Value         string
	explicitlySet bool
}

// NewExplicitString creates a string flag with the provided default value.
func NewExplicitString(defaultValue string) *ExplicitString {
	return &ExplicitString{Value: defaultValue, explicitlySet: false}
}

// ExplicitlySet returns whether the flag was explicitly set through the
// flags.Unmarshaler interface.
func (e *ExplicitString) ExplicitlySet() bool { return e.explicitlySet }

// MarshalFlag implements the flags.Marshaler interface.
func (e *ExplicitString) MarshalFlag() (string, error) { return e.Value, nil }

// UnmarshalFlag implements the flags.Unmarshaler interface.
func (e *ExplicitString) UnmarshalFlag(value string) error {
	e.Value = value
	e.explicitlySet = true
	return nil
}

// String implements the fmt.Stringer interface.
func (e *ExplicitString) String() string { return e.Value }
<|endoftext|>


<commit_before>/*
© Copyright IBM Corporation 2018, 2019

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package mqinimerge merges user-supplied INI files into qm.ini and mqat.ini
package mqinimerge

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/ibm-messaging/mq-container/pkg/mqini"
)

var qmgrDir string

var stanzasQMINI []string
var stanzasMQATINI []string

// AddStanzas reads supplied MQ INI configuration files and updates the stanzas
// in the queue manager's INI configuration files.
func AddStanzas(qmname string) error {
	// Find the qmgr directory.
	qm, err := mqini.GetQueueManager(qmname)
	if err != nil {
		return err
	}
	qmgrDir = mqini.GetDataDirectory(qm)
	// Find the users ini configuration file
	files, err := getIniFileList()
	if err != nil {
		return err
	}
	if len(files) > 1 {
		msg := fmt.Sprintf("[ %v ]", files)
		return errors.New("Only a single INI file can be provided. Following INI files were found:" + msg)
	}
	if len(files) == 0 {
		// No INI file update required.
		return nil
	}

	//read user supplied config file.
	iniFileBytes, err := ioutil.ReadFile(files[0])
	if err != nil {
		return err
	}
	userconfig := string(iniFileBytes)
	if len(userconfig) == 0 {
		return nil
	}

	// Prepare a list of all supported stanzas
	PopulateAllAvailableStanzas()

	// Update the qmgr ini file with user config.
	qminiConfiglist, qmatConfiglist, err := PrepareConfigStanzasToWrite(userconfig)
	if err != nil {
		return err
	}
	err = writeConfigStanzas(qminiConfiglist, qmatConfiglist)
	if err != nil {
		return err
	}

	return nil
}

// PopulateAllAvailableStanzas initializes the INI stanzas prescribed by MQ specification.
func PopulateAllAvailableStanzas() {
	stanzasQMINI = []string{"ExitPath",
		"Log",
		"Service",
		"ServiceComponent",
		"Channels",
		"TCP",
		"ApiExitLocal",
		"AccessMode",
		"RestrictedMode",
		"XAResourceManager",
		"DefaultBindType",
		"SSL",
		"DiagnosticMessages",
		"Filesystem",
		"Security",
		"TuningParameters",
		"ExitPropertiesLocal",
		"LU62",
		"NETBIOS"}

	stanzasMQATINI = []string{"AllActivityTrace", "ApplicationTrace"}
}

// getIniFileList checks for the user supplied INI file in `/etc/mqm` directory.
func getIniFileList() ([]string, error) {
	fileList := []string{}
	err := filepath.Walk("/etc/mqm", func(path string, f os.FileInfo, err error) error {
		if strings.HasSuffix(path, ".ini") {
			fileList = append(fileList, path)
		}
		return nil
	})
	if err != nil {
		return nil, err
	}
	return fileList, nil
}

// PrepareConfigStanzasToWrite Reads through the user supplied INI config file and prepares list of
// updates to be written into corresponding mq ini files (qm.ini and/or mqat.ini files)
func PrepareConfigStanzasToWrite(userconfig string) ([]string, []string, error) {
	var qminiConfigStr string
	var mqatiniConfigStr string

	//read the initial version.
	// #nosec G304 - qmgrDir filepath is derived from dspmqinf
	iniFileBytes, err := ioutil.ReadFile(filepath.Join(qmgrDir, "qm.ini"))
	if err != nil {
		return nil, nil, err
	}
	qminiConfigStr = string(iniFileBytes)
	qminiConfiglist := strings.Split(qminiConfigStr, "\n")

	// #nosec G304 - qmgrDir filepath is derived from dspmqinf
	iniFileBytes, err = ioutil.ReadFile(filepath.Join(qmgrDir, "mqat.ini"))
	if err != nil {
		return nil, nil, err
	}
	mqatiniConfigStr = string(iniFileBytes)
	qmatConfiglist := strings.Split(mqatiniConfigStr, "\n")

	stanzaListMerge := make(map[string]strings.Builder)
	stanzaListAppend := make(map[string]strings.Builder)
	var sbAppend strings.Builder
	var sbMerger strings.Builder

	scanner := bufio.NewScanner(strings.NewReader(userconfig))
	scanner.Split(bufio.ScanLines)
	consumetoAppend := false
	consumeToMerge := false
	var stanza string

	// Read through the user file and prepare what we want.
	for scanner.Scan() {
		//if this is comment or an empty line, ignore it.
		if strings.HasPrefix(scanner.Text(), "#") || len(strings.TrimSpace(scanner.Text())) == 0 {
			continue
		}
		//thumb rule - all stanzas have ":".
		if strings.Contains(scanner.Text(), ":") {
			stanza = strings.TrimSpace(scanner.Text())
			consumetoAppend = false
			consumeToMerge = false

			// Check if this stanza exists in the qm.ini/mqat.ini files
			if strings.Contains(qminiConfigStr, stanza) ||
				(strings.Contains(mqatiniConfigStr, stanza) && !(strings.Contains(stanza, "ApplicationTrace"))) {
				consumeToMerge = true
				sbMerger = strings.Builder{}

				stanzaListMerge[stanza] = sbMerger
			} else {
				consumetoAppend = true
				sbAppend = strings.Builder{}
				stanzaListAppend[stanza] = sbAppend
			}
		} else {
			if consumetoAppend {
				sb := stanzaListAppend[stanza]
				_, err := sb.WriteString(scanner.Text() + "\n")
				if err != nil {
					return nil, nil, err
				}
				stanzaListAppend[stanza] = sb
			}
			if consumeToMerge {
				sb := stanzaListMerge[stanza]
				_, err := sb.WriteString(scanner.Text() + "\n")
				if err != nil {
					return nil, nil, err
				}
				stanzaListMerge[stanza] = sb
			}
		}
	}

	// do merge.
	if len(stanzaListMerge) > 0 {
		for key := range stanzaListMerge {
			toWrite, filename := ValidateStanzaToWrite(key)
			if toWrite {
				attrList := stanzaListMerge[key]
				switch filename {
				case "qm.ini":
					qminiConfiglist, err = prepareStanzasToMerge(key, attrList, qminiConfiglist)
					if err != nil {
						return nil, nil, err
					}
				case "mqat.ini":
					qmatConfiglist, err = prepareStanzasToMerge(key, attrList, qmatConfiglist)
					if err != nil {
						return nil, nil, err
					}
				default:
				}
			}
		}
	}

	// do append.
	if len(stanzaListAppend) > 0 {
		for key := range stanzaListAppend {
			attrList := stanzaListAppend[key]
			if strings.Contains(strings.Join(stanzasMQATINI, ", "), strings.TrimSuffix(strings.TrimSpace(key), ":")) {
				qmatConfiglist = prepareStanzasToAppend(key, attrList, qmatConfiglist)
			} else {
				qminiConfiglist = prepareStanzasToAppend(key, attrList, qminiConfiglist)
			}
		}
	}

	return qminiConfiglist, qmatConfiglist, nil
}

// ValidateStanzaToWrite validates stanza to be written and the file it belongs to.
func ValidateStanzaToWrite(stanza string) (bool, string) {
	stanza = strings.TrimSuffix(strings.TrimSpace(stanza), ":")
	if strings.Contains(strings.Join(stanzasQMINI, ", "), stanza) {
		return true, "qm.ini"
	} else if strings.Contains(strings.Join(stanzasMQATINI, ", "), stanza) {
		return true, "mqat.ini"
	} else {
		return false, ""
	}
}

// prepareStanzasToAppend Prepares list of stanzas that are to be appended into qm ini files(qm.ini/mqat.ini)
func prepareStanzasToAppend(key string, attrList strings.Builder, iniConfigList []string) []string {
	newVal := key + "\n" + attrList.String()
	list := strings.Split(newVal, "\n")
	iniConfigList = append(iniConfigList, list...)
	return iniConfigList
}

// prepareStanzasToMerge Prepares list of stanzas that are to be updated into qm ini files(qm.ini/mqat.ini)
// These stanzas are already present in mq ini files and their values have to be updated with user supplied ini.
func prepareStanzasToMerge(key string, attrList strings.Builder, iniConfigList []string) ([]string, error) {

	pos := -1
	//find the index of current stanza in qm's ini file.
	for i := 0; i < len(iniConfigList); i++ {
		if strings.Contains(iniConfigList[i], key) {
			pos = i
			break
		}
	}

	var appList strings.Builder
	lineScanner := bufio.NewScanner(strings.NewReader(attrList.String()))
	lineScanner.Split(bufio.ScanLines)

	//Now go through the array and merge the values.
	for lineScanner.Scan() {
		attrLine := lineScanner.Text()
		keyvalue := strings.Split(attrLine, "=")
		merged := false
		for i := pos + 1; i < len(iniConfigList); i++ {
			if strings.HasPrefix(iniConfigList[i], "#") {
				continue
			}
			if strings.Contains(iniConfigList[i], ":") {
				break
			}
			if strings.Contains(iniConfigList[i], keyvalue[0]) {
				iniConfigList[i] = attrLine
				merged = true
				break
			}
		}
		//If this is not merged, then its a new parameter in existing stanza.
		if !merged && len(strings.TrimSpace(attrLine)) > 0 {
			_, err := appList.WriteString(attrLine)
			if err != nil {
				return nil, err
			}
			merged = false
		}

		if len(appList.String()) > 0 {
			temp := make([]string, pos+1)
			for i := 0; i < pos+1; i++ {
				temp[i] = iniConfigList[i]
			}
			list := strings.Split(appList.String(), "\n")
			temp = append(temp, list...)
			temp1 := iniConfigList[pos+1:]
			iniConfigList = append(temp, temp1...)
		}

	}
	return iniConfigList, nil
}

// writeFileIfChanged writes the specified data to the specified file path
// (just like ioutil.WriteFile), but first checks if this is needed
func writeFileIfChanged(path string, data []byte, perm os.FileMode) error {
	current, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}
	// Only write the new file if the it's different from the current file
	if !bytes.Equal(current, data) {
		err = ioutil.WriteFile(path, data, perm)
		if err != nil {
			return err
		}
	}
	return nil
}

// writeConfigStanzas writes the INI file updates into corresponding MQ INI files.
func writeConfigStanzas(qmConfig []string, atConfig []string) error {
	err := writeFileIfChanged(filepath.Join(qmgrDir, "qm.ini"), []byte(strings.Join(qmConfig, "\n")), 0644)
	if err != nil {
		return err
	}

	err = writeFileIfChanged(filepath.Join(qmgrDir, "mqat.ini"), []byte(strings.Join(atConfig, "\n")), 0644)
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>Fix gosec for writeFileIfChanged<commit_after>/*
© Copyright IBM Corporation 2018, 2019

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package mqinimerge merges user-supplied INI files into qm.ini and mqat.ini
package mqinimerge

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/ibm-messaging/mq-container/pkg/mqini"
)

var qmgrDir string

var stanzasQMINI []string
var stanzasMQATINI []string

// AddStanzas reads supplied MQ INI configuration files and updates the stanzas
// in the queue manager's INI configuration files.
func AddStanzas(qmname string) error {
	// Find the qmgr directory.
	qm, err := mqini.GetQueueManager(qmname)
	if err != nil {
		return err
	}
	qmgrDir = mqini.GetDataDirectory(qm)
	// Find the users ini configuration file
	files, err := getIniFileList()
	if err != nil {
		return err
	}
	if len(files) > 1 {
		msg := fmt.Sprintf("[ %v ]", files)
		return errors.New("Only a single INI file can be provided. Following INI files were found:" + msg)
	}
	if len(files) == 0 {
		// No INI file update required.
		return nil
	}

	//read user supplied config file.
	iniFileBytes, err := ioutil.ReadFile(files[0])
	if err != nil {
		return err
	}
	userconfig := string(iniFileBytes)
	if len(userconfig) == 0 {
		return nil
	}

	// Prepare a list of all supported stanzas
	PopulateAllAvailableStanzas()

	// Update the qmgr ini file with user config.
	qminiConfiglist, qmatConfiglist, err := PrepareConfigStanzasToWrite(userconfig)
	if err != nil {
		return err
	}
	err = writeConfigStanzas(qminiConfiglist, qmatConfiglist)
	if err != nil {
		return err
	}

	return nil
}

// PopulateAllAvailableStanzas initializes the INI stanzas prescribed by MQ specification.
func PopulateAllAvailableStanzas() {
	stanzasQMINI = []string{"ExitPath",
		"Log",
		"Service",
		"ServiceComponent",
		"Channels",
		"TCP",
		"ApiExitLocal",
		"AccessMode",
		"RestrictedMode",
		"XAResourceManager",
		"DefaultBindType",
		"SSL",
		"DiagnosticMessages",
		"Filesystem",
		"Security",
		"TuningParameters",
		"ExitPropertiesLocal",
		"LU62",
		"NETBIOS"}

	stanzasMQATINI = []string{"AllActivityTrace", "ApplicationTrace"}
}

// getIniFileList checks for the user supplied INI file in `/etc/mqm` directory.
func getIniFileList() ([]string, error) {
	fileList := []string{}
	err := filepath.Walk("/etc/mqm", func(path string, f os.FileInfo, err error) error {
		if strings.HasSuffix(path, ".ini") {
			fileList = append(fileList, path)
		}
		return nil
	})
	if err != nil {
		return nil, err
	}
	return fileList, nil
}

// PrepareConfigStanzasToWrite Reads through the user supplied INI config file and prepares list of
// updates to be written into corresponding mq ini files (qm.ini and/or mqat.ini files)
func PrepareConfigStanzasToWrite(userconfig string) ([]string, []string, error) {
	var qminiConfigStr string
	var mqatiniConfigStr string

	//read the initial version.
	// #nosec G304 - qmgrDir filepath is derived from dspmqinf
	iniFileBytes, err := ioutil.ReadFile(filepath.Join(qmgrDir, "qm.ini"))
	if err != nil {
		return nil, nil, err
	}
	qminiConfigStr = string(iniFileBytes)
	qminiConfiglist := strings.Split(qminiConfigStr, "\n")

	// #nosec G304 - qmgrDir filepath is derived from dspmqinf
	iniFileBytes, err = ioutil.ReadFile(filepath.Join(qmgrDir, "mqat.ini"))
	if err != nil {
		return nil, nil, err
	}
	mqatiniConfigStr = string(iniFileBytes)
	qmatConfiglist := strings.Split(mqatiniConfigStr, "\n")

	stanzaListMerge := make(map[string]strings.Builder)
	stanzaListAppend := make(map[string]strings.Builder)
	var sbAppend strings.Builder
	var sbMerger strings.Builder

	scanner := bufio.NewScanner(strings.NewReader(userconfig))
	scanner.Split(bufio.ScanLines)
	consumetoAppend := false
	consumeToMerge := false
	var stanza string

	// Read through the user file and prepare what we want.
	for scanner.Scan() {
		//if this is comment or an empty line, ignore it.
		if strings.HasPrefix(scanner.Text(), "#") || len(strings.TrimSpace(scanner.Text())) == 0 {
			continue
		}
		//thumb rule - all stanzas have ":".
		if strings.Contains(scanner.Text(), ":") {
			stanza = strings.TrimSpace(scanner.Text())
			consumetoAppend = false
			consumeToMerge = false

			// Check if this stanza exists in the qm.ini/mqat.ini files
			if strings.Contains(qminiConfigStr, stanza) ||
				(strings.Contains(mqatiniConfigStr, stanza) && !(strings.Contains(stanza, "ApplicationTrace"))) {
				consumeToMerge = true
				sbMerger = strings.Builder{}

				stanzaListMerge[stanza] = sbMerger
			} else {
				consumetoAppend = true
				sbAppend = strings.Builder{}
				stanzaListAppend[stanza] = sbAppend
			}
		} else {
			if consumetoAppend {
				sb := stanzaListAppend[stanza]
				_, err := sb.WriteString(scanner.Text() + "\n")
				if err != nil {
					return nil, nil, err
				}
				stanzaListAppend[stanza] = sb
			}
			if consumeToMerge {
				sb := stanzaListMerge[stanza]
				_, err := sb.WriteString(scanner.Text() + "\n")
				if err != nil {
					return nil, nil, err
				}
				stanzaListMerge[stanza] = sb
			}
		}
	}

	// do merge.
	if len(stanzaListMerge) > 0 {
		for key := range stanzaListMerge {
			toWrite, filename := ValidateStanzaToWrite(key)
			if toWrite {
				attrList := stanzaListMerge[key]
				switch filename {
				case "qm.ini":
					qminiConfiglist, err = prepareStanzasToMerge(key, attrList, qminiConfiglist)
					if err != nil {
						return nil, nil, err
					}
				case "mqat.ini":
					qmatConfiglist, err = prepareStanzasToMerge(key, attrList, qmatConfiglist)
					if err != nil {
						return nil, nil, err
					}
				default:
				}
			}
		}
	}

	// do append.
	if len(stanzaListAppend) > 0 {
		for key := range stanzaListAppend {
			attrList := stanzaListAppend[key]
			if strings.Contains(strings.Join(stanzasMQATINI, ", "), strings.TrimSuffix(strings.TrimSpace(key), ":")) {
				qmatConfiglist = prepareStanzasToAppend(key, attrList, qmatConfiglist)
			} else {
				qminiConfiglist = prepareStanzasToAppend(key, attrList, qminiConfiglist)
			}
		}
	}

	return qminiConfiglist, qmatConfiglist, nil
}

// ValidateStanzaToWrite validates stanza to be written and the file it belongs to.
func ValidateStanzaToWrite(stanza string) (bool, string) {
	stanza = strings.TrimSuffix(strings.TrimSpace(stanza), ":")
	if strings.Contains(strings.Join(stanzasQMINI, ", "), stanza) {
		return true, "qm.ini"
	} else if strings.Contains(strings.Join(stanzasMQATINI, ", "), stanza) {
		return true, "mqat.ini"
	} else {
		return false, ""
	}
}

// prepareStanzasToAppend Prepares list of stanzas that are to be appended into qm ini files(qm.ini/mqat.ini)
func prepareStanzasToAppend(key string, attrList strings.Builder, iniConfigList []string) []string {
	newVal := key + "\n" + attrList.String()
	list := strings.Split(newVal, "\n")
	iniConfigList = append(iniConfigList, list...)
	return iniConfigList
}

// prepareStanzasToMerge Prepares list of stanzas that are to be updated into qm ini files(qm.ini/mqat.ini)
// These stanzas are already present in mq ini files and their values have to be updated with user supplied ini.
func prepareStanzasToMerge(key string, attrList strings.Builder, iniConfigList []string) ([]string, error) {

	pos := -1
	//find the index of current stanza in qm's ini file.
	for i := 0; i < len(iniConfigList); i++ {
		if strings.Contains(iniConfigList[i], key) {
			pos = i
			break
		}
	}

	var appList strings.Builder
	lineScanner := bufio.NewScanner(strings.NewReader(attrList.String()))
	lineScanner.Split(bufio.ScanLines)

	//Now go through the array and merge the values.
	for lineScanner.Scan() {
		attrLine := lineScanner.Text()
		keyvalue := strings.Split(attrLine, "=")
		merged := false
		for i := pos + 1; i < len(iniConfigList); i++ {
			if strings.HasPrefix(iniConfigList[i], "#") {
				continue
			}
			if strings.Contains(iniConfigList[i], ":") {
				break
			}
			if strings.Contains(iniConfigList[i], keyvalue[0]) {
				iniConfigList[i] = attrLine
				merged = true
				break
			}
		}
		//If this is not merged, then its a new parameter in existing stanza.
		if !merged && len(strings.TrimSpace(attrLine)) > 0 {
			_, err := appList.WriteString(attrLine)
			if err != nil {
				return nil, err
			}
			merged = false
		}

		if len(appList.String()) > 0 {
			temp := make([]string, pos+1)
			for i := 0; i < pos+1; i++ {
				temp[i] = iniConfigList[i]
			}
			list := strings.Split(appList.String(), "\n")
			temp = append(temp, list...)
			temp1 := iniConfigList[pos+1:]
			iniConfigList = append(temp, temp1...)
		}

	}
	return iniConfigList, nil
}

// writeFileIfChanged writes the specified data to the specified file path
// (just like ioutil.WriteFile), but first checks if this is needed
func writeFileIfChanged(path string, data []byte, perm os.FileMode) error {
	// #nosec G304 - internal utility using file name derived from dspmqinf
	current, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}
	// Only write the new file if the it's different from the current file
	if !bytes.Equal(current, data) {
		err = ioutil.WriteFile(path, data, perm)
		if err != nil {
			return err
		}
	}
	return nil
}

// writeConfigStanzas writes the INI file updates into corresponding MQ INI files.
func writeConfigStanzas(qmConfig []string, atConfig []string) error {
	err := writeFileIfChanged(filepath.Join(qmgrDir, "qm.ini"), []byte(strings.Join(qmConfig, "\n")), 0644)
	if err != nil {
		return err
	}

	err = writeFileIfChanged(filepath.Join(qmgrDir, "mqat.ini"), []byte(strings.Join(atConfig, "\n")), 0644)
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Only build with go1.7 or above because b.Run did not exist before.
// +build integration

package mtail_test

import (
	"flag"
	"fmt"
	"io"
	"os"
	"path"
	"testing"

	"github.com/google/mtail/internal/metrics"
	"github.com/google/mtail/internal/mtail"
	"github.com/google/mtail/internal/testutil"
	"github.com/google/mtail/internal/watcher"
)

var (
	recordBenchmark = flag.Bool("record_benchmark", false, "Record the benchmark results to 'benchmark_results.csv'.")
)

func BenchmarkProgram(b *testing.B) {
	// exampleProgramTests live in ex_test.go
	for _, bm := range exampleProgramTests {
		bm := bm
		b.Run(fmt.Sprintf("%s on %s", bm.programfile, bm.logfile), func(b *testing.B) {
			b.ReportAllocs()
			logDir, rmLogDir := testutil.TestTempDir(b)
			defer rmLogDir()
			logFile := path.Join(logDir, "test.log")
			log := testutil.TestOpenFile(b, logFile)
			w := watcher.NewFakeWatcher()
			store := metrics.NewStore()
			programFile := path.Join("../..", bm.programfile)
			mtail, err := mtail.New(store, w, mtail.ProgramPath(programFile), mtail.LogPathPatterns(log.Name()))
			if err != nil {
				b.Fatalf("Failed to create mtail: %s", err)
			}
			err = mtail.StartTailing()
			if err != nil {
				b.Fatalf("starttailing failed: %s", err)
			}

			var total int64
			dataLogFile := path.Join("../..", bm.logfile)
			b.ResetTimer()
			for i := 0; i < b.N; i++ {
				l, err := os.Open(dataLogFile)
				if err != nil {
					b.Fatalf("Couldn't open logfile: %s", err)
				}
				count, err := io.Copy(log, l)
				if err != nil {
					b.Fatalf("Write of test data failed to test file: %s", err)
				}
				total += count
				w.InjectUpdate(log.Name())
			}
			mtail.Close()
			b.StopTimer()
			b.SetBytes(total)
		})
	}
}
<commit_msg>Remove unused flag.<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Only build with go1.7 or above because b.Run did not exist before.
// +build integration

package mtail_test

import (
	"fmt"
	"io"
	"os"
	"path"
	"testing"

	"github.com/google/mtail/internal/metrics"
	"github.com/google/mtail/internal/mtail"
	"github.com/google/mtail/internal/testutil"
	"github.com/google/mtail/internal/watcher"
)

func BenchmarkProgram(b *testing.B) {
	// exampleProgramTests live in ex_test.go
	for _, bm := range exampleProgramTests {
		bm := bm
		b.Run(fmt.Sprintf("%s on %s", bm.programfile, bm.logfile), func(b *testing.B) {
			b.ReportAllocs()
			logDir, rmLogDir := testutil.TestTempDir(b)
			defer rmLogDir()
			logFile := path.Join(logDir, "test.log")
			log := testutil.TestOpenFile(b, logFile)
			w := watcher.NewFakeWatcher()
			store := metrics.NewStore()
			programFile := path.Join("../..", bm.programfile)
			mtail, err := mtail.New(store, w, mtail.ProgramPath(programFile), mtail.LogPathPatterns(log.Name()))
			if err != nil {
				b.Fatalf("Failed to create mtail: %s", err)
			}
			err = mtail.StartTailing()
			if err != nil {
				b.Fatalf("starttailing failed: %s", err)
			}

			var total int64
			dataLogFile := path.Join("../..", bm.logfile)
			b.ResetTimer()
			for i := 0; i < b.N; i++ {
				l, err := os.Open(dataLogFile)
				if err != nil {
					b.Fatalf("Couldn't open logfile: %s", err)
				}
				count, err := io.Copy(log, l)
				if err != nil {
					b.Fatalf("Write of test data failed to test file: %s", err)
				}
				total += count
				w.InjectUpdate(log.Name())
			}
			mtail.Close()
			b.StopTimer()
			b.SetBytes(total)
		})
	}
}
<|endoftext|>


<commit_before>// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build typeparams && go1.18
// +build typeparams,go1.18

package typeparams

import (
	"go/ast"
	"go/types"
)

// NOTE: doc comments must be kept in sync with notypeparams.go.

// Enabled reports whether type parameters are enabled in the current build
// environment.
const Enabled = true

// GetIndexExprData extracts data from AST nodes that represent index
// expressions.
//
// For an ast.IndexExpr, the resulting IndexExprData will have exactly one
// index expression. For an ast.MultiIndexExpr (go1.18+), it may have a
// variable number of index expressions.
//
// For nodes that don't represent index expressions, GetIndexExprData returns
// nil.
func GetIndexExprData(n ast.Node) *IndexExprData {
	switch e := n.(type) {
	case *ast.IndexExpr:
		return &IndexExprData{
			X:       e.X,
			Lbrack:  e.Lbrack,
			Indices: []ast.Expr{e.Index},
			Rbrack:  e.Rbrack,
		}
	case *ast.MultiIndexExpr:
		return (*IndexExprData)(e)
	}
	return nil
}

// ForTypeDecl extracts the (possibly nil) type parameter node list from n.
func ForTypeDecl(n *ast.TypeSpec) *ast.FieldList {
	return n.TypeParams
}

// ForFuncDecl extracts the (possibly nil) type parameter node list from n.
func ForFuncDecl(n *ast.FuncDecl) *ast.FieldList {
	if n.Type != nil {
		return n.Type.TypeParams
	}
	return nil
}

// ForSignature extracts the (possibly empty) type parameter object list from
// sig.
func ForSignature(sig *types.Signature) []*types.TypeName {
	return tparamsSlice(sig.TypeParams())
}

// IsComparable reports if iface is the comparable interface.
func IsComparable(iface *types.Interface) bool {
	return iface.IsComparable()
}

// IsConstraint reports whether iface may only be used as a type parameter
// constraint (i.e. has a type set or is the comparable interface).
func IsConstraint(iface *types.Interface) bool {
	return iface.IsConstraint()
}

// ForNamed extracts the (possibly empty) type parameter object list from
// named.
func ForNamed(named *types.Named) []*types.TypeName {
	return tparamsSlice(named.TypeParams())
}

func tparamsSlice(tparams *types.TypeParamList) []*types.TypeName {
	length := tparams.Len()
	if length == 0 {
		return nil
	}

	result := make([]*types.TypeName, length)
	for i := 0; i < length; i++ {
		result[i] = tparams.At(i).Obj()
	}

	return result
}

// NamedTArgs extracts the (possibly empty) type argument list from named.
func NamedTArgs(named *types.Named) []types.Type {
	targs := named.TypeArgs()
	numArgs := targs.Len()

	typs := make([]types.Type, numArgs)
	for i := 0; i < numArgs; i++ {
		typs[i] = targs.At(i)
	}

	return typs
}

// InitInferred initializes info to record inferred type information.
func InitInferred(info *types.Info) {
	info.Inferred = make(map[ast.Expr]types.Inferred)
}

// GetInferred extracts inferred type information from info for e.
//
// The expression e may have an inferred type if it is an *ast.IndexExpr
// representing partial instantiation of a generic function type for which type
// arguments have been inferred using constraint type inference, or if it is an
// *ast.CallExpr for which type type arguments have be inferred using both
// constraint type inference and function argument inference.
func GetInferred(info *types.Info, e ast.Expr) ([]types.Type, *types.Signature) {
	if info.Inferred == nil {
		return nil, nil
	}
	inf := info.Inferred[e]

	length := inf.TArgs.Len()

	typs := make([]types.Type, length)
	for i := 0; i < length; i++ {
		typs[i] = inf.TArgs.At(i)
	}

	return typs, inf.Sig
}
<commit_msg>internal/typeparams: update MultiIndexExpr to IndexListExpr<commit_after>// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build typeparams && go1.18
// +build typeparams,go1.18

package typeparams

import (
	"go/ast"
	"go/types"
)

// NOTE: doc comments must be kept in sync with notypeparams.go.

// Enabled reports whether type parameters are enabled in the current build
// environment.
const Enabled = true

// GetIndexExprData extracts data from AST nodes that represent index
// expressions.
//
// For an ast.IndexExpr, the resulting IndexExprData will have exactly one
// index expression. For an ast.IndexListExpr (go1.18+), it may have a
// variable number of index expressions.
//
// For nodes that don't represent index expressions, GetIndexExprData returns
// nil.
func GetIndexExprData(n ast.Node) *IndexExprData {
	switch e := n.(type) {
	case *ast.IndexExpr:
		return &IndexExprData{
			X:       e.X,
			Lbrack:  e.Lbrack,
			Indices: []ast.Expr{e.Index},
			Rbrack:  e.Rbrack,
		}
	case *ast.IndexListExpr:
		return (*IndexExprData)(e)
	}
	return nil
}

// ForTypeDecl extracts the (possibly nil) type parameter node list from n.
func ForTypeDecl(n *ast.TypeSpec) *ast.FieldList {
	return n.TypeParams
}

// ForFuncDecl extracts the (possibly nil) type parameter node list from n.
func ForFuncDecl(n *ast.FuncDecl) *ast.FieldList {
	if n.Type != nil {
		return n.Type.TypeParams
	}
	return nil
}

// ForSignature extracts the (possibly empty) type parameter object list from
// sig.
func ForSignature(sig *types.Signature) []*types.TypeName {
	return tparamsSlice(sig.TypeParams())
}

// IsComparable reports if iface is the comparable interface.
func IsComparable(iface *types.Interface) bool {
	return iface.IsComparable()
}

// IsConstraint reports whether iface may only be used as a type parameter
// constraint (i.e. has a type set or is the comparable interface).
func IsConstraint(iface *types.Interface) bool {
	return iface.IsConstraint()
}

// ForNamed extracts the (possibly empty) type parameter object list from
// named.
func ForNamed(named *types.Named) []*types.TypeName {
	return tparamsSlice(named.TypeParams())
}

func tparamsSlice(tparams *types.TypeParamList) []*types.TypeName {
	length := tparams.Len()
	if length == 0 {
		return nil
	}

	result := make([]*types.TypeName, length)
	for i := 0; i < length; i++ {
		result[i] = tparams.At(i).Obj()
	}

	return result
}

// NamedTArgs extracts the (possibly empty) type argument list from named.
func NamedTArgs(named *types.Named) []types.Type {
	targs := named.TypeArgs()
	numArgs := targs.Len()

	typs := make([]types.Type, numArgs)
	for i := 0; i < numArgs; i++ {
		typs[i] = targs.At(i)
	}

	return typs
}

// InitInferred initializes info to record inferred type information.
func InitInferred(info *types.Info) {
	info.Inferred = make(map[ast.Expr]types.Inferred)
}

// GetInferred extracts inferred type information from info for e.
//
// The expression e may have an inferred type if it is an *ast.IndexExpr
// representing partial instantiation of a generic function type for which type
// arguments have been inferred using constraint type inference, or if it is an
// *ast.CallExpr for which type type arguments have be inferred using both
// constraint type inference and function argument inference.
func GetInferred(info *types.Info, e ast.Expr) ([]types.Type, *types.Signature) {
	if info.Inferred == nil {
		return nil, nil
	}
	inf := info.Inferred[e]

	length := inf.TArgs.Len()

	typs := make([]types.Type, length)
	for i := 0; i < length; i++ {
		typs[i] = inf.TArgs.At(i)
	}

	return typs, inf.Sig
}
<|endoftext|>


<commit_before>package slt

import (
	"errors"
	"fmt"
	"log"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"syscall"
	"time"

	"github.com/capnm/sysinfo" // TODO: can be thrown using syscall!!

)

func KiraWrap(icsFileName, intTime, randomNumber string, noGPU bool) () {
	var (
		timeLimit string
		err error
		kiraString string
		kiraArgs []string
		kiraWrappedCmd *exec.Cmd
		pathName, icsName, outName, errName, ext string
		icsFile, outFile, errFile *os.File
		regRes map[string]string
		done = make(chan string, 1)
		randomSeed string = ""
		u  *user.User
		host, wd string
	)

	pathName = filepath.Dir(icsFileName)
	icsName = filepath.Base(icsFileName)
	timeLimit = intTime
	randomSeed = randomNumber

	fmt.Println("###################################################")

	// Extract fileNameBody, round and ext
	log.Println("Extract files names")
	ext = filepath.Ext(icsName)
	regRes, err = Reg(icsName)
	if err != nil {
		log.Println("Can't derive standard names from STDOUT => wrap it!!")
		errName = "err-" + icsName + ext
		outName = "out-" + icsName + ext
	} else {
		if regRes["prefix"] != "ics" {
			log.Fatalf("Please specify a STDIN file, found %v prefix", regRes["prefix"])
		}

		// Creating new filenames
		errName = "err-" + regRes["baseName"] + "-run" + regRes["run"] + "-rnd" + regRes["rnd"] + ext
		outName = "out-" + regRes["baseName"] + "-run" + regRes["run"] + "-rnd" + regRes["rnd"] + ext
	}

	if icsName, err = filepath.Abs(filepath.Join(pathName, icsName)); err != nil {
		log.Fatal("Error composing icsName: ", err)
	}
	if errName, err = filepath.Abs(filepath.Join(pathName, errName)); err != nil {
		log.Fatal("Error composing errName: ", err)
	}
	if outName, err = filepath.Abs(filepath.Join(pathName, outName)); err != nil {
		log.Fatal("Error composing outName: ", err)
	}

	if icsFile, err = os.Open(icsName); err != nil {log.Fatal(err)}
	defer icsFile.Close()
	if outFile, err = os.Create(outName); err != nil {log.Fatal(err)}
	defer outFile.Close()
	if errFile, err = os.Create(errName); err != nil {log.Fatal(err)}
	defer errFile.Close()

	errFile.WriteString("\n#==============================\n")
	errFile.WriteString(fmt.Sprintf("\n#   %v Start with kiraWrap.\n", time.Now().Format(time.RFC850)))
	errFile.WriteString("\n#==============================\n")

	outFile.WriteString("\n#==============================\n")
	outFile.WriteString(fmt.Sprintf("\n#   %v Start with kiraWrap.\n", time.Now().Format(time.RFC850)))
	outFile.WriteString("\n#==============================\n")

	kiraArgs =  []string{"-t", timeLimit,// +  // number of timesteps to compute
		"-d", "1",// +  // log output interval
		"-D", "1",// +  // snapshot interval
		// 				"-b", "1",// +  // frequency of full binary output
		"-f", "0",// +  // dynamical friction (0 = no friction, 1 = friction)
		"-n", "10",// +  // terminate if the cluster remains with only 10 particles
		"-e", "0.000",// + // softening
		// 				"-B",// // switch on binary evolution
		//"-s 36543" // random seed
	}

	if noGPU {
		log.Println("Selected the no GPU integration.")
		log.Println("Assuming kira is in $HOME/bin/kira-no-GPU, if not, please copy it there... for sake of simplicity!:P")
		kiraString = filepath.Join(os.Getenv("HOME"), "/bin/", "kira-no-GPU")
	} else if tf {
		log.Println("Selected TF versionn.")
		log.Println("Assuming kira is in $HOME/bin/kiraTF, if not, please copy it there... for sake of simplicity!:P")
		kiraString = filepath.Join(os.Getenv("HOME"), "/bin/", "kiraTF")
// 		kiraArgs = append(kiraArgs, "-F") // F now means dynamical friction
	} else if as {
		log.Println("Selected TF versionn.")
		log.Println("Assuming kira is in $HOME/bin/kira-AS, if not, please copy it there... for sake of simplicity!:P")
		kiraString = filepath.Join(os.Getenv("HOME"), "/bin/", "kira-AS")
	}
	} else {
		log.Println("Assuming kira is in $HOME/bin/kira, if not, please copy it there... for sake of simplicity!:P")
		kiraString = filepath.Join(os.Getenv("HOME"), "/bin/", "kira")
	}

	if !noBinaries {
		log.Println("Binary evolution on, adding flags to kira")
		kiraArgs = append(kiraArgs, "-b", "1", "-B")
	} else {
		kiraArgs = append(kiraArgs, "-S")
		log.Println("Binary evolution off")
	}

	// Add the random seed if specified
	if randomSeed != "" {
		kiraArgs = append(kiraArgs, "-s", randomSeed)
	}

	kiraWrappedCmd = exec.Command(kiraString, kiraArgs...)
	if kiraWrappedCmd.Stdin = icsFile; err != nil {log.Fatal("Error connecting ICs to kira STDIN: ", err)}
	if kiraWrappedCmd.Stdout = outFile; err != nil {log.Fatal("Error connecting ICs to kira STDOUT: ", err)}
	if kiraWrappedCmd.Stderr = errFile; err != nil {log.Fatal("Error connecting ICs to kira STDERR: ", err)}

	if u, err = user.Current(); err != nil {
		log.Fatal("Can't retrieve username: ", err)
	}
	if host, err = os.Hostname(); err != nil {
		log.Fatal("Can't retrieve hostname: ", err)
	}
	if wd, err = os.Getwd(); err != nil {
		log.Fatal("Can't retrieve working dir: ", err)
	}

	log.Println("Run summary:")

	log.Printf("Username: %v (%v)\n", u.Username, u.Name)
	log.Println("Hostname: ", host)
	log.Println("Working dir: ", wd)
	log.Println("LD_LIBRARY_PATH: ", os.Getenv("LD_LIBRARY_PATH"))

	log.Println("Command: ", kiraString, kiraArgs)
	log.Println("STDIN = ", icsName)
	log.Println("STDOUT = ", outName)
	log.Println("STDERR = ", errName)

	log.Println("Ready... steady... Go!")

	if err = kiraWrappedCmd.Start(); err != nil {
		log.Fatal("Error starting kiraWrappedCmd: ", err)
	}

	log.Println("Waiting kira to finish while checking for problems...")
	// Wait for the process to end normally
	go waitProcess(kiraWrappedCmd, done)
	// Check for pp3-stalling situations
	go killTrigger(errName, kiraWrappedCmd, done)

	if err = errors.New(<-done); err.Error() != "" {
		errFile.WriteString("\n"+err.Error()+"\n")
	}

	errFile.WriteString("\n#==============================\n")
	errFile.WriteString(fmt.Sprintf("\n#   %v Done with kiraWrap.\n", time.Now().Format(time.RFC850)))
	errFile.WriteString(fmt.Sprintf("\n#   Username: %v (%V)\n", u.Username, u.Name))
	errFile.WriteString(fmt.Sprintf("\n#   Hostname: %v\n", host))
	errFile.WriteString("\n==============================\n")

	outFile.WriteString("\n#==============================\n")
	outFile.WriteString(fmt.Sprintf("\n#   %v Done with kiraWrap.\n", time.Now().Format(time.RFC850)))
	outFile.WriteString("\n#==============================\n")

	fmt.Print("\x07") // Beep when finish!!:D
}

func killTrigger(errName string, kiraWrappedCmd *exec.Cmd, done chan string) () {
	const toGB = float64(1. / (1024*1024*1024))
	const maxStderrGB = float64(2)
	const minDiskGB = float64(5)
	const maxMemPerCent = float64(98)

	var (
		fileInfo os.FileInfo
		sysInfo *sysinfo.SI
		memAvail float64
		diskAvailGB float64
		wd string
		reason string
		err error
	)

	if wd, err = os.Getwd(); err != nil {
		log.Fatal("Can't retrieve local dir: ", err)
	}

	fsInfo := syscall.Statfs_t{}

	for {
		// Check STDERR file size
		if fileInfo, err = os.Stat(errName); err != nil {
			log.Fatal("Error checking STDERR file size, err")
		}
		// STDERR exceesing aloowed dimension of 2GB
		// probably the simulation is stalling because
		// of pp3 locked on a binary
		if float64(fileInfo.Size()) * toGB > maxStderrGB {
			reason = "probable pp3 stalling"
			break
		}

		// Check memory availability
		sysInfo = sysinfo.Get()
		memAvail = float64(sysInfo.FreeRam) / float64(sysInfo.TotalRam) * 100
		if memAvail > maxMemPerCent {
			reason = fmt.Sprintf("memory less than 95%% on the system: %2.2f",  memAvail)
			break
		}

		// Check HDD space availability
		if err = syscall.Statfs(wd, &fsInfo); err!= nil{
			log.Fatal("Cant't retrieve file system information: ", err)
		}
		if diskAvailGB = float64(fsInfo.Bavail) * float64(fsInfo.Frsize) * toGB; diskAvailGB < minDiskGB {
			reason = " available disk space less than 5 GB on the system"
			break
		}

		// Wait some time
		time.Sleep(time.Duration(2) * time.Minute)
	}
	log.Println("Kill kira because ", reason)
	if err := kiraWrappedCmd.Process.Kill(); err != nil {
		log.Fatal("failed to kill: ", err)
	}
	done <- "killed because " + reason
}

func waitProcess(kiraWrappedCmd *exec.Cmd, done chan string) () {
	err := kiraWrappedCmd.Wait()
	log.Println("Process exited with error ", err)
	if err == nil {
		log.Println("Maybe the simulation is COMPLETE or the STD* are mixed and wrong.")
	}
	done <- ""
}

<commit_msg>fix kiraWrap<commit_after>package slt

import (
	"errors"
	"fmt"
	"log"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"syscall"
	"time"

	"github.com/capnm/sysinfo" // TODO: can be thrown using syscall!!

)

func KiraWrap(icsFileName, intTime, randomNumber string, noGPU bool) () {
	var (
		timeLimit string
		err error
		kiraString string
		kiraArgs []string
		kiraWrappedCmd *exec.Cmd
		pathName, icsName, outName, errName, ext string
		icsFile, outFile, errFile *os.File
		regRes map[string]string
		done = make(chan string, 1)
		randomSeed string = ""
		u  *user.User
		host, wd string
	)

	pathName = filepath.Dir(icsFileName)
	icsName = filepath.Base(icsFileName)
	timeLimit = intTime
	randomSeed = randomNumber

	fmt.Println("###################################################")

	// Extract fileNameBody, round and ext
	log.Println("Extract files names")
	ext = filepath.Ext(icsName)
	regRes, err = Reg(icsName)
	if err != nil {
		log.Println("Can't derive standard names from STDOUT => wrap it!!")
		errName = "err-" + icsName + ext
		outName = "out-" + icsName + ext
	} else {
		if regRes["prefix"] != "ics" {
			log.Fatalf("Please specify a STDIN file, found %v prefix", regRes["prefix"])
		}

		// Creating new filenames
		errName = "err-" + regRes["baseName"] + "-run" + regRes["run"] + "-rnd" + regRes["rnd"] + ext
		outName = "out-" + regRes["baseName"] + "-run" + regRes["run"] + "-rnd" + regRes["rnd"] + ext
	}

	if icsName, err = filepath.Abs(filepath.Join(pathName, icsName)); err != nil {
		log.Fatal("Error composing icsName: ", err)
	}
	if errName, err = filepath.Abs(filepath.Join(pathName, errName)); err != nil {
		log.Fatal("Error composing errName: ", err)
	}
	if outName, err = filepath.Abs(filepath.Join(pathName, outName)); err != nil {
		log.Fatal("Error composing outName: ", err)
	}

	if icsFile, err = os.Open(icsName); err != nil {log.Fatal(err)}
	defer icsFile.Close()
	if outFile, err = os.Create(outName); err != nil {log.Fatal(err)}
	defer outFile.Close()
	if errFile, err = os.Create(errName); err != nil {log.Fatal(err)}
	defer errFile.Close()

	errFile.WriteString("\n#==============================\n")
	errFile.WriteString(fmt.Sprintf("\n#   %v Start with kiraWrap.\n", time.Now().Format(time.RFC850)))
	errFile.WriteString("\n#==============================\n")

	outFile.WriteString("\n#==============================\n")
	outFile.WriteString(fmt.Sprintf("\n#   %v Start with kiraWrap.\n", time.Now().Format(time.RFC850)))
	outFile.WriteString("\n#==============================\n")

	kiraArgs =  []string{"-t", timeLimit,// +  // number of timesteps to compute
		"-d", "1",// +  // log output interval
		"-D", "1",// +  // snapshot interval
		// 				"-b", "1",// +  // frequency of full binary output
		"-f", "0",// +  // dynamical friction (0 = no friction, 1 = friction)
		"-n", "10",// +  // terminate if the cluster remains with only 10 particles
		"-e", "0.000",// + // softening
		// 				"-B",// // switch on binary evolution
		//"-s 36543" // random seed
	}

	if noGPU {
		log.Println("Selected the no GPU integration.")
		log.Println("Assuming kira is in $HOME/bin/kira-no-GPU, if not, please copy it there... for sake of simplicity!:P")
		kiraString = filepath.Join(os.Getenv("HOME"), "/bin/", "kira-no-GPU")
	} else if tf {
		log.Println("Selected TF versionn.")
		log.Println("Assuming kira is in $HOME/bin/kiraTF, if not, please copy it there... for sake of simplicity!:P")
		kiraString = filepath.Join(os.Getenv("HOME"), "/bin/", "kiraTF")
// 		kiraArgs = append(kiraArgs, "-F") // F now means dynamical friction
	} else if as {
		log.Println("Selected TF versionn.")
		log.Println("Assuming kira is in $HOME/bin/kira-AS, if not, please copy it there... for sake of simplicity!:P")
		kiraString = filepath.Join(os.Getenv("HOME"), "/bin/", "kira-AS")
	} else {
		log.Println("Assuming kira is in $HOME/bin/kira, if not, please copy it there... for sake of simplicity!:P")
		kiraString = filepath.Join(os.Getenv("HOME"), "/bin/", "kira")
	}

	if !noBinaries {
		log.Println("Binary evolution on, adding flags to kira")
		kiraArgs = append(kiraArgs, "-b", "1", "-B")
	} else {
		kiraArgs = append(kiraArgs, "-S")
		log.Println("Binary evolution off")
	}

	// Add the random seed if specified
	if randomSeed != "" {
		kiraArgs = append(kiraArgs, "-s", randomSeed)
	}

	kiraWrappedCmd = exec.Command(kiraString, kiraArgs...)
	if kiraWrappedCmd.Stdin = icsFile; err != nil {log.Fatal("Error connecting ICs to kira STDIN: ", err)}
	if kiraWrappedCmd.Stdout = outFile; err != nil {log.Fatal("Error connecting ICs to kira STDOUT: ", err)}
	if kiraWrappedCmd.Stderr = errFile; err != nil {log.Fatal("Error connecting ICs to kira STDERR: ", err)}

	if u, err = user.Current(); err != nil {
		log.Fatal("Can't retrieve username: ", err)
	}
	if host, err = os.Hostname(); err != nil {
		log.Fatal("Can't retrieve hostname: ", err)
	}
	if wd, err = os.Getwd(); err != nil {
		log.Fatal("Can't retrieve working dir: ", err)
	}

	log.Println("Run summary:")

	log.Printf("Username: %v (%v)\n", u.Username, u.Name)
	log.Println("Hostname: ", host)
	log.Println("Working dir: ", wd)
	log.Println("LD_LIBRARY_PATH: ", os.Getenv("LD_LIBRARY_PATH"))

	log.Println("Command: ", kiraString, kiraArgs)
	log.Println("STDIN = ", icsName)
	log.Println("STDOUT = ", outName)
	log.Println("STDERR = ", errName)

	log.Println("Ready... steady... Go!")

	if err = kiraWrappedCmd.Start(); err != nil {
		log.Fatal("Error starting kiraWrappedCmd: ", err)
	}

	log.Println("Waiting kira to finish while checking for problems...")
	// Wait for the process to end normally
	go waitProcess(kiraWrappedCmd, done)
	// Check for pp3-stalling situations
	go killTrigger(errName, kiraWrappedCmd, done)

	if err = errors.New(<-done); err.Error() != "" {
		errFile.WriteString("\n"+err.Error()+"\n")
	}

	errFile.WriteString("\n#==============================\n")
	errFile.WriteString(fmt.Sprintf("\n#   %v Done with kiraWrap.\n", time.Now().Format(time.RFC850)))
	errFile.WriteString(fmt.Sprintf("\n#   Username: %v (%V)\n", u.Username, u.Name))
	errFile.WriteString(fmt.Sprintf("\n#   Hostname: %v\n", host))
	errFile.WriteString("\n==============================\n")

	outFile.WriteString("\n#==============================\n")
	outFile.WriteString(fmt.Sprintf("\n#   %v Done with kiraWrap.\n", time.Now().Format(time.RFC850)))
	outFile.WriteString("\n#==============================\n")

	fmt.Print("\x07") // Beep when finish!!:D
}

func killTrigger(errName string, kiraWrappedCmd *exec.Cmd, done chan string) () {
	const toGB = float64(1. / (1024*1024*1024))
	const maxStderrGB = float64(2)
	const minDiskGB = float64(5)
	const maxMemPerCent = float64(98)

	var (
		fileInfo os.FileInfo
		sysInfo *sysinfo.SI
		memAvail float64
		diskAvailGB float64
		wd string
		reason string
		err error
	)

	if wd, err = os.Getwd(); err != nil {
		log.Fatal("Can't retrieve local dir: ", err)
	}

	fsInfo := syscall.Statfs_t{}

	for {
		// Check STDERR file size
		if fileInfo, err = os.Stat(errName); err != nil {
			log.Fatal("Error checking STDERR file size, err")
		}
		// STDERR exceesing aloowed dimension of 2GB
		// probably the simulation is stalling because
		// of pp3 locked on a binary
		if float64(fileInfo.Size()) * toGB > maxStderrGB {
			reason = "probable pp3 stalling"
			break
		}

		// Check memory availability
		sysInfo = sysinfo.Get()
		memAvail = float64(sysInfo.FreeRam) / float64(sysInfo.TotalRam) * 100
		if memAvail > maxMemPerCent {
			reason = fmt.Sprintf("memory less than 95%% on the system: %2.2f",  memAvail)
			break
		}

		// Check HDD space availability
		if err = syscall.Statfs(wd, &fsInfo); err!= nil{
			log.Fatal("Cant't retrieve file system information: ", err)
		}
		if diskAvailGB = float64(fsInfo.Bavail) * float64(fsInfo.Frsize) * toGB; diskAvailGB < minDiskGB {
			reason = " available disk space less than 5 GB on the system"
			break
		}

		// Wait some time
		time.Sleep(time.Duration(2) * time.Minute)
	}
	log.Println("Kill kira because ", reason)
	if err := kiraWrappedCmd.Process.Kill(); err != nil {
		log.Fatal("failed to kill: ", err)
	}
	done <- "killed because " + reason
}

func waitProcess(kiraWrappedCmd *exec.Cmd, done chan string) () {
	err := kiraWrappedCmd.Wait()
	log.Println("Process exited with error ", err)
	if err == nil {
		log.Println("Maybe the simulation is COMPLETE or the STD* are mixed and wrong.")
	}
	done <- ""
}

<|endoftext|>