<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"runtime/pprof"
	"strconv"
	"strings"

	"github.com/belogik/goes"
	"github.com/miku/estab"
)

func main() {

	host := flag.String("host", "localhost", "elasticsearch host")
	port := flag.String("port", "9200", "elasticsearch port")
	indicesString := flag.String("indices", "", "indices to search (or all)")
	fieldsString := flag.String("f", "_id _index", "field or fields space separated")
	timeout := flag.String("timeout", "10m", "scroll timeout")
	size := flag.Int("size", 10000, "scroll batch size")
	nullValue := flag.String("null", "NOT_AVAILABLE", "value for empty fields")
	separator := flag.String("separator", "|", "separator to use for multiple field values")
	delimiter := flag.String("delimiter", "\t", "column delimiter")
	version := flag.Bool("v", false, "prints current program version")
	cpuprofile := flag.String("cpuprofile", "", "write cpu profile to file")

	flag.Parse()

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *version {
		fmt.Println(estab.Version)
		os.Exit(0)
	}

	var indices []string
	trimmed := strings.TrimSpace(*indicesString)
	if len(trimmed) > 0 {
		indices = strings.Fields(trimmed)
	}

	fields := strings.Fields(*fieldsString)
	conn := goes.NewConnection(*host, *port)
	var query = map[string]interface{}{
		"query": map[string]interface{}{
			"match_all": map[string]interface{}{},
		},
		"fields": fields,
	}

	scanResponse, err := conn.Scan(query, indices, []string{""}, *timeout, *size)
	if err != nil {
		log.Fatalln(err)
	}

	for {
		scrollResponse, err := conn.Scroll(scanResponse.ScrollId, *timeout)
		if err != nil {
			log.Fatalln(err)
		}
		if len(scrollResponse.Hits.Hits) == 0 {
			break
		}
		for _, hit := range scrollResponse.Hits.Hits {
			var columns []string
			for _, f := range fields {
				var c []string
				switch f {
				case "_id":
					c = append(c, hit.Id)
				case "_index":
					c = append(c, hit.Index)
				case "_type":
					c = append(c, hit.Type)
				case "_score":
					c = append(c, strconv.FormatFloat(hit.Score, 'f', 6, 64))
				default:
					switch value := hit.Fields[f].(type) {
					case nil:
						c = []string{*nullValue}
					case []interface{}:
						for _, e := range value {
							c = append(c, e.(string))
						}
					default:
						log.Fatalf("unknown field type in response: %+v", hit.Fields[f])
					}
				}
				columns = append(columns, strings.Join(c, *separator))
			}
			fmt.Println(strings.Join(columns, *delimiter))
		}
	}
}
<commit_msg>added -limit flag<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"runtime/pprof"
	"strconv"
	"strings"

	"github.com/belogik/goes"
	"github.com/miku/estab"
)

func main() {

	host := flag.String("host", "localhost", "elasticsearch host")
	port := flag.String("port", "9200", "elasticsearch port")
	indicesString := flag.String("indices", "", "indices to search (or all)")
	fieldsString := flag.String("f", "_id _index", "field or fields space separated")
	timeout := flag.String("timeout", "10m", "scroll timeout")
	size := flag.Int("size", 10000, "scroll batch size")
	nullValue := flag.String("null", "NOT_AVAILABLE", "value for empty fields")
	separator := flag.String("separator", "|", "separator to use for multiple field values")
	delimiter := flag.String("delimiter", "\t", "column delimiter")
	limit := flag.Int("limit", 0, "maximum number of docs to return (return all by default)")
	version := flag.Bool("v", false, "prints current program version")
	cpuprofile := flag.String("cpuprofile", "", "write cpu profile to file")

	flag.Parse()

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *version {
		fmt.Println(estab.Version)
		os.Exit(0)
	}

	var indices []string
	trimmed := strings.TrimSpace(*indicesString)
	if len(trimmed) > 0 {
		indices = strings.Fields(trimmed)
	}

	fields := strings.Fields(*fieldsString)
	conn := goes.NewConnection(*host, *port)
	var query = map[string]interface{}{
		"query": map[string]interface{}{
			"match_all": map[string]interface{}{},
		},
		"fields": fields,
	}

	scanResponse, err := conn.Scan(query, indices, []string{""}, *timeout, *size)
	if err != nil {
		log.Fatalln(err)
	}

	counter := 0

	for {
		scrollResponse, err := conn.Scroll(scanResponse.ScrollId, *timeout)
		if err != nil {
			log.Fatalln(err)
		}
		if len(scrollResponse.Hits.Hits) == 0 {
			break
		}
		for _, hit := range scrollResponse.Hits.Hits {
			if *limit > 0 && counter == *limit {
				return
			}
			var columns []string
			for _, f := range fields {
				var c []string
				switch f {
				case "_id":
					c = append(c, hit.Id)
				case "_index":
					c = append(c, hit.Index)
				case "_type":
					c = append(c, hit.Type)
				case "_score":
					c = append(c, strconv.FormatFloat(hit.Score, 'f', 6, 64))
				default:
					switch value := hit.Fields[f].(type) {
					case nil:
						c = []string{*nullValue}
					case []interface{}:
						for _, e := range value {
							c = append(c, e.(string))
						}
					default:
						log.Fatalf("unknown field type in response: %+v", hit.Fields[f])
					}
				}
				columns = append(columns, strings.Join(c, *separator))
			}
			fmt.Println(strings.Join(columns, *delimiter))
			counter++
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strconv"

	"flag"

	"github.com/ammario/fastpass"
)

func cmdList(fp *fastpass.FastPass) {
	search := flag.Arg(1)

	entries := fp.Entries.FuzzyMatch(search)

	fmt.Println("------------")
	fmt.Printf("%v entries\n", len(entries))
	fmt.Println("------------")
	largestName := 0
	for _, e := range entries {
		if runes := []rune(e.Name); len(runes) > largestName {
			largestName = len(runes)
		}
	}
	for _, e := range entries.SortByName() {
		fmt.Printf("%-"+strconv.Itoa(largestName)+"v [hits:%v created:%v]", e.Name, e.Stats.Hits, e.CreatedAt.Format("01/02/2006 15:04:05"))
		if e.Notes != "" {
			fmt.Printf(" Notes: %v", e.Notes)
		}
		fmt.Printf("\n")
	}
}
<commit_msg>add db location to list cmd<commit_after>package main

import (
	"fmt"
	"strconv"

	"flag"

	"github.com/ammario/fastpass"
)

func cmdList(fp *fastpass.FastPass) {
	search := flag.Arg(1)

	entries := fp.Entries.FuzzyMatch(search)

	fmt.Println("------------")
	fmt.Printf("%v: %v entries\n", config.DB, len(entries))
	fmt.Println("------------")
	largestName := 0
	for _, e := range entries {
		if runes := []rune(e.Name); len(runes) > largestName {
			largestName = len(runes)
		}
	}
	for _, e := range entries.SortByName() {
		fmt.Printf("%-"+strconv.Itoa(largestName)+"v [hits:%v created:%v]", e.Name, e.Stats.Hits, e.CreatedAt.Format("01/02/2006 15:04:05"))
		if e.Notes != "" {
			fmt.Printf(" Notes: %v", e.Notes)
		}
		fmt.Printf("\n")
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"errors"
	"flag"
	"fmt"
	"go/build"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"code.google.com/p/go.tools/go/types"
)

var (
	source  = flag.String("s", "", "only consider packages from src, where src is one of the supported compilers")
	verbose = flag.Bool("v", false, "verbose mode")
)

// lists of registered sources and corresponding importers
var (
	sources      []string
	importers    []types.Importer
	importFailed = errors.New("import failed")
)

// map of imported packages
var packages = make(map[string]*types.Package)

func usage() {
	fmt.Fprintln(os.Stderr, "usage: godex [flags] {path|qualifiedIdent}")
	flag.PrintDefaults()
	os.Exit(2)
}

func report(msg string) {
	fmt.Fprintln(os.Stderr, "error: "+msg)
	os.Exit(2)
}

func main() {
	flag.Usage = usage
	flag.Parse()

	if flag.NArg() == 0 {
		report("no package name, path, or file provided")
	}

	imp := tryImports
	if *source != "" {
		imp = lookup(*source)
		if imp == nil {
			report("source (-s argument) must be one of: " + strings.Join(sources, ", "))
		}
	}

	for _, arg := range flag.Args() {
		path, name := splitPathIdent(arg)
		logf("\tprocessing %q: path = %q, name = %s\n", arg, path, name)

		// generate possible package path prefixes
		// (at the moment we do this for each argument - should probably cache the generated prefixes)
		prefixes := make(chan string)
		go genPrefixes(prefixes, !filepath.IsAbs(path) && !build.IsLocalImport(path))

		// import package
		pkg, err := tryPrefixes(packages, prefixes, path, imp)
		if err != nil {
			logf("\t=> ignoring %q: %s\n", path, err)
			continue
		}

		// filter objects if needed
		var filter func(types.Object) bool
		if name != "" {
			filter = func(obj types.Object) bool {
				// TODO(gri) perhaps use regular expression matching here?
				return obj.Name() == name
			}
		}

		// print contents
		print(os.Stdout, pkg, filter)
	}
}

func logf(format string, args ...interface{}) {
	if *verbose {
		fmt.Fprintf(os.Stderr, format, args...)
	}
}

// splitPathIdent splits a path.name argument into its components.
// All but the last path element may contain dots.
func splitPathIdent(arg string) (path, name string) {
	if i := strings.LastIndex(arg, "."); i >= 0 {
		if j := strings.LastIndex(arg, "/"); j < i {
			// '.' is not part of path
			path = arg[:i]
			name = arg[i+1:]
			return
		}
	}
	path = arg
	return
}

// tryPrefixes tries to import the package given by (the possibly partial) path using the given importer imp
// by prepending all possible prefixes to path. It returns with the first package that it could import, or
// with an error.
func tryPrefixes(packages map[string]*types.Package, prefixes chan string, path string, imp types.Importer) (pkg *types.Package, err error) {
	for prefix := range prefixes {
		logf("\ttrying prefix %q\n", prefix)
		prepath := filepath.Join(prefix, path)
		pkg, err = imp(packages, prepath)
		if err == nil {
			break
		}
		logf("\t=> importing %q failed: %s\n", prepath, err)
	}
	return
}

// tryImports is an importer that tries all registered importers
// successively until one of them succeeds or all of them failed.
func tryImports(packages map[string]*types.Package, path string) (pkg *types.Package, err error) {
	for i, imp := range importers {
		logf("\t\ttrying %s import\n", sources[i])
		pkg, err = imp(packages, path)
		if err == nil {
			break
		}
		logf("\t\t=> %s import failed: %s\n", sources[i], err)
	}
	return
}

// protect protects an importer imp from panics and returns the protected importer.
func protect(imp types.Importer) types.Importer {
	return func(packages map[string]*types.Package, path string) (pkg *types.Package, err error) {
		defer func() {
			if recover() != nil {
				pkg = nil
				err = importFailed
			}
		}()
		return imp(packages, path)
	}
}

// register registers an importer imp for a given source src.
func register(src string, imp types.Importer) {
	if lookup(src) != nil {
		panic(src + " importer already registered")
	}
	sources = append(sources, src)
	importers = append(importers, protect(imp))
}

// lookup returns the importer imp for a given source src.
func lookup(src string) types.Importer {
	for i, s := range sources {
		if s == src {
			return importers[i]
		}
	}
	return nil
}

func genPrefixes(out chan string, all bool) {
	out <- ""
	if all {
		platform := build.Default.GOOS + "_" + build.Default.GOARCH
		dirnames := append([]string{build.Default.GOROOT}, filepath.SplitList(build.Default.GOPATH)...)
		for _, dirname := range dirnames {
			walkDir(filepath.Join(dirname, "pkg", platform), "", out)
		}
	}
	close(out)
}

func walkDir(dirname, prefix string, out chan string) {
	fiList, err := ioutil.ReadDir(dirname)
	if err != nil {
		return
	}
	for _, fi := range fiList {
		if fi.IsDir() && !strings.HasPrefix(fi.Name(), ".") {
			prefix := filepath.Join(prefix, fi.Name())
			out <- prefix
			walkDir(filepath.Join(dirname, fi.Name()), prefix, out)
		}
	}
}
<commit_msg>go.tools/cmd/godex: make relative package paths work<commit_after>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"errors"
	"flag"
	"fmt"
	"go/build"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"code.google.com/p/go.tools/go/types"
)

var (
	source  = flag.String("s", "", "only consider packages from src, where src is one of the supported compilers")
	verbose = flag.Bool("v", false, "verbose mode")
)

// lists of registered sources and corresponding importers
var (
	sources      []string
	importers    []types.Importer
	importFailed = errors.New("import failed")
)

// map of imported packages
var packages = make(map[string]*types.Package)

func usage() {
	fmt.Fprintln(os.Stderr, "usage: godex [flags] {path|qualifiedIdent}")
	flag.PrintDefaults()
	os.Exit(2)
}

func report(msg string) {
	fmt.Fprintln(os.Stderr, "error: "+msg)
	os.Exit(2)
}

func main() {
	flag.Usage = usage
	flag.Parse()

	if flag.NArg() == 0 {
		report("no package name, path, or file provided")
	}

	imp := tryImports
	if *source != "" {
		imp = lookup(*source)
		if imp == nil {
			report("source (-s argument) must be one of: " + strings.Join(sources, ", "))
		}
	}

	for _, arg := range flag.Args() {
		path, name := splitPathIdent(arg)
		logf("\tprocessing %q: path = %q, name = %s\n", arg, path, name)

		// generate possible package path prefixes
		// (at the moment we do this for each argument - should probably cache the generated prefixes)
		prefixes := make(chan string)
		go genPrefixes(prefixes, !filepath.IsAbs(path) && !build.IsLocalImport(path))

		// import package
		pkg, err := tryPrefixes(packages, prefixes, path, imp)
		if err != nil {
			logf("\t=> ignoring %q: %s\n", path, err)
			continue
		}

		// filter objects if needed
		var filter func(types.Object) bool
		if name != "" {
			filter = func(obj types.Object) bool {
				// TODO(gri) perhaps use regular expression matching here?
				return obj.Name() == name
			}
		}

		// print contents
		print(os.Stdout, pkg, filter)
	}
}

func logf(format string, args ...interface{}) {
	if *verbose {
		fmt.Fprintf(os.Stderr, format, args...)
	}
}

// splitPathIdent splits a path.name argument into its components.
// All but the last path element may contain dots.
func splitPathIdent(arg string) (path, name string) {
	if i := strings.LastIndex(arg, "."); i >= 0 {
		if j := strings.LastIndex(arg, "/"); j < i {
			// '.' is not part of path
			path = arg[:i]
			name = arg[i+1:]
			return
		}
	}
	path = arg
	return
}

// tryPrefixes tries to import the package given by (the possibly partial) path using the given importer imp
// by prepending all possible prefixes to path. It returns with the first package that it could import, or
// with an error.
func tryPrefixes(packages map[string]*types.Package, prefixes chan string, path string, imp types.Importer) (pkg *types.Package, err error) {
	for prefix := range prefixes {
		actual := path
		if prefix == "" {
			// don't use filepath.Join as it will sanitize the path and remove
			// a leading dot and then the path is not recognized as a relative
			// package path by the importers anymore
			logf("\ttrying no prefix\n")
		} else {
			actual = filepath.Join(prefix, path)
			logf("\ttrying prefix %q\n", prefix)
		}
		pkg, err = imp(packages, actual)
		if err == nil {
			break
		}
		logf("\t=> importing %q failed: %s\n", actual, err)
	}
	return
}

// tryImports is an importer that tries all registered importers
// successively until one of them succeeds or all of them failed.
func tryImports(packages map[string]*types.Package, path string) (pkg *types.Package, err error) {
	for i, imp := range importers {
		logf("\t\ttrying %s import\n", sources[i])
		pkg, err = imp(packages, path)
		if err == nil {
			break
		}
		logf("\t\t=> %s import failed: %s\n", sources[i], err)
	}
	return
}

// protect protects an importer imp from panics and returns the protected importer.
func protect(imp types.Importer) types.Importer {
	return func(packages map[string]*types.Package, path string) (pkg *types.Package, err error) {
		defer func() {
			if recover() != nil {
				pkg = nil
				err = importFailed
			}
		}()
		return imp(packages, path)
	}
}

// register registers an importer imp for a given source src.
func register(src string, imp types.Importer) {
	if lookup(src) != nil {
		panic(src + " importer already registered")
	}
	sources = append(sources, src)
	importers = append(importers, protect(imp))
}

// lookup returns the importer imp for a given source src.
func lookup(src string) types.Importer {
	for i, s := range sources {
		if s == src {
			return importers[i]
		}
	}
	return nil
}

func genPrefixes(out chan string, all bool) {
	out <- ""
	if all {
		platform := build.Default.GOOS + "_" + build.Default.GOARCH
		dirnames := append([]string{build.Default.GOROOT}, filepath.SplitList(build.Default.GOPATH)...)
		for _, dirname := range dirnames {
			walkDir(filepath.Join(dirname, "pkg", platform), "", out)
		}
	}
	close(out)
}

func walkDir(dirname, prefix string, out chan string) {
	fiList, err := ioutil.ReadDir(dirname)
	if err != nil {
		return
	}
	for _, fi := range fiList {
		if fi.IsDir() && !strings.HasPrefix(fi.Name(), ".") {
			prefix := filepath.Join(prefix, fi.Name())
			out <- prefix
			walkDir(filepath.Join(dirname, fi.Name()), prefix, out)
		}
	}
}
<|endoftext|>


<commit_before>// cmd/gofile shows the platform string of Go executables
//
// cmd/gofile takes one argument, which is either program name or abosolute or
// relative path to an executable; when a program name is provided, it's looked up
// up in the $PATH.
//
// Example usage
//
//   ~ $ gowhich godoc
//   darwin_amd64
package main

import (
	"fmt"
	"os"
	"os/exec"

	"github.com/rjeczalik/which"
)

func die(v interface{}) {
	fmt.Fprintln(os.Stderr, v)
	os.Exit(1)
}

const usage = `NAME:
	gofile - shows the platform string of Go executables

USAGE:
	gofile name|path

EXAMPLES:
	gofile godoc
	gofile ~/bin/godoc`

func ishelp(s string) bool {
	return s == "-h" || s == "-help" || s == "help" || s == "--help" || s == "/?"
}

func main() {
	if len(os.Args) != 2 {
		die(usage)
	}
	if ishelp(os.Args[1]) {
		fmt.Println(usage)
		return
	}
	path, err := exec.LookPath(os.Args[1])
	if err != nil {
		die(err)
	}
	ex, err := which.NewExec(path)
	if err != nil {
		die(err)
	}
	fmt.Println(ex.Type)
}
<commit_msg>Fix typo in cmd/gofile inline doc<commit_after>// cmd/gofile shows the platform string of Go executables
//
// cmd/gofile takes one argument, which is either program name or abosolute or
// relative path to an executable; when a program name is provided, it's looked up
// up in the $PATH.
//
// Example usage
//
//   ~ $ gofile godoc
//   darwin_amd64
package main

import (
	"fmt"
	"os"
	"os/exec"

	"github.com/rjeczalik/which"
)

func die(v interface{}) {
	fmt.Fprintln(os.Stderr, v)
	os.Exit(1)
}

const usage = `NAME:
	gofile - shows the platform string of Go executables

USAGE:
	gofile name|path

EXAMPLES:
	gofile godoc
	gofile ~/bin/godoc`

func ishelp(s string) bool {
	return s == "-h" || s == "-help" || s == "help" || s == "--help" || s == "/?"
}

func main() {
	if len(os.Args) != 2 {
		die(usage)
	}
	if ishelp(os.Args[1]) {
		fmt.Println(usage)
		return
	}
	path, err := exec.LookPath(os.Args[1])
	if err != nil {
		die(err)
	}
	ex, err := which.NewExec(path)
	if err != nil {
		die(err)
	}
	fmt.Println(ex.Type)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"log"
	"os"
	"strconv"
	"strings"
	"unsafe"

	"github.com/alexflint/go-arg"
	"github.com/brentp/gsort"
	"github.com/brentp/xopen"
	ggd_utils "github.com/gogetdata/ggd-utils"
)

// DEFAULT_MEM is the number of megabytes of mem to use.
var DEFAULT_MEM = 1800

// VERSION is the program version number
const VERSION = "0.1.0"

var FileCols map[string][]int = map[string][]int{
	"BED": []int{0, 1, 2},
	"VCF": []int{0, 1, -1},
	"GFF": []int{0, 3, -1, 4},
	"GTF": []int{0, 3, -1, 4},
}

var CHECK_ORDER = []string{"BED", "GTF"}

var args struct {
	Path              string `arg:"positional,help:a tab-delimited file to sort"`
	Genome            string `arg:"positional,help:a genome file of chromosome sizes and order"`
	ChromosomMappings string `arg:"-c,help:a file used to re-map chromosome names for example from hg19 to GRCh37"`
	Memory            int    `arg:"-m,help:megabytes of memory to use before writing to temp files."`
	Parent            bool   `arg:"-p,help:for gff only. given rows with same chrom and start put those with a 'Parent' attribute first"`
}

func unsafeString(b []byte) string {
	return *(*string)(unsafe.Pointer(&b))
}

// get the start and end of a column given the index
func getAt(line []byte, idx int) (int, int) {
	if idx == 0 {
		return 0, bytes.IndexRune(line, '\t')
	}
	off := 0
	for i := 0; i < idx; i++ {
		off += 1 + bytes.IndexRune(line[off:], '\t')
	}
	e := bytes.IndexRune(line[off:], '\t')
	if e == -1 {
		e = len(line)
		for line[e-1] == '\n' || line[e-1] == '\r' {
			e--
		}
	} else {
		e = off + e
	}
	return off, e
}

// the last function is used when a column is -1
func sortFnFromCols(cols []int, gf *ggd_utils.GenomeFile, getter endGetter) func([]byte) []int {
	m := 0
	for _, c := range cols {
		if c > m {
			m = c
		}
	}
	m += 2
	if getter != nil && m < 6 {
		m = 6
	}
	H := 0 // keep order of header
	fn := func(line []byte) []int {
		l := make([]int, len(cols))
		s, e := getAt(line, cols[0])
		var ok bool
		if s < 0 || e < 0 {
			ok = false
		} else {
			chrom := string(line[s:e])
			if gf.ReMap != nil && len(gf.ReMap) > 0 {
				newchrom, ok := gf.ReMap[chrom]
				if !ok {
					log.Printf("[gsort] warning! no mapping given for chromosome: %s", chrom)
				} else {
					chrom = newchrom
				}
			}
			l[0], ok = gf.Order[chrom]
		}
		if !ok {
			if line[0] == '#' || hasAnyHeader(string(line)) {
				H++
				return []int{gsort.HEADER_LINE, H}
			}
			log.Fatalf("unknown chromosome: %s (known: %v)", line[s:e], gf.Order)
		}
		for k, col := range cols[1:] {
			i := k + 1
			if col == -1 {
				l[i] = getter(l[i-1], line)
			} else {
				s, e := getAt(line, col)
				subset := line[s:e]
				v, err := strconv.Atoi(unsafeString(subset))
				if err != nil {
					log.Fatal(err)
				}
				l[i] = v
			}
		}
		return l
	}
	return fn
}

var allowedHeaders = []string{"browser", "track"}

func hasAnyHeader(line string) bool {
	for _, a := range allowedHeaders {
		if strings.HasPrefix(line, a) {
			return true
		}
	}
	return false
}

func sniff(rdr *bufio.Reader) (string, *bufio.Reader, error) {
	lines := make([]string, 0, 200)
	var ftype string
	for len(lines) < 50000 {
		line, err := rdr.ReadString('\n')
		if len(line) > 0 {
			lines = append(lines, line)
			if line[0] == '#' {
				if strings.HasPrefix(line, "##fileformat=VCF") || strings.HasPrefix(line, "#CHROM\tPOS\tID") {
					ftype = "VCF"
					break
				} else {
					continue
				}
			} else {
				toks := strings.Split(line, "\t")
				if len(toks) < 3 {
					if hasAnyHeader(string(line)) {
						continue
					}
					return "", nil, fmt.Errorf("file has fewer than 3 columns")
				}
				for _, t := range CHECK_ORDER {
					cols := FileCols[t]
					ok := true
					last := 0
					for _, c := range cols[1:] {
						if c == -1 {
							continue
						}
						if c >= len(toks) {
							ok = false
							break
						}
						v, err := strconv.Atoi(strings.TrimRight(toks[c], "\r\n"))
						if err != nil {
							ok = false
							break
						}
						// check that 0 <= start col <= end_col
						if v < last {
							ok = false
							break
						}
						last = v
					}
					if ok {
						ftype = t
						break
					}
				}
				if hasAnyHeader(string(line)) {
					continue
				}
				if ftype == "" {
					return "", nil, fmt.Errorf("unknown file format: %s", string(line))
				}
				break
			}
		}
		if err != nil {
			return "", nil, err
		}

	}
	nrdr := io.MultiReader(strings.NewReader(strings.Join(lines, "")), rdr)
	return ftype, bufio.NewReader(nrdr), nil
}

func find(key []byte, info []byte) (int, int) {
	l := len(key)
	if pos := bytes.Index(info, key); pos != -1 {
		var end int
		for end = pos + l + 1; end < len(info); end++ {
			if info[end] == ';' {
				break
			}
		}
		return pos + l, end
	}
	return -1, -1

}

func getMax(i []byte) (int, error) {
	if !bytes.Contains(i, []byte(",")) {
		return strconv.Atoi(unsafeString(i))
	}
	all := bytes.Split(i, []byte{','})
	max := -1
	for _, b := range all {
		v, err := strconv.Atoi(unsafeString(b))
		if err != nil {
			return max, err
		}
		if v > max {
			max = v
		}
	}
	return max, nil
}

type endGetter func(start int, line []byte) int

var vcfEndGetter = endGetter(func(start int, line []byte) int {

	col4s, col4e := getAt(line, 4)
	col4 := line[col4s:col4e]
	if bytes.Contains(col4, []byte{'<'}) && (bytes.Contains(col4, []byte("<DEL")) ||
		bytes.Contains(col4, []byte("<DUP")) ||
		bytes.Contains(col4, []byte("<INV")) ||
		bytes.Contains(col4, []byte("<CN"))) {
		// need to look at INFO for this.

		is, ie := getAt(line, 7)
		info := line[is:ie]
		if s, e := find([]byte("END="), info); s != -1 {
			end, err := getMax(info[s:e])
			if err != nil {
				log.Fatal(err)
			}
			return end
		}
		s, e := find([]byte("SVLEN="), info)
		if s == -1 {
			log.Printf("warning: cant find end for %s", string(line))
			s3, e3 := getAt(line, 3)
			return start + e3 - s3
		}
		svlen, err := getMax(info[s:e])
		if err != nil {
			log.Fatal(err)
		}
		return start + svlen

	}
	// length of reference.
	s3, e3 := getAt(line, 3)
	return start + e3 - s3

})

// remap chromosome names.
func remap(gf *ggd_utils.GenomeFile, mappings map[string]string) {

	if mappings == nil || len(mappings) == 0 {
		return
	}

	chroms := make([]string, 0, len(gf.Lengths))
	for chrom := range gf.Lengths {
		if _, ok := mappings[chrom]; !ok {
			log.Printf("[gsort] WARNING! %s not found in chromosome mappings", chrom)
		} else {
			chroms = append(chroms, chrom)
		}
	}
	for _, chrom := range chroms {
		if chrom != mappings[chrom] {
			gf.Lengths[mappings[chrom]] = gf.Lengths[chrom]
			gf.Lengths[mappings[chrom]] = gf.Order[chrom]
			delete(gf.Lengths, chrom)
			delete(gf.Order, chrom)
		}

	}

}

func main() {

	args.Memory = DEFAULT_MEM
	p := arg.MustParse(&args)
	fmt.Fprintf(os.Stderr, "> gsort version %s\n", VERSION)
	if args.Path == "" || args.Genome == "" {
		p.Fail("must specify a tab-delimited file and a genome file")
	}

	rdr, err := xopen.Ropen(args.Path)
	if err == io.EOF {
		log.Println("gsort: empty file")
		os.Exit(0)
	}
	if err != nil {
		log.Fatal(err)
	}
	defer rdr.Close()

	ftype, brdr, err := sniff(rdr.Reader)
	if err != nil && err != io.EOF {
		log.Fatal(err)
	}

	gf, err := ggd_utils.ReadGenomeFile(args.Genome, args.ChromosomMappings)

	if err != nil && err != io.EOF {
		log.Fatal(err)
	}
	var getter endGetter

	if ftype == "VCF" {
		getter = vcfEndGetter
	} else if args.Parent && (ftype == "GFF" || ftype == "GTF") {

		seen := make(map[string]int, 20)
		cnt := 2
		getter = endGetter(func(start int, line []byte) int {
			ix := bytes.Index(line, []byte("\tID="))
			if ix == -1 {
				ix = bytes.Index(line, []byte(";ID="))
			}
			if ix != -1 {
				ix += 4
				ixEnd := bytes.IndexByte(line[ix:], ';')
				if ixEnd == -1 {
					seen[string(line[ix:len(line)-1])] = cnt
				} else {
					seen[string(line[ix:ix+ixEnd])] = cnt
				}
				cnt++
			}
			// want parent lines to come first. so lines containing a parent come last.

			if ix := bytes.Index(line, []byte("Parent=")); ix != -1 {
				ix += 7
				ie := bytes.IndexByte(line[ix:], ';')
				if ie == -1 {
					ie = len(line) - 1 - ix
				}
				if o, ok := seen[string(line[ix:ix+ie])]; ok {
					return o
				}
				return 1
			}
			return 0
		})
	} else if ftype == "GFF" || ftype == "GTF" {
		FileCols[ftype] = []int{0, 3, 4}
	}

	sortFn := sortFnFromCols(FileCols[ftype], gf, getter)
	wtr := bufio.NewWriter(os.Stdout)

	if err := gsort.Sort(brdr, wtr, sortFn, args.Memory, gf.ReMap); err != nil {
		log.Fatal("error from gsort.Sort", err)
	}
}
<commit_msg>fix spelling<commit_after>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"log"
	"os"
	"strconv"
	"strings"
	"unsafe"

	"github.com/alexflint/go-arg"
	"github.com/brentp/gsort"
	"github.com/brentp/xopen"
	ggd_utils "github.com/gogetdata/ggd-utils"
)

// DEFAULT_MEM is the number of megabytes of mem to use.
var DEFAULT_MEM = 1800

// VERSION is the program version number
const VERSION = "0.1.0"

var FileCols map[string][]int = map[string][]int{
	"BED": []int{0, 1, 2},
	"VCF": []int{0, 1, -1},
	"GFF": []int{0, 3, -1, 4},
	"GTF": []int{0, 3, -1, 4},
}

var CHECK_ORDER = []string{"BED", "GTF"}

var args struct {
	Path               string `arg:"positional,help:a tab-delimited file to sort"`
	Genome             string `arg:"positional,help:a genome file of chromosome sizes and order"`
	ChromosomeMappings string `arg:"-c,help:a file used to re-map chromosome names for example from hg19 to GRCh37"`
	Memory             int    `arg:"-m,help:megabytes of memory to use before writing to temp files."`
	Parent             bool   `arg:"-p,help:for gff only. given rows with same chrom and start put those with a 'Parent' attribute first"`
}

func unsafeString(b []byte) string {
	return *(*string)(unsafe.Pointer(&b))
}

// get the start and end of a column given the index
func getAt(line []byte, idx int) (int, int) {
	if idx == 0 {
		return 0, bytes.IndexRune(line, '\t')
	}
	off := 0
	for i := 0; i < idx; i++ {
		off += 1 + bytes.IndexRune(line[off:], '\t')
	}
	e := bytes.IndexRune(line[off:], '\t')
	if e == -1 {
		e = len(line)
		for line[e-1] == '\n' || line[e-1] == '\r' {
			e--
		}
	} else {
		e = off + e
	}
	return off, e
}

// the last function is used when a column is -1
func sortFnFromCols(cols []int, gf *ggd_utils.GenomeFile, getter endGetter) func([]byte) []int {
	m := 0
	for _, c := range cols {
		if c > m {
			m = c
		}
	}
	m += 2
	if getter != nil && m < 6 {
		m = 6
	}
	H := 0 // keep order of header
	fn := func(line []byte) []int {
		l := make([]int, len(cols))
		s, e := getAt(line, cols[0])
		var ok bool
		if s < 0 || e < 0 {
			ok = false
		} else {
			chrom := string(line[s:e])
			if gf.ReMap != nil && len(gf.ReMap) > 0 {
				newchrom, ok := gf.ReMap[chrom]
				if !ok {
					log.Printf("[gsort] warning! no mapping given for chromosome: %s", chrom)
				} else {
					chrom = newchrom
				}
			}
			l[0], ok = gf.Order[chrom]
		}
		if !ok {
			if line[0] == '#' || hasAnyHeader(string(line)) {
				H++
				return []int{gsort.HEADER_LINE, H}
			}
			log.Fatalf("unknown chromosome: %s (known: %v)", line[s:e], gf.Order)
		}
		for k, col := range cols[1:] {
			i := k + 1
			if col == -1 {
				l[i] = getter(l[i-1], line)
			} else {
				s, e := getAt(line, col)
				subset := line[s:e]
				v, err := strconv.Atoi(unsafeString(subset))
				if err != nil {
					log.Fatal(err)
				}
				l[i] = v
			}
		}
		return l
	}
	return fn
}

var allowedHeaders = []string{"browser", "track"}

func hasAnyHeader(line string) bool {
	for _, a := range allowedHeaders {
		if strings.HasPrefix(line, a) {
			return true
		}
	}
	return false
}

func sniff(rdr *bufio.Reader) (string, *bufio.Reader, error) {
	lines := make([]string, 0, 200)
	var ftype string
	for len(lines) < 50000 {
		line, err := rdr.ReadString('\n')
		if len(line) > 0 {
			lines = append(lines, line)
			if line[0] == '#' {
				if strings.HasPrefix(line, "##fileformat=VCF") || strings.HasPrefix(line, "#CHROM\tPOS\tID") {
					ftype = "VCF"
					break
				} else {
					continue
				}
			} else {
				toks := strings.Split(line, "\t")
				if len(toks) < 3 {
					if hasAnyHeader(string(line)) {
						continue
					}
					return "", nil, fmt.Errorf("file has fewer than 3 columns")
				}
				for _, t := range CHECK_ORDER {
					cols := FileCols[t]
					ok := true
					last := 0
					for _, c := range cols[1:] {
						if c == -1 {
							continue
						}
						if c >= len(toks) {
							ok = false
							break
						}
						v, err := strconv.Atoi(strings.TrimRight(toks[c], "\r\n"))
						if err != nil {
							ok = false
							break
						}
						// check that 0 <= start col <= end_col
						if v < last {
							ok = false
							break
						}
						last = v
					}
					if ok {
						ftype = t
						break
					}
				}
				if hasAnyHeader(string(line)) {
					continue
				}
				if ftype == "" {
					return "", nil, fmt.Errorf("unknown file format: %s", string(line))
				}
				break
			}
		}
		if err != nil {
			return "", nil, err
		}

	}
	nrdr := io.MultiReader(strings.NewReader(strings.Join(lines, "")), rdr)
	return ftype, bufio.NewReader(nrdr), nil
}

func find(key []byte, info []byte) (int, int) {
	l := len(key)
	if pos := bytes.Index(info, key); pos != -1 {
		var end int
		for end = pos + l + 1; end < len(info); end++ {
			if info[end] == ';' {
				break
			}
		}
		return pos + l, end
	}
	return -1, -1

}

func getMax(i []byte) (int, error) {
	if !bytes.Contains(i, []byte(",")) {
		return strconv.Atoi(unsafeString(i))
	}
	all := bytes.Split(i, []byte{','})
	max := -1
	for _, b := range all {
		v, err := strconv.Atoi(unsafeString(b))
		if err != nil {
			return max, err
		}
		if v > max {
			max = v
		}
	}
	return max, nil
}

type endGetter func(start int, line []byte) int

var vcfEndGetter = endGetter(func(start int, line []byte) int {

	col4s, col4e := getAt(line, 4)
	col4 := line[col4s:col4e]
	if bytes.Contains(col4, []byte{'<'}) && (bytes.Contains(col4, []byte("<DEL")) ||
		bytes.Contains(col4, []byte("<DUP")) ||
		bytes.Contains(col4, []byte("<INV")) ||
		bytes.Contains(col4, []byte("<CN"))) {
		// need to look at INFO for this.

		is, ie := getAt(line, 7)
		info := line[is:ie]
		if s, e := find([]byte("END="), info); s != -1 {
			end, err := getMax(info[s:e])
			if err != nil {
				log.Fatal(err)
			}
			return end
		}
		s, e := find([]byte("SVLEN="), info)
		if s == -1 {
			log.Printf("warning: cant find end for %s", string(line))
			s3, e3 := getAt(line, 3)
			return start + e3 - s3
		}
		svlen, err := getMax(info[s:e])
		if err != nil {
			log.Fatal(err)
		}
		return start + svlen

	}
	// length of reference.
	s3, e3 := getAt(line, 3)
	return start + e3 - s3

})

// remap chromosome names.
func remap(gf *ggd_utils.GenomeFile, mappings map[string]string) {

	if mappings == nil || len(mappings) == 0 {
		return
	}

	chroms := make([]string, 0, len(gf.Lengths))
	for chrom := range gf.Lengths {
		if _, ok := mappings[chrom]; !ok {
			log.Printf("[gsort] WARNING! %s not found in chromosome mappings", chrom)
		} else {
			chroms = append(chroms, chrom)
		}
	}
	for _, chrom := range chroms {
		if chrom != mappings[chrom] {
			gf.Lengths[mappings[chrom]] = gf.Lengths[chrom]
			gf.Lengths[mappings[chrom]] = gf.Order[chrom]
			delete(gf.Lengths, chrom)
			delete(gf.Order, chrom)
		}

	}

}

func main() {

	args.Memory = DEFAULT_MEM
	p := arg.MustParse(&args)
	fmt.Fprintf(os.Stderr, "> gsort version %s\n", VERSION)
	if args.Path == "" || args.Genome == "" {
		p.Fail("must specify a tab-delimited file and a genome file")
	}

	rdr, err := xopen.Ropen(args.Path)
	if err == io.EOF {
		log.Println("gsort: empty file")
		os.Exit(0)
	}
	if err != nil {
		log.Fatal(err)
	}
	defer rdr.Close()

	ftype, brdr, err := sniff(rdr.Reader)
	if err != nil && err != io.EOF {
		log.Fatal(err)
	}

	gf, err := ggd_utils.ReadGenomeFile(args.Genome, args.ChromosomeMappings)

	if err != nil && err != io.EOF {
		log.Fatal(err)
	}
	var getter endGetter

	if ftype == "VCF" {
		getter = vcfEndGetter
	} else if args.Parent && (ftype == "GFF" || ftype == "GTF") {

		seen := make(map[string]int, 20)
		cnt := 2
		getter = endGetter(func(start int, line []byte) int {
			ix := bytes.Index(line, []byte("\tID="))
			if ix == -1 {
				ix = bytes.Index(line, []byte(";ID="))
			}
			if ix != -1 {
				ix += 4
				ixEnd := bytes.IndexByte(line[ix:], ';')
				if ixEnd == -1 {
					seen[string(line[ix:len(line)-1])] = cnt
				} else {
					seen[string(line[ix:ix+ixEnd])] = cnt
				}
				cnt++
			}
			// want parent lines to come first. so lines containing a parent come last.

			if ix := bytes.Index(line, []byte("Parent=")); ix != -1 {
				ix += 7
				ie := bytes.IndexByte(line[ix:], ';')
				if ie == -1 {
					ie = len(line) - 1 - ix
				}
				if o, ok := seen[string(line[ix:ix+ie])]; ok {
					return o
				}
				return 1
			}
			return 0
		})
	} else if ftype == "GFF" || ftype == "GTF" {
		FileCols[ftype] = []int{0, 3, 4}
	}

	sortFn := sortFnFromCols(FileCols[ftype], gf, getter)
	wtr := bufio.NewWriter(os.Stdout)

	if err := gsort.Sort(brdr, wtr, sortFn, args.Memory, gf.ReMap); err != nil {
		log.Fatal("error from gsort.Sort", err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The Periph Authors. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

// i2c-io communicates to an I²C device.
package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strconv"

	"periph.io/x/periph/conn/i2c"
	"periph.io/x/periph/conn/i2c/i2creg"
	"periph.io/x/periph/host"
)

func mainImpl() error {
	addr := flag.Int("a", -1, "I²C device address to query")
	busName := flag.String("b", "", "I²C bus to use")
	verbose := flag.Bool("v", false, "verbose mode")
	// TODO(maruel): This is not generic enough.
	write := flag.Bool("w", false, "write instead of reading")
	reg := flag.Int("r", -1, "register to address")
	hz := flag.Int("hz", 0, "I²C bus speed (may require root)")
	l := flag.Int("l", 1, "length of data to read; ignored if -w is specified")
	flag.Parse()
	if !*verbose {
		log.SetOutput(ioutil.Discard)
	}
	log.SetFlags(log.Lmicroseconds)

	if *addr < 0 || *addr >= 1<<9 {
		return fmt.Errorf("-a is required and must be between 0 and %d", 1<<9-1)
	}
	if *reg < 0 || *reg > 255 {
		return errors.New("-r must be between 0 and 255")
	}
	if *l <= 0 || *l > 255 {
		return errors.New("-l must be between 1 and 255")
	}

	if _, err := host.Init(); err != nil {
		return err
	}

	var buf []byte
	if *write {
		if flag.NArg() == 0 {
			return errors.New("specify data to write as a list of hex encoded bytes")
		}
		buf = make([]byte, 0, flag.NArg())
		for _, a := range flag.Args() {
			b, err := strconv.ParseUint(a, 0, 8)
			if err != nil {
				return err
			}
			buf = append(buf, byte(b))
		}
	} else {
		if flag.NArg() != 0 {
			return errors.New("do not specify bytes when reading")
		}
		buf = make([]byte, *l)
	}

	bus, err := i2creg.Open(*busName)
	if err != nil {
		return err
	}
	defer bus.Close()

	if *hz != 0 {
		if err := bus.SetSpeed(int64(*hz)); err != nil {
			return err
		}
	}
	if *verbose {
		if p, ok := bus.(i2c.Pins); ok {
			log.Printf("Using pins SCL: %s  SDA: %s", p.SCL(), p.SDA())
		}
	}
	d := i2c.Dev{Bus: bus, Addr: uint16(*addr)}
	if *write {
		_, err = d.Write(buf)
	} else {
		if err = d.Tx([]byte{byte(*reg)}, buf); err != nil {
			return err
		}
		for i, b := range buf {
			if i != 0 {
				if _, err = fmt.Print(", "); err != nil {
					break
				}
			}
			if _, err = fmt.Printf("0x%02X", b); err != nil {
				break
			}
		}
		_, err = fmt.Print("\n")
	}
	return err
}

func main() {
	if err := mainImpl(); err != nil {
		fmt.Fprintf(os.Stderr, "i2c-io: %s.\n", err)
		os.Exit(1)
	}
}
<commit_msg> Fix I2C write in i2c-io. (#210)<commit_after>// Copyright 2016 The Periph Authors. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

// i2c-io communicates to an I²C device.
package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strconv"

	"periph.io/x/periph/conn/i2c"
	"periph.io/x/periph/conn/i2c/i2creg"
	"periph.io/x/periph/host"
)

func mainImpl() error {
	addr := flag.Int("a", -1, "I²C device address to query")
	busName := flag.String("b", "", "I²C bus to use")
	verbose := flag.Bool("v", false, "verbose mode")
	// TODO(maruel): This is not generic enough.
	write := flag.Bool("w", false, "write instead of reading")
	reg := flag.Int("r", -1, "register to address")
	hz := flag.Int("hz", 0, "I²C bus speed (may require root)")
	l := flag.Int("l", 1, "length of data to read; ignored if -w is specified")
	flag.Parse()
	if !*verbose {
		log.SetOutput(ioutil.Discard)
	}
	log.SetFlags(log.Lmicroseconds)

	if *addr < 0 || *addr >= 1<<9 {
		return fmt.Errorf("-a is required and must be between 0 and %d", 1<<9-1)
	}
	if *reg < 0 || *reg > 255 {
		return errors.New("-r must be between 0 and 255")
	}
	if *l <= 0 || *l > 255 {
		return errors.New("-l must be between 1 and 255")
	}

	if _, err := host.Init(); err != nil {
		return err
	}

	var buf []byte
	if *write {
		if flag.NArg() == 0 {
			return errors.New("specify data to write as a list of hex encoded bytes")
		}
		buf = make([]byte, 1, flag.NArg()+1)
		buf[0] = byte(*reg)
		for _, a := range flag.Args() {
			b, err := strconv.ParseUint(a, 0, 8)
			if err != nil {
				return err
			}
			buf = append(buf, byte(b))
		}
	} else {
		if flag.NArg() != 0 {
			return errors.New("do not specify bytes when reading")
		}
		buf = make([]byte, *l)
	}

	bus, err := i2creg.Open(*busName)
	if err != nil {
		return err
	}
	defer bus.Close()

	if *hz != 0 {
		if err := bus.SetSpeed(int64(*hz)); err != nil {
			return err
		}
	}
	if *verbose {
		if p, ok := bus.(i2c.Pins); ok {
			log.Printf("Using pins SCL: %s  SDA: %s", p.SCL(), p.SDA())
		}
	}
	d := i2c.Dev{Bus: bus, Addr: uint16(*addr)}
	if *write {
		_, err = d.Write(buf)
	} else {
		if err = d.Tx([]byte{byte(*reg)}, buf); err != nil {
			return err
		}
		for i, b := range buf {
			if i != 0 {
				if _, err = fmt.Print(", "); err != nil {
					break
				}
			}
			if _, err = fmt.Printf("0x%02X", b); err != nil {
				break
			}
		}
		_, err = fmt.Print("\n")
	}
	return err
}

func main() {
	if err := mainImpl(); err != nil {
		fmt.Fprintf(os.Stderr, "i2c-io: %s.\n", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package main

import (
	"fmt"
	"io"
	"time"

	"launchpad.net/gnuflag"

	"launchpad.net/juju-core/agent"
	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/errors"
	"launchpad.net/juju-core/log"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/state/api"
	apiagent "launchpad.net/juju-core/state/api/agent"
	apideployer "launchpad.net/juju-core/state/api/deployer"
	"launchpad.net/juju-core/state/api/params"
	"launchpad.net/juju-core/tools"
	"launchpad.net/juju-core/worker"
	"launchpad.net/juju-core/worker/deployer"
	"launchpad.net/juju-core/worker/upgrader"
)

// requiredError is useful when complaining about missing command-line options.
func requiredError(name string) error {
	return fmt.Errorf("--%s option must be set", name)
}

// AgentConf handles command-line flags shared by all agents.
type AgentConf struct {
	dataDir string
	config  agent.Config
}

// addFlags injects common agent flags into f.
func (c *AgentConf) addFlags(f *gnuflag.FlagSet) {
	f.StringVar(&c.dataDir, "data-dir", "/var/lib/juju", "directory for juju data")
}

func (c *AgentConf) checkArgs(args []string) error {
	if c.dataDir == "" {
		return requiredError("data-dir")
	}
	return cmd.CheckEmpty(args)
}

func (c *AgentConf) read(tag string) (err error) {
	c.config, err = agent.ReadConf(c.dataDir, tag)
	return
}

func importance(err error) int {
	switch {
	case err == nil:
		return 0
	default:
		return 1
	case isUpgraded(err):
		return 2
	case err == worker.ErrTerminateAgent:
		return 3
	}
}

// moreImportant returns whether err0 is
// more important than err1 - that is, whether
// we should act on err0 in preference to err1.
func moreImportant(err0, err1 error) bool {
	return importance(err0) > importance(err1)
}

func isUpgraded(err error) bool {
	_, ok := err.(*upgrader.UpgradeReadyError)
	return ok
}

type Agent interface {
	Entity(st *state.State) (AgentState, error)
	Tag() string
}

// The AgentState interface is implemented by state types
// that represent running agents.
type AgentState interface {
	// SetAgentTools sets the tools that the agent is currently running.
	SetAgentTools(tools *tools.Tools) error
	Tag() string
	SetMongoPassword(password string) error
	Life() state.Life
}

type fatalError struct {
	Err string
}

func (e *fatalError) Error() string {
	return e.Err
}

func isFatal(err error) bool {
	isTerminate := err == worker.ErrTerminateAgent
	notProvisioned := params.ErrCode(err) == params.CodeNotProvisioned
	if isTerminate || notProvisioned || isUpgraded(err) {
		return true
	}
	_, ok := err.(*fatalError)
	return ok
}

// isleep waits for the given duration or until it receives a value on
// stop.  It returns whether the full duration was slept without being
// stopped.
func isleep(d time.Duration, stop <-chan struct{}) bool {
	select {
	case <-stop:
		return false
	case <-time.After(d):
	}
	return true
}

func openState(agentConfig agent.Config, a Agent) (*state.State, AgentState, error) {
	st, err := agentConfig.OpenState()
	if err != nil {
		return nil, nil, err
	}
	entity, err := a.Entity(st)
	if errors.IsNotFoundError(err) || err == nil && entity.Life() == state.Dead {
		err = worker.ErrTerminateAgent
	}
	if err != nil {
		st.Close()
		return nil, nil, err
	}
	return st, entity, nil
}

func openAPIState(agentConfig agent.Config, a Agent) (*api.State, *apiagent.Entity, error) {
	// We let the API dial fail immediately because the
	// runner's loop outside the caller of openAPIState will
	// keep on retrying. If we block for ages here,
	// then the worker that's calling this cannot
	// be interrupted.
	st, newPassword, err := agentConfig.OpenAPI(api.DialOpts{})
	if err != nil {
		return nil, nil, err
	}
	entity, err := st.Agent().Entity(a.Tag())
	if params.ErrCode(err) == params.CodeNotFound || err == nil && entity.Life() == params.Dead {
		err = worker.ErrTerminateAgent
	}
	if err != nil {
		st.Close()
		return nil, nil, err
	}
	if newPassword != "" {
		if err := entity.SetPassword(newPassword); err != nil {
			return nil, nil, err
		}
	}
	return st, entity, nil

}

// agentDone processes the error returned by
// an exiting agent.
func agentDone(err error) error {
	if err == worker.ErrTerminateAgent {
		err = nil
	}
	if ug, ok := err.(*upgrader.UpgradeReadyError); ok {
		if err := ug.ChangeAgentTools(); err != nil {
			// Return and let upstart deal with the restart.
			return err
		}
	}
	return err
}

type closeWorker struct {
	worker worker.Worker
	closer io.Closer
}

// newCloseWorker returns a task that wraps the given task,
// closing the given closer when it finishes.
func newCloseWorker(worker worker.Worker, closer io.Closer) worker.Worker {
	return &closeWorker{
		worker: worker,
		closer: closer,
	}
}

func (c *closeWorker) Kill() {
	c.worker.Kill()
}

func (c *closeWorker) Wait() error {
	err := c.worker.Wait()
	if err := c.closer.Close(); err != nil {
		log.Errorf("closeWorker: close error: %v", err)
	}
	return err
}

// newDeployContext gives the tests the opportunity to create a deployer.Context
// that can be used for testing so as to avoid (1) deploying units to the system
// running the tests and (2) get access to the *State used internally, so that
// tests can be run without waiting for the 5s watcher refresh time to which we would
// otherwise be restricted.
var newDeployContext = func(st *apideployer.State, agentConfig agent.Config) deployer.Context {
	return deployer.NewSimpleContext(agentConfig, st)
}
<commit_msg>Changed openAPIState as suggested<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package main

import (
	"fmt"
	"io"
	"time"

	"launchpad.net/gnuflag"

	"launchpad.net/juju-core/agent"
	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/errors"
	"launchpad.net/juju-core/log"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/state/api"
	apiagent "launchpad.net/juju-core/state/api/agent"
	apideployer "launchpad.net/juju-core/state/api/deployer"
	"launchpad.net/juju-core/state/api/params"
	"launchpad.net/juju-core/tools"
	"launchpad.net/juju-core/worker"
	"launchpad.net/juju-core/worker/deployer"
	"launchpad.net/juju-core/worker/upgrader"
)

// requiredError is useful when complaining about missing command-line options.
func requiredError(name string) error {
	return fmt.Errorf("--%s option must be set", name)
}

// AgentConf handles command-line flags shared by all agents.
type AgentConf struct {
	dataDir string
	config  agent.Config
}

// addFlags injects common agent flags into f.
func (c *AgentConf) addFlags(f *gnuflag.FlagSet) {
	f.StringVar(&c.dataDir, "data-dir", "/var/lib/juju", "directory for juju data")
}

func (c *AgentConf) checkArgs(args []string) error {
	if c.dataDir == "" {
		return requiredError("data-dir")
	}
	return cmd.CheckEmpty(args)
}

func (c *AgentConf) read(tag string) (err error) {
	c.config, err = agent.ReadConf(c.dataDir, tag)
	return
}

func importance(err error) int {
	switch {
	case err == nil:
		return 0
	default:
		return 1
	case isUpgraded(err):
		return 2
	case err == worker.ErrTerminateAgent:
		return 3
	}
}

// moreImportant returns whether err0 is
// more important than err1 - that is, whether
// we should act on err0 in preference to err1.
func moreImportant(err0, err1 error) bool {
	return importance(err0) > importance(err1)
}

func isUpgraded(err error) bool {
	_, ok := err.(*upgrader.UpgradeReadyError)
	return ok
}

type Agent interface {
	Entity(st *state.State) (AgentState, error)
	Tag() string
}

// The AgentState interface is implemented by state types
// that represent running agents.
type AgentState interface {
	// SetAgentTools sets the tools that the agent is currently running.
	SetAgentTools(tools *tools.Tools) error
	Tag() string
	SetMongoPassword(password string) error
	Life() state.Life
}

type fatalError struct {
	Err string
}

func (e *fatalError) Error() string {
	return e.Err
}

func isFatal(err error) bool {
	isTerminate := err == worker.ErrTerminateAgent
	notProvisioned := params.ErrCode(err) == params.CodeNotProvisioned
	if isTerminate || notProvisioned || isUpgraded(err) {
		return true
	}
	_, ok := err.(*fatalError)
	return ok
}

// isleep waits for the given duration or until it receives a value on
// stop.  It returns whether the full duration was slept without being
// stopped.
func isleep(d time.Duration, stop <-chan struct{}) bool {
	select {
	case <-stop:
		return false
	case <-time.After(d):
	}
	return true
}

func openState(agentConfig agent.Config, a Agent) (*state.State, AgentState, error) {
	st, err := agentConfig.OpenState()
	if err != nil {
		return nil, nil, err
	}
	entity, err := a.Entity(st)
	if errors.IsNotFoundError(err) || err == nil && entity.Life() == state.Dead {
		err = worker.ErrTerminateAgent
	}
	if err != nil {
		st.Close()
		return nil, nil, err
	}
	return st, entity, nil
}

func openAPIState(agentConfig agent.Config, a Agent) (*api.State, *apiagent.Entity, error) {
	// We let the API dial fail immediately because the
	// runner's loop outside the caller of openAPIState will
	// keep on retrying. If we block for ages here,
	// then the worker that's calling this cannot
	// be interrupted.
	st, newPassword, err := agentConfig.OpenAPI(api.DialOpts{})
	if err != nil {
		if params.ErrCode(err) == params.CodeUnauthorized {
			err = worker.ErrTerminateAgent
		}
		return nil, nil, err
	}
	entity, err := st.Agent().Entity(a.Tag())
	if params.ErrCode(err) == params.CodeUnauthorized ||
		err == nil && entity.Life() == params.Dead {
		err = worker.ErrTerminateAgent
	}
	if err != nil {
		st.Close()
		return nil, nil, err
	}
	if newPassword != "" {
		if err := entity.SetPassword(newPassword); err != nil {
			return nil, nil, err
		}
	}
	return st, entity, nil

}

// agentDone processes the error returned by
// an exiting agent.
func agentDone(err error) error {
	if err == worker.ErrTerminateAgent {
		err = nil
	}
	if ug, ok := err.(*upgrader.UpgradeReadyError); ok {
		if err := ug.ChangeAgentTools(); err != nil {
			// Return and let upstart deal with the restart.
			return err
		}
	}
	return err
}

type closeWorker struct {
	worker worker.Worker
	closer io.Closer
}

// newCloseWorker returns a task that wraps the given task,
// closing the given closer when it finishes.
func newCloseWorker(worker worker.Worker, closer io.Closer) worker.Worker {
	return &closeWorker{
		worker: worker,
		closer: closer,
	}
}

func (c *closeWorker) Kill() {
	c.worker.Kill()
}

func (c *closeWorker) Wait() error {
	err := c.worker.Wait()
	if err := c.closer.Close(); err != nil {
		log.Errorf("closeWorker: close error: %v", err)
	}
	return err
}

// newDeployContext gives the tests the opportunity to create a deployer.Context
// that can be used for testing so as to avoid (1) deploying units to the system
// running the tests and (2) get access to the *State used internally, so that
// tests can be run without waiting for the 5s watcher refresh time to which we would
// otherwise be restricted.
var newDeployContext = func(st *apideployer.State, agentConfig agent.Config) deployer.Context {
	return deployer.NewSimpleContext(agentConfig, st)
}
<|endoftext|>


<commit_before>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kvm

import (
	"fmt"
	"path"
	"strings"

	"github.com/apigee/apigeecli/apiclient"
	"github.com/apigee/apigeecli/client/kvm"
	"github.com/apigee/apigeecli/cmd/utils"
	"github.com/spf13/cobra"
)

const kVMFileName = "kvms.json"

// ImpCmd to import kvm entries from files
var ImpCmd = &cobra.Command{
	Use:   "import",
	Short: "Import KVM Entries from a folder containing KVM files",
	Long:  "Import KVM Entries from a folder containing KVM files",
	Args: func(cmd *cobra.Command, args []string) (err error) {
		return apiclient.SetApigeeOrg(org)
	},
	RunE: func(cmd *cobra.Command, args []string) (err error) {

		var kvmList []string

		orgKVMFileList, envKVMFileList, proxyKVMFileList, err := utils.ListKVMFiles(folder)
		if err != nil {
			return err
		}

		if utils.FileExists(path.Join(folder, "org_"+org+"_"+kVMFileName)) {
			fmt.Println("Importing Org scoped KVMs...")
			if kvmList, err = utils.ReadEntityFile(path.Join(folder, "org_"+org+"_"+kVMFileName)); err != nil {
				return err
			}
			for _, kvmName := range kvmList {
				//create only encrypted KVMs
				fmt.Printf("\tCreating KVM %s\n", kvmName)
				if _, err = kvm.Create("", kvmName, true); err != nil {
					return err
				}
				fmt.Printf("\tImporting entries for %s\n", kvmName)
				if orgKVMFileList[kvmName] != "" {
					if err = kvm.ImportEntries("", kvmName, conn, orgKVMFileList[kvmName]); err != nil {
						return err
					}
				}
			}
		}

		if len(envKVMFileList) > 0 {
			fmt.Println("Importing env scoped KVMs...")
			for _, envKVMFile := range envKVMFileList {
				kvmMetadata := strings.Split(envKVMFile, "_")
				apiclient.SetApigeeEnv(kvmMetadata[1])
				fmt.Printf("\tCreating KVM %s\n", envKVMFile)
				if _, err = kvm.Create("", kvmMetadata[2], true); err != nil {
					return err
				}
				fmt.Printf("\tImporting entries for %s\n", envKVMFile)
				if err = kvm.ImportEntries("", kvmMetadata[2], conn, envKVMFile); err != nil {
					return err
				}
			}
		}

		if len(proxyKVMFileList) > 0 {
			fmt.Println("Importing proxy scoped KVMs...")
			for _, proxyKVMFile := range proxyKVMFileList {
				kvmMetadata := strings.Split(proxyKVMFile, "_")
				fmt.Printf("\tCreating KVM %s\n", proxyKVMFile)
				if _, err = kvm.Create(kvmMetadata[1], "", true); err != nil {
					return err
				}
				fmt.Printf("\tImporting entries for %s\n", proxyKVMFile)
				if err = kvm.ImportEntries(kvmMetadata[1], kvmMetadata[2], conn, proxyKVMFile); err != nil {
					return err
				}
			}
		}

		return err
	},
}

var folder string

func init() {

	ImpCmd.Flags().StringVarP(&folder, "folder", "f",
		"", "The absolute path to the folder containing KVM entries")
	ImpCmd.Flags().IntVarP(&conn, "conn", "c",
		4, "Number of connections")

	_ = ImpCmd.MarkFlagRequired("folder")
}
<commit_msg>turn off printing<commit_after>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kvm

import (
	"fmt"
	"path"
	"strings"

	"github.com/apigee/apigeecli/apiclient"
	"github.com/apigee/apigeecli/client/kvm"
	"github.com/apigee/apigeecli/cmd/utils"
	"github.com/spf13/cobra"
)

const kVMFileName = "kvms.json"

// ImpCmd to import kvm entries from files
var ImpCmd = &cobra.Command{
	Use:   "import",
	Short: "Import KVM Entries from a folder containing KVM files",
	Long:  "Import KVM Entries from a folder containing KVM files",
	Args: func(cmd *cobra.Command, args []string) (err error) {
		return apiclient.SetApigeeOrg(org)
	},
	RunE: func(cmd *cobra.Command, args []string) (err error) {

		var kvmList []string

		apiclient.SetPrintOutput(false)

		orgKVMFileList, envKVMFileList, proxyKVMFileList, err := utils.ListKVMFiles(folder)
		if err != nil {
			return err
		}

		if utils.FileExists(path.Join(folder, "org_"+org+"_"+kVMFileName)) {
			fmt.Println("Importing Org scoped KVMs...")
			if kvmList, err = utils.ReadEntityFile(path.Join(folder, "org_"+org+"_"+kVMFileName)); err != nil {
				return err
			}
			for _, kvmName := range kvmList {
				//create only encrypted KVMs
				fmt.Printf("\tCreating KVM %s\n", kvmName)
				if _, err = kvm.Create("", kvmName, true); err != nil {
					return err
				}
				fmt.Printf("\tImporting entries for %s\n", kvmName)
				if orgKVMFileList[kvmName] != "" {
					if err = kvm.ImportEntries("", kvmName, conn, orgKVMFileList[kvmName]); err != nil {
						return err
					}
				}
			}
		}

		if len(envKVMFileList) > 0 {
			fmt.Println("Importing env scoped KVMs...")
			for _, envKVMFile := range envKVMFileList {
				kvmMetadata := strings.Split(envKVMFile, "_")
				apiclient.SetApigeeEnv(kvmMetadata[1])
				fmt.Printf("\tCreating KVM %s\n", envKVMFile)
				if _, err = kvm.Create("", kvmMetadata[2], true); err != nil {
					return err
				}
				fmt.Printf("\tImporting entries for %s\n", envKVMFile)
				if err = kvm.ImportEntries("", kvmMetadata[2], conn, envKVMFile); err != nil {
					return err
				}
			}
		}

		if len(proxyKVMFileList) > 0 {
			fmt.Println("Importing proxy scoped KVMs...")
			for _, proxyKVMFile := range proxyKVMFileList {
				kvmMetadata := strings.Split(proxyKVMFile, "_")
				fmt.Printf("\tCreating KVM %s\n", proxyKVMFile)
				if _, err = kvm.Create(kvmMetadata[1], "", true); err != nil {
					return err
				}
				fmt.Printf("\tImporting entries for %s\n", proxyKVMFile)
				if err = kvm.ImportEntries(kvmMetadata[1], kvmMetadata[2], conn, proxyKVMFile); err != nil {
					return err
				}
			}
		}

		return err
	},
}

var folder string

func init() {

	ImpCmd.Flags().StringVarP(&folder, "folder", "f",
		"", "The absolute path to the folder containing KVM entries")
	ImpCmd.Flags().IntVarP(&conn, "conn", "c",
		4, "Number of connections")

	_ = ImpCmd.MarkFlagRequired("folder")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/signal"
	"strings"
	"time"

	"github.com/bwmarrin/discordgo"
)

func init() {
	config.DiscordToken = os.Getenv("DISCORD_TOKEN")
	config.Debug = os.Getenv("STATE") == "debug"
}

func main() {
	discord, err := discordgo.New(config.DiscordToken)
	if err != nil {
		fmt.Println(err)
		return
	}
	// Get the account information.
	u, err := discord.User("@me")
	if err != nil {
		fmt.Println("error obtaining account details,", err)
	}

	// Store the account ID for later use.
	config.BotID = u.ID

	discord.AddHandler(messageCreate)

	err = discord.Open()
	if err != nil {
		fmt.Println(err)
	}

	fmt.Println("MoeBot is up and running!")
	handleExit()
}

func messageCreate(s *discordgo.Session, m *discordgo.MessageCreate) {
	// log message
	if config.Debug {
		fmt.Printf("%20s %10s %10s > %s\n", m.ChannelID, time.Now().Format(time.Stamp), m.Author.Username, m.Content)
	}
	if m.Author.ID != config.BotID {
		parseMessage(s, m)
	}
}

func parseMessage(session *discordgo.Session, message *discordgo.MessageCreate) {
	content := strings.ToLower(message.Content)
	if strings.HasPrefix(content, "hi moebot") || strings.HasPrefix(content, "hello moebot") {
		session.ChannelMessageSend(message.ChannelID, "Hello "+message.Author.Username+"!")
	}
}

func handleExit() {
	sigs := make(chan os.Signal, 1)
	exit := make(chan bool, 1)
	signal.Notify(sigs)

	go func() {
		sig := <-sigs
		fmt.Println()
		fmt.Println(sig)
		exit <- true
	}()

	<-exit
	fmt.Println("Disconnectiong from Discord")
	fmt.Println("Exiting from MoeBot...")
}

var config struct {
	Debug        bool
	DiscordToken string
	BotID        string
}
<commit_msg>Added the "moebot whats moe" feature<commit_after>package main

import (
	"fmt"
	"os"
	"os/signal"
	"strings"
	"time"

	"github.com/bwmarrin/discordgo"
)

func init() {
	config.DiscordToken = os.Getenv("DISCORD_TOKEN")
	config.Debug = os.Getenv("STATE") == "debug"
}

func main() {
	discord, err := discordgo.New(config.DiscordToken)
	if err != nil {
		fmt.Println(err)
		return
	}
	// Get the account information.
	u, err := discord.User("@me")
	if err != nil {
		fmt.Println("error obtaining account details,", err)
	}

	// Store the account ID for later use.
	config.BotID = u.ID

	discord.AddHandler(messageCreate)

	err = discord.Open()
	if err != nil {
		fmt.Println(err)
	}

	fmt.Println("MoeBot is up and running!")
	handleExit()
}

func messageCreate(s *discordgo.Session, m *discordgo.MessageCreate) {
	// log message
	if config.Debug {
		fmt.Printf("%20s %10s %10s > %s\n", m.ChannelID, time.Now().Format(time.Stamp), m.Author.Username, m.Content)
	}
	if m.Author.ID != config.BotID {
		parseMessage(s, m)
	}
}

func parseMessage(session *discordgo.Session, message *discordgo.MessageCreate) {
	content := strings.ToLower(message.Content)
	if strings.HasPrefix(content, "hi moebot") || strings.HasPrefix(content, "hello moebot") {
		session.ChannelMessageSend(message.ChannelID, "Hello "+message.Author.Username+"!")
	} else if strings.HasPrefix(content, "moebot what's moe") || strings.HasPrefix(content, "moebot what is moe") {
		session.ChannelMessageSend(message.ChannelID, "Check this out: https://vimeo.com/70285271")
	}
}

func handleExit() {
	sigs := make(chan os.Signal, 1)
	exit := make(chan bool, 1)
	signal.Notify(sigs)

	go func() {
		sig := <-sigs
		fmt.Println()
		fmt.Println(sig)
		exit <- true
	}()

	<-exit
	fmt.Println("Disconnectiong from Discord")
	fmt.Println("Exiting from MoeBot...")
}

var config struct {
	Debug        bool
	DiscordToken string
	BotID        string
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"os/exec"
	"time"

	"github.com/Songmu/retry"
	"github.com/Songmu/wrapcommander"
	"github.com/jessevdk/go-flags"
)

type opts struct {
	Retry    int64   `short:"r" long:"retry" description:"hoge"`
	Interval float64 `short:"i" long:"interval" description:"hoge"`
}

func main() {
	os.Exit(run(os.Args[1:]))
}

func run(args []string) int {
	p, o, cmdArgs, err := parseArgs(args)
	if err != nil || len(cmdArgs) < 1 {
		if ferr, ok := err.(*flags.Error); !ok || ferr.Type != flags.ErrHelp {
			p.WriteHelp(os.Stderr)
		}
		return 2
	}
	err = o.run(cmdArgs)
	return wrapcommander.ResolveExitCode(err)
}

func parseArgs(args []string) (*flags.Parser, *opts, []string, error) {
	o := &opts{}
	p := flags.NewParser(o, flags.Default)

	rest, err := p.ParseArgs(args)
	return p, o, rest, err
}

func (o *opts) run(args []string) error {
	return retry.Retry(uint(o.Retry), time.Duration(o.Interval*float64(time.Second)), func() error {
		cmd := exec.Command(args[0], args[1:]...)
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		return cmd.Run()
	})
}
<commit_msg>fix interface<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"strconv"
	"time"

	"github.com/Songmu/retry"
	"github.com/Songmu/wrapcommander"
)

type opts struct {
	retry    uint
	interval float64
	cmd      []string
}

func main() {
	os.Exit(run(os.Args[1:]))
}

func run(args []string) int {
	o, err := parseArgs(args)
	if err != nil {
		if err.Error() != "" {
			fmt.Fprintln(os.Stderr, err)
		}
		writeHelp()
		return 2
	}
	err = o.run()
	return wrapcommander.ResolveExitCode(err)
}

func writeHelp() {
	fmt.Fprint(os.Stderr, `Usage: retry COUNT [INTERVAL] COMMAND [ARG]...

Start COMMAND, and retry it until success up to COUNT with INTERVAL seconds.

 COUNT: max retry count
 INTERVAL: interval seconds (default 1.0)
`)
}

func parseArgs(args []string) (*opts, error) {
	if len(args) < 2 {
		return nil, fmt.Errorf("")
	}
	o := &opts{}
	retry, err := strconv.ParseUint(args[0], 10, 64)
	if err != nil {
		return nil, fmt.Errorf("retry COUNT should be an integer")
	}
	o.retry = uint(retry)
	cmdIdx := 1
	interval, err := strconv.ParseFloat(args[1], 64)
	if err != nil {
		o.interval = 1.0 // default
	} else {
		// custom interval
		o.interval = interval
		cmdIdx++
	}
	o.cmd = args[cmdIdx:]
	if len(o.cmd) < 1 {
		return nil, fmt.Errorf("no command arguments are specified")
	}
	return o, nil
}

func (o *opts) run() error {
	return retry.Retry(uint(o.retry), time.Duration(o.interval*float64(time.Second)), func() error {
		cmd := exec.Command(o.cmd[0], o.cmd[1:]...)
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		err := cmd.Run()
		if !wrapcommander.IsInvoked(err) {
			fmt.Fprintln(os.Stderr, err)
		}
		return err
	})
}
<|endoftext|>


<commit_before>// +build !windows

// Copyright 2015 Richard Lehane. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"os"
	"time"

	"github.com/richardlehane/siegfried"
)

func retryOpen(path string, err error) (*os.File, error) {
	return nil, err
}

func retryStat(path string, err error) (os.FileInfo, error) {
	return nil, err
}

func multiIdentifyS(w writer, s *siegfried.Siegfried, root, orig string, norecurse bool) error {
	walkFunc := func(path string, info os.FileInfo, err error) error {
		if *throttlef > 0 {
			<-throttle.C
		}
		if err != nil {
			return fmt.Errorf("walking %s; got %v", path, err)
		}
		if info.IsDir() {
			if norecurse && path != root {
				return filepath.SkipDir
			}
			if *droido {
				w.writeFile(shortpath(path, orig), -1, info.ModTime().Format(time.RFC3339), nil, nil, nil) // write directory with a -1 size for droid output only
			}
			return nil
		}
		identifyFile(w, s, path, info.Size(), info.ModTime().Format(time.RFC3339))
		return nil
	}
	return filepath.Walk(root, walkFunc)
}
<commit_msg>missing import<commit_after>// +build !windows

// Copyright 2015 Richard Lehane. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"os"
	"path/filepath"
	"time"

	"github.com/richardlehane/siegfried"
)

func retryOpen(path string, err error) (*os.File, error) {
	return nil, err
}

func retryStat(path string, err error) (os.FileInfo, error) {
	return nil, err
}

func multiIdentifyS(w writer, s *siegfried.Siegfried, root, orig string, norecurse bool) error {
	walkFunc := func(path string, info os.FileInfo, err error) error {
		if *throttlef > 0 {
			<-throttle.C
		}
		if err != nil {
			return fmt.Errorf("walking %s; got %v", path, err)
		}
		if info.IsDir() {
			if norecurse && path != root {
				return filepath.SkipDir
			}
			if *droido {
				w.writeFile(path, -1, info.ModTime().Format(time.RFC3339), nil, nil, nil) // write directory with a -1 size for droid output only
			}
			return nil
		}
		identifyFile(w, s, path, info.Size(), info.ModTime().Format(time.RFC3339))
		return nil
	}
	return filepath.Walk(root, walkFunc)
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/sha256"
	"fmt"
	"io"
	"os"

	"github.com/fd0/khepri/chunker"
)

func main() {
	count, bytes := 0, 0
	min := 0
	max := 0

	var (
		err  error
		file *os.File = os.Stdin
	)

	if len(os.Args) > 1 {
		file, err = os.Open(os.Args[1])
		if err != nil {
			panic(err)
		}
	}

	ch := chunker.New(file)

	for {
		chunk, err := ch.Next()

		if chunk != nil {
			fmt.Printf("%d %016x %02x\n", chunk.Length, chunk.Cut, sha256.Sum256(chunk.Data))
			count++
			bytes += chunk.Length

			if chunk.Length == chunker.MaxSize {
				max++
			} else if chunk.Length == chunker.MinSize {
				min++
			}

		}

		if err == io.EOF {
			break
		}
	}

	var avg int
	if count > 0 {
		avg = bytes / count
	}

	fmt.Fprintf(os.Stderr, "%d chunks, average size %d (%d min size, %d max size chunks)\n",
		count, avg, min, max)
}
<commit_msg>Extend cmd/split<commit_after>package main

import (
	"crypto/sha256"
	"fmt"
	"io"
	"os"

	"github.com/fd0/khepri/chunker"
)

func main() {
	count, bytes := 0, 0
	min := 0
	max := 0

	var (
		err  error
		file *os.File = os.Stdin
	)

	if len(os.Args) > 1 {
		file, err = os.Open(os.Args[1])
		if err != nil {
			panic(err)
		}
	}

	ch := chunker.New(file)

	for {
		chunk, err := ch.Next()

		if err == io.EOF {
			break
		}

		if err != nil {
			panic(err)
		}

		fmt.Printf("%d %016x %02x\n", chunk.Length, chunk.Cut, sha256.Sum256(chunk.Data))
		count++
		bytes += chunk.Length

		if chunk.Length == chunker.MaxSize {
			max++
		} else if chunk.Length == chunker.MinSize {
			min++
		}
	}

	var avg int
	if count > 0 {
		avg = bytes / count
	}

	fmt.Fprintf(os.Stderr, "%d chunks from %d bytes, average size %d (%d min size, %d max size chunks)\n",
		count, bytes, avg, min, max)
}
<|endoftext|>


<commit_before>package main

import (
	"compress/gzip"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"strconv"
	"strings"
	"sync"

	"github.com/thomersch/grandine/lib/mvt"
	"github.com/thomersch/grandine/lib/progressbar"
	"github.com/thomersch/grandine/lib/spaten"
	"github.com/thomersch/grandine/lib/spatial"
	"github.com/thomersch/grandine/lib/tile"
)

type zmLvl []int

func (zm *zmLvl) String() string {
	return fmt.Sprintf("%d", *zm)
}

func (zm *zmLvl) Set(value string) error {
	for _, s := range strings.Split(value, ",") {
		v, err := strconv.Atoi(strings.TrimSpace(s))
		if err != nil {
			return fmt.Errorf("%s (only integer values are allowed)", value)
		}
		*zm = append(*zm, v)
	}
	return nil
}

type bbox spatial.BBox

func (b *bbox) String() string {
	return fmt.Sprintf("%v", *b)
}

func (b *bbox) Set(value string) error {
	var (
		fl    [4]float64
		parts = strings.Split(value, ",")
		err   error
	)
	if len(parts) != 4 {
		return errors.New("bbox takes exactly 4 parameters: SW Lon, SW Lat, NE Lon, NE Lat")
	}
	for i, s := range parts {
		fl[i], err = strconv.ParseFloat(s, 64)
		if err != nil {
			return fmt.Errorf("could not parse bbox expression: %v", err)
		}
	}
	b.SW = spatial.Point{fl[0], fl[1]}
	b.NE = spatial.Point{fl[2], fl[3]}
	return nil
}

var (
	zoomlevels zmLvl
	quiet      *bool
)

func main() {
	var (
		sourceStdIn bool
		tileCodec   tile.Codec
	)
	source := flag.String("in", "", "file to read from, supported format: spaten")
	target := flag.String("out", "tiles", "path where the tiles will be written")
	defaultLayer := flag.Bool("default-layer", true, "if no layer name is specified in the feature, whether it will be put into a default layer")
	workersNumber := flag.Int("workers", runtime.GOMAXPROCS(0), "number of workers")
	cpuProfile := flag.String("cpuprof", "", "writes CPU profiling data into a file")
	geojsonCodec := flag.Bool("geojson", false, "encode tiles into geojson instead of MVT, for debugging purposes")
	compressTiles := flag.Bool("compress", false, "compress tiles with gzip")
	quiet = flag.Bool("q", false, "argument to use if program should be run in quiet mode with reduced logging")

	flag.Var(&zoomlevels, "zoom", "one or more zoom level of which the tiles will be rendered")
	flag.Parse()

	if len(*source) == 0 {
		log.Print("grandine-tiler: Reading from stdin.")
		sourceStdIn = true
	}

	if len(zoomlevels) == 0 {
		log.Fatal("no zoom levels specified")
	}

	if len(*cpuProfile) != 0 {
		f, err := os.Create(*cpuProfile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *geojsonCodec {
		tileCodec = &tile.GeoJSONCodec{}
	} else {
		tileCodec = &mvt.Codec{}
	}

	var (
		f   io.Reader
		err error
	)

	if !sourceStdIn {
		f, err = os.Open(*source)
		if err != nil {
			log.Fatal(err)
		}
		defer f.(io.Closer).Close()
	} else {
		f = os.Stdin
	}

	err = os.MkdirAll(*target, 0777)
	if err != nil {
		log.Fatal(err)
	}

	log.Println("parsing input...")
	fc := spatial.FeatureCollection{}
	codec := spaten.Codec{}
	err = codec.Decode(f, &fc)
	if err != nil {
		log.Fatalf("could not read incoming file: %v", err)
	}

	if len(fc.Features) == 0 {
		log.Fatal("no features in input file")
	}

	log.Printf("read %d features", len(fc.Features))

	var (
		bboxPts []spatial.Point
		ft      = featureTable(zoomlevels)
	)
	for featID, feat := range fc.Features {
		for _, zl := range zoomlevels {
			if !renderable(feat.Props, zl) {
				continue
			}
			for _, tid := range tile.Coverage(feat.Geometry.BBox(), zl) {
				ft[zl][tid.X][tid.Y] = append(ft[zl][tid.X][tid.Y], &fc.Features[featID])
			}
		}
		bb := feat.Geometry.BBox()
		bboxPts = append(bboxPts, bb.SW, bb.NE)
	}

	log.Println("determining which tiles need to be generated")
	var tc []tile.ID
	for _, zoomlevel := range zoomlevels {
		tc = append(tc, tile.Coverage(spatial.Line(bboxPts).BBox(), zoomlevel)...)
	}

	log.Printf("starting to generate %d tiles...", len(tc))

	dtw := diskTileWriter{basedir: *target, compressTiles: *compressTiles}
	dlm := defaultLayerMapper{defaultLayer: *defaultLayer}

	shuffleWork(tc) // randomize order for better worker saturation
	var (
		wg       sync.WaitGroup
		ws       = workerSlices(tc, *workersNumber)
		pb, done = progressbar.NewBar(len(tc), len(ws))
	)
	for wrk := 0; wrk < len(ws); wrk++ {
		wg.Add(1)
		go func(i int) {
			generateTiles(ws[i], ft, &dtw, tileCodec, &dlm, pb)
			wg.Done()
		}(wrk)
	}
	wg.Wait()
	done()
}

func renderable(props map[string]interface{}, zl int) bool {
	var propInt = func(props map[string]interface{}, name string, defaultVal int) int {
		v, ok := props[name]
		if !ok {
			return defaultVal
		}
		i, ok := v.(int)
		if ok {
			return i
		}
		f, ok := v.(float64)
		if ok {
			return int(f)
		}
		log.Printf("%v is neither int nor float: %v", props, v)
		return defaultVal
	}
	return (zl >= propInt(props, "@zoom:min", 0)) && (zl <= propInt(props, "@zoom:max", 99))
}

func featureTable(zls []int) map[int][][][]*spatial.Feature {
	r := map[int][][][]*spatial.Feature{}
	for _, zl := range zls {
		l := pow(2, zl)
		r[zl] = make([][][]*spatial.Feature, l)
		for x := range r[zl] {
			r[zl][x] = make([][]*spatial.Feature, l)
		}
	}
	return r
}

func workerSlices(tiles []tile.ID, wrkNum int) [][]tile.ID {
	var r [][]tile.ID
	if len(tiles) <= wrkNum {
		for t := 0; t < len(tiles); t++ {
			r = append(r, []tile.ID{tiles[t]})
		}
		return r
	}
	for wrkr := 0; wrkr < wrkNum; wrkr++ {
		start := (len(tiles) / wrkNum) * wrkr
		end := (len(tiles) / wrkNum) * (wrkr + 1)
		if wrkr == wrkNum-1 {
			end = len(tiles)
		}
		r = append(r, tiles[start:end])
	}
	return r
}

type diskTileWriter struct {
	basedir       string
	compressTiles bool
}

func (tw *diskTileWriter) WriteTile(tID tile.ID, buf []byte) error {
	err := os.MkdirAll(filepath.Join(tw.basedir, strconv.Itoa(tID.Z), strconv.Itoa(tID.X)), 0777)
	if err != nil {
		return err
	}
	tf, err := os.Create(filepath.Join(tw.basedir, strconv.Itoa(tID.Z), strconv.Itoa(tID.X), strconv.Itoa(tID.Y)+".mvt")) // TODO: file extension based on codec
	if err != nil {
		return err
	}
	defer tf.Close()

	if tw.compressTiles {
		_, err = gzip.NewWriter(tf).Write(buf)
	} else {
		_, err = tf.Write(buf)
	}
	if err != nil {
		return err
	}
	return nil
}

type defaultLayerMapper struct {
	defaultLayer bool
}

func (dlm *defaultLayerMapper) LayerName(props map[string]interface{}) string {
	if layerName, ok := props["@layer"]; ok {
		return layerName.(string)
	}
	if dlm.defaultLayer {
		return "default"
	}
	return ""
}

type layerMapper interface {
	LayerName(map[string]interface{}) string
}

type tileWriter interface {
	WriteTile(tile.ID, []byte) error
}

func generateTiles(tIDs []tile.ID, features map[int][][][]*spatial.Feature, tw tileWriter, encoder tile.Codec, lm layerMapper, pb chan<- struct{}) {
	for _, tID := range tIDs {
		var (
			layers = map[string][]spatial.Feature{}
			ln     string
		)
		defer func() { pb <- struct{}{} }()
		for _, ft := range features[tID.Z][tID.X][tID.Y] {
			feat := *ft
			ln = lm.LayerName(feat.Props)
			if len(ln) != 0 {
				if _, ok := layers[ln]; !ok {
					layers[ln] = []spatial.Feature{feat}
				} else {
					layers[ln] = append(layers[ln], feat)
				}
			}
		}
		if !anyFeatures(layers) {
			continue
		}
		buf, err := encoder.EncodeTile(layers, tID)
		if err != nil {
			log.Fatal(err)
		}

		err = tw.WriteTile(tID, buf)
		if err != nil {
			log.Fatal(err)
		}
	}
}

func anyFeatures(layers map[string][]spatial.Feature) bool {
	for i := range layers {
		if len(layers[i]) > 0 {
			return true
		}
	}
	return false
}

func pow(x, y int) int {
	var res = 1
	for i := 1; i <= y; i++ {
		res *= x
	}
	return res
}
<commit_msg>cmd/tiler: clarify zoom level syntax, allow trailing commas<commit_after>package main

import (
	"compress/gzip"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"strconv"
	"strings"
	"sync"

	"github.com/thomersch/grandine/lib/mvt"
	"github.com/thomersch/grandine/lib/progressbar"
	"github.com/thomersch/grandine/lib/spaten"
	"github.com/thomersch/grandine/lib/spatial"
	"github.com/thomersch/grandine/lib/tile"
)

type zmLvl []int

func (zm *zmLvl) String() string {
	return fmt.Sprintf("%d", *zm)
}

func (zm *zmLvl) Set(value string) error {
	for _, s := range strings.Split(value, ",") {
		zs := strings.TrimSpace(s)
		if zs == "" {
			continue
		}
		v, err := strconv.Atoi(zs)
		if err != nil {
			return fmt.Errorf("%s (only integer values are allowed)", value)
		}
		*zm = append(*zm, v)
	}
	return nil
}

type bbox spatial.BBox

func (b *bbox) String() string {
	return fmt.Sprintf("%v", *b)
}

func (b *bbox) Set(value string) error {
	var (
		fl    [4]float64
		parts = strings.Split(value, ",")
		err   error
	)
	if len(parts) != 4 {
		return errors.New("bbox takes exactly 4 parameters: SW Lon, SW Lat, NE Lon, NE Lat")
	}
	for i, s := range parts {
		fl[i], err = strconv.ParseFloat(s, 64)
		if err != nil {
			return fmt.Errorf("could not parse bbox expression: %v", err)
		}
	}
	b.SW = spatial.Point{fl[0], fl[1]}
	b.NE = spatial.Point{fl[2], fl[3]}
	return nil
}

var (
	zoomlevels zmLvl
	quiet      *bool
)

func main() {
	var (
		sourceStdIn bool
		tileCodec   tile.Codec
	)
	source := flag.String("in", "", "file to read from, supported format: spaten")
	target := flag.String("out", "tiles", "path where the tiles will be written")
	defaultLayer := flag.Bool("default-layer", true, "if no layer name is specified in the feature, whether it will be put into a default layer")
	workersNumber := flag.Int("workers", runtime.GOMAXPROCS(0), "number of workers")
	cpuProfile := flag.String("cpuprof", "", "writes CPU profiling data into a file")
	geojsonCodec := flag.Bool("geojson", false, "encode tiles into geojson instead of MVT, for debugging purposes")
	compressTiles := flag.Bool("compress", false, "compress tiles with gzip")
	quiet = flag.Bool("q", false, "argument to use if program should be run in quiet mode with reduced logging")

	flag.Var(&zoomlevels, "zoom", "one or more zoom levels (comma separated) of which the tiles will be rendered")
	flag.Parse()

	if len(*source) == 0 {
		log.Print("grandine-tiler: Reading from stdin.")
		sourceStdIn = true
	}

	if len(zoomlevels) == 0 {
		log.Fatal("no zoom levels specified")
	}

	if len(*cpuProfile) != 0 {
		f, err := os.Create(*cpuProfile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *geojsonCodec {
		tileCodec = &tile.GeoJSONCodec{}
	} else {
		tileCodec = &mvt.Codec{}
	}

	var (
		f   io.Reader
		err error
	)

	if !sourceStdIn {
		f, err = os.Open(*source)
		if err != nil {
			log.Fatal(err)
		}
		defer f.(io.Closer).Close()
	} else {
		f = os.Stdin
	}

	err = os.MkdirAll(*target, 0777)
	if err != nil {
		log.Fatal(err)
	}

	log.Println("parsing input...")
	fc := spatial.FeatureCollection{}
	codec := spaten.Codec{}
	err = codec.Decode(f, &fc)
	if err != nil {
		log.Fatalf("could not read incoming file: %v", err)
	}

	if len(fc.Features) == 0 {
		log.Fatal("no features in input file")
	}

	log.Printf("read %d features", len(fc.Features))

	var (
		bboxPts []spatial.Point
		ft      = featureTable(zoomlevels)
	)
	for featID, feat := range fc.Features {
		for _, zl := range zoomlevels {
			if !renderable(feat.Props, zl) {
				continue
			}
			for _, tid := range tile.Coverage(feat.Geometry.BBox(), zl) {
				ft[zl][tid.X][tid.Y] = append(ft[zl][tid.X][tid.Y], &fc.Features[featID])
			}
		}
		bb := feat.Geometry.BBox()
		bboxPts = append(bboxPts, bb.SW, bb.NE)
	}

	log.Println("determining which tiles need to be generated")
	var tc []tile.ID
	for _, zoomlevel := range zoomlevels {
		tc = append(tc, tile.Coverage(spatial.Line(bboxPts).BBox(), zoomlevel)...)
	}

	log.Printf("starting to generate %d tiles...", len(tc))

	dtw := diskTileWriter{basedir: *target, compressTiles: *compressTiles}
	dlm := defaultLayerMapper{defaultLayer: *defaultLayer}

	shuffleWork(tc) // randomize order for better worker saturation
	var (
		wg       sync.WaitGroup
		ws       = workerSlices(tc, *workersNumber)
		pb, done = progressbar.NewBar(len(tc), len(ws))
	)
	for wrk := 0; wrk < len(ws); wrk++ {
		wg.Add(1)
		go func(i int) {
			generateTiles(ws[i], ft, &dtw, tileCodec, &dlm, pb)
			wg.Done()
		}(wrk)
	}
	wg.Wait()
	done()
}

func renderable(props map[string]interface{}, zl int) bool {
	var propInt = func(props map[string]interface{}, name string, defaultVal int) int {
		v, ok := props[name]
		if !ok {
			return defaultVal
		}
		i, ok := v.(int)
		if ok {
			return i
		}
		f, ok := v.(float64)
		if ok {
			return int(f)
		}
		log.Printf("%v is neither int nor float: %v", props, v)
		return defaultVal
	}
	return (zl >= propInt(props, "@zoom:min", 0)) && (zl <= propInt(props, "@zoom:max", 99))
}

func featureTable(zls []int) map[int][][][]*spatial.Feature {
	r := map[int][][][]*spatial.Feature{}
	for _, zl := range zls {
		l := pow(2, zl)
		r[zl] = make([][][]*spatial.Feature, l)
		for x := range r[zl] {
			r[zl][x] = make([][]*spatial.Feature, l)
		}
	}
	return r
}

func workerSlices(tiles []tile.ID, wrkNum int) [][]tile.ID {
	var r [][]tile.ID
	if len(tiles) <= wrkNum {
		for t := 0; t < len(tiles); t++ {
			r = append(r, []tile.ID{tiles[t]})
		}
		return r
	}
	for wrkr := 0; wrkr < wrkNum; wrkr++ {
		start := (len(tiles) / wrkNum) * wrkr
		end := (len(tiles) / wrkNum) * (wrkr + 1)
		if wrkr == wrkNum-1 {
			end = len(tiles)
		}
		r = append(r, tiles[start:end])
	}
	return r
}

type diskTileWriter struct {
	basedir       string
	compressTiles bool
}

func (tw *diskTileWriter) WriteTile(tID tile.ID, buf []byte) error {
	err := os.MkdirAll(filepath.Join(tw.basedir, strconv.Itoa(tID.Z), strconv.Itoa(tID.X)), 0777)
	if err != nil {
		return err
	}
	tf, err := os.Create(filepath.Join(tw.basedir, strconv.Itoa(tID.Z), strconv.Itoa(tID.X), strconv.Itoa(tID.Y)+".mvt")) // TODO: file extension based on codec
	if err != nil {
		return err
	}
	defer tf.Close()

	if tw.compressTiles {
		_, err = gzip.NewWriter(tf).Write(buf)
	} else {
		_, err = tf.Write(buf)
	}
	if err != nil {
		return err
	}
	return nil
}

type defaultLayerMapper struct {
	defaultLayer bool
}

func (dlm *defaultLayerMapper) LayerName(props map[string]interface{}) string {
	if layerName, ok := props["@layer"]; ok {
		return layerName.(string)
	}
	if dlm.defaultLayer {
		return "default"
	}
	return ""
}

type layerMapper interface {
	LayerName(map[string]interface{}) string
}

type tileWriter interface {
	WriteTile(tile.ID, []byte) error
}

func generateTiles(tIDs []tile.ID, features map[int][][][]*spatial.Feature, tw tileWriter, encoder tile.Codec, lm layerMapper, pb chan<- struct{}) {
	for _, tID := range tIDs {
		var (
			layers = map[string][]spatial.Feature{}
			ln     string
		)
		defer func() { pb <- struct{}{} }()
		for _, ft := range features[tID.Z][tID.X][tID.Y] {
			feat := *ft
			ln = lm.LayerName(feat.Props)
			if len(ln) != 0 {
				if _, ok := layers[ln]; !ok {
					layers[ln] = []spatial.Feature{feat}
				} else {
					layers[ln] = append(layers[ln], feat)
				}
			}
		}
		if !anyFeatures(layers) {
			continue
		}
		buf, err := encoder.EncodeTile(layers, tID)
		if err != nil {
			log.Fatal(err)
		}

		err = tw.WriteTile(tID, buf)
		if err != nil {
			log.Fatal(err)
		}
	}
}

func anyFeatures(layers map[string][]spatial.Feature) bool {
	for i := range layers {
		if len(layers[i]) > 0 {
			return true
		}
	}
	return false
}

func pow(x, y int) int {
	var res = 1
	for i := 1; i <= y; i++ {
		res *= x
	}
	return res
}
<|endoftext|>


<commit_before>package authentication

import (
	"context"
	"fmt"
	"log"
	"strings"

	"github.com/Azure/go-autorest/autorest"
	"github.com/Azure/go-autorest/autorest/adal"
	"github.com/manicminer/hamilton/auth"
	"github.com/manicminer/hamilton/environments"
)

// Config is the configuration structure used to instantiate a
// new Azure management client.
type Config struct {
	ClientID           string
	SubscriptionID     string
	TenantID           string
	AuxiliaryTenantIDs []string
	Environment        string
	MetadataHost       string

	GetAuthenticatedObjectID         func(context.Context) (*string, error)
	AuthenticatedAsAServicePrincipal bool

	// A Custom Resource Manager Endpoint
	// at this time this should only be applicable for Azure Stack.
	CustomResourceManagerEndpoint string

	// Beta opt-in for Microsoft Graph
	UseMicrosoftGraph bool

	authMethod authMethod
}

type OAuthConfig struct {
	OAuth            *adal.OAuthConfig
	MultiTenantOauth *adal.MultiTenantOAuthConfig
}

// GetAuthorizationToken returns an authorization token for the authentication method defined in the Config
func (c Config) GetOAuthConfig(activeDirectoryEndpoint string) (*adal.OAuthConfig, error) {
	log.Printf("Getting OAuth config for endpoint %s with  tenant %s", activeDirectoryEndpoint, c.TenantID)

	// fix for ADFS environments, if the login endpoint ends in `/adfs` it's an adfs environment
	// the login endpoint ends up residing in `ActiveDirectoryEndpoint`
	oAuthTenant := c.TenantID
	if strings.HasSuffix(strings.ToLower(activeDirectoryEndpoint), "/adfs") {
		log.Printf("[DEBUG] ADFS environment detected - overriding Tenant ID to `adfs`!")
		oAuthTenant = "adfs"
	}

	oauth, err := adal.NewOAuthConfig(activeDirectoryEndpoint, oAuthTenant)
	if err != nil {
		return nil, err
	}

	// OAuthConfigForTenant returns a pointer, which can be nil.
	if oauth == nil {
		return nil, fmt.Errorf("unable to configure OAuthConfig for tenant %s", c.TenantID)
	}

	return oauth, nil
}

// GetMultiTenantOAuthConfig returns a multi-tenant authorization token for the authentication method defined in the Config
func (c Config) GetMultiTenantOAuthConfig(activeDirectoryEndpoint string) (*adal.MultiTenantOAuthConfig, error) {
	log.Printf("Getting multi OAuth config for endpoint %s with  tenant %s (aux tenants: %v)", activeDirectoryEndpoint, c.TenantID, c.AuxiliaryTenantIDs)
	oauth, err := adal.NewMultiTenantOAuthConfig(activeDirectoryEndpoint, c.TenantID, c.AuxiliaryTenantIDs, adal.OAuthOptions{})
	if err != nil {
		return nil, err
	}

	// OAuthConfigForTenant returns a pointer, which can be nil.
	if oauth == nil {
		return nil, fmt.Errorf("unable to configure OAuthConfig for tenant %s (auxiliary tenants %v)", c.TenantID, c.AuxiliaryTenantIDs)
	}

	return &oauth, nil
}

// BuildOAuthConfig builds the authorization configuration for the specified Active Directory Endpoint
func (c Config) BuildOAuthConfig(activeDirectoryEndpoint string) (*OAuthConfig, error) {
	multiAuth := OAuthConfig{}
	var err error

	multiAuth.OAuth, err = c.GetOAuthConfig(activeDirectoryEndpoint)
	if err != nil {
		return nil, err
	}

	if len(c.AuxiliaryTenantIDs) > 0 {
		multiAuth.MultiTenantOauth, err = c.GetMultiTenantOAuthConfig(activeDirectoryEndpoint)
		if err != nil {
			return nil, err
		}
	}

	return &multiAuth, nil
}

// ADALBearerAuthorizerCallback returns a BearerAuthorizer valid only for the Primary Tenant
// this signs a request using the AccessToken returned from the primary Resource Manager authorizer
func (c Config) ADALBearerAuthorizerCallback(ctx context.Context, sender autorest.Sender, oauthConfig *OAuthConfig) *autorest.BearerAuthorizerCallback {
	return autorest.NewBearerAuthorizerCallback(sender, func(tenantID, resource string) (*autorest.BearerAuthorizer, error) {
		// a BearerAuthorizer is only valid for the primary tenant
		newAuthConfig := &OAuthConfig{
			OAuth: oauthConfig.OAuth,
		}

		storageSpt, err := c.GetADALToken(ctx, sender, newAuthConfig, resource)
		if err != nil {
			return nil, err
		}

		cast, ok := storageSpt.(*autorest.BearerAuthorizer)
		if !ok {
			return nil, fmt.Errorf("converting %+v to a BearerAuthorizer", storageSpt)
		}

		return cast, nil
	})
}

// MSALBearerAuthorizerCallback returns a BearerAuthorizer valid only for the Primary Tenant
// this signs a request using the AccessToken returned from the primary Resource Manager authorizer
func (c Config) MSALBearerAuthorizerCallback(ctx context.Context, api environments.Api, sender autorest.Sender, oauthConfig *OAuthConfig, endpoint string) *autorest.BearerAuthorizerCallback {
	authorizer, err := c.GetMSALToken(ctx, api, sender, oauthConfig, endpoint)
	if err != nil {
		return nil // BOOM
	}

	cast, ok := authorizer.(*auth.CachedAuthorizer)
	if !ok {
		return nil // BOOM
	}

	return cast.BearerAuthorizerCallback()
}

// GetADALToken returns an autorest.Authorizer using an ADAL token via the authentication method defined in the Config
func (c Config) GetADALToken(ctx context.Context, sender autorest.Sender, oauth *OAuthConfig, endpoint string) (autorest.Authorizer, error) {
	return c.authMethod.getADALToken(ctx, sender, oauth, endpoint)
}

// GetMSALToken returns an autorest.Authorizer using an MSAL token via the authentication method defined in the Config
func (c Config) GetMSALToken(ctx context.Context, api environments.Api, sender autorest.Sender, oauth *OAuthConfig, endpoint string) (autorest.Authorizer, error) {
	return c.authMethod.getMSALToken(ctx, api, sender, oauth, endpoint)
}
<commit_msg>Better error handling for auth.Config.MSALBearerAuthorizerCallback; don't return a nil function<commit_after>package authentication

import (
	"context"
	"fmt"
	"log"
	"strings"

	"github.com/Azure/go-autorest/autorest"
	"github.com/Azure/go-autorest/autorest/adal"
	"github.com/manicminer/hamilton/auth"
	"github.com/manicminer/hamilton/environments"
)

// Config is the configuration structure used to instantiate a
// new Azure management client.
type Config struct {
	ClientID           string
	SubscriptionID     string
	TenantID           string
	AuxiliaryTenantIDs []string
	Environment        string
	MetadataHost       string

	GetAuthenticatedObjectID         func(context.Context) (*string, error)
	AuthenticatedAsAServicePrincipal bool

	// A Custom Resource Manager Endpoint
	// at this time this should only be applicable for Azure Stack.
	CustomResourceManagerEndpoint string

	// Beta opt-in for Microsoft Graph
	UseMicrosoftGraph bool

	authMethod authMethod
}

type OAuthConfig struct {
	OAuth            *adal.OAuthConfig
	MultiTenantOauth *adal.MultiTenantOAuthConfig
}

// GetAuthorizationToken returns an authorization token for the authentication method defined in the Config
func (c Config) GetOAuthConfig(activeDirectoryEndpoint string) (*adal.OAuthConfig, error) {
	log.Printf("Getting OAuth config for endpoint %s with  tenant %s", activeDirectoryEndpoint, c.TenantID)

	// fix for ADFS environments, if the login endpoint ends in `/adfs` it's an adfs environment
	// the login endpoint ends up residing in `ActiveDirectoryEndpoint`
	oAuthTenant := c.TenantID
	if strings.HasSuffix(strings.ToLower(activeDirectoryEndpoint), "/adfs") {
		log.Printf("[DEBUG] ADFS environment detected - overriding Tenant ID to `adfs`!")
		oAuthTenant = "adfs"
	}

	oauth, err := adal.NewOAuthConfig(activeDirectoryEndpoint, oAuthTenant)
	if err != nil {
		return nil, err
	}

	// OAuthConfigForTenant returns a pointer, which can be nil.
	if oauth == nil {
		return nil, fmt.Errorf("unable to configure OAuthConfig for tenant %s", c.TenantID)
	}

	return oauth, nil
}

// GetMultiTenantOAuthConfig returns a multi-tenant authorization token for the authentication method defined in the Config
func (c Config) GetMultiTenantOAuthConfig(activeDirectoryEndpoint string) (*adal.MultiTenantOAuthConfig, error) {
	log.Printf("Getting multi OAuth config for endpoint %s with  tenant %s (aux tenants: %v)", activeDirectoryEndpoint, c.TenantID, c.AuxiliaryTenantIDs)
	oauth, err := adal.NewMultiTenantOAuthConfig(activeDirectoryEndpoint, c.TenantID, c.AuxiliaryTenantIDs, adal.OAuthOptions{})
	if err != nil {
		return nil, err
	}

	// OAuthConfigForTenant returns a pointer, which can be nil.
	if oauth == nil {
		return nil, fmt.Errorf("unable to configure OAuthConfig for tenant %s (auxiliary tenants %v)", c.TenantID, c.AuxiliaryTenantIDs)
	}

	return &oauth, nil
}

// BuildOAuthConfig builds the authorization configuration for the specified Active Directory Endpoint
func (c Config) BuildOAuthConfig(activeDirectoryEndpoint string) (*OAuthConfig, error) {
	multiAuth := OAuthConfig{}
	var err error

	multiAuth.OAuth, err = c.GetOAuthConfig(activeDirectoryEndpoint)
	if err != nil {
		return nil, err
	}

	if len(c.AuxiliaryTenantIDs) > 0 {
		multiAuth.MultiTenantOauth, err = c.GetMultiTenantOAuthConfig(activeDirectoryEndpoint)
		if err != nil {
			return nil, err
		}
	}

	return &multiAuth, nil
}

// ADALBearerAuthorizerCallback returns a BearerAuthorizer valid only for the Primary Tenant
// this signs a request using the AccessToken returned from the primary Resource Manager authorizer
func (c Config) ADALBearerAuthorizerCallback(ctx context.Context, sender autorest.Sender, oauthConfig *OAuthConfig) *autorest.BearerAuthorizerCallback {
	return autorest.NewBearerAuthorizerCallback(sender, func(tenantID, resource string) (*autorest.BearerAuthorizer, error) {
		// a BearerAuthorizer is only valid for the primary tenant
		newAuthConfig := &OAuthConfig{
			OAuth: oauthConfig.OAuth,
		}

		storageSpt, err := c.GetADALToken(ctx, sender, newAuthConfig, resource)
		if err != nil {
			return nil, err
		}

		cast, ok := storageSpt.(*autorest.BearerAuthorizer)
		if !ok {
			return nil, fmt.Errorf("converting %+v to a BearerAuthorizer", storageSpt)
		}

		return cast, nil
	})
}

// MSALBearerAuthorizerCallback returns a BearerAuthorizer valid only for the Primary Tenant
// this signs a request using the AccessToken returned from the primary Resource Manager authorizer
func (c Config) MSALBearerAuthorizerCallback(ctx context.Context, api environments.Api, sender autorest.Sender, oauthConfig *OAuthConfig, endpoint string) *autorest.BearerAuthorizerCallback {
	authorizer, err := c.GetMSALToken(ctx, api, sender, oauthConfig, endpoint)
	if err != nil {
		return autorest.NewBearerAuthorizerCallback(nil, func(_, _ string) (*autorest.BearerAuthorizer, error) {
			return nil, fmt.Errorf("failed to acquire MSAL token for %s", api.Endpoint)
		})
	}

	cast, ok := authorizer.(*auth.CachedAuthorizer)
	if !ok {
		return autorest.NewBearerAuthorizerCallback(nil, func(_, _ string) (*autorest.BearerAuthorizer, error) {
			return nil, fmt.Errorf("authorizer was not an auth.CachedAuthorizer for %s", api.Endpoint)
		})
	}

	return cast.BearerAuthorizerCallback()
}

// GetADALToken returns an autorest.Authorizer using an ADAL token via the authentication method defined in the Config
func (c Config) GetADALToken(ctx context.Context, sender autorest.Sender, oauth *OAuthConfig, endpoint string) (autorest.Authorizer, error) {
	return c.authMethod.getADALToken(ctx, sender, oauth, endpoint)
}

// GetMSALToken returns an autorest.Authorizer using an MSAL token via the authentication method defined in the Config
func (c Config) GetMSALToken(ctx context.Context, api environments.Api, sender autorest.Sender, oauth *OAuthConfig, endpoint string) (autorest.Authorizer, error) {
	return c.authMethod.getMSALToken(ctx, api, sender, oauth, endpoint)
}
<|endoftext|>