<commit_before>// Package cmount implements a FUSE mounting system for rclone remotes.
//
// This uses the cgo based cgofuse library

// +build cmount
// +build cgo
// +build linux darwin freebsd windows

package cmount

import (
	"fmt"
	"os"
	"runtime"
	"time"

	"github.com/billziss-gh/cgofuse/fuse"
	"github.com/pkg/errors"
	"github.com/rclone/rclone/cmd/mountlib"
	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/vfs"
)

func init() {
	name := "cmount"
	if runtime.GOOS == "windows" {
		name = "mount"
	}
	mountlib.NewMountCommand(name, false, mount)
	mountlib.AddRc("cmount", mount)
}

// mountOptions configures the options from the command line flags
func mountOptions(VFS *vfs.VFS, device string, mountpoint string, opt *mountlib.Options) (options []string) {
	// Options
	options = []string{
		"-o", "fsname=" + device,
		"-o", "subtype=rclone",
		"-o", fmt.Sprintf("max_readahead=%d", opt.MaxReadAhead),
		"-o", fmt.Sprintf("attr_timeout=%g", opt.AttrTimeout.Seconds()),
		// This causes FUSE to supply O_TRUNC with the Open
		// call which is more efficient for cmount.  However
		// it does not work with cgofuse on Windows with
		// WinFSP so cmount must work with or without it.
		"-o", "atomic_o_trunc",
	}
	if opt.DebugFUSE {
		options = append(options, "-o", "debug")
	}

	// OSX options
	if runtime.GOOS == "darwin" {
		if opt.NoAppleDouble {
			options = append(options, "-o", "noappledouble")
		}
		if opt.NoAppleXattr {
			options = append(options, "-o", "noapplexattr")
		}
	}

	// Windows options
	if runtime.GOOS == "windows" {
		// These cause WinFsp to mean the current user
		options = append(options, "-o", "uid=-1")
		options = append(options, "-o", "gid=-1")
		options = append(options, "--FileSystemName=rclone")
	}

	if runtime.GOOS == "darwin" || runtime.GOOS == "windows" {
		if opt.VolumeName != "" {
			options = append(options, "-o", "volname="+opt.VolumeName)
		}
	}
	if opt.AllowNonEmpty {
		options = append(options, "-o", "nonempty")
	}
	if opt.AllowOther {
		options = append(options, "-o", "allow_other")
	}
	if opt.AllowRoot {
		options = append(options, "-o", "allow_root")
	}
	if opt.DefaultPermissions {
		options = append(options, "-o", "default_permissions")
	}
	if VFS.Opt.ReadOnly {
		options = append(options, "-o", "ro")
	}
	if opt.WritebackCache {
		// FIXME? options = append(options, "-o", WritebackCache())
	}
	if opt.DaemonTimeout != 0 {
		options = append(options, "-o", fmt.Sprintf("daemon_timeout=%d", int(opt.DaemonTimeout.Seconds())))
	}
	for _, option := range opt.ExtraOptions {
		options = append(options, "-o", option)
	}
	for _, option := range opt.ExtraFlags {
		options = append(options, option)
	}
	return options
}

// waitFor runs fn() until it returns true or the timeout expires
func waitFor(fn func() bool) (ok bool) {
	const totalWait = 10 * time.Second
	const individualWait = 10 * time.Millisecond
	for i := 0; i < int(totalWait/individualWait); i++ {
		ok = fn()
		if ok {
			return ok
		}
		time.Sleep(individualWait)
	}
	return false
}

// mount the file system
//
// The mount point will be ready when this returns.
//
// returns an error, and an error channel for the serve process to
// report an error when fusermount is called.
func mount(VFS *vfs.VFS, mountpoint string, opt *mountlib.Options) (<-chan error, func() error, error) {
	f := VFS.Fs()
	fs.Debugf(f, "Mounting on %q", mountpoint)

	// Check the mountpoint - in Windows the mountpoint mustn't exist before the mount
	if runtime.GOOS != "windows" {
		fi, err := os.Stat(mountpoint)
		if err != nil {
			return nil, nil, errors.Wrap(err, "mountpoint")
		}
		if !fi.IsDir() {
			return nil, nil, errors.New("mountpoint is not a directory")
		}
	}

	// Create underlying FS
	fsys := NewFS(VFS)
	host := fuse.NewFileSystemHost(fsys)
	host.SetCapReaddirPlus(true) // only works on Windows
	host.SetCapCaseInsensitive(f.Features().CaseInsensitive)

	// Create options
	options := mountOptions(VFS, f.Name()+":"+f.Root(), mountpoint, opt)
	fs.Debugf(f, "Mounting with options: %q", options)

	// Serve the mount point in the background returning error to errChan
	errChan := make(chan error, 1)
	go func() {
		defer func() {
			if r := recover(); r != nil {
				errChan <- errors.Errorf("mount failed: %v", r)
			}
		}()
		var err error
		ok := host.Mount(mountpoint, options)
		if !ok {
			err = errors.New("mount failed")
			fs.Errorf(f, "Mount failed")
		}
		errChan <- err
	}()

	// unmount
	unmount := func() error {
		// Shutdown the VFS
		fsys.VFS.Shutdown()
		fs.Debugf(nil, "Calling host.Unmount")
		if host.Unmount() {
			fs.Debugf(nil, "host.Unmount succeeded")
			if runtime.GOOS == "windows" {
				if !waitFor(func() bool {
					_, err := os.Stat(mountpoint)
					return err != nil
				}) {
					fs.Errorf(nil, "mountpoint %q didn't disappear after unmount - continuing anyway", mountpoint)
				}
			}
			return nil
		}
		fs.Debugf(nil, "host.Unmount failed")
		return errors.New("host unmount failed")
	}

	// Wait for the filesystem to become ready, checking the file
	// system didn't blow up before starting
	select {
	case err := <-errChan:
		err = errors.Wrap(err, "mount stopped before calling Init")
		return nil, nil, err
	case <-fsys.ready:
	}

	// Wait for the mount point to be available on Windows
	// On Windows the Init signal comes slightly before the mount is ready
	if runtime.GOOS == "windows" {
		if !waitFor(func() bool {
			_, err := os.Stat(mountpoint)
			return err == nil
		}) {
			fs.Errorf(nil, "mountpoint %q didn't became available on mount - continuing anyway", mountpoint)
		}
	}

	return errChan, unmount, nil
}
<commit_msg>mount: On Windows don't add -o uid/gid=-1 if user supplies -o uid/gid.<commit_after>// Package cmount implements a FUSE mounting system for rclone remotes.
//
// This uses the cgo based cgofuse library

// +build cmount
// +build cgo
// +build linux darwin freebsd windows

package cmount

import (
	"fmt"
	"os"
	"runtime"
	"strings"
	"time"

	"github.com/billziss-gh/cgofuse/fuse"
	"github.com/pkg/errors"
	"github.com/rclone/rclone/cmd/mountlib"
	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/vfs"
)

func init() {
	name := "cmount"
	if runtime.GOOS == "windows" {
		name = "mount"
	}
	mountlib.NewMountCommand(name, false, mount)
	mountlib.AddRc("cmount", mount)
}

// mountOptions configures the options from the command line flags
func mountOptions(VFS *vfs.VFS, device string, mountpoint string, opt *mountlib.Options) (options []string) {
	// Options
	options = []string{
		"-o", "fsname=" + device,
		"-o", "subtype=rclone",
		"-o", fmt.Sprintf("max_readahead=%d", opt.MaxReadAhead),
		"-o", fmt.Sprintf("attr_timeout=%g", opt.AttrTimeout.Seconds()),
		// This causes FUSE to supply O_TRUNC with the Open
		// call which is more efficient for cmount.  However
		// it does not work with cgofuse on Windows with
		// WinFSP so cmount must work with or without it.
		"-o", "atomic_o_trunc",
	}
	if opt.DebugFUSE {
		options = append(options, "-o", "debug")
	}

	// OSX options
	if runtime.GOOS == "darwin" {
		if opt.NoAppleDouble {
			options = append(options, "-o", "noappledouble")
		}
		if opt.NoAppleXattr {
			options = append(options, "-o", "noapplexattr")
		}
	}

	// determine if ExtraOptions already has an opt in
	hasOption := func(optionName string) bool {
		optionName += "="
		for _, option := range opt.ExtraOptions {
			if strings.HasPrefix(option, optionName) {
				return true
			}
		}
		return false
	}

	// Windows options
	if runtime.GOOS == "windows" {
		// These cause WinFsp to mean the current user
		if !hasOption("uid") {
			options = append(options, "-o", "uid=-1")
		}
		if !hasOption("gid") {
			options = append(options, "-o", "gid=-1")
		}
		options = append(options, "--FileSystemName=rclone")
	}

	if runtime.GOOS == "darwin" || runtime.GOOS == "windows" {
		if opt.VolumeName != "" {
			options = append(options, "-o", "volname="+opt.VolumeName)
		}
	}
	if opt.AllowNonEmpty {
		options = append(options, "-o", "nonempty")
	}
	if opt.AllowOther {
		options = append(options, "-o", "allow_other")
	}
	if opt.AllowRoot {
		options = append(options, "-o", "allow_root")
	}
	if opt.DefaultPermissions {
		options = append(options, "-o", "default_permissions")
	}
	if VFS.Opt.ReadOnly {
		options = append(options, "-o", "ro")
	}
	if opt.WritebackCache {
		// FIXME? options = append(options, "-o", WritebackCache())
	}
	if opt.DaemonTimeout != 0 {
		options = append(options, "-o", fmt.Sprintf("daemon_timeout=%d", int(opt.DaemonTimeout.Seconds())))
	}
	for _, option := range opt.ExtraOptions {
		options = append(options, "-o", option)
	}
	for _, option := range opt.ExtraFlags {
		options = append(options, option)
	}
	return options
}

// waitFor runs fn() until it returns true or the timeout expires
func waitFor(fn func() bool) (ok bool) {
	const totalWait = 10 * time.Second
	const individualWait = 10 * time.Millisecond
	for i := 0; i < int(totalWait/individualWait); i++ {
		ok = fn()
		if ok {
			return ok
		}
		time.Sleep(individualWait)
	}
	return false
}

// mount the file system
//
// The mount point will be ready when this returns.
//
// returns an error, and an error channel for the serve process to
// report an error when fusermount is called.
func mount(VFS *vfs.VFS, mountpoint string, opt *mountlib.Options) (<-chan error, func() error, error) {
	f := VFS.Fs()
	fs.Debugf(f, "Mounting on %q", mountpoint)

	// Check the mountpoint - in Windows the mountpoint mustn't exist before the mount
	if runtime.GOOS != "windows" {
		fi, err := os.Stat(mountpoint)
		if err != nil {
			return nil, nil, errors.Wrap(err, "mountpoint")
		}
		if !fi.IsDir() {
			return nil, nil, errors.New("mountpoint is not a directory")
		}
	}

	// Create underlying FS
	fsys := NewFS(VFS)
	host := fuse.NewFileSystemHost(fsys)
	host.SetCapReaddirPlus(true) // only works on Windows
	host.SetCapCaseInsensitive(f.Features().CaseInsensitive)

	// Create options
	options := mountOptions(VFS, f.Name()+":"+f.Root(), mountpoint, opt)
	fs.Debugf(f, "Mounting with options: %q", options)

	// Serve the mount point in the background returning error to errChan
	errChan := make(chan error, 1)
	go func() {
		defer func() {
			if r := recover(); r != nil {
				errChan <- errors.Errorf("mount failed: %v", r)
			}
		}()
		var err error
		ok := host.Mount(mountpoint, options)
		if !ok {
			err = errors.New("mount failed")
			fs.Errorf(f, "Mount failed")
		}
		errChan <- err
	}()

	// unmount
	unmount := func() error {
		// Shutdown the VFS
		fsys.VFS.Shutdown()
		fs.Debugf(nil, "Calling host.Unmount")
		if host.Unmount() {
			fs.Debugf(nil, "host.Unmount succeeded")
			if runtime.GOOS == "windows" {
				if !waitFor(func() bool {
					_, err := os.Stat(mountpoint)
					return err != nil
				}) {
					fs.Errorf(nil, "mountpoint %q didn't disappear after unmount - continuing anyway", mountpoint)
				}
			}
			return nil
		}
		fs.Debugf(nil, "host.Unmount failed")
		return errors.New("host unmount failed")
	}

	// Wait for the filesystem to become ready, checking the file
	// system didn't blow up before starting
	select {
	case err := <-errChan:
		err = errors.Wrap(err, "mount stopped before calling Init")
		return nil, nil, err
	case <-fsys.ready:
	}

	// Wait for the mount point to be available on Windows
	// On Windows the Init signal comes slightly before the mount is ready
	if runtime.GOOS == "windows" {
		if !waitFor(func() bool {
			_, err := os.Stat(mountpoint)
			return err == nil
		}) {
			fs.Errorf(nil, "mountpoint %q didn't became available on mount - continuing anyway", mountpoint)
		}
	}

	return errChan, unmount, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/koron/gomigemo/migemo"
	"log"
)

func dictdir() string {
	return "./_dict"
}

func query(d migemo.Dict, s string) (string, error) {
	return migemo.Pattern(d, s)
}

func main() {
	dict, err := migemo.Load(dictdir())
	if err != nil {
		log.Fatal(err)
	}
	for q := range cl {
		p, err := query(dict, q)
		if err != nil {
			log.Error(err)
			continue
		}
		fmt.Println("PATTERN: %s\n", p)
	}
}
<commit_msg>impl query loop<commit_after>package main

import (
	"bufio"
	"fmt"
	"github.com/koron/gomigemo/migemo"
	"log"
	"os"
	"strings"
)

func dictdir() string {
	return "./_dict"
}

func setupOption(m migemo.Matcher) {
	o := m.GetOptions()
	o.OpWSpaces = ""
	m.SetOptions(o)
}

func query(d migemo.Dict, s string) (string, error) {
	m, err := d.Matcher(s)
	if err != nil {
		return "", err
	}
	setupOption(m)
	p, err := m.Pattern()
	if err != nil {
		return "", err
	}
	return p, nil
}

func readInput(ch chan<- string) {
	r := bufio.NewReader(os.Stdin)
	for {
		fmt.Print("QUERY: ")
		l, err := r.ReadString('\n')
		if l != "" {
			ch <- strings.TrimSpace(l)
		}
		if err != nil {
			return
		}
	}
}

func openQueries() <-chan string {
	ch := make(chan string, 1)
	go func() {
		readInput(ch)
		close(ch)
	}()
	return ch
}

func main() {
	dict, err := migemo.Load(dictdir())
	if err != nil {
		log.Fatal(err)
	}
	ch := openQueries()
	for q := range ch {
		p, err := query(dict, q)
		if err != nil {
			log.Print(err)
			continue
		}
		fmt.Printf("PATTERN: %s\n", p)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"path/filepath"
	"time"

	"gnd.la/log"

	"gopkgs.com/command.v1"
)

var (
	commands = []*command.Cmd{
		{
			Name: "dev",
			Help: "Start the Gondola development server",
			Func: devCommand,
			Options: &devOptions{
				Dir:  ".",
				Port: 8888,
			},
		},
		{
			Name:    "new",
			Help:    "Create a new Gondola project",
			Usage:   "<dir>",
			Func:    newCommand,
			Options: &newOptions{Template: "hello"},
		},
		{
			Name:     "build",
			Help:     "Build packages",
			Usage:    "[package-1] [package-2] ... [package-n]",
			LongHelp: buildHelp,
			Func:     buildCommand,
			Options:  &buildOptions{Go: "go"},
		},
		{
			Name: "clean",
			Help: "Cleans any Gondola packages which use conditional compilation - DO THIS BEFORE BUILDING A BINARY FOR DEPLOYMENT - see golang.org/issue/3172",
			Func: cleanCommand,
		},
		{
			Name:    "profile",
			Help:    "Show profiling information for a remote server running a Gondola app",
			Usage:   "<url>",
			Func:    profileCommand,
			Options: &profileOptions{Method: "GET"},
		},
		{
			Name:    "gen-app",
			Help:    "Generate boilerplate code for a Gondola app from the appfile.yaml file",
			Func:    genAppCommand,
			Options: &genAppOptions{},
		},
		{
			Name:    "bake",
			Help:    "Converts all assets in <dir> into Go code and generates a VFS named with <name>",
			Usage:   "-dir=<dir> -name=<name> ... additional flags",
			Options: &bakeOptions{},
			Func:    bakeCommand,
		},
		{
			Name:    "random-string",
			Help:    "Generates a random string suitable for use as the app secret",
			Func:    randomStringCommand,
			Options: &randomStringOptions{Length: defaultRandomLength},
		},
		{
			Name:  "rm-gen",
			Help:  "Remove Gondola generated files (identified by *.gen.*)",
			Usage: "[dir]",
			Func:  rmGenCommand,
		},
		{
			Name:    "make-messages",
			Help:    "Generate strings files from the current package (including its non-package subdirectories, like templates)",
			Func:    makeMessagesCommand,
			Options: &makeMessagesOptions{Out: filepath.Join("_messages", "messages.pot")},
		},
		{
			Name:    "compile-messages",
			Help:    "Compiles all po files from the current directory and its subdirectories",
			Func:    compileMessagesCommand,
			Options: &compileMessagesOptions{Out: "messages.go"},
		},
		{
			Name:    "gen",
			Help:    "Perform code generation in the current directory according the rules in the config file",
			Func:    genCommand,
			Options: &genOptions{Genfile: "genfile.yaml"},
		},
		{
			Name:    "gae-dev",
			Help:    "Start the Gondola App Engine development server",
			Func:    gaeDevCommand,
			Options: &gaeDevOptions{Host: "localhost", Port: 8080, AdminPort: 8000},
		},
		{
			Name:    "gae-test",
			Help:    "Start serving your app on localhost and run gnd.la/app/tester tests against it",
			Func:    gaeTestCommand,
			Options: &gaeTestOptions{},
		},
		{
			Name:    "gae-deploy",
			Help:    "Deploy your application to App Engine",
			Func:    gaeDeployCommand,
			Options: &gaeDeployOptions{},
		},
	}
)

type commonOptions struct {
	Quiet bool `name:"q" help:"Disable verbose output"`
}

func main() {
	opts := &command.Options{
		Options: &commonOptions{},
		Func: func(opts *commonOptions) {
			if opts.Quiet {
				log.SetLevel(log.LError)
			} else {
				log.SetLevel(log.LDebug)
			}
		},
	}
	command.Exit(command.RunOpts(nil, opts, commands))
}

func init() {
	math.Seed(time.Now().Unix())
}
<commit_msg>Fix build error<commit_after>package main

import (
	"math/rand"
	"path/filepath"
	"time"

	"gnd.la/log"

	"gopkgs.com/command.v1"
)

var (
	commands = []*command.Cmd{
		{
			Name: "dev",
			Help: "Start the Gondola development server",
			Func: devCommand,
			Options: &devOptions{
				Dir:  ".",
				Port: 8888,
			},
		},
		{
			Name:    "new",
			Help:    "Create a new Gondola project",
			Usage:   "<dir>",
			Func:    newCommand,
			Options: &newOptions{Template: "hello"},
		},
		{
			Name:     "build",
			Help:     "Build packages",
			Usage:    "[package-1] [package-2] ... [package-n]",
			LongHelp: buildHelp,
			Func:     buildCommand,
			Options:  &buildOptions{Go: "go"},
		},
		{
			Name: "clean",
			Help: "Cleans any Gondola packages which use conditional compilation - DO THIS BEFORE BUILDING A BINARY FOR DEPLOYMENT - see golang.org/issue/3172",
			Func: cleanCommand,
		},
		{
			Name:    "profile",
			Help:    "Show profiling information for a remote server running a Gondola app",
			Usage:   "<url>",
			Func:    profileCommand,
			Options: &profileOptions{Method: "GET"},
		},
		{
			Name:    "gen-app",
			Help:    "Generate boilerplate code for a Gondola app from the appfile.yaml file",
			Func:    genAppCommand,
			Options: &genAppOptions{},
		},
		{
			Name:    "bake",
			Help:    "Converts all assets in <dir> into Go code and generates a VFS named with <name>",
			Usage:   "-dir=<dir> -name=<name> ... additional flags",
			Options: &bakeOptions{},
			Func:    bakeCommand,
		},
		{
			Name:    "random-string",
			Help:    "Generates a random string suitable for use as the app secret",
			Func:    randomStringCommand,
			Options: &randomStringOptions{Length: defaultRandomLength},
		},
		{
			Name:  "rm-gen",
			Help:  "Remove Gondola generated files (identified by *.gen.*)",
			Usage: "[dir]",
			Func:  rmGenCommand,
		},
		{
			Name:    "make-messages",
			Help:    "Generate strings files from the current package (including its non-package subdirectories, like templates)",
			Func:    makeMessagesCommand,
			Options: &makeMessagesOptions{Out: filepath.Join("_messages", "messages.pot")},
		},
		{
			Name:    "compile-messages",
			Help:    "Compiles all po files from the current directory and its subdirectories",
			Func:    compileMessagesCommand,
			Options: &compileMessagesOptions{Out: "messages.go"},
		},
		{
			Name:    "gen",
			Help:    "Perform code generation in the current directory according the rules in the config file",
			Func:    genCommand,
			Options: &genOptions{Genfile: "genfile.yaml"},
		},
		{
			Name:    "gae-dev",
			Help:    "Start the Gondola App Engine development server",
			Func:    gaeDevCommand,
			Options: &gaeDevOptions{Host: "localhost", Port: 8080, AdminPort: 8000},
		},
		{
			Name:    "gae-test",
			Help:    "Start serving your app on localhost and run gnd.la/app/tester tests against it",
			Func:    gaeTestCommand,
			Options: &gaeTestOptions{},
		},
		{
			Name:    "gae-deploy",
			Help:    "Deploy your application to App Engine",
			Func:    gaeDeployCommand,
			Options: &gaeDeployOptions{},
		},
	}
)

type commonOptions struct {
	Quiet bool `name:"q" help:"Disable verbose output"`
}

func main() {
	opts := &command.Options{
		Options: &commonOptions{},
		Func: func(opts *commonOptions) {
			if opts.Quiet {
				log.SetLevel(log.LError)
			} else {
				log.SetLevel(log.LDebug)
			}
		},
	}
	command.Exit(command.RunOpts(nil, opts, commands))
}

func init() {
	rand.Seed(time.Now().Unix())
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 Square, Inc
// +build linux darwin

package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"runtime"
	"runtime/debug"
	"sync"
	"time"

	"github.com/gizak/termui"

	"github.com/square/inspect/cmd/inspect/osmain"
	"github.com/square/inspect/metrics"
)

func main() {
	// never use more than one process
	runtime.GOMAXPROCS(1)
	// wait group to let the main program run forever in batchmode
	var wg sync.WaitGroup
	// options
	var batchmode, servermode bool
	var address string
	var stepSec int
	var nIter int
	var evt <-chan termui.Event
	var widgets *osmain.DisplayWidgets
	var uiSummaryBody *termui.Grid
	var uiHelpBody *termui.Grid
	var uiDetailList *termui.List

	flag.BoolVar(&batchmode, "b", false, "Run in batch mode; suitable for parsing")
	flag.BoolVar(&batchmode, "batchmode", false, "Run in batch mode; suitable for parsing")
	flag.IntVar(&nIter, "n", 0, "Quit after these many iterations")
	flag.IntVar(&nIter, "iterations", 0, "Quit after these many iterations")
	flag.BoolVar(&servermode, "server", false,
		"Runs continously and exposes metrics as JSON on HTTP")
	flag.StringVar(&address, "address", ":19999",
		"address to listen on for http if running in server mode")
	flag.IntVar(&stepSec, "step", 2,
		"metrics are collected every step seconds")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Options \n")
		fmt.Fprintf(os.Stderr, "------- \n")
		flag.PrintDefaults()
		fmt.Fprintf(os.Stderr, "Notes \n")
		fmt.Fprintf(os.Stderr, "------- \n")
		fmt.Fprintf(os.Stderr, "All CPU percentages are normalized to total number of logical cpus \n")
	}
	flag.Parse()

	if servermode {
		batchmode = true
	}

	if !batchmode {
		err := termui.Init()
		termui.UseTheme("helloworld")
		if err != nil {
			log.Fatalf("Unable to initialize termui", err)
		}
		widgets = uiWidgets()
		uiSummaryBody = uiSummary(widgets)
		uiHelpBody = uiHelp()
		uiDetailList = termui.NewList()
		evt = termui.EventCh()
		// display summary view
		termui.Body = uiSummaryBody
		uiRefresh()
	}
	// Initialize a metric context
	m := metrics.NewMetricContext("system")
	// Default step for collectors
	step := time.Millisecond * time.Duration(stepSec) * 1000
	// Register various stats we are interested in tracking
	stats := osmain.Register(m, step)
	// run http server
	if servermode {
		go func() {
			http.HandleFunc("/api/v1/metrics.json", m.HttpJsonHandler)
			log.Fatal(http.ListenAndServe(address, nil))
		}()
	}

	iterationsRun := 0
	// runs forever
	wg.Add(1)
	go func() {
		defer wg.Done()
		for {
			// Clear previous problems
			var problems []string
			stats.Problems = problems
			// Quit after n iterations if specified
			iterationsRun++
			if nIter > 0 && iterationsRun > nIter {
				break
			}
			stats.Print(batchmode, widgets)
			if !batchmode {
				termui.Render(termui.Body)
			}
			time.Sleep(step)
			// be aggressive about reclaiming memory
			// tradeoff with CPU usage
			runtime.GC()
			debug.FreeOSMemory()
		}
	}()

	if !batchmode {
		for {
			e := <-evt
			if e.Type == termui.EventKey {
				switch e.Ch {
				case 'q':
					termui.Close()
					return
				case 'c':
					uiDetailList = widgets.ProcessesByCPU
					termui.Body = uiDetail(uiDetailList)
				case 'd':
					uiDetailList = widgets.DiskIOUsage
					termui.Body = uiDetail(uiDetailList)
				case 'C':
					uiDetailList = widgets.CgroupsCPU
					termui.Body = uiDetail(uiDetailList)
				case 'f':
					uiDetailList = widgets.FileSystemUsage
					termui.Body = uiDetail(uiDetailList)
				case 'm':
					uiDetailList = widgets.ProcessesByMemory
					termui.Body = uiDetail(uiDetailList)
				case 'p':
					uiDetailList = widgets.Problems
					termui.Body = uiDetail(uiDetailList)
				case 'M':
					uiDetailList = widgets.CgroupsMem
					termui.Body = uiDetail(uiDetailList)
				case 'n':
					uiDetailList = widgets.InterfaceUsage
					termui.Body = uiDetail(uiDetailList)
				case 'i':
					uiDetailList = widgets.ProcessesByIO
					termui.Body = uiDetail(uiDetailList)
				case 's':
					uiResetAttributes(widgets)
					termui.Body = uiSummaryBody
				case 'h':
					termui.Body = uiHelpBody
				}
				uiRefresh()
			}
			if e.Type == termui.EventResize {
				uiRefresh()
			}
		}
	}
	wg.Wait()
}
<commit_msg>Add /health endpoint for checking health<commit_after>// Copyright (c) 2014 Square, Inc
// +build linux darwin

package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"runtime"
	"runtime/debug"
	"sync"
	"time"

	"github.com/gizak/termui"

	"github.com/square/inspect/cmd/inspect/osmain"
	"github.com/square/inspect/metrics"
)

func main() {
	// never use more than one process
	runtime.GOMAXPROCS(1)
	// wait group to let the main program run forever in batchmode
	var wg sync.WaitGroup
	// options
	var batchmode, servermode bool
	var address string
	var stepSec int
	var nIter int
	var evt <-chan termui.Event
	var widgets *osmain.DisplayWidgets
	var uiSummaryBody *termui.Grid
	var uiHelpBody *termui.Grid
	var uiDetailList *termui.List

	flag.BoolVar(&batchmode, "b", false, "Run in batch mode; suitable for parsing")
	flag.BoolVar(&batchmode, "batchmode", false, "Run in batch mode; suitable for parsing")
	flag.IntVar(&nIter, "n", 0, "Quit after these many iterations")
	flag.IntVar(&nIter, "iterations", 0, "Quit after these many iterations")
	flag.BoolVar(&servermode, "server", false,
		"Runs continously and exposes metrics as JSON on HTTP")
	flag.StringVar(&address, "address", ":19999",
		"address to listen on for http if running in server mode")
	flag.IntVar(&stepSec, "step", 2,
		"metrics are collected every step seconds")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Options \n")
		fmt.Fprintf(os.Stderr, "------- \n")
		flag.PrintDefaults()
		fmt.Fprintf(os.Stderr, "Notes \n")
		fmt.Fprintf(os.Stderr, "------- \n")
		fmt.Fprintf(os.Stderr, "All CPU percentages are normalized to total number of logical cpus \n")
	}
	flag.Parse()

	if servermode {
		batchmode = true
	}

	if !batchmode {
		err := termui.Init()
		termui.UseTheme("helloworld")
		if err != nil {
			log.Fatalf("Unable to initialize termui", err)
		}
		widgets = uiWidgets()
		uiSummaryBody = uiSummary(widgets)
		uiHelpBody = uiHelp()
		uiDetailList = termui.NewList()
		evt = termui.EventCh()
		// display summary view
		termui.Body = uiSummaryBody
		uiRefresh()
	}
	// Initialize a metric context
	m := metrics.NewMetricContext("system")
	// Default step for collectors
	step := time.Millisecond * time.Duration(stepSec) * 1000
	// Register various stats we are interested in tracking
	stats := osmain.Register(m, step)
	// run http server
	if servermode {
		go func() {
			http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {})
			http.HandleFunc("/api/v1/metrics.json", m.HttpJsonHandler)
			log.Fatal(http.ListenAndServe(address, nil))
		}()
	}

	iterationsRun := 0
	// runs forever
	wg.Add(1)
	go func() {
		defer wg.Done()
		for {
			// Clear previous problems
			var problems []string
			stats.Problems = problems
			// Quit after n iterations if specified
			iterationsRun++
			if nIter > 0 && iterationsRun > nIter {
				break
			}
			stats.Print(batchmode, widgets)
			if !batchmode {
				termui.Render(termui.Body)
			}
			time.Sleep(step)
			// be aggressive about reclaiming memory
			// tradeoff with CPU usage
			runtime.GC()
			debug.FreeOSMemory()
		}
	}()

	if !batchmode {
		for {
			e := <-evt
			if e.Type == termui.EventKey {
				switch e.Ch {
				case 'q':
					termui.Close()
					return
				case 'c':
					uiDetailList = widgets.ProcessesByCPU
					termui.Body = uiDetail(uiDetailList)
				case 'd':
					uiDetailList = widgets.DiskIOUsage
					termui.Body = uiDetail(uiDetailList)
				case 'C':
					uiDetailList = widgets.CgroupsCPU
					termui.Body = uiDetail(uiDetailList)
				case 'f':
					uiDetailList = widgets.FileSystemUsage
					termui.Body = uiDetail(uiDetailList)
				case 'm':
					uiDetailList = widgets.ProcessesByMemory
					termui.Body = uiDetail(uiDetailList)
				case 'p':
					uiDetailList = widgets.Problems
					termui.Body = uiDetail(uiDetailList)
				case 'M':
					uiDetailList = widgets.CgroupsMem
					termui.Body = uiDetail(uiDetailList)
				case 'n':
					uiDetailList = widgets.InterfaceUsage
					termui.Body = uiDetail(uiDetailList)
				case 'i':
					uiDetailList = widgets.ProcessesByIO
					termui.Body = uiDetail(uiDetailList)
				case 's':
					uiResetAttributes(widgets)
					termui.Body = uiSummaryBody
				case 'h':
					termui.Body = uiHelpBody
				}
				uiRefresh()
			}
			if e.Type == termui.EventResize {
				uiRefresh()
			}
		}
	}
	wg.Wait()
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"

	"golang.org/x/net/context"

	"github.com/gophergala2016/magopie"
	"github.com/rs/xhandler"
	"github.com/rs/xmux"
	"github.com/spf13/afero"
)

func init() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
}

func main() {
	a := &app{
		fs: afero.OsFs{},
		sites: sitedb{
			sites: []site{
				kickAssTorrents,
				thePirateBay,
			},
		},
		torcacheURL: "http://torcache.net",
		downloadDir: ".",
	}

	var port string
	if port = os.Getenv("PORT"); port == "" {
		port = "8080"
	}

	addr := fmt.Sprintf("%s:%s", "localhost", port)
	log.Printf("Listening on %s", addr)
	log.Fatal(http.ListenAndServe(addr, router(a)))
}

func router(a *app) http.Handler {
	mux := xmux.New()

	// TODO switch to a muxer where I define the method as part of the route
	mux.GET("/sites", xhandler.HandlerFuncC(a.handleAllSites))
	mux.GET("/sites/:id", xhandler.HandlerFuncC(a.handleSingleSite))
	mux.GET("/torrents", xhandler.HandlerFuncC(a.handleTorrents))
	mux.POST("/download/:hash", xhandler.HandlerFuncC(a.handleDownload))

	return xhandler.New(context.Background(), mux)
}

type app struct {
	// sites is the collection of upstream sites registered for this app
	sites sitedb

	// torcacheURL is the base URL for the service that serves torrent files
	torcacheURL string

	// fs is the filesystem where we'll store .torrent files
	fs afero.Fs

	// downloadDir is the path to the the dir in fs where we'll store files
	downloadDir string
}

func (a *app) handleAllSites(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	err := json.NewEncoder(w).Encode(a.sites.GetAllSites())
	if err != nil {
		log.Println(err)
	}
}

func (a *app) handleSingleSite(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	id := xmux.Param(ctx, "id")
	err := json.NewEncoder(w).Encode(a.sites.GetSite(id))
	if err != nil {
		log.Println(err)
	}
}

func (a *app) handleTorrents(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	// TODO better error response?
	term := r.FormValue("q")
	if term == "" {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	// TODO do this concurrently
	torrents := []magopie.Torrent{}
	for _, s := range a.sites.GetEnabledSites() {
		results, err := s.search(term)
		if err != nil {
			log.Printf("Error searching %q on %v: %v", term, s.ID, err)
			continue
		}
		for _, t := range results {
			torrents = append(torrents, t)
		}
	}

	if err := json.NewEncoder(w).Encode(torrents); err != nil {
		log.Println(err)
	}
}

func (a *app) handleDownload(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	hash := strings.ToUpper(xmux.Param(ctx, "hash"))
	url := fmt.Sprintf(
		"%s/torrent/%s.torrent",
		a.torcacheURL,
		url.QueryEscape(hash),
	)

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		log.Print(err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	req.Header.Set("User-Agent", "Magopie Server")

	res, err := http.DefaultClient.Do(req)
	if err != nil {
		log.Print(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	defer res.Body.Close()

	// Ensure download directory exists
	// TODO should this be done every request?
	err = a.fs.MkdirAll(a.downloadDir, 0755)
	if err != nil {
		log.Print("Error making download dir", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	// Create file to write torrent
	// TODO create with .part or something then move when done
	path := filepath.Join(a.downloadDir, hash+".torrent")
	file, err := a.fs.Create(path)
	if err != nil {
		log.Print(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	// Copy file contents from torcache response to os file
	_, err = io.Copy(file, res.Body)
	if err != nil {
		log.Print(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusCreated)
}
<commit_msg>Log requests<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"

	"golang.org/x/net/context"

	"github.com/gophergala2016/magopie"
	"github.com/rs/xhandler"
	"github.com/rs/xmux"
	"github.com/spf13/afero"
)

func init() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
}

func main() {
	a := &app{
		fs: afero.OsFs{},
		sites: sitedb{
			sites: []site{
				kickAssTorrents,
				thePirateBay,
			},
		},
		torcacheURL: "http://torcache.net",
		downloadDir: ".",
	}

	var port string
	if port = os.Getenv("PORT"); port == "" {
		port = "8080"
	}

	addr := fmt.Sprintf("%s:%s", "localhost", port)
	log.Printf("Listening on %s", addr)
	log.Fatal(http.ListenAndServe(addr, router(a)))
}

func router(a *app) http.Handler {
	mux := xmux.New()

	c := xhandler.Chain{}

	c.Use(mwLogger)

	mux.GET("/sites", c.HandlerCF(xhandler.HandlerFuncC(a.handleAllSites)))
	mux.GET("/sites/:id", c.HandlerCF(xhandler.HandlerFuncC(a.handleSingleSite)))
	mux.GET("/torrents", c.HandlerCF(xhandler.HandlerFuncC(a.handleTorrents)))
	mux.POST("/download/:hash", c.HandlerCF(xhandler.HandlerFuncC(a.handleDownload)))

	return xhandler.New(context.Background(), mux)
}

func mwLogger(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Println("Serving", r.Method, r.URL.String(), "to", r.RemoteAddr)
		next.ServeHTTP(w, r)
	})
}

type app struct {
	// sites is the collection of upstream sites registered for this app
	sites sitedb

	// torcacheURL is the base URL for the service that serves torrent files
	torcacheURL string

	// fs is the filesystem where we'll store .torrent files
	fs afero.Fs

	// downloadDir is the path to the the dir in fs where we'll store files
	downloadDir string
}

func (a *app) handleAllSites(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	err := json.NewEncoder(w).Encode(a.sites.GetAllSites())
	if err != nil {
		log.Println(err)
	}
}

func (a *app) handleSingleSite(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	id := xmux.Param(ctx, "id")
	err := json.NewEncoder(w).Encode(a.sites.GetSite(id))
	if err != nil {
		log.Println(err)
	}
}

func (a *app) handleTorrents(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	// TODO better error response?
	term := r.FormValue("q")
	if term == "" {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	// TODO do this concurrently
	torrents := []magopie.Torrent{}
	for _, s := range a.sites.GetEnabledSites() {
		results, err := s.search(term)
		if err != nil {
			log.Printf("Error searching %q on %v: %v", term, s.ID, err)
			continue
		}
		for _, t := range results {
			torrents = append(torrents, t)
		}
	}

	if err := json.NewEncoder(w).Encode(torrents); err != nil {
		log.Println(err)
	}
}

func (a *app) handleDownload(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	hash := strings.ToUpper(xmux.Param(ctx, "hash"))
	url := fmt.Sprintf(
		"%s/torrent/%s.torrent",
		a.torcacheURL,
		url.QueryEscape(hash),
	)

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		log.Print(err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	req.Header.Set("User-Agent", "Magopie Server")

	res, err := http.DefaultClient.Do(req)
	if err != nil {
		log.Print(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	defer res.Body.Close()

	// Ensure download directory exists
	// TODO should this be done every request?
	err = a.fs.MkdirAll(a.downloadDir, 0755)
	if err != nil {
		log.Print("Error making download dir", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	// Create file to write torrent
	// TODO create with .part or something then move when done
	path := filepath.Join(a.downloadDir, hash+".torrent")
	file, err := a.fs.Create(path)
	if err != nil {
		log.Print(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	// Copy file contents from torcache response to os file
	_, err = io.Copy(file, res.Body)
	if err != nil {
		log.Print(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusCreated)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"

	"github.com/scipipe/scipipe"
)

func main() {
	flag.Parse()
	cmd := flag.Arg(0)
	if cmd == "new" {
		wfcode := `// Workflow written in SciPipe.
// For more information about SciPipe, see: http://scipipe.org
package main

import sp "github.com/scipipe/scipipe"

func main() {
	// Create a workflow, using 4 cpu cores
	wf := sp.NewWorkflow("my_workflow", 4)

	// Initialize processes
	foo := wf.NewProc("fooer", "echo foo > {o:foo}")
	foo.SetPathStatic("foo", "foo.txt")

	f2b := wf.NewProc("foo2bar", "sed 's/foo/bar/g' {i:foo} > {o:bar}")
	f2b.SetPathExtend("foo", "bar", ".bar.txt")

	// From workflow dependency network
	f2b.In("foo").From(foo.Out("foo"))

	// Run the workflow
	wf.Run()
}`
		fileName := flag.Arg(1)
		if fileName == "" {
			fmt.Println("ERROR: No filename specified!")
			printUsage()
			os.Exit(1)
		}

		f, err := os.Create(fileName)
		if err != nil {
			scipipe.Fail("Could not create file:", fileName)
		}
		defer f.Close()

		_, err = f.Write([]byte(wfcode))
		if err != nil {
			scipipe.Fail("Could not write to file:", fileName)
		}

		fmt.Println("Successfully wrote new workflow file to:", fileName, "\n\nNow you can run it with:\ngo run ", fileName)
	} else {
		printUsage()
	}
}

func printUsage() {
	fmt.Println("Usage of scipipe:\nscipipe new <filename.go>")
}
<commit_msg>Refactor scipipe command<commit_after>package main

import (
	"flag"
	"fmt"
	"os"

	"github.com/scipipe/scipipe"
)

func main() {
	flag.Parse()
	cmd := flag.Arg(0)
	switch cmd {
	case "new":
		writeNewWorkflowFile(flag.Arg(1))
	default:
		printUsage()
	}
}

func printUsage() {
	fmt.Println("Usage of scipipe:\nscipipe new <filename.go>")
}

func writeNewWorkflowFile(fileName string) {
	wfcode := `// Workflow written in SciPipe.
// For more information about SciPipe, see: http://scipipe.org
package main

import sp "github.com/scipipe/scipipe"

func main() {
	// Create a workflow, using 4 cpu cores
	wf := sp.NewWorkflow("my_workflow", 4)

	// Initialize processes
	foo := wf.NewProc("fooer", "echo foo > {o:foo}")
	foo.SetPathStatic("foo", "foo.txt")

	f2b := wf.NewProc("foo2bar", "sed 's/foo/bar/g' {i:foo} > {o:bar}")
	f2b.SetPathExtend("foo", "bar", ".bar.txt")

	// From workflow dependency network
	f2b.In("foo").From(foo.Out("foo"))

	// Run the workflow
	wf.Run()
}`
	if fileName == "" {
		fmt.Println("ERROR: No filename specified!")
		printUsage()
		os.Exit(1)
	}
	f, err := os.Create(fileName)
	if err != nil {
		scipipe.Fail("Could not create file:", fileName)
	}
	defer f.Close()
	_, err = f.Write([]byte(wfcode))
	if err != nil {
		scipipe.Fail("Could not write to file:", fileName)
	}
	fmt.Println("Successfully wrote new workflow file to:", fileName, "\n\nNow you can run it with:\ngo run ", fileName)
}
<|endoftext|>


<commit_before>// Downloads torrents from the command-line.
package main

import (
	"expvar"
	"fmt"
	stdLog "log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/alexflint/go-arg"
	"github.com/anacrolix/missinggo"
	"github.com/dustin/go-humanize"
	"golang.org/x/xerrors"

	"github.com/anacrolix/log"

	"github.com/anacrolix/envpprof"
	"github.com/anacrolix/tagflag"
	"golang.org/x/time/rate"

	"github.com/anacrolix/torrent"
	"github.com/anacrolix/torrent/iplist"
	"github.com/anacrolix/torrent/metainfo"
	"github.com/anacrolix/torrent/storage"
)

func torrentBar(t *torrent.Torrent, pieceStates bool) {
	go func() {
		if t.Info() == nil {
			fmt.Printf("getting info for %q\n", t.Name())
			<-t.GotInfo()
		}
		var lastLine string
		for {
			var completedPieces, partialPieces int
			psrs := t.PieceStateRuns()
			for _, r := range psrs {
				if r.Complete {
					completedPieces += r.Length
				}
				if r.Partial {
					partialPieces += r.Length
				}
			}
			line := fmt.Sprintf(
				"downloading %q: %s/%s, %d/%d pieces completed (%d partial)\n",
				t.Name(),
				humanize.Bytes(uint64(t.BytesCompleted())),
				humanize.Bytes(uint64(t.Length())),
				completedPieces,
				t.NumPieces(),
				partialPieces,
			)
			if line != lastLine {
				lastLine = line
				os.Stdout.WriteString(line)
			}
			if pieceStates {
				fmt.Println(psrs)
			}
			time.Sleep(time.Second)
		}
	}()
}

type stringAddr string

func (stringAddr) Network() string   { return "" }
func (me stringAddr) String() string { return string(me) }

func resolveTestPeers(addrs []string) (ret []torrent.PeerInfo) {
	for _, ta := range flags.TestPeer {
		ret = append(ret, torrent.PeerInfo{
			Addr: stringAddr(ta),
		})
	}
	return
}

func addTorrents(client *torrent.Client) error {
	testPeers := resolveTestPeers(flags.TestPeer)
	for _, arg := range flags.Torrent {
		t, err := func() (*torrent.Torrent, error) {
			if strings.HasPrefix(arg, "magnet:") {
				t, err := client.AddMagnet(arg)
				if err != nil {
					return nil, xerrors.Errorf("error adding magnet: %w", err)
				}
				return t, nil
			} else if strings.HasPrefix(arg, "http://") || strings.HasPrefix(arg, "https://") {
				response, err := http.Get(arg)
				if err != nil {
					return nil, xerrors.Errorf("Error downloading torrent file: %s", err)
				}

				metaInfo, err := metainfo.Load(response.Body)
				defer response.Body.Close()
				if err != nil {
					return nil, xerrors.Errorf("error loading torrent file %q: %s\n", arg, err)
				}
				t, err := client.AddTorrent(metaInfo)
				if err != nil {
					return nil, xerrors.Errorf("adding torrent: %w", err)
				}
				return t, nil
			} else if strings.HasPrefix(arg, "infohash:") {
				t, _ := client.AddTorrentInfoHash(metainfo.NewHashFromHex(strings.TrimPrefix(arg, "infohash:")))
				return t, nil
			} else {
				metaInfo, err := metainfo.LoadFromFile(arg)
				if err != nil {
					return nil, xerrors.Errorf("error loading torrent file %q: %s\n", arg, err)
				}
				t, err := client.AddTorrent(metaInfo)
				if err != nil {
					return nil, xerrors.Errorf("adding torrent: %w", err)
				}
				return t, nil
			}
		}()
		if err != nil {
			return xerrors.Errorf("adding torrent for %q: %w", arg, err)
		}
		if flags.Progress {
			torrentBar(t, flags.PieceStates)
		}
		t.AddPeers(testPeers)
		go func() {
			<-t.GotInfo()
			if len(flags.File) == 0 {
				t.DownloadAll()
			} else {
				for _, f := range t.Files() {
					for _, fileArg := range flags.File {
						if f.DisplayPath() == fileArg {
							f.Download()
						}
					}
				}
			}
		}()
	}
	return nil
}

var flags struct {
	Debug bool
	Stats *bool

	*DownloadCmd  `arg:"subcommand:download"`
	*ListFilesCmd `arg:"subcommand:list-files"`
}

//DownloadCmd: &DownloadCmd{
//	UploadRate:   -1,
//	DownloadRate: -1,
//	Progress:     true,
//	Dht:          true,
//
//	TcpPeers:   true,
//	UtpPeers:   true,
//	Webtorrent: true,
//
//	Ipv4: true,
//	Ipv6: true,
//	Pex:  true,
//},

type DownloadCmd struct {
	Mmap            bool          `help:"memory-map torrent data"`
	TestPeer        []string      `help:"addresses of some starting peers"`
	Seed            bool          `help:"seed after download is complete"`
	Addr            string        `help:"network listen addr"`
	UploadRate      tagflag.Bytes `help:"max piece bytes to send per second" default:"-1"`
	DownloadRate    tagflag.Bytes `help:"max bytes per second down from peers"`
	PackedBlocklist string
	PublicIP        net.IP
	Progress        bool
	PieceStates     bool
	Quiet           bool `help:"discard client logging"`
	Dht             bool

	TcpPeers        bool
	UtpPeers        bool
	Webtorrent      bool
	DisableWebseeds bool

	Ipv4 bool
	Ipv6 bool
	Pex  bool

	File    []string
	Torrent []string `arity:"+" help:"torrent file path or magnet uri" arg:"positional"`
}

type ListFilesCmd struct {
	TorrentPath string `arg:"positional"`
}

func stdoutAndStderrAreSameFile() bool {
	fi1, _ := os.Stdout.Stat()
	fi2, _ := os.Stderr.Stat()
	return os.SameFile(fi1, fi2)
}

func statsEnabled() bool {
	if flags.Stats == nil {
		return flags.Debug
	}
	return *flags.Stats
}

func exitSignalHandlers(notify *missinggo.SynchronizedEvent) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
	for {
		log.Printf("close signal received: %+v", <-c)
		notify.Set()
	}
}

func main() {
	if err := mainErr(); err != nil {
		log.Printf("error in main: %v", err)
		os.Exit(1)
	}
}

func mainErr() error {
	stdLog.SetFlags(stdLog.Flags() | stdLog.Lshortfile)
	p := arg.MustParse(&flags)
	switch {
	//case :
	//	return announceErr(flags.Args, parser)
	case flags.DownloadCmd != nil:
		return downloadErr()
	case flags.ListFilesCmd != nil:
		mi, err := metainfo.LoadFromFile(flags.ListFilesCmd.TorrentPath)
		if err != nil {
			return fmt.Errorf("loading from file %q: %v", flags.ListFilesCmd.TorrentPath, err)
		}
		info, err := mi.UnmarshalInfo()
		if err != nil {
			return fmt.Errorf("unmarshalling info from metainfo at %q: %v", flags.ListFilesCmd.TorrentPath, err)
		}
		for _, f := range info.UpvertedFiles() {
			fmt.Println(f.DisplayPath(&info))
		}
		return nil
	default:
		p.Fail(fmt.Sprintf("unexpected subcommand: %v", p.Subcommand()))
		panic("unreachable")
	}
}

func downloadErr() error {
	defer envpprof.Stop()
	clientConfig := torrent.NewDefaultClientConfig()
	clientConfig.DisableWebseeds = flags.DisableWebseeds
	clientConfig.DisableTCP = !flags.TcpPeers
	clientConfig.DisableUTP = !flags.UtpPeers
	clientConfig.DisableIPv4 = !flags.Ipv4
	clientConfig.DisableIPv6 = !flags.Ipv6
	clientConfig.DisableAcceptRateLimiting = true
	clientConfig.NoDHT = !flags.Dht
	clientConfig.Debug = flags.Debug
	clientConfig.Seed = flags.Seed
	clientConfig.PublicIp4 = flags.PublicIP
	clientConfig.PublicIp6 = flags.PublicIP
	clientConfig.DisablePEX = !flags.Pex
	clientConfig.DisableWebtorrent = !flags.Webtorrent
	if flags.PackedBlocklist != "" {
		blocklist, err := iplist.MMapPackedFile(flags.PackedBlocklist)
		if err != nil {
			return xerrors.Errorf("loading blocklist: %v", err)
		}
		defer blocklist.Close()
		clientConfig.IPBlocklist = blocklist
	}
	if flags.Mmap {
		clientConfig.DefaultStorage = storage.NewMMap("")
	}
	if flags.Addr != "" {
		clientConfig.SetListenAddr(flags.Addr)
	}
	if flags.UploadRate != -1 {
		clientConfig.UploadRateLimiter = rate.NewLimiter(rate.Limit(flags.UploadRate), 256<<10)
	}
	if flags.DownloadRate != -1 {
		clientConfig.DownloadRateLimiter = rate.NewLimiter(rate.Limit(flags.DownloadRate), 1<<20)
	}
	if flags.Quiet {
		clientConfig.Logger = log.Discard
	}

	var stop missinggo.SynchronizedEvent
	defer func() {
		stop.Set()
	}()

	client, err := torrent.NewClient(clientConfig)
	if err != nil {
		return xerrors.Errorf("creating client: %v", err)
	}
	defer client.Close()
	go exitSignalHandlers(&stop)
	go func() {
		<-stop.C()
		client.Close()
	}()

	// Write status on the root path on the default HTTP muxer. This will be bound to localhost
	// somewhere if GOPPROF is set, thanks to the envpprof import.
	http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		client.WriteStatus(w)
	})
	err = addTorrents(client)
	if err != nil {
		return fmt.Errorf("adding torrents: %w", err)
	}
	defer outputStats(client)
	if client.WaitAll() {
		log.Print("downloaded ALL the torrents")
	} else {
		return xerrors.New("y u no complete torrents?!")
	}
	if flags.Seed {
		outputStats(client)
		<-stop.C()
	}
	return nil
}

func outputStats(cl *torrent.Client) {
	if !statsEnabled() {
		return
	}
	expvar.Do(func(kv expvar.KeyValue) {
		fmt.Printf("%s: %s\n", kv.Key, kv.Value)
	})
	cl.WriteStatus(os.Stdout)
}
<commit_msg>cmd/torrent: Fix download command default flags<commit_after>// Downloads torrents from the command-line.
package main

import (
	"expvar"
	"fmt"
	stdLog "log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/alexflint/go-arg"
	"github.com/anacrolix/missinggo"
	"github.com/dustin/go-humanize"
	"golang.org/x/xerrors"

	"github.com/anacrolix/log"

	"github.com/anacrolix/envpprof"
	"github.com/anacrolix/tagflag"
	"golang.org/x/time/rate"

	"github.com/anacrolix/torrent"
	"github.com/anacrolix/torrent/iplist"
	"github.com/anacrolix/torrent/metainfo"
	"github.com/anacrolix/torrent/storage"
)

func torrentBar(t *torrent.Torrent, pieceStates bool) {
	go func() {
		if t.Info() == nil {
			fmt.Printf("getting info for %q\n", t.Name())
			<-t.GotInfo()
		}
		var lastLine string
		for {
			var completedPieces, partialPieces int
			psrs := t.PieceStateRuns()
			for _, r := range psrs {
				if r.Complete {
					completedPieces += r.Length
				}
				if r.Partial {
					partialPieces += r.Length
				}
			}
			line := fmt.Sprintf(
				"downloading %q: %s/%s, %d/%d pieces completed (%d partial)\n",
				t.Name(),
				humanize.Bytes(uint64(t.BytesCompleted())),
				humanize.Bytes(uint64(t.Length())),
				completedPieces,
				t.NumPieces(),
				partialPieces,
			)
			if line != lastLine {
				lastLine = line
				os.Stdout.WriteString(line)
			}
			if pieceStates {
				fmt.Println(psrs)
			}
			time.Sleep(time.Second)
		}
	}()
}

type stringAddr string

func (stringAddr) Network() string   { return "" }
func (me stringAddr) String() string { return string(me) }

func resolveTestPeers(addrs []string) (ret []torrent.PeerInfo) {
	for _, ta := range flags.TestPeer {
		ret = append(ret, torrent.PeerInfo{
			Addr: stringAddr(ta),
		})
	}
	return
}

func addTorrents(client *torrent.Client) error {
	testPeers := resolveTestPeers(flags.TestPeer)
	for _, arg := range flags.Torrent {
		t, err := func() (*torrent.Torrent, error) {
			if strings.HasPrefix(arg, "magnet:") {
				t, err := client.AddMagnet(arg)
				if err != nil {
					return nil, xerrors.Errorf("error adding magnet: %w", err)
				}
				return t, nil
			} else if strings.HasPrefix(arg, "http://") || strings.HasPrefix(arg, "https://") {
				response, err := http.Get(arg)
				if err != nil {
					return nil, xerrors.Errorf("Error downloading torrent file: %s", err)
				}

				metaInfo, err := metainfo.Load(response.Body)
				defer response.Body.Close()
				if err != nil {
					return nil, xerrors.Errorf("error loading torrent file %q: %s\n", arg, err)
				}
				t, err := client.AddTorrent(metaInfo)
				if err != nil {
					return nil, xerrors.Errorf("adding torrent: %w", err)
				}
				return t, nil
			} else if strings.HasPrefix(arg, "infohash:") {
				t, _ := client.AddTorrentInfoHash(metainfo.NewHashFromHex(strings.TrimPrefix(arg, "infohash:")))
				return t, nil
			} else {
				metaInfo, err := metainfo.LoadFromFile(arg)
				if err != nil {
					return nil, xerrors.Errorf("error loading torrent file %q: %s\n", arg, err)
				}
				t, err := client.AddTorrent(metaInfo)
				if err != nil {
					return nil, xerrors.Errorf("adding torrent: %w", err)
				}
				return t, nil
			}
		}()
		if err != nil {
			return xerrors.Errorf("adding torrent for %q: %w", arg, err)
		}
		if flags.Progress {
			torrentBar(t, flags.PieceStates)
		}
		t.AddPeers(testPeers)
		go func() {
			<-t.GotInfo()
			if len(flags.File) == 0 {
				t.DownloadAll()
			} else {
				for _, f := range t.Files() {
					for _, fileArg := range flags.File {
						if f.DisplayPath() == fileArg {
							f.Download()
						}
					}
				}
			}
		}()
	}
	return nil
}

var flags struct {
	Debug bool
	Stats *bool

	*DownloadCmd  `arg:"subcommand:download"`
	*ListFilesCmd `arg:"subcommand:list-files"`
}

//DownloadCmd: &DownloadCmd{
//	UploadRate:   -1,
//	DownloadRate: -1,
//	Progress:     true,
//	Dht:          true,
//
//	TcpPeers:   true,
//	UtpPeers:   true,
//	Webtorrent: true,
//
//	Ipv4: true,
//	Ipv6: true,
//	Pex:  true,
//},

type DownloadCmd struct {
	Mmap            bool          `help:"memory-map torrent data"`
	TestPeer        []string      `help:"addresses of some starting peers"`
	Seed            bool          `help:"seed after download is complete"`
	Addr            string        `help:"network listen addr"`
	UploadRate      tagflag.Bytes `help:"max piece bytes to send per second" default:"-1"`
	DownloadRate    tagflag.Bytes `help:"max bytes per second down from peers" default:"-1"`
	PackedBlocklist string
	PublicIP        net.IP
	Progress        bool `default:"true"`
	PieceStates     bool
	Quiet           bool `help:"discard client logging"`
	Dht             bool `default:"true"`

	TcpPeers        bool `default:"true"`
	UtpPeers        bool `default:"true"`
	Webtorrent      bool `default:"true"`
	DisableWebseeds bool

	Ipv4 bool `default:"true"`
	Ipv6 bool `default:"true"`
	Pex  bool `default:"true"`

	File    []string
	Torrent []string `arity:"+" help:"torrent file path or magnet uri" arg:"positional"`
}

type ListFilesCmd struct {
	TorrentPath string `arg:"positional"`
}

func stdoutAndStderrAreSameFile() bool {
	fi1, _ := os.Stdout.Stat()
	fi2, _ := os.Stderr.Stat()
	return os.SameFile(fi1, fi2)
}

func statsEnabled() bool {
	if flags.Stats == nil {
		return flags.Debug
	}
	return *flags.Stats
}

func exitSignalHandlers(notify *missinggo.SynchronizedEvent) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
	for {
		log.Printf("close signal received: %+v", <-c)
		notify.Set()
	}
}

func main() {
	if err := mainErr(); err != nil {
		log.Printf("error in main: %v", err)
		os.Exit(1)
	}
}

func mainErr() error {
	stdLog.SetFlags(stdLog.Flags() | stdLog.Lshortfile)
	p := arg.MustParse(&flags)
	switch {
	//case :
	//	return announceErr(flags.Args, parser)
	case flags.DownloadCmd != nil:
		return downloadErr()
	case flags.ListFilesCmd != nil:
		mi, err := metainfo.LoadFromFile(flags.ListFilesCmd.TorrentPath)
		if err != nil {
			return fmt.Errorf("loading from file %q: %v", flags.ListFilesCmd.TorrentPath, err)
		}
		info, err := mi.UnmarshalInfo()
		if err != nil {
			return fmt.Errorf("unmarshalling info from metainfo at %q: %v", flags.ListFilesCmd.TorrentPath, err)
		}
		for _, f := range info.UpvertedFiles() {
			fmt.Println(f.DisplayPath(&info))
		}
		return nil
	default:
		p.Fail(fmt.Sprintf("unexpected subcommand: %v", p.Subcommand()))
		panic("unreachable")
	}
}

func downloadErr() error {
	defer envpprof.Stop()
	clientConfig := torrent.NewDefaultClientConfig()
	clientConfig.DisableWebseeds = flags.DisableWebseeds
	clientConfig.DisableTCP = !flags.TcpPeers
	clientConfig.DisableUTP = !flags.UtpPeers
	clientConfig.DisableIPv4 = !flags.Ipv4
	clientConfig.DisableIPv6 = !flags.Ipv6
	clientConfig.DisableAcceptRateLimiting = true
	clientConfig.NoDHT = !flags.Dht
	clientConfig.Debug = flags.Debug
	clientConfig.Seed = flags.Seed
	clientConfig.PublicIp4 = flags.PublicIP
	clientConfig.PublicIp6 = flags.PublicIP
	clientConfig.DisablePEX = !flags.Pex
	clientConfig.DisableWebtorrent = !flags.Webtorrent
	if flags.PackedBlocklist != "" {
		blocklist, err := iplist.MMapPackedFile(flags.PackedBlocklist)
		if err != nil {
			return xerrors.Errorf("loading blocklist: %v", err)
		}
		defer blocklist.Close()
		clientConfig.IPBlocklist = blocklist
	}
	if flags.Mmap {
		clientConfig.DefaultStorage = storage.NewMMap("")
	}
	if flags.Addr != "" {
		clientConfig.SetListenAddr(flags.Addr)
	}
	if flags.UploadRate != -1 {
		clientConfig.UploadRateLimiter = rate.NewLimiter(rate.Limit(flags.UploadRate), 256<<10)
	}
	if flags.DownloadRate != -1 {
		clientConfig.DownloadRateLimiter = rate.NewLimiter(rate.Limit(flags.DownloadRate), 1<<20)
	}
	if flags.Quiet {
		clientConfig.Logger = log.Discard
	}

	var stop missinggo.SynchronizedEvent
	defer func() {
		stop.Set()
	}()

	client, err := torrent.NewClient(clientConfig)
	if err != nil {
		return xerrors.Errorf("creating client: %v", err)
	}
	defer client.Close()
	go exitSignalHandlers(&stop)
	go func() {
		<-stop.C()
		client.Close()
	}()

	// Write status on the root path on the default HTTP muxer. This will be bound to localhost
	// somewhere if GOPPROF is set, thanks to the envpprof import.
	http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		client.WriteStatus(w)
	})
	err = addTorrents(client)
	if err != nil {
		return fmt.Errorf("adding torrents: %w", err)
	}
	defer outputStats(client)
	if client.WaitAll() {
		log.Print("downloaded ALL the torrents")
	} else {
		return xerrors.New("y u no complete torrents?!")
	}
	if flags.Seed {
		outputStats(client)
		<-stop.C()
	}
	return nil
}

func outputStats(cl *torrent.Client) {
	if !statsEnabled() {
		return
	}
	expvar.Do(func(kv expvar.KeyValue) {
		fmt.Printf("%s: %s\n", kv.Key, kv.Value)
	})
	cl.WriteStatus(os.Stdout)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"strings"

	"github.com/anacrolix/libtorgo/metainfo"

	"bitbucket.org/anacrolix/go.torrent"
	"bitbucket.org/anacrolix/go.torrent/util"
)

var (
	downloadDir = flag.String("downloadDir", "", "directory to store download torrent data")
	testPeer    = flag.String("testPeer", "", "bootstrap peer address")
	httpAddr    = flag.String("httpAddr", "localhost:0", "http serve address")
	// TODO: Check the default torrent listen port.
	listenAddr      = flag.String("listenAddr", ":6882", "incoming connection address")
	disableTrackers = flag.Bool("disableTrackers", false, "disable trackers")
	seed            = flag.Bool("seed", false, "seed after downloading")
)

func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	flag.Parse()
	if *httpAddr != "" {
		util.LoggedHTTPServe(*httpAddr)
	}
	client, err := torrent.NewClient(&torrent.Config{
		DataDir:         *downloadDir,
		DisableTrackers: *disableTrackers,
		ListenAddr:      *listenAddr,
	})
	if err != nil {
		log.Fatalf("error creating client: %s", err)
	}
	http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		client.WriteStatus(w)
	})
	defer client.Stop()
	if flag.NArg() == 0 {
		fmt.Fprintln(os.Stderr, "no torrents specified")
		return
	}
	for _, arg := range flag.Args() {
		var ih torrent.InfoHash
		if strings.HasPrefix(arg, "magnet:") {
			m, err := torrent.ParseMagnetURI(arg)
			if err != nil {
				log.Fatalf("error parsing magnet uri: %s", err)
			}
			ih = m.InfoHash
			_, err = client.AddMagnet(arg)
			if err != nil {
				log.Fatalf("error adding magnet: %s", err)
			}
		} else {
			metaInfo, err := metainfo.LoadFromFile(arg)
			if err != nil {
				log.Fatal(err)
			}
			err = client.AddTorrent(metaInfo)
			if err != nil {
				log.Fatal(err)
			}
			util.CopyExact(&ih, metaInfo.Info.Hash)
		}
		// client.PrioritizeDataRegion(ih, 0, 999999999)
		err := client.AddPeers(ih, func() []torrent.Peer {
			if *testPeer == "" {
				return nil
			}
			addr, err := net.ResolveTCPAddr("tcp", *testPeer)
			if err != nil {
				log.Fatal(err)
			}
			return []torrent.Peer{{
				IP:   addr.IP,
				Port: addr.Port,
			}}
		}())
		if err != nil {
			log.Fatal(err)
		}
	}
	if *seed {
		select {}
	}
	if client.WaitAll() {
		log.Print("downloaded ALL the torrents")
	} else {
		log.Fatal("y u no complete torrents?!")
	}
}
<commit_msg>Change cmd/torrent default ports<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"strings"

	"github.com/anacrolix/libtorgo/metainfo"

	"bitbucket.org/anacrolix/go.torrent"
	"bitbucket.org/anacrolix/go.torrent/util"
)

var (
	downloadDir = flag.String("downloadDir", "", "directory to store download torrent data")
	testPeer    = flag.String("testPeer", "", "bootstrap peer address")
	httpAddr    = flag.String("httpAddr", "localhost:6061", "http serve address")
	// TODO: Check the default torrent listen port.
	listenAddr      = flag.String("listenAddr", ":50007", "incoming connection address")
	disableTrackers = flag.Bool("disableTrackers", false, "disable trackers")
	seed            = flag.Bool("seed", false, "seed after downloading")
)

func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	flag.Parse()
	if *httpAddr != "" {
		util.LoggedHTTPServe(*httpAddr)
	}
	client, err := torrent.NewClient(&torrent.Config{
		DataDir:         *downloadDir,
		DisableTrackers: *disableTrackers,
		ListenAddr:      *listenAddr,
	})
	if err != nil {
		log.Fatalf("error creating client: %s", err)
	}
	http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		client.WriteStatus(w)
	})
	defer client.Stop()
	if flag.NArg() == 0 {
		fmt.Fprintln(os.Stderr, "no torrents specified")
		return
	}
	for _, arg := range flag.Args() {
		var ih torrent.InfoHash
		if strings.HasPrefix(arg, "magnet:") {
			m, err := torrent.ParseMagnetURI(arg)
			if err != nil {
				log.Fatalf("error parsing magnet uri: %s", err)
			}
			ih = m.InfoHash
			_, err = client.AddMagnet(arg)
			if err != nil {
				log.Fatalf("error adding magnet: %s", err)
			}
		} else {
			metaInfo, err := metainfo.LoadFromFile(arg)
			if err != nil {
				log.Fatal(err)
			}
			err = client.AddTorrent(metaInfo)
			if err != nil {
				log.Fatal(err)
			}
			util.CopyExact(&ih, metaInfo.Info.Hash)
		}
		// client.PrioritizeDataRegion(ih, 0, 999999999)
		err := client.AddPeers(ih, func() []torrent.Peer {
			if *testPeer == "" {
				return nil
			}
			addr, err := net.ResolveTCPAddr("tcp", *testPeer)
			if err != nil {
				log.Fatal(err)
			}
			return []torrent.Peer{{
				IP:   addr.IP,
				Port: addr.Port,
			}}
		}())
		if err != nil {
			log.Fatal(err)
		}
	}
	if *seed {
		select {}
	}
	if client.WaitAll() {
		log.Print("downloaded ALL the torrents")
	} else {
		log.Fatal("y u no complete torrents?!")
	}
}
<|endoftext|>


<commit_before>package confparse

// #include "confparse.h"
import (
	"C"
)

type ConfigFile struct {
	Filename string
	Entries *ConfigEntry
	Next *ConfigFile
}

type ConfigEntry struct {
	// variable and section line numbers
	VarLineNum int
	SectLineNum int

	// variable name and optional data section
	VarName string
	VarData string

	// child entries
	Entries *ConfigEntry

	// next entry at same level
	Next *ConfigEntry
}

func LoadConfigFile (filename string) *ConfigFile {
	return WalkCTree(C.config_file_load(C.CString(filename)))
}

// Walk roots
func WalkCTree(cfptr *C.ConfigFile) *ConfigFile {
	var entries *ConfigEntry = nil
	if cfptr.entries != nil {
		entries = WalkCETree(cfptr.entries)
	}

	var child *ConfigFile = nil
	if cfptr.next != nil {
		child = WalkCTree(cfptr.next)
	}

	return &ConfigFile {
		C.GoString(cfptr.filename),
		entries,
		child,
	}
}

// Walk entries
func WalkCETree(ceptr *C.ConfigEntry) *ConfigEntry {
	var entries *ConfigEntry = nil
	var next *ConfigEntry = nil
	var vardata string = "<undefined>"

	if ceptr.vardata != nil {
		vardata = C.GoString(ceptr.vardata);
	}

	if ceptr.entries != nil {
		entries = WalkCETree(ceptr.entries)
	}

	if ceptr.next != nil {
		next = WalkCETree(ceptr.next)
	}

	return &ConfigEntry {
		int(ceptr.varlinenum),
		int(ceptr.sectlinenum),
		C.GoString(ceptr.varname),
		vardata,
		entries,
		next,
	}
}
<commit_msg>confparse: handle C library failure<commit_after>package confparse

// #include "confparse.h"
import (
	"C"
)

type ConfigFile struct {
	Filename string
	Entries *ConfigEntry
	Next *ConfigFile
}

type ConfigEntry struct {
	// variable and section line numbers
	VarLineNum int
	SectLineNum int

	// variable name and optional data section
	VarName string
	VarData string

	// child entries
	Entries *ConfigEntry

	// next entry at same level
	Next *ConfigEntry
}

func LoadConfigFile (filename string) *ConfigFile {
	cfptr := C.config_file_load(C.CString(filename))
	if cfptr != nil {
		return WalkCTree(cfptr)
	}

	return nil
}

// Walk roots
func WalkCTree(cfptr *C.ConfigFile) *ConfigFile {
	var entries *ConfigEntry = nil
	if cfptr.entries != nil {
		entries = WalkCETree(cfptr.entries)
	}

	var child *ConfigFile = nil
	if cfptr.next != nil {
		child = WalkCTree(cfptr.next)
	}

	return &ConfigFile {
		C.GoString(cfptr.filename),
		entries,
		child,
	}
}

// Walk entries
func WalkCETree(ceptr *C.ConfigEntry) *ConfigEntry {
	var entries *ConfigEntry = nil
	var next *ConfigEntry = nil
	var vardata string = "<undefined>"

	if ceptr.vardata != nil {
		vardata = C.GoString(ceptr.vardata);
	}

	if ceptr.entries != nil {
		entries = WalkCETree(ceptr.entries)
	}

	if ceptr.next != nil {
		next = WalkCETree(ceptr.next)
	}

	return &ConfigEntry {
		int(ceptr.varlinenum),
		int(ceptr.sectlinenum),
		C.GoString(ceptr.varname),
		vardata,
		entries,
		next,
	}
}
<|endoftext|>


<commit_before>package http2

import (
	"bytes"
	"crypto/rand"
	"crypto/tls"
	"net"
	"sync"
	"sync/atomic"
	"testing"
	"time"
)

func BenchmarkConnReadWriteTCP_1K_C1(b *testing.B) {
	benchmarkConnReadWrite(b, false, 1024, 1)
}

func BenchmarkConnReadWriteTCP_1K_C8(b *testing.B) {
	benchmarkConnReadWrite(b, false, 1024, 8)
}

func BenchmarkConnReadWriteTCP_1K_C64(b *testing.B) {
	benchmarkConnReadWrite(b, false, 1024, 64)
}

func BenchmarkConnReadWriteTCP_1K_C512(b *testing.B) {
	benchmarkConnReadWrite(b, false, 1024, 512)
}

func BenchmarkConnReadWriteTCP_1M_C1(b *testing.B) {
	benchmarkConnReadWrite(b, false, 1024*1024, 1)
}

func BenchmarkConnReadWriteTCP_1M_C8(b *testing.B) {
	benchmarkConnReadWrite(b, false, 1024*1024, 8)
}

func BenchmarkConnReadWriteTCP_1M_C64(b *testing.B) {
	benchmarkConnReadWrite(b, false, 1024*1024, 64)
}

func BenchmarkConnReadWriteTCP_1M_C512(b *testing.B) {
	benchmarkConnReadWrite(b, false, 1024*1024, 512)
}

func BenchmarkConnReadWriteTLS_1K_C1(b *testing.B) {
	benchmarkConnReadWrite(b, true, 1024, 1)
}

func BenchmarkConnReadWriteTLS_1K_C8(b *testing.B) {
	benchmarkConnReadWrite(b, true, 1024, 8)
}

func BenchmarkConnReadWriteTLS_1K_C64(b *testing.B) {
	benchmarkConnReadWrite(b, true, 1024, 64)
}

func BenchmarkConnReadWriteTLS_1K_C512(b *testing.B) {
	benchmarkConnReadWrite(b, true, 1024, 512)
}

func BenchmarkConnReadWriteTLS_1M_C1(b *testing.B) {
	benchmarkConnReadWrite(b, true, 1024*1024, 1)
}

func BenchmarkConnReadWriteTLS_1M_C8(b *testing.B) {
	benchmarkConnReadWrite(b, true, 1024*1024, 8)
}

func BenchmarkConnReadWriteTLS_1M_C64(b *testing.B) {
	benchmarkConnReadWrite(b, true, 1024*1024, 64)
}

func BenchmarkConnReadWriteTLS_1M_C512(b *testing.B) {
	benchmarkConnReadWrite(b, true, 1024*1024, 512)
}

func benchmarkConnReadWrite(b *testing.B, overTLS bool, n, c int) {
	sc, cc := pipe(overTLS)
	server, client := &conn{Conn: sc, pending: map[uint32]int64{}}, &conn{Conn: cc, pending: map[uint32]int64{}}
	go server.serve()
	go client.serve()
	ch := make(chan int, c*4)
	var wg sync.WaitGroup
	for i := 0; i < c; i++ {
		wg.Add(1)
		go func() {
			for range ch {
				streamID, err := client.NextStreamID()
				if err != nil {
					b.Fatal(err)
				}
				err = client.writeBytes(streamID, n)
				if err != nil {
					b.Fatal(err)
				}
			}
			wg.Done()
		}()
	}
	b.StartTimer()
	for i := 0; i < b.N; i++ {
		ch <- i
	}
	b.StopTimer()
	close(ch)
	wg.Wait()
	if err := client.Close(); err != nil {
		b.Log(err)
	}
	if err := server.Close(); err != nil {
		b.Log(err)
	}
	time.Sleep(100 * time.Millisecond)
	if atomic.LoadInt64(&client.tx) != server.rx {
		b.Fatal("lost data")
	}
	if atomic.LoadInt64(&server.tx) != client.rx {
		b.Fatal("lost data")
	}
	if int64(b.N*n) != client.rx {
		b.Fatal("lost data")
	}
}

type conn struct {
	*Conn
	rx, tx  int64
	rb      bytes.Buffer
	pending map[uint32]int64
}

func (c *conn) serve() {
	for !c.Closed() {
		frame, err := c.ReadFrame()
		if err != nil {
			return
		}
		var endStream bool
		switch v := frame.(type) {
		case *DataFrame:
			c.rb.Reset()
			var n int64
			n, err = c.rb.ReadFrom(v.Data)
			c.rx += n
			c.pending[v.StreamID] += n
			if err != nil {
				return
			}
			endStream = v.EndStream
		case *HeadersFrame:
			endStream = v.EndStream
		}
		if endStream && c.ServerConn() {
			go c.writeBytes(frame.Stream(), int(c.pending[frame.Stream()]))
		}
	}
}

func (c *conn) writeBytes(streamID uint32, n int) (err error) {
	if streamID == 0 {
		if streamID, err = c.NextStreamID(); err != nil {
			return
		}
	}
	err = c.WriteFrame(&HeadersFrame{streamID, nil, Priority{}, 0, n == 0})
	if n > 0 && err == nil {
		if err = c.WriteFrame(&DataFrame{streamID, bytes.NewBuffer(make([]byte, n)), n, 0, true}); err == nil {
			atomic.AddInt64(&c.tx, int64(n))
		}
	}
	return
}

func pipe(overTLS bool) (server *Conn, client *Conn) {
	done := make(chan struct{})
	addr := &net.TCPAddr{Port: 8989}
	sc := &Config{}
	cc := &Config{}
	for {
		lis, err := net.Listen("tcp", addr.String())
		if err != nil {
			if addr.Port > 65535 {
				panic(err)
			}
			addr.Port++
			continue
		}
		go func() {
			s, err := lis.Accept()
			if err != nil {
				panic(err)
			}
			s.(*net.TCPConn).SetNoDelay(true)
			if overTLS {
				cert, err := tls.LoadX509KeyPair("testdata/server.pem", "testdata/server.key")
				if err != nil {
					panic(err)
				}
				sc.TLSConfig = &tls.Config{
					Certificates:             []tls.Certificate{cert},
					Rand:                     rand.Reader,
					NextProtos:               []string{VersionTLS},
					PreferServerCipherSuites: true,
				}
				s = tls.Server(s, sc.TLSConfig)
			}
			server = newConn(s, true, sc)
			if err = server.Handshake(); err != nil {
				panic(err)
			}
			lis.Close()
			close(done)
		}()
		break
	}
	c, err := net.Dial("tcp", addr.String())
	if err != nil {
		panic(err)
	}
	c.(*net.TCPConn).SetNoDelay(true)
	if overTLS {
		cc.TLSConfig = &tls.Config{
			Rand:               rand.Reader,
			NextProtos:         []string{VersionTLS},
			InsecureSkipVerify: true,
		}
		c = tls.Client(c, cc.TLSConfig)
	}
	client = newConn(c, false, cc)
	if err = client.Handshake(); err != nil {
		panic(err)
	}
	select {
	case <-done:
	case <-time.After(1 * time.Second):
		panic("pipe: timed out")
	}
	return
}
<commit_msg>apply added Frame Interface/remove handshake<commit_after>package http2

import (
	"bytes"
	"crypto/rand"
	"crypto/tls"
	"net"
	"sync"
	"sync/atomic"
	"testing"
	"time"
)

func BenchmarkConnReadWriteTCP_1K_C1(b *testing.B) {
	benchmarkConnReadWrite(b, false, 1024, 1)
}

func BenchmarkConnReadWriteTCP_1K_C8(b *testing.B) {
	benchmarkConnReadWrite(b, false, 1024, 8)
}

func BenchmarkConnReadWriteTCP_1K_C64(b *testing.B) {
	benchmarkConnReadWrite(b, false, 1024, 64)
}

func BenchmarkConnReadWriteTCP_1K_C512(b *testing.B) {
	benchmarkConnReadWrite(b, false, 1024, 512)
}

func BenchmarkConnReadWriteTCP_1M_C1(b *testing.B) {
	benchmarkConnReadWrite(b, false, 1024*1024, 1)
}

func BenchmarkConnReadWriteTCP_1M_C8(b *testing.B) {
	benchmarkConnReadWrite(b, false, 1024*1024, 8)
}

func BenchmarkConnReadWriteTCP_1M_C64(b *testing.B) {
	benchmarkConnReadWrite(b, false, 1024*1024, 64)
}

func BenchmarkConnReadWriteTCP_1M_C512(b *testing.B) {
	benchmarkConnReadWrite(b, false, 1024*1024, 512)
}

func BenchmarkConnReadWriteTLS_1K_C1(b *testing.B) {
	benchmarkConnReadWrite(b, true, 1024, 1)
}

func BenchmarkConnReadWriteTLS_1K_C8(b *testing.B) {
	benchmarkConnReadWrite(b, true, 1024, 8)
}

func BenchmarkConnReadWriteTLS_1K_C64(b *testing.B) {
	benchmarkConnReadWrite(b, true, 1024, 64)
}

func BenchmarkConnReadWriteTLS_1K_C512(b *testing.B) {
	benchmarkConnReadWrite(b, true, 1024, 512)
}

func BenchmarkConnReadWriteTLS_1M_C1(b *testing.B) {
	benchmarkConnReadWrite(b, true, 1024*1024, 1)
}

func BenchmarkConnReadWriteTLS_1M_C8(b *testing.B) {
	benchmarkConnReadWrite(b, true, 1024*1024, 8)
}

func BenchmarkConnReadWriteTLS_1M_C64(b *testing.B) {
	benchmarkConnReadWrite(b, true, 1024*1024, 64)
}

func BenchmarkConnReadWriteTLS_1M_C512(b *testing.B) {
	benchmarkConnReadWrite(b, true, 1024*1024, 512)
}

func benchmarkConnReadWrite(b *testing.B, overTLS bool, n, c int) {
	sc, cc := pipe(overTLS)
	server, client := &conn{Conn: sc, pending: map[uint32]int64{}}, &conn{Conn: cc, pending: map[uint32]int64{}}
	go server.serve()
	go client.serve()
	ch := make(chan int, c*4)
	var wg sync.WaitGroup
	for i := 0; i < c; i++ {
		wg.Add(1)
		go func() {
			for range ch {
				streamID, err := client.NextStreamID()
				if err != nil {
					b.Fatal(err)
				}
				err = client.writeBytes(streamID, n)
				if err != nil {
					b.Fatal(err)
				}
			}
			wg.Done()
		}()
	}
	b.StartTimer()
	for i := 0; i < b.N; i++ {
		ch <- i
	}
	b.StopTimer()
	close(ch)
	wg.Wait()
	if err := client.Close(); err != nil {
		b.Log(err)
	}
	if err := server.Close(); err != nil {
		b.Log(err)
	}
	time.Sleep(100 * time.Millisecond)
	if atomic.LoadInt64(&client.tx) != server.rx {
		b.Fatal("lost data")
	}
	if atomic.LoadInt64(&server.tx) != client.rx {
		b.Fatal("lost data")
	}
	if int64(b.N*n) != client.rx {
		b.Fatal("lost data")
	}
}

type conn struct {
	*Conn
	rx, tx  int64
	rb      bytes.Buffer
	pending map[uint32]int64
}

func (c *conn) serve() {
	for !c.Closed() {
		frame, err := c.ReadFrame()
		if err != nil {
			return
		}
		switch v := frame.(type) {
		case *DataFrame:
			c.rb.Reset()
			var n int64
			n, err = c.rb.ReadFrom(v.Data)
			c.rx += n
			c.pending[v.StreamID] += n
			if err != nil {
				return
			}
		}
		if frame.EndOfStream() && c.ServerConn() {
			go c.writeBytes(frame.Stream(), int(c.pending[frame.Stream()]))
		}
	}
}

func (c *conn) writeBytes(streamID uint32, n int) (err error) {
	if streamID == 0 {
		if streamID, err = c.NextStreamID(); err != nil {
			return
		}
	}
	err = c.WriteFrame(&HeadersFrame{streamID, nil, Priority{}, 0, n == 0})
	if n > 0 && err == nil {
		if err = c.WriteFrame(&DataFrame{streamID, bytes.NewBuffer(make([]byte, n)), n, 0, true}); err == nil {
			atomic.AddInt64(&c.tx, int64(n))
		}
	}
	return
}

func pipe(overTLS bool) (server *Conn, client *Conn) {
	done := make(chan struct{})
	addr := &net.TCPAddr{Port: 8989}
	sc := &Config{}
	cc := &Config{}
	for {
		lis, err := net.Listen("tcp", addr.String())
		if err != nil {
			if addr.Port > 65535 {
				panic(err)
			}
			addr.Port++
			continue
		}
		go func() {
			s, err := lis.Accept()
			if err != nil {
				panic(err)
			}
			s.(*net.TCPConn).SetNoDelay(true)
			if overTLS {
				cert, err := tls.LoadX509KeyPair("testdata/server.pem", "testdata/server.key")
				if err != nil {
					panic(err)
				}
				sc.TLSConfig = &tls.Config{
					Certificates:             []tls.Certificate{cert},
					Rand:                     rand.Reader,
					NextProtos:               []string{VersionTLS},
					PreferServerCipherSuites: true,
				}
				s = tls.Server(s, sc.TLSConfig)
			}
			server = newConn(s, true, sc)
			lis.Close()
			close(done)
		}()
		break
	}
	c, err := net.Dial("tcp", addr.String())
	if err != nil {
		panic(err)
	}
	c.(*net.TCPConn).SetNoDelay(true)
	if overTLS {
		cc.TLSConfig = &tls.Config{
			Rand:               rand.Reader,
			NextProtos:         []string{VersionTLS},
			InsecureSkipVerify: true,
		}
		c = tls.Client(c, cc.TLSConfig)
	}
	client = newConn(c, false, cc)
	select {
	case <-done:
	case <-time.After(1 * time.Second):
		panic("pipe: timed out")
	}
	return
}
<|endoftext|>


<commit_before>package gcloudcleanup

import (
	"time"

	"gopkg.in/urfave/cli.v2"
)

var (
	Flags = []cli.Flag{
		&cli.StringFlag{
			Name:    "account-json",
			Value:   "{}",
			Usage:   "file path to or json blob of GCE account stuff",
			EnvVars: []string{"GCLOUD_CLEANUP_ACCOUNT_JSON", "GOOGLE_CREDENTIALS"},
		},
		&cli.StringFlag{
			Name:    "project-id",
			Usage:   "name of GCE project",
			EnvVars: []string{"GCLOUD_CLEANUP_PROJECT_ID", "GCLOUD_PROJECT"},
		},
		&cli.DurationFlag{
			Name:    "instance-max-age",
			Value:   3 * time.Hour,
			Usage:   "max age for an instance to be considered deletable",
			EnvVars: []string{"GCLOUD_CLEANUP_INSTANCE_MAX_AGE"},
		},
		&cli.StringSliceFlag{
			Name:    "instance-filters",
			Usage:   "filters used when fetching instances for deletion",
			EnvVars: []string{"GCLOUD_CLEANUP_INSTANCE_FILTERS"},
		},
		&cli.StringSliceFlag{
			Name:    "image-filters",
			Usage:   "filters used when fetching images for deletion",
			EnvVars: []string{"GCLOUD_CLEANUP_IMAGE_FILTERS"},
		},
		&cli.StringSliceFlag{
			Name:    "entities",
			Usage:   "entities to clean up",
			EnvVars: []string{"GCLOUD_CLEANUP_ENTITIES"},
		},
		&cli.DurationFlag{
			Name:    "loop-sleep",
			Value:   5 * time.Minute,
			Usage:   "sleep time between loops",
			EnvVars: []string{"GCLOUD_CLEANUP_LOOP_SLEEP"},
		},
		&cli.BoolFlag{
			Name:    "once",
			Usage:   "only run once, no looping",
			EnvVars: []string{"GCLOUD_CLEANUP_ONCE"},
		},
		&cli.StringFlag{
			Name:    "rate-limit-redis-url",
			Usage:   "URL to Redis instance to use for rate limiting",
			EnvVars: []string{"GCLOUD_CLEANUP_RATE_LIMIT_REDIS_URL"},
		},
		&cli.StringFlag{
			Name:    "rate-limit-prefix",
			Usage:   "prefix for the rate limit key in Redis",
			EnvVars: []string{"GCLOUD_CLEANUP_RATE_LIMIT_PREFIX"},
		},
		&cli.IntFlag{
			Name:    "rate-limit-max-calls",
			Value:   10,
			Usage:   "number of calls per duration to let through to the GCE API",
			EnvVars: []string{"GCLOUD_CLEANUP_RATE_LIMIT_MAX_CALLS"},
		},
		&cli.DurationFlag{
			Name:    "rate-limit-duration",
			Value:   1 * time.Second,
			Usage:   "interval in which to let max-calls through to the GCE API",
			EnvVars: []string{"GCLOUD_CLEANUP_RATE_LIMIT_DURATION"},
		},
		&cli.StringFlag{
			Name:    "job-board-url",
			Value:   "http://localhost:4567",
			Usage:   "url to job-board instance for fetching registered images",
			EnvVars: []string{"GCLOUD_CLEANUP_JOB_BOARD_URL", "JOB_BOARD_URL"},
		},
		&cli.BoolFlag{
			Name:    "archive-serial",
			Usage:   "archive instance serial output before deleting",
			EnvVars: []string{"GCLOUD_CLEANUP_ARCHIVE_SERIAL", "ARCHIVE_SERIAL"},
		},
		&cli.StringFlag{
			Name:    "archive-bucket",
			Value:   "gcloud-cleanup-serial-output",
			Usage:   "bucket to which instance serial output will be archived before deleting",
			EnvVars: []string{"GCLOUD_CLEANUP_ARCHIVE_BUCKET", "ARCHIVE_BUCKET"},
		},
		&cli.Int64Flag{
			Name:    "archive-sample-rate",
			Value:   1,
			Usage:   "sample rate for archiving as an inverse fraction - for sample rate n, every nth event will be sampled",
			EnvVars: []string{"GCLOUD_CLEANUP_ARCHIVE_SAMPLE_RATE", "ARCHIVE_SAMPLE_RATE"},
		},
		&cli.Int64Flag{
			Name:    "trace-sample-rate",
			Value:   8,
			Usage:   "sample rate for trace as an inverse fraction - for sample rate n, every nth event will be sampled",
			EnvVars: []string{"GCLOUD_CLEANUP_TRACE_SAMPLE_RATE", "TRACE_SAMPLE_RATE"},
		},
		&cli.BoolFlag{
			Name:    "debug",
			Usage:   "output more stuff",
			EnvVars: []string{"GCLOUD_CLEANUP_DEBUG", "DEBUG"},
		},
		&cli.BoolFlag{
			Name:    "opencensus-enable",
			Usage:   "disable tracing for gcloud-cleanup",
			EnvVars: []string{"GCLOUD_CLEANUP_OPENCENSUS_ENABLE", "OPENCENSUS_DISABLE"},
		},
		&cli.BoolFlag{
			Name:    "noop",
			Usage:   "don't do mutative stuff",
			EnvVars: []string{"GCLOUD_CLEANUP_NOOP", "NOOP"},
		},
	}
)
<commit_msg>make name, usage, envvars consistent, default as enabled<commit_after>package gcloudcleanup

import (
	"time"

	"gopkg.in/urfave/cli.v2"
)

var (
	Flags = []cli.Flag{
		&cli.StringFlag{
			Name:    "account-json",
			Value:   "{}",
			Usage:   "file path to or json blob of GCE account stuff",
			EnvVars: []string{"GCLOUD_CLEANUP_ACCOUNT_JSON", "GOOGLE_CREDENTIALS"},
		},
		&cli.StringFlag{
			Name:    "project-id",
			Usage:   "name of GCE project",
			EnvVars: []string{"GCLOUD_CLEANUP_PROJECT_ID", "GCLOUD_PROJECT"},
		},
		&cli.DurationFlag{
			Name:    "instance-max-age",
			Value:   3 * time.Hour,
			Usage:   "max age for an instance to be considered deletable",
			EnvVars: []string{"GCLOUD_CLEANUP_INSTANCE_MAX_AGE"},
		},
		&cli.StringSliceFlag{
			Name:    "instance-filters",
			Usage:   "filters used when fetching instances for deletion",
			EnvVars: []string{"GCLOUD_CLEANUP_INSTANCE_FILTERS"},
		},
		&cli.StringSliceFlag{
			Name:    "image-filters",
			Usage:   "filters used when fetching images for deletion",
			EnvVars: []string{"GCLOUD_CLEANUP_IMAGE_FILTERS"},
		},
		&cli.StringSliceFlag{
			Name:    "entities",
			Usage:   "entities to clean up",
			EnvVars: []string{"GCLOUD_CLEANUP_ENTITIES"},
		},
		&cli.DurationFlag{
			Name:    "loop-sleep",
			Value:   5 * time.Minute,
			Usage:   "sleep time between loops",
			EnvVars: []string{"GCLOUD_CLEANUP_LOOP_SLEEP"},
		},
		&cli.BoolFlag{
			Name:    "once",
			Usage:   "only run once, no looping",
			EnvVars: []string{"GCLOUD_CLEANUP_ONCE"},
		},
		&cli.StringFlag{
			Name:    "rate-limit-redis-url",
			Usage:   "URL to Redis instance to use for rate limiting",
			EnvVars: []string{"GCLOUD_CLEANUP_RATE_LIMIT_REDIS_URL"},
		},
		&cli.StringFlag{
			Name:    "rate-limit-prefix",
			Usage:   "prefix for the rate limit key in Redis",
			EnvVars: []string{"GCLOUD_CLEANUP_RATE_LIMIT_PREFIX"},
		},
		&cli.IntFlag{
			Name:    "rate-limit-max-calls",
			Value:   10,
			Usage:   "number of calls per duration to let through to the GCE API",
			EnvVars: []string{"GCLOUD_CLEANUP_RATE_LIMIT_MAX_CALLS"},
		},
		&cli.DurationFlag{
			Name:    "rate-limit-duration",
			Value:   1 * time.Second,
			Usage:   "interval in which to let max-calls through to the GCE API",
			EnvVars: []string{"GCLOUD_CLEANUP_RATE_LIMIT_DURATION"},
		},
		&cli.StringFlag{
			Name:    "job-board-url",
			Value:   "http://localhost:4567",
			Usage:   "url to job-board instance for fetching registered images",
			EnvVars: []string{"GCLOUD_CLEANUP_JOB_BOARD_URL", "JOB_BOARD_URL"},
		},
		&cli.BoolFlag{
			Name:    "archive-serial",
			Usage:   "archive instance serial output before deleting",
			EnvVars: []string{"GCLOUD_CLEANUP_ARCHIVE_SERIAL", "ARCHIVE_SERIAL"},
		},
		&cli.StringFlag{
			Name:    "archive-bucket",
			Value:   "gcloud-cleanup-serial-output",
			Usage:   "bucket to which instance serial output will be archived before deleting",
			EnvVars: []string{"GCLOUD_CLEANUP_ARCHIVE_BUCKET", "ARCHIVE_BUCKET"},
		},
		&cli.Int64Flag{
			Name:    "archive-sample-rate",
			Value:   1,
			Usage:   "sample rate for archiving as an inverse fraction - for sample rate n, every nth event will be sampled",
			EnvVars: []string{"GCLOUD_CLEANUP_ARCHIVE_SAMPLE_RATE", "ARCHIVE_SAMPLE_RATE"},
		},
		&cli.Int64Flag{
			Name:    "trace-sample-rate",
			Value:   8,
			Usage:   "sample rate for trace as an inverse fraction - for sample rate n, every nth event will be sampled",
			EnvVars: []string{"GCLOUD_CLEANUP_TRACE_SAMPLE_RATE", "TRACE_SAMPLE_RATE"},
		},
		&cli.BoolFlag{
			Name:    "debug",
			Usage:   "output more stuff",
			EnvVars: []string{"GCLOUD_CLEANUP_DEBUG", "DEBUG"},
		},
		&cli.BoolFlag{
			Name:    "opencensus-enable",
			Usage:   "enable tracing for gcloud-cleanup",
			EnvVars: []string{"GCLOUD_CLEANUP_OPENCENSUS_ENABLE", "OPENCENSUS_ENABLE"},
		},
		&cli.BoolFlag{
			Name:    "noop",
			Usage:   "don't do mutative stuff",
			EnvVars: []string{"GCLOUD_CLEANUP_NOOP", "NOOP"},
		},
	}
)
<|endoftext|>


<commit_before>/* https://leetcode.com/problems/search-in-rotated-sorted-array/#/description
Suppose an array sorted in ascending order is rotated at some pivot unknown to you beforehand.

(i.e., 0 1 2 4 5 6 7 might become 4 5 6 7 0 1 2).

You are given a target value to search. If found in the array return its index, otherwise return -1.

You may assume no duplicate exists in the array.
*/

package leetcode

func search(nums []int, target int) int {
	var (
		lenn        = len(nums)
		left, right = 0, lenn - 1
	)

	if lenn == 0 {
		return -1
	}

	if nums[0] > nums[lenn-1] {
		// is Rotated Sorted Array
		if target <= nums[lenn-1] {
			// right block, find left index
			tmp := right
			for tmp >= left {
				mid := left + (tmp-left)/2
				if nums[mid] > nums[lenn-1] {
					left = mid + 1
				} else {
					tmp = mid - 1
				}
			}
			left = tmp + 1
		}
		if target >= nums[0] {
			// left block, find right index
			tmp := left
			for right >= tmp {
				mid := tmp + (right-tmp)/2
				if nums[mid] < nums[0] {
					right = mid - 1
				} else {
					tmp = mid + 1
				}
			}
			right = tmp - 1
		}
	}

	for right >= left {
		mid := left + (right-left)/2
		if nums[mid] == target {
			return mid
		} else if nums[mid] > target {
			right = mid - 1
		} else {
			left = mid + 1
		}
	}

	return -1
}
<commit_msg>use another thinking<commit_after>/* https://leetcode.com/problems/search-in-rotated-sorted-array/#/description
Suppose an array sorted in ascending order is rotated at some pivot unknown to you beforehand.

(i.e., 0 1 2 4 5 6 7 might become 4 5 6 7 0 1 2).

You are given a target value to search. If found in the array return its index, otherwise return -1.

You may assume no duplicate exists in the array.
*/

package leetcode

func search(nums []int, target int) int {
	var (
		lenn        = len(nums)
		left, right = 0, lenn - 1
	)

	for right >= left {
		mid := left + (right-left)/2
		if nums[mid] == target {
			return mid
		}
		if nums[left] <= nums[mid] {
			if nums[left] <= target && target < nums[mid] {
				right = mid
			} else {
				left = mid + 1
			}
		} else {
			if nums[mid] < target && target <= nums[right-1] {
				left = mid + 1
			} else {
				right = mid
			}
		}
	}

	return -1
}
<|endoftext|>


<commit_before>package invdapi

import (
	"bytes"
	"encoding/json"
	"errors"
	"github.com/Invoiced/invoiced-go/invdendpoint"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"
	"strings"
)

const requestURL = "https://api.invoiced.com"
const devRequestURL = "https://api.sandbox.invoiced.com"
const requestType = "application/json"
const InvoicedTokenString = "invoicedToken"

const version = "1.3.0"

func Version() string {
	return version
}

type Connection struct {
	key             string
	itemsPerRequest int
	client          *http.Client
	url             string
}

type InvoicedToken struct {
	Key string `json:"invoicedApiKey"`
}

func NewConnection(key string, devMode bool) *Connection {
	c := new(Connection)
	c.key = key
	c.client = new(http.Client)
	if devMode {
		c.url = devRequestURL
	} else {
		c.url = requestURL
	}

	return c
}

func checkStatusForError(status int, r io.Reader) error {
	if status < 400 {
		return nil
	}

	body, err := ioutil.ReadAll(r)
	if err != nil {
		panic(err)
	}

	apiError := new(APIError)

	err = json.Unmarshal(body, apiError)

	if err != nil {
		apiError.Type = string(body)

	}

	return errors.New(apiError.Error())

}

func pushDataIntoStruct(endPointData interface{}, respBody io.Reader) {

	body, err := ioutil.ReadAll(respBody)

	if err != nil {
		panic(err)
	}

	log.Println("INVOICED-GO => ", string(body))
	err = json.Unmarshal(body, endPointData)

	if err != nil {
		panic(err)
	}

}

func parseURLLinksFromString(s string) map[string]string {

	urlAndLinkMap := make(map[string]string)

	rawURLLinksAndRelations := strings.Split(s, ",")

	for _, rawURLLinkRelation := range rawURLLinksAndRelations {
		parsedRawURLAndRelation := strings.Split(rawURLLinkRelation, ";")
		url := parseRawURL(parsedRawURLAndRelation[0])
		relation := parseRawRelation(parsedRawURLAndRelation[1])

		urlAndLinkMap[relation] = url
	}

	return urlAndLinkMap
}

func parseRawRelation(s string) string {
	//parse rel="last" => last

	first := strings.Index(s, "\"")
	last := strings.LastIndex(s, "\"")

	trimmed := s[first+1 : last]

	trimmed = strings.TrimSpace(trimmed)

	return trimmed

}

func parseRawURL(s string) string {
	//<https://api.invoiced.com/invoices?page=1>

	trimmed := strings.Trim(s, " < ")

	trimmed = strings.Trim(trimmed, " > ")

	trimmed = strings.TrimSpace(trimmed)

	return trimmed

}

func addFilterSortToEndPoint(endpoint string, filter *invdendpoint.Filter, sort *invdendpoint.Sort) string {

	emptyFilter := true
	emptySort := true

	if filter != nil && filter.String() != "" {
		emptyFilter = false
	}

	if sort != nil && sort.String() != "" {
		emptySort = false
	}

	if !emptyFilter && !emptySort {
		return endpoint + "?" + filter.String() + "&" + sort.String()
	} else if !emptyFilter && emptySort {
		return endpoint + "?" + filter.String()
	} else if emptyFilter && !emptySort {
		return endpoint + "?" + sort.String()
	}

	return endpoint

}

func addIncludeToEndPoint(endpoint string, includeValue string) string {

	finalEndpoint := ""
	if strings.Contains(endpoint, "?") {
		finalEndpoint = endpoint + "&" + "include=" + includeValue
	} else {
		finalEndpoint = endpoint + "?" + "include=" + includeValue
	}

	return finalEndpoint

}

func makeEndPointSingular(endpoint string, id int64) string {
	return endpoint + "/" + strconv.FormatInt(id, 10)
}

func (c *Connection) makeEndPointURL(endPoint string) string {

	return c.url + endPoint
}

func (c *Connection) setItemsPerRequest(items int) {
	c.itemsPerRequest = items
}

func (c *Connection) get(endPoint string) *http.Response {

	req, err := http.NewRequest("GET", endPoint, nil)
	req.SetBasicAuth(c.key, "")

	resp, err := c.client.Do(req)

	if err != nil {
		panic(err)
	}

	return resp

}

func (c *Connection) post(endPoint string, body io.Reader) *http.Response {

	req, err := http.NewRequest("POST", endPoint, body)
	req.SetBasicAuth(c.key, "")
	req.Header.Set("Content-Type", requestType)

	resp, err := c.client.Do(req)

	if err != nil {
		panic(err)
	}

	return resp

}

func (c *Connection) patch(endPoint string, body io.Reader) *http.Response {

	req, err := http.NewRequest("PATCH", endPoint, body)
	req.SetBasicAuth(c.key, "")
	req.Header.Set("Content-Type", requestType)

	resp, err := c.client.Do(req)

	if err != nil {
		panic(err)
	}

	return resp

}

func (c *Connection) deleteRequest(endPoint string) *http.Response {

	req, err := http.NewRequest("DELETE", endPoint, nil)
	req.SetBasicAuth(c.key, "")
	req.Header.Set("Content-Type", requestType)

	resp, err := c.client.Do(req)

	if err != nil {
		panic(err)
	}

	return resp

}

func (c *Connection) create(endPoint string, requestData interface{}, responseData interface{}) error {

	b, err := json.Marshal(requestData)

	if err != nil {
		panic(err)
	}

	body := bytes.NewBuffer(b)

	resp := c.post(endPoint, body)

	apiError := checkStatusForError(resp.StatusCode, resp.Body)

	if apiError != nil {
		return apiError
	}

	pushDataIntoStruct(responseData, resp.Body)

	return nil

}

func (c *Connection) delete(endPoint string) error {

	resp := c.deleteRequest(endPoint)

	apiError := checkStatusForError(resp.StatusCode, resp.Body)

	if apiError != nil {
		return apiError
	}

	return nil

}

func (c *Connection) update(endPoint string, requestData interface{}, responseData interface{}) error {

	b, err := json.Marshal(requestData)

	if err != nil {
		panic(err)
	}

	body := bytes.NewBuffer(b)

	resp := c.patch(endPoint, body)

	apiError := checkStatusForError(resp.StatusCode, resp.Body)

	if apiError != nil {
		return apiError
	}

	pushDataIntoStruct(responseData, resp.Body)

	return nil

}

func (c *Connection) count(endPoint string) (int64, error) {
	resp := c.get(endPoint)

	defer resp.Body.Close()

	apiErr := checkStatusForError(resp.StatusCode, resp.Body)

	if apiErr != nil {
		return 0, apiErr
	}

	s := resp.Header.Get("X-Total-Count")

	i, err := strconv.ParseInt(s, 10, 64)

	if err != nil {
		panic(err)
	}

	return i, nil

}

func (c *Connection) retrieveDataFromAPI(endPoint string, endPointData interface{}) (string, error) {

	nextURL := ""

	resp := c.get(endPoint)

	defer resp.Body.Close()

	link := resp.Header.Get("Link")

	if link != "" {
		nextMap := parseURLLinksFromString(link)

		if nextMap["self"] != nextMap["next"] {
			nextURL = nextMap["next"]
		}
	}

	apiError := checkStatusForError(resp.StatusCode, resp.Body)

	if apiError != nil {
		return "", apiError
	}

	pushDataIntoStruct(endPointData, resp.Body)

	return nextURL, nil

}
<commit_msg>Turn off debugging<commit_after>package invdapi

import (
	"bytes"
	"encoding/json"
	"errors"
	"github.com/Invoiced/invoiced-go/invdendpoint"
	"io"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
)

const requestURL = "https://api.invoiced.com"
const devRequestURL = "https://api.sandbox.invoiced.com"
const requestType = "application/json"
const InvoicedTokenString = "invoicedToken"

const version = "1.3.0"

func Version() string {
	return version
}

type Connection struct {
	key             string
	itemsPerRequest int
	client          *http.Client
	url             string
}

type InvoicedToken struct {
	Key string `json:"invoicedApiKey"`
}

func NewConnection(key string, devMode bool) *Connection {
	c := new(Connection)
	c.key = key
	c.client = new(http.Client)
	if devMode {
		c.url = devRequestURL
	} else {
		c.url = requestURL
	}

	return c
}

func checkStatusForError(status int, r io.Reader) error {
	if status < 400 {
		return nil
	}

	body, err := ioutil.ReadAll(r)
	if err != nil {
		panic(err)
	}

	apiError := new(APIError)

	err = json.Unmarshal(body, apiError)

	if err != nil {
		apiError.Type = string(body)

	}

	return errors.New(apiError.Error())

}

func pushDataIntoStruct(endPointData interface{}, respBody io.Reader) {

	body, err := ioutil.ReadAll(respBody)

	if err != nil {
		panic(err)
	}

	err = json.Unmarshal(body, endPointData)

	if err != nil {
		panic(err)
	}

}

func parseURLLinksFromString(s string) map[string]string {

	urlAndLinkMap := make(map[string]string)

	rawURLLinksAndRelations := strings.Split(s, ",")

	for _, rawURLLinkRelation := range rawURLLinksAndRelations {
		parsedRawURLAndRelation := strings.Split(rawURLLinkRelation, ";")
		url := parseRawURL(parsedRawURLAndRelation[0])
		relation := parseRawRelation(parsedRawURLAndRelation[1])

		urlAndLinkMap[relation] = url
	}

	return urlAndLinkMap
}

func parseRawRelation(s string) string {
	//parse rel="last" => last

	first := strings.Index(s, "\"")
	last := strings.LastIndex(s, "\"")

	trimmed := s[first+1 : last]

	trimmed = strings.TrimSpace(trimmed)

	return trimmed

}

func parseRawURL(s string) string {
	//<https://api.invoiced.com/invoices?page=1>

	trimmed := strings.Trim(s, " < ")

	trimmed = strings.Trim(trimmed, " > ")

	trimmed = strings.TrimSpace(trimmed)

	return trimmed

}

func addFilterSortToEndPoint(endpoint string, filter *invdendpoint.Filter, sort *invdendpoint.Sort) string {

	emptyFilter := true
	emptySort := true

	if filter != nil && filter.String() != "" {
		emptyFilter = false
	}

	if sort != nil && sort.String() != "" {
		emptySort = false
	}

	if !emptyFilter && !emptySort {
		return endpoint + "?" + filter.String() + "&" + sort.String()
	} else if !emptyFilter && emptySort {
		return endpoint + "?" + filter.String()
	} else if emptyFilter && !emptySort {
		return endpoint + "?" + sort.String()
	}

	return endpoint

}

func addIncludeToEndPoint(endpoint string, includeValue string) string {

	finalEndpoint := ""
	if strings.Contains(endpoint, "?") {
		finalEndpoint = endpoint + "&" + "include=" + includeValue
	} else {
		finalEndpoint = endpoint + "?" + "include=" + includeValue
	}

	return finalEndpoint

}

func makeEndPointSingular(endpoint string, id int64) string {
	return endpoint + "/" + strconv.FormatInt(id, 10)
}

func (c *Connection) makeEndPointURL(endPoint string) string {

	return c.url + endPoint
}

func (c *Connection) setItemsPerRequest(items int) {
	c.itemsPerRequest = items
}

func (c *Connection) get(endPoint string) *http.Response {

	req, err := http.NewRequest("GET", endPoint, nil)
	req.SetBasicAuth(c.key, "")

	resp, err := c.client.Do(req)

	if err != nil {
		panic(err)
	}

	return resp

}

func (c *Connection) post(endPoint string, body io.Reader) *http.Response {

	req, err := http.NewRequest("POST", endPoint, body)
	req.SetBasicAuth(c.key, "")
	req.Header.Set("Content-Type", requestType)

	resp, err := c.client.Do(req)

	if err != nil {
		panic(err)
	}

	return resp

}

func (c *Connection) patch(endPoint string, body io.Reader) *http.Response {

	req, err := http.NewRequest("PATCH", endPoint, body)
	req.SetBasicAuth(c.key, "")
	req.Header.Set("Content-Type", requestType)

	resp, err := c.client.Do(req)

	if err != nil {
		panic(err)
	}

	return resp

}

func (c *Connection) deleteRequest(endPoint string) *http.Response {

	req, err := http.NewRequest("DELETE", endPoint, nil)
	req.SetBasicAuth(c.key, "")
	req.Header.Set("Content-Type", requestType)

	resp, err := c.client.Do(req)

	if err != nil {
		panic(err)
	}

	return resp

}

func (c *Connection) create(endPoint string, requestData interface{}, responseData interface{}) error {

	b, err := json.Marshal(requestData)

	if err != nil {
		panic(err)
	}

	body := bytes.NewBuffer(b)

	resp := c.post(endPoint, body)

	apiError := checkStatusForError(resp.StatusCode, resp.Body)

	if apiError != nil {
		return apiError
	}

	pushDataIntoStruct(responseData, resp.Body)

	return nil

}

func (c *Connection) delete(endPoint string) error {

	resp := c.deleteRequest(endPoint)

	apiError := checkStatusForError(resp.StatusCode, resp.Body)

	if apiError != nil {
		return apiError
	}

	return nil

}

func (c *Connection) update(endPoint string, requestData interface{}, responseData interface{}) error {

	b, err := json.Marshal(requestData)

	if err != nil {
		panic(err)
	}

	body := bytes.NewBuffer(b)

	resp := c.patch(endPoint, body)

	apiError := checkStatusForError(resp.StatusCode, resp.Body)

	if apiError != nil {
		return apiError
	}

	pushDataIntoStruct(responseData, resp.Body)

	return nil

}

func (c *Connection) count(endPoint string) (int64, error) {
	resp := c.get(endPoint)

	defer resp.Body.Close()

	apiErr := checkStatusForError(resp.StatusCode, resp.Body)

	if apiErr != nil {
		return 0, apiErr
	}

	s := resp.Header.Get("X-Total-Count")

	i, err := strconv.ParseInt(s, 10, 64)

	if err != nil {
		panic(err)
	}

	return i, nil

}

func (c *Connection) retrieveDataFromAPI(endPoint string, endPointData interface{}) (string, error) {

	nextURL := ""

	resp := c.get(endPoint)

	defer resp.Body.Close()

	link := resp.Header.Get("Link")

	if link != "" {
		nextMap := parseURLLinksFromString(link)

		if nextMap["self"] != nextMap["next"] {
			nextURL = nextMap["next"]
		}
	}

	apiError := checkStatusForError(resp.StatusCode, resp.Body)

	if apiError != nil {
		return "", apiError
	}

	pushDataIntoStruct(endPointData, resp.Body)

	return nextURL, nil

}
<|endoftext|>