<commit_before>// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"context"
	"time"

	"github.com/fatih/color"
	"github.com/minio/cli"
	json "github.com/minio/colorjson"
	"github.com/minio/madmin-go"
	"github.com/minio/mc/pkg/probe"
	"github.com/minio/minio/pkg/console"
)

var adminServiceRestartCmd = cli.Command{
	Name:         "restart",
	Usage:        "restart all MinIO servers",
	Action:       mainAdminServiceRestart,
	OnUsageError: onUsageError,
	Before:       setGlobalsFromContext,
	Flags:        globalFlags,
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} TARGET

FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
EXAMPLES:
  1. Restart MinIO server represented by its alias 'play'.
     {{.Prompt}} {{.HelpName}} play/
`,
}

// serviceRestartCommand is container for service restart command success and failure messages.
type serviceRestartCommand struct {
	Status    string `json:"status"`
	ServerURL string `json:"serverURL"`
}

// String colorized service restart command message.
func (s serviceRestartCommand) String() string {
	return console.Colorize("ServiceRestart", "Restart command successfully sent to `"+s.ServerURL+"`. Type Ctrl-C or wait to see the status of the restart.")
}

// JSON jsonified service restart command message.
func (s serviceRestartCommand) JSON() string {
	serviceRestartJSONBytes, e := json.MarshalIndent(s, "", " ")
	fatalIf(probe.NewError(e), "Unable to marshal into JSON.")

	return string(serviceRestartJSONBytes)
}

// serviceRestartMessage is container for service restart success and failure messages.
type serviceRestartMessage struct {
	Status    string `json:"status"`
	ServerURL string `json:"serverURL"`
	Err       error  `json:"error,omitempty"`
}

// String colorized service restart message.
func (s serviceRestartMessage) String() string {
	if s.Err == nil {
		return console.Colorize("ServiceRestart", "\nRestarted `"+s.ServerURL+"` successfully.")
	}
	return console.Colorize("FailedServiceRestart", "Failed to restart `"+s.ServerURL+"`. error: "+s.Err.Error())
}

// JSON jsonified service restart message.
func (s serviceRestartMessage) JSON() string {
	serviceRestartJSONBytes, e := json.MarshalIndent(s, "", " ")
	fatalIf(probe.NewError(e), "Unable to marshal into JSON.")

	return string(serviceRestartJSONBytes)
}

// checkAdminServiceRestartSyntax - validate all the passed arguments
func checkAdminServiceRestartSyntax(ctx *cli.Context) {
	if len(ctx.Args()) == 0 || len(ctx.Args()) > 2 {
		cli.ShowCommandHelpAndExit(ctx, "restart", 1) // last argument is exit code
	}
}

func mainAdminServiceRestart(ctx *cli.Context) error {

	// Validate serivce restart syntax.
	checkAdminServiceRestartSyntax(ctx)

	// Set color.
	console.SetColor("ServiceOffline", color.New(color.FgRed, color.Bold))
	console.SetColor("ServiceInitializing", color.New(color.FgYellow, color.Bold))
	console.SetColor("ServiceRestart", color.New(color.FgGreen, color.Bold))
	console.SetColor("FailedServiceRestart", color.New(color.FgRed, color.Bold))

	// Get the alias parameter from cli
	args := ctx.Args()
	aliasedURL := args.Get(0)

	client, err := newAdminClient(aliasedURL)
	fatalIf(err, "Unable to initialize admin connection.")

	// Restart the specified MinIO server
	fatalIf(probe.NewError(client.ServiceRestart(globalContext)), "Unable to restart the server.")

	// Success..
	printMsg(serviceRestartCommand{Status: "success", ServerURL: aliasedURL})

	coloring := color.New(color.FgRed)
	mark := "."

	// Print restart progress
	printProgress := func() {
		if !globalQuiet && !globalJSON {
			coloring.Printf(mark)
		}
	}

	for {
		select {
		case <-globalContext.Done():
			return globalContext.Err()
		case <-time.NewTimer(3 * time.Second).C:
			ctx, cancel := context.WithTimeout(globalContext, 1*time.Second)
			// Fetch the service status of the specified MinIO server
			info, e := client.ServerInfo(ctx)
			cancel()
			switch {
			case e == nil && info.Mode == string(madmin.ItemOnline):
				printMsg(serviceRestartMessage{Status: "success", ServerURL: aliasedURL})
				return nil
			case err == nil && info.Mode == string(madmin.ItemInitializing):
				coloring = color.New(color.FgYellow)
				mark = "!"
				fallthrough
			default:
				printProgress()
			}
		}
	}
}
<commit_msg>restart: Avoid bad empty line (#3729)<commit_after>// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"context"
	"time"

	"github.com/fatih/color"
	"github.com/minio/cli"
	json "github.com/minio/colorjson"
	"github.com/minio/madmin-go"
	"github.com/minio/mc/pkg/probe"
	"github.com/minio/minio/pkg/console"
)

var adminServiceRestartCmd = cli.Command{
	Name:         "restart",
	Usage:        "restart all MinIO servers",
	Action:       mainAdminServiceRestart,
	OnUsageError: onUsageError,
	Before:       setGlobalsFromContext,
	Flags:        globalFlags,
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} TARGET

FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
EXAMPLES:
  1. Restart MinIO server represented by its alias 'play'.
     {{.Prompt}} {{.HelpName}} play/
`,
}

// serviceRestartCommand is container for service restart command success and failure messages.
type serviceRestartCommand struct {
	Status    string `json:"status"`
	ServerURL string `json:"serverURL"`
}

// String colorized service restart command message.
func (s serviceRestartCommand) String() string {
	return console.Colorize("ServiceRestart", "Restart command successfully sent to `"+s.ServerURL+"`. Type Ctrl-C to quit or wait to follow the status of the restart process.")
}

// JSON jsonified service restart command message.
func (s serviceRestartCommand) JSON() string {
	serviceRestartJSONBytes, e := json.MarshalIndent(s, "", " ")
	fatalIf(probe.NewError(e), "Unable to marshal into JSON.")

	return string(serviceRestartJSONBytes)
}

// serviceRestartMessage is container for service restart success and failure messages.
type serviceRestartMessage struct {
	Status    string `json:"status"`
	ServerURL string `json:"serverURL"`
	Err       error  `json:"error,omitempty"`
}

// String colorized service restart message.
func (s serviceRestartMessage) String() string {
	if s.Err == nil {
		return console.Colorize("ServiceRestart", "\nRestarted `"+s.ServerURL+"` successfully.")
	}
	return console.Colorize("FailedServiceRestart", "Failed to restart `"+s.ServerURL+"`. error: "+s.Err.Error())
}

// JSON jsonified service restart message.
func (s serviceRestartMessage) JSON() string {
	serviceRestartJSONBytes, e := json.MarshalIndent(s, "", " ")
	fatalIf(probe.NewError(e), "Unable to marshal into JSON.")

	return string(serviceRestartJSONBytes)
}

// checkAdminServiceRestartSyntax - validate all the passed arguments
func checkAdminServiceRestartSyntax(ctx *cli.Context) {
	if len(ctx.Args()) == 0 || len(ctx.Args()) > 2 {
		cli.ShowCommandHelpAndExit(ctx, "restart", 1) // last argument is exit code
	}
}

func mainAdminServiceRestart(ctx *cli.Context) error {

	// Validate serivce restart syntax.
	checkAdminServiceRestartSyntax(ctx)

	// Set color.
	console.SetColor("ServiceOffline", color.New(color.FgRed, color.Bold))
	console.SetColor("ServiceInitializing", color.New(color.FgYellow, color.Bold))
	console.SetColor("ServiceRestart", color.New(color.FgGreen, color.Bold))
	console.SetColor("FailedServiceRestart", color.New(color.FgRed, color.Bold))

	// Get the alias parameter from cli
	args := ctx.Args()
	aliasedURL := args.Get(0)

	client, err := newAdminClient(aliasedURL)
	fatalIf(err, "Unable to initialize admin connection.")

	// Restart the specified MinIO server
	fatalIf(probe.NewError(client.ServiceRestart(globalContext)), "Unable to restart the server.")

	// Success..
	printMsg(serviceRestartCommand{Status: "success", ServerURL: aliasedURL})

	coloring := color.New(color.FgRed)
	mark := "..."

	// Print restart progress
	printProgress := func() {
		if !globalQuiet && !globalJSON {
			coloring.Printf(mark)
		}
	}

	printProgress()
	mark = "."

	for {
		select {
		case <-globalContext.Done():
			return globalContext.Err()
		case <-time.NewTimer(3 * time.Second).C:
			ctx, cancel := context.WithTimeout(globalContext, 1*time.Second)
			// Fetch the service status of the specified MinIO server
			info, e := client.ServerInfo(ctx)
			cancel()
			switch {
			case e == nil && info.Mode == string(madmin.ItemOnline):
				printMsg(serviceRestartMessage{Status: "success", ServerURL: aliasedURL})
				return nil
			case err == nil && info.Mode == string(madmin.ItemInitializing):
				coloring = color.New(color.FgYellow)
				mark = "!"
				fallthrough
			default:
				printProgress()
			}
		}
	}
}
<|endoftext|>


<commit_before>package common

import (
	"context"
	"os"

	"github.com/moby/buildkit/util/appcontext"
	"github.com/moby/buildkit/util/tracing/detect"
	"github.com/urfave/cli"
	"go.opentelemetry.io/otel/attribute"
	"go.opentelemetry.io/otel/codes"
	"go.opentelemetry.io/otel/trace"
)

func AttachAppContext(app *cli.App) error {
	ctx := appcontext.Context()

	tp, err := detect.TracerProvider()
	if err != nil {
		return err
	}
	tracer := tp.Tracer("")

	var span trace.Span

	for i, cmd := range app.Commands {
		func(before cli.BeforeFunc) {
			name := cmd.Name
			app.Commands[i].Before = func(clicontext *cli.Context) error {
				if before != nil {
					if err := before(clicontext); err != nil {
						return err
					}
				}

				ctx, span = tracer.Start(ctx, name, trace.WithAttributes(
					attribute.StringSlice("command", os.Args),
				))

				clicontext.App.Metadata["context"] = ctx
				return nil
			}
		}(cmd.Before)
	}

	app.ExitErrHandler = func(clicontext *cli.Context, err error) {
		if span != nil {
			span.SetStatus(codes.Error, err.Error())
		}
		cli.HandleExitCoder(err)
	}

	after := app.After
	app.After = func(clicontext *cli.Context) error {
		if after != nil {
			if err := after(clicontext); err != nil {
				return err
			}
		}
		if span != nil {
			span.End()
		}
		return detect.Shutdown(context.TODO())
	}
	return nil
}

func CommandContext(c *cli.Context) context.Context {
	return c.App.Metadata["context"].(context.Context)
}
<commit_msg>Fix nil error on `buildctl debug`<commit_after>package common

import (
	"context"
	"os"

	"github.com/moby/buildkit/util/appcontext"
	"github.com/moby/buildkit/util/tracing/detect"
	"github.com/urfave/cli"
	"go.opentelemetry.io/otel/attribute"
	"go.opentelemetry.io/otel/codes"
	"go.opentelemetry.io/otel/trace"
)

func AttachAppContext(app *cli.App) error {
	ctx := appcontext.Context()

	tp, err := detect.TracerProvider()
	if err != nil {
		return err
	}
	tracer := tp.Tracer("")

	var span trace.Span

	for i, cmd := range app.Commands {
		func(before cli.BeforeFunc) {
			name := cmd.Name
			app.Commands[i].Before = func(clicontext *cli.Context) error {
				if before != nil {
					if err := before(clicontext); err != nil {
						return err
					}
				}

				ctx, span = tracer.Start(ctx, name, trace.WithAttributes(
					attribute.StringSlice("command", os.Args),
				))

				clicontext.App.Metadata["context"] = ctx
				return nil
			}
		}(cmd.Before)
	}

	app.ExitErrHandler = func(clicontext *cli.Context, err error) {
		if span != nil && err != nil {
			span.SetStatus(codes.Error, err.Error())
		}
		cli.HandleExitCoder(err)
	}

	after := app.After
	app.After = func(clicontext *cli.Context) error {
		if after != nil {
			if err := after(clicontext); err != nil {
				return err
			}
		}
		if span != nil {
			span.End()
		}
		return detect.Shutdown(context.TODO())
	}
	return nil
}

func CommandContext(c *cli.Context) context.Context {
	return c.App.Metadata["context"].(context.Context)
}
<|endoftext|>


<commit_before>// bulk_load_influx loads an InfluxDB daemon with data from stdin.
//
// The caller is responsible for assuring that the database is empty before
// bulk load.
package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/pkg/profile"
	"github.com/valyala/fasthttp"
)

// Program option vars:
var (
	csvDaemonUrls string
	daemonUrls    []string
	dbName            string
	replicationFactor int
	workers           int
	lineLimit         int64
	batchSize         int
	backoff           time.Duration
	doLoad            bool
	doDBCreate        bool
	useGzip           bool
	doAbortOnExist    bool
	memprofile        bool
)

// Global vars
var (
	bufPool        sync.Pool
	batchChan      chan *bytes.Buffer
	inputDone      chan struct{}
	workersGroup   sync.WaitGroup
	backingOffChans []chan bool
	backingOffDones []chan struct{}
)

// Parse args:
func init() {
	flag.StringVar(&csvDaemonUrls, "urls", "http://localhost:8086", "InfluxDB URLs, comma-separated. Will be used in a round-robin fashion.")
	flag.StringVar(&dbName, "db", "benchmark_db", "Database name.")
	flag.IntVar(&replicationFactor, "replication-factor", 2, "Cluster replication factor (only applies to clustered databases).")
	flag.IntVar(&batchSize, "batch-size", 5000, "Batch size (input lines).")
	flag.IntVar(&workers, "workers", 1, "Number of parallel requests to make.")
	flag.Int64Var(&lineLimit, "line-limit", -1, "Number of lines to read from stdin before quitting.")
	flag.DurationVar(&backoff, "backoff", time.Second, "Time to sleep between requests when server indicates backpressure is needed.")
	flag.BoolVar(&useGzip, "gzip", false, "Whether to gzip encode requests.")
	flag.BoolVar(&doLoad, "do-load", true, "Whether to write data. Set this flag to false to check input read speed.")
	flag.BoolVar(&doDBCreate, "do-db-create", true, "Whether to create the database.")
	flag.BoolVar(&doAbortOnExist, "do-abort-on-exist", true, "Whether to abort if the destination database already exists.")
	flag.BoolVar(&memprofile, "memprofile", false, "Whether to write a memprofile (file automatically determined).")

	flag.Parse()

	daemonUrls = strings.Split(csvDaemonUrls, ",")
	if len(daemonUrls) == 0 {
		log.Fatal("missing 'urls' flag")
	}
	fmt.Printf("daemon URLs: %v\n", daemonUrls)
}

func main() {
	if memprofile {
		p := profile.Start(profile.MemProfile)
		defer p.Stop()
	}
	if doLoad && doDBCreate {
		// check that there are no pre-existing databases:
		existingDatabases, err := listDatabases(daemonUrls[0])
		if err != nil {
			log.Fatal(err)
		}

		if len(existingDatabases) > 0 {
			if doAbortOnExist {
				log.Fatalf("There are databases already in the data store. If you know what you are doing, run the command:\ncurl 'http://localhost:8086/query?q=drop%%20database%%20%s'\n", existingDatabases[0])
			} else {
				log.Printf("Info: there are databases already in the data store.")
			}
		}

		if len(existingDatabases) == 0 {
			err = createDb(daemonUrls[0], dbName, replicationFactor)
			if err != nil {
				log.Fatal(err)
			}
			time.Sleep(2000*time.Millisecond)
		}
	}

	bufPool = sync.Pool{
		New: func() interface{} {
			return bytes.NewBuffer(make([]byte, 0, 4*1024*1024))
		},
	}

	batchChan = make(chan *bytes.Buffer, workers)
	inputDone = make(chan struct{})

	backingOffChans = make([]chan bool, workers)
	backingOffDones = make([]chan struct{}, workers)

	for i := 0; i < workers; i++ {
		daemonUrl := daemonUrls[i%len(daemonUrls)]
		backingOffChans[i] = make(chan bool, 100)
		backingOffDones[i] = make(chan struct{})
		workersGroup.Add(1)
		cfg := HTTPWriterConfig{
			DebugInfo: fmt.Sprintf("worker #%d, dest url: %s", i, daemonUrl),
			Host:      daemonUrl,
			Database:  dbName,
			BackingOffChan: backingOffChans[i],
			BackingOffDone: backingOffDones[i],
		}
		go processBatches(NewHTTPWriter(cfg), backingOffChans[i], backingOffDones[i])
		go processBackoffMessages(i, backingOffChans[i], backingOffDones[i])
	}

	start := time.Now()
	itemsRead := scan(batchSize)

	<-inputDone
	close(batchChan)

	workersGroup.Wait()

	for i := range backingOffChans {
		close(backingOffChans[i])
		<-backingOffDones[i]
	}

	end := time.Now()
	took := end.Sub(start)
	rate := float64(itemsRead) / float64(took.Seconds())

	fmt.Printf("loaded %d items in %fsec with %d workers (mean rate %f/sec)\n", itemsRead, took.Seconds(), workers, rate)
}

// scan reads one line at a time from stdin.
// When the requested number of lines per batch is met, send a batch over batchChan for the workers to write.
func scan(linesPerBatch int) int64 {
	buf := bufPool.Get().(*bytes.Buffer)

	var n int
	var itemsRead int64
	newline := []byte("\n")

	scanner := bufio.NewScanner(bufio.NewReaderSize(os.Stdin, 4*1024*1024))
	for scanner.Scan() {
		if itemsRead == lineLimit {
			break
		}

		itemsRead++

		buf.Write(scanner.Bytes())
		buf.Write(newline)

		n++
		if n >= linesPerBatch {
			batchChan <- buf
			buf = bufPool.Get().(*bytes.Buffer)
			n = 0
		}
	}

	if err := scanner.Err(); err != nil {
		log.Fatalf("Error reading input: %s", err.Error())
	}

	// Finished reading input, make sure last batch goes out.
	if n > 0 {
		batchChan <- buf
	}

	// Closing inputDone signals to the application that we've read everything and can now shut down.
	close(inputDone)

	return itemsRead
}

// processBatches reads byte buffers from batchChan and writes them to the target server, while tracking stats on the write.
func processBatches(w *HTTPWriter, backoffSrc chan bool, backoffDst chan struct{}) {
	for batch := range batchChan {
		// Write the batch: try until backoff is not needed.
		if doLoad {
			var err error
			for {
				if useGzip {
					compressedBatch := bufPool.Get().(*bytes.Buffer)
					fasthttp.WriteGzip(compressedBatch, batch.Bytes())
					_, err = w.WriteLineProtocol(compressedBatch.Bytes(), true)
					// Return the compressed batch buffer to the pool.
					compressedBatch.Reset()
					bufPool.Put(compressedBatch)
				} else {
					_, err = w.WriteLineProtocol(batch.Bytes(), false)
				}

				if err == BackoffError {
					backoffSrc <- true
					time.Sleep(backoff)
				} else {
					backoffSrc <- false
					break
				}
			}
			if err != nil {
				log.Fatalf("Error writing: %s\n", err.Error())
			}
		}

		// Return the batch buffer to the pool.
		batch.Reset()
		bufPool.Put(batch)
	}
	workersGroup.Done()
}

func processBackoffMessages(workerId int, src chan bool, dst chan struct{}) {
	var totalBackoffSecs float64
	var start time.Time
	last := false
	for this := range src {
		if this && !last {
			start = time.Now()
			last = true
		} else if !this && last {
			took := time.Now().Sub(start)
			fmt.Printf("[worker %d] backoff took %.02fsec\n", workerId, took.Seconds())
			totalBackoffSecs += took.Seconds()
			last = false
			start = time.Now()
		}
	}
	fmt.Printf("[worker %d] backoffs took a total of %fsec of runtime\n", workerId, totalBackoffSecs)
	dst <- struct{}{}
}

func createDb(daemon_url, dbname string, replicationFactor int) error {
	u, err := url.Parse(daemon_url)
	if err != nil {
		return err
	}

	// serialize params the right way:
	u.Path = "query"
	v := u.Query()
	v.Set("consistency", "all")
	v.Set("q", fmt.Sprintf("CREATE DATABASE %s WITH REPLICATION %d", dbname, replicationFactor))
	u.RawQuery = v.Encode()

	req, err := http.NewRequest("GET", u.String(), nil)
	if err != nil {
		return err
	}

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	// does the body need to be read into the void?

	if resp.StatusCode != 200 {
		return fmt.Errorf("bad db create")
	}
	return nil
}

// listDatabases lists the existing databases in InfluxDB.
func listDatabases(daemonUrl string) ([]string, error) {
	u := fmt.Sprintf("%s/query?q=show%%20databases", daemonUrl)
	resp, err := http.Get(u)
	if err != nil {
		return nil, fmt.Errorf("listDatabases error: %s", err.Error())
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	// Do ad-hoc parsing to find existing database names:
	// {"results":[{"series":[{"name":"databases","columns":["name"],"values":[["_internal"],["benchmark_db"]]}]}]}%
	type listingType struct {
		Results []struct {
			Series []struct {
				Values [][]string
			}
		}
	}
	var listing listingType
	err = json.Unmarshal(body, &listing)
	if err != nil {
		return nil, err
	}

	ret := []string{}
	for _, nestedName := range listing.Results[0].Series[0].Values {
		name := nestedName[0]
		// the _internal database is skipped:
		if name == "_internal" {
			continue
		}
		ret = append(ret, name)
	}
	return ret, nil
}
<commit_msg>support bulk load time limits<commit_after>// bulk_load_influx loads an InfluxDB daemon with data from stdin.
//
// The caller is responsible for assuring that the database is empty before
// bulk load.
package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/pkg/profile"
	"github.com/valyala/fasthttp"
)

// Program option vars:
var (
	csvDaemonUrls     string
	daemonUrls        []string
	dbName            string
	replicationFactor int
	workers           int
	lineLimit         int64
	batchSize         int
	backoff           time.Duration
	timeLimit         time.Duration
	doLoad            bool
	doDBCreate        bool
	useGzip           bool
	doAbortOnExist    bool
	memprofile        bool
)

// Global vars
var (
	bufPool         sync.Pool
	batchChan       chan *bytes.Buffer
	inputDone       chan struct{}
	workersGroup    sync.WaitGroup
	backingOffChans []chan bool
	backingOffDones []chan struct{}
)

// Parse args:
func init() {
	flag.StringVar(&csvDaemonUrls, "urls", "http://localhost:8086", "InfluxDB URLs, comma-separated. Will be used in a round-robin fashion.")
	flag.StringVar(&dbName, "db", "benchmark_db", "Database name.")
	flag.IntVar(&replicationFactor, "replication-factor", 2, "Cluster replication factor (only applies to clustered databases).")
	flag.IntVar(&batchSize, "batch-size", 5000, "Batch size (input lines).")
	flag.IntVar(&workers, "workers", 1, "Number of parallel requests to make.")
	flag.Int64Var(&lineLimit, "line-limit", -1, "Number of lines to read from stdin before quitting.")
	flag.DurationVar(&backoff, "backoff", time.Second, "Time to sleep between requests when server indicates backpressure is needed.")
	flag.DurationVar(&timeLimit, "time-limit", -1, "Maximum duration to run (-1 is the default: no limit).")
	flag.BoolVar(&useGzip, "gzip", false, "Whether to gzip encode requests.")
	flag.BoolVar(&doLoad, "do-load", true, "Whether to write data. Set this flag to false to check input read speed.")
	flag.BoolVar(&doDBCreate, "do-db-create", true, "Whether to create the database.")
	flag.BoolVar(&doAbortOnExist, "do-abort-on-exist", true, "Whether to abort if the destination database already exists.")
	flag.BoolVar(&memprofile, "memprofile", false, "Whether to write a memprofile (file automatically determined).")

	flag.Parse()

	daemonUrls = strings.Split(csvDaemonUrls, ",")
	if len(daemonUrls) == 0 {
		log.Fatal("missing 'urls' flag")
	}
	fmt.Printf("daemon URLs: %v\n", daemonUrls)
}

func main() {
	if memprofile {
		p := profile.Start(profile.MemProfile)
		defer p.Stop()
	}
	if doLoad && doDBCreate {
		// check that there are no pre-existing databases:
		existingDatabases, err := listDatabases(daemonUrls[0])
		if err != nil {
			log.Fatal(err)
		}

		if len(existingDatabases) > 0 {
			if doAbortOnExist {
				log.Fatalf("There are databases already in the data store. If you know what you are doing, run the command:\ncurl 'http://localhost:8086/query?q=drop%%20database%%20%s'\n", existingDatabases[0])
			} else {
				log.Printf("Info: there are databases already in the data store.")
			}
		}

		if len(existingDatabases) == 0 {
			err = createDb(daemonUrls[0], dbName, replicationFactor)
			if err != nil {
				log.Fatal(err)
			}
			time.Sleep(1000 * time.Millisecond)
		}
	}

	bufPool = sync.Pool{
		New: func() interface{} {
			return bytes.NewBuffer(make([]byte, 0, 4*1024*1024))
		},
	}

	batchChan = make(chan *bytes.Buffer, workers)
	inputDone = make(chan struct{})

	backingOffChans = make([]chan bool, workers)
	backingOffDones = make([]chan struct{}, workers)

	for i := 0; i < workers; i++ {
		daemonUrl := daemonUrls[i%len(daemonUrls)]
		backingOffChans[i] = make(chan bool, 100)
		backingOffDones[i] = make(chan struct{})
		workersGroup.Add(1)
		cfg := HTTPWriterConfig{
			DebugInfo:      fmt.Sprintf("worker #%d, dest url: %s", i, daemonUrl),
			Host:           daemonUrl,
			Database:       dbName,
			BackingOffChan: backingOffChans[i],
			BackingOffDone: backingOffDones[i],
		}
		go processBatches(NewHTTPWriter(cfg), backingOffChans[i], backingOffDones[i])
		go processBackoffMessages(i, backingOffChans[i], backingOffDones[i])
	}

	start := time.Now()
	itemsRead := scan(batchSize)

	<-inputDone
	close(batchChan)

	workersGroup.Wait()

	for i := range backingOffChans {
		close(backingOffChans[i])
		<-backingOffDones[i]
	}

	end := time.Now()
	took := end.Sub(start)
	rate := float64(itemsRead) / float64(took.Seconds())

	fmt.Printf("loaded %d items in %fsec with %d workers (mean rate %f/sec)\n", itemsRead, took.Seconds(), workers, rate)
}

// scan reads one line at a time from stdin.
// When the requested number of lines per batch is met, send a batch over batchChan for the workers to write.
func scan(linesPerBatch int) int64 {
	buf := bufPool.Get().(*bytes.Buffer)

	var n int
	var itemsRead int64
	newline := []byte("\n")
	var deadline time.Time
	if timeLimit >= 0 {
		deadline = time.Now().Add(timeLimit)
	}

	scanner := bufio.NewScanner(bufio.NewReaderSize(os.Stdin, 4*1024*1024))
outer:
	for scanner.Scan() {
		if itemsRead == lineLimit {
			break
		}

		itemsRead++

		buf.Write(scanner.Bytes())
		buf.Write(newline)

		n++
		if n >= linesPerBatch {
			if timeLimit >= 0 && time.Now().After(deadline) {
				break outer
			}
			batchChan <- buf
			buf = bufPool.Get().(*bytes.Buffer)
			n = 0
		}
	}

	if err := scanner.Err(); err != nil {
		log.Fatalf("Error reading input: %s", err.Error())
	}

	// Finished reading input, make sure last batch goes out.
	if n > 0 {
		batchChan <- buf
	}

	// Closing inputDone signals to the application that we've read everything and can now shut down.
	close(inputDone)

	return itemsRead
}

// processBatches reads byte buffers from batchChan and writes them to the target server, while tracking stats on the write.
func processBatches(w *HTTPWriter, backoffSrc chan bool, backoffDst chan struct{}) {
	for batch := range batchChan {
		// Write the batch: try until backoff is not needed.
		if doLoad {
			var err error
			for {
				if useGzip {
					compressedBatch := bufPool.Get().(*bytes.Buffer)
					fasthttp.WriteGzip(compressedBatch, batch.Bytes())
					_, err = w.WriteLineProtocol(compressedBatch.Bytes(), true)
					// Return the compressed batch buffer to the pool.
					compressedBatch.Reset()
					bufPool.Put(compressedBatch)
				} else {
					_, err = w.WriteLineProtocol(batch.Bytes(), false)
				}

				if err == BackoffError {
					backoffSrc <- true
					time.Sleep(backoff)
				} else {
					backoffSrc <- false
					break
				}
			}
			if err != nil {
				log.Fatalf("Error writing: %s\n", err.Error())
			}
		}

		// Return the batch buffer to the pool.
		batch.Reset()
		bufPool.Put(batch)
	}
	workersGroup.Done()
}

func processBackoffMessages(workerId int, src chan bool, dst chan struct{}) {
	var totalBackoffSecs float64
	var start time.Time
	last := false
	for this := range src {
		if this && !last {
			start = time.Now()
			last = true
		} else if !this && last {
			took := time.Now().Sub(start)
			fmt.Printf("[worker %d] backoff took %.02fsec\n", workerId, took.Seconds())
			totalBackoffSecs += took.Seconds()
			last = false
			start = time.Now()
		}
	}
	fmt.Printf("[worker %d] backoffs took a total of %fsec of runtime\n", workerId, totalBackoffSecs)
	dst <- struct{}{}
}

func createDb(daemon_url, dbname string, replicationFactor int) error {
	u, err := url.Parse(daemon_url)
	if err != nil {
		return err
	}

	// serialize params the right way:
	u.Path = "query"
	v := u.Query()
	v.Set("consistency", "all")
	v.Set("q", fmt.Sprintf("CREATE DATABASE %s WITH REPLICATION %d", dbname, replicationFactor))
	u.RawQuery = v.Encode()

	req, err := http.NewRequest("GET", u.String(), nil)
	if err != nil {
		return err
	}

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	// does the body need to be read into the void?

	if resp.StatusCode != 200 {
		return fmt.Errorf("bad db create")
	}
	return nil
}

// listDatabases lists the existing databases in InfluxDB.
func listDatabases(daemonUrl string) ([]string, error) {
	u := fmt.Sprintf("%s/query?q=show%%20databases", daemonUrl)
	resp, err := http.Get(u)
	if err != nil {
		return nil, fmt.Errorf("listDatabases error: %s", err.Error())
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	// Do ad-hoc parsing to find existing database names:
	// {"results":[{"series":[{"name":"databases","columns":["name"],"values":[["_internal"],["benchmark_db"]]}]}]}%
	type listingType struct {
		Results []struct {
			Series []struct {
				Values [][]string
			}
		}
	}
	var listing listingType
	err = json.Unmarshal(body, &listing)
	if err != nil {
		return nil, err
	}

	ret := []string{}
	for _, nestedName := range listing.Results[0].Series[0].Values {
		name := nestedName[0]
		// the _internal database is skipped:
		if name == "_internal" {
			continue
		}
		ret = append(ret, name)
	}
	return ret, nil
}
<|endoftext|>


<commit_before>package playtak

import (
	"context"
	"flag"
	"log"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/google/subcommands"
	"github.com/nelhage/taktician/playtak"
	"github.com/nelhage/taktician/playtak/bot"
)

type Command struct {
	server    string
	user      string
	pass      string
	accept    string
	observe   string
	gameTime  time.Duration
	increment time.Duration
	size      int
	once      bool
	takbot    string

	friendly bool
	fpa      string
	logFile  string

	debug           int
	depth           int
	multicut        bool
	limit           time.Duration
	sort            bool
	tableMem        int64
	useOpponentTime bool

	book bool

	debugClient bool
}

func (*Command) Name() string     { return "playtak" }
func (*Command) Synopsis() string { return "Play Tak on playtak.com using the Taktician AI" }
func (*Command) Usage() string {
	return `playtak [flags]
`
}

func (c *Command) SetFlags(flags *flag.FlagSet) {
	flags.StringVar(&c.server, "server", "playtak.com:10000", "playtak.com server to connect to")
	flags.StringVar(&c.user, "user", "", "username for login")
	flags.StringVar(&c.pass, "pass", "", "password for login")
	flags.StringVar(&c.accept, "accept", "", "accept a game from specified user")
	flags.StringVar(&c.observe, "observe", "", "observe a game by a specified user")
	flags.DurationVar(&c.gameTime, "time", 20*time.Minute, "Length of game to offer")
	flags.DurationVar(&c.increment, "increment", 0, "time increment to offer")
	flags.IntVar(&c.size, "size", 5, "size of game to offer")
	flags.BoolVar(&c.once, "once", false, "play a single game and exit")
	flags.StringVar(&c.takbot, "takbot", "", "challenge TakBot AI")

	flags.BoolVar(&c.friendly, "friendly", false, "play as FriendlyBot")
	flags.StringVar(&c.fpa, "fpa", "", "select an alternate FPA rule set")
	flags.StringVar(&c.logFile, "log-file", "", "Log friendly/FPA games")

	flags.IntVar(&c.debug, "debug", 1, "debug level")
	flags.IntVar(&c.depth, "depth", 0, "minimax depth")
	flags.BoolVar(&c.multicut, "multi-cut", false, "use multi-cut")
	flags.DurationVar(&c.limit, "limit", time.Minute, "time limit per move")
	flags.BoolVar(&c.sort, "sort", true, "sort moves via history heuristic")
	flags.Int64Var(&c.tableMem, "table-mem", 0, "set table size")
	flags.BoolVar(&c.useOpponentTime, "use-opponent-time", true, "think on opponent's time")

	flags.BoolVar(&c.book, "book", true, "use built-in opening book")

	flags.BoolVar(&c.debugClient, "debug-client", false, "log debug output for playtak connection")
}

const ClientName = "Taktician AI"

func (c *Command) Execute(ctx context.Context, flag *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
	if c.accept != "" || c.takbot != "" || c.observe != "" {
		c.once = true
	}
	var fpaRuleset FPARule
	if c.fpa != "" {
		c.friendly = true
		switch c.fpa {
		case "true", "center":
			fpaRuleset = &CenterBlack{}
		case "doublestack":
			fpaRuleset = &DoubleStack{}
		default:
			log.Fatalf("Unknown FPA ruleset: %s", c.fpa)
		}
	}

	sigs := make(chan os.Signal, 1)
	if c.observe == "" {
		signal.Notify(sigs, syscall.SIGTERM, syscall.SIGINT)
	}

	backoff := 1 * time.Second
	var b bot.Bot
	for {
		var client *playtak.Commands
		cl, err := playtak.Dial(c.debugClient, c.server)
		if err != nil {
			goto reconnect
		}
		backoff = time.Second
		client = &playtak.Commands{"", cl}
		client.SendClient(ClientName)
		if c.user != "" {
			err = client.Login(c.user, c.pass)
		} else {
			err = client.LoginGuest()
		}
		if err != nil {
			log.Fatal("login: ", err)
		}
		log.Printf("login OK")
		if c.friendly {
			b = &Friendly{
				cmd:     c,
				logFile: c.logFile,
				client:  client,
				fpa:     fpaRuleset,
			}
		} else {
			b = &Taktician{cmd: c, client: client}
		}
		for {
			if c.accept == "" && c.observe == "" {
				client.SendCommand("Seek",
					strconv.Itoa(c.size),
					strconv.Itoa(int(c.gameTime.Seconds())),
					strconv.Itoa(int(c.increment.Seconds())))
				log.Printf("Seek OK")
				if c.takbot != "" {
					client.SendCommand("Shout", "takbot: play", c.takbot)
				}
			}

		recvLoop:
			for {
				select {
				case line, ok := <-client.Recv():
					if !ok {
						break recvLoop
					}
					switch {
					case strings.HasPrefix(line, "Seek new"):
						bits := strings.Split(line, " ")
						if bits[3] == c.accept {
							log.Printf("accepting game %s from %s", bits[2], bits[3])
							client.SendCommand("Accept", bits[2])
						}
					case strings.HasPrefix(line, "Game Start"):
						bot.PlayGame(client, b, line)
						time.Sleep(100 * time.Millisecond)
						break recvLoop
					case strings.HasPrefix(line, "Shout"):
						who, msg := playtak.ParseShout(line)
						if who != "" {
							b.HandleChat("", who, msg)
						}
					case strings.HasPrefix(line, "GameList Add"):
						bits := strings.Split(line, " ")
						white := bits[3]
						black := strings.TrimRight(bits[5], ",")
						if white == c.observe || black == c.observe {
							client.SendCommand("Observe", bits[2][len("Game#"):])
						}
					case strings.HasPrefix(line, "Observe"):
						bot.ObserveGame(client, b, line)
						break recvLoop
					}
				case <-sigs:
					return subcommands.ExitSuccess
				}
			}
			if c.once {
				return subcommands.ExitSuccess
			}
			if client.Error() != nil {
				log.Printf("Disconnected: %v", client.Error())
				break
			}
		}
	reconnect:
		log.Printf("sleeping %s before reconnect...", backoff)
		select {
		case <-time.After(backoff):
		case <-sigs:
			return subcommands.ExitSuccess
		}
		backoff = backoff * 2
		if backoff > time.Minute {
			backoff = time.Minute
		}
	}
}
<commit_msg>-fpa cairn<commit_after>package playtak

import (
	"context"
	"flag"
	"log"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/google/subcommands"
	"github.com/nelhage/taktician/playtak"
	"github.com/nelhage/taktician/playtak/bot"
)

type Command struct {
	server    string
	user      string
	pass      string
	accept    string
	observe   string
	gameTime  time.Duration
	increment time.Duration
	size      int
	once      bool
	takbot    string

	friendly bool
	fpa      string
	logFile  string

	debug           int
	depth           int
	multicut        bool
	limit           time.Duration
	sort            bool
	tableMem        int64
	useOpponentTime bool

	book bool

	debugClient bool
}

func (*Command) Name() string     { return "playtak" }
func (*Command) Synopsis() string { return "Play Tak on playtak.com using the Taktician AI" }
func (*Command) Usage() string {
	return `playtak [flags]
`
}

func (c *Command) SetFlags(flags *flag.FlagSet) {
	flags.StringVar(&c.server, "server", "playtak.com:10000", "playtak.com server to connect to")
	flags.StringVar(&c.user, "user", "", "username for login")
	flags.StringVar(&c.pass, "pass", "", "password for login")
	flags.StringVar(&c.accept, "accept", "", "accept a game from specified user")
	flags.StringVar(&c.observe, "observe", "", "observe a game by a specified user")
	flags.DurationVar(&c.gameTime, "time", 20*time.Minute, "Length of game to offer")
	flags.DurationVar(&c.increment, "increment", 0, "time increment to offer")
	flags.IntVar(&c.size, "size", 5, "size of game to offer")
	flags.BoolVar(&c.once, "once", false, "play a single game and exit")
	flags.StringVar(&c.takbot, "takbot", "", "challenge TakBot AI")

	flags.BoolVar(&c.friendly, "friendly", false, "play as FriendlyBot")
	flags.StringVar(&c.fpa, "fpa", "", "select an alternate FPA rule set")
	flags.StringVar(&c.logFile, "log-file", "", "Log friendly/FPA games")

	flags.IntVar(&c.debug, "debug", 1, "debug level")
	flags.IntVar(&c.depth, "depth", 0, "minimax depth")
	flags.BoolVar(&c.multicut, "multi-cut", false, "use multi-cut")
	flags.DurationVar(&c.limit, "limit", time.Minute, "time limit per move")
	flags.BoolVar(&c.sort, "sort", true, "sort moves via history heuristic")
	flags.Int64Var(&c.tableMem, "table-mem", 0, "set table size")
	flags.BoolVar(&c.useOpponentTime, "use-opponent-time", true, "think on opponent's time")

	flags.BoolVar(&c.book, "book", true, "use built-in opening book")

	flags.BoolVar(&c.debugClient, "debug-client", false, "log debug output for playtak connection")
}

const ClientName = "Taktician AI"

func (c *Command) Execute(ctx context.Context, flag *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
	if c.accept != "" || c.takbot != "" || c.observe != "" {
		c.once = true
	}
	var fpaRuleset FPARule
	if c.fpa != "" {
		c.friendly = true
		switch c.fpa {
		case "true", "center":
			fpaRuleset = &CenterBlack{}
		case "doublestack":
			fpaRuleset = &DoubleStack{}
		case "cairn":
			fpaRuleset = &Cairn{}
		default:
			log.Fatalf("Unknown FPA ruleset: %s", c.fpa)
		}
	}

	sigs := make(chan os.Signal, 1)
	if c.observe == "" {
		signal.Notify(sigs, syscall.SIGTERM, syscall.SIGINT)
	}

	backoff := 1 * time.Second
	var b bot.Bot
	for {
		var client *playtak.Commands
		cl, err := playtak.Dial(c.debugClient, c.server)
		if err != nil {
			goto reconnect
		}
		backoff = time.Second
		client = &playtak.Commands{"", cl}
		client.SendClient(ClientName)
		if c.user != "" {
			err = client.Login(c.user, c.pass)
		} else {
			err = client.LoginGuest()
		}
		if err != nil {
			log.Fatal("login: ", err)
		}
		log.Printf("login OK")
		if c.friendly {
			b = &Friendly{
				cmd:     c,
				logFile: c.logFile,
				client:  client,
				fpa:     fpaRuleset,
			}
		} else {
			b = &Taktician{cmd: c, client: client}
		}
		for {
			if c.accept == "" && c.observe == "" {
				client.SendCommand("Seek",
					strconv.Itoa(c.size),
					strconv.Itoa(int(c.gameTime.Seconds())),
					strconv.Itoa(int(c.increment.Seconds())))
				log.Printf("Seek OK")
				if c.takbot != "" {
					client.SendCommand("Shout", "takbot: play", c.takbot)
				}
			}

		recvLoop:
			for {
				select {
				case line, ok := <-client.Recv():
					if !ok {
						break recvLoop
					}
					switch {
					case strings.HasPrefix(line, "Seek new"):
						bits := strings.Split(line, " ")
						if bits[3] == c.accept {
							log.Printf("accepting game %s from %s", bits[2], bits[3])
							client.SendCommand("Accept", bits[2])
						}
					case strings.HasPrefix(line, "Game Start"):
						bot.PlayGame(client, b, line)
						time.Sleep(100 * time.Millisecond)
						break recvLoop
					case strings.HasPrefix(line, "Shout"):
						who, msg := playtak.ParseShout(line)
						if who != "" {
							b.HandleChat("", who, msg)
						}
					case strings.HasPrefix(line, "GameList Add"):
						bits := strings.Split(line, " ")
						white := bits[3]
						black := strings.TrimRight(bits[5], ",")
						if white == c.observe || black == c.observe {
							client.SendCommand("Observe", bits[2][len("Game#"):])
						}
					case strings.HasPrefix(line, "Observe"):
						bot.ObserveGame(client, b, line)
						break recvLoop
					}
				case <-sigs:
					return subcommands.ExitSuccess
				}
			}
			if c.once {
				return subcommands.ExitSuccess
			}
			if client.Error() != nil {
				log.Printf("Disconnected: %v", client.Error())
				break
			}
		}
	reconnect:
		log.Printf("sleeping %s before reconnect...", backoff)
		select {
		case <-time.After(backoff):
		case <-sigs:
			return subcommands.ExitSuccess
		}
		backoff = backoff * 2
		if backoff > time.Minute {
			backoff = time.Minute
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package memfs

import (
	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/syncutil"
)

// Common attributes for files and directories.
//
// TODO(jacobsa): Add tests for interacting with a file/directory after it has
// been unlinked, including creating a new file. Make sure we don't screw up
// and reuse an inode ID while it is still in use.
type inode struct {
	/////////////////////////
	// Constant data
	/////////////////////////

	// Is this a directory? If not, it is a file.
	dir bool

	/////////////////////////
	// Mutable state
	/////////////////////////

	mu syncutil.InvariantMutex

	// The current attributes of this inode.
	//
	// INVARIANT: No non-permission mode bits are set besides os.ModeDir
	// INVARIANT: If dir, then os.ModeDir is set
	// INVARIANT: If !dir, then os.ModeDir is not set
	attributes fuse.InodeAttributes // GUARDED_BY(mu)

	// For directories, entries describing the children of the directory.
	//
	// This array can never be shortened, nor can its elements be moved, because
	// we use its indices for Dirent.Offset, which is exposed to the user who
	// might be calling readdir in a loop while concurrently modifying the
	// directory. Unused entries can, however, be reused.
	//
	// TODO(jacobsa): Add good tests exercising concurrent modifications while
	// doing readdir, seekdir, etc. calls.
	//
	// INVARIANT: If dir is false, this is nil.
	// INVARIANT: For each i, entries[i].Offset == i+1
	entries []fuseutil.Dirent // GUARDED_BY(mu)

	// For files, the current contents of the file.
	//
	// INVARIANT: If dir is true, this is nil.
	contents []byte // GUARDED_BY(mu)
}

func newInode(dir bool) (in *inode) {
	in = &inode{
		dir: dir,
	}

	in.mu = syncutil.NewInvariantMutex(in.checkInvariants)
	return
}

func (inode *inode) checkInvariants()

// Find an entry for the given child name and return its inode ID.
//
// REQUIRES: inode.dir
// SHARED_LOCKS_REQUIRED(inode.mu)
func (inode *inode) LookUpChild(name string) (id fuse.InodeID, ok bool) {
	if !inode.dir {
		panic("LookUpChild called on non-directory.")
	}

	panic("TODO")
}

// Serve a ReadDir request.
//
// REQUIRED: inode.dir
// SHARED_LOCKS_REQUIRED(inode.mu)
func (inode *inode) ReadDir(offset int, size int) (data []byte, err error) {
	if !inode.dir {
		panic("ReadDir called on non-directory.")
	}

	for i := offset; i < len(inode.entries); i++ {
		data = fuseutil.AppendDirent(data, inode.entries[i])

		// Trim and stop early if we've exceeded the requested size.
		if len(data) > size {
			data = data[:size]
			break
		}
	}

	return
}
<commit_msg>Finished inode.LookUpChild.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package memfs

import (
	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/syncutil"
)

// Common attributes for files and directories.
//
// TODO(jacobsa): Add tests for interacting with a file/directory after it has
// been unlinked, including creating a new file. Make sure we don't screw up
// and reuse an inode ID while it is still in use.
type inode struct {
	/////////////////////////
	// Constant data
	/////////////////////////

	// Is this a directory? If not, it is a file.
	dir bool

	/////////////////////////
	// Mutable state
	/////////////////////////

	mu syncutil.InvariantMutex

	// The current attributes of this inode.
	//
	// INVARIANT: No non-permission mode bits are set besides os.ModeDir
	// INVARIANT: If dir, then os.ModeDir is set
	// INVARIANT: If !dir, then os.ModeDir is not set
	attributes fuse.InodeAttributes // GUARDED_BY(mu)

	// For directories, entries describing the children of the directory.
	//
	// This array can never be shortened, nor can its elements be moved, because
	// we use its indices for Dirent.Offset, which is exposed to the user who
	// might be calling readdir in a loop while concurrently modifying the
	// directory. Unused entries can, however, be reused.
	//
	// TODO(jacobsa): Add good tests exercising concurrent modifications while
	// doing readdir, seekdir, etc. calls.
	//
	// INVARIANT: If dir is false, this is nil.
	// INVARIANT: For each i, entries[i].Offset == i+1
	entries []fuseutil.Dirent // GUARDED_BY(mu)

	// For files, the current contents of the file.
	//
	// INVARIANT: If dir is true, this is nil.
	contents []byte // GUARDED_BY(mu)
}

func newInode(dir bool) (in *inode) {
	in = &inode{
		dir: dir,
	}

	in.mu = syncutil.NewInvariantMutex(in.checkInvariants)
	return
}

func (inode *inode) checkInvariants()

// Find an entry for the given child name and return its inode ID.
//
// REQUIRES: inode.dir
// SHARED_LOCKS_REQUIRED(inode.mu)
func (inode *inode) LookUpChild(name string) (id fuse.InodeID, ok bool) {
	if !inode.dir {
		panic("LookUpChild called on non-directory.")
	}

	for _, e := range inode.entries {
		if e.Name == name {
			id = e.Inode
			ok = true
			return
		}
	}

	return
}

// Serve a ReadDir request.
//
// REQUIRED: inode.dir
// SHARED_LOCKS_REQUIRED(inode.mu)
func (inode *inode) ReadDir(offset int, size int) (data []byte, err error) {
	if !inode.dir {
		panic("ReadDir called on non-directory.")
	}

	for i := offset; i < len(inode.entries); i++ {
		data = fuseutil.AppendDirent(data, inode.entries[i])

		// Trim and stop early if we've exceeded the requested size.
		if len(data) > size {
			data = data[:size]
			break
		}
	}

	return
}
<|endoftext|>


<commit_before>// Copyright 2013, Cong Ding. All rights reserved.
// Use of this source code is governed by a GPLv2
// license that can be found in the LICENSE file.
//
// author: Cong Ding <dinggnu@gmail.com>
package logging

import (
	"runtime"
	"strconv"
	"sync/atomic"
	"time"
)

type field func(*logging) string

var fields = map[string]field{
	"seqid":           (*logging).seqid,
	"name":            (*logging).name,
	"levelno":         (*logging).levelno,
	"levelname":       (*logging).levelname,
	"pathname":        (*logging).pathname,
	"filename":        (*logging).filename,
	"module":          (*logging).module,
	"lineno":          (*logging).lineno,
	"funcName":        (*logging).funcName,
	"created":         (*logging).created,
	"asctime":         (*logging).asctime,
	"msecs":           (*logging).msecs,
	"relativeCreated": (*logging).relativeCreated,
	"thread":          (*logging).thread,
	"threadName":      (*logging).threadName,
	"process":         (*logging).process,
	"message":         (*logging).message,
}

// GetGoId returns the id of goroutine, which is defined in ./get_go_id.c
func GetGoId() int32

func init() {
}

func (logger *logging) seqid() string {
	return strconv.FormatUint(atomic.AddUint64(&(logger.Seqid), 1), 10)
}

func (logger *logging) name() string {
	return logger.Name
}

func (logger *logging) levelno() string {
	return strconv.Itoa(int(logger.Level))
}

func (logger *logging) levelname() string {
	return levelNames[logger.Level]
}

func (logger *logging) pathname() string {
	calldepth := 5
	_, file, _, ok := runtime.Caller(calldepth)
	if !ok {
		file = "???"
	}
	return file
}

func (logger *logging) filename() string {
	calldepth := 5
	_, file, _, ok := runtime.Caller(calldepth)
	if !ok {
		file = "???"
	}
	short := file
	for i := len(file) - 1; i > 0; i-- {
		if file[i] == '/' {
			short = file[i+1:]
			break
		}
	}
	file = short
	return file
}

func (logger *logging) module() string {
	return ""
}

func (logger *logging) lineno() string {
	calldepth := 5
	_, _, line, ok := runtime.Caller(calldepth)
	if !ok {
		line = 0
	}
	return strconv.Itoa(line)
}

func (logger *logging) funcName() string {
	calldepth := 5
	pc, _, _, ok := runtime.Caller(calldepth)
	if !ok {
		return "???"
	}
	return runtime.FuncForPC(pc).Name()
}

func (logger *logging) created() string {
	return strconv.FormatInt(logger.startTime, 10)
}

func (logger *logging) asctime() string {
	return time.Now().String()
}

func (logger *logging) msecs() string {
	return strconv.Itoa(int(logger.startTime % 1000))
}

func (logger *logging) timestamp() string {
	return strconv.FormatInt(time.Now().UnixNano(), 10)
}

func (logger *logging) relativeCreated() string {
	return strconv.FormatInt(time.Now().UnixNano()-logger.startTime, 10)
}

func (logger *logging) thread() string {
	return strconv.Itoa(int(GetGoId()))
}

func (logger *logging) threadName() string {
	return strconv.Itoa(int(GetGoId()))
}

func (logger *logging) process() string {
	return ""
}

func (logger *logging) message() string {
	return ""
}
<commit_msg>move calldepth to the beginning as global var<commit_after>// Copyright 2013, Cong Ding. All rights reserved.
// Use of this source code is governed by a GPLv2
// license that can be found in the LICENSE file.
//
// author: Cong Ding <dinggnu@gmail.com>
package logging

import (
	"runtime"
	"strconv"
	"sync/atomic"
	"time"
)

type field func(*logging) string

var fields = map[string]field{
	"seqid":           (*logging).seqid,
	"name":            (*logging).name,
	"levelno":         (*logging).levelno,
	"levelname":       (*logging).levelname,
	"pathname":        (*logging).pathname,
	"filename":        (*logging).filename,
	"module":          (*logging).module,
	"lineno":          (*logging).lineno,
	"funcName":        (*logging).funcName,
	"created":         (*logging).created,
	"asctime":         (*logging).asctime,
	"msecs":           (*logging).msecs,
	"relativeCreated": (*logging).relativeCreated,
	"thread":          (*logging).thread,
	"threadName":      (*logging).threadName,
	"process":         (*logging).process,
	"message":         (*logging).message,
}

// the calling depth of these function, which is used to call the
// runtime.Caller() function to get the line number and file name
var calldepth = 5

// GetGoId returns the id of goroutine, which is defined in ./get_go_id.c
func GetGoId() int32

func init() {
}

func (logger *logging) seqid() string {
	return strconv.FormatUint(atomic.AddUint64(&(logger.Seqid), 1), 10)
}

func (logger *logging) name() string {
	return logger.Name
}

func (logger *logging) levelno() string {
	return strconv.Itoa(int(logger.Level))
}

func (logger *logging) levelname() string {
	return levelNames[logger.Level]
}

func (logger *logging) pathname() string {
	_, file, _, ok := runtime.Caller(calldepth)
	if !ok {
		file = "???"
	}
	return file
}

func (logger *logging) filename() string {
	_, file, _, ok := runtime.Caller(calldepth)
	if !ok {
		file = "???"
	}
	short := file
	for i := len(file) - 1; i > 0; i-- {
		if file[i] == '/' {
			short = file[i+1:]
			break
		}
	}
	file = short
	return file
}

func (logger *logging) module() string {
	return ""
}

func (logger *logging) lineno() string {
	_, _, line, ok := runtime.Caller(calldepth)
	if !ok {
		line = 0
	}
	return strconv.Itoa(line)
}

func (logger *logging) funcName() string {
	pc, _, _, ok := runtime.Caller(calldepth)
	if !ok {
		return "???"
	}
	return runtime.FuncForPC(pc).Name()
}

func (logger *logging) created() string {
	return strconv.FormatInt(logger.startTime, 10)
}

func (logger *logging) asctime() string {
	return time.Now().String()
}

func (logger *logging) msecs() string {
	return strconv.Itoa(int(logger.startTime % 1000))
}

func (logger *logging) timestamp() string {
	return strconv.FormatInt(time.Now().UnixNano(), 10)
}

func (logger *logging) relativeCreated() string {
	return strconv.FormatInt(time.Now().UnixNano()-logger.startTime, 10)
}

func (logger *logging) thread() string {
	return strconv.Itoa(int(GetGoId()))
}

func (logger *logging) threadName() string {
	return strconv.Itoa(int(GetGoId()))
}

func (logger *logging) process() string {
	return ""
}

func (logger *logging) message() string {
	return ""
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package backups

import (
	"fmt"
	"io"
	"os"

	"github.com/juju/cmd"
	"github.com/juju/errors"
	"launchpad.net/gnuflag"
)

const archivePrefix = "juju-backup-"

const downloadDoc = `
"download" pulls down a backup archive file.

If --filename is not used, the archive is downloaded to a temporary
location.  The filename is printed to stdout.
`

// DownloadCommand is the sub-command for downloading a backup archive.
type DownloadCommand struct {
	CommandBase
	// Filename is where to save the downloaded archive.
	Filename string
	// ID is the backup ID to download.
	ID string
}

// Info implements Command.Info.
func (c *DownloadCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "download",
		Args:    "<ID>",
		Purpose: "get an archive file",
		Doc:     downloadDoc,
	}
}

// SetFlags implements Command.SetFlags.
func (c *DownloadCommand) SetFlags(f *gnuflag.FlagSet) {
	f.StringVar(&c.Filename, "filename", "", "download target")
}

// Init implements Command.Init.
func (c *DownloadCommand) Init(args []string) error {
	if len(args) == 0 {
		return errors.New("missing ID")
	}
	id, args := args[0], args[1:]
	if err := cmd.CheckEmpty(args); err != nil {
		return errors.Trace(err)
	}
	c.ID = id
	return nil
}

// Run implements Command.Run.
func (c *DownloadCommand) Run(ctx *cmd.Context) error {
	client, err := c.NewAPIClient()
	if err != nil {
		return errors.Trace(err)
	}
	defer client.Close()

	// Download the archive.
	resultArchive, err := client.Download(c.ID)
	if err != nil {
		return errors.Trace(err)
	}
	defer resultArchive.Close()

	// Prepare the local archive.
	filename := c.ResolveFilename()
	archive, err := os.Create(filename)
	if err != nil {
		return errors.Annotate(err, "while creating local archive file")
	}
	defer archive.Close()

	// Write out the archive.
	_, err = io.Copy(archive, resultArchive)
	if err != nil {
		return errors.Annotate(err, "while creating local archive file")
	}

	// Print the local filename.
	fmt.Fprintln(ctx.Stdout, filename)
	return nil
}

// ResolveFilename returns the filename used by the command.
func (c *DownloadCommand) ResolveFilename() string {
	filename := c.Filename
	if filename == "" {
		filename = archivePrefix + c.ID + ".tar.gz"
	}
	return filename
}
<commit_msg>Clarify a command help string.<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package backups

import (
	"fmt"
	"io"
	"os"

	"github.com/juju/cmd"
	"github.com/juju/errors"
	"launchpad.net/gnuflag"
)

const archivePrefix = "juju-backup-"

const downloadDoc = `
"download" retrieves a backup archive file.

If --filename is not used, the archive is downloaded to a temporary
location and the filename is printed to stdout.
`

// DownloadCommand is the sub-command for downloading a backup archive.
type DownloadCommand struct {
	CommandBase
	// Filename is where to save the downloaded archive.
	Filename string
	// ID is the backup ID to download.
	ID string
}

// Info implements Command.Info.
func (c *DownloadCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "download",
		Args:    "<ID>",
		Purpose: "get an archive file",
		Doc:     downloadDoc,
	}
}

// SetFlags implements Command.SetFlags.
func (c *DownloadCommand) SetFlags(f *gnuflag.FlagSet) {
	f.StringVar(&c.Filename, "filename", "", "download target")
}

// Init implements Command.Init.
func (c *DownloadCommand) Init(args []string) error {
	if len(args) == 0 {
		return errors.New("missing ID")
	}
	id, args := args[0], args[1:]
	if err := cmd.CheckEmpty(args); err != nil {
		return errors.Trace(err)
	}
	c.ID = id
	return nil
}

// Run implements Command.Run.
func (c *DownloadCommand) Run(ctx *cmd.Context) error {
	client, err := c.NewAPIClient()
	if err != nil {
		return errors.Trace(err)
	}
	defer client.Close()

	// Download the archive.
	resultArchive, err := client.Download(c.ID)
	if err != nil {
		return errors.Trace(err)
	}
	defer resultArchive.Close()

	// Prepare the local archive.
	filename := c.ResolveFilename()
	archive, err := os.Create(filename)
	if err != nil {
		return errors.Annotate(err, "while creating local archive file")
	}
	defer archive.Close()

	// Write out the archive.
	_, err = io.Copy(archive, resultArchive)
	if err != nil {
		return errors.Annotate(err, "while creating local archive file")
	}

	// Print the local filename.
	fmt.Fprintln(ctx.Stdout, filename)
	return nil
}

// ResolveFilename returns the filename used by the command.
func (c *DownloadCommand) ResolveFilename() string {
	filename := c.Filename
	if filename == "" {
		filename = archivePrefix + c.ID + ".tar.gz"
	}
	return filename
}
<|endoftext|>


<commit_before>package upload

import (
	"crypto/tls"
	"fmt"
	"os"
	"path/filepath"

	"github.com/codegangsta/cli"
	"github.com/materials-commons/gohandy/file"
	"github.com/materials-commons/mcstore/cmd/pkg/project"
	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/files"
	"github.com/materials-commons/mcstore/server/mcstored/service/rest/upload"
	"github.com/parnurzeal/gorequest"
)

// Command contains the arguments and functions for the cli upload command.
var Command = cli.Command{
	Name:    "upload",
	Aliases: []string{"up", "u"},
	Usage:   "Upload data to MaterialsCommons",
	Flags: []cli.Flag{
		cli.IntFlag{
			Name:  "parallel, n",
			Value: 3,
			Usage: "Number of simultaneous uploads to perform, defaults to 3",
		},
	},
	Action: uploadCLI,
}

const oneMeg = 1024 * 1024
const twoMeg = oneMeg * 2
const largeFileSize = oneMeg * 25
const maxSimultaneous = 5

var proj *project.MCProject

//var pbPool = &pb.Pool{}

// uploadCLI implements the cli command upload.
func uploadCLI(c *cli.Context) {
	fmt.Println("upload: ", c.Args())
	if len(c.Args()) != 1 {
		fmt.Println("You must give the directory to walk")
		os.Exit(1)
	}
	dir := c.Args()[0]
	numThreads := getNumThreads(c)

	if !file.IsDir(dir) {
		fmt.Printf("Invalid directory: %s\n", dir)
		os.Exit(1)
	}

	uploadToServer(dir, numThreads)
}

// getNumThreads ensures that the number of parallel downloads is valid.
func getNumThreads(c *cli.Context) int {
	numThreads := c.Int("parallel")

	if numThreads < 1 {
		fmt.Println("Simultaneous downloads must be positive: ", numThreads)
		os.Exit(1)
	} else if numThreads > maxSimultaneous {
		fmt.Printf("You may not set simultaneous downloads greater than %d: %d\n", maxSimultaneous, numThreads)
		os.Exit(1)
	}

	return numThreads
}

// uploadToServer
func uploadToServer(dir string, numThreads int) {
	var err error
	proj, err = project.Find(dir)
	if err != nil {
		fmt.Println("Unable to locate project dir is in.")
		os.Exit(1)
	}
	fmt.Printf("project = '%s'\n", proj.ID)
	// _, errc := files.PWalk(dir, numThreads, processFiles)
	// if err := <-errc; err != nil {
	// 	fmt.Println("Got error: ", err)
	// }
}

// findDotMCProject will walk up from directory looking for the .mcproject
// directory. If it cannot find it, then the directory isn't in a
// known project. findProject will call os.Exit on any errors or if
// it cannot find a .mcproject directory.
func findDotMCProject(dir string) string {
	// Normalize the directory path, and convert all path separators to a
	// forward slash (/).
	dirPath, err := filepath.Abs(dir)
	if err != nil {
		fmt.Printf("Bad directory %s: %s", dir, err)
		os.Exit(1)
	}

	dirPath = filepath.ToSlash(dirPath)
	for {
		if dirPath == "/" {
			// Projects at root level not allowed
			fmt.Println("Your directory is not in a project.")
			fmt.Println("Upload a directory in a project or create a project by running the create-project command.")
			os.Exit(1)
		}

		mcprojectDir := filepath.Join(dirPath, ".mcproject")
		if file.IsDir(mcprojectDir) {
			// found it
			return mcprojectDir
		}
		dirPath = filepath.Dir(dirPath)
	}
}

// processFiles is the callback passed into PWalk. It processes each file, determines
// if it should be uploaded, and if so uploads the file. There can be a maxSimultaneous
// processFiles routines running.
func processFiles(done <-chan struct{}, entries <-chan files.TreeEntry, result chan<- string) {
	fmt.Println("processFiles")
	u := &uploader{
		client: gorequest.New().TLSClientConfig(&tls.Config{InsecureSkipVerify: true}),
	}
	for entry := range entries {
		select {
		case result <- u.sendFile(entry):
		case <-done:
			// Received done, so stop processing requests.
			return
		}
	}
}

type uploader struct {
	client *gorequest.SuperAgent
}

// sendFile needs to:
//   if file is not on server then
//       send file up and send hash up at end
//           -- here we are computing hash as we send blocks up
//   else
//       compute hash
//       if hash is on server then
//          tell server to create a new entry pointing to
//          the already uploaded file
//       else
//          upload the file
//       end
//  end
//
func (u *uploader) sendFile(fileEntry files.TreeEntry) string {
	u.createUploadRequest()
	// buf := make([]byte, twoMeg)
	// f, err := os.Open(fileEntry.Path)
	// if err != nil {
	// 	return ""
	// }
	// fileHash := ""
	// // For small files just transfer the bytes. For
	// // large files compute the file hash.
	// if fileEntry.Finfo.Size() > largeFileSize {
	// 	fileHash, _ = file.HashStr(md5.New(), fileEntry.Path)
	// }
	// for {
	// 	read, err := f.Read(buf)
	// 	sendFlowChunk(buf)
	// }
	return fileEntry.Path
}

func fileChanged(oinfo, ninfo file.ExFileInfo) bool {
	switch {
	case oinfo.Size() != ninfo.Size():
		return true
	case oinfo.CTime().Before(ninfo.CTime()):
		return true
	case oinfo.ModTime().Before(ninfo.ModTime()):
		return true
	default:
		return false
	}
}

func (u *uploader) createUploadRequest() {
	fmt.Println("createUploadRequest")
	req := upload.CreateRequest{
		ProjectID:   "9ead5bbf-f7eb-4010-bc1f-e4a063f56226",
		DirectoryID: "c54a77d6-cd6d-4cd1-8f19-44facc761da6",
		FileName:    "abc.txt",
		FileSize:    10,
		FileMTime:   "Thu, 30 Apr 2015 13:10:04 EST",
	}

	var resp upload.CreateResponse
	fmt.Println("url =", app.MCApi.APIUrl("/upload"))
	r, body, errs := u.client.Post(app.MCApi.APIUrl("/upload")).Send(req).End()
	if err := app.MCApi.APIError(r, errs); err != nil {
		fmt.Println("got err from Post:", err)
		return
	}
	app.MCApi.ToJSON(body, &resp)
	fmt.Printf("%#v\n", resp)
}

func sendFlowChunk(buf []byte) {

}
<commit_msg>Make sure fileChanged will work.<commit_after>package upload

import (
	"crypto/tls"
	"fmt"
	"os"
	"path/filepath"

	"github.com/codegangsta/cli"
	"github.com/materials-commons/gohandy/file"
	"github.com/materials-commons/mcstore/cmd/pkg/project"
	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/files"
	"github.com/materials-commons/mcstore/server/mcstored/service/rest/upload"
	"github.com/parnurzeal/gorequest"
)

// Command contains the arguments and functions for the cli upload command.
var Command = cli.Command{
	Name:    "upload",
	Aliases: []string{"up", "u"},
	Usage:   "Upload data to MaterialsCommons",
	Flags: []cli.Flag{
		cli.IntFlag{
			Name:  "parallel, n",
			Value: 3,
			Usage: "Number of simultaneous uploads to perform, defaults to 3",
		},
	},
	Action: uploadCLI,
}

const oneMeg = 1024 * 1024
const twoMeg = oneMeg * 2
const largeFileSize = oneMeg * 25
const maxSimultaneous = 5

var proj *project.MCProject

//var pbPool = &pb.Pool{}

// uploadCLI implements the cli command upload.
func uploadCLI(c *cli.Context) {
	fmt.Println("upload: ", c.Args())
	if len(c.Args()) != 1 {
		fmt.Println("You must give the directory to walk")
		os.Exit(1)
	}
	dir := c.Args()[0]
	numThreads := getNumThreads(c)

	if !file.IsDir(dir) {
		fmt.Printf("Invalid directory: %s\n", dir)
		os.Exit(1)
	}

	uploadToServer(dir, numThreads)
}

// getNumThreads ensures that the number of parallel downloads is valid.
func getNumThreads(c *cli.Context) int {
	numThreads := c.Int("parallel")

	if numThreads < 1 {
		fmt.Println("Simultaneous downloads must be positive: ", numThreads)
		os.Exit(1)
	} else if numThreads > maxSimultaneous {
		fmt.Printf("You may not set simultaneous downloads greater than %d: %d\n", maxSimultaneous, numThreads)
		os.Exit(1)
	}

	return numThreads
}

// uploadToServer
func uploadToServer(dir string, numThreads int) {
	var err error
	proj, err = project.Find(dir)
	if err != nil {
		fmt.Println("Unable to locate project dir is in.")
		os.Exit(1)
	}
	fmt.Printf("project = '%s'\n", proj.ID)
	// _, errc := files.PWalk(dir, numThreads, processFiles)
	// if err := <-errc; err != nil {
	// 	fmt.Println("Got error: ", err)
	// }
}

// findDotMCProject will walk up from directory looking for the .mcproject
// directory. If it cannot find it, then the directory isn't in a
// known project. findProject will call os.Exit on any errors or if
// it cannot find a .mcproject directory.
func findDotMCProject(dir string) string {
	// Normalize the directory path, and convert all path separators to a
	// forward slash (/).
	dirPath, err := filepath.Abs(dir)
	if err != nil {
		fmt.Printf("Bad directory %s: %s", dir, err)
		os.Exit(1)
	}

	dirPath = filepath.ToSlash(dirPath)
	for {
		if dirPath == "/" {
			// Projects at root level not allowed
			fmt.Println("Your directory is not in a project.")
			fmt.Println("Upload a directory in a project or create a project by running the create-project command.")
			os.Exit(1)
		}

		mcprojectDir := filepath.Join(dirPath, ".mcproject")
		if file.IsDir(mcprojectDir) {
			// found it
			return mcprojectDir
		}
		dirPath = filepath.Dir(dirPath)
	}
}

// processFiles is the callback passed into PWalk. It processes each file, determines
// if it should be uploaded, and if so uploads the file. There can be a maxSimultaneous
// processFiles routines running.
func processFiles(done <-chan struct{}, entries <-chan files.TreeEntry, result chan<- string) {
	fmt.Println("processFiles")
	u := &uploader{
		client: gorequest.New().TLSClientConfig(&tls.Config{InsecureSkipVerify: true}),
	}
	for entry := range entries {
		select {
		case result <- u.sendFile(entry):
		case <-done:
			// Received done, so stop processing requests.
			return
		}
	}
}

type uploader struct {
	client *gorequest.SuperAgent
}

// sendFile needs to:
//   if file is not on server then
//       send file up and send hash up at end
//           -- here we are computing hash as we send blocks up
//   else
//       compute hash
//       if hash is on server then
//          tell server to create a new entry pointing to
//          the already uploaded file
//       else
//          upload the file
//       end
//  end
//
func (u *uploader) sendFile(fileEntry files.TreeEntry) string {
	u.createUploadRequest()
	//s1 := file.ExInfoFrom(20, time.Now(), time.Now(), time.Now(), file.FID{})
	//fileChanged(s1, s1)
	// buf := make([]byte, twoMeg)
	// f, err := os.Open(fileEntry.Path)
	// if err != nil {
	// 	return ""
	// }
	// fileHash := ""
	// // For small files just transfer the bytes. For
	// // large files compute the file hash.
	// if fileEntry.Finfo.Size() > largeFileSize {
	// 	fileHash, _ = file.HashStr(md5.New(), fileEntry.Path)
	// }
	// for {
	// 	read, err := f.Read(buf)
	// 	sendFlowChunk(buf)
	// }
	return fileEntry.Path
}

func fileChanged(oinfo, ninfo file.ExFileInfo) bool {
	switch {
	case oinfo.Size() != ninfo.Size():
		return true
	case oinfo.CTime().Before(ninfo.CTime()):
		return true
	case oinfo.ModTime().Before(ninfo.ModTime()):
		return true
	default:
		return false
	}
}

func (u *uploader) createUploadRequest() {
	fmt.Println("createUploadRequest")
	req := upload.CreateRequest{
		ProjectID:   "9ead5bbf-f7eb-4010-bc1f-e4a063f56226",
		DirectoryID: "c54a77d6-cd6d-4cd1-8f19-44facc761da6",
		FileName:    "abc.txt",
		FileSize:    10,
		FileMTime:   "Thu, 30 Apr 2015 13:10:04 EST",
	}

	var resp upload.CreateResponse
	fmt.Println("url =", app.MCApi.APIUrl("/upload"))
	r, body, errs := u.client.Post(app.MCApi.APIUrl("/upload")).Send(req).End()
	if err := app.MCApi.APIError(r, errs); err != nil {
		fmt.Println("got err from Post:", err)
		return
	}
	app.MCApi.ToJSON(body, &resp)
	fmt.Printf("%#v\n", resp)
}

func sendFlowChunk(buf []byte) {

}
<|endoftext|>


<commit_before>package main

import (
	"math/rand"

	"github.com/foolusion/elwinprotos/storage"
	"github.com/pkg/errors"
	"github.com/spf13/viper"
	"k8s.io/contrib/compare/Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/utils"
)

type namespace struct {
	Name        string `bson:"_id"`
	NumSegments int
	Segments    []byte
}

func (n *namespace) sampleSegments(num int) []byte {
	if num <= 0 {
		return make([]byte, len(n.Segments))
	}
	available := n.availableSegments()
	p := rand.Perm(len(available))
	out := make([]byte, len(n.Segments))
	if num > len(available) {
		num = len(available)
	}
	for i := 0; i < num; i++ {
		set(out, available[p[i]])
	}
	return out
}

func set(b []byte, index int) {
	i, pos := index/8, uint8(index%8)
	b[i] |= 1 << pos
}

func clear(b []byte, index int) {
	i, pos := index/8, uint8(index%8)
	b[i] &= ^(1 << pos)
}

func (n *namespace) availableSegments() []int {
	out := make([]int, 0, n.NumSegments)
	for i := range n.Segments {
		for shift := uint8(0); shift < 8; shift++ {
			if i*8+int(shift) > int(n.NumSegments) {
				break
			}
			if n.Segments[i]&(1<<shift) != 1<<shift {
				out = append(out, i*8+int(shift))
			}
		}
	}
	return out
}

func (s *server) getNamespace(name string) (*namespace, error) {
	var ns namespace
	if err := s.DB(viper.GetString(cfgMongoDatabase)).C(collNamespaces).FindId(name).One(&ns); err != nil {
		return nil, errors.Wrap(err, "could not find namespace")
	}
	return &ns, nil
}

func (s *server) createNamespace(name string, numSegments int) (*namespace, error) {
	if numSegments <= 0 {
		return nil, errors.New("must supply a positive number of segments")
	}
	numBytes := numSegments / 8
	if numSegments%8 != 0 {
		numBytes++
	}
	return &namespace{
		Name:        name,
		NumSegments: numSegments,
		Segments:    make([]byte, numBytes),
	}, nil
}

func (s *server) createExperiment(name, namespace string, nsSegments, expSegments int, labels map[string]string, params []*storage.Param) error {
	ns, err := s.getNamespace(namespace)
	if err != nil {
		name, err := utils.GenerateRandomName("", 7)
		if err != nil {
			return errors.Wrap(err, "could not genereate new namespace name")
		}
		if ns, err = s.createNamespace(name, nsSegments); err != nil {
			return errors.Wrap(err, "could not create new namespace")
		}
	}

	seg := ns.sampleSegments(expSegments)

	if _, err := s.DB(viper.GetString(cfgMongoDatabase)).C(collNamespaces).UpsertId(ns.Name, *ns); err != nil {
		return errors.Wrap(err, "could not update namespace")
	}

	id, err := utils.GenerateRandomName(name, 32)
	if err != nil {
		return errors.Wrap(err, "could not generate experiment id")
	}

	exp := experiment{
		Id: id,
		Experiment: storage.Experiment{
			Id:        id,
			Name:      name,
			Namespace: ns.Name,
			Labels:    labels,
			Params:    params,
			Segments:  seg,
		},
	}
	if _, err := s.DB(viper.GetString(cfgMongoDatabase)).C(collExperiments).UpsertId(id, exp); err != nil {
		return errors.Wrap(err, "could not update experiment")
	}
	return nil
}
<commit_msg>cmd/mongo-store: don't import from vendor<commit_after>package main

import (
	crand "crypto/rand"
	"encoding/hex"
	"io"
	"math/rand"

	"github.com/foolusion/elwinprotos/storage"
	"github.com/pkg/errors"
	"github.com/spf13/viper"
)

type namespace struct {
	Name        string `bson:"_id"`
	NumSegments int
	Segments    []byte
}

func (n *namespace) sampleSegments(num int) []byte {
	if num <= 0 {
		return make([]byte, len(n.Segments))
	}
	available := n.availableSegments()
	p := rand.Perm(len(available))
	out := make([]byte, len(n.Segments))
	if num > len(available) {
		num = len(available)
	}
	for i := 0; i < num; i++ {
		set(out, available[p[i]])
	}
	return out
}

func set(b []byte, index int) {
	i, pos := index/8, uint8(index%8)
	b[i] |= 1 << pos
}

func clear(b []byte, index int) {
	i, pos := index/8, uint8(index%8)
	b[i] &= ^(1 << pos)
}

func (n *namespace) availableSegments() []int {
	out := make([]int, 0, n.NumSegments)
	for i := range n.Segments {
		for shift := uint8(0); shift < 8; shift++ {
			if i*8+int(shift) > int(n.NumSegments) {
				break
			}
			if n.Segments[i]&(1<<shift) != 1<<shift {
				out = append(out, i*8+int(shift))
			}
		}
	}
	return out
}

func (s *server) getNamespace(name string) (*namespace, error) {
	var ns namespace
	if err := s.DB(viper.GetString(cfgMongoDatabase)).C(collNamespaces).FindId(name).One(&ns); err != nil {
		return nil, errors.Wrap(err, "could not find namespace")
	}
	return &ns, nil
}

func (s *server) createNamespace(name string, numSegments int) (*namespace, error) {
	if numSegments <= 0 {
		return nil, errors.New("must supply a positive number of segments")
	}
	numBytes := numSegments / 8
	if numSegments%8 != 0 {
		numBytes++
	}
	return &namespace{
		Name:        name,
		NumSegments: numSegments,
		Segments:    make([]byte, numBytes),
	}, nil
}

// GenerateRandomName returns a new name joined with a prefix.  This size
// specified is used to truncate the randomly generated value
func GenerateRandomName(prefix string, size int) (string, error) {
	id := make([]byte, 32)
	if _, err := io.ReadFull(crand.Reader, id); err != nil {
		return "", err
	}
	if size > 64 {
		size = 64
	}
	return prefix + hex.EncodeToString(id)[:size], nil
}

func (s *server) createExperiment(name, namespace string, nsSegments, expSegments int, labels map[string]string, params []*storage.Param) error {
	ns, err := s.getNamespace(namespace)
	if err != nil {
		name, err := GenerateRandomName("", 7)
		if err != nil {
			return errors.Wrap(err, "could not genereate new namespace name")
		}
		if ns, err = s.createNamespace(name, nsSegments); err != nil {
			return errors.Wrap(err, "could not create new namespace")
		}
	}

	seg := ns.sampleSegments(expSegments)

	if _, err := s.DB(viper.GetString(cfgMongoDatabase)).C(collNamespaces).UpsertId(ns.Name, *ns); err != nil {
		return errors.Wrap(err, "could not update namespace")
	}

	id, err := GenerateRandomName(name, 32)
	if err != nil {
		return errors.Wrap(err, "could not generate experiment id")
	}

	exp := experiment{
		Id: id,
		Experiment: storage.Experiment{
			Id:        id,
			Name:      name,
			Namespace: ns.Name,
			Labels:    labels,
			Params:    params,
			Segments:  seg,
		},
	}
	if _, err := s.DB(viper.GetString(cfgMongoDatabase)).C(collExperiments).UpsertId(id, exp); err != nil {
		return errors.Wrap(err, "could not update experiment")
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"os"
	"sync"

	"github.com/thomersch/grandine/lib/cugdf"
	"github.com/thomersch/grandine/lib/spatial"

	"github.com/thomersch/gosmparse"
)

type dataHandler struct {
	cond condition

	ec *elemCache

	nodes    []gosmparse.Node
	nodesMtx sync.Mutex
	ways     []gosmparse.Way
	waysMtx  sync.Mutex
	rels     []gosmparse.Relation
	relsMtx  sync.Mutex
}

func (d *dataHandler) ReadNode(n gosmparse.Node) {
	// TODO: make it possible to specify condition type (node/way/rel)
	if d.cond.Matches(n.Tags) {
		d.nodesMtx.Lock()
		d.nodes = append(d.nodes, n)
		d.nodesMtx.Unlock()
	}
}

func (d *dataHandler) ReadWay(w gosmparse.Way) {
	// TODO: make it possible to specify condition type (node/way/rel)
	if d.cond.Matches(w.Tags) {
		d.ec.AddNodes(w.NodeIDs...)
		d.ec.setMembers(w.ID, w.NodeIDs)

		d.waysMtx.Lock()
		d.ways = append(d.ways, w)
		d.waysMtx.Unlock()
	}
}

func (d *dataHandler) ReadRelation(r gosmparse.Relation) {
	if d.cond.Matches(r.Tags) {
		d.relsMtx.Lock()
		d.rels = append(d.rels, r)
		d.relsMtx.Unlock()

		for _, memb := range r.Members {
			switch memb.Type {
			case gosmparse.WayType:
				d.ec.AddWay(memb.ID)
			} // TODO: check if relations of nodes/relations are necessary
		}
	}
}

type elemCache struct {
	nodes    map[int64]spatial.Point
	nodesMtx sync.Mutex
	ways     map[int64][]int64
	waysMtx  sync.Mutex
}

func NewElemCache() *elemCache {
	return &elemCache{
		nodes: map[int64]spatial.Point{},
		ways:  map[int64][]int64{},
	}
}

func (d *elemCache) AddNodes(nIDs ...int64) {
	d.nodesMtx.Lock()
	for _, nID := range nIDs {
		d.nodes[nID] = spatial.Point{}
	}
	d.nodesMtx.Unlock()
}

func (d *elemCache) AddWay(wID int64) {
	d.waysMtx.Lock()
	d.ways[wID] = []int64{}
	d.waysMtx.Unlock()
}

func (d *elemCache) SetCoord(nID int64, coord spatial.Point) {
	d.nodesMtx.Lock()
	d.nodes[nID] = coord
	d.nodesMtx.Unlock()
}

func (d *elemCache) setMembers(wID int64, members []int64) {
	d.waysMtx.Lock()
	d.ways[wID] = members
	d.waysMtx.Unlock()
}

func (d *elemCache) ReadWay(w gosmparse.Way) {
	d.waysMtx.Lock()
	_, ok := d.ways[w.ID]
	d.waysMtx.Unlock()
	if ok {
		d.setMembers(w.ID, w.NodeIDs)
		d.AddNodes(w.NodeIDs...)
	}
}

func (d *elemCache) Line(wID int64) spatial.Line {
	// check if mutex is needed
	membs, ok := d.ways[wID]
	if !ok {
		log.Fatalf("missing referenced way: %v", wID)
	}

	var l spatial.Line
	for _, memb := range membs {
		l = append(l, d.nodes[memb])
	}
	return l
}

// Interface enforces this. Probably I should change the behavior.
func (d *elemCache) ReadNode(n gosmparse.Node)         {}
func (d *elemCache) ReadRelation(r gosmparse.Relation) {}

type nodeCollector struct {
	ec *elemCache
}

func (d *nodeCollector) ReadNode(n gosmparse.Node) {
	d.ec.SetCoord(n.ID, spatial.Point{float64(n.Lon), float64(n.Lat)})
}
func (d *nodeCollector) ReadWay(w gosmparse.Way)           {}
func (d *nodeCollector) ReadRelation(r gosmparse.Relation) {}

// const (
// 	typAny      = 0
// 	typNode     = 1
// 	typWay      = 2
// 	typRelation = 3
// )

type tagMapFn func(map[string]string) map[string]interface{}

type mapper struct {
	cond     condition
	mapper   tagMapFn
	elemType spatial.GeomType
}

type condition struct {
	key   string
	value string
}

func (c *condition) Matches(kv map[string]string) bool {
	if v, ok := kv[c.key]; ok {
		if len(c.value) == 0 || c.value == v {
			return true
		}
	}
	return false
}

func main() {
	cond := condition{"building", ""}

	source := flag.String("src", "osm.pbf", "")
	outfile := flag.String("out", "osm.cugdf", "")
	flag.Parse()

	f, err := os.Open(*source)
	if err != nil {
		log.Fatal(err)
	}
	dec := gosmparse.NewDecoder(f)

	// First pass
	ec := NewElemCache()
	dh := dataHandler{
		cond: cond,
		ec:   ec,
	}
	log.Println("Starting 3 step parsing")
	log.Println("Reading data (1/3)...")
	err = dec.Parse(&dh)
	if err != nil {
		log.Fatal(err)
	}

	_, err = f.Seek(0, 0) // jumps to beginning of file
	if err != nil {
		log.Fatal(err)
	}

	// Second pass
	log.Println("Collecting nodes (2/3)...")
	err = dec.Parse(ec)
	if err != nil {
		log.Fatal(err)
	}

	_, err = f.Seek(0, 0)
	if err != nil {
		log.Fatal(err)
	}

	// Third pass
	log.Println("Resolving dependent objects (3/3)...")
	rc := nodeCollector{
		ec: ec,
	}
	err = dec.Parse(&rc)
	if err != nil {
		log.Fatal(err)
	}

	var fc []spatial.Feature

	log.Println("Parsing completed.")

	log.Println("Collecting points...")
	for _, pt := range dh.nodes {
		props := map[string]interface{}{}
		for k, v := range pt.Tags {
			props[k] = v
		}
		fc = append(fc, spatial.Feature{
			Props:    props,
			Geometry: spatial.MustNewGeom(spatial.Point{float64(pt.Lon), float64(pt.Lat)}),
		})
	}

	log.Println("Assembling ways...")
	// TODO: auto-detect if linestring or polygon, based on tags
	for _, wy := range dh.ways {
		props := map[string]interface{}{}
		for k, v := range wy.Tags {
			props[k] = v
		}
		ln := ec.Line(wy.ID)
		if !ln.Clockwise() {
			ln.Reverse()
		}
		fc = append(fc, spatial.Feature{
			Props:    props,
			Geometry: spatial.MustNewGeom(ln),
		})
	}

	log.Println("Assembling relations...")
	for _, rl := range dh.rels {
		if v, ok := rl.Tags["type"]; !ok || v != "multipolygon" {
			continue
		}
		var poly spatial.Polygon

		for _, memb := range rl.Members {
			if memb.Role == "outer" || memb.Role == "inner" {
				ring := ec.Line(memb.ID)
				if (memb.Role == "outer" && !ring.Clockwise()) || (memb.Role == "inner" && ring.Clockwise()) {
					ring.Reverse()
				}
				poly = append(poly, ring)
			}
		}
	}

	log.Println("Writing out")
	of, err := os.Create(*outfile)
	if err != nil {
		log.Fatal(err)
	}
	err = cugdf.Marshal(fc, of)
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>cmd/spatialize: tag mapper for nodes started<commit_after>package main

import (
	"flag"
	"log"
	"os"
	"sync"

	"github.com/thomersch/grandine/lib/cugdf"
	"github.com/thomersch/grandine/lib/spatial"

	"github.com/thomersch/gosmparse"
)

type nd struct {
	Lat, Lon float64
	Tags     map[string]interface{}
}
type wy gosmparse.Way
type rl gosmparse.Relation

type dataHandler struct {
	cond condition

	ec *elemCache

	nodes    []nd
	nodesMtx sync.Mutex
	ways     []wy
	waysMtx  sync.Mutex
	rels     []rl
	relsMtx  sync.Mutex
}

func (d *dataHandler) ReadNode(n gosmparse.Node) {
	if d.cond.Matches(n.Tags) {
		d.nodesMtx.Lock()
		d.nodes = append(d.nodes, nd{
			Lat:  n.Lat,
			Lon:  n.Lon,
			Tags: d.cond.Map(n.Tags),
		})
		d.nodesMtx.Unlock()
	}
}

func (d *dataHandler) ReadWay(w gosmparse.Way) {
	if d.cond.Matches(w.Tags) {
		d.ec.AddNodes(w.NodeIDs...)
		d.ec.setMembers(w.ID, w.NodeIDs)

		d.waysMtx.Lock()
		d.ways = append(d.ways, wy(w))
		d.waysMtx.Unlock()
	}
}

func (d *dataHandler) ReadRelation(r gosmparse.Relation) {
	if d.cond.Matches(r.Tags) {
		d.relsMtx.Lock()
		d.rels = append(d.rels, rl(r))
		d.relsMtx.Unlock()

		for _, memb := range r.Members {
			switch memb.Type {
			case gosmparse.WayType:
				d.ec.AddWay(memb.ID)
			} // TODO: check if relations of nodes/relations are necessary
		}
	}
}

type elemCache struct {
	nodes    map[int64]spatial.Point
	nodesMtx sync.Mutex
	ways     map[int64][]int64
	waysMtx  sync.Mutex
}

func NewElemCache() *elemCache {
	return &elemCache{
		nodes: map[int64]spatial.Point{},
		ways:  map[int64][]int64{},
	}
}

func (d *elemCache) AddNodes(nIDs ...int64) {
	d.nodesMtx.Lock()
	for _, nID := range nIDs {
		d.nodes[nID] = spatial.Point{}
	}
	d.nodesMtx.Unlock()
}

func (d *elemCache) AddWay(wID int64) {
	d.waysMtx.Lock()
	d.ways[wID] = []int64{}
	d.waysMtx.Unlock()
}

func (d *elemCache) SetCoord(nID int64, coord spatial.Point) {
	d.nodesMtx.Lock()
	d.nodes[nID] = coord
	d.nodesMtx.Unlock()
}

func (d *elemCache) setMembers(wID int64, members []int64) {
	d.waysMtx.Lock()
	d.ways[wID] = members
	d.waysMtx.Unlock()
}

func (d *elemCache) ReadWay(w gosmparse.Way) {
	d.waysMtx.Lock()
	_, ok := d.ways[w.ID]
	d.waysMtx.Unlock()
	if ok {
		d.setMembers(w.ID, w.NodeIDs)
		d.AddNodes(w.NodeIDs...)
	}
}

func (d *elemCache) Line(wID int64) spatial.Line {
	// check if mutex is needed
	membs, ok := d.ways[wID]
	if !ok {
		log.Fatalf("missing referenced way: %v", wID)
	}

	var l spatial.Line
	for _, memb := range membs {
		l = append(l, d.nodes[memb])
	}
	return l
}

// Interface enforces this. Probably I should change the behavior.
func (d *elemCache) ReadNode(n gosmparse.Node)         {}
func (d *elemCache) ReadRelation(r gosmparse.Relation) {}

type nodeCollector struct {
	ec *elemCache
}

func (d *nodeCollector) ReadNode(n gosmparse.Node) {
	d.ec.SetCoord(n.ID, spatial.Point{float64(n.Lon), float64(n.Lat)})
}
func (d *nodeCollector) ReadWay(w gosmparse.Way)           {}
func (d *nodeCollector) ReadRelation(r gosmparse.Relation) {}

type tagMapFn func(map[string]string) map[string]interface{}

type mapper struct {
	cond     condition
	mapper   tagMapFn
	elemType spatial.GeomType
}

type condition struct {
	// TODO: make it possible to specify condition type (node/way/rel)
	key    string
	value  string
	mapper tagMapFn
}

func (c *condition) Matches(kv map[string]string) bool {
	if v, ok := kv[c.key]; ok {
		if len(c.value) == 0 || c.value == v {
			return true
		}
	}
	return false
}

func (c *condition) Map(kv map[string]string) map[string]interface{} {
	return c.mapper(kv)
}

func main() {
	cond := condition{"highway", "primary", func(map[string]string) map[string]interface{} { return nil }}

	source := flag.String("src", "osm.pbf", "")
	outfile := flag.String("out", "osm.cugdf", "")
	flag.Parse()

	f, err := os.Open(*source)
	if err != nil {
		log.Fatal(err)
	}
	dec := gosmparse.NewDecoder(f)

	// First pass
	ec := NewElemCache()
	dh := dataHandler{
		cond: cond,
		ec:   ec,
	}
	log.Println("Starting 3 step parsing")
	log.Println("Reading data (1/3)...")
	err = dec.Parse(&dh)
	if err != nil {
		log.Fatal(err)
	}

	_, err = f.Seek(0, 0) // jumps to beginning of file
	if err != nil {
		log.Fatal(err)
	}

	// Second pass
	log.Println("Collecting nodes (2/3)...")
	err = dec.Parse(ec)
	if err != nil {
		log.Fatal(err)
	}

	_, err = f.Seek(0, 0)
	if err != nil {
		log.Fatal(err)
	}

	// Third pass
	log.Println("Resolving dependent objects (3/3)...")
	rc := nodeCollector{
		ec: ec,
	}
	err = dec.Parse(&rc)
	if err != nil {
		log.Fatal(err)
	}

	var fc []spatial.Feature

	log.Println("Parsing completed.")

	log.Println("Collecting points...")
	for _, pt := range dh.nodes {
		props := map[string]interface{}{}
		for k, v := range pt.Tags {
			props[k] = v
		}
		fc = append(fc, spatial.Feature{
			Props:    props,
			Geometry: spatial.MustNewGeom(spatial.Point{float64(pt.Lon), float64(pt.Lat)}),
		})
	}

	log.Println("Assembling ways...")
	// TODO: auto-detect if linestring or polygon, based on tags
	for _, wy := range dh.ways {
		props := map[string]interface{}{}
		for k, v := range wy.Tags {
			props[k] = v
		}
		ln := ec.Line(wy.ID)
		if !ln.Clockwise() {
			ln.Reverse()
		}
		fc = append(fc, spatial.Feature{
			Props:    props,
			Geometry: spatial.MustNewGeom(ln),
		})
	}

	log.Println("Assembling relations...")
	for _, rl := range dh.rels {
		if v, ok := rl.Tags["type"]; !ok || v != "multipolygon" {
			continue
		}
		var poly spatial.Polygon

		for _, memb := range rl.Members {
			if memb.Role == "outer" || memb.Role == "inner" {
				ring := ec.Line(memb.ID)
				if (memb.Role == "outer" && !ring.Clockwise()) || (memb.Role == "inner" && ring.Clockwise()) {
					ring.Reverse()
				}
				poly = append(poly, ring)
			}
		}
	}

	log.Println("Writing out")
	of, err := os.Create(*outfile)
	if err != nil {
		log.Fatal(err)
	}
	err = cugdf.Marshal(fc, of)
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore
//
// This build tag means that "go install golang.org/x/image/..." doesn't
// install this manual test. Use "go run main.go" to explicitly run it.

// Program webp-manual-test checks that the Go WEBP library's decodings match
// the C WEBP library's.
package main // import "golang.org/x/image/cmd/webp-manual-test"

import (
	"bytes"
	"encoding/hex"
	"flag"
	"fmt"
	"image"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"

	"golang.org/x/image/webp"
	"golang.org/x/image/webp/nycbcra"
)

var (
	dwebp = flag.String("dwebp", "/usr/bin/dwebp", "path to the dwebp program "+
		"installed from https://developers.google.com/speed/webp/download")
	testdata = flag.String("testdata", "", "path to the libwebp-test-data directory "+
		"checked out from https://chromium.googlesource.com/webm/libwebp-test-data")
)

func main() {
	flag.Parse()
	if *dwebp == "" {
		flag.Usage()
		log.Fatal("dwebp flag was not specified")
	}
	if _, err := os.Stat(*dwebp); err != nil {
		flag.Usage()
		log.Fatalf("could not find dwebp program at %q", *dwebp)
	}
	if *testdata == "" {
		flag.Usage()
		log.Fatal("testdata flag was not specified")
	}

	f, err := os.Open(*testdata)
	if err != nil {
		log.Fatalf("Open: %v", err)
	}
	defer f.Close()
	names, err := f.Readdirnames(-1)
	if err != nil {
		log.Fatalf("Readdirnames: %v", err)
	}
	sort.Strings(names)

	nFail, nPass := 0, 0
	for _, name := range names {
		if !strings.HasSuffix(name, "webp") {
			continue
		}
		if err := test(name); err != nil {
			fmt.Printf("FAIL\t%s\t%v\n", name, err)
			nFail++
		} else {
			fmt.Printf("PASS\t%s\n", name)
			nPass++
		}
	}
	fmt.Printf("%d PASS, %d FAIL, %d TOTAL\n", nPass, nFail, nPass+nFail)
	if nFail != 0 {
		os.Exit(1)
	}
}

// test tests a single WEBP image.
func test(name string) error {
	filename := filepath.Join(*testdata, name)
	f, err := os.Open(filename)
	if err != nil {
		return fmt.Errorf("Open: %v", err)
	}
	defer f.Close()

	gotImage, err := webp.Decode(f)
	if err != nil {
		return fmt.Errorf("Decode: %v", err)
	}
	format, encode := "-pgm", encodePGM
	if _, lossless := gotImage.(*image.NRGBA); lossless {
		format, encode = "-pam", encodePAM
	}
	got, err := encode(gotImage)
	if err != nil {
		return fmt.Errorf("encode: %v", err)
	}

	stdout := new(bytes.Buffer)
	stderr := new(bytes.Buffer)
	c := exec.Command(*dwebp, filename, format, "-o", "/dev/stdout")
	c.Stdout = stdout
	c.Stderr = stderr
	if err := c.Run(); err != nil {
		os.Stderr.Write(stderr.Bytes())
		return fmt.Errorf("executing dwebp: %v", err)
	}
	want := stdout.Bytes()

	if len(got) != len(want) {
		return fmt.Errorf("encodings have different length: got %d, want %d", len(got), len(want))
	}
	for i, g := range got {
		if w := want[i]; g != w {
			return fmt.Errorf("encodings differ at position 0x%x: got 0x%02x, want 0x%02x", i, g, w)
		}
	}
	return nil
}

// encodePAM encodes gotImage in the PAM format.
func encodePAM(gotImage image.Image) ([]byte, error) {
	m, ok := gotImage.(*image.NRGBA)
	if !ok {
		return nil, fmt.Errorf("lossless image did not decode to an *image.NRGBA")
	}
	b := m.Bounds()
	w, h := b.Dx(), b.Dy()
	buf := new(bytes.Buffer)
	fmt.Fprintf(buf, "P7\nWIDTH %d\nHEIGHT %d\nDEPTH 4\nMAXVAL 255\nTUPLTYPE RGB_ALPHA\nENDHDR\n", w, h)
	for y := b.Min.Y; y < b.Max.Y; y++ {
		o := m.PixOffset(b.Min.X, y)
		buf.Write(m.Pix[o : o+4*w])
	}
	return buf.Bytes(), nil
}

// encodePGM encodes gotImage in the PGM format in the IMC4 layout.
func encodePGM(gotImage image.Image) ([]byte, error) {
	var (
		m  *image.YCbCr
		ma *nycbcra.Image
	)
	switch g := gotImage.(type) {
	case *image.YCbCr:
		m = g
	case *nycbcra.Image:
		m = &g.YCbCr
		ma = g
	default:
		return nil, fmt.Errorf("lossy image did not decode to an *image.YCbCr")
	}
	if m.SubsampleRatio != image.YCbCrSubsampleRatio420 {
		return nil, fmt.Errorf("lossy image did not decode to a 4:2:0 YCbCr")
	}
	b := m.Bounds()
	w, h := b.Dx(), b.Dy()
	w2, h2 := (w+1)/2, (h+1)/2
	outW, outH := 2*w2, h+h2
	if ma != nil {
		outH += h
	}
	buf := new(bytes.Buffer)
	fmt.Fprintf(buf, "P5\n%d %d\n255\n", outW, outH)
	for y := b.Min.Y; y < b.Max.Y; y++ {
		o := m.YOffset(b.Min.X, y)
		buf.Write(m.Y[o : o+w])
		if w&1 != 0 {
			buf.WriteByte(0x00)
		}
	}
	for y := b.Min.Y; y < b.Max.Y; y += 2 {
		o := m.COffset(b.Min.X, y)
		buf.Write(m.Cb[o : o+w2])
		buf.Write(m.Cr[o : o+w2])
	}
	if ma != nil {
		for y := b.Min.Y; y < b.Max.Y; y++ {
			o := ma.AOffset(b.Min.X, y)
			buf.Write(ma.A[o : o+w])
			if w&1 != 0 {
				buf.WriteByte(0x00)
			}
		}
	}
	return buf.Bytes(), nil
}

// dump can be useful for debugging.
func dump(w io.Writer, b []byte) {
	h := hex.Dumper(w)
	h.Write(b)
	h.Close()
}
<commit_msg>cmd/webp-manual-test: blacklist some versions of the dwebp binary.<commit_after>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore
//
// This build tag means that "go install golang.org/x/image/..." doesn't
// install this manual test. Use "go run main.go" to explicitly run it.

// Program webp-manual-test checks that the Go WEBP library's decodings match
// the C WEBP library's.
package main // import "golang.org/x/image/cmd/webp-manual-test"

import (
	"bytes"
	"encoding/hex"
	"flag"
	"fmt"
	"image"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"

	"golang.org/x/image/webp"
	"golang.org/x/image/webp/nycbcra"
)

var (
	dwebp = flag.String("dwebp", "/usr/bin/dwebp", "path to the dwebp program "+
		"installed from https://developers.google.com/speed/webp/download")
	testdata = flag.String("testdata", "", "path to the libwebp-test-data directory "+
		"checked out from https://chromium.googlesource.com/webm/libwebp-test-data")
)

func main() {
	flag.Parse()
	if err := checkDwebp(); err != nil {
		flag.Usage()
		log.Fatal(err)
	}
	if *testdata == "" {
		flag.Usage()
		log.Fatal("testdata flag was not specified")
	}

	f, err := os.Open(*testdata)
	if err != nil {
		log.Fatalf("Open: %v", err)
	}
	defer f.Close()
	names, err := f.Readdirnames(-1)
	if err != nil {
		log.Fatalf("Readdirnames: %v", err)
	}
	sort.Strings(names)

	nFail, nPass := 0, 0
	for _, name := range names {
		if !strings.HasSuffix(name, "webp") {
			continue
		}
		if err := test(name); err != nil {
			fmt.Printf("FAIL\t%s\t%v\n", name, err)
			nFail++
		} else {
			fmt.Printf("PASS\t%s\n", name)
			nPass++
		}
	}
	fmt.Printf("%d PASS, %d FAIL, %d TOTAL\n", nPass, nFail, nPass+nFail)
	if nFail != 0 {
		os.Exit(1)
	}
}

func checkDwebp() error {
	if *dwebp == "" {
		return fmt.Errorf("dwebp flag was not specified")
	}
	if _, err := os.Stat(*dwebp); err != nil {
		return fmt.Errorf("could not find dwebp program at %q", *dwebp)
	}
	b, err := exec.Command(*dwebp, "-version").Output()
	if err != nil {
		return fmt.Errorf("could not determine the dwebp program version for %q: %v", *dwebp, err)
	}
	switch s := string(bytes.TrimSpace(b)); s {
	case "0.4.0", "0.4.1", "0.4.2":
		return fmt.Errorf("the dwebp program version %q for %q has a known bug "+
			"(https://bugs.chromium.org/p/webp/issues/detail?id=239). Please use a newer version.", s, *dwebp)
	}
	return nil
}

// test tests a single WEBP image.
func test(name string) error {
	filename := filepath.Join(*testdata, name)
	f, err := os.Open(filename)
	if err != nil {
		return fmt.Errorf("Open: %v", err)
	}
	defer f.Close()

	gotImage, err := webp.Decode(f)
	if err != nil {
		return fmt.Errorf("Decode: %v", err)
	}
	format, encode := "-pgm", encodePGM
	if _, lossless := gotImage.(*image.NRGBA); lossless {
		format, encode = "-pam", encodePAM
	}
	got, err := encode(gotImage)
	if err != nil {
		return fmt.Errorf("encode: %v", err)
	}

	stdout := new(bytes.Buffer)
	stderr := new(bytes.Buffer)
	c := exec.Command(*dwebp, filename, format, "-o", "/dev/stdout")
	c.Stdout = stdout
	c.Stderr = stderr
	if err := c.Run(); err != nil {
		os.Stderr.Write(stderr.Bytes())
		return fmt.Errorf("executing dwebp: %v", err)
	}
	want := stdout.Bytes()

	if len(got) != len(want) {
		return fmt.Errorf("encodings have different length: got %d, want %d", len(got), len(want))
	}
	for i, g := range got {
		if w := want[i]; g != w {
			return fmt.Errorf("encodings differ at position 0x%x: got 0x%02x, want 0x%02x", i, g, w)
		}
	}
	return nil
}

// encodePAM encodes gotImage in the PAM format.
func encodePAM(gotImage image.Image) ([]byte, error) {
	m, ok := gotImage.(*image.NRGBA)
	if !ok {
		return nil, fmt.Errorf("lossless image did not decode to an *image.NRGBA")
	}
	b := m.Bounds()
	w, h := b.Dx(), b.Dy()
	buf := new(bytes.Buffer)
	fmt.Fprintf(buf, "P7\nWIDTH %d\nHEIGHT %d\nDEPTH 4\nMAXVAL 255\nTUPLTYPE RGB_ALPHA\nENDHDR\n", w, h)
	for y := b.Min.Y; y < b.Max.Y; y++ {
		o := m.PixOffset(b.Min.X, y)
		buf.Write(m.Pix[o : o+4*w])
	}
	return buf.Bytes(), nil
}

// encodePGM encodes gotImage in the PGM format in the IMC4 layout.
func encodePGM(gotImage image.Image) ([]byte, error) {
	var (
		m  *image.YCbCr
		ma *nycbcra.Image
	)
	switch g := gotImage.(type) {
	case *image.YCbCr:
		m = g
	case *nycbcra.Image:
		m = &g.YCbCr
		ma = g
	default:
		return nil, fmt.Errorf("lossy image did not decode to an *image.YCbCr")
	}
	if m.SubsampleRatio != image.YCbCrSubsampleRatio420 {
		return nil, fmt.Errorf("lossy image did not decode to a 4:2:0 YCbCr")
	}
	b := m.Bounds()
	w, h := b.Dx(), b.Dy()
	w2, h2 := (w+1)/2, (h+1)/2
	outW, outH := 2*w2, h+h2
	if ma != nil {
		outH += h
	}
	buf := new(bytes.Buffer)
	fmt.Fprintf(buf, "P5\n%d %d\n255\n", outW, outH)
	for y := b.Min.Y; y < b.Max.Y; y++ {
		o := m.YOffset(b.Min.X, y)
		buf.Write(m.Y[o : o+w])
		if w&1 != 0 {
			buf.WriteByte(0x00)
		}
	}
	for y := b.Min.Y; y < b.Max.Y; y += 2 {
		o := m.COffset(b.Min.X, y)
		buf.Write(m.Cb[o : o+w2])
		buf.Write(m.Cr[o : o+w2])
	}
	if ma != nil {
		for y := b.Min.Y; y < b.Max.Y; y++ {
			o := ma.AOffset(b.Min.X, y)
			buf.Write(ma.A[o : o+w])
			if w&1 != 0 {
				buf.WriteByte(0x00)
			}
		}
	}
	return buf.Bytes(), nil
}

// dump can be useful for debugging.
func dump(w io.Writer, b []byte) {
	h := hex.Dumper(w)
	h.Write(b)
	h.Close()
}
<|endoftext|>