<commit_before>package restore

import (
	"archive/tar"
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io"
	"net"
	"os"
	"path/filepath"
	"sync"

	"github.com/influxdb/influxdb/cmd/influxd/backup"
	"github.com/influxdb/influxdb/meta"
)

// Command represents the program execution for "influxd restore".
type Command struct {
	Stdout io.Writer
	Stderr io.Writer

	backupFilesPath string
	metadir         string
	datadir         string
	database        string
	retention       string
	shard           string

	// TODO: when the new meta stuff is done this should not be exported or be gone
	MetaConfig *meta.Config
}

// NewCommand returns a new instance of Command with default settings.
func NewCommand() *Command {
	return &Command{
		Stdout:     os.Stdout,
		Stderr:     os.Stderr,
		MetaConfig: meta.NewConfig(),
	}
}

// Run executes the program.
func (cmd *Command) Run(args ...string) error {
	if err := cmd.parseFlags(args); err != nil {
		return err
	}

	if cmd.metadir != "" {
		if err := cmd.unpackMeta(); err != nil {
			return err
		}
	}

	if cmd.shard != "" {
		return cmd.unpackShard(cmd.shard)
	} else if cmd.retention != "" {
		return cmd.unpackRetention()
	}

	return cmd.unpackDatabase()
}

// parseFlags parses and validates the command line arguments.
func (cmd *Command) parseFlags(args []string) error {
	fs := flag.NewFlagSet("", flag.ContinueOnError)
	fs.StringVar(&cmd.metadir, "metadir", "", "")
	fs.StringVar(&cmd.datadir, "datadir", "", "")
	fs.StringVar(&cmd.database, "database", "", "")
	fs.StringVar(&cmd.retention, "retention", "", "")
	fs.StringVar(&cmd.shard, "shard", "", "")
	fs.SetOutput(cmd.Stderr)
	fs.Usage = cmd.printUsage
	if err := fs.Parse(args); err != nil {
		return err
	}

	cmd.MetaConfig = meta.NewConfig()
	cmd.MetaConfig.Dir = cmd.metadir

	// Require output path.
	cmd.backupFilesPath = fs.Arg(0)
	if cmd.backupFilesPath == "" {
		return fmt.Errorf("path with backup files required")
	}

	// validate the arguments
	if cmd.metadir == "" && cmd.database == "" {
		return fmt.Errorf("either a metadir or database are required to restore")
	}

	if cmd.datadir == "" {
		return fmt.Errorf("datadir is required to restore")
	}

	if cmd.shard != "" {
		if cmd.database == "" {
			return fmt.Errorf("database is required to restore shard")
		}
		if cmd.retention == "" {
			return fmt.Errorf("retention is required to restore shard")
		}
	} else if cmd.retention != "" && cmd.database == "" {
		return fmt.Errorf("database is required to restore retention policy")
	}

	return nil
}

// unpackMeta reads the metadata from the backup directory and initializes a raft
// cluster and replaces the root metadata.
func (cmd *Command) unpackMeta() error {
	// find the meta file
	metaFiles, err := filepath.Glob(filepath.Join(cmd.backupFilesPath, backup.Metafile+".*"))
	if err != nil {
		return err
	}
	if len(metaFiles) == 0 {
		return fmt.Errorf("no metastore backups in %s", cmd.backupFilesPath)
	}

	// Read the metastore backup
	f, err := os.Open(metaFiles[len(metaFiles)-1])
	if err != nil {
		return err
	}

	var buf bytes.Buffer
	if _, err := io.Copy(&buf, f); err != nil {
		return fmt.Errorf("copy: %s", err)
	}

	// Unpack into metadata.
	var data meta.Data
	if err := data.UnmarshalBinary(buf.Bytes()); err != nil {
		return fmt.Errorf("unmarshal: %s", err)
	}

	// Initialize meta store.
	store := meta.NewStore(cmd.MetaConfig)
	store.RaftListener = newNopListener()
	store.ExecListener = newNopListener()
	store.RPCListener = newNopListener()

	// Determine advertised address.
	_, port, err := net.SplitHostPort(cmd.MetaConfig.BindAddress)
	if err != nil {
		return fmt.Errorf("split bind address: %s", err)
	}
	hostport := net.JoinHostPort(cmd.MetaConfig.Hostname, port)

	// Resolve address.
	addr, err := net.ResolveTCPAddr("tcp", hostport)
	if err != nil {
		return fmt.Errorf("resolve tcp: addr=%s, err=%s", hostport, err)
	}
	store.Addr = addr
	store.RemoteAddr = addr

	// Open the meta store.
	if err := store.Open(); err != nil {
		return fmt.Errorf("open store: %s", err)
	}
	defer store.Close()

	// Wait for the store to be ready or error.
	select {
	case <-store.Ready():
	case err := <-store.Err():
		return err
	}

	// Force set the full metadata.
	if err := store.SetData(&data); err != nil {
		return fmt.Errorf("set data: %s", err)
	}

	return nil
}

// unpackShard will look for all backup files in the path matching this shard ID
// and restore them to the data dir
func (cmd *Command) unpackShard(shardID string) error {
	// make sure the shard isn't already there so we don't clobber anything
	restorePath := filepath.Join(cmd.datadir, cmd.database, cmd.retention, shardID)
	if _, err := os.Stat(restorePath); err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("shard already present: %s", restorePath)
	}

	// find the shard backup files
	pat := filepath.Join(cmd.backupFilesPath, fmt.Sprintf(backup.BackupFilePattern, cmd.database, cmd.retention, shardID))
	return cmd.unpackFiles(pat + ".*")
}

// unpackDatabase will look for all backup files in the path matching this database
// and restore them to the data dir
func (cmd *Command) unpackDatabase() error {
	// make sure the shard isn't already there so we don't clobber anything
	restorePath := filepath.Join(cmd.datadir, cmd.database)
	if _, err := os.Stat(restorePath); err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("database already present: %s", restorePath)
	}

	// find the database backup files
	pat := filepath.Join(cmd.backupFilesPath, cmd.database)
	return cmd.unpackFiles(pat + ".*")
}

// unpackRetention will look for all backup files in the path matching this retention
// and restore them to the data dir
func (cmd *Command) unpackRetention() error {
	// make sure the shard isn't already there so we don't clobber anything
	restorePath := filepath.Join(cmd.datadir, cmd.database, cmd.retention)
	if _, err := os.Stat(restorePath); err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("retention already present: %s", restorePath)
	}

	// find the retention backup files
	pat := filepath.Join(cmd.backupFilesPath, cmd.database)
	return cmd.unpackFiles(fmt.Sprintf("%s.%s.*", pat, cmd.retention))
}

// unpackFiles will look for backup files matching the pattern and restore them to the data dir
func (cmd *Command) unpackFiles(pat string) error {
	fmt.Printf("Restoring from backup %s\n", pat)

	backupFiles, err := filepath.Glob(pat)
	if err != nil {
		return err
	}

	if len(backupFiles) == 0 {
		return fmt.Errorf("no backup files for %s in %s", pat, cmd.backupFilesPath)
	}

	for _, fn := range backupFiles {
		if err := cmd.unpackTar(fn); err != nil {
			return err
		}
	}

	return nil
}

// unpackTar will restore a single tar archive to the data dir
func (cmd *Command) unpackTar(tarFile string) error {
	f, err := os.Open(tarFile)
	if err != nil {
		return err
	}
	defer f.Close()

	tr := tar.NewReader(f)

	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			return nil
		} else if err != nil {
			return err
		}

		if err := cmd.unpackFile(tr, hdr.Name); err != nil {
			return err
		}
	}
}

// unpackFile will copy the current file from the tar archive to the data dir
func (cmd *Command) unpackFile(tr *tar.Reader, fileName string) error {
	fn := filepath.Join(cmd.datadir, fileName)
	fmt.Printf("unpacking %s", fn)

	if err := os.MkdirAll(filepath.Dir(fn), 0777); err != nil {
		return fmt.Errorf("error making restore dir: %s", err.Error())
	}

	ff, err := os.Create(fn)
	if err != nil {
		return err
	}
	defer ff.Close()

	if _, err := io.Copy(ff, tr); err != nil {
		return err
	}

	return nil
}

// printUsage prints the usage message to STDERR.
func (cmd *Command) printUsage() {
	fmt.Fprintf(cmd.Stderr, `usage: influxd restore [flags] PATH

restore uses backups from the PATH to restore the metastore, databases, retention policies, or specific shards. The InfluxDB process must not be running during restore.

        -metadir <path>
                    	Optional. If set the metastore will be recovered to the given path.
        -datadir <path>
        				Optional. If set the restore process will recover the specified
        				database, retention policy or shard to the given directory.
       	-database <name>
       					Optional. Required if no metadir given. Will restore the database TSM files.
       	-retention <name>
       					Optional. If given, database is required. Will restore the retention policy's TSM files.
       	-shard <id>
       					Optional. If given, database and retention are required. Will restore the shard's TSM files.
`)
}

type nopListener struct {
	mu      sync.Mutex
	closing chan struct{}
}

func newNopListener() *nopListener {
	return &nopListener{closing: make(chan struct{})}
}

func (ln *nopListener) Accept() (net.Conn, error) {
	ln.mu.Lock()
	defer ln.mu.Unlock()

	<-ln.closing
	return nil, errors.New("listener closing")
}

func (ln *nopListener) Close() error {
	if ln.closing != nil {
		close(ln.closing)
		ln.mu.Lock()
		defer ln.mu.Unlock()

		ln.closing = nil
	}
	return nil
}

func (ln *nopListener) Addr() net.Addr { return nil }
<commit_msg>Fix go vet in restore.go<commit_after>package restore

import (
	"archive/tar"
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io"
	"net"
	"os"
	"path/filepath"
	"strconv"
	"sync"

	"github.com/influxdb/influxdb/cmd/influxd/backup"
	"github.com/influxdb/influxdb/meta"
)

// Command represents the program execution for "influxd restore".
type Command struct {
	Stdout io.Writer
	Stderr io.Writer

	backupFilesPath string
	metadir         string
	datadir         string
	database        string
	retention       string
	shard           string

	// TODO: when the new meta stuff is done this should not be exported or be gone
	MetaConfig *meta.Config
}

// NewCommand returns a new instance of Command with default settings.
func NewCommand() *Command {
	return &Command{
		Stdout:     os.Stdout,
		Stderr:     os.Stderr,
		MetaConfig: meta.NewConfig(),
	}
}

// Run executes the program.
func (cmd *Command) Run(args ...string) error {
	if err := cmd.parseFlags(args); err != nil {
		return err
	}

	if cmd.metadir != "" {
		if err := cmd.unpackMeta(); err != nil {
			return err
		}
	}

	if cmd.shard != "" {
		return cmd.unpackShard(cmd.shard)
	} else if cmd.retention != "" {
		return cmd.unpackRetention()
	}

	return cmd.unpackDatabase()
}

// parseFlags parses and validates the command line arguments.
func (cmd *Command) parseFlags(args []string) error {
	fs := flag.NewFlagSet("", flag.ContinueOnError)
	fs.StringVar(&cmd.metadir, "metadir", "", "")
	fs.StringVar(&cmd.datadir, "datadir", "", "")
	fs.StringVar(&cmd.database, "database", "", "")
	fs.StringVar(&cmd.retention, "retention", "", "")
	fs.StringVar(&cmd.shard, "shard", "", "")
	fs.SetOutput(cmd.Stderr)
	fs.Usage = cmd.printUsage
	if err := fs.Parse(args); err != nil {
		return err
	}

	cmd.MetaConfig = meta.NewConfig()
	cmd.MetaConfig.Dir = cmd.metadir

	// Require output path.
	cmd.backupFilesPath = fs.Arg(0)
	if cmd.backupFilesPath == "" {
		return fmt.Errorf("path with backup files required")
	}

	// validate the arguments
	if cmd.metadir == "" && cmd.database == "" {
		return fmt.Errorf("either a metadir or database are required to restore")
	}

	if cmd.datadir == "" {
		return fmt.Errorf("datadir is required to restore")
	}

	if cmd.shard != "" {
		if cmd.database == "" {
			return fmt.Errorf("database is required to restore shard")
		}
		if cmd.retention == "" {
			return fmt.Errorf("retention is required to restore shard")
		}
	} else if cmd.retention != "" && cmd.database == "" {
		return fmt.Errorf("database is required to restore retention policy")
	}

	return nil
}

// unpackMeta reads the metadata from the backup directory and initializes a raft
// cluster and replaces the root metadata.
func (cmd *Command) unpackMeta() error {
	// find the meta file
	metaFiles, err := filepath.Glob(filepath.Join(cmd.backupFilesPath, backup.Metafile+".*"))
	if err != nil {
		return err
	}
	if len(metaFiles) == 0 {
		return fmt.Errorf("no metastore backups in %s", cmd.backupFilesPath)
	}

	// Read the metastore backup
	f, err := os.Open(metaFiles[len(metaFiles)-1])
	if err != nil {
		return err
	}

	var buf bytes.Buffer
	if _, err := io.Copy(&buf, f); err != nil {
		return fmt.Errorf("copy: %s", err)
	}

	// Unpack into metadata.
	var data meta.Data
	if err := data.UnmarshalBinary(buf.Bytes()); err != nil {
		return fmt.Errorf("unmarshal: %s", err)
	}

	// Initialize meta store.
	store := meta.NewStore(cmd.MetaConfig)
	store.RaftListener = newNopListener()
	store.ExecListener = newNopListener()
	store.RPCListener = newNopListener()

	// Determine advertised address.
	_, port, err := net.SplitHostPort(cmd.MetaConfig.BindAddress)
	if err != nil {
		return fmt.Errorf("split bind address: %s", err)
	}
	hostport := net.JoinHostPort(cmd.MetaConfig.Hostname, port)

	// Resolve address.
	addr, err := net.ResolveTCPAddr("tcp", hostport)
	if err != nil {
		return fmt.Errorf("resolve tcp: addr=%s, err=%s", hostport, err)
	}
	store.Addr = addr
	store.RemoteAddr = addr

	// Open the meta store.
	if err := store.Open(); err != nil {
		return fmt.Errorf("open store: %s", err)
	}
	defer store.Close()

	// Wait for the store to be ready or error.
	select {
	case <-store.Ready():
	case err := <-store.Err():
		return err
	}

	// Force set the full metadata.
	if err := store.SetData(&data); err != nil {
		return fmt.Errorf("set data: %s", err)
	}

	return nil
}

// unpackShard will look for all backup files in the path matching this shard ID
// and restore them to the data dir
func (cmd *Command) unpackShard(shardID string) error {
	// make sure the shard isn't already there so we don't clobber anything
	restorePath := filepath.Join(cmd.datadir, cmd.database, cmd.retention, shardID)
	if _, err := os.Stat(restorePath); err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("shard already present: %s", restorePath)
	}

	id, err := strconv.ParseUint(shardID, 10, 64)
	if err != nil {
		return err
	}

	// find the shard backup files
	pat := filepath.Join(cmd.backupFilesPath, fmt.Sprintf(backup.BackupFilePattern, cmd.database, cmd.retention, id))
	return cmd.unpackFiles(pat + ".*")
}

// unpackDatabase will look for all backup files in the path matching this database
// and restore them to the data dir
func (cmd *Command) unpackDatabase() error {
	// make sure the shard isn't already there so we don't clobber anything
	restorePath := filepath.Join(cmd.datadir, cmd.database)
	if _, err := os.Stat(restorePath); err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("database already present: %s", restorePath)
	}

	// find the database backup files
	pat := filepath.Join(cmd.backupFilesPath, cmd.database)
	return cmd.unpackFiles(pat + ".*")
}

// unpackRetention will look for all backup files in the path matching this retention
// and restore them to the data dir
func (cmd *Command) unpackRetention() error {
	// make sure the shard isn't already there so we don't clobber anything
	restorePath := filepath.Join(cmd.datadir, cmd.database, cmd.retention)
	if _, err := os.Stat(restorePath); err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("retention already present: %s", restorePath)
	}

	// find the retention backup files
	pat := filepath.Join(cmd.backupFilesPath, cmd.database)
	return cmd.unpackFiles(fmt.Sprintf("%s.%s.*", pat, cmd.retention))
}

// unpackFiles will look for backup files matching the pattern and restore them to the data dir
func (cmd *Command) unpackFiles(pat string) error {
	fmt.Printf("Restoring from backup %s\n", pat)

	backupFiles, err := filepath.Glob(pat)
	if err != nil {
		return err
	}

	if len(backupFiles) == 0 {
		return fmt.Errorf("no backup files for %s in %s", pat, cmd.backupFilesPath)
	}

	for _, fn := range backupFiles {
		if err := cmd.unpackTar(fn); err != nil {
			return err
		}
	}

	return nil
}

// unpackTar will restore a single tar archive to the data dir
func (cmd *Command) unpackTar(tarFile string) error {
	f, err := os.Open(tarFile)
	if err != nil {
		return err
	}
	defer f.Close()

	tr := tar.NewReader(f)

	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			return nil
		} else if err != nil {
			return err
		}

		if err := cmd.unpackFile(tr, hdr.Name); err != nil {
			return err
		}
	}
}

// unpackFile will copy the current file from the tar archive to the data dir
func (cmd *Command) unpackFile(tr *tar.Reader, fileName string) error {
	fn := filepath.Join(cmd.datadir, fileName)
	fmt.Printf("unpacking %s", fn)

	if err := os.MkdirAll(filepath.Dir(fn), 0777); err != nil {
		return fmt.Errorf("error making restore dir: %s", err.Error())
	}

	ff, err := os.Create(fn)
	if err != nil {
		return err
	}
	defer ff.Close()

	if _, err := io.Copy(ff, tr); err != nil {
		return err
	}

	return nil
}

// printUsage prints the usage message to STDERR.
func (cmd *Command) printUsage() {
	fmt.Fprintf(cmd.Stderr, `usage: influxd restore [flags] PATH

restore uses backups from the PATH to restore the metastore, databases, retention policies, or specific shards. The InfluxDB process must not be running during restore.

        -metadir <path>
                    	Optional. If set the metastore will be recovered to the given path.
        -datadir <path>
        				Optional. If set the restore process will recover the specified
        				database, retention policy or shard to the given directory.
       	-database <name>
       					Optional. Required if no metadir given. Will restore the database TSM files.
       	-retention <name>
       					Optional. If given, database is required. Will restore the retention policy's TSM files.
       	-shard <id>
       					Optional. If given, database and retention are required. Will restore the shard's TSM files.
`)
}

type nopListener struct {
	mu      sync.Mutex
	closing chan struct{}
}

func newNopListener() *nopListener {
	return &nopListener{closing: make(chan struct{})}
}

func (ln *nopListener) Accept() (net.Conn, error) {
	ln.mu.Lock()
	defer ln.mu.Unlock()

	<-ln.closing
	return nil, errors.New("listener closing")
}

func (ln *nopListener) Close() error {
	if ln.closing != nil {
		close(ln.closing)
		ln.mu.Lock()
		defer ln.mu.Unlock()

		ln.closing = nil
	}
	return nil
}

func (ln *nopListener) Addr() net.Addr { return nil }
<|endoftext|>


<commit_before>// Copyright 2021 Google Inc.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// This executable builds the Docker images based off the Skia executables in the
// gcr.io/skia-public/skia-release image. It then issues a PubSub notification to have those apps
// tagged and deployed by docker_pushes_watcher.
// See //docker_pushes_watcher/README.md in the infra repo for more.
package main

import (
	"context"
	"flag"
	"fmt"
	"os"
	"path"
	"path/filepath"

	"cloud.google.com/go/pubsub"
	"google.golang.org/api/option"

	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	docker_pubsub "go.skia.org/infra/go/docker/build/pubsub"
	sk_exec "go.skia.org/infra/go/exec"
	"go.skia.org/infra/task_driver/go/lib/auth_steps"
	"go.skia.org/infra/task_driver/go/lib/bazel"
	"go.skia.org/infra/task_driver/go/lib/checkout"
	"go.skia.org/infra/task_driver/go/lib/docker"
	"go.skia.org/infra/task_driver/go/lib/golang"
	"go.skia.org/infra/task_driver/go/lib/os_steps"
	"go.skia.org/infra/task_driver/go/td"
	"go.skia.org/infra/task_scheduler/go/types"
)

var (
	// Required properties for this task.
	projectId     = flag.String("project_id", "", "ID of the Google Cloud project.")
	taskId        = flag.String("task_id", "", "ID of this task.")
	taskName      = flag.String("task_name", "", "Name of the task.")
	workdir       = flag.String("workdir", ".", "Working directory")
	infraRevision = flag.String("infra_revision", "origin/main", "Specifies which revision of the infra repo the images should be built off")

	checkoutFlags = checkout.SetupFlags(nil)

	// Optional flags.
	local  = flag.Bool("local", false, "True if running locally (as opposed to on the bots)")
	output = flag.String("o", "", "If provided, dump a JSON blob of step data to the given file. Prints to stdout if '-' is given.")
)

const (
	fiddlerImageName = "fiddler"
	apiImageName     = "api"
)

func buildPushFiddlerImage(ctx context.Context, dkr *docker.Docker, tag, infraCheckoutDir string, topic *pubsub.Topic) error {
	// Run skia-release image and extract products out of /tmp/skia/skia. See
	// https://skia.googlesource.com/skia/+/0e845dc8b05cb2d40d1c880184e33dd76081283a/docker/skia-release/Dockerfile#33
	productsDir, err := os_steps.TempDir(ctx, "", "")
	if err != nil {
		return err
	}
	volumes := []string{
		fmt.Sprintf("%s:/OUT", productsDir),
	}
	skiaCopyCmd := []string{"/bin/sh", "-c", "cd /tmp; tar cvzf skia.tar.gz --directory=/tmp/skia skia; cp /tmp/skia.tar.gz /OUT/"}
	releaseImg := fmt.Sprintf("gcr.io/skia-public/skia-release:%s", tag)
	if err := dkr.Run(ctx, releaseImg, skiaCopyCmd, volumes, nil); err != nil {
		return err
	}

	err = td.Do(ctx, td.Props("Build "+fiddlerImageName+" image").Infra(), func(ctx context.Context) error {
		runCmd := &sk_exec.Command{
			Name:       "make",
			Args:       []string{"release-fiddler-ci"},
			InheritEnv: true,
			Env: []string{
				"COPY_FROM_DIR=" + productsDir,
				"STABLE_DOCKER_TAG=" + tag,
			},
			Dir:       filepath.Join(infraCheckoutDir, "fiddlek"),
			LogStdout: true,
			LogStderr: true,
		}
		_, err := sk_exec.RunCommand(ctx, runCmd)
		if err != nil {
			return err
		}
		return nil
	})
	if err != nil {
		return err
	}
	if err := docker.PublishToTopic(ctx, "gcr.io/skia-public/"+fiddlerImageName, tag, common.REPO_SKIA, topic); err != nil {
		return err
	}

	return cleanupTempFiles(ctx, dkr, releaseImg, volumes)
}

func cleanupTempFiles(ctx context.Context, dkr *docker.Docker, image string, volumes []string) error {
	// Remove all temporary files from the host machine. Swarming gets upset if there are root-owned
	// files it cannot clean up.
	cleanupCmd := []string{"/bin/sh", "-c", "rm -rf /OUT/*"}
	return dkr.Run(ctx, image, cleanupCmd, volumes, nil)
}

func buildPushApiImage(ctx context.Context, dkr *docker.Docker, tag, checkoutDir, infraCheckoutDir string, topic *pubsub.Topic) error {
	tempDir, err := os_steps.TempDir(ctx, "", "")
	if err != nil {
		return err
	}
	// Change perms of the directory for doxygen to be able to write to it.
	if err := os.Chmod(tempDir, 0777); err != nil {
		return err
	}
	// Run Doxygen pointing to the location of the checkout and the out dir.
	volumes := []string{
		fmt.Sprintf("%s:/OUT", tempDir),
		fmt.Sprintf("%s:/CHECKOUT", checkoutDir),
	}
	env := []string{
		"OUTPUT_DIRECTORY=/OUT",
	}
	doxygenCmd := []string{"/bin/sh", "-c", "cd /CHECKOUT/tools/doxygen && doxygen ProdDoxyfile"}
	doxygenImg := "gcr.io/skia-public/doxygen:testing-slim"
	if err := dkr.Run(ctx, doxygenImg, doxygenCmd, volumes, env); err != nil {
		return err
	}

	err = td.Do(ctx, td.Props("Build "+apiImageName+" image").Infra(), func(ctx context.Context) error {
		runCmd := &sk_exec.Command{
			Name:       "make",
			Args:       []string{"release-api-ci"},
			InheritEnv: true,
			Env: []string{
				"COPY_FROM_DIR=" + filepath.Join(tempDir, "html"),
				"STABLE_DOCKER_TAG=" + tag,
			},
			Dir:       filepath.Join(infraCheckoutDir, "api"),
			LogStdout: true,
			LogStderr: true,
		}
		_, err := sk_exec.RunCommand(ctx, runCmd)
		if err != nil {
			return err
		}
		return nil
	})
	if err != nil {
		return err
	}
	if err := docker.PublishToTopic(ctx, "gcr.io/skia-public/"+apiImageName, tag, common.REPO_SKIA, topic); err != nil {
		return err
	}

	return cleanupTempFiles(ctx, dkr, doxygenImg, volumes)
}

func main() {
	// Setup.
	ctx := td.StartRun(projectId, taskId, taskName, output, local)
	defer td.EndRun(ctx)

	if *infraRevision == "" {
		td.Fatalf(ctx, "Must specify --infra_revision")
	}

	rs, err := checkout.GetRepoState(checkoutFlags)
	if err != nil {
		td.Fatal(ctx, err)
	}
	wd, err := os_steps.Abs(ctx, *workdir)
	if err != nil {
		td.Fatal(ctx, err)
	}
	// Check out the Skia repo code.
	co, err := checkout.EnsureGitCheckout(ctx, path.Join(wd, "repo"), rs)
	if err != nil {
		td.Fatal(ctx, err)
	}
	skiaCheckoutDir := co.Dir()

	// Checkout out the Skia infra repo at the specified commit.
	infraRS := types.RepoState{
		Repo:     common.REPO_SKIA_INFRA,
		Revision: *infraRevision,
	}
	infraCheckoutDir := filepath.Join("infra_repo")
	if _, err := checkout.EnsureGitCheckout(ctx, infraCheckoutDir, infraRS); err != nil {
		td.Fatal(ctx, err)
	}

	// Setup go.
	ctx = golang.WithEnv(ctx, wd)

	// Ensure that the bazel cache is setup.
	opts := bazel.BazelOptions{
		CachePath: "/mnt/pd0/bazel_cache",
	}
	if err := bazel.EnsureBazelRCFile(ctx, opts); err != nil {
		td.Fatal(ctx, err)
	}

	// Create token source with scope for cloud registry (storage) and pubsub.
	ts, err := auth_steps.Init(ctx, *local, auth.ScopeUserinfoEmail, auth.ScopeFullControl, pubsub.ScopePubSub)
	if err != nil {
		td.Fatal(ctx, err)
	}

	// Create pubsub client.
	client, err := pubsub.NewClient(ctx, docker_pubsub.TOPIC_PROJECT_ID, option.WithTokenSource(ts))
	if err != nil {
		td.Fatal(ctx, err)
	}
	topic := client.Topic(docker_pubsub.TOPIC)

	// Figure out which tag to use for docker build and push.
	tag := rs.Revision
	if rs.Issue != "" && rs.Patchset != "" {
		tag = fmt.Sprintf("%s_%s", rs.Issue, rs.Patchset)
	}

	// Instantiate docker.
	dkr, err := docker.New(ctx, ts)
	if err != nil {
		td.Fatal(ctx, err)
	}

	// Build and push all apps of interest below.
	if err := buildPushApiImage(ctx, dkr, tag, skiaCheckoutDir, infraCheckoutDir, topic); err != nil {
		td.Fatal(ctx, err)
	}
	if err := buildPushFiddlerImage(ctx, dkr, tag, infraCheckoutDir, topic); err != nil {
		td.Fatal(ctx, err)
	}
}
<commit_msg>Pull the latest doxygen image<commit_after>// Copyright 2021 Google Inc.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// This executable builds the Docker images based off the Skia executables in the
// gcr.io/skia-public/skia-release image. It then issues a PubSub notification to have those apps
// tagged and deployed by docker_pushes_watcher.
// See //docker_pushes_watcher/README.md in the infra repo for more.
package main

import (
	"context"
	"flag"
	"fmt"
	"os"
	"path"
	"path/filepath"

	"cloud.google.com/go/pubsub"
	"google.golang.org/api/option"

	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	docker_pubsub "go.skia.org/infra/go/docker/build/pubsub"
	sk_exec "go.skia.org/infra/go/exec"
	"go.skia.org/infra/task_driver/go/lib/auth_steps"
	"go.skia.org/infra/task_driver/go/lib/bazel"
	"go.skia.org/infra/task_driver/go/lib/checkout"
	"go.skia.org/infra/task_driver/go/lib/docker"
	"go.skia.org/infra/task_driver/go/lib/golang"
	"go.skia.org/infra/task_driver/go/lib/os_steps"
	"go.skia.org/infra/task_driver/go/td"
	"go.skia.org/infra/task_scheduler/go/types"
)

var (
	// Required properties for this task.
	projectId     = flag.String("project_id", "", "ID of the Google Cloud project.")
	taskId        = flag.String("task_id", "", "ID of this task.")
	taskName      = flag.String("task_name", "", "Name of the task.")
	workdir       = flag.String("workdir", ".", "Working directory")
	infraRevision = flag.String("infra_revision", "origin/main", "Specifies which revision of the infra repo the images should be built off")

	checkoutFlags = checkout.SetupFlags(nil)

	// Optional flags.
	local  = flag.Bool("local", false, "True if running locally (as opposed to on the bots)")
	output = flag.String("o", "", "If provided, dump a JSON blob of step data to the given file. Prints to stdout if '-' is given.")
)

const (
	fiddlerImageName = "fiddler"
	apiImageName     = "api"
)

func buildPushFiddlerImage(ctx context.Context, dkr *docker.Docker, tag, infraCheckoutDir string, topic *pubsub.Topic) error {
	// Run skia-release image and extract products out of /tmp/skia/skia. See
	// https://skia.googlesource.com/skia/+/0e845dc8b05cb2d40d1c880184e33dd76081283a/docker/skia-release/Dockerfile#33
	productsDir, err := os_steps.TempDir(ctx, "", "")
	if err != nil {
		return err
	}
	volumes := []string{
		fmt.Sprintf("%s:/OUT", productsDir),
	}
	skiaCopyCmd := []string{"/bin/sh", "-c", "cd /tmp; tar cvzf skia.tar.gz --directory=/tmp/skia skia; cp /tmp/skia.tar.gz /OUT/"}
	releaseImg := fmt.Sprintf("gcr.io/skia-public/skia-release:%s", tag)
	if err := dkr.Run(ctx, releaseImg, skiaCopyCmd, volumes, nil); err != nil {
		return err
	}

	err = td.Do(ctx, td.Props("Build "+fiddlerImageName+" image").Infra(), func(ctx context.Context) error {
		runCmd := &sk_exec.Command{
			Name:       "make",
			Args:       []string{"release-fiddler-ci"},
			InheritEnv: true,
			Env: []string{
				"COPY_FROM_DIR=" + productsDir,
				"STABLE_DOCKER_TAG=" + tag,
			},
			Dir:       filepath.Join(infraCheckoutDir, "fiddlek"),
			LogStdout: true,
			LogStderr: true,
		}
		_, err := sk_exec.RunCommand(ctx, runCmd)
		if err != nil {
			return err
		}
		return nil
	})
	if err != nil {
		return err
	}
	if err := docker.PublishToTopic(ctx, "gcr.io/skia-public/"+fiddlerImageName, tag, common.REPO_SKIA, topic); err != nil {
		return err
	}

	return cleanupTempFiles(ctx, dkr, releaseImg, volumes)
}

func cleanupTempFiles(ctx context.Context, dkr *docker.Docker, image string, volumes []string) error {
	// Remove all temporary files from the host machine. Swarming gets upset if there are root-owned
	// files it cannot clean up.
	cleanupCmd := []string{"/bin/sh", "-c", "rm -rf /OUT/*"}
	return dkr.Run(ctx, image, cleanupCmd, volumes, nil)
}

func buildPushApiImage(ctx context.Context, dkr *docker.Docker, tag, checkoutDir, infraCheckoutDir string, topic *pubsub.Topic) error {
	tempDir, err := os_steps.TempDir(ctx, "", "")
	if err != nil {
		return err
	}
	// Change perms of the directory for doxygen to be able to write to it.
	if err := os.Chmod(tempDir, 0777); err != nil {
		return err
	}
	// Run Doxygen pointing to the location of the checkout and the out dir.
	volumes := []string{
		fmt.Sprintf("%s:/OUT", tempDir),
		fmt.Sprintf("%s:/CHECKOUT", checkoutDir),
	}
	env := []string{
		"OUTPUT_DIRECTORY=/OUT",
	}
	doxygenCmd := []string{"/bin/sh", "-c", "cd /CHECKOUT/tools/doxygen && doxygen ProdDoxyfile"}
	doxygenImg := "gcr.io/skia-public/doxygen:testing-slim"
	// Make sure we have the latest doxygen image.
	if err := dkr.Pull(ctx, doxygenImg); err != nil {
		return err
	}
	if err := dkr.Run(ctx, doxygenImg, doxygenCmd, volumes, env); err != nil {
		return err
	}

	err = td.Do(ctx, td.Props("Build "+apiImageName+" image").Infra(), func(ctx context.Context) error {
		runCmd := &sk_exec.Command{
			Name:       "make",
			Args:       []string{"release-api-ci"},
			InheritEnv: true,
			Env: []string{
				"COPY_FROM_DIR=" + filepath.Join(tempDir, "html"),
				"STABLE_DOCKER_TAG=" + tag,
			},
			Dir:       filepath.Join(infraCheckoutDir, "api"),
			LogStdout: true,
			LogStderr: true,
		}
		_, err := sk_exec.RunCommand(ctx, runCmd)
		if err != nil {
			return err
		}
		return nil
	})
	if err != nil {
		return err
	}
	if err := docker.PublishToTopic(ctx, "gcr.io/skia-public/"+apiImageName, tag, common.REPO_SKIA, topic); err != nil {
		return err
	}

	return cleanupTempFiles(ctx, dkr, doxygenImg, volumes)
}

func main() {
	// Setup.
	ctx := td.StartRun(projectId, taskId, taskName, output, local)
	defer td.EndRun(ctx)

	if *infraRevision == "" {
		td.Fatalf(ctx, "Must specify --infra_revision")
	}

	rs, err := checkout.GetRepoState(checkoutFlags)
	if err != nil {
		td.Fatal(ctx, err)
	}
	wd, err := os_steps.Abs(ctx, *workdir)
	if err != nil {
		td.Fatal(ctx, err)
	}
	// Check out the Skia repo code.
	co, err := checkout.EnsureGitCheckout(ctx, path.Join(wd, "repo"), rs)
	if err != nil {
		td.Fatal(ctx, err)
	}
	skiaCheckoutDir := co.Dir()

	// Checkout out the Skia infra repo at the specified commit.
	infraRS := types.RepoState{
		Repo:     common.REPO_SKIA_INFRA,
		Revision: *infraRevision,
	}
	infraCheckoutDir := filepath.Join("infra_repo")
	if _, err := checkout.EnsureGitCheckout(ctx, infraCheckoutDir, infraRS); err != nil {
		td.Fatal(ctx, err)
	}

	// Setup go.
	ctx = golang.WithEnv(ctx, wd)

	// Ensure that the bazel cache is setup.
	opts := bazel.BazelOptions{
		CachePath: "/mnt/pd0/bazel_cache",
	}
	if err := bazel.EnsureBazelRCFile(ctx, opts); err != nil {
		td.Fatal(ctx, err)
	}

	// Create token source with scope for cloud registry (storage) and pubsub.
	ts, err := auth_steps.Init(ctx, *local, auth.ScopeUserinfoEmail, auth.ScopeFullControl, pubsub.ScopePubSub)
	if err != nil {
		td.Fatal(ctx, err)
	}

	// Create pubsub client.
	client, err := pubsub.NewClient(ctx, docker_pubsub.TOPIC_PROJECT_ID, option.WithTokenSource(ts))
	if err != nil {
		td.Fatal(ctx, err)
	}
	topic := client.Topic(docker_pubsub.TOPIC)

	// Figure out which tag to use for docker build and push.
	tag := rs.Revision
	if rs.Issue != "" && rs.Patchset != "" {
		tag = fmt.Sprintf("%s_%s", rs.Issue, rs.Patchset)
	}

	// Instantiate docker.
	dkr, err := docker.New(ctx, ts)
	if err != nil {
		td.Fatal(ctx, err)
	}

	// Build and push all apps of interest below.
	if err := buildPushApiImage(ctx, dkr, tag, skiaCheckoutDir, infraCheckoutDir, topic); err != nil {
		td.Fatal(ctx, err)
	}
	if err := buildPushFiddlerImage(ctx, dkr, tag, infraCheckoutDir, topic); err != nil {
		td.Fatal(ctx, err)
	}
}
<|endoftext|>


<commit_before>/*
 * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"path/filepath"
	"sort"
	"strings"
)

func listDirHealFactory(isLeaf isLeafFunc, disks ...StorageAPI) listDirFunc {
	// Returns sorted merged entries from all the disks.
	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string, delayIsLeaf bool, err error) {
		for _, disk := range disks {
			if disk == nil {
				continue
			}
			var entries []string
			var newEntries []string
			entries, err = disk.ListDir(bucket, prefixDir)
			if err != nil {
				continue
			}
			// Listing needs to be sorted.
			sort.Strings(entries)

			// Filter entries that have the prefix prefixEntry.
			entries = filterMatchingPrefix(entries, prefixEntry)

			// isLeaf() check has to happen here so that trailing "/" for objects can be removed.
			for i, entry := range entries {
				if isLeaf(bucket, pathJoin(prefixDir, entry)) {
					entries[i] = strings.TrimSuffix(entry, slashSeparator)
				}
			}
			// Sort again after removing trailing "/" for objects as the previous sort
			// does not hold good anymore.
			sort.Strings(entries)
			if len(mergedEntries) == 0 {
				// For the first successful disk.ListDir()
				mergedEntries = entries
				sort.Strings(mergedEntries)
				continue
			}
			// find elements in entries which are not in mergedentries
			for _, entry := range entries {
				idx := sort.SearchStrings(mergedEntries, entry)
				// idx different from len(mergedEntries) means entry is not found
				// in mergedEntries
				if idx < len(mergedEntries) {
					continue
				}
				newEntries = append(newEntries, entry)
			}
			if len(newEntries) > 0 {
				// Merge the entries and sort it.
				mergedEntries = append(mergedEntries, newEntries...)
				sort.Strings(mergedEntries)
			}
		}
		return mergedEntries, false, nil
	}
	return listDir
}

// listObjectsHeal - wrapper function implemented over file tree walk.
func (xl xlObjects) listObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
	// Default is recursive, if delimiter is set then list non recursive.
	recursive := true
	if delimiter == slashSeparator {
		recursive = false
	}

	// "heal" true for listObjectsHeal() and false for listObjects()
	heal := true
	walkResultCh, endWalkCh := xl.listPool.Release(listParams{bucket, recursive, marker, prefix, heal})
	if walkResultCh == nil {
		endWalkCh = make(chan struct{})
		isLeaf := xl.isObject
		listDir := listDirHealFactory(isLeaf, xl.storageDisks...)
		walkResultCh = startTreeWalk(bucket, prefix, marker, recursive, listDir, nil, endWalkCh)
	}

	var objInfos []ObjectInfo
	var eof bool
	var nextMarker string
	for i := 0; i < maxKeys; {
		walkResult, ok := <-walkResultCh
		if !ok {
			// Closed channel.
			eof = true
			break
		}
		// For any walk error return right away.
		if walkResult.err != nil {
			return ListObjectsInfo{}, toObjectErr(walkResult.err, bucket, prefix)
		}
		entry := walkResult.entry
		var objInfo ObjectInfo
		if hasSuffix(entry, slashSeparator) {
			// Object name needs to be full path.
			objInfo.Bucket = bucket
			objInfo.Name = entry
			objInfo.IsDir = true
		} else {
			var err error
			objInfo, err = xl.getObjectInfo(bucket, entry)
			if err != nil {
				// Ignore errFileNotFound
				if errorCause(err) == errFileNotFound {
					continue
				}
				return ListObjectsInfo{}, toObjectErr(err, bucket, prefix)
			}
		}
		nextMarker = objInfo.Name
		objInfos = append(objInfos, objInfo)
		i++
		if walkResult.end {
			eof = true
			break
		}
	}

	params := listParams{bucket, recursive, nextMarker, prefix, heal}
	if !eof {
		xl.listPool.Set(params, walkResultCh, endWalkCh)
	}

	result := ListObjectsInfo{IsTruncated: !eof}
	for _, objInfo := range objInfos {
		result.NextMarker = objInfo.Name
		if objInfo.IsDir {
			result.Prefixes = append(result.Prefixes, objInfo.Name)
			continue
		}

		// Check if the current object needs healing
		objectLock := globalNSMutex.NewNSLock(bucket, objInfo.Name)
		objectLock.RLock()
		partsMetadata, errs := readAllXLMetadata(xl.storageDisks, bucket, objInfo.Name)
		if xlShouldHeal(partsMetadata, errs) {
			healStat := xlHealStat(xl, partsMetadata, errs)
			result.Objects = append(result.Objects, ObjectInfo{
				Name:           objInfo.Name,
				ModTime:        objInfo.ModTime,
				Size:           objInfo.Size,
				IsDir:          false,
				HealObjectInfo: &healStat,
			})
		}
		objectLock.RUnlock()
	}
	return result, nil
}

// ListObjects - list all objects at prefix, delimited by '/'.
func (xl xlObjects) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
	if err := checkListObjsArgs(bucket, prefix, marker, delimiter, xl); err != nil {
		return ListObjectsInfo{}, err
	}

	// With max keys of zero we have reached eof, return right here.
	if maxKeys == 0 {
		return ListObjectsInfo{}, nil
	}

	// For delimiter and prefix as '/' we do not list anything at all
	// since according to s3 spec we stop at the 'delimiter' along
	// with the prefix. On a flat namespace with 'prefix' as '/'
	// we don't have any entries, since all the keys are of form 'keyName/...'
	if delimiter == slashSeparator && prefix == slashSeparator {
		return ListObjectsInfo{}, nil
	}

	// Over flowing count - reset to maxObjectList.
	if maxKeys < 0 || maxKeys > maxObjectList {
		maxKeys = maxObjectList
	}

	// Initiate a list operation, if successful filter and return quickly.
	listObjInfo, err := xl.listObjectsHeal(bucket, prefix, marker, delimiter, maxKeys)
	if err == nil {
		// We got the entries successfully return.
		return listObjInfo, nil
	}

	// Return error at the end.
	return ListObjectsInfo{}, toObjectErr(err, bucket, prefix)
}

// ListUploadsHeal - lists ongoing multipart uploads that require
// healing in one or more disks.
func (xl xlObjects) ListUploadsHeal(bucket, prefix, marker, uploadIDMarker,
	delimiter string, maxUploads int) (ListMultipartsInfo, error) {

	// For delimiter and prefix as '/' we do not list anything at all
	// since according to s3 spec we stop at the 'delimiter' along
	// with the prefix. On a flat namespace with 'prefix' as '/'
	// we don't have any entries, since all the keys are of form 'keyName/...'
	if delimiter == slashSeparator && prefix == slashSeparator {
		return ListMultipartsInfo{}, nil
	}

	// Initiate a list operation.
	listMultipartInfo, err := xl.listMultipartUploadsHeal(bucket, prefix,
		marker, uploadIDMarker, delimiter, maxUploads)
	if err != nil {
		return ListMultipartsInfo{}, toObjectErr(err, bucket, prefix)
	}

	// We got the entries successfully return.
	return listMultipartInfo, nil
}

// Fetches list of multipart uploadIDs given bucket, keyMarker, uploadIDMarker.
func fetchMultipartUploadIDs(bucket, keyMarker, uploadIDMarker string,
	maxUploads int, disks []StorageAPI) (uploads []uploadMetadata, end bool,
	err error) {

	// Hold a read lock on keyMarker path.
	keyMarkerLock := globalNSMutex.NewNSLock(minioMetaMultipartBucket,
		pathJoin(bucket, keyMarker))
	keyMarkerLock.RLock()
	for _, disk := range disks {
		if disk == nil {
			continue
		}
		uploads, end, err = listMultipartUploadIDs(bucket, keyMarker,
			uploadIDMarker, maxUploads, disk)
		if err == nil ||
			!isErrIgnored(err, objMetadataOpIgnoredErrs...) {
			break
		}
	}
	keyMarkerLock.RUnlock()
	return uploads, end, err
}

// listMultipartUploadsHeal - Returns a list of incomplete multipart
// uploads that need to be healed.
func (xl xlObjects) listMultipartUploadsHeal(bucket, prefix, keyMarker,
	uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, error) {

	result := ListMultipartsInfo{
		IsTruncated: true,
		MaxUploads:  maxUploads,
		KeyMarker:   keyMarker,
		Prefix:      prefix,
		Delimiter:   delimiter,
	}

	recursive := delimiter != slashSeparator

	var uploads []uploadMetadata
	var err error
	// List all upload ids for the given keyMarker, starting from
	// uploadIDMarker.
	if uploadIDMarker != "" {
		uploads, _, err = fetchMultipartUploadIDs(bucket, keyMarker,
			uploadIDMarker, maxUploads, xl.getLoadBalancedDisks())
		if err != nil {
			return ListMultipartsInfo{}, err
		}
		maxUploads = maxUploads - len(uploads)
	}

	// We can't use path.Join() as it strips off the trailing '/'.
	multipartPrefixPath := pathJoin(bucket, prefix)
	// multipartPrefixPath should have a trailing '/' when prefix = "".
	if prefix == "" {
		multipartPrefixPath += slashSeparator
	}

	multipartMarkerPath := ""
	if keyMarker != "" {
		multipartMarkerPath = pathJoin(bucket, keyMarker)
	}

	// `heal bool` is used to differentiate listing of incomplete
	// uploads (and parts) from a regular listing of incomplete
	// parts by client SDKs or mc-like commands, within a treewalk
	// pool.
	heal := true
	// The listing is truncated if we have maxUploads entries and
	// there are more entries to be listed.
	truncated := true
	var walkerCh chan treeWalkResult
	var walkerDoneCh chan struct{}
	// Check if we have room left to send more uploads.
	if maxUploads > 0 {
		uploadsLeft := maxUploads

		walkerCh, walkerDoneCh = xl.listPool.Release(listParams{
			bucket:    minioMetaMultipartBucket,
			recursive: recursive,
			marker:    multipartMarkerPath,
			prefix:    multipartPrefixPath,
			heal:      heal,
		})
		if walkerCh == nil {
			walkerDoneCh = make(chan struct{})
			isLeaf := xl.isMultipartUpload
			listDir := listDirFactory(isLeaf, xlTreeWalkIgnoredErrs,
				xl.getLoadBalancedDisks()...)
			walkerCh = startTreeWalk(minioMetaMultipartBucket,
				multipartPrefixPath, multipartMarkerPath,
				recursive, listDir, isLeaf, walkerDoneCh)
		}
		// Collect uploads until leftUploads limit is reached.
		for {
			walkResult, ok := <-walkerCh
			if !ok {
				truncated = false
				break
			}
			// For any error during tree walk, we should return right away.
			if walkResult.err != nil {
				return ListMultipartsInfo{}, walkResult.err
			}

			entry := strings.TrimPrefix(walkResult.entry,
				retainSlash(bucket))
			// Skip entries that are not object directory.
			if hasSuffix(walkResult.entry, slashSeparator) {
				uploads = append(uploads, uploadMetadata{
					Object: entry,
				})
				uploadsLeft--
				if uploadsLeft == 0 {
					break
				}
				continue
			}

			// For an object entry we get all its pending
			// uploadIDs.
			var newUploads []uploadMetadata
			var end bool
			uploadIDMarker = ""
			newUploads, end, err = fetchMultipartUploadIDs(bucket, entry, uploadIDMarker,
				uploadsLeft, xl.getLoadBalancedDisks())
			if err != nil {
				return ListMultipartsInfo{}, err
			}
			uploads = append(uploads, newUploads...)
			uploadsLeft -= len(newUploads)
			if end && walkResult.end {
				truncated = false
				break
			}
			if uploadsLeft == 0 {
				break
			}
		}

	}

	// For all received uploads fill in the multiparts result.
	for _, upload := range uploads {
		var objectName string
		var uploadID string
		if hasSuffix(upload.Object, slashSeparator) {
			// All directory entries are common
			// prefixes. For common prefixes, upload ids
			// are empty.
			uploadID = ""
			objectName = upload.Object
			result.CommonPrefixes = append(result.CommonPrefixes, objectName)
		} else {
			// Check if upload needs healing.
			uploadIDPath := filepath.Join(bucket, upload.Object, upload.UploadID)
			partsMetadata, errs := readAllXLMetadata(xl.storageDisks,
				minioMetaMultipartBucket, uploadIDPath)
			if xlShouldHeal(partsMetadata, errs) {
				healUploadInfo := xlHealStat(xl, partsMetadata, errs)
				upload.HealUploadInfo = &healUploadInfo
				result.Uploads = append(result.Uploads, upload)
			}
			uploadID = upload.UploadID
			objectName = upload.Object
		}

		result.NextKeyMarker = objectName
		result.NextUploadIDMarker = uploadID
	}

	if truncated {
		// Put back the tree walk go-routine into the pool for
		// subsequent use.
		xl.listPool.Set(listParams{
			bucket:    bucket,
			recursive: recursive,
			marker:    result.NextKeyMarker,
			prefix:    prefix,
			heal:      heal,
		}, walkerCh, walkerDoneCh)
	}

	result.IsTruncated = truncated
	// Result is not truncated, reset the markers.
	if !result.IsTruncated {
		result.NextKeyMarker = ""
		result.NextUploadIDMarker = ""
	}
	return result, nil
}
<commit_msg>Fix listDirHealFactory merging of entries across disks (#3959)<commit_after>/*
 * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"path/filepath"
	"sort"
	"strings"
)

func listDirHealFactory(isLeaf isLeafFunc, disks ...StorageAPI) listDirFunc {
	// Returns sorted merged entries from all the disks.
	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string, delayIsLeaf bool, err error) {
		for _, disk := range disks {
			if disk == nil {
				continue
			}
			var entries []string
			var newEntries []string
			entries, err = disk.ListDir(bucket, prefixDir)
			if err != nil {
				continue
			}

			// Filter entries that have the prefix prefixEntry.
			entries = filterMatchingPrefix(entries, prefixEntry)

			// isLeaf() check has to happen here so that
			// trailing "/" for objects can be removed.
			for i, entry := range entries {
				if isLeaf(bucket, pathJoin(prefixDir, entry)) {
					entries[i] = strings.TrimSuffix(entry, slashSeparator)
				}
			}

			// Find elements in entries which are not in mergedEntries
			for _, entry := range entries {
				idx := sort.SearchStrings(mergedEntries, entry)
				// if entry is already present in mergedEntries don't add.
				if idx < len(mergedEntries) && mergedEntries[idx] == entry {
					continue
				}
				newEntries = append(newEntries, entry)
			}

			if len(newEntries) > 0 {
				// Merge the entries and sort it.
				mergedEntries = append(mergedEntries, newEntries...)
				sort.Strings(mergedEntries)
			}
		}
		return mergedEntries, false, nil
	}
	return listDir
}

// listObjectsHeal - wrapper function implemented over file tree walk.
func (xl xlObjects) listObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
	// Default is recursive, if delimiter is set then list non recursive.
	recursive := true
	if delimiter == slashSeparator {
		recursive = false
	}

	// "heal" true for listObjectsHeal() and false for listObjects()
	heal := true
	walkResultCh, endWalkCh := xl.listPool.Release(listParams{bucket, recursive, marker, prefix, heal})
	if walkResultCh == nil {
		endWalkCh = make(chan struct{})
		isLeaf := xl.isObject
		listDir := listDirHealFactory(isLeaf, xl.storageDisks...)
		walkResultCh = startTreeWalk(bucket, prefix, marker, recursive, listDir, nil, endWalkCh)
	}

	var objInfos []ObjectInfo
	var eof bool
	var nextMarker string
	for i := 0; i < maxKeys; {
		walkResult, ok := <-walkResultCh
		if !ok {
			// Closed channel.
			eof = true
			break
		}
		// For any walk error return right away.
		if walkResult.err != nil {
			return ListObjectsInfo{}, toObjectErr(walkResult.err, bucket, prefix)
		}
		entry := walkResult.entry
		var objInfo ObjectInfo
		if hasSuffix(entry, slashSeparator) {
			// Object name needs to be full path.
			objInfo.Bucket = bucket
			objInfo.Name = entry
			objInfo.IsDir = true
		} else {
			var err error
			objInfo, err = xl.getObjectInfo(bucket, entry)
			if err != nil {
				// Ignore errFileNotFound
				if errorCause(err) == errFileNotFound {
					continue
				}
				return ListObjectsInfo{}, toObjectErr(err, bucket, prefix)
			}
		}
		nextMarker = objInfo.Name
		objInfos = append(objInfos, objInfo)
		i++
		if walkResult.end {
			eof = true
			break
		}
	}

	params := listParams{bucket, recursive, nextMarker, prefix, heal}
	if !eof {
		xl.listPool.Set(params, walkResultCh, endWalkCh)
	}

	result := ListObjectsInfo{IsTruncated: !eof}
	for _, objInfo := range objInfos {
		result.NextMarker = objInfo.Name
		if objInfo.IsDir {
			result.Prefixes = append(result.Prefixes, objInfo.Name)
			continue
		}

		// Check if the current object needs healing
		objectLock := globalNSMutex.NewNSLock(bucket, objInfo.Name)
		objectLock.RLock()
		partsMetadata, errs := readAllXLMetadata(xl.storageDisks, bucket, objInfo.Name)
		if xlShouldHeal(partsMetadata, errs) {
			healStat := xlHealStat(xl, partsMetadata, errs)
			result.Objects = append(result.Objects, ObjectInfo{
				Name:           objInfo.Name,
				ModTime:        objInfo.ModTime,
				Size:           objInfo.Size,
				IsDir:          false,
				HealObjectInfo: &healStat,
			})
		}
		objectLock.RUnlock()
	}
	return result, nil
}

// ListObjects - list all objects at prefix, delimited by '/'.
func (xl xlObjects) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
	if err := checkListObjsArgs(bucket, prefix, marker, delimiter, xl); err != nil {
		return ListObjectsInfo{}, err
	}

	// With max keys of zero we have reached eof, return right here.
	if maxKeys == 0 {
		return ListObjectsInfo{}, nil
	}

	// For delimiter and prefix as '/' we do not list anything at all
	// since according to s3 spec we stop at the 'delimiter' along
	// with the prefix. On a flat namespace with 'prefix' as '/'
	// we don't have any entries, since all the keys are of form 'keyName/...'
	if delimiter == slashSeparator && prefix == slashSeparator {
		return ListObjectsInfo{}, nil
	}

	// Over flowing count - reset to maxObjectList.
	if maxKeys < 0 || maxKeys > maxObjectList {
		maxKeys = maxObjectList
	}

	// Initiate a list operation, if successful filter and return quickly.
	listObjInfo, err := xl.listObjectsHeal(bucket, prefix, marker, delimiter, maxKeys)
	if err == nil {
		// We got the entries successfully return.
		return listObjInfo, nil
	}

	// Return error at the end.
	return ListObjectsInfo{}, toObjectErr(err, bucket, prefix)
}

// ListUploadsHeal - lists ongoing multipart uploads that require
// healing in one or more disks.
func (xl xlObjects) ListUploadsHeal(bucket, prefix, marker, uploadIDMarker,
	delimiter string, maxUploads int) (ListMultipartsInfo, error) {

	// For delimiter and prefix as '/' we do not list anything at all
	// since according to s3 spec we stop at the 'delimiter' along
	// with the prefix. On a flat namespace with 'prefix' as '/'
	// we don't have any entries, since all the keys are of form 'keyName/...'
	if delimiter == slashSeparator && prefix == slashSeparator {
		return ListMultipartsInfo{}, nil
	}

	// Initiate a list operation.
	listMultipartInfo, err := xl.listMultipartUploadsHeal(bucket, prefix,
		marker, uploadIDMarker, delimiter, maxUploads)
	if err != nil {
		return ListMultipartsInfo{}, toObjectErr(err, bucket, prefix)
	}

	// We got the entries successfully return.
	return listMultipartInfo, nil
}

// Fetches list of multipart uploadIDs given bucket, keyMarker, uploadIDMarker.
func fetchMultipartUploadIDs(bucket, keyMarker, uploadIDMarker string,
	maxUploads int, disks []StorageAPI) (uploads []uploadMetadata, end bool,
	err error) {

	// Hold a read lock on keyMarker path.
	keyMarkerLock := globalNSMutex.NewNSLock(minioMetaMultipartBucket,
		pathJoin(bucket, keyMarker))
	keyMarkerLock.RLock()
	for _, disk := range disks {
		if disk == nil {
			continue
		}
		uploads, end, err = listMultipartUploadIDs(bucket, keyMarker,
			uploadIDMarker, maxUploads, disk)
		if err == nil ||
			!isErrIgnored(err, objMetadataOpIgnoredErrs...) {
			break
		}
	}
	keyMarkerLock.RUnlock()
	return uploads, end, err
}

// listMultipartUploadsHeal - Returns a list of incomplete multipart
// uploads that need to be healed.
func (xl xlObjects) listMultipartUploadsHeal(bucket, prefix, keyMarker,
	uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, error) {

	result := ListMultipartsInfo{
		IsTruncated: true,
		MaxUploads:  maxUploads,
		KeyMarker:   keyMarker,
		Prefix:      prefix,
		Delimiter:   delimiter,
	}

	recursive := delimiter != slashSeparator

	var uploads []uploadMetadata
	var err error
	// List all upload ids for the given keyMarker, starting from
	// uploadIDMarker.
	if uploadIDMarker != "" {
		uploads, _, err = fetchMultipartUploadIDs(bucket, keyMarker,
			uploadIDMarker, maxUploads, xl.getLoadBalancedDisks())
		if err != nil {
			return ListMultipartsInfo{}, err
		}
		maxUploads = maxUploads - len(uploads)
	}

	// We can't use path.Join() as it strips off the trailing '/'.
	multipartPrefixPath := pathJoin(bucket, prefix)
	// multipartPrefixPath should have a trailing '/' when prefix = "".
	if prefix == "" {
		multipartPrefixPath += slashSeparator
	}

	multipartMarkerPath := ""
	if keyMarker != "" {
		multipartMarkerPath = pathJoin(bucket, keyMarker)
	}

	// `heal bool` is used to differentiate listing of incomplete
	// uploads (and parts) from a regular listing of incomplete
	// parts by client SDKs or mc-like commands, within a treewalk
	// pool.
	heal := true
	// The listing is truncated if we have maxUploads entries and
	// there are more entries to be listed.
	truncated := true
	var walkerCh chan treeWalkResult
	var walkerDoneCh chan struct{}
	// Check if we have room left to send more uploads.
	if maxUploads > 0 {
		uploadsLeft := maxUploads

		walkerCh, walkerDoneCh = xl.listPool.Release(listParams{
			bucket:    minioMetaMultipartBucket,
			recursive: recursive,
			marker:    multipartMarkerPath,
			prefix:    multipartPrefixPath,
			heal:      heal,
		})
		if walkerCh == nil {
			walkerDoneCh = make(chan struct{})
			isLeaf := xl.isMultipartUpload
			listDir := listDirFactory(isLeaf, xlTreeWalkIgnoredErrs,
				xl.getLoadBalancedDisks()...)
			walkerCh = startTreeWalk(minioMetaMultipartBucket,
				multipartPrefixPath, multipartMarkerPath,
				recursive, listDir, isLeaf, walkerDoneCh)
		}
		// Collect uploads until leftUploads limit is reached.
		for {
			walkResult, ok := <-walkerCh
			if !ok {
				truncated = false
				break
			}
			// For any error during tree walk, we should return right away.
			if walkResult.err != nil {
				return ListMultipartsInfo{}, walkResult.err
			}

			entry := strings.TrimPrefix(walkResult.entry,
				retainSlash(bucket))
			// Skip entries that are not object directory.
			if hasSuffix(walkResult.entry, slashSeparator) {
				uploads = append(uploads, uploadMetadata{
					Object: entry,
				})
				uploadsLeft--
				if uploadsLeft == 0 {
					break
				}
				continue
			}

			// For an object entry we get all its pending
			// uploadIDs.
			var newUploads []uploadMetadata
			var end bool
			uploadIDMarker = ""
			newUploads, end, err = fetchMultipartUploadIDs(bucket, entry, uploadIDMarker,
				uploadsLeft, xl.getLoadBalancedDisks())
			if err != nil {
				return ListMultipartsInfo{}, err
			}
			uploads = append(uploads, newUploads...)
			uploadsLeft -= len(newUploads)
			if end && walkResult.end {
				truncated = false
				break
			}
			if uploadsLeft == 0 {
				break
			}
		}

	}

	// For all received uploads fill in the multiparts result.
	for _, upload := range uploads {
		var objectName string
		var uploadID string
		if hasSuffix(upload.Object, slashSeparator) {
			// All directory entries are common
			// prefixes. For common prefixes, upload ids
			// are empty.
			uploadID = ""
			objectName = upload.Object
			result.CommonPrefixes = append(result.CommonPrefixes, objectName)
		} else {
			// Check if upload needs healing.
			uploadIDPath := filepath.Join(bucket, upload.Object, upload.UploadID)
			partsMetadata, errs := readAllXLMetadata(xl.storageDisks,
				minioMetaMultipartBucket, uploadIDPath)
			if xlShouldHeal(partsMetadata, errs) {
				healUploadInfo := xlHealStat(xl, partsMetadata, errs)
				upload.HealUploadInfo = &healUploadInfo
				result.Uploads = append(result.Uploads, upload)
			}
			uploadID = upload.UploadID
			objectName = upload.Object
		}

		result.NextKeyMarker = objectName
		result.NextUploadIDMarker = uploadID
	}

	if truncated {
		// Put back the tree walk go-routine into the pool for
		// subsequent use.
		xl.listPool.Set(listParams{
			bucket:    bucket,
			recursive: recursive,
			marker:    result.NextKeyMarker,
			prefix:    prefix,
			heal:      heal,
		}, walkerCh, walkerDoneCh)
	}

	result.IsTruncated = truncated
	// Result is not truncated, reset the markers.
	if !result.IsTruncated {
		result.NextKeyMarker = ""
		result.NextUploadIDMarker = ""
	}
	return result, nil
}
<|endoftext|>


<commit_before>// Copyright (C) 2017 Michał Matczuk
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tunnel_test

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"net/http"
	"net/http/httptest"
	"net/url"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/mmatczuk/go-http-tunnel"
	"github.com/mmatczuk/go-http-tunnel/id"
	"github.com/mmatczuk/go-http-tunnel/log"
	"github.com/mmatczuk/go-http-tunnel/proto"
)

const (
	payloadInitialSize = 512
	payloadLen         = 10
)

// echoHTTP starts serving HTTP requests on listener l, it accepts connections,
// reads request body and writes is back in response.
func echoHTTP(t *testing.T, l net.Listener) {
	http.Serve(l, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		prior := strings.Join(r.Header["X-Forwarded-For"], ", ")
		if len(strings.Split(prior, ",")) != 2 {
			t.Fatal(r.Header)
		}
		if !strings.Contains(r.Header.Get("X-Forwarded-Host"), "localhost:") {
			t.Fatal(r.Header)
		}
		if r.Header.Get("X-Forwarded-Proto") != "http" {
			t.Fatal(r.Header)
		}

		w.WriteHeader(http.StatusOK)
		if r.Body != nil {
			body, err := ioutil.ReadAll(r.Body)
			if err != nil {
				panic(err)
			}
			w.Write(body)
		}
	}))
}

// echoTCP accepts connections and copies back received bytes.
func echoTCP(l net.Listener) {
	for {
		conn, err := l.Accept()
		if err != nil {
			return
		}
		go func() {
			io.Copy(conn, conn)
		}()
	}
}

func makeEcho(t *testing.T) (http net.Listener, tcp net.Listener) {
	var err error

	// TCP echo
	tcp, err = net.Listen("tcp", ":0")
	if err != nil {
		t.Fatal(err)
	}
	go echoTCP(tcp)

	// HTTP echo
	http, err = net.Listen("tcp", ":0")
	if err != nil {
		t.Fatal(err)
	}
	go echoHTTP(t, http)

	return
}

func makeTunnelServer(t *testing.T) *tunnel.Server {
	cert, identifier := selfSignedCert()
	s, err := tunnel.NewServer(&tunnel.ServerConfig{
		Addr:      ":0",
		TLSConfig: tlsConfig(cert),
		Logger:    log.NewStdLogger(),
	})
	if err != nil {
		t.Fatal(err)
	}
	s.Subscribe(identifier)
	go s.Start()

	return s
}

func makeTunnelClient(t *testing.T, serverAddr string, httpLocalAddr, httpAddr, tcpLocalAddr, tcpAddr net.Addr) *tunnel.Client {
	httpProxy := tunnel.NewMultiHTTPProxy(map[string]*url.URL{
		"localhost:" + port(httpLocalAddr): {
			Scheme: "http",
			Host:   "127.0.0.1:" + port(httpAddr),
		},
	}, log.NewStdLogger())

	tcpProxy := tunnel.NewMultiTCPProxy(map[string]string{
		port(tcpLocalAddr): tcpAddr.String(),
	}, log.NewStdLogger())

	tunnels := map[string]*proto.Tunnel{
		proto.HTTP: {
			Protocol: proto.HTTP,
			Host:     "localhost",
			Auth:     "user:password",
		},
		proto.TCP: {
			Protocol: proto.TCP,
			Addr:     tcpLocalAddr.String(),
		},
	}

	cert, _ := selfSignedCert()
	c := tunnel.NewClient(&tunnel.ClientConfig{
		ServerAddr:      serverAddr,
		TLSClientConfig: tlsConfig(cert),
		Tunnels:         tunnels,
		Proxy: tunnel.Proxy(tunnel.ProxyFuncs{
			HTTP: httpProxy.Proxy,
			TCP:  tcpProxy.Proxy,
		}),
		Logger: log.NewStdLogger(),
	})
	go c.Start()

	return c
}

func TestIntegration(t *testing.T) {
	// local services
	http, tcp := makeEcho(t)
	defer http.Close()
	defer tcp.Close()

	// server
	s := makeTunnelServer(t)
	defer s.Stop()
	h := httptest.NewServer(s)
	defer h.Close()

	httpLocalAddr := h.Listener.Addr()
	tcpLocalAddr := freeAddr()

	// client
	c := makeTunnelClient(t, s.Addr(),
		httpLocalAddr, http.Addr(),
		tcpLocalAddr, tcp.Addr(),
	)
	// FIXME: replace sleep with client state change watch when ready
	time.Sleep(500 * time.Millisecond)
	defer c.Stop()

	payload := randPayload(payloadInitialSize, payloadLen)
	table := []struct {
		S []uint
	}{
		{[]uint{200, 160, 120, 80, 40, 20}},
		{[]uint{40, 80, 120, 160, 200}},
		{[]uint{0, 0, 0, 0, 0, 0, 0, 0, 0, 200}},
	}

	var wg sync.WaitGroup
	for _, test := range table {
		for i, repeat := range test.S {
			p := payload[i]
			r := repeat

			wg.Add(1)
			go func() {
				testHTTP(t, h.Listener.Addr(), p, r)
				wg.Done()
			}()
			wg.Add(1)
			go func() {
				testTCP(t, tcpLocalAddr, p, r)
				wg.Done()
			}()
		}
	}
	wg.Wait()
}

func testHTTP(t *testing.T, addr net.Addr, payload []byte, repeat uint) {
	url := fmt.Sprintf("http://localhost:%s/some/path", port(addr))

	for repeat > 0 {
		r, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(payload))
		if err != nil {
			t.Fatal("Failed to create request")
		}
		r.SetBasicAuth("user", "password")

		resp, err := http.DefaultClient.Do(r)
		if err != nil {
			t.Error(err)
		}
		if resp.StatusCode != http.StatusOK {
			t.Error("Unexpected status code", resp)
		}
		b, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			t.Error("Read error")
		}
		n, m := len(b), len(payload)
		if n != m {
			t.Error("Write read mismatch", n, m)
		}
		repeat--
	}
}

func testTCP(t *testing.T, addr net.Addr, payload []byte, repeat uint) {
	conn, err := net.Dial("tcp", addr.String())
	if err != nil {
		t.Fatal("Dial failed", err)
	}
	defer conn.Close()

	var buf = make([]byte, 10*1024*1024)
	var read, write int
	for repeat > 0 {
		m, err := conn.Write(payload)
		if err != nil {
			t.Error("Write failed", err)
		}
		if m != len(payload) {
			t.Log("Write mismatch", m, len(payload))
		}
		write += m

		n, err := conn.Read(buf)
		if err != nil {
			t.Error("Read failed", err)
		}
		read += n
		repeat--
	}

	for read < write {
		t.Log("No yet read everything", "write", write, "read", read)
		time.Sleep(50 * time.Millisecond)
		n, err := conn.Read(buf)
		if err != nil {
			t.Error("Read failed", err)
		}
		read += n
	}

	if read != write {
		t.Fatal("Write read mismatch", read, write)
	}
}

//
// helpers
//

// randPayload returns slice of randomly initialised data buffers.
func randPayload(initialSize, n int) [][]byte {
	payload := make([][]byte, n)
	l := initialSize
	for i := 0; i < n; i++ {
		payload[i] = randBytes(l)
		l *= 2
	}
	return payload
}

func randBytes(n int) []byte {
	b := make([]byte, n)
	read, err := rand.Read(b)
	if err != nil {
		panic(err)
	}
	if read != n {
		panic("read did not fill whole slice")
	}
	return b
}

func freeAddr() net.Addr {
	l, err := net.Listen("tcp", ":0")
	if err != nil {
		panic(err)
	}
	defer l.Close()
	return l.Addr()
}

func port(addr net.Addr) string {
	return fmt.Sprint(addr.(*net.TCPAddr).Port)
}

func selfSignedCert() (tls.Certificate, id.ID) {
	cert, err := tls.LoadX509KeyPair("./testdata/selfsigned.crt", "./testdata/selfsigned.key")
	if err != nil {
		panic(err)
	}
	x509Cert, err := x509.ParseCertificate(cert.Certificate[0])
	if err != nil {
		panic(err)
	}

	return cert, id.New(x509Cert.Raw)
}

func tlsConfig(cert tls.Certificate) *tls.Config {
	c := &tls.Config{
		Certificates:             []tls.Certificate{cert},
		ClientAuth:               tls.RequestClientCert,
		SessionTicketsDisabled:   true,
		InsecureSkipVerify:       true,
		MinVersion:               tls.VersionTLS12,
		CipherSuites:             []uint16{tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256},
		PreferServerCipherSuites: true,
		NextProtos:               []string{"h2"},
	}
	c.BuildNameToCertificate()
	return c
}
<commit_msg>integration test: make test helpers use testing.TB<commit_after>// Copyright (C) 2017 Michał Matczuk
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tunnel_test

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"net/http"
	"net/http/httptest"
	"net/url"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/mmatczuk/go-http-tunnel"
	"github.com/mmatczuk/go-http-tunnel/id"
	"github.com/mmatczuk/go-http-tunnel/log"
	"github.com/mmatczuk/go-http-tunnel/proto"
)

const (
	payloadInitialSize = 512
	payloadLen         = 10
)

// echoHTTP starts serving HTTP requests on listener l, it accepts connections,
// reads request body and writes is back in response.
func echoHTTP(t testing.TB, l net.Listener) {
	http.Serve(l, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		prior := strings.Join(r.Header["X-Forwarded-For"], ", ")
		if len(strings.Split(prior, ",")) != 2 {
			t.Fatal(r.Header)
		}
		if !strings.Contains(r.Header.Get("X-Forwarded-Host"), "localhost:") {
			t.Fatal(r.Header)
		}
		if r.Header.Get("X-Forwarded-Proto") != "http" {
			t.Fatal(r.Header)
		}

		w.WriteHeader(http.StatusOK)
		if r.Body != nil {
			body, err := ioutil.ReadAll(r.Body)
			if err != nil {
				panic(err)
			}
			w.Write(body)
		}
	}))
}

// echoTCP accepts connections and copies back received bytes.
func echoTCP(l net.Listener) {
	for {
		conn, err := l.Accept()
		if err != nil {
			return
		}
		go func() {
			io.Copy(conn, conn)
		}()
	}
}

func makeEcho(t testing.TB) (http net.Listener, tcp net.Listener) {
	var err error

	// TCP echo
	tcp, err = net.Listen("tcp", ":0")
	if err != nil {
		t.Fatal(err)
	}
	go echoTCP(tcp)

	// HTTP echo
	http, err = net.Listen("tcp", ":0")
	if err != nil {
		t.Fatal(err)
	}
	go echoHTTP(t, http)

	return
}

func makeTunnelServer(t testing.TB) *tunnel.Server {
	cert, identifier := selfSignedCert()
	s, err := tunnel.NewServer(&tunnel.ServerConfig{
		Addr:      ":0",
		TLSConfig: tlsConfig(cert),
		Logger:    log.NewStdLogger(),
	})
	if err != nil {
		t.Fatal(err)
	}
	s.Subscribe(identifier)
	go s.Start()

	return s
}

func makeTunnelClient(t testing.TB, serverAddr string, httpLocalAddr, httpAddr, tcpLocalAddr, tcpAddr net.Addr) *tunnel.Client {
	httpProxy := tunnel.NewMultiHTTPProxy(map[string]*url.URL{
		"localhost:" + port(httpLocalAddr): {
			Scheme: "http",
			Host:   "127.0.0.1:" + port(httpAddr),
		},
	}, log.NewStdLogger())

	tcpProxy := tunnel.NewMultiTCPProxy(map[string]string{
		port(tcpLocalAddr): tcpAddr.String(),
	}, log.NewStdLogger())

	tunnels := map[string]*proto.Tunnel{
		proto.HTTP: {
			Protocol: proto.HTTP,
			Host:     "localhost",
			Auth:     "user:password",
		},
		proto.TCP: {
			Protocol: proto.TCP,
			Addr:     tcpLocalAddr.String(),
		},
	}

	cert, _ := selfSignedCert()
	c := tunnel.NewClient(&tunnel.ClientConfig{
		ServerAddr:      serverAddr,
		TLSClientConfig: tlsConfig(cert),
		Tunnels:         tunnels,
		Proxy: tunnel.Proxy(tunnel.ProxyFuncs{
			HTTP: httpProxy.Proxy,
			TCP:  tcpProxy.Proxy,
		}),
		Logger: log.NewStdLogger(),
	})
	go c.Start()

	return c
}

func TestIntegration(t *testing.T) {
	// local services
	http, tcp := makeEcho(t)
	defer http.Close()
	defer tcp.Close()

	// server
	s := makeTunnelServer(t)
	defer s.Stop()
	h := httptest.NewServer(s)
	defer h.Close()

	httpLocalAddr := h.Listener.Addr()
	tcpLocalAddr := freeAddr()

	// client
	c := makeTunnelClient(t, s.Addr(),
		httpLocalAddr, http.Addr(),
		tcpLocalAddr, tcp.Addr(),
	)
	// FIXME: replace sleep with client state change watch when ready
	time.Sleep(500 * time.Millisecond)
	defer c.Stop()

	payload := randPayload(payloadInitialSize, payloadLen)
	table := []struct {
		S []uint
	}{
		{[]uint{200, 160, 120, 80, 40, 20}},
		{[]uint{40, 80, 120, 160, 200}},
		{[]uint{0, 0, 0, 0, 0, 0, 0, 0, 0, 200}},
	}

	var wg sync.WaitGroup
	for _, test := range table {
		for i, repeat := range test.S {
			p := payload[i]
			r := repeat

			wg.Add(1)
			go func() {
				testHTTP(t, h.Listener.Addr(), p, r)
				wg.Done()
			}()
			wg.Add(1)
			go func() {
				testTCP(t, tcpLocalAddr, p, r)
				wg.Done()
			}()
		}
	}
	wg.Wait()
}

func testHTTP(t testing.TB, addr net.Addr, payload []byte, repeat uint) {
	url := fmt.Sprintf("http://localhost:%s/some/path", port(addr))

	for repeat > 0 {
		r, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(payload))
		if err != nil {
			t.Fatal("Failed to create request")
		}
		r.SetBasicAuth("user", "password")

		resp, err := http.DefaultClient.Do(r)
		if err != nil {
			t.Error(err)
		}
		if resp.StatusCode != http.StatusOK {
			t.Error("Unexpected status code", resp)
		}
		b, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			t.Error("Read error")
		}
		n, m := len(b), len(payload)
		if n != m {
			t.Error("Write read mismatch", n, m)
		}
		repeat--
	}
}

func testTCP(t testing.TB, addr net.Addr, payload []byte, repeat uint) {
	conn, err := net.Dial("tcp", addr.String())
	if err != nil {
		t.Fatal("Dial failed", err)
	}
	defer conn.Close()

	var buf = make([]byte, 10*1024*1024)
	var read, write int
	for repeat > 0 {
		m, err := conn.Write(payload)
		if err != nil {
			t.Error("Write failed", err)
		}
		if m != len(payload) {
			t.Log("Write mismatch", m, len(payload))
		}
		write += m

		n, err := conn.Read(buf)
		if err != nil {
			t.Error("Read failed", err)
		}
		read += n
		repeat--
	}

	for read < write {
		t.Log("No yet read everything", "write", write, "read", read)
		time.Sleep(50 * time.Millisecond)
		n, err := conn.Read(buf)
		if err != nil {
			t.Error("Read failed", err)
		}
		read += n
	}

	if read != write {
		t.Fatal("Write read mismatch", read, write)
	}
}

//
// helpers
//

// randPayload returns slice of randomly initialised data buffers.
func randPayload(initialSize, n int) [][]byte {
	payload := make([][]byte, n)
	l := initialSize
	for i := 0; i < n; i++ {
		payload[i] = randBytes(l)
		l *= 2
	}
	return payload
}

func randBytes(n int) []byte {
	b := make([]byte, n)
	read, err := rand.Read(b)
	if err != nil {
		panic(err)
	}
	if read != n {
		panic("read did not fill whole slice")
	}
	return b
}

func freeAddr() net.Addr {
	l, err := net.Listen("tcp", ":0")
	if err != nil {
		panic(err)
	}
	defer l.Close()
	return l.Addr()
}

func port(addr net.Addr) string {
	return fmt.Sprint(addr.(*net.TCPAddr).Port)
}

func selfSignedCert() (tls.Certificate, id.ID) {
	cert, err := tls.LoadX509KeyPair("./testdata/selfsigned.crt", "./testdata/selfsigned.key")
	if err != nil {
		panic(err)
	}
	x509Cert, err := x509.ParseCertificate(cert.Certificate[0])
	if err != nil {
		panic(err)
	}

	return cert, id.New(x509Cert.Raw)
}

func tlsConfig(cert tls.Certificate) *tls.Config {
	c := &tls.Config{
		Certificates:             []tls.Certificate{cert},
		ClientAuth:               tls.RequestClientCert,
		SessionTicketsDisabled:   true,
		InsecureSkipVerify:       true,
		MinVersion:               tls.VersionTLS12,
		CipherSuites:             []uint16{tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256},
		PreferServerCipherSuites: true,
		NextProtos:               []string{"h2"},
	}
	c.BuildNameToCertificate()
	return c
}
<|endoftext|>


<commit_before>package filestore

import (
	"errors"
	"io"
	"os"
	"path/filepath"

	ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore"
	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore/query"
	k "github.com/ipfs/go-ipfs/blocks/key"
	//mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash"
	u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
)

type datastore struct {
	ds           ds.Datastore
	alwaysVerify bool
}

func New(d ds.Datastore, fileStorePath string) (ds.Datastore, error) {
	return &datastore{d, true}, nil
}

func (d *datastore) Put(key ds.Key, value interface{}) (err error) {
	val, ok := value.(*DataWOpts)
	if !ok {
		panic(ds.ErrInvalidType)
	}

	addType, ok := val.AddOpts.(int)
	if !ok {
		panic(ds.ErrInvalidType)
	}
	if addType != AddNoCopy {
		return errors.New("Only \"no-copy\" mode supported for now.")
	}

	dataObj, ok := val.DataObj.(*DataObj)
	if !ok {
		panic(ds.ErrInvalidType)
	}

	// Make sure the filename is an absolute path
	if !filepath.IsAbs(dataObj.FilePath) {
		return errors.New("datastore put: non-absolute filename: " + dataObj.FilePath)
	}

	// Make sure we can read the file as a sanity check
	if file, err := os.Open(dataObj.FilePath); err != nil {
		return err
	} else {
		file.Close()
	}

	data, err := dataObj.Marshal()
	if err != nil {
		return err
	}
	return d.ds.Put(key, data)
}

func (d *datastore) Get(key ds.Key) (value interface{}, err error) {
	dataObj, err := d.ds.Get(key)
	if err != nil {
		return nil, err
	}
	data := dataObj.([]byte)
	val := new(DataObj)
	err = val.Unmarshal(data)
	if err != nil {
		return nil, err
	}
	if val.NoBlockData {
		file, err := os.Open(val.FilePath)
		if err != nil {
			return nil, err
		}
		_, err = file.Seek(int64(val.Offset), 0)
		if err != nil {
			return nil, err
		}
		buf := make([]byte, val.Size)
		_, err = io.ReadFull(file, buf)
		if err != nil {
			return nil, err
		}
		data, err := reconstruct(val.Data, buf)
		if err != nil {
			return nil, err
		}
		if d.alwaysVerify {
			newKey := k.Key(u.Hash(data)).DsKey()
			if newKey != key {
				return nil, errors.New("Filestore: Block Verification Failed")
			}
		}
		return data, nil
	} else {
		return val.Data, nil
	}
}

func (d *datastore) Has(key ds.Key) (exists bool, err error) {
	return d.ds.Has(key)
}

func (d *datastore) Delete(key ds.Key) error {
	return ds.ErrNotFound
}

func (d *datastore) Query(q query.Query) (query.Results, error) {
	return nil, errors.New("queries not supported yet")
}

func (d *datastore) Close() error {
	c, ok := d.ds.(io.Closer)
	if ok {
		return c.Close()
	} else {
		return nil
	}
}

func (d *datastore) Batch() (ds.Batch, error) {
	return ds.NewBasicBatch(d), nil
}
<commit_msg>Add Basic Query and "Direct" commands to filestore.  Needs Testing.<commit_after>package filestore

import (
	"errors"
	"io"
	"os"
	"path/filepath"

	ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore"
	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore/query"
	k "github.com/ipfs/go-ipfs/blocks/key"
	//mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash"
	u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
)

type datastore struct {
	ds           ds.Datastore
	alwaysVerify bool
}

func New(d ds.Datastore, fileStorePath string) (ds.Datastore, error) {
	return &datastore{d, true}, nil
}

func (d *datastore) Put(key ds.Key, value interface{}) (err error) {
	val, ok := value.(*DataWOpts)
	if !ok {
		panic(ds.ErrInvalidType)
	}

	addType, ok := val.AddOpts.(int)
	if !ok {
		panic(ds.ErrInvalidType)
	}
	if addType != AddNoCopy {
		return errors.New("Only \"no-copy\" mode supported for now.")
	}

	dataObj, ok := val.DataObj.(*DataObj)
	if !ok {
		panic(ds.ErrInvalidType)
	}

	// Make sure the filename is an absolute path
	if !filepath.IsAbs(dataObj.FilePath) {
		return errors.New("datastore put: non-absolute filename: " + dataObj.FilePath)
	}

	// Make sure we can read the file as a sanity check
	if file, err := os.Open(dataObj.FilePath); err != nil {
		return err
	} else {
		file.Close()
	}

	data, err := dataObj.Marshal()
	if err != nil {
		return err
	}
	return d.ds.Put(key, data)
}

func (d *datastore) Get(key ds.Key) (value interface{}, err error) {
	dataObj, err := d.ds.Get(key)
	if err != nil {
		return nil, err
	}
	val, err := d.decode(dataObj)
	if err != nil {
		return nil, err
	}
	return d.GetData(key, val, d.alwaysVerify)
}

// Get the key as a DataObj
func (d *datastore) GetDirect(key ds.Key) (*DataObj, error) {
	dataObj, err := d.ds.Get(key)
	if err != nil {
		return nil, err
	}
	return d.decode(dataObj)
}

func (d *datastore) decode(dataObj interface{}) (*DataObj, error) {
	data := dataObj.([]byte)
	val := new(DataObj)
	err := val.Unmarshal(data)
	if err != nil {
		return nil, err
	}
	return val, nil
}

// Get the orignal data out of the DataObj
func (d *datastore) GetData(key ds.Key, val *DataObj, verify bool) ([]byte, error) {
	if val.NoBlockData {
		file, err := os.Open(val.FilePath)
		if err != nil {
			return nil, err
		}
		_, err = file.Seek(int64(val.Offset), 0)
		if err != nil {
			return nil, err
		}
		buf := make([]byte, val.Size)
		_, err = io.ReadFull(file, buf)
		if err != nil {
			return nil, err
		}
		data, err := reconstruct(val.Data, buf)
		if err != nil {
			return nil, err
		}
		if verify {
			newKey := k.Key(u.Hash(data)).DsKey()
			if newKey != key {
				return nil, errors.New("Filestore: Block Verification Failed")
			}
		}
		return data, nil
	} else {
		return val.Data, nil
	}
}

func (d *datastore) Has(key ds.Key) (exists bool, err error) {
	return d.ds.Has(key)
}

func (d *datastore) Delete(key ds.Key) error {
	return ds.ErrNotFound
}

func (d *datastore) DeleteDirect(key ds.Key) error {
	return d.ds.Delete(key)
}

func (d *datastore) Query(q query.Query) (query.Results, error) {
	res, err := d.ds.Query(q)
	if err != nil {
		return nil, err
	}
	if q.KeysOnly {
		return res, nil
	}
	return nil, errors.New("filestore currently only supports keyonly queries")
	// return &queryResult{res, func(r query.Result) query.Result {
	// 	val, err := d.decode(r.Value)
	// 	if err != nil {
	// 		return query.Result{query.Entry{r.Key, nil}, err}
	// 	}
	// 	// Note: It should not be necessary to reclean the key
	// 	// here (by calling ds.NewKey) just to convert the
	// 	// string back to a ds.Key
	// 	data, err := d.GetData(ds.NewKey(r.Key), val, d.alwaysVerify)
	// 	if err != nil {
	// 		return query.Result{query.Entry{r.Key, nil}, err}
	// 	}
	// 	return query.Result{query.Entry{r.Key, data}, r.Error}
	// }}, nil
}

func (d *datastore) QueryDirect(q query.Query) (query.Results, error) {
	res, err := d.ds.Query(q)
	if err != nil {
		return nil, err
	}
	if q.KeysOnly {
		return res, nil
	}
	return nil, errors.New("filestore currently only supports keyonly queries")
	// return &queryResult{res, func(r query.Result) query.Result {
	// 	val, err := d.decode(r.Value)
	// 	if err != nil {
	// 		return query.Result{query.Entry{r.Key, nil}, err}
	// 	}
	// 	return query.Result{query.Entry{r.Key, val}, r.Error}
	// }}, nil
}

// type queryResult struct {
// 	query.Results
// 	adjResult func(query.Result) query.Result
// }

// func (q *queryResult) Next() <-chan query.Result {
// 	in := q.Results.Next()
// 	out := make(chan query.Result)
// 	go func() {
// 		res := <-in
// 		if res.Error == nil {
// 			out <- res
// 		}
// 		out <- q.adjResult(res)
// 	}()
// 	return out
// }

// func (q *queryResult) Rest() ([]query.Entry, error) {
// 	res, err := q.Results.Rest()
// 	if err != nil {
// 		return nil, err
// 	}
// 	for _, entry := range res {
// 		newRes := q.adjResult(query.Result{entry, nil})
// 		if newRes.Error != nil {
// 			return nil, newRes.Error
// 		}
// 		entry.Value = newRes.Value
// 	}
// 	return res, nil
// }

func (d *datastore) Close() error {
	c, ok := d.ds.(io.Closer)
	if ok {
		return c.Close()
	} else {
		return nil
	}
}

func (d *datastore) Batch() (ds.Batch, error) {
	return ds.NewBasicBatch(d), nil
}
<|endoftext|>


<commit_before>package ast

import (
	"fmt"
)

// Iterator starts uninitialized, you need to call Next() first.
//
// For example:
//
// it := n.Children()
// for it.Next() {
// 		it.Node()
// }
type Iterator interface {
	// Next moves the iterator forward and returns true if points to a node, false
	// otherwise.
	Next() bool
	// Node returns a copy of the node pointed at by the iterator.
	Node() Node
}

type chainIterator struct {
	started bool
	node    Node
}

func (c *chainIterator) Next() bool {
	if !c.started {
		c.started = true
	} else if c.node.Valid() {
		c.node = c.node.Next()
	}
	return c.node.Valid()
}

func (c *chainIterator) Node() Node {
	return c.node
}

type Root struct {
	nodes []Node
}

func (r *Root) Iterator() Iterator {
	it := &chainIterator{}
	if len(r.nodes) > 0 {
		it.node = r.nodes[0]
	}
	return it
}

func (r *Root) Reset() {
	r.nodes = r.nodes[:0]
}

func (r *Root) at(idx int) Node {
	// TODO: unsafe to point to the node directly
	return r.nodes[idx]
}

// Arrays have one child per element in the array.
// InlineTables have one child per key-value pair in the table.
// KeyValues have at least two children. The first one is the value. The
// rest make a potentially dotted key.
// Table and Array table have one child per element of the key they
// represent (same as KeyValue, but without the last node being the value).
// children []Node
type Node struct {
	Kind Kind
	Data []byte // Raw bytes from the input

	// next idx (in the root array). 0 if last of the collection.
	next int
	// child idx (in the root array). 0 if no child.
	child int
	// pointer to the root array
	root *Root
}

// Next returns a copy of the next node, or an invalid Node if there is no
// next node.
func (n Node) Next() Node {
	if n.next <= 0 {
		return NoNode
	}
	return n.root.at(n.next)
}

// Child returns a copy of the first child node of this node. Other children
// can be accessed calling Next on the first child.
// Returns an invalid Node if there is none.
func (n Node) Child() Node {
	if n.child <= 0 {
		return NoNode
	}
	return n.root.at(n.child)
}

func (n Node) Valid() bool {
	return n.Kind != Invalid
}

var NoNode = Node{}

// Key returns the child nodes making the Key on a supported node. Panics
// otherwise.
// They are guaranteed to be all be of the Kind Key. A simple key would return
// just one element.
func (n *Node) Key() Iterator {
	switch n.Kind {
	case KeyValue:
		value := n.Child()
		if !value.Valid() {
			panic(fmt.Errorf("KeyValue should have at least two children"))
		}
		return &chainIterator{node: value.Next()}
	case Table, ArrayTable:
		return &chainIterator{node: n.Child()}
	default:
		panic(fmt.Errorf("Key() is not supported on a %s", n.Kind))
	}
}

// Value returns a pointer to the value node of a KeyValue.
// Guaranteed to be non-nil.
// Panics if not called on a KeyValue node, or if the Children are malformed.
func (n Node) Value() Node {
	assertKind(KeyValue, n)
	return n.Child()
}

func (n Node) Children() Iterator {
	return &chainIterator{node: n.Child()}
}

func assertKind(k Kind, n Node) {
	if n.Kind != k {
		panic(fmt.Errorf("method was expecting a %s, not a %s", k, n.Kind))
	}
}
<commit_msg>Remove unused interface<commit_after>package ast

import (
	"fmt"
)

// Iterator starts uninitialized, you need to call Next() first.
//
// For example:
//
// it := n.Children()
// for it.Next() {
// 		it.Node()
// }
type Iterator struct {
	started bool
	node    Node
}

// Next moves the iterator forward and returns true if points to a node, false
// otherwise.
func (c *Iterator) Next() bool {
	if !c.started {
		c.started = true
	} else if c.node.Valid() {
		c.node = c.node.Next()
	}
	return c.node.Valid()
}

// Node returns a copy of the node pointed at by the iterator.
func (c *Iterator) Node() Node {
	return c.node
}

type Root struct {
	nodes []Node
}

func (r *Root) Iterator() Iterator {
	it := Iterator{}
	if len(r.nodes) > 0 {
		it.node = r.nodes[0]
	}
	return it
}

func (r *Root) Reset() {
	r.nodes = r.nodes[:0]
}

func (r *Root) at(idx int) Node {
	// TODO: unsafe to point to the node directly
	return r.nodes[idx]
}

// Arrays have one child per element in the array.
// InlineTables have one child per key-value pair in the table.
// KeyValues have at least two children. The first one is the value. The
// rest make a potentially dotted key.
// Table and Array table have one child per element of the key they
// represent (same as KeyValue, but without the last node being the value).
// children []Node
type Node struct {
	Kind Kind
	Data []byte // Raw bytes from the input

	// next idx (in the root array). 0 if last of the collection.
	next int
	// child idx (in the root array). 0 if no child.
	child int
	// pointer to the root array
	root *Root
}

// Next returns a copy of the next node, or an invalid Node if there is no
// next node.
func (n Node) Next() Node {
	if n.next <= 0 {
		return NoNode
	}
	return n.root.at(n.next)
}

// Child returns a copy of the first child node of this node. Other children
// can be accessed calling Next on the first child.
// Returns an invalid Node if there is none.
func (n Node) Child() Node {
	if n.child <= 0 {
		return NoNode
	}
	return n.root.at(n.child)
}

func (n Node) Valid() bool {
	return n.Kind != Invalid
}

var NoNode = Node{}

// Key returns the child nodes making the Key on a supported node. Panics
// otherwise.
// They are guaranteed to be all be of the Kind Key. A simple key would return
// just one element.
func (n *Node) Key() Iterator {
	switch n.Kind {
	case KeyValue:
		value := n.Child()
		if !value.Valid() {
			panic(fmt.Errorf("KeyValue should have at least two children"))
		}
		return Iterator{node: value.Next()}
	case Table, ArrayTable:
		return Iterator{node: n.Child()}
	default:
		panic(fmt.Errorf("Key() is not supported on a %s", n.Kind))
	}
}

// Value returns a pointer to the value node of a KeyValue.
// Guaranteed to be non-nil.
// Panics if not called on a KeyValue node, or if the Children are malformed.
func (n Node) Value() Node {
	assertKind(KeyValue, n)
	return n.Child()
}

func (n Node) Children() Iterator {
	return Iterator{node: n.Child()}
}

func assertKind(k Kind, n Node) {
	if n.Kind != k {
		panic(fmt.Errorf("method was expecting a %s, not a %s", k, n.Kind))
	}
}
<|endoftext|>


<commit_before>package wspacego

import (
	. "github.com/r7kamura/gospel"
	"path"
	"runtime"
	"testing"
)

func TestInterpretor(t *testing.T) {
	Describe(t, "インタープリターのテスト", func() {
		data := []byte{'A', ' ', 'B', '\t', '\r', '\n'}
		Context("インスタンス生成", func() {
			It("インスタンスが生成できること", func() {
				actual := NewInterpreter(data)
				Expect(actual).To(Exist)
				Expect(actual.origin).To(Equal, data)
			})
		})
		Context("不要な文字を排除する関数", func() {
			It("不要なデータ以外排除されていること", func() {
				sut := NewInterpreter(data)
				sut.filter()
				Expect(sut.source).To(Equal, []byte{' ', '\t', '\n'})
			})
			It("不要なデータ以外排除されていること（改行を増やす）", func() {
				data = append(data, '\n')
				sut := NewInterpreter(data)
				sut.filter()
				Expect(sut.source).To(Equal, []byte{' ', '\t', '\n', '\n'})
			})
		})
		Context("ソースファイルをコマンドリストに変換する関数", func() {
			It("指定したコマンドが作成されること", func() {
				data = []byte{'P', 'u', 's', 'h', ' ', ' ', '\t', '\n'}
				sut := NewInterpreter(data)
				sut.filter()
				sut.parseCommands()
				Expect(sut.commands).To(Exist)
				Expect(sut.commands.Len()).To(Equal, 1)
				Expect(sut.commands.Get(1)).To(Equal, NewSubCommandWithParam("stack", "push", 1))
			})
		})
	})
}

func current_dir() string {
	_, fpath, _, _ := runtime.Caller(0)
	return path.Dir(fpath)
}
<commit_msg>テストの説明文を変更<commit_after>package wspacego

import (
	. "github.com/r7kamura/gospel"
	"path"
	"runtime"
	"testing"
)

func TestInterpretor(t *testing.T) {
	Describe(t, "インタープリターのテスト", func() {
		data := []byte{'A', ' ', 'B', '\t', '\r', '\n'}
		Context("インスタンス生成", func() {
			It("インスタンスが生成できること", func() {
				actual := NewInterpreter(data)
				Expect(actual).To(Exist)
				Expect(actual.origin).To(Equal, data)
			})
		})
		Context("不要な文字を排除する関数", func() {
			It("不要なデータ以外排除されていること", func() {
				sut := NewInterpreter(data)
				sut.filter()
				Expect(sut.source).To(Equal, []byte{' ', '\t', '\n'})
			})
			It("不要なデータ以外排除されていること（改行を増やす）", func() {
				data = append(data, '\n')
				sut := NewInterpreter(data)
				sut.filter()
				Expect(sut.source).To(Equal, []byte{' ', '\t', '\n', '\n'})
			})
		})
		Context("ソースファイルをコマンドリストに変換する関数", func() {
			It("スタックに１をプッシュするコマンドが作成されること", func() {
				data = []byte{'P', 'u', 's', 'h', ' ', ' ', '\t', '\n'}
				sut := NewInterpreter(data)
				sut.filter()
				sut.parseCommands()
				Expect(sut.commands).To(Exist)
				Expect(sut.commands.Len()).To(Equal, 1)
				Expect(sut.commands.Get(1)).To(Equal, NewSubCommandWithParam("stack", "push", 1))
			})
		})
	})
}

func current_dir() string {
	_, fpath, _, _ := runtime.Caller(0)
	return path.Dir(fpath)
}
<|endoftext|>