<commit_before>package graphdriver

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/vbatts/tar-split/tar/storage"

	"github.com/docker/docker/pkg/archive"
	"github.com/docker/docker/pkg/idtools"
)

// FsMagic unsigned id of the filesystem in use.
type FsMagic uint32

const (
	// FsMagicUnsupported is a predefined constant value other than a valid filesystem id.
	FsMagicUnsupported = FsMagic(0x00000000)
)

var (
	// All registered drivers
	drivers map[string]InitFunc

	// ErrNotSupported returned when driver is not supported.
	ErrNotSupported = errors.New("driver not supported")
	// ErrPrerequisites retuned when driver does not meet prerequisites.
	ErrPrerequisites = errors.New("prerequisites for driver not satisfied (wrong filesystem?)")
	// ErrIncompatibleFS returned when file system is not supported.
	ErrIncompatibleFS = fmt.Errorf("backing file system is unsupported for this graph driver")
)

// InitFunc initializes the storage driver.
type InitFunc func(root string, options []string, uidMaps, gidMaps []idtools.IDMap) (Driver, error)

// ProtoDriver defines the basic capabilities of a driver.
// This interface exists solely to be a minimum set of methods
// for client code which choose not to implement the entire Driver
// interface and use the NaiveDiffDriver wrapper constructor.
//
// Use of ProtoDriver directly by client code is not recommended.
type ProtoDriver interface {
	// String returns a string representation of this driver.
	String() string
	// Create creates a new, empty, filesystem layer with the
	// specified id and parent and mountLabel. Parent and mountLabel may be "".
	Create(id, parent, mountLabel string) error
	// Remove attempts to remove the filesystem layer with this id.
	Remove(id string) error
	// Get returns the mountpoint for the layered filesystem referred
	// to by this id. You can optionally specify a mountLabel or "".
	// Returns the absolute path to the mounted layered filesystem.
	Get(id, mountLabel string) (dir string, err error)
	// Put releases the system resources for the specified id,
	// e.g, unmounting layered filesystem.
	Put(id string) error
	// Exists returns whether a filesystem layer with the specified
	// ID exists on this driver.
	Exists(id string) bool
	// Status returns a set of key-value pairs which give low
	// level diagnostic status about this driver.
	Status() [][2]string
	// Returns a set of key-value pairs which give low level information
	// about the image/container driver is managing.
	GetMetadata(id string) (map[string]string, error)
	// Cleanup performs necessary tasks to release resources
	// held by the driver, e.g., unmounting all layered filesystems
	// known to this driver.
	Cleanup() error
}

// Driver is the interface for layered/snapshot file system drivers.
type Driver interface {
	ProtoDriver
	// Diff produces an archive of the changes between the specified
	// layer and its parent layer which may be "".
	Diff(id, parent string) (archive.Archive, error)
	// Changes produces a list of changes between the specified layer
	// and its parent layer. If parent is "", then all changes will be ADD changes.
	Changes(id, parent string) ([]archive.Change, error)
	// ApplyDiff extracts the changeset from the given diff into the
	// layer with the specified id and parent, returning the size of the
	// new layer in bytes.
	// The archive.Reader must be an uncompressed stream.
	ApplyDiff(id, parent string, diff archive.Reader) (size int64, err error)
	// DiffSize calculates the changes between the specified id
	// and its parent and returns the size in bytes of the changes
	// relative to its base filesystem directory.
	DiffSize(id, parent string) (size int64, err error)
}

// DiffGetterDriver is the interface for layered file system drivers that
// provide a specialized function for getting file contents for tar-split.
type DiffGetterDriver interface {
	Driver
	// DiffGetter returns an interface to efficiently retrieve the contents
	// of files in a layer.
	DiffGetter(id string) (FileGetCloser, error)
}

// FileGetCloser extends the storage.FileGetter interface with a Close method
// for cleaning up.
type FileGetCloser interface {
	storage.FileGetter
	// Close cleans up any resources associated with the FileGetCloser.
	Close() error
}

func init() {
	drivers = make(map[string]InitFunc)
}

// Register registers a InitFunc for the driver.
func Register(name string, initFunc InitFunc) error {
	if _, exists := drivers[name]; exists {
		return fmt.Errorf("Name already registered %s", name)
	}
	drivers[name] = initFunc

	return nil
}

// GetDriver initializes and returns the registered driver
func GetDriver(name, home string, options []string, uidMaps, gidMaps []idtools.IDMap) (Driver, error) {
	if initFunc, exists := drivers[name]; exists {
		return initFunc(filepath.Join(home, name), options, uidMaps, gidMaps)
	}
	if pluginDriver, err := lookupPlugin(name, home, options); err == nil {
		return pluginDriver, nil
	}
	logrus.Errorf("Failed to GetDriver graph %s %s", name, home)
	return nil, ErrNotSupported
}

// getBuiltinDriver initializes and returns the registered driver, but does not try to load from plugins
func getBuiltinDriver(name, home string, options []string, uidMaps, gidMaps []idtools.IDMap) (Driver, error) {
	if initFunc, exists := drivers[name]; exists {
		return initFunc(filepath.Join(home, name), options, uidMaps, gidMaps)
	}
	logrus.Errorf("Failed to built-in GetDriver graph %s %s", name, home)
	return nil, ErrNotSupported
}

// New creates the driver and initializes it at the specified root.
func New(root string, name string, options []string, uidMaps, gidMaps []idtools.IDMap) (driver Driver, err error) {
	if name != "" {
		logrus.Debugf("[graphdriver] trying provided driver %q", name) // so the logs show specified driver
		return GetDriver(name, root, options, uidMaps, gidMaps)
	}

	// Guess for prior driver
	priorDrivers := scanPriorDrivers(root)
	for _, name := range priority {
		if name == "vfs" {
			// don't use vfs even if there is state present.
			continue
		}
		for _, prior := range priorDrivers {
			// of the state found from prior drivers, check in order of our priority
			// which we would prefer
			if prior == name {
				driver, err = getBuiltinDriver(name, root, options, uidMaps, gidMaps)
				if err != nil {
					// unlike below, we will return error here, because there is prior
					// state, and now it is no longer supported/prereq/compatible, so
					// something changed and needs attention. Otherwise the daemon's
					// images would just "disappear".
					logrus.Errorf("[graphdriver] prior storage driver %q failed: %s", name, err)
					return nil, err
				}
				if err := checkPriorDriver(name, root); err != nil {
					return nil, err
				}
				logrus.Infof("[graphdriver] using prior storage driver %q", name)
				return driver, nil
			}
		}
	}

	// Check for priority drivers first
	for _, name := range priority {
		driver, err = getBuiltinDriver(name, root, options, uidMaps, gidMaps)
		if err != nil {
			if err == ErrNotSupported || err == ErrPrerequisites || err == ErrIncompatibleFS {
				continue
			}
			return nil, err
		}
		return driver, nil
	}

	// Check all registered drivers if no priority driver is found
	for _, initFunc := range drivers {
		if driver, err = initFunc(root, options, uidMaps, gidMaps); err != nil {
			if err == ErrNotSupported || err == ErrPrerequisites || err == ErrIncompatibleFS {
				continue
			}
			return nil, err
		}
		return driver, nil
	}
	return nil, fmt.Errorf("No supported storage backend found")
}

// scanPriorDrivers returns an un-ordered scan of directories of prior storage drivers
func scanPriorDrivers(root string) []string {
	priorDrivers := []string{}
	for driver := range drivers {
		p := filepath.Join(root, driver)
		if _, err := os.Stat(p); err == nil && driver != "vfs" {
			priorDrivers = append(priorDrivers, driver)
		}
	}
	return priorDrivers
}

func checkPriorDriver(name, root string) error {
	priorDrivers := []string{}
	for _, prior := range scanPriorDrivers(root) {
		if prior != name && prior != "vfs" {
			if _, err := os.Stat(filepath.Join(root, prior)); err == nil {
				priorDrivers = append(priorDrivers, prior)
			}
		}
	}

	if len(priorDrivers) > 0 {

		return fmt.Errorf("%q contains other graphdrivers: %s; Please cleanup or explicitly choose storage driver (-s <DRIVER>)", root, strings.Join(priorDrivers, ","))
	}
	return nil
}
<commit_msg>Make sure we call every graph init with the same root path.<commit_after>package graphdriver

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/vbatts/tar-split/tar/storage"

	"github.com/docker/docker/pkg/archive"
	"github.com/docker/docker/pkg/idtools"
)

// FsMagic unsigned id of the filesystem in use.
type FsMagic uint32

const (
	// FsMagicUnsupported is a predefined constant value other than a valid filesystem id.
	FsMagicUnsupported = FsMagic(0x00000000)
)

var (
	// All registered drivers
	drivers map[string]InitFunc

	// ErrNotSupported returned when driver is not supported.
	ErrNotSupported = errors.New("driver not supported")
	// ErrPrerequisites retuned when driver does not meet prerequisites.
	ErrPrerequisites = errors.New("prerequisites for driver not satisfied (wrong filesystem?)")
	// ErrIncompatibleFS returned when file system is not supported.
	ErrIncompatibleFS = fmt.Errorf("backing file system is unsupported for this graph driver")
)

// InitFunc initializes the storage driver.
type InitFunc func(root string, options []string, uidMaps, gidMaps []idtools.IDMap) (Driver, error)

// ProtoDriver defines the basic capabilities of a driver.
// This interface exists solely to be a minimum set of methods
// for client code which choose not to implement the entire Driver
// interface and use the NaiveDiffDriver wrapper constructor.
//
// Use of ProtoDriver directly by client code is not recommended.
type ProtoDriver interface {
	// String returns a string representation of this driver.
	String() string
	// Create creates a new, empty, filesystem layer with the
	// specified id and parent and mountLabel. Parent and mountLabel may be "".
	Create(id, parent, mountLabel string) error
	// Remove attempts to remove the filesystem layer with this id.
	Remove(id string) error
	// Get returns the mountpoint for the layered filesystem referred
	// to by this id. You can optionally specify a mountLabel or "".
	// Returns the absolute path to the mounted layered filesystem.
	Get(id, mountLabel string) (dir string, err error)
	// Put releases the system resources for the specified id,
	// e.g, unmounting layered filesystem.
	Put(id string) error
	// Exists returns whether a filesystem layer with the specified
	// ID exists on this driver.
	Exists(id string) bool
	// Status returns a set of key-value pairs which give low
	// level diagnostic status about this driver.
	Status() [][2]string
	// Returns a set of key-value pairs which give low level information
	// about the image/container driver is managing.
	GetMetadata(id string) (map[string]string, error)
	// Cleanup performs necessary tasks to release resources
	// held by the driver, e.g., unmounting all layered filesystems
	// known to this driver.
	Cleanup() error
}

// Driver is the interface for layered/snapshot file system drivers.
type Driver interface {
	ProtoDriver
	// Diff produces an archive of the changes between the specified
	// layer and its parent layer which may be "".
	Diff(id, parent string) (archive.Archive, error)
	// Changes produces a list of changes between the specified layer
	// and its parent layer. If parent is "", then all changes will be ADD changes.
	Changes(id, parent string) ([]archive.Change, error)
	// ApplyDiff extracts the changeset from the given diff into the
	// layer with the specified id and parent, returning the size of the
	// new layer in bytes.
	// The archive.Reader must be an uncompressed stream.
	ApplyDiff(id, parent string, diff archive.Reader) (size int64, err error)
	// DiffSize calculates the changes between the specified id
	// and its parent and returns the size in bytes of the changes
	// relative to its base filesystem directory.
	DiffSize(id, parent string) (size int64, err error)
}

// DiffGetterDriver is the interface for layered file system drivers that
// provide a specialized function for getting file contents for tar-split.
type DiffGetterDriver interface {
	Driver
	// DiffGetter returns an interface to efficiently retrieve the contents
	// of files in a layer.
	DiffGetter(id string) (FileGetCloser, error)
}

// FileGetCloser extends the storage.FileGetter interface with a Close method
// for cleaning up.
type FileGetCloser interface {
	storage.FileGetter
	// Close cleans up any resources associated with the FileGetCloser.
	Close() error
}

func init() {
	drivers = make(map[string]InitFunc)
}

// Register registers a InitFunc for the driver.
func Register(name string, initFunc InitFunc) error {
	if _, exists := drivers[name]; exists {
		return fmt.Errorf("Name already registered %s", name)
	}
	drivers[name] = initFunc

	return nil
}

// GetDriver initializes and returns the registered driver
func GetDriver(name, home string, options []string, uidMaps, gidMaps []idtools.IDMap) (Driver, error) {
	if initFunc, exists := drivers[name]; exists {
		return initFunc(filepath.Join(home, name), options, uidMaps, gidMaps)
	}
	if pluginDriver, err := lookupPlugin(name, home, options); err == nil {
		return pluginDriver, nil
	}
	logrus.Errorf("Failed to GetDriver graph %s %s", name, home)
	return nil, ErrNotSupported
}

// getBuiltinDriver initializes and returns the registered driver, but does not try to load from plugins
func getBuiltinDriver(name, home string, options []string, uidMaps, gidMaps []idtools.IDMap) (Driver, error) {
	if initFunc, exists := drivers[name]; exists {
		return initFunc(filepath.Join(home, name), options, uidMaps, gidMaps)
	}
	logrus.Errorf("Failed to built-in GetDriver graph %s %s", name, home)
	return nil, ErrNotSupported
}

// New creates the driver and initializes it at the specified root.
func New(root string, name string, options []string, uidMaps, gidMaps []idtools.IDMap) (Driver, error) {
	if name != "" {
		logrus.Debugf("[graphdriver] trying provided driver %q", name) // so the logs show specified driver
		return GetDriver(name, root, options, uidMaps, gidMaps)
	}

	// Guess for prior driver
	driversMap := scanPriorDrivers(root)
	for _, name := range priority {
		if name == "vfs" {
			// don't use vfs even if there is state present.
			continue
		}
		if _, prior := driversMap[name]; prior {
			// of the state found from prior drivers, check in order of our priority
			// which we would prefer
			driver, err := getBuiltinDriver(name, root, options, uidMaps, gidMaps)
			if err != nil {
				// unlike below, we will return error here, because there is prior
				// state, and now it is no longer supported/prereq/compatible, so
				// something changed and needs attention. Otherwise the daemon's
				// images would just "disappear".
				logrus.Errorf("[graphdriver] prior storage driver %q failed: %s", name, err)
				return nil, err
			}

			// abort starting when there are other prior configured drivers
			// to ensure the user explicitly selects the driver to load
			if len(driversMap)-1 > 0 {
				var driversSlice []string
				for name := range driversMap {
					driversSlice = append(driversSlice, name)
				}

				return nil, fmt.Errorf("%q contains several valid graphdrivers: %s; Please cleanup or explicitly choose storage driver (-s <DRIVER>)", root, strings.Join(driversSlice, ", "))
			}

			logrus.Infof("[graphdriver] using prior storage driver %q", name)
			return driver, nil
		}
	}

	// Check for priority drivers first
	for _, name := range priority {
		driver, err := getBuiltinDriver(name, root, options, uidMaps, gidMaps)
		if err != nil {
			if isDriverNotSupported(err) {
				continue
			}
			return nil, err
		}
		return driver, nil
	}

	// Check all registered drivers if no priority driver is found
	for name, initFunc := range drivers {
		driver, err := initFunc(filepath.Join(root, name), options, uidMaps, gidMaps)
		if err != nil {
			if isDriverNotSupported(err) {
				continue
			}
			return nil, err
		}
		return driver, nil
	}
	return nil, fmt.Errorf("No supported storage backend found")
}

// isDriverNotSupported returns true if the error initializing
// the graph driver is a non-supported error.
func isDriverNotSupported(err error) bool {
	return err == ErrNotSupported || err == ErrPrerequisites || err == ErrIncompatibleFS
}

// scanPriorDrivers returns an un-ordered scan of directories of prior storage drivers
func scanPriorDrivers(root string) map[string]bool {
	driversMap := make(map[string]bool)

	for driver := range drivers {
		p := filepath.Join(root, driver)
		if _, err := os.Stat(p); err == nil && driver != "vfs" {
			driversMap[driver] = true
		}
	}
	return driversMap
}
<|endoftext|>


<commit_before>// Copyright 2018 The ObjectHash-Proto Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tests

import (
	"testing"

	"github.com/golang/protobuf/proto"

	pb2_latest "github.com/deepmind/objecthash-proto/test_protos/generated/latest/proto2"
	pb3_latest "github.com/deepmind/objecthash-proto/test_protos/generated/latest/proto3"
)

// TestOneOfFields checks that oneof fields are handled properly.
func TestOneOfFields(t *testing.T, hashers ProtoHashers) {
	hasher := hashers.DefaultHasher

	testCases := []testCase{
		//////////////////////////
		//  Empty oneof fields. //
		//////////////////////////
		{
			protos: []proto.Message{
				&pb2_latest.Singleton{},
				&pb3_latest.Singleton{},

				&pb2_latest.Empty{},
				&pb3_latest.Empty{},
			},
			equivalentJSONString: "{}",
			equivalentObject:     map[int64]string{},
			expectedHashString:   "18ac3e7343f016890c510e93f935261169d9e3f565436429830faf0934f4f8e4",
		},

		/////////////////////////////////////////////
		//  One of the options selected but empty. //
		/////////////////////////////////////////////
		{
			protos: []proto.Message{
				// Only proto2 has empty values.
				&pb2_latest.Simple{BoolField: proto.Bool(false)},

				&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheBool{}},
				&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheBool{}},

				&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheBool{TheBool: false}},
				&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheBool{TheBool: false}},
			},
			// No equivalent JSON because JSON maps have to have strings as keys.
			equivalentObject:   map[int64]bool{1: false},
			expectedHashString: "8a956cfa8e9b45b738cb8dc8a3dc7126dab3cbd2c07c80fa1ec312a1a31ed709",
		},

		{
			protos: []proto.Message{
				// Only proto2 has empty values.
				&pb2_latest.Simple{StringField: proto.String("")},

				&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheString{}},
				&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheString{}},

				&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheString{TheString: ""}},
				&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheString{TheString: ""}},
			},
			// No equivalent JSON because JSON maps have to have strings as keys.
			equivalentObject:   map[int64]string{25: ""},
			expectedHashString: "79cff9d2d0ee6c6071c82b58d1a2fcf056b58c4501606862489e5731644c755a",
		},

		{
			protos: []proto.Message{
				// Only proto2 has empty values.
				&pb2_latest.Simple{Int32Field: proto.Int32(0)},

				&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheInt32{}},
				&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheInt32{}},

				&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheInt32{TheInt32: 0}},
				&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheInt32{TheInt32: 0}},
			},
			// No equivalent JSON because JSON maps have to have strings as keys.
			equivalentObject:   map[int64]int32{13: 0},
			expectedHashString: "bafd42680c987c47a76f72e08ed975877162efdb550d2c564c758dc7d988468f",
		},

		////////////////////////////////////////////////
		//  One of the options selected with content. //
		////////////////////////////////////////////////
		{
			protos: []proto.Message{
				&pb2_latest.Simple{StringField: proto.String("TEST!")},
				&pb3_latest.Simple{StringField: "TEST!"},

				&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheString{TheString: "TEST!"}},
				&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheString{TheString: "TEST!"}},
			},
			// No equivalent JSON because JSON maps have to have strings as keys.
			equivalentObject:   map[int64]string{25: "TEST!"},
			expectedHashString: "336cdbca99fd46157bc47bcc456f0ac7f1ef3be7a79acf3535f671434b53944f",
		},

		{
			protos: []proto.Message{
				&pb2_latest.Simple{Int32Field: proto.Int32(99)},
				&pb3_latest.Simple{Int32Field: 99},

				&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheInt32{TheInt32: 99}},
				&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheInt32{TheInt32: 99}},
			},
			// No equivalent JSON because JSON maps have to have strings as keys.
			equivalentObject:   map[int64]int32{13: 99},
			expectedHashString: "65517521bc278528d25caf1643da0f094fd88dad50205c9743e3c984a7c53b7d",
		},

		///////////////////////////
		//  Nested oneof fields. //
		///////////////////////////
		{
			protos: []proto.Message{
				&pb2_latest.Simple{SingletonField: &pb2_latest.Singleton{}},
				&pb3_latest.Simple{SingletonField: &pb3_latest.Singleton{}},

				&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheSingleton{TheSingleton: &pb2_latest.Singleton{}}},
				&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheSingleton{TheSingleton: &pb3_latest.Singleton{}}},
			},
			// No equivalent JSON because JSON maps have to have strings as keys.
			equivalentObject: map[int64]map[int64]int64{35: {}},
			// equivalentObject:   map[int64]map[int64]map[int64]int64{35: {35: {}}},
			expectedHashString: "4967c72525c764229f9fbf1294764c9aedc0d4f9f4c52e04a19c7f35ca65f517",
		},

		{
			protos: []proto.Message{
				&pb2_latest.Simple{SingletonField: &pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheSingleton{TheSingleton: &pb2_latest.Singleton{}}}},
				&pb3_latest.Simple{SingletonField: &pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheSingleton{TheSingleton: &pb3_latest.Singleton{}}}},

				&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheSingleton{TheSingleton: &pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheSingleton{TheSingleton: &pb2_latest.Singleton{}}}}},
				&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheSingleton{TheSingleton: &pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheSingleton{TheSingleton: &pb3_latest.Singleton{}}}}},
			},
			// No equivalent JSON because JSON maps have to have strings as keys.
			equivalentObject:   map[int64]map[int64]map[int64]int64{35: {35: {}}},
			expectedHashString: "8ea95bbda0f42073a61f46f9f375f48d5a7cb034fce56b44f958470fda5236d0",
		},
	}

	for _, tc := range testCases {
		if err := tc.check(hasher); err != nil {
			t.Errorf("%s", err)
		}

		if err := checkAsSingletonOnTheWire(tc, hasher); err != nil {
			t.Errorf("%s", err)
		}
	}
}

// Checks the provided test case after all its proto messages have been cycled
// to their wire format and unmarshalled back as a Singleton message.
func checkAsSingletonOnTheWire(tc testCase, hasher ProtoHasher) error {
	testCaseAfterAWireTransfer := testCase{
		protos:               tc.protos,
		equivalentJSONString: tc.equivalentJSONString,
		equivalentObject:     tc.equivalentObject,
		expectedHashString:   tc.expectedHashString,
	}

	for i, pb := range tc.protos {
		testCaseAfterAWireTransfer.protos[i] = unmarshalAsSingletonOnTheWire(pb)
	}

	return testCaseAfterAWireTransfer.check(hasher)
}

// Marshals a proto message to its wire format and returns its
// unmarshalled Singleton message.
func unmarshalAsSingletonOnTheWire(original proto.Message) proto.Message {
	binary, err := proto.Marshal(original)
	if err != nil {
		panic(err)
	}

	singleton := &pb3_latest.Singleton{}
	err = proto.Unmarshal(binary, singleton)
	if err != nil {
		panic(err)
	}

	return singleton
}
<commit_msg>Add explanation for equivalence of oneof fields with non-oneof versions.<commit_after>// Copyright 2018 The ObjectHash-Proto Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tests

import (
	"testing"

	"github.com/golang/protobuf/proto"

	pb2_latest "github.com/deepmind/objecthash-proto/test_protos/generated/latest/proto2"
	pb3_latest "github.com/deepmind/objecthash-proto/test_protos/generated/latest/proto3"
)

// TestOneOfFields checks that oneof fields are handled properly.
func TestOneOfFields(t *testing.T, hashers ProtoHashers) {
	hasher := hashers.DefaultHasher

	testCases := []testCase{
		//////////////////////////
		//  Empty oneof fields. //
		//////////////////////////
		{
			protos: []proto.Message{
				&pb2_latest.Singleton{},
				&pb3_latest.Singleton{},

				&pb2_latest.Empty{},
				&pb3_latest.Empty{},
			},
			equivalentJSONString: "{}",
			equivalentObject:     map[int64]string{},
			expectedHashString:   "18ac3e7343f016890c510e93f935261169d9e3f565436429830faf0934f4f8e4",
		},

		/////////////////////////////////////////////
		//  One of the options selected but empty. //
		/////////////////////////////////////////////
		{
			protos: []proto.Message{
				// Only proto2 has empty values.
				&pb2_latest.Simple{BoolField: proto.Bool(false)},

				&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheBool{}},
				&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheBool{}},

				&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheBool{TheBool: false}},
				&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheBool{TheBool: false}},
			},
			// No equivalent JSON because JSON maps have to have strings as keys.
			equivalentObject:   map[int64]bool{1: false},
			expectedHashString: "8a956cfa8e9b45b738cb8dc8a3dc7126dab3cbd2c07c80fa1ec312a1a31ed709",
		},

		{
			protos: []proto.Message{
				// Only proto2 has empty values.
				&pb2_latest.Simple{StringField: proto.String("")},

				&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheString{}},
				&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheString{}},

				&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheString{TheString: ""}},
				&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheString{TheString: ""}},
			},
			// No equivalent JSON because JSON maps have to have strings as keys.
			equivalentObject:   map[int64]string{25: ""},
			expectedHashString: "79cff9d2d0ee6c6071c82b58d1a2fcf056b58c4501606862489e5731644c755a",
		},

		{
			protos: []proto.Message{
				// Only proto2 has empty values.
				&pb2_latest.Simple{Int32Field: proto.Int32(0)},

				&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheInt32{}},
				&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheInt32{}},

				&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheInt32{TheInt32: 0}},
				&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheInt32{TheInt32: 0}},
			},
			// No equivalent JSON because JSON maps have to have strings as keys.
			equivalentObject:   map[int64]int32{13: 0},
			expectedHashString: "bafd42680c987c47a76f72e08ed975877162efdb550d2c564c758dc7d988468f",
		},

		////////////////////////////////////////////////
		//  One of the options selected with content. //
		////////////////////////////////////////////////
		//
		// For protobufs, it is legal (and backwards-compatible) to update a message by wrapping
		// an existing field within a oneof rule. Therefore, both objects (using old schem and
		// the new schema) should result in the same objecthash.
		//
		// Example:
		//
		// # Old schema:               | # New schema:
		// message Simple {            | message Singleton {
		//   string string_field = 25; |   oneof singleton {
		// }                           |     string the_string = 25;
		//                             |   }
		//                             | }
		//
		// The following examples demonstrate this equivalence.

		{
			protos: []proto.Message{
				&pb2_latest.Simple{StringField: proto.String("TEST!")},
				&pb3_latest.Simple{StringField: "TEST!"},

				&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheString{TheString: "TEST!"}},
				&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheString{TheString: "TEST!"}},
			},
			// No equivalent JSON because JSON maps have to have strings as keys.
			equivalentObject:   map[int64]string{25: "TEST!"},
			expectedHashString: "336cdbca99fd46157bc47bcc456f0ac7f1ef3be7a79acf3535f671434b53944f",
		},

		{
			protos: []proto.Message{
				&pb2_latest.Simple{Int32Field: proto.Int32(99)},
				&pb3_latest.Simple{Int32Field: 99},

				&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheInt32{TheInt32: 99}},
				&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheInt32{TheInt32: 99}},
			},
			// No equivalent JSON because JSON maps have to have strings as keys.
			equivalentObject:   map[int64]int32{13: 99},
			expectedHashString: "65517521bc278528d25caf1643da0f094fd88dad50205c9743e3c984a7c53b7d",
		},

		///////////////////////////
		//  Nested oneof fields. //
		///////////////////////////
		{
			protos: []proto.Message{
				&pb2_latest.Simple{SingletonField: &pb2_latest.Singleton{}},
				&pb3_latest.Simple{SingletonField: &pb3_latest.Singleton{}},

				&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheSingleton{TheSingleton: &pb2_latest.Singleton{}}},
				&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheSingleton{TheSingleton: &pb3_latest.Singleton{}}},
			},
			// No equivalent JSON because JSON maps have to have strings as keys.
			equivalentObject: map[int64]map[int64]int64{35: {}},
			// equivalentObject:   map[int64]map[int64]map[int64]int64{35: {35: {}}},
			expectedHashString: "4967c72525c764229f9fbf1294764c9aedc0d4f9f4c52e04a19c7f35ca65f517",
		},

		{
			protos: []proto.Message{
				&pb2_latest.Simple{SingletonField: &pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheSingleton{TheSingleton: &pb2_latest.Singleton{}}}},
				&pb3_latest.Simple{SingletonField: &pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheSingleton{TheSingleton: &pb3_latest.Singleton{}}}},

				&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheSingleton{TheSingleton: &pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheSingleton{TheSingleton: &pb2_latest.Singleton{}}}}},
				&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheSingleton{TheSingleton: &pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheSingleton{TheSingleton: &pb3_latest.Singleton{}}}}},
			},
			// No equivalent JSON because JSON maps have to have strings as keys.
			equivalentObject:   map[int64]map[int64]map[int64]int64{35: {35: {}}},
			expectedHashString: "8ea95bbda0f42073a61f46f9f375f48d5a7cb034fce56b44f958470fda5236d0",
		},
	}

	for _, tc := range testCases {
		if err := tc.check(hasher); err != nil {
			t.Errorf("%s", err)
		}

		if err := checkAsSingletonOnTheWire(tc, hasher); err != nil {
			t.Errorf("%s", err)
		}
	}
}

// Checks the provided test case after all its proto messages have been cycled
// to their wire format and unmarshalled back as a Singleton message.
func checkAsSingletonOnTheWire(tc testCase, hasher ProtoHasher) error {
	testCaseAfterAWireTransfer := testCase{
		protos:               tc.protos,
		equivalentJSONString: tc.equivalentJSONString,
		equivalentObject:     tc.equivalentObject,
		expectedHashString:   tc.expectedHashString,
	}

	for i, pb := range tc.protos {
		testCaseAfterAWireTransfer.protos[i] = unmarshalAsSingletonOnTheWire(pb)
	}

	return testCaseAfterAWireTransfer.check(hasher)
}

// Marshals a proto message to its wire format and returns its
// unmarshalled Singleton message.
func unmarshalAsSingletonOnTheWire(original proto.Message) proto.Message {
	binary, err := proto.Marshal(original)
	if err != nil {
		panic(err)
	}

	singleton := &pb3_latest.Singleton{}
	err = proto.Unmarshal(binary, singleton)
	if err != nil {
		panic(err)
	}

	return singleton
}
<|endoftext|>


<commit_before>// Copyright (c) 2017, Boise State University All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"regexp"
	"testing"
	"time"
)

func init() {
	f, _ := os.Create(os.DevNull)
	log.SetOutput(f)
}

func TestImageMatch(t *testing.T) {
	match := regexp.MustCompile(jupyterNotebookImageMatch)
	tests := []struct {
		s string
		m bool
	}{
		{"ksshannon/geo-notebook", true},
		{"ksshannon/geo-notebook:latest", true},
		{"ksshannon/geo-notebook:sometag", true},
		{"ksshannon/notanotebook", false},
		{"notanotebook", false},
		{"notanotebook:invalid", false},
		{"jupyter/tmpnb:latest", false},
		{"jupyter/configurable-http-proxy:latest", false},
	}
	for _, test := range tests {
		if match.MatchString(test.s) != test.m {
			t.Errorf("missed match: %v", test)
		}
	}
}

const skipDocker = "skipping docker dependent test"

func TestNewNotebook(t *testing.T) {
	if testing.Short() {
		t.Skip(skipDocker)
	}
	p, err := newNotebookPool(".*", 2, time.Minute*2)
	if err != nil {
		t.Fatal(err)
	}
	p.disableJupyterAuth = true
	p.stopCollector()
	nb, err := p.newNotebook("jupyter/minimal-notebook", false, "")
	if err != nil {
		t.Error(err)
	}
	time.Sleep(time.Second * 10)
	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/?token=", nb.port))
	if err != nil {
		t.Fatal(err)
	}
	resp.Body.Close()
	p.stopAndKillContainer(nb.id)
}

func TestCollection(t *testing.T) {
	if testing.Short() {
		t.Skip(skipDocker)
	}
	p, err := newNotebookPool(".*", 2, time.Second*5)
	if err != nil {
		t.Fatal(err)
	}
	p.disableJupyterAuth = true
	// Stop the collector, then restart it with an aggressive rate
	p.stopCollector()
	p.startCollector(time.Second)
	nb, err := p.newNotebook("jupyter/minimal-notebook", false, "")
	if err != nil {
		t.Error(err)
	}
	time.Sleep(time.Second * 6)
	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/?token=", nb.port))
	if err == nil {
		resp.Body.Close()
		t.Error("container should be dead")
	}
	n := len(p.activeNotebooks())
	if n != 0 {
		t.Errorf("pool not drained (%d)", n)
	}
	p.stopAndKillContainer(nb.id)
}

func TestZombies(t *testing.T) {
	if testing.Short() {
		t.Skip(skipDocker)
	}
	p, err := newNotebookPool(".*", 2, time.Minute*2)
	if err != nil {
		t.Fatal(err)
	}
	p.disableJupyterAuth = true
	p.stopCollector()
	nb, err := p.newNotebook("jupyter/minimal-notebook", false, "")
	if err != nil {
		t.Error(err)
	}
	if len(p.containerMap) != 1 {
		t.Error("failed to create container")
	}
	// manually remove the container from the container map, and drop the port
	p.portSet.Drop(nb.port)
	p.Lock()
	delete(p.containerMap, nb.hash)
	p.Unlock()
	c, err := p.zombieContainers()
	if len(c) != 1 {
		t.Error("failed to locate zombie")
	}
	err = p.killZombieContainers()
	if err != nil {
		t.Error(err)
	}
	c, err = p.zombieContainers()
	if len(c) != 0 {
		t.Error("failed to kill zombie")
	}
}
<commit_msg>check availableNotebooks in test<commit_after>// Copyright (c) 2017, Boise State University All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"regexp"
	"testing"
	"time"
)

func init() {
	f, _ := os.Create(os.DevNull)
	log.SetOutput(f)
}

func TestImageMatch(t *testing.T) {
	match := regexp.MustCompile(jupyterNotebookImageMatch)
	tests := []struct {
		s string
		m bool
	}{
		{"ksshannon/geo-notebook", true},
		{"ksshannon/geo-notebook:latest", true},
		{"ksshannon/geo-notebook:sometag", true},
		{"ksshannon/notanotebook", false},
		{"notanotebook", false},
		{"notanotebook:invalid", false},
		{"jupyter/tmpnb:latest", false},
		{"jupyter/configurable-http-proxy:latest", false},
	}
	for _, test := range tests {
		if match.MatchString(test.s) != test.m {
			t.Errorf("missed match: %v", test)
		}
	}
}

const skipDocker = "skipping docker dependent test"

func TestNewNotebook(t *testing.T) {
	if testing.Short() {
		t.Skip(skipDocker)
	}
	p, err := newNotebookPool(".*", 2, time.Minute*2)
	if err != nil {
		t.Fatal(err)
	}
	p.disableJupyterAuth = true
	p.stopCollector()
	nb, err := p.newNotebook("jupyter/minimal-notebook", false, "")
	if err != nil {
		t.Error(err)
	}
	time.Sleep(time.Second * 10)
	if len(p.activeNotebooks()) != 1 {
		t.Fatal("failed to create a notebook")
	}
	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/?token=", nb.port))
	if err != nil {
		t.Fatal(err)
	}
	resp.Body.Close()
	p.stopAndKillContainer(nb.id)
}

func TestCollection(t *testing.T) {
	if testing.Short() {
		t.Skip(skipDocker)
	}
	p, err := newNotebookPool(".*", 2, time.Second*5)
	if err != nil {
		t.Fatal(err)
	}
	p.disableJupyterAuth = true
	// Stop the collector, then restart it with an aggressive rate
	p.stopCollector()
	p.startCollector(time.Second)
	nb, err := p.newNotebook("jupyter/minimal-notebook", false, "")
	if err != nil {
		t.Error(err)
	}
	time.Sleep(time.Second * 6)
	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/?token=", nb.port))
	if err == nil {
		resp.Body.Close()
		t.Error("container should be dead")
	}
	n := len(p.activeNotebooks())
	if n != 0 {
		t.Errorf("pool not drained (%d)", n)
	}
	p.stopAndKillContainer(nb.id)
}

func TestZombies(t *testing.T) {
	if testing.Short() {
		t.Skip(skipDocker)
	}
	p, err := newNotebookPool(".*", 2, time.Minute*2)
	if err != nil {
		t.Fatal(err)
	}
	p.disableJupyterAuth = true
	p.stopCollector()
	nb, err := p.newNotebook("jupyter/minimal-notebook", false, "")
	if err != nil {
		t.Error(err)
	}
	if len(p.containerMap) != 1 {
		t.Error("failed to create container")
	}
	// manually remove the container from the container map, and drop the port
	p.portSet.Drop(nb.port)
	p.Lock()
	delete(p.containerMap, nb.hash)
	p.Unlock()
	c, err := p.zombieContainers()
	if len(c) != 1 {
		t.Error("failed to locate zombie")
	}
	err = p.killZombieContainers()
	if err != nil {
		t.Error(err)
	}
	c, err = p.zombieContainers()
	if len(c) != 0 {
		t.Error("failed to kill zombie")
	}
}
<|endoftext|>


<commit_before>package backend

import (
	"fmt"
	"io"
	"log"
	"path"
	"sort"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3iface"
)

type S3Backend struct {
	bucket     string
	path       string
	maxRetries int
	svc        s3iface.S3API
}

func NewS3Backend(bucket string, s3path string, maxRetries int, svc s3iface.S3API) *S3Backend {
	return &S3Backend{
		bucket:     bucket,
		path:       strings.TrimPrefix(path.Clean(s3path), "/"),
		maxRetries: maxRetries,
		svc:        svc,
	}
}

func (s *S3Backend) ListDBs() ([]string, error) {
	return s.listDirs(s.path, "")
}

func (s *S3Backend) ListVersions(db, after string, checkForSuccess bool) ([]string, error) {
	versions, err := s.listDirs(path.Join(s.path, db), after)
	if err != nil {
		return nil, err
	}

	if checkForSuccess {
		var filtered []string
		for _, version := range versions {
			successFile := path.Join(s.path, db, version, "_SUCCESS")
			exists := s.exists(successFile)

			if exists {
				filtered = append(filtered, version)
			}
		}

		versions = filtered
	}

	return versions, nil
}

func (s *S3Backend) listDirs(dir, after string) ([]string, error) {
	// This code assumes you're using S3 like a filesystem, with directories
	// separated by /'s. It also ignores the trailing slash on a prefix (for the
	// purposes of sorting lexicographically), to be consistent with other
	// backends.
	var res []string

	for {
		params := &s3.ListObjectsInput{
			Bucket:    aws.String(s.bucket),
			Delimiter: aws.String("/"),
			Marker:    aws.String(after),
			MaxKeys:   aws.Int64(1000),
			Prefix:    aws.String(dir + "/"),
		}
		resp, err := s.svc.ListObjects(params)

		if err != nil {
			return nil, s.s3error(err)
		} else if resp.CommonPrefixes == nil {
			break
		}

		for _, p := range resp.CommonPrefixes {
			prefix := strings.TrimSuffix(*p.Prefix, "/")

			// List the prefix, to make sure it's a "directory"
			isDir := false
			params := &s3.ListObjectsInput{
				Bucket:    aws.String(s.bucket),
				Delimiter: aws.String(""),
				Marker:    aws.String(after),
				MaxKeys:   aws.Int64(3),
				Prefix:    aws.String(prefix),
			}
			resp, err := s.svc.ListObjects(params)
			if err != nil {
				return nil, err
			}

			for _, key := range resp.Contents {
				if strings.TrimSpace(path.Base(*key.Key)) != "" {
					isDir = true
					break
				}
			}

			if isDir {
				res = append(res, path.Base(prefix))
			}
		}

		if !*resp.IsTruncated || len(resp.CommonPrefixes) == 0 {
			break
		} else {
			after = resp.CommonPrefixes[len(resp.CommonPrefixes)-1].String()
		}
	}

	sort.Strings(res)
	return res, nil
}

func (s *S3Backend) ListFiles(db, version string) ([]string, error) {
	versionPrefix := path.Join(s.path, db, version)

	// We use a set here because S3 sometimes returns duplicate keys.
	res := make(map[string]bool)

	params := &s3.ListObjectsInput{
		Bucket:    aws.String(s.bucket),
		Delimiter: aws.String(""),
		MaxKeys:   aws.Int64(1000),
		Prefix:    aws.String(versionPrefix),
	}

	datasetSize := int64(0)
	numFiles := int64(0)

	err := s.svc.ListObjectsPages(params, func(page *s3.ListObjectsOutput, isLastPage bool) bool {
		for _, key := range page.Contents {
			name := path.Base(*key.Key)
			// S3 sometimes has keys that are the same as the "directory"
			if strings.TrimSpace(name) != "" && !strings.HasPrefix(name, "_") && !strings.HasPrefix(name, ".") {
				datasetSize += *key.Size
				numFiles++
				res[name] = true
			}
		}

		log.Printf("call_site=s3.ListFiles sequins_db=%q sequins_db_version=%q dataset_size=%d file_count=%d", db, version, datasetSize, numFiles)

		return true
	})

	if err != nil {
		return nil, s.s3error(err)
	}

	sorted := make([]string, 0, len(res))
	for name := range res {
		sorted = append(sorted, name)
	}

	sort.Strings(sorted)
	return sorted, nil
}

func (s *S3Backend) Open(db, version, file string) (io.ReadCloser, error) {
	src := path.Join(s.path, db, version, file)
	params := &s3.GetObjectInput{
		Bucket: aws.String(s.bucket),
		Key:    aws.String(src),
	}
	resp, err := s.svc.GetObject(params)

	// If the download failed, due to the key not being found, retry
	// maxRetries number of times with an exponential backoff as it may
	// have been due to latency.
	backoff := time.Duration(1)
	for i := 0; i < s.maxRetries && err != nil; i++ {
		aerr, ok := err.(awserr.Error)
		if ok && aerr.Code() == s3.ErrCodeNoSuchKey {
			time.Sleep(backoff * time.Second)
			resp, err = s.svc.GetObject(params)
			backoff *= 2
		} else {
			break
		}
	}

	if err != nil {
		return nil, fmt.Errorf("error opening S3 path %s: %s", s.path, err)
	}

	return resp.Body, nil
}

func (s *S3Backend) DisplayPath(parts ...string) string {
	allParts := append([]string{s.path}, parts...)
	return s.displayURL(allParts...)
}

func (s *S3Backend) displayURL(parts ...string) string {
	key := strings.TrimPrefix(path.Join(parts...), "/")
	return fmt.Sprintf("s3://%s/%s", s.bucket, key)
}

func (s *S3Backend) exists(key string) bool {
	params := &s3.GetObjectInput{
		Bucket: aws.String(s.bucket),
		Key:    aws.String(key),
	}

	resp, err := s.svc.GetObject(params)
	if err != nil {
		return false
	}

	defer resp.Body.Close()

	return true
}

func (s *S3Backend) s3error(err error) error {
	return fmt.Errorf("unexpected S3 error on bucket %s: %s", s.bucket, err)
}
<commit_msg>only log after pagination<commit_after>package backend

import (
	"fmt"
	"io"
	"log"
	"path"
	"sort"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3iface"
)

type S3Backend struct {
	bucket     string
	path       string
	maxRetries int
	svc        s3iface.S3API
}

func NewS3Backend(bucket string, s3path string, maxRetries int, svc s3iface.S3API) *S3Backend {
	return &S3Backend{
		bucket:     bucket,
		path:       strings.TrimPrefix(path.Clean(s3path), "/"),
		maxRetries: maxRetries,
		svc:        svc,
	}
}

func (s *S3Backend) ListDBs() ([]string, error) {
	return s.listDirs(s.path, "")
}

func (s *S3Backend) ListVersions(db, after string, checkForSuccess bool) ([]string, error) {
	versions, err := s.listDirs(path.Join(s.path, db), after)
	if err != nil {
		return nil, err
	}

	if checkForSuccess {
		var filtered []string
		for _, version := range versions {
			successFile := path.Join(s.path, db, version, "_SUCCESS")
			exists := s.exists(successFile)

			if exists {
				filtered = append(filtered, version)
			}
		}

		versions = filtered
	}

	return versions, nil
}

func (s *S3Backend) listDirs(dir, after string) ([]string, error) {
	// This code assumes you're using S3 like a filesystem, with directories
	// separated by /'s. It also ignores the trailing slash on a prefix (for the
	// purposes of sorting lexicographically), to be consistent with other
	// backends.
	var res []string

	for {
		params := &s3.ListObjectsInput{
			Bucket:    aws.String(s.bucket),
			Delimiter: aws.String("/"),
			Marker:    aws.String(after),
			MaxKeys:   aws.Int64(1000),
			Prefix:    aws.String(dir + "/"),
		}
		resp, err := s.svc.ListObjects(params)

		if err != nil {
			return nil, s.s3error(err)
		} else if resp.CommonPrefixes == nil {
			break
		}

		for _, p := range resp.CommonPrefixes {
			prefix := strings.TrimSuffix(*p.Prefix, "/")

			// List the prefix, to make sure it's a "directory"
			isDir := false
			params := &s3.ListObjectsInput{
				Bucket:    aws.String(s.bucket),
				Delimiter: aws.String(""),
				Marker:    aws.String(after),
				MaxKeys:   aws.Int64(3),
				Prefix:    aws.String(prefix),
			}
			resp, err := s.svc.ListObjects(params)
			if err != nil {
				return nil, err
			}

			for _, key := range resp.Contents {
				if strings.TrimSpace(path.Base(*key.Key)) != "" {
					isDir = true
					break
				}
			}

			if isDir {
				res = append(res, path.Base(prefix))
			}
		}

		if !*resp.IsTruncated || len(resp.CommonPrefixes) == 0 {
			break
		} else {
			after = resp.CommonPrefixes[len(resp.CommonPrefixes)-1].String()
		}
	}

	sort.Strings(res)
	return res, nil
}

func (s *S3Backend) ListFiles(db, version string) ([]string, error) {
	versionPrefix := path.Join(s.path, db, version)

	// We use a set here because S3 sometimes returns duplicate keys.
	res := make(map[string]bool)

	params := &s3.ListObjectsInput{
		Bucket:    aws.String(s.bucket),
		Delimiter: aws.String(""),
		MaxKeys:   aws.Int64(1000),
		Prefix:    aws.String(versionPrefix),
	}

	datasetSize := int64(0)
	numFiles := int64(0)

	err := s.svc.ListObjectsPages(params, func(page *s3.ListObjectsOutput, isLastPage bool) bool {
		for _, key := range page.Contents {
			name := path.Base(*key.Key)
			// S3 sometimes has keys that are the same as the "directory"
			if strings.TrimSpace(name) != "" && !strings.HasPrefix(name, "_") && !strings.HasPrefix(name, ".") {
				datasetSize += *key.Size
				numFiles++
				res[name] = true
			}
		}
		return true
	})

	if err != nil {
		return nil, s.s3error(err)
	}

	log.Printf("call_site=s3.ListFiles sequins_db=%q sequins_db_version=%q dataset_size=%d file_count=%d", db, version, datasetSize, numFiles)

	sorted := make([]string, 0, len(res))
	for name := range res {
		sorted = append(sorted, name)
	}

	sort.Strings(sorted)
	return sorted, nil
}

func (s *S3Backend) Open(db, version, file string) (io.ReadCloser, error) {
	src := path.Join(s.path, db, version, file)
	params := &s3.GetObjectInput{
		Bucket: aws.String(s.bucket),
		Key:    aws.String(src),
	}
	resp, err := s.svc.GetObject(params)

	// If the download failed, due to the key not being found, retry
	// maxRetries number of times with an exponential backoff as it may
	// have been due to latency.
	backoff := time.Duration(1)
	for i := 0; i < s.maxRetries && err != nil; i++ {
		aerr, ok := err.(awserr.Error)
		if ok && aerr.Code() == s3.ErrCodeNoSuchKey {
			time.Sleep(backoff * time.Second)
			resp, err = s.svc.GetObject(params)
			backoff *= 2
		} else {
			break
		}
	}

	if err != nil {
		return nil, fmt.Errorf("error opening S3 path %s: %s", s.path, err)
	}

	return resp.Body, nil
}

func (s *S3Backend) DisplayPath(parts ...string) string {
	allParts := append([]string{s.path}, parts...)
	return s.displayURL(allParts...)
}

func (s *S3Backend) displayURL(parts ...string) string {
	key := strings.TrimPrefix(path.Join(parts...), "/")
	return fmt.Sprintf("s3://%s/%s", s.bucket, key)
}

func (s *S3Backend) exists(key string) bool {
	params := &s3.GetObjectInput{
		Bucket: aws.String(s.bucket),
		Key:    aws.String(key),
	}

	resp, err := s.svc.GetObject(params)
	if err != nil {
		return false
	}

	defer resp.Body.Close()

	return true
}

func (s *S3Backend) s3error(err error) error {
	return fmt.Errorf("unexpected S3 error on bucket %s: %s", s.bucket, err)
}
<|endoftext|>


<commit_before>package gocardless_pro_go

import (
	"fmt"
	"net/url"
)

type RefundService service

type Refund struct {
	ID							string				`json:"id,omitempty"`
	CreatedAt					string				`json:"created_at,omitempty"`
	Reference 					string				`json:"reference,omitempty"`
	Currency					string				`json:"currency,omitempty"`
	Amount						int64				`json:"amount,omitempty"`
	Links						[]RefundLink		`json:"links,omitempty"`
	Metadata					map[string]string	`json:"metadata,omitempty"`
}

type RefundListRequest struct {
	CreatedAt 			CreatedAt	`json:"created_at,omitempty"`
	Limit				int			`json:"limit,omitempty"`
	Before				string		`json:"before,omitempty"`
	After				string		`json:"after,omitempty"`
	Payment				string		`json:"payment,omitempty"`
}

type RefundList struct {
	Meta   ListMeta
	Values []Refund `json:"data"`
}

type RefundCreateRequest struct {
	Metadata				map[string]string		`json:"metadata,omitempty"`
	Reference				string					`json:"reference,omitempty"`
	Amount					string					`json:"amount,omitempty"`
	TotalAmountConfirmation	string					`json:"total_amount_confirmation,omitempty"`
	Links					[]string				`json:"links,omitempty"`
}

// Create creates a new refund
func (s *RefundService) CreateRefund(refundReq *RefundCreateRequest) (*Refund, error) {
	u := fmt.Sprintf("/refunds")
	refund := &Refund{}
	err := s.client.Call("POST", u, refundReq, refund)

	return refund, err
}

// List returns a list of refunds
func (s *RefundService) ListRefunds(req *RefundListRequest) (*RefundList, error) {
	return s.ListNRefunds(10, 0, req)
}

func (s *RefundService) ListNRefunds(count, offset int, req *RefundListRequest) (*RefundList, error) {
	params := url.Values{}
	params.Add("after", req.After)
	params.Add("before", req.Before)
	params.Add("created_at[gt]", req.CreatedAt.Gt)
	params.Add("created_at[gte]", req.CreatedAt.Gte)
	params.Add("created_at[lt]", req.CreatedAt.Lt)
	params.Add("created_at[lte]", req.CreatedAt.Lte)
	params.Add("limit", string(req.Limit))
	params.Add("payment", req.Payment)

	u := paginateURL("/refunds", count, offset)
	refunds := &RefundList{}
	err := s.client.Call("GET", u, params, refunds)

	return refunds, err
}

func (s *PaymentService) GetRefund(id string) (*Payment, error) {
	u := fmt.Sprintf("/payments/%s", id)
	payment := &Payment{}
	err := s.client.Call("GET", u, nil, payment)

	return payment, err
}

func (s *PaymentService) UpdatePayment(updatedPayment *Payment, metadata map[string]string) (*Payment, error) {
	params := url.Values{}
	params.Add("metadata", string(metadata))
	u := fmt.Sprintf("payments/%d", updatedPayment.ID)
	payment := &Payment{}
	err := s.client.Call("PUT", u, params, payment)

	return payment, err
}
<commit_msg>Add getting of refund<commit_after>package gocardless_pro_go

import (
	"fmt"
	"net/url"
)

type RefundService service

type Refund struct {
	ID							string				`json:"id,omitempty"`
	CreatedAt					string				`json:"created_at,omitempty"`
	Reference 					string				`json:"reference,omitempty"`
	Currency					string				`json:"currency,omitempty"`
	Amount						int64				`json:"amount,omitempty"`
	Links						[]RefundLink		`json:"links,omitempty"`
	Metadata					map[string]string	`json:"metadata,omitempty"`
}

type RefundListRequest struct {
	CreatedAt 			CreatedAt	`json:"created_at,omitempty"`
	Limit				int			`json:"limit,omitempty"`
	Before				string		`json:"before,omitempty"`
	After				string		`json:"after,omitempty"`
	Payment				string		`json:"payment,omitempty"`
}

type RefundList struct {
	Meta   ListMeta
	Values []Refund `json:"data"`
}

type RefundCreateRequest struct {
	Metadata				map[string]string		`json:"metadata,omitempty"`
	Reference				string					`json:"reference,omitempty"`
	Amount					string					`json:"amount,omitempty"`
	TotalAmountConfirmation	string					`json:"total_amount_confirmation,omitempty"`
	Links					[]string				`json:"links,omitempty"`
}

// Create creates a new refund
func (s *RefundService) CreateRefund(refundReq *RefundCreateRequest) (*Refund, error) {
	u := fmt.Sprintf("/refunds")
	refund := &Refund{}
	err := s.client.Call("POST", u, refundReq, refund)

	return refund, err
}

// List returns a list of refunds
func (s *RefundService) ListRefunds(req *RefundListRequest) (*RefundList, error) {
	return s.ListNRefunds(10, 0, req)
}

func (s *RefundService) ListNRefunds(count, offset int, req *RefundListRequest) (*RefundList, error) {
	params := url.Values{}
	params.Add("after", req.After)
	params.Add("before", req.Before)
	params.Add("created_at[gt]", req.CreatedAt.Gt)
	params.Add("created_at[gte]", req.CreatedAt.Gte)
	params.Add("created_at[lt]", req.CreatedAt.Lt)
	params.Add("created_at[lte]", req.CreatedAt.Lte)
	params.Add("limit", string(req.Limit))
	params.Add("payment", req.Payment)

	u := paginateURL("/refunds", count, offset)
	refunds := &RefundList{}
	err := s.client.Call("GET", u, params, refunds)

	return refunds, err
}

func (s *RefundService) GetRefund(id string) (*Refund, error) {
	u := fmt.Sprintf("/refunds/%s", id)
	refund := &Refund{}
	err := s.client.Call("GET", u, nil, payment)

	return refund, err
}

func (s *PaymentService) UpdatePayment(updatedPayment *Payment, metadata map[string]string) (*Payment, error) {
	params := url.Values{}
	params.Add("metadata", string(metadata))
	u := fmt.Sprintf("payments/%d", updatedPayment.ID)
	payment := &Payment{}
	err := s.client.Call("PUT", u, params, payment)

	return payment, err
}
<|endoftext|>


<commit_before>package types

import "fmt"

// A RegionID represents the ID of a Region
type RegionID string

// A Region holds information about a geographical region, including its ID, name & shape
type Region struct {
	ID     RegionID `json:"id"`
	Name   string   `json:"name"`
	Status string   `json:"status"`

	//DatasetCreation time.Time `json:"dataset_created_at"`
	//LastLoaded      time.Time `json:"last_load_at"`

	//ProductionStart time.Time `json:"start_production_date"`
	//ProductionEnd   time.Time `json:"end_production_date"`

	Error error `json:"error"`
}

// String stringifies a region
func (r Region) String() string {
	format := `ID: %s
Name: %s
Status: %s
Error: %v
`
	return fmt.Sprintf(format, r.ID, r.Name, r.Status, r.Error)
}
<commit_msg>Eliminate remnant of RegionID<commit_after>package types

import "fmt"

// A Region holds information about a geographical region, including its ID, name & shape
type Region struct {
	ID     ID     `json:"id"`
	Name   string `json:"name"`
	Status string `json:"status"`

	//DatasetCreation time.Time `json:"dataset_created_at"`
	//LastLoaded      time.Time `json:"last_load_at"`

	//ProductionStart time.Time `json:"start_production_date"`
	//ProductionEnd   time.Time `json:"end_production_date"`

	Error error `json:"error"`
}

// String stringifies a region
func (r Region) String() string {
	format := `ID: %s
Name: %s
Status: %s
Error: %v
`
	return fmt.Sprintf(format, r.ID, r.Name, r.Status, r.Error)
}
<|endoftext|>


<commit_before>package blockchain

import (
	"bytes"
	"errors"
	"fmt"
	"reflect"
	"time"

	. "github.com/tendermint/go-common"
	"github.com/tendermint/go-p2p"
	"github.com/tendermint/go-wire"
	"github.com/tendermint/tendermint/events"
	"github.com/tendermint/tendermint/proxy"
	sm "github.com/tendermint/tendermint/state"
	"github.com/tendermint/tendermint/types"
)

const (
	BlockchainChannel      = byte(0x40)
	defaultChannelCapacity = 100
	defaultSleepIntervalMS = 500
	trySyncIntervalMS      = 100
	// stop syncing when last block's time is
	// within this much of the system time.
	// stopSyncingDurationMinutes = 10

	// ask for best height every 10s
	statusUpdateIntervalSeconds = 10
	// check if we should switch to consensus reactor
	switchToConsensusIntervalSeconds = 1
	maxBlockchainResponseSize        = types.MaxBlockSize + 2
)

type consensusReactor interface {
	// for when we switch from blockchain reactor and fast sync to
	// the consensus machine
	SwitchToConsensus(*sm.State)
}

// BlockchainReactor handles long-term catchup syncing.
type BlockchainReactor struct {
	p2p.BaseReactor

	sw         *p2p.Switch
	state      *sm.State
	proxyAppCtx   proxy.AppContext // same as consensus.proxyAppCtx
	store      *BlockStore
	pool       *BlockPool
	sync       bool
	requestsCh chan BlockRequest
	timeoutsCh chan string
	lastBlock  *types.Block

	evsw events.Fireable
}

func NewBlockchainReactor(state *sm.State, proxyAppCtx proxy.AppContext, store *BlockStore, sync bool) *BlockchainReactor {
	if state.LastBlockHeight != store.Height() &&
		state.LastBlockHeight != store.Height()-1 { // XXX double check this logic.
		PanicSanity(Fmt("state (%v) and store (%v) height mismatch", state.LastBlockHeight, store.Height()))
	}
	requestsCh := make(chan BlockRequest, defaultChannelCapacity)
	timeoutsCh := make(chan string, defaultChannelCapacity)
	pool := NewBlockPool(
		store.Height()+1,
		requestsCh,
		timeoutsCh,
	)
	bcR := &BlockchainReactor{
		state:      state,
		proxyAppCtx:   proxyAppCtx,
		store:      store,
		pool:       pool,
		sync:       sync,
		requestsCh: requestsCh,
		timeoutsCh: timeoutsCh,
	}
	bcR.BaseReactor = *p2p.NewBaseReactor(log, "BlockchainReactor", bcR)
	return bcR
}

func (bcR *BlockchainReactor) OnStart() error {
	bcR.BaseReactor.OnStart()
	if bcR.sync {
		_, err := bcR.pool.Start()
		if err != nil {
			return err
		}
		go bcR.poolRoutine()
	}
	return nil
}

func (bcR *BlockchainReactor) OnStop() {
	bcR.BaseReactor.OnStop()
	bcR.pool.Stop()
}

// Implements Reactor
func (bcR *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor {
	return []*p2p.ChannelDescriptor{
		&p2p.ChannelDescriptor{
			ID:                BlockchainChannel,
			Priority:          5,
			SendQueueCapacity: 100,
		},
	}
}

// Implements Reactor
func (bcR *BlockchainReactor) AddPeer(peer *p2p.Peer) {
	// Send peer our state.
	peer.Send(BlockchainChannel, &bcStatusResponseMessage{bcR.store.Height()})
}

// Implements Reactor
func (bcR *BlockchainReactor) RemovePeer(peer *p2p.Peer, reason interface{}) {
	// Remove peer from the pool.
	bcR.pool.RemovePeer(peer.Key)
}

// Implements Reactor
func (bcR *BlockchainReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte) {
	_, msg, err := DecodeMessage(msgBytes)
	if err != nil {
		log.Warn("Error decoding message", "error", err)
		return
	}

	log.Notice("Received message", "src", src, "chID", chID, "msg", msg)

	switch msg := msg.(type) {
	case *bcBlockRequestMessage:
		// Got a request for a block. Respond with block if we have it.
		block := bcR.store.LoadBlock(msg.Height)
		if block != nil {
			msg := &bcBlockResponseMessage{Block: block}
			queued := src.TrySend(BlockchainChannel, msg)
			if !queued {
				// queue is full, just ignore.
			}
		} else {
			// TODO peer is asking for things we don't have.
		}
	case *bcBlockResponseMessage:
		// Got a block.
		bcR.pool.AddBlock(src.Key, msg.Block, len(msgBytes))
	case *bcStatusRequestMessage:
		// Send peer our state.
		queued := src.TrySend(BlockchainChannel, &bcStatusResponseMessage{bcR.store.Height()})
		if !queued {
			// sorry
		}
	case *bcStatusResponseMessage:
		// Got a peer status. Unverified.
		bcR.pool.SetPeerHeight(src.Key, msg.Height)
	default:
		log.Warn(Fmt("Unknown message type %v", reflect.TypeOf(msg)))
	}
}

// Handle messages from the poolReactor telling the reactor what to do.
// NOTE: Don't sleep in the FOR_LOOP or otherwise slow it down!
// (Except for the SYNC_LOOP, which is the primary purpose and must be synchronous.)
func (bcR *BlockchainReactor) poolRoutine() {

	trySyncTicker := time.NewTicker(trySyncIntervalMS * time.Millisecond)
	statusUpdateTicker := time.NewTicker(statusUpdateIntervalSeconds * time.Second)
	switchToConsensusTicker := time.NewTicker(switchToConsensusIntervalSeconds * time.Second)

FOR_LOOP:
	for {
		select {
		case request := <-bcR.requestsCh: // chan BlockRequest
			peer := bcR.Switch.Peers().Get(request.PeerID)
			if peer == nil {
				continue FOR_LOOP // Peer has since been disconnected.
			}
			msg := &bcBlockRequestMessage{request.Height}
			queued := peer.TrySend(BlockchainChannel, msg)
			if !queued {
				// We couldn't make the request, send-queue full.
				// The pool handles timeouts, just let it go.
				continue FOR_LOOP
			}
		case peerID := <-bcR.timeoutsCh: // chan string
			// Peer timed out.
			peer := bcR.Switch.Peers().Get(peerID)
			if peer != nil {
				bcR.Switch.StopPeerForError(peer, errors.New("BlockchainReactor Timeout"))
			}
		case _ = <-statusUpdateTicker.C:
			// ask for status updates
			go bcR.BroadcastStatusRequest()
		case _ = <-switchToConsensusTicker.C:
			height, numPending := bcR.pool.GetStatus()
			outbound, inbound, _ := bcR.Switch.NumPeers()
			log.Info("Consensus ticker", "numPending", numPending, "total", len(bcR.pool.requesters),
				"outbound", outbound, "inbound", inbound)
			if bcR.pool.IsCaughtUp() {
				log.Notice("Time to switch to consensus reactor!", "height", height)
				bcR.pool.Stop()

				conR := bcR.Switch.Reactor("CONSENSUS").(consensusReactor)
				conR.SwitchToConsensus(bcR.state)

				break FOR_LOOP
			}
		case _ = <-trySyncTicker.C: // chan time
			// This loop can be slow as long as it's doing syncing work.
		SYNC_LOOP:
			for i := 0; i < 10; i++ {
				// See if there are any blocks to sync.
				first, second := bcR.pool.PeekTwoBlocks()
				//log.Info("TrySync peeked", "first", first, "second", second)
				if first == nil || second == nil {
					// We need both to sync the first block.
					break SYNC_LOOP
				}
				firstParts := first.MakePartSet()
				firstPartsHeader := firstParts.Header()
				// Finally, verify the first block using the second's validation.
				err := bcR.state.Validators.VerifyValidation(
					bcR.state.ChainID, first.Hash(), firstPartsHeader, first.Height, second.LastValidation)
				if err != nil {
					log.Info("error in validation", "error", err)
					bcR.pool.RedoRequest(first.Height)
					break SYNC_LOOP
				} else {
					bcR.pool.PopRequest()
					err := bcR.state.ExecBlock(bcR.proxyAppCtx, first, firstPartsHeader)
					if err != nil {
						// TODO This is bad, are we zombie?
						PanicQ(Fmt("Failed to process committed block: %v", err))
					}
					err = bcR.state.Commit(bcR.proxyAppCtx)
					if err != nil {
						// TODO Handle gracefully.
						PanicQ(Fmt("Failed to commit block at application: %v", err))
					}
					bcR.store.SaveBlock(first, firstParts, second.LastValidation)
					bcR.state.Save()
				}
			}
			continue FOR_LOOP
		case <-bcR.Quit:
			break FOR_LOOP
		}
	}
}

func (bcR *BlockchainReactor) BroadcastStatusResponse() error {
	bcR.Switch.Broadcast(BlockchainChannel, &bcStatusResponseMessage{bcR.store.Height()})
	return nil
}

func (bcR *BlockchainReactor) BroadcastStatusRequest() error {
	bcR.Switch.Broadcast(BlockchainChannel, &bcStatusRequestMessage{bcR.store.Height()})
	return nil
}

// implements events.Eventable
func (bcR *BlockchainReactor) SetFireable(evsw events.Fireable) {
	bcR.evsw = evsw
}

//-----------------------------------------------------------------------------
// Messages

const (
	msgTypeBlockRequest   = byte(0x10)
	msgTypeBlockResponse  = byte(0x11)
	msgTypeStatusResponse = byte(0x20)
	msgTypeStatusRequest  = byte(0x21)
)

type BlockchainMessage interface{}

var _ = wire.RegisterInterface(
	struct{ BlockchainMessage }{},
	wire.ConcreteType{&bcBlockRequestMessage{}, msgTypeBlockRequest},
	wire.ConcreteType{&bcBlockResponseMessage{}, msgTypeBlockResponse},
	wire.ConcreteType{&bcStatusResponseMessage{}, msgTypeStatusResponse},
	wire.ConcreteType{&bcStatusRequestMessage{}, msgTypeStatusRequest},
)

// TODO: ensure that bz is completely read.
func DecodeMessage(bz []byte) (msgType byte, msg BlockchainMessage, err error) {
	msgType = bz[0]
	n := int(0)
	r := bytes.NewReader(bz)
	msg = wire.ReadBinary(struct{ BlockchainMessage }{}, r, maxBlockchainResponseSize, &n, &err).(struct{ BlockchainMessage }).BlockchainMessage
	if err != nil && n != len(bz) {
		err = errors.New("DecodeMessage() had bytes left over.")
	}
	return
}

//-------------------------------------

type bcBlockRequestMessage struct {
	Height int
}

func (m *bcBlockRequestMessage) String() string {
	return fmt.Sprintf("[bcBlockRequestMessage %v]", m.Height)
}

//-------------------------------------

// NOTE: keep up-to-date with maxBlockchainResponseSize
type bcBlockResponseMessage struct {
	Block *types.Block
}

func (m *bcBlockResponseMessage) String() string {
	return fmt.Sprintf("[bcBlockResponseMessage %v]", m.Block.Height)
}

//-------------------------------------

type bcStatusRequestMessage struct {
	Height int
}

func (m *bcStatusRequestMessage) String() string {
	return fmt.Sprintf("[bcStatusRequestMessage %v]", m.Height)
}

//-------------------------------------

type bcStatusResponseMessage struct {
	Height int
}

func (m *bcStatusResponseMessage) String() string {
	return fmt.Sprintf("[bcStatusResponseMessage %v]", m.Height)
}
<commit_msg>Fix BlockchainReactor bug w/ mismatched state.LastBlockHeight vs store.Height<commit_after>package blockchain

import (
	"bytes"
	"errors"
	"fmt"
	"reflect"
	"time"

	. "github.com/tendermint/go-common"
	"github.com/tendermint/go-p2p"
	"github.com/tendermint/go-wire"
	"github.com/tendermint/tendermint/events"
	"github.com/tendermint/tendermint/proxy"
	sm "github.com/tendermint/tendermint/state"
	"github.com/tendermint/tendermint/types"
)

const (
	BlockchainChannel      = byte(0x40)
	defaultChannelCapacity = 100
	defaultSleepIntervalMS = 500
	trySyncIntervalMS      = 100
	// stop syncing when last block's time is
	// within this much of the system time.
	// stopSyncingDurationMinutes = 10

	// ask for best height every 10s
	statusUpdateIntervalSeconds = 10
	// check if we should switch to consensus reactor
	switchToConsensusIntervalSeconds = 1
	maxBlockchainResponseSize        = types.MaxBlockSize + 2
)

type consensusReactor interface {
	// for when we switch from blockchain reactor and fast sync to
	// the consensus machine
	SwitchToConsensus(*sm.State)
}

// BlockchainReactor handles long-term catchup syncing.
type BlockchainReactor struct {
	p2p.BaseReactor

	sw          *p2p.Switch
	state       *sm.State
	proxyAppCtx proxy.AppContext // same as consensus.proxyAppCtx
	store       *BlockStore
	pool        *BlockPool
	sync        bool
	requestsCh  chan BlockRequest
	timeoutsCh  chan string
	lastBlock   *types.Block

	evsw events.Fireable
}

func NewBlockchainReactor(state *sm.State, proxyAppCtx proxy.AppContext, store *BlockStore, sync bool) *BlockchainReactor {
	if state.LastBlockHeight == store.Height()-1 {
		store.height -= 1 // XXX HACK, make this better
	}
	if state.LastBlockHeight != store.Height() {
		PanicSanity(Fmt("state (%v) and store (%v) height mismatch", state.LastBlockHeight, store.Height()))
	}
	requestsCh := make(chan BlockRequest, defaultChannelCapacity)
	timeoutsCh := make(chan string, defaultChannelCapacity)
	pool := NewBlockPool(
		store.Height()+1,
		requestsCh,
		timeoutsCh,
	)
	bcR := &BlockchainReactor{
		state:       state,
		proxyAppCtx: proxyAppCtx,
		store:       store,
		pool:        pool,
		sync:        sync,
		requestsCh:  requestsCh,
		timeoutsCh:  timeoutsCh,
	}
	bcR.BaseReactor = *p2p.NewBaseReactor(log, "BlockchainReactor", bcR)
	return bcR
}

func (bcR *BlockchainReactor) OnStart() error {
	bcR.BaseReactor.OnStart()
	if bcR.sync {
		_, err := bcR.pool.Start()
		if err != nil {
			return err
		}
		go bcR.poolRoutine()
	}
	return nil
}

func (bcR *BlockchainReactor) OnStop() {
	bcR.BaseReactor.OnStop()
	bcR.pool.Stop()
}

// Implements Reactor
func (bcR *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor {
	return []*p2p.ChannelDescriptor{
		&p2p.ChannelDescriptor{
			ID:                BlockchainChannel,
			Priority:          5,
			SendQueueCapacity: 100,
		},
	}
}

// Implements Reactor
func (bcR *BlockchainReactor) AddPeer(peer *p2p.Peer) {
	// Send peer our state.
	peer.Send(BlockchainChannel, &bcStatusResponseMessage{bcR.store.Height()})
}

// Implements Reactor
func (bcR *BlockchainReactor) RemovePeer(peer *p2p.Peer, reason interface{}) {
	// Remove peer from the pool.
	bcR.pool.RemovePeer(peer.Key)
}

// Implements Reactor
func (bcR *BlockchainReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte) {
	_, msg, err := DecodeMessage(msgBytes)
	if err != nil {
		log.Warn("Error decoding message", "error", err)
		return
	}

	log.Notice("Received message", "src", src, "chID", chID, "msg", msg)

	switch msg := msg.(type) {
	case *bcBlockRequestMessage:
		// Got a request for a block. Respond with block if we have it.
		block := bcR.store.LoadBlock(msg.Height)
		if block != nil {
			msg := &bcBlockResponseMessage{Block: block}
			queued := src.TrySend(BlockchainChannel, msg)
			if !queued {
				// queue is full, just ignore.
			}
		} else {
			// TODO peer is asking for things we don't have.
		}
	case *bcBlockResponseMessage:
		// Got a block.
		bcR.pool.AddBlock(src.Key, msg.Block, len(msgBytes))
	case *bcStatusRequestMessage:
		// Send peer our state.
		queued := src.TrySend(BlockchainChannel, &bcStatusResponseMessage{bcR.store.Height()})
		if !queued {
			// sorry
		}
	case *bcStatusResponseMessage:
		// Got a peer status. Unverified.
		bcR.pool.SetPeerHeight(src.Key, msg.Height)
	default:
		log.Warn(Fmt("Unknown message type %v", reflect.TypeOf(msg)))
	}
}

// Handle messages from the poolReactor telling the reactor what to do.
// NOTE: Don't sleep in the FOR_LOOP or otherwise slow it down!
// (Except for the SYNC_LOOP, which is the primary purpose and must be synchronous.)
func (bcR *BlockchainReactor) poolRoutine() {

	trySyncTicker := time.NewTicker(trySyncIntervalMS * time.Millisecond)
	statusUpdateTicker := time.NewTicker(statusUpdateIntervalSeconds * time.Second)
	switchToConsensusTicker := time.NewTicker(switchToConsensusIntervalSeconds * time.Second)

FOR_LOOP:
	for {
		select {
		case request := <-bcR.requestsCh: // chan BlockRequest
			peer := bcR.Switch.Peers().Get(request.PeerID)
			if peer == nil {
				continue FOR_LOOP // Peer has since been disconnected.
			}
			msg := &bcBlockRequestMessage{request.Height}
			queued := peer.TrySend(BlockchainChannel, msg)
			if !queued {
				// We couldn't make the request, send-queue full.
				// The pool handles timeouts, just let it go.
				continue FOR_LOOP
			}
		case peerID := <-bcR.timeoutsCh: // chan string
			// Peer timed out.
			peer := bcR.Switch.Peers().Get(peerID)
			if peer != nil {
				bcR.Switch.StopPeerForError(peer, errors.New("BlockchainReactor Timeout"))
			}
		case _ = <-statusUpdateTicker.C:
			// ask for status updates
			go bcR.BroadcastStatusRequest()
		case _ = <-switchToConsensusTicker.C:
			height, numPending := bcR.pool.GetStatus()
			outbound, inbound, _ := bcR.Switch.NumPeers()
			log.Info("Consensus ticker", "numPending", numPending, "total", len(bcR.pool.requesters),
				"outbound", outbound, "inbound", inbound)
			if bcR.pool.IsCaughtUp() {
				log.Notice("Time to switch to consensus reactor!", "height", height)
				bcR.pool.Stop()

				conR := bcR.Switch.Reactor("CONSENSUS").(consensusReactor)
				conR.SwitchToConsensus(bcR.state)

				break FOR_LOOP
			}
		case _ = <-trySyncTicker.C: // chan time
			// This loop can be slow as long as it's doing syncing work.
		SYNC_LOOP:
			for i := 0; i < 10; i++ {
				// See if there are any blocks to sync.
				first, second := bcR.pool.PeekTwoBlocks()
				//log.Info("TrySync peeked", "first", first, "second", second)
				if first == nil || second == nil {
					// We need both to sync the first block.
					break SYNC_LOOP
				}
				firstParts := first.MakePartSet()
				firstPartsHeader := firstParts.Header()
				// Finally, verify the first block using the second's validation.
				err := bcR.state.Validators.VerifyValidation(
					bcR.state.ChainID, first.Hash(), firstPartsHeader, first.Height, second.LastValidation)
				if err != nil {
					log.Info("error in validation", "error", err)
					bcR.pool.RedoRequest(first.Height)
					break SYNC_LOOP
				} else {
					bcR.pool.PopRequest()
					err := bcR.state.ExecBlock(bcR.proxyAppCtx, first, firstPartsHeader)
					if err != nil {
						// TODO This is bad, are we zombie?
						PanicQ(Fmt("Failed to process committed block: %v", err))
					}
					err = bcR.state.Commit(bcR.proxyAppCtx)
					if err != nil {
						// TODO Handle gracefully.
						PanicQ(Fmt("Failed to commit block at application: %v", err))
					}
					bcR.store.SaveBlock(first, firstParts, second.LastValidation)
					bcR.state.Save()
				}
			}
			continue FOR_LOOP
		case <-bcR.Quit:
			break FOR_LOOP
		}
	}
}

func (bcR *BlockchainReactor) BroadcastStatusResponse() error {
	bcR.Switch.Broadcast(BlockchainChannel, &bcStatusResponseMessage{bcR.store.Height()})
	return nil
}

func (bcR *BlockchainReactor) BroadcastStatusRequest() error {
	bcR.Switch.Broadcast(BlockchainChannel, &bcStatusRequestMessage{bcR.store.Height()})
	return nil
}

// implements events.Eventable
func (bcR *BlockchainReactor) SetFireable(evsw events.Fireable) {
	bcR.evsw = evsw
}

//-----------------------------------------------------------------------------
// Messages

const (
	msgTypeBlockRequest   = byte(0x10)
	msgTypeBlockResponse  = byte(0x11)
	msgTypeStatusResponse = byte(0x20)
	msgTypeStatusRequest  = byte(0x21)
)

type BlockchainMessage interface{}

var _ = wire.RegisterInterface(
	struct{ BlockchainMessage }{},
	wire.ConcreteType{&bcBlockRequestMessage{}, msgTypeBlockRequest},
	wire.ConcreteType{&bcBlockResponseMessage{}, msgTypeBlockResponse},
	wire.ConcreteType{&bcStatusResponseMessage{}, msgTypeStatusResponse},
	wire.ConcreteType{&bcStatusRequestMessage{}, msgTypeStatusRequest},
)

// TODO: ensure that bz is completely read.
func DecodeMessage(bz []byte) (msgType byte, msg BlockchainMessage, err error) {
	msgType = bz[0]
	n := int(0)
	r := bytes.NewReader(bz)
	msg = wire.ReadBinary(struct{ BlockchainMessage }{}, r, maxBlockchainResponseSize, &n, &err).(struct{ BlockchainMessage }).BlockchainMessage
	if err != nil && n != len(bz) {
		err = errors.New("DecodeMessage() had bytes left over.")
	}
	return
}

//-------------------------------------

type bcBlockRequestMessage struct {
	Height int
}

func (m *bcBlockRequestMessage) String() string {
	return fmt.Sprintf("[bcBlockRequestMessage %v]", m.Height)
}

//-------------------------------------

// NOTE: keep up-to-date with maxBlockchainResponseSize
type bcBlockResponseMessage struct {
	Block *types.Block
}

func (m *bcBlockResponseMessage) String() string {
	return fmt.Sprintf("[bcBlockResponseMessage %v]", m.Block.Height)
}

//-------------------------------------

type bcStatusRequestMessage struct {
	Height int
}

func (m *bcStatusRequestMessage) String() string {
	return fmt.Sprintf("[bcStatusRequestMessage %v]", m.Height)
}

//-------------------------------------

type bcStatusResponseMessage struct {
	Height int
}

func (m *bcStatusResponseMessage) String() string {
	return fmt.Sprintf("[bcStatusResponseMessage %v]", m.Height)
}
<|endoftext|>


<commit_before>package booklitcmd

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"

	"github.com/sirupsen/logrus"
	"github.com/vito/booklit"
	"github.com/vito/booklit/baselit"
	"github.com/vito/booklit/load"
	"github.com/vito/booklit/render"
)

type Command struct {
	Version func() `short:"v" long:"version" description:"Print the version of Boooklit and exit."`

	In  string `long:"in"  short:"i" required:"true" description:"Input .lit file to load."`
	Out string `long:"out" short:"o" description:"Directory into which sections will be rendered."`

	SectionTag  string `long:"section-tag"  description:"Section tag to render."`
	SectionPath string `long:"section-path" description:"Section path to load and render with --in as its parent."`

	SaveSearchIndex bool `long:"save-search-index" description:"Save a search index JSON file in the destination."`

	ServerPort int `long:"serve" short:"s" description:"Start an HTTP server on the given port."`

	Plugins []string `long:"plugin" short:"p" description:"Package to import, providing a plugin."`

	Debug bool `long:"debug" short:"d" description:"Log at debug level."`

	AllowBrokenReferences bool `long:"allow-broken-references" description:"Replace broken references with a bogus tag."`

	HTMLEngine struct {
		Templates string `long:"templates" description:"Directory containing .tmpl files to load."`
	} `group:"HTML Rendering Engine" namespace:"html"`

	TextEngine struct {
		FileExtension string `long:"file-extension" description:"File extension to use for generated files."`
		Templates     string `long:"templates"      description:"Directory containing .tmpl files to load."`
	} `group:"Text Rendering Engine" namespace:"text"`
}

func (cmd *Command) Execute(args []string) error {
	if cmd.Debug {
		logrus.SetLevel(logrus.DebugLevel)
	}

	isReexec := os.Getenv("BOOKLIT_REEXEC") != ""
	if !isReexec && len(cmd.Plugins) > 0 {
		logrus.Debug("plugins configured; reexecing")

		exitCode, err := cmd.reexec()
		if err != nil {
			return err
		}

		os.Exit(exitCode)

		return nil
	}

	if cmd.ServerPort != 0 {
		return cmd.Serve()
	}

	return cmd.Build()
}

func (cmd *Command) Serve() error {
	http.Handle("/", &Server{
		In: cmd.In,
		Processor: &load.Processor{
			AllowBrokenReferences: cmd.AllowBrokenReferences,
		},

		Templates:  cmd.HTMLEngine.Templates,
		Engine:     render.NewHTMLEngine(),
		FileServer: http.FileServer(http.Dir(cmd.Out)),
	})

	logrus.WithField("port", cmd.ServerPort).Info("listening")

	return http.ListenAndServe(fmt.Sprintf(":%d", cmd.ServerPort), nil)
}

var basePluginFactories = []booklit.PluginFactory{
	baselit.NewPlugin,
}

func (cmd *Command) Build() error {
	processor := &load.Processor{
		AllowBrokenReferences: cmd.AllowBrokenReferences,
	}

	var engine render.Engine
	if cmd.TextEngine.FileExtension != "" {
		textEngine := render.NewTextEngine(cmd.TextEngine.FileExtension)

		if cmd.TextEngine.Templates != "" {
			err := textEngine.LoadTemplates(cmd.TextEngine.Templates)
			if err != nil {
				return err
			}
		}

		engine = textEngine
	} else {
		htmlEngine := render.NewHTMLEngine()

		if cmd.HTMLEngine.Templates != "" {
			err := htmlEngine.LoadTemplates(cmd.HTMLEngine.Templates)
			if err != nil {
				return err
			}
		}

		engine = htmlEngine
	}

	section, err := processor.LoadFile(cmd.In, basePluginFactories)
	if err != nil {
		return err
	}

	sectionToRender := section
	if cmd.SectionTag != "" {
		tags := section.FindTag(cmd.SectionTag)
		if len(tags) == 0 {
			return fmt.Errorf("unknown tag: %s", cmd.SectionTag)
		}

		sectionToRender = tags[0].Section
	} else if cmd.SectionPath != "" {
		sectionToRender, err = processor.LoadFileIn(section, cmd.SectionPath, basePluginFactories)
		if err != nil {
			return err
		}
	}

	if cmd.Out == "" {
		return engine.RenderSection(os.Stdout, sectionToRender)
	}

	err = os.MkdirAll(cmd.Out, 0755)
	if err != nil {
		return err
	}

	writer := render.Writer{
		Engine:      engine,
		Destination: cmd.Out,
	}

	err = writer.WriteSection(sectionToRender)
	if err != nil {
		return err
	}

	if cmd.SaveSearchIndex {
		err = writer.WriteSearchIndex(section, "search_index.json")
		if err != nil {
			return err
		}
	}

	return nil
}

func (cmd *Command) reexec() (int, error) {
	tmpdir, err := ioutil.TempDir("", "booklit-reexec")
	if err != nil {
		return 0, err
	}

	defer func() {
		_ = os.RemoveAll(tmpdir)
	}()

	src := filepath.Join(tmpdir, "main.go")
	bin := filepath.Join(tmpdir, "main")

	goSrc := "package main\n"
	goSrc += "import \"github.com/vito/booklit/booklitcmd\"\n"
	for _, p := range cmd.Plugins {
		goSrc += "import _ \"" + p + "\"\n"
	}
	goSrc += "func main() {\n"
	goSrc += "	booklitcmd.Main()\n"
	goSrc += "}\n"

	err = ioutil.WriteFile(src, []byte(goSrc), 0644)
	if err != nil {
		return 0, err
	}

	build := exec.Command("go", "install", src)
	build.Env = append(os.Environ(), "GOBIN="+tmpdir)
	build.Stdout = os.Stdout
	build.Stderr = os.Stderr

	logrus.Debug("building reexec binary")

	err = build.Run()
	if err != nil {
		return 0, fmt.Errorf("build failed: %w", err)
	}

	run := exec.Command(bin, os.Args[1:]...)
	run.Env = append(os.Environ(), "BOOKLIT_REEXEC=1")
	run.Stdout = os.Stdout
	run.Stderr = os.Stderr

	logrus.Debug("reexecing")

	err = run.Run()
	if err != nil {
		if exitErr, ok := err.(*exec.ExitError); ok {
			return exitErr.ExitCode(), nil
		}

		return 0, fmt.Errorf("reexec failed: %w", err)
	}

	return 0, nil
}
<commit_msg>add --http-profile and --cpu-profile<commit_after>package booklitcmd

import (
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/exec"
	"path/filepath"
	"runtime/pprof"

	"github.com/sirupsen/logrus"
	"github.com/vito/booklit"
	"github.com/vito/booklit/baselit"
	"github.com/vito/booklit/load"
	"github.com/vito/booklit/render"
)

type Command struct {
	Version func() `short:"v" long:"version" description:"Print the version of Boooklit and exit."`

	In  string `long:"in"  short:"i" required:"true" description:"Input .lit file to load."`
	Out string `long:"out" short:"o" description:"Directory into which sections will be rendered."`

	SectionTag  string `long:"section-tag"  description:"Section tag to render."`
	SectionPath string `long:"section-path" description:"Section path to load and render with --in as its parent."`

	SaveSearchIndex bool `long:"save-search-index" description:"Save a search index JSON file in the destination."`

	ServerPort int `long:"serve" short:"s" description:"Start an HTTP server on the given port."`

	Plugins []string `long:"plugin" short:"p" description:"Package to import, providing a plugin."`

	Debug bool `long:"debug" short:"d" description:"Log at debug level."`

	AllowBrokenReferences bool `long:"allow-broken-references" description:"Replace broken references with a bogus tag."`

	HTTPProfilePort int    `long:"http-profile" description:"Start the Go net/http/pprof server on this port."`
	CPUProfilePath  string `long:"cpu-profile"  description:"Write a CPU profile to this path."`

	HTMLEngine struct {
		Templates string `long:"templates" description:"Directory containing .tmpl files to load."`
	} `group:"HTML Rendering Engine" namespace:"html"`

	TextEngine struct {
		FileExtension string `long:"file-extension" description:"File extension to use for generated files."`
		Templates     string `long:"templates"      description:"Directory containing .tmpl files to load."`
	} `group:"Text Rendering Engine" namespace:"text"`
}

func (cmd *Command) Execute(args []string) error {
	if cmd.Debug {
		logrus.SetLevel(logrus.DebugLevel)
	}

	isReexec := os.Getenv("BOOKLIT_REEXEC") != ""
	if !isReexec && len(cmd.Plugins) > 0 {
		logrus.Debug("plugins configured; reexecing")

		exitCode, err := cmd.reexec()
		if err != nil {
			return err
		}

		os.Exit(exitCode)

		return nil
	}

	if cmd.HTTPProfilePort != 0 {
		logrus.Debugf("serving pprof on :%d", cmd.HTTPProfilePort)

		l, err := net.Listen("tcp", fmt.Sprintf(":%d", cmd.HTTPProfilePort))
		if err != nil {
			return err
		}

		go http.Serve(l, nil)
	}

	if cmd.CPUProfilePath != "" {
		profFile, err := os.Create(cmd.CPUProfilePath)
		if err != nil {
			return err
		}

		defer profFile.Close()

		pprof.StartCPUProfile(profFile)
		defer pprof.StopCPUProfile()
	}

	if cmd.ServerPort != 0 {
		return cmd.Serve()
	}

	return cmd.Build()
}

func (cmd *Command) Serve() error {
	http.Handle("/", &Server{
		In: cmd.In,
		Processor: &load.Processor{
			AllowBrokenReferences: cmd.AllowBrokenReferences,
		},

		Templates:  cmd.HTMLEngine.Templates,
		Engine:     render.NewHTMLEngine(),
		FileServer: http.FileServer(http.Dir(cmd.Out)),
	})

	logrus.WithField("port", cmd.ServerPort).Info("listening")

	return http.ListenAndServe(fmt.Sprintf(":%d", cmd.ServerPort), nil)
}

var basePluginFactories = []booklit.PluginFactory{
	baselit.NewPlugin,
}

func (cmd *Command) Build() error {
	processor := &load.Processor{
		AllowBrokenReferences: cmd.AllowBrokenReferences,
	}

	var engine render.Engine
	if cmd.TextEngine.FileExtension != "" {
		textEngine := render.NewTextEngine(cmd.TextEngine.FileExtension)

		if cmd.TextEngine.Templates != "" {
			err := textEngine.LoadTemplates(cmd.TextEngine.Templates)
			if err != nil {
				return err
			}
		}

		engine = textEngine
	} else {
		htmlEngine := render.NewHTMLEngine()

		if cmd.HTMLEngine.Templates != "" {
			err := htmlEngine.LoadTemplates(cmd.HTMLEngine.Templates)
			if err != nil {
				return err
			}
		}

		engine = htmlEngine
	}

	section, err := processor.LoadFile(cmd.In, basePluginFactories)
	if err != nil {
		return err
	}

	sectionToRender := section
	if cmd.SectionTag != "" {
		tags := section.FindTag(cmd.SectionTag)
		if len(tags) == 0 {
			return fmt.Errorf("unknown tag: %s", cmd.SectionTag)
		}

		sectionToRender = tags[0].Section
	} else if cmd.SectionPath != "" {
		sectionToRender, err = processor.LoadFileIn(section, cmd.SectionPath, basePluginFactories)
		if err != nil {
			return err
		}
	}

	if cmd.Out == "" {
		return engine.RenderSection(os.Stdout, sectionToRender)
	}

	err = os.MkdirAll(cmd.Out, 0755)
	if err != nil {
		return err
	}

	writer := render.Writer{
		Engine:      engine,
		Destination: cmd.Out,
	}

	err = writer.WriteSection(sectionToRender)
	if err != nil {
		return err
	}

	if cmd.SaveSearchIndex {
		err = writer.WriteSearchIndex(section, "search_index.json")
		if err != nil {
			return err
		}
	}

	return nil
}

func (cmd *Command) reexec() (int, error) {
	tmpdir, err := ioutil.TempDir("", "booklit-reexec")
	if err != nil {
		return 0, err
	}

	defer func() {
		_ = os.RemoveAll(tmpdir)
	}()

	src := filepath.Join(tmpdir, "main.go")
	bin := filepath.Join(tmpdir, "main")

	goSrc := "package main\n"
	goSrc += "import \"github.com/vito/booklit/booklitcmd\"\n"
	for _, p := range cmd.Plugins {
		goSrc += "import _ \"" + p + "\"\n"
	}
	goSrc += "func main() {\n"
	goSrc += "	booklitcmd.Main()\n"
	goSrc += "}\n"

	err = ioutil.WriteFile(src, []byte(goSrc), 0644)
	if err != nil {
		return 0, err
	}

	build := exec.Command("go", "install", src)
	build.Env = append(os.Environ(), "GOBIN="+tmpdir)
	build.Stdout = os.Stdout
	build.Stderr = os.Stderr

	logrus.Debug("building reexec binary")

	err = build.Run()
	if err != nil {
		return 0, fmt.Errorf("build failed: %w", err)
	}

	run := exec.Command(bin, os.Args[1:]...)
	run.Env = append(os.Environ(), "BOOKLIT_REEXEC=1")
	run.Stdout = os.Stdout
	run.Stderr = os.Stderr

	logrus.Debug("reexecing")

	err = run.Run()
	if err != nil {
		if exitErr, ok := err.(*exec.ExitError); ok {
			return exitErr.ExitCode(), nil
		}

		return 0, fmt.Errorf("reexec failed: %w", err)
	}

	return 0, nil
}
<|endoftext|>