<commit_before>package db

import (
	"fmt"
	"go/ast"
	"net/url"
	"reflect"
	"sort"
	"strings"

	"github.com/lxc/lxd/lxd/db/generate/lex"
	"github.com/pkg/errors"
)

// Packages returns the the AST packages in which to search for structs.
//
// By default it includes the lxd/db and shared/api packages.
func Packages() (map[string]*ast.Package, error) {
	packages := map[string]*ast.Package{}

	for _, name := range defaultPackages {
		pkg, err := lex.Parse(name)
		if err != nil {
			return nil, errors.Wrapf(err, "Parse %q", name)
		}
		parts := strings.Split(name, "/")
		packages[parts[len(parts)-1]] = pkg
	}

	return packages, nil
}

var defaultPackages = []string{
	"github.com/lxc/lxd/shared/api",
	"github.com/lxc/lxd/lxd/db",
}

// Filters parses all filtering statement defined for the given entity. It
// returns all supported combinations of filters, sorted by number of criteria.
func Filters(pkg *ast.Package, entity string) [][]string {
	objects := pkg.Scope.Objects
	filters := [][]string{}

	prefix := fmt.Sprintf("%sObjectsBy", lex.Minuscule(lex.Camel(entity)))

	for name := range objects {
		if !strings.HasPrefix(name, prefix) {
			continue
		}
		rest := name[len(prefix):]
		filters = append(filters, strings.Split(rest, "And"))
	}

	return sortFilters(filters)
}

// RefFilters parses all filtering statement defined for the given entity reference.
func RefFilters(pkg *ast.Package, entity string, ref string) [][]string {
	objects := pkg.Scope.Objects
	filters := [][]string{}

	prefix := fmt.Sprintf("%s%sRefBy", lex.Minuscule(lex.Camel(entity)), lex.Capital(ref))

	for name := range objects {
		if !strings.HasPrefix(name, prefix) {
			continue
		}
		rest := name[len(prefix):]
		filters = append(filters, strings.Split(rest, "And"))
	}

	return sortFilters(filters)
}

func sortFilters(filters [][]string) [][]string {
	sort.Slice(filters, func(i, j int) bool {
		n1 := len(filters[i])
		n2 := len(filters[j])
		if n1 != n2 {
			return n1 > n2
		}
		f1 := sortFilter(filters[i])
		f2 := sortFilter(filters[j])
		for k := range f1 {
			if f1[k] == f2[k] {
				continue
			}
			return f1[k] > f2[k]
		}
		panic("duplicate filter")
	})
	return filters
}

func sortFilter(filter []string) []string {
	f := make([]string, len(filter))
	copy(f, filter)
	sort.Sort(sort.Reverse(sort.StringSlice(f)))
	return f
}

// Criteria returns a list of criteria
func Criteria(pkg *ast.Package, entity string) ([]string, error) {
	name := fmt.Sprintf("%sFilter", lex.Camel(entity))
	str := findStruct(pkg.Scope, name)

	if str == nil {
		return nil, fmt.Errorf("No filter declared for %q", entity)
	}

	criteria := []string{}

	for _, f := range str.Fields.List {
		if len(f.Names) != 1 {
			return nil, fmt.Errorf("Unexpected fields number")
		}

		if !f.Names[0].IsExported() {
			return nil, fmt.Errorf("Unexported field name")
		}

		criteria = append(criteria, f.Names[0].Name)
	}

	return criteria, nil
}

// Parse the structure declaration with the given name found in the given Go
// package.
func Parse(pkg *ast.Package, name string) (*Mapping, error) {
	str := findStruct(pkg.Scope, name)
	if str == nil {
		return nil, fmt.Errorf("No declaration found for %q", name)
	}

	fields, err := parseStruct(str)
	if err != nil {
		return nil, errors.Wrapf(err, "Failed to parse %q", name)
	}

	m := &Mapping{
		Package: pkg.Name,
		Name:    name,
		Fields:  fields,
	}

	return m, nil
}

// Find the StructType node for the structure with the given name
func findStruct(scope *ast.Scope, name string) *ast.StructType {
	obj := scope.Lookup(name)
	if obj == nil {
		return nil
	}

	typ, ok := obj.Decl.(*ast.TypeSpec)
	if !ok {
		return nil
	}

	str, ok := typ.Type.(*ast.StructType)
	if !ok {
		return nil
	}

	return str
}

// Extract field information from the given structure.
func parseStruct(str *ast.StructType) ([]*Field, error) {
	fields := make([]*Field, 0)

	for _, f := range str.Fields.List {
		if len(f.Names) == 0 {
			// Check if this is a parent struct.
			ident, ok := f.Type.(*ast.Ident)
			if !ok {
				continue
			}

			typ, ok := ident.Obj.Decl.(*ast.TypeSpec)
			if !ok {
				continue
			}

			parentStr, ok := typ.Type.(*ast.StructType)
			if !ok {
				continue
			}

			parentFields, err := parseStruct(parentStr)
			if err != nil {
				return nil, errors.Wrapf(err, "Failed to parse parent struct")
			}
			fields = append(fields, parentFields...)

			continue
		}

		if len(f.Names) != 1 {
			return nil, fmt.Errorf("Expected a single field name, got %q", f.Names)
		}

		field, err := parseField(f)
		if err != nil {
			return nil, err
		}

		// Don't add field if it has been ignored.
		if field != nil {
			fields = append(fields, field)
		}
	}

	return fields, nil
}

func parseField(f *ast.Field) (*Field, error) {
	name := f.Names[0]

	if !name.IsExported() {
		//return nil, fmt.Errorf("Unexported field name %q", name.Name)
	}

	// Ignore fields that are marked with a tag of `db:"ingore"`
	if f.Tag != nil {
		tag := f.Tag.Value
		tagValue := reflect.StructTag(tag[1 : len(tag)-1]).Get("db")
		if tagValue == "ignore" {
			return nil, nil
		}
	}

	typeName := parseType(f.Type)
	if typeName == "" {
		return nil, fmt.Errorf("Unsupported type for field %q", name.Name)
	}

	typeObj := Type{
		Name: typeName,
	}

	if IsColumnType(typeName) {
		typeObj.Code = TypeColumn
	} else if strings.HasPrefix(typeName, "[]") {
		typeObj.Code = TypeSlice
	} else if strings.HasPrefix(typeName, "map[") {
		typeObj.Code = TypeMap
	} else {
		return nil, fmt.Errorf("Unsupported type for field %q", name.Name)
	}

	var config url.Values
	if f.Tag != nil {
		tag := f.Tag.Value
		var err error
		config, err = url.ParseQuery(reflect.StructTag(tag[1 : len(tag)-1]).Get("db"))
		if err != nil {
			return nil, errors.Wrap(err, "Parse 'db' structure tag")
		}
	}

	field := Field{
		Name:   name.Name,
		Type:   typeObj,
		Config: config,
	}

	return &field, nil
}

func parseType(x ast.Expr) string {
	switch t := x.(type) {
	case *ast.StarExpr:
		// Pointers are not supported.
		return ""
	case *ast.SelectorExpr:
		return parseType(t.X) + "." + t.Sel.String()
	case *ast.Ident:
		s := t.String()
		if s == "byte" {
			return "uint8"
		}
		return s
	case *ast.ArrayType:
		return "[" + parseType(t.Len) + "]" + parseType(t.Elt)
	case *ast.MapType:
		return "map[" + parseType(t.Key) + "]" + parseType(t.Value)
	case *ast.BasicLit:
		return t.Value
	case nil:
		return ""
	default:
		return ""
	}
}
<commit_msg>lxd/db/generate/db/parse: use kind parameter for Filter generation<commit_after>package db

import (
	"fmt"
	"go/ast"
	"net/url"
	"reflect"
	"sort"
	"strings"

	"github.com/lxc/lxd/lxd/db/generate/lex"
	"github.com/pkg/errors"
)

// Packages returns the the AST packages in which to search for structs.
//
// By default it includes the lxd/db and shared/api packages.
func Packages() (map[string]*ast.Package, error) {
	packages := map[string]*ast.Package{}

	for _, name := range defaultPackages {
		pkg, err := lex.Parse(name)
		if err != nil {
			return nil, errors.Wrapf(err, "Parse %q", name)
		}
		parts := strings.Split(name, "/")
		packages[parts[len(parts)-1]] = pkg
	}

	return packages, nil
}

var defaultPackages = []string{
	"github.com/lxc/lxd/shared/api",
	"github.com/lxc/lxd/lxd/db",
}

// Filters parses all filtering statement defined for the given entity. It
// returns all supported combinations of filters, sorted by number of criteria.
func Filters(pkg *ast.Package, kind string, entity string) [][]string {
	objects := pkg.Scope.Objects
	filters := [][]string{}

	prefix := fmt.Sprintf("%s%sBy", lex.Minuscule(lex.Camel(entity)), lex.Camel(kind))

	for name := range objects {
		if !strings.HasPrefix(name, prefix) {
			continue
		}
		rest := name[len(prefix):]
		filters = append(filters, strings.Split(rest, "And"))
	}

	return sortFilters(filters)
}

// RefFilters parses all filtering statement defined for the given entity reference.
func RefFilters(pkg *ast.Package, entity string, ref string) [][]string {
	objects := pkg.Scope.Objects
	filters := [][]string{}

	prefix := fmt.Sprintf("%s%sRefBy", lex.Minuscule(lex.Camel(entity)), lex.Capital(ref))

	for name := range objects {
		if !strings.HasPrefix(name, prefix) {
			continue
		}
		rest := name[len(prefix):]
		filters = append(filters, strings.Split(rest, "And"))
	}

	return sortFilters(filters)
}

func sortFilters(filters [][]string) [][]string {
	sort.Slice(filters, func(i, j int) bool {
		n1 := len(filters[i])
		n2 := len(filters[j])
		if n1 != n2 {
			return n1 > n2
		}
		f1 := sortFilter(filters[i])
		f2 := sortFilter(filters[j])
		for k := range f1 {
			if f1[k] == f2[k] {
				continue
			}
			return f1[k] > f2[k]
		}
		panic("duplicate filter")
	})
	return filters
}

func sortFilter(filter []string) []string {
	f := make([]string, len(filter))
	copy(f, filter)
	sort.Sort(sort.Reverse(sort.StringSlice(f)))
	return f
}

// Criteria returns a list of criteria
func Criteria(pkg *ast.Package, entity string) ([]string, error) {
	name := fmt.Sprintf("%sFilter", lex.Camel(entity))
	str := findStruct(pkg.Scope, name)

	if str == nil {
		return nil, fmt.Errorf("No filter declared for %q", entity)
	}

	criteria := []string{}

	for _, f := range str.Fields.List {
		if len(f.Names) != 1 {
			return nil, fmt.Errorf("Unexpected fields number")
		}

		if !f.Names[0].IsExported() {
			return nil, fmt.Errorf("Unexported field name")
		}

		criteria = append(criteria, f.Names[0].Name)
	}

	return criteria, nil
}

// Parse the structure declaration with the given name found in the given Go
// package.
func Parse(pkg *ast.Package, name string) (*Mapping, error) {
	str := findStruct(pkg.Scope, name)
	if str == nil {
		return nil, fmt.Errorf("No declaration found for %q", name)
	}

	fields, err := parseStruct(str)
	if err != nil {
		return nil, errors.Wrapf(err, "Failed to parse %q", name)
	}

	m := &Mapping{
		Package: pkg.Name,
		Name:    name,
		Fields:  fields,
	}

	return m, nil
}

// Find the StructType node for the structure with the given name
func findStruct(scope *ast.Scope, name string) *ast.StructType {
	obj := scope.Lookup(name)
	if obj == nil {
		return nil
	}

	typ, ok := obj.Decl.(*ast.TypeSpec)
	if !ok {
		return nil
	}

	str, ok := typ.Type.(*ast.StructType)
	if !ok {
		return nil
	}

	return str
}

// Extract field information from the given structure.
func parseStruct(str *ast.StructType) ([]*Field, error) {
	fields := make([]*Field, 0)

	for _, f := range str.Fields.List {
		if len(f.Names) == 0 {
			// Check if this is a parent struct.
			ident, ok := f.Type.(*ast.Ident)
			if !ok {
				continue
			}

			typ, ok := ident.Obj.Decl.(*ast.TypeSpec)
			if !ok {
				continue
			}

			parentStr, ok := typ.Type.(*ast.StructType)
			if !ok {
				continue
			}

			parentFields, err := parseStruct(parentStr)
			if err != nil {
				return nil, errors.Wrapf(err, "Failed to parse parent struct")
			}
			fields = append(fields, parentFields...)

			continue
		}

		if len(f.Names) != 1 {
			return nil, fmt.Errorf("Expected a single field name, got %q", f.Names)
		}

		field, err := parseField(f)
		if err != nil {
			return nil, err
		}

		// Don't add field if it has been ignored.
		if field != nil {
			fields = append(fields, field)
		}
	}

	return fields, nil
}

func parseField(f *ast.Field) (*Field, error) {
	name := f.Names[0]

	if !name.IsExported() {
		//return nil, fmt.Errorf("Unexported field name %q", name.Name)
	}

	// Ignore fields that are marked with a tag of `db:"ingore"`
	if f.Tag != nil {
		tag := f.Tag.Value
		tagValue := reflect.StructTag(tag[1 : len(tag)-1]).Get("db")
		if tagValue == "ignore" {
			return nil, nil
		}
	}

	typeName := parseType(f.Type)
	if typeName == "" {
		return nil, fmt.Errorf("Unsupported type for field %q", name.Name)
	}

	typeObj := Type{
		Name: typeName,
	}

	if IsColumnType(typeName) {
		typeObj.Code = TypeColumn
	} else if strings.HasPrefix(typeName, "[]") {
		typeObj.Code = TypeSlice
	} else if strings.HasPrefix(typeName, "map[") {
		typeObj.Code = TypeMap
	} else {
		return nil, fmt.Errorf("Unsupported type for field %q", name.Name)
	}

	var config url.Values
	if f.Tag != nil {
		tag := f.Tag.Value
		var err error
		config, err = url.ParseQuery(reflect.StructTag(tag[1 : len(tag)-1]).Get("db"))
		if err != nil {
			return nil, errors.Wrap(err, "Parse 'db' structure tag")
		}
	}

	field := Field{
		Name:   name.Name,
		Type:   typeObj,
		Config: config,
	}

	return &field, nil
}

func parseType(x ast.Expr) string {
	switch t := x.(type) {
	case *ast.StarExpr:
		// Pointers are not supported.
		return ""
	case *ast.SelectorExpr:
		return parseType(t.X) + "." + t.Sel.String()
	case *ast.Ident:
		s := t.String()
		if s == "byte" {
			return "uint8"
		}
		return s
	case *ast.ArrayType:
		return "[" + parseType(t.Len) + "]" + parseType(t.Elt)
	case *ast.MapType:
		return "map[" + parseType(t.Key) + "]" + parseType(t.Value)
	case *ast.BasicLit:
		return t.Value
	case nil:
		return ""
	default:
		return ""
	}
}
<|endoftext|>


<commit_before>package endpoints_test

import (
	"io/ioutil"
	"net"
	"os"
	"testing"

	"github.com/lxc/lxd/shared"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// If no socket-based activation is detected, a new local unix socket will be
// created.
func TestEndpoints_LocalCreateUnixSocket(t *testing.T) {
	endpoints, config, cleanup := newEndpoints(t)
	defer cleanup()

	require.NoError(t, endpoints.Up(config))

	path := endpoints.LocalSocketPath()
	assert.NoError(t, httpGetOverUnixSocket(path))

	// The unix socket file gets removed after shutdown.
	cleanup()
	assert.Equal(t, false, shared.PathExists(path))
}

// If socket-based activation is detected, it will be used for binding the API
// Endpoints' unix socket.
func TestEndpoints_LocalSocketBasedActivation(t *testing.T) {
	listener := newUnixListener(t)
	defer listener.Close() // This will also remove the underlying file

	file, err := listener.File()
	require.NoError(t, err)
	defer file.Close()

	endpoints, config, cleanup := newEndpoints(t)
	defer cleanup()

	setupSocketBasedActivation(endpoints, file)

	require.NoError(t, endpoints.Up(config))

	assertNoSocketBasedActivation(t)

	path := endpoints.LocalSocketPath()
	assert.NoError(t, httpGetOverUnixSocket(path))

	// The unix socket file does not get removed after shutdown (thanks to
	// this change in Go 1.6:
	//
	// https://github.com/golang/go/commit/a4fd325c178ea29f554d69de4f2c3ffa09b53874
	//
	// which prevents listeners created from file descriptors from removing
	// their socket files on close).
	cleanup()
	assert.Equal(t, true, shared.PathExists(path))
}

// If a custom group for the unix socket is specified, but no such one exists,
// an error is returned.
func TestEndpoints_LocalUnknownUnixGroup(t *testing.T) {
	endpoints, config, cleanup := newEndpoints(t)
	defer cleanup()

	config.LocalUnixSocketGroup = "xquibaz"
	err := endpoints.Up(config)

	assert.EqualError(
		t, err, "local endpoint: cannot get group ID of 'xquibaz': unknown group xquibaz")
}

// If another LXD endpoints is already listening on the unix socket, an error is
// returned.
func TestEndpoints_LocalAlreadyRunning(t *testing.T) {
	endpoints1, config1, cleanup1 := newEndpoints(t)
	defer cleanup1()

	require.NoError(t, endpoints1.Up(config1))

	endpoints2, config2, cleanup2 := newEndpoints(t)
	config2.Dir = config1.Dir
	config2.UnixSocket = config1.UnixSocket
	defer cleanup2()

	err := endpoints2.Up(config2)
	assert.EqualError(t, err, "local endpoint: LXD is already running")
}

// Create a UnixListener using a random and unique file name.
func newUnixListener(t *testing.T) *net.UnixListener {
	file, err := ioutil.TempFile("", "lxd-endpoints-test")
	require.NoError(t, err)

	path := file.Name()
	file.Close()
	os.Remove(path)

	addr, err := net.ResolveUnixAddr("unix", path)
	require.NoError(t, err)

	listener, err := net.ListenUnix("unix", addr)

	require.NoError(t, err)
	return listener
}
<commit_msg>lxd/endpoints: Update error string in test<commit_after>package endpoints_test

import (
	"io/ioutil"
	"net"
	"os"
	"testing"

	"github.com/lxc/lxd/shared"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// If no socket-based activation is detected, a new local unix socket will be
// created.
func TestEndpoints_LocalCreateUnixSocket(t *testing.T) {
	endpoints, config, cleanup := newEndpoints(t)
	defer cleanup()

	require.NoError(t, endpoints.Up(config))

	path := endpoints.LocalSocketPath()
	assert.NoError(t, httpGetOverUnixSocket(path))

	// The unix socket file gets removed after shutdown.
	cleanup()
	assert.Equal(t, false, shared.PathExists(path))
}

// If socket-based activation is detected, it will be used for binding the API
// Endpoints' unix socket.
func TestEndpoints_LocalSocketBasedActivation(t *testing.T) {
	listener := newUnixListener(t)
	defer listener.Close() // This will also remove the underlying file

	file, err := listener.File()
	require.NoError(t, err)
	defer file.Close()

	endpoints, config, cleanup := newEndpoints(t)
	defer cleanup()

	setupSocketBasedActivation(endpoints, file)

	require.NoError(t, endpoints.Up(config))

	assertNoSocketBasedActivation(t)

	path := endpoints.LocalSocketPath()
	assert.NoError(t, httpGetOverUnixSocket(path))

	// The unix socket file does not get removed after shutdown (thanks to
	// this change in Go 1.6:
	//
	// https://github.com/golang/go/commit/a4fd325c178ea29f554d69de4f2c3ffa09b53874
	//
	// which prevents listeners created from file descriptors from removing
	// their socket files on close).
	cleanup()
	assert.Equal(t, true, shared.PathExists(path))
}

// If a custom group for the unix socket is specified, but no such one exists,
// an error is returned.
func TestEndpoints_LocalUnknownUnixGroup(t *testing.T) {
	endpoints, config, cleanup := newEndpoints(t)
	defer cleanup()

	config.LocalUnixSocketGroup = "xquibaz"
	err := endpoints.Up(config)

	assert.EqualError(
		t, err, "local endpoint: cannot get group ID of 'xquibaz': group: unknown group xquibaz")
}

// If another LXD endpoints is already listening on the unix socket, an error is
// returned.
func TestEndpoints_LocalAlreadyRunning(t *testing.T) {
	endpoints1, config1, cleanup1 := newEndpoints(t)
	defer cleanup1()

	require.NoError(t, endpoints1.Up(config1))

	endpoints2, config2, cleanup2 := newEndpoints(t)
	config2.Dir = config1.Dir
	config2.UnixSocket = config1.UnixSocket
	defer cleanup2()

	err := endpoints2.Up(config2)
	assert.EqualError(t, err, "local endpoint: LXD is already running")
}

// Create a UnixListener using a random and unique file name.
func newUnixListener(t *testing.T) *net.UnixListener {
	file, err := ioutil.TempFile("", "lxd-endpoints-test")
	require.NoError(t, err)

	path := file.Name()
	file.Close()
	os.Remove(path)

	addr, err := net.ResolveUnixAddr("unix", path)
	require.NoError(t, err)

	listener, err := net.ListenUnix("unix", addr)

	require.NoError(t, err)
	return listener
}
<|endoftext|>


<commit_before>package storage

import (
	"io"

	"github.com/lxc/lxd/lxd/migration"
	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/lxd/storage/drivers"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/logger"
)

type mockBackend struct {
	name   string
	state  *state.State
	logger logger.Logger
}

func (b *mockBackend) DaemonState() *state.State {
	return b.state
}

func (b *mockBackend) ID() int64 {
	return -1
}

func (b *mockBackend) Name() string {
	return b.name
}

func (b *mockBackend) Driver() drivers.Driver {
	return nil
}

func (b *mockBackend) MigrationTypes(contentType drivers.ContentType) []migration.Type {
	return []migration.Type{
		{
			FSType:   migration.MigrationFSType_RSYNC,
			Features: []string{"xattrs", "delete", "compress", "bidirectional"},
		},
	}
}

func (b *mockBackend) GetResources() (*api.ResourcesStoragePool, error) {
	return nil, nil
}

func (b *mockBackend) Delete(op *operations.Operation) error {
	return nil
}

func (b *mockBackend) Mount() (bool, error) {
	return true, nil
}

func (b *mockBackend) Unmount() (bool, error) {
	return true, nil
}

func (b *mockBackend) CreateInstance(i Instance, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) CreateInstanceFromBackup(i Instance, sourcePath string, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) CreateInstanceFromCopy(i Instance, src Instance, snapshots bool, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) CreateInstanceFromImage(i Instance, fingerprint string, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) CreateInstanceFromMigration(i Instance, conn io.ReadWriteCloser, args migration.SinkArgs, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) RenameInstance(i Instance, newName string, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) DeleteInstance(i Instance, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) MigrateInstance(i Instance, snapshots bool, args migration.SourceArgs) (migration.StorageSourceDriver, error) {
	return nil, nil
}

func (b *mockBackend) RefreshInstance(i Instance, src Instance, snapshots bool, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) BackupInstance(i Instance, targetPath string, optimized bool, snapshots bool, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) GetInstanceUsage(i Instance) (int64, error) {
	return 0, nil
}

func (b *mockBackend) SetInstanceQuota(i Instance, quota uint64) error {
	return nil
}

func (b *mockBackend) MountInstance(i Instance) (bool, error) {
	return true, nil
}

func (b *mockBackend) UnmountInstance(i Instance) (bool, error) {
	return true, nil
}

func (b *mockBackend) GetInstanceDisk(i Instance) (string, string, error) {
	return "", "", nil
}

func (b *mockBackend) CreateInstanceSnapshot(i Instance, name string, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) RenameInstanceSnapshot(i Instance, newName string, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) DeleteInstanceSnapshot(i Instance, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) RestoreInstanceSnapshot(i Instance, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) MountInstanceSnapshot(i Instance) (bool, error) {
	return true, nil
}

func (b *mockBackend) UnmountInstanceSnapshot(i Instance) (bool, error) {
	return true, nil
}

func (b *mockBackend) CreateImage(fingerprint string, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) DeleteImage(fingerprint string, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) CreateCustomVolume(volName, desc string, config map[string]string, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) CreateCustomVolumeFromCopy(volName, desc string, config map[string]string, srcPoolName, srcVolName string, srcVolOnly bool, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) RenameCustomVolume(volName string, newName string, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) UpdateCustomVolume(volName, newDesc string, newConfig map[string]string, op *operations.Operation) error {
	return ErrNotImplemented
}

func (b *mockBackend) DeleteCustomVolume(volName string, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) MigrateCustomVolume(conn io.ReadWriteCloser, args migration.VolumeSourceArgs, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) CreateCustomVolumeFromMigration(conn io.ReadWriteCloser, args migration.VolumeTargetArgs, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) GetCustomVolumeUsage(volName string) (int64, error) {
	return 0, nil
}

func (b *mockBackend) MountCustomVolume(volName string, op *operations.Operation) (bool, error) {
	return true, nil
}

func (b *mockBackend) UnmountCustomVolume(volName string, op *operations.Operation) (bool, error) {
	return true, nil
}

func (b *mockBackend) CreateCustomVolumeSnapshot(volName string, newSnapshotName string, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) RenameCustomVolumeSnapshot(volName string, newName string, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) DeleteCustomVolumeSnapshot(volName string, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) RestoreCustomVolume(volName string, snapshotName string, op *operations.Operation) error {
	return nil
}
<commit_msg>lxd/storage/backend/mock: Updates with EnsureImage<commit_after>package storage

import (
	"io"

	"github.com/lxc/lxd/lxd/migration"
	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/lxd/storage/drivers"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/logger"
)

type mockBackend struct {
	name   string
	state  *state.State
	logger logger.Logger
}

func (b *mockBackend) DaemonState() *state.State {
	return b.state
}

func (b *mockBackend) ID() int64 {
	return -1
}

func (b *mockBackend) Name() string {
	return b.name
}

func (b *mockBackend) Driver() drivers.Driver {
	return nil
}

func (b *mockBackend) MigrationTypes(contentType drivers.ContentType) []migration.Type {
	return []migration.Type{
		{
			FSType:   migration.MigrationFSType_RSYNC,
			Features: []string{"xattrs", "delete", "compress", "bidirectional"},
		},
	}
}

func (b *mockBackend) GetResources() (*api.ResourcesStoragePool, error) {
	return nil, nil
}

func (b *mockBackend) Delete(op *operations.Operation) error {
	return nil
}

func (b *mockBackend) Mount() (bool, error) {
	return true, nil
}

func (b *mockBackend) Unmount() (bool, error) {
	return true, nil
}

func (b *mockBackend) CreateInstance(i Instance, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) CreateInstanceFromBackup(i Instance, sourcePath string, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) CreateInstanceFromCopy(i Instance, src Instance, snapshots bool, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) CreateInstanceFromImage(i Instance, fingerprint string, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) CreateInstanceFromMigration(i Instance, conn io.ReadWriteCloser, args migration.SinkArgs, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) RenameInstance(i Instance, newName string, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) DeleteInstance(i Instance, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) MigrateInstance(i Instance, snapshots bool, args migration.SourceArgs) (migration.StorageSourceDriver, error) {
	return nil, nil
}

func (b *mockBackend) RefreshInstance(i Instance, src Instance, snapshots bool, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) BackupInstance(i Instance, targetPath string, optimized bool, snapshots bool, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) GetInstanceUsage(i Instance) (int64, error) {
	return 0, nil
}

func (b *mockBackend) SetInstanceQuota(i Instance, quota uint64) error {
	return nil
}

func (b *mockBackend) MountInstance(i Instance) (bool, error) {
	return true, nil
}

func (b *mockBackend) UnmountInstance(i Instance) (bool, error) {
	return true, nil
}

func (b *mockBackend) GetInstanceDisk(i Instance) (string, string, error) {
	return "", "", nil
}

func (b *mockBackend) CreateInstanceSnapshot(i Instance, name string, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) RenameInstanceSnapshot(i Instance, newName string, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) DeleteInstanceSnapshot(i Instance, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) RestoreInstanceSnapshot(i Instance, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) MountInstanceSnapshot(i Instance) (bool, error) {
	return true, nil
}

func (b *mockBackend) UnmountInstanceSnapshot(i Instance) (bool, error) {
	return true, nil
}

func (b *mockBackend) EnsureImage(fingerprint string, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) DeleteImage(fingerprint string, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) CreateCustomVolume(volName, desc string, config map[string]string, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) CreateCustomVolumeFromCopy(volName, desc string, config map[string]string, srcPoolName, srcVolName string, srcVolOnly bool, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) RenameCustomVolume(volName string, newName string, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) UpdateCustomVolume(volName, newDesc string, newConfig map[string]string, op *operations.Operation) error {
	return ErrNotImplemented
}

func (b *mockBackend) DeleteCustomVolume(volName string, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) MigrateCustomVolume(conn io.ReadWriteCloser, args migration.VolumeSourceArgs, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) CreateCustomVolumeFromMigration(conn io.ReadWriteCloser, args migration.VolumeTargetArgs, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) GetCustomVolumeUsage(volName string) (int64, error) {
	return 0, nil
}

func (b *mockBackend) MountCustomVolume(volName string, op *operations.Operation) (bool, error) {
	return true, nil
}

func (b *mockBackend) UnmountCustomVolume(volName string, op *operations.Operation) (bool, error) {
	return true, nil
}

func (b *mockBackend) CreateCustomVolumeSnapshot(volName string, newSnapshotName string, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) RenameCustomVolumeSnapshot(volName string, newName string, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) DeleteCustomVolumeSnapshot(volName string, op *operations.Operation) error {
	return nil
}

func (b *mockBackend) RestoreCustomVolume(volName string, snapshotName string, op *operations.Operation) error {
	return nil
}
<|endoftext|>


<commit_before>package shadowsocks_test

import (
// . "v2ray.com/core/proxy/shadowsocks"
)
<commit_msg>remove unused file<commit_after><|endoftext|>


<commit_before>// Copyright 2016 CNI authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"path/filepath"
	"strings"

	"github.com/containernetworking/cni/pkg/skel"
	"github.com/containernetworking/cni/pkg/testutils"
	"github.com/containernetworking/cni/pkg/types"
	"github.com/containernetworking/cni/pkg/types/020"
	"github.com/containernetworking/cni/pkg/types/current"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("host-local Operations", func() {
	It("allocates and releases an address with ADD/DEL", func() {
		const ifname string = "eth0"
		const nspath string = "/some/where"

		tmpDir, err := ioutil.TempDir("", "host_local_artifacts")
		Expect(err).NotTo(HaveOccurred())
		defer os.RemoveAll(tmpDir)

		err = ioutil.WriteFile(filepath.Join(tmpDir, "resolv.conf"), []byte("nameserver 192.0.2.3"), 0644)
		Expect(err).NotTo(HaveOccurred())

		conf := fmt.Sprintf(`{
    "cniVersion": "0.3.0",
    "name": "mynet",
    "type": "ipvlan",
    "master": "foo0",
    "ipam": {
        "type": "host-local",
        "subnet": "10.1.2.0/24",
        "dataDir": "%s",
		"resolvConf": "%s/resolv.conf"
    }
}`, tmpDir, tmpDir)

		args := &skel.CmdArgs{
			ContainerID: "dummy",
			Netns:       nspath,
			IfName:      ifname,
			StdinData:   []byte(conf),
		}

		// Allocate the IP
		r, raw, err := testutils.CmdAddWithResult(nspath, ifname, []byte(conf), func() error {
			return cmdAdd(args)
		})
		Expect(err).NotTo(HaveOccurred())
		Expect(strings.Index(string(raw), "\"version\":")).Should(BeNumerically(">", 0))

		result, err := current.GetResult(r)
		Expect(err).NotTo(HaveOccurred())

		expectedAddress, err := types.ParseCIDR("10.1.2.2/24")
		Expect(err).NotTo(HaveOccurred())
		Expect(len(result.IPs)).To(Equal(1))
		expectedAddress.IP = expectedAddress.IP.To16()
		Expect(result.IPs[0].Address).To(Equal(*expectedAddress))
		Expect(result.IPs[0].Gateway).To(Equal(net.ParseIP("10.1.2.1")))

		ipFilePath := filepath.Join(tmpDir, "mynet", "10.1.2.2")
		contents, err := ioutil.ReadFile(ipFilePath)
		Expect(err).NotTo(HaveOccurred())
		Expect(string(contents)).To(Equal("dummy"))

		lastFilePath := filepath.Join(tmpDir, "mynet", "last_reserved_ip")
		contents, err = ioutil.ReadFile(lastFilePath)
		Expect(err).NotTo(HaveOccurred())
		Expect(string(contents)).To(Equal("10.1.2.2"))

		// Release the IP
		err = testutils.CmdDelWithResult(nspath, ifname, func() error {
			return cmdDel(args)
		})
		Expect(err).NotTo(HaveOccurred())

		_, err = os.Stat(ipFilePath)
		Expect(err).To(HaveOccurred())
	})

	It("allocates and releases an address with ADD/DEL and 0.1.0 config", func() {
		const ifname string = "eth0"
		const nspath string = "/some/where"

		tmpDir, err := ioutil.TempDir("", "host_local_artifacts")
		Expect(err).NotTo(HaveOccurred())
		defer os.RemoveAll(tmpDir)

		conf := fmt.Sprintf(`{
    "cniVersion": "0.1.0",
    "name": "mynet",
    "type": "ipvlan",
    "master": "foo0",
    "ipam": {
        "type": "host-local",
        "subnet": "10.1.2.0/24",
        "dataDir": "%s"
    }
}`, tmpDir)

		args := &skel.CmdArgs{
			ContainerID: "dummy",
			Netns:       nspath,
			IfName:      ifname,
			StdinData:   []byte(conf),
		}

		// Allocate the IP
		r, raw, err := testutils.CmdAddWithResult(nspath, ifname, []byte(conf), func() error {
			return cmdAdd(args)
		})
		Expect(err).NotTo(HaveOccurred())
		Expect(strings.Index(string(raw), "\"ip4\":")).Should(BeNumerically(">", 0))

		result, err := types020.GetResult(r)
		Expect(err).NotTo(HaveOccurred())

		expectedAddress, err := types.ParseCIDR("10.1.2.2/24")
		Expect(err).NotTo(HaveOccurred())
		expectedAddress.IP = expectedAddress.IP.To16()
		Expect(result.IP4.IP).To(Equal(*expectedAddress))
		Expect(result.IP4.Gateway).To(Equal(net.ParseIP("10.1.2.1")))

		ipFilePath := filepath.Join(tmpDir, "mynet", "10.1.2.2")
		contents, err := ioutil.ReadFile(ipFilePath)
		Expect(err).NotTo(HaveOccurred())
		Expect(string(contents)).To(Equal("dummy"))

		lastFilePath := filepath.Join(tmpDir, "mynet", "last_reserved_ip")
		contents, err = ioutil.ReadFile(lastFilePath)
		Expect(err).NotTo(HaveOccurred())
		Expect(string(contents)).To(Equal("10.1.2.2"))

		Expect(result.DNS).To(Equal(types.DNS{Nameservers: []string{"192.0.2.3"}}))

		// Release the IP
		err = testutils.CmdDelWithResult(nspath, ifname, func() error {
			return cmdDel(args)
		})
		Expect(err).NotTo(HaveOccurred())

		_, err = os.Stat(ipFilePath)
		Expect(err).To(HaveOccurred())
	})

	It("ignores whitespace in disk files", func() {
		const ifname string = "eth0"
		const nspath string = "/some/where"

		tmpDir, err := ioutil.TempDir("", "host_local_artifacts")
		Expect(err).NotTo(HaveOccurred())
		defer os.RemoveAll(tmpDir)

		conf := fmt.Sprintf(`{
    "cniVersion": "0.2.0",
    "name": "mynet",
    "type": "ipvlan",
    "master": "foo0",
    "ipam": {
        "type": "host-local",
        "subnet": "10.1.2.0/24",
        "dataDir": "%s"
    }
}`, tmpDir)

		args := &skel.CmdArgs{
			ContainerID: "   dummy\n ",
			Netns:       nspath,
			IfName:      ifname,
			StdinData:   []byte(conf),
		}

		// Allocate the IP
		r, _, err := testutils.CmdAddWithResult(nspath, ifname, []byte(conf), func() error {
			return cmdAdd(args)
		})
		Expect(err).NotTo(HaveOccurred())

		result, err := current.GetResult(r)
		Expect(err).NotTo(HaveOccurred())

		ipFilePath := filepath.Join(tmpDir, "mynet", result.IPs[0].Address.IP.String())
		contents, err := ioutil.ReadFile(ipFilePath)
		Expect(err).NotTo(HaveOccurred())
		Expect(string(contents)).To(Equal("dummy"))

		// Release the IP
		err = testutils.CmdDelWithResult(nspath, ifname, func() error {
			return cmdDel(args)
		})
		Expect(err).NotTo(HaveOccurred())

		_, err = os.Stat(ipFilePath)
		Expect(err).To(HaveOccurred())
	})

	It("does not output an error message upon initial subnet creation", func() {
		const ifname string = "eth0"
		const nspath string = "/some/where"

		tmpDir, err := ioutil.TempDir("", "host_local_artifacts")
		Expect(err).NotTo(HaveOccurred())
		defer os.RemoveAll(tmpDir)

		conf := fmt.Sprintf(`{
    "cniVersion": "0.2.0",
    "name": "mynet",
    "type": "ipvlan",
    "master": "foo0",
    "ipam": {
        "type": "host-local",
        "subnet": "10.1.2.0/24",
        "dataDir": "%s"
    }
}`, tmpDir)

		args := &skel.CmdArgs{
			ContainerID: "testing",
			Netns:       nspath,
			IfName:      ifname,
			StdinData:   []byte(conf),
		}

		// Allocate the IP
		_, out, err := testutils.CmdAddWithResult(nspath, ifname, []byte(conf), func() error {
			return cmdAdd(args)
		})
		Expect(err).NotTo(HaveOccurred())
		Expect(strings.Index(string(out), "Error retriving last reserved ip")).To(Equal(-1))
	})
})
<commit_msg>fix unrelated failing tests<commit_after>// Copyright 2016 CNI authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"path/filepath"
	"strings"

	"github.com/containernetworking/cni/pkg/skel"
	"github.com/containernetworking/cni/pkg/testutils"
	"github.com/containernetworking/cni/pkg/types"
	"github.com/containernetworking/cni/pkg/types/020"
	"github.com/containernetworking/cni/pkg/types/current"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("host-local Operations", func() {
	It("allocates and releases an address with ADD/DEL", func() {
		const ifname string = "eth0"
		const nspath string = "/some/where"

		tmpDir, err := ioutil.TempDir("", "host_local_artifacts")
		Expect(err).NotTo(HaveOccurred())
		defer os.RemoveAll(tmpDir)

		err = ioutil.WriteFile(filepath.Join(tmpDir, "resolv.conf"), []byte("nameserver 192.0.2.3"), 0644)
		Expect(err).NotTo(HaveOccurred())

		conf := fmt.Sprintf(`{
    "cniVersion": "0.3.0",
    "name": "mynet",
    "type": "ipvlan",
    "master": "foo0",
    "ipam": {
        "type": "host-local",
        "subnet": "10.1.2.0/24",
        "dataDir": "%s",
		"resolvConf": "%s/resolv.conf"
    }
}`, tmpDir, tmpDir)

		args := &skel.CmdArgs{
			ContainerID: "dummy",
			Netns:       nspath,
			IfName:      ifname,
			StdinData:   []byte(conf),
		}

		// Allocate the IP
		r, raw, err := testutils.CmdAddWithResult(nspath, ifname, []byte(conf), func() error {
			return cmdAdd(args)
		})
		Expect(err).NotTo(HaveOccurred())
		Expect(strings.Index(string(raw), "\"version\":")).Should(BeNumerically(">", 0))

		result, err := current.GetResult(r)
		Expect(err).NotTo(HaveOccurred())

		expectedAddress, err := types.ParseCIDR("10.1.2.2/24")
		Expect(err).NotTo(HaveOccurred())
		Expect(len(result.IPs)).To(Equal(1))
		expectedAddress.IP = expectedAddress.IP.To16()
		Expect(result.IPs[0].Address).To(Equal(*expectedAddress))
		Expect(result.IPs[0].Gateway).To(Equal(net.ParseIP("10.1.2.1")))

		ipFilePath := filepath.Join(tmpDir, "mynet", "10.1.2.2")
		contents, err := ioutil.ReadFile(ipFilePath)
		Expect(err).NotTo(HaveOccurred())
		Expect(string(contents)).To(Equal("dummy"))

		lastFilePath := filepath.Join(tmpDir, "mynet", "last_reserved_ip")
		contents, err = ioutil.ReadFile(lastFilePath)
		Expect(err).NotTo(HaveOccurred())
		Expect(string(contents)).To(Equal("10.1.2.2"))

		// Release the IP
		err = testutils.CmdDelWithResult(nspath, ifname, func() error {
			return cmdDel(args)
		})
		Expect(err).NotTo(HaveOccurred())

		_, err = os.Stat(ipFilePath)
		Expect(err).To(HaveOccurred())
	})

	It("allocates and releases an address with ADD/DEL and 0.1.0 config", func() {
		const ifname string = "eth0"
		const nspath string = "/some/where"

		tmpDir, err := ioutil.TempDir("", "host_local_artifacts")
		Expect(err).NotTo(HaveOccurred())
		defer os.RemoveAll(tmpDir)

		err = ioutil.WriteFile(filepath.Join(tmpDir, "resolv.conf"), []byte("nameserver 192.0.2.3"), 0644)
		Expect(err).NotTo(HaveOccurred())

		conf := fmt.Sprintf(`{
    "cniVersion": "0.1.0",
    "name": "mynet",
    "type": "ipvlan",
    "master": "foo0",
    "ipam": {
        "type": "host-local",
        "subnet": "10.1.2.0/24",
        "dataDir": "%s",
	"resolvConf": "%s/resolv.conf"
    }
}`, tmpDir, tmpDir)

		args := &skel.CmdArgs{
			ContainerID: "dummy",
			Netns:       nspath,
			IfName:      ifname,
			StdinData:   []byte(conf),
		}

		// Allocate the IP
		r, raw, err := testutils.CmdAddWithResult(nspath, ifname, []byte(conf), func() error {
			return cmdAdd(args)
		})
		Expect(err).NotTo(HaveOccurred())
		Expect(strings.Index(string(raw), "\"ip4\":")).Should(BeNumerically(">", 0))

		result, err := types020.GetResult(r)
		Expect(err).NotTo(HaveOccurred())

		expectedAddress, err := types.ParseCIDR("10.1.2.2/24")
		Expect(err).NotTo(HaveOccurred())
		expectedAddress.IP = expectedAddress.IP.To16()
		Expect(result.IP4.IP).To(Equal(*expectedAddress))
		Expect(result.IP4.Gateway).To(Equal(net.ParseIP("10.1.2.1")))

		ipFilePath := filepath.Join(tmpDir, "mynet", "10.1.2.2")
		contents, err := ioutil.ReadFile(ipFilePath)
		Expect(err).NotTo(HaveOccurred())
		Expect(string(contents)).To(Equal("dummy"))

		lastFilePath := filepath.Join(tmpDir, "mynet", "last_reserved_ip")
		contents, err = ioutil.ReadFile(lastFilePath)
		Expect(err).NotTo(HaveOccurred())
		Expect(string(contents)).To(Equal("10.1.2.2"))

		Expect(result.DNS).To(Equal(types.DNS{Nameservers: []string{"192.0.2.3"}}))

		// Release the IP
		err = testutils.CmdDelWithResult(nspath, ifname, func() error {
			return cmdDel(args)
		})
		Expect(err).NotTo(HaveOccurred())

		_, err = os.Stat(ipFilePath)
		Expect(err).To(HaveOccurred())
	})

	It("ignores whitespace in disk files", func() {
		const ifname string = "eth0"
		const nspath string = "/some/where"

		tmpDir, err := ioutil.TempDir("", "host_local_artifacts")
		Expect(err).NotTo(HaveOccurred())
		defer os.RemoveAll(tmpDir)

		conf := fmt.Sprintf(`{
    "cniVersion": "0.3.0",
    "name": "mynet",
    "type": "ipvlan",
    "master": "foo0",
    "ipam": {
        "type": "host-local",
        "subnet": "10.1.2.0/24",
        "dataDir": "%s"
    }
}`, tmpDir)

		args := &skel.CmdArgs{
			ContainerID: "   dummy\n ",
			Netns:       nspath,
			IfName:      ifname,
			StdinData:   []byte(conf),
		}

		// Allocate the IP
		r, _, err := testutils.CmdAddWithResult(nspath, ifname, []byte(conf), func() error {
			return cmdAdd(args)
		})
		Expect(err).NotTo(HaveOccurred())

		result, err := current.GetResult(r)
		Expect(err).NotTo(HaveOccurred())

		ipFilePath := filepath.Join(tmpDir, "mynet", result.IPs[0].Address.IP.String())
		contents, err := ioutil.ReadFile(ipFilePath)
		Expect(err).NotTo(HaveOccurred())
		Expect(string(contents)).To(Equal("dummy"))

		// Release the IP
		err = testutils.CmdDelWithResult(nspath, ifname, func() error {
			return cmdDel(args)
		})
		Expect(err).NotTo(HaveOccurred())

		_, err = os.Stat(ipFilePath)
		Expect(err).To(HaveOccurred())
	})

	It("does not output an error message upon initial subnet creation", func() {
		const ifname string = "eth0"
		const nspath string = "/some/where"

		tmpDir, err := ioutil.TempDir("", "host_local_artifacts")
		Expect(err).NotTo(HaveOccurred())
		defer os.RemoveAll(tmpDir)

		conf := fmt.Sprintf(`{
    "cniVersion": "0.2.0",
    "name": "mynet",
    "type": "ipvlan",
    "master": "foo0",
    "ipam": {
        "type": "host-local",
        "subnet": "10.1.2.0/24",
        "dataDir": "%s"
    }
}`, tmpDir)

		args := &skel.CmdArgs{
			ContainerID: "testing",
			Netns:       nspath,
			IfName:      ifname,
			StdinData:   []byte(conf),
		}

		// Allocate the IP
		_, out, err := testutils.CmdAddWithResult(nspath, ifname, []byte(conf), func() error {
			return cmdAdd(args)
		})
		Expect(err).NotTo(HaveOccurred())
		Expect(strings.Index(string(out), "Error retriving last reserved ip")).To(Equal(-1))
	})
})
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"net"
	"net/http"
	"os"
	"runtime"

	"github.com/goraft/raft"
)

var (
	port       = flag.Int("p", 7001, "raft port")
	hostname   = flag.String("h", "localhost", "raft hostname")
	verbose    = flag.Bool("v", false, "debug info")
	leader     = flag.String("l", "", "the address of the leader")
	peerNumber = flag.Int("peer-number", 3, "number of expected peers")
)

var peerCnt = 1

func main() {
	runtime.GOMAXPROCS(2)
	flag.Parse()
	if *verbose == true {
		raft.SetLogLevel(2)
	}

	t := raft.NewHTTPTransporter("/raft")

	name := fmt.Sprintf("raft-%v", *port)
	dir := fmt.Sprintf("./raft-%v", *port)
	connStr := fmt.Sprintf("http://%v:%v", *hostname, *port)

	os.RemoveAll(dir)
	os.Mkdir(dir, 0700)

	server, err := raft.NewServer(name, dir, t, nil, nil, connStr)
	if err != nil {
		panic(err)
	}

	err = server.Start()
	if err != nil {
		panic(err)
	}

	// Create listener for HTTP server and start it.
	listener, err := net.Listen("tcp", fmt.Sprintf(":%d", *port))
	if err != nil {
		panic(err)
	}
	defer listener.Close()

	joinCommand := &raft.DefaultJoinCommand{
		Name:             name,
		ConnectionString: connStr,
	}

	if *leader != "" {
		var b bytes.Buffer
		json.NewEncoder(&b).Encode(joinCommand)
		resp, err := http.Post(fmt.Sprintf("http://%s/join", *leader), "application/json", &b)
		if err != nil {
			panic(err)
		}

		resp.Body.Close()
		fmt.Println("start as a follower of ", *leader)
	} else {
		_, err = server.Do(joinCommand)
		if err != nil {
			panic(err)
		}

		server.AddEventListener("addPeer", createAddListenerFunc(server))
		fmt.Println("start as a leader")
	}
	server.AddEventListener("stateChange", createStateChangeListenerFunc(server))
	// Create wrapping HTTP server.
	mux := http.NewServeMux()
	t.Install(server, mux)
	mux.HandleFunc("/join", createJoinHandler(server))
	httpServer := &http.Server{Addr: connStr, Handler: mux}
	httpServer.Serve(listener)
}

func createJoinHandler(s raft.Server) func(w http.ResponseWriter, req *http.Request) {
	return func(w http.ResponseWriter, req *http.Request) {
		fmt.Println("recv join!")
		command := &raft.DefaultJoinCommand{}

		if err := json.NewDecoder(req.Body).Decode(&command); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		if _, err := s.Do(command); err != nil {
			fmt.Println("error...", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}
}

func createAddListenerFunc(s raft.Server) func(e raft.Event) {
	return func(e raft.Event) {
		fmt.Println("add peer")
		if len(s.Peers()) == *peerNumber-1 {
			go send(s, 1000)
		}
	}
}

func createStateChangeListenerFunc(s raft.Server) func(e raft.Event) {
	return func(e raft.Event) {
		if e.Value() == "leader" {
			go send(s, 1000)
		}
	}
}
<commit_msg>better bench<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"net"
	"net/http"
	"os"
	"runtime"
	"time"

	"github.com/goraft/raft"
)

var (
	port       = flag.Int("p", 7001, "raft port")
	hostname   = flag.String("h", "localhost", "raft hostname")
	verbose    = flag.Bool("v", false, "debug info")
	leader     = flag.String("l", "", "the address of the leader")
	peerNumber = flag.Int("peer-number", 3, "number of expected peers")
)

var peerCnt = 1

func main() {
	runtime.GOMAXPROCS(2)
	flag.Parse()
	if *verbose == true {
		raft.SetLogLevel(2)
	}

	t := raft.NewHTTPTransporter("/raft")

	name := fmt.Sprintf("raft-%v", *port)
	dir := fmt.Sprintf("./raft-%v", *port)
	connStr := fmt.Sprintf("http://%v:%v", *hostname, *port)

	os.RemoveAll(dir)
	os.Mkdir(dir, 0700)

	server, err := raft.NewServer(name, dir, t, nil, nil, connStr)
	if err != nil {
		panic(err)
	}

	server.SetHeartbeatTimeout(time.Millisecond * 2)

	err = server.Start()
	if err != nil {
		panic(err)
	}

	// Create listener for HTTP server and start it.
	listener, err := net.Listen("tcp", fmt.Sprintf(":%d", *port))
	if err != nil {
		panic(err)
	}
	defer listener.Close()

	joinCommand := &raft.DefaultJoinCommand{
		Name:             name,
		ConnectionString: connStr,
	}

	if *leader != "" {
		var b bytes.Buffer
		json.NewEncoder(&b).Encode(joinCommand)
		resp, err := http.Post(fmt.Sprintf("http://%s/join", *leader), "application/json", &b)
		if err != nil {
			panic(err)
		}

		resp.Body.Close()
		fmt.Println("start as a follower of ", *leader)
	} else {
		_, err = server.Do(joinCommand)
		if err != nil {
			panic(err)
		}

		server.AddEventListener("addPeer", createAddListenerFunc(server))
		fmt.Println("start as a leader")
	}
	server.AddEventListener("stateChange", createStateChangeListenerFunc(server))
	// Create wrapping HTTP server.
	mux := http.NewServeMux()
	t.Install(server, mux)
	mux.HandleFunc("/join", createJoinHandler(server))
	httpServer := &http.Server{Addr: connStr, Handler: mux}
	httpServer.Serve(listener)
}

func createJoinHandler(s raft.Server) func(w http.ResponseWriter, req *http.Request) {
	return func(w http.ResponseWriter, req *http.Request) {
		fmt.Println("recv join!")
		command := &raft.DefaultJoinCommand{}

		if err := json.NewDecoder(req.Body).Decode(&command); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		if _, err := s.Do(command); err != nil {
			fmt.Println("error...", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}
}

func createAddListenerFunc(s raft.Server) func(e raft.Event) {
	return func(e raft.Event) {
		fmt.Println("add peer")
		if len(s.Peers()) == *peerNumber-1 {
			go send(s, 1000)
		}
	}
}

func createStateChangeListenerFunc(s raft.Server) func(e raft.Event) {
	return func(e raft.Event) {
		if e.Value() == "leader" {
			go send(s, 1000)
		}
	}
}
<|endoftext|>


<commit_before>package master

import (
	"fmt"
	. "github.com/KIT-MAMID/mamid/model"
	"github.com/jinzhu/gorm"
)

type ClusterAllocator struct {
}

type persistence uint

const (
	Persistent persistence = 0
	Volatile   persistence = 1
)

type memberCountTuple map[persistence]uint

func (c *ClusterAllocator) CompileMongodLayout(tx *gorm.DB) (err error) {

	defer func() {
		r := recover()
		if r == nil {
			return
		}
		switch r {
		case r == nil:
			return
		case r == gorm.ErrInvalidTransaction:
			err = r.(error)
		default:
			panic(r)
		}
	}()

	replicaSets := c.replicaSets(tx)
	for _, r := range replicaSets {
		c.removeUnneededMembers(tx, r)
	}

	c.addMembers(tx, replicaSets)

	return err
}

func (c *ClusterAllocator) replicaSets(tx *gorm.DB) (replicaSets []*ReplicaSet) {

	if err := tx.Where(ReplicaSet{}).Find(&replicaSets).Error; err != nil {
		panic(err)
	}

	for _, r := range replicaSets {

		if err := tx.Model(r).Related(&r.Mongods, "Mongods").Error; err != nil {
			panic(err)
		}

		for _, m := range r.Mongods {

			if err := tx.Model(m).Related(&m.ObservedState, "ObservedState").Error; err != nil {
				panic(err)
			}
			if err := tx.Model(m).Related(&m.DesiredState, "DesiredState").Error; err != nil {
				panic(err)
			}
			if err := tx.Model(m).Related(&m.ParentSlave, "ParentSlave").Error; err != nil {
				panic(err)
			}

		}

	}

	return replicaSets
}

func (c *ClusterAllocator) removeUnneededMembers(tx *gorm.DB, r *ReplicaSet) {
	for persistence, count := range c.effectiveMemberCount(tx, r) {
		c.removeUnneededMembersByPersistence(tx, r, persistence, count)
	}
}

func slavePersistence(s *Slave) persistence {
	switch s.PersistentStorage {
	case true:
		return Persistent
	default:
		return Volatile
	}
}

func (c *ClusterAllocator) removeUnneededMembersByPersistence(tx *gorm.DB, r *ReplicaSet, p persistence, initialCount uint) {

	var configuredMemberCount uint
	if p == Persistent {
		configuredMemberCount = r.PersistentMemberCount
	} else if p == Volatile {
		configuredMemberCount = r.VolatileMemberCount
	}

	// Destroy any Mongod running on disabled slaves (no specific priority)
	for initialCount > configuredMemberCount {
		for _, m := range r.Mongods {

			if m.ParentSlave.ConfiguredState == SlaveStateDisabled &&
				slavePersistence(m.ParentSlave) == p {

				c.destroyMongod(tx, m)
				initialCount--
			}
		}
	}

	// Remove superfluous Mongods on busiest slaves first
	removalPQ := c.pqMongods(r.Mongods, p)
	for initialCount > configuredMemberCount {
		// Destroy any Mongod (lower priority)
		m := removalPQ.PopMongodOnBusiestSlave()

		if m == nil {
			break
		}

		// destroy
		c.destroyMongod(tx, m)
		initialCount--

	}

}

func (c *ClusterAllocator) destroyMongod(tx *gorm.DB, m *Mongod) {

	// Set the desired execution state to disabled

	m.DesiredState.ExecutionState = MongodExecutionStateDestroyed
	if err := tx.Model(&m.DesiredState).Update("execution_state", MongodExecutionStateDestroyed); err != nil {
		panic(err)
	}

	// TODO MongodMatchStatus

}

func (c *ClusterAllocator) effectiveMemberCount(tx *gorm.DB, r *ReplicaSet) memberCountTuple {

	var res memberCountTuple

	for _, m := range r.Mongods {

		if m.ObservedState.ExecutionState == MongodExecutionStateRunning &&
			m.DesiredState.ExecutionState == MongodExecutionStateRunning {
			if m.ParentSlave.PersistentStorage {
				res[Persistent]++
			} else {
				res[Volatile]++
			}
		}
	}

	return res
}

func (c *ClusterAllocator) addMembers(tx *gorm.DB, replicaSets []*ReplicaSet) {

	for _, persistence := range []persistence{Volatile, Persistent} {

		// build prioritization datastructures
		// will only return items that match current persistence and actually need more members

		pqReplicaSets := c.pqReplicaSets(replicaSets, persistence)
		pqRiskGroups := c.pqRiskGroups(tx, persistence)

		for r := pqReplicaSets.Pop(); r != nil; {

			if s := pqRiskGroups.PopSlaveinNonconflictingRiskGroup(r); s != nil {

				// spawn new Mongod m on s and add it to r.Mongods
				// compute MongodState for m and set the DesiredState variable
				_ = c.spawnMongodOnSlave(tx, s, r)
				// TODO send DesiredReplicaSetConstraintStatus

				pqReplicaSets.PushIfDegraded(r)
				pqRiskGroups.PushSlaveIfFreePorts(s)

			} else {

				// TODO send DesiredReplicaSetConstraintStatus
				panic("not implemented")

			}
		}

	}
}

func (c *ClusterAllocator) spawnMongodOnSlave(tx *gorm.DB, s *Slave, r *ReplicaSet) *Mongod {

	// Get a port number, validates expected invariant that there's a free port as a side effect
	portNumber, err := c.slaveNextMongodPort(tx, s)
	if err != nil {
		panic(err)
	}

	m := &Mongod{
		Port:        portNumber,
		ReplSetName: r.Name,
		ParentSlave: s,
		ReplicaSet:  r,
		DesiredState: MongodState{ // TODO verify this nested initialization works with gorm
			IsShardingConfigServer: r.ConfigureAsShardingConfigServer,
			ExecutionState:         MongodExecutionStateRunning,
		},
	}

	if err := tx.Create(&m).Error; err != nil {
		panic(err)
	}

	return m

}

func (c *ClusterAllocator) slaveNextMongodPort(tx *gorm.DB, s *Slave) (portNumber PortNumber, err error) {

	var mongods []*Mongod

	if err = tx.Model(s).Related(&mongods).Error; err != nil {
		return PortNumber(0), err
	}

	maxMongodCount := slaveMaxNumberOfMongods(s)
	if len(mongods) >= int(maxMongodCount) {
		return PortNumber(0), fmt.Errorf("slave '%s' is full or is running more than maximum of '%d' Mongods", s.Hostname, maxMongodCount)
	}

	if len(mongods) <= 0 {
		return s.MongodPortRangeBegin, nil
	}

	portsUsed := make([]bool, maxMongodCount)
	for _, m := range mongods {
		portsUsed[m.Port-s.MongodPortRangeBegin] = true
	}
	for i := PortNumber(0); i < maxMongodCount; i++ {
		if !portsUsed[i] {
			return s.MongodPortRangeBegin + i, nil
		}
	}

	panic("algorithm invariant violated: this code should not be reached")
	return PortNumber(0), nil
}

func slaveMaxNumberOfMongods(s *Slave) PortNumber {
	res := s.MongodPortRangeEnd - s.MongodPortRangeBegin + PortNumber(1)
	if res <= 0 {
		panic("datastructure invariant violated: the range of Mongod ports for a slave must be sized greater than 0")
	}
	return res
}

func (c *ClusterAllocator) alreadyAddedMemberCount(tx *gorm.DB, r *ReplicaSet) memberCountTuple {

	var res memberCountTuple

	for _, m := range r.Mongods {

		if m.ParentSlave.ConfiguredState != SlaveStateDisabled &&
			m.DesiredState.ExecutionState != MongodExecutionStateNotRunning &&
			m.DesiredState.ExecutionState != MongodExecutionStateDestroyed {
			if m.ParentSlave.PersistentStorage {
				res[Persistent]++
			} else {
				res[Volatile]++
			}
		}

	}

	return res
}

func slaveUsage(s *Slave) (runningMongods, maxMongods uint) {
	return uint(len(s.Mongods)), uint(slaveMaxNumberOfMongods(s))
}

func slaveBusyRate(s *Slave) float64 {
	runningMongods, maxMongods := slaveUsage(s)
	return float64(runningMongods) / float64(maxMongods)
}
<commit_msg>Change interface required from pq_risk_groups<commit_after>package master

import (
	"fmt"
	. "github.com/KIT-MAMID/mamid/model"
	"github.com/jinzhu/gorm"
)

type ClusterAllocator struct {
}

type persistence uint

const (
	Persistent persistence = 0
	Volatile   persistence = 1
)

type memberCountTuple map[persistence]uint

func (c *ClusterAllocator) CompileMongodLayout(tx *gorm.DB) (err error) {

	defer func() {
		r := recover()
		if r == nil {
			return
		}
		switch r {
		case r == nil:
			return
		case r == gorm.ErrInvalidTransaction:
			err = r.(error)
		default:
			panic(r)
		}
	}()

	replicaSets := c.replicaSets(tx)
	for _, r := range replicaSets {
		c.removeUnneededMembers(tx, r)
	}

	c.addMembers(tx, replicaSets)

	return err
}

func (c *ClusterAllocator) replicaSets(tx *gorm.DB) (replicaSets []*ReplicaSet) {

	if err := tx.Where(ReplicaSet{}).Find(&replicaSets).Error; err != nil {
		panic(err)
	}

	for _, r := range replicaSets {

		if err := tx.Model(r).Related(&r.Mongods, "Mongods").Error; err != nil {
			panic(err)
		}

		for _, m := range r.Mongods {

			if err := tx.Model(m).Related(&m.ObservedState, "ObservedState").Error; err != nil {
				panic(err)
			}
			if err := tx.Model(m).Related(&m.DesiredState, "DesiredState").Error; err != nil {
				panic(err)
			}
			if err := tx.Model(m).Related(&m.ParentSlave, "ParentSlave").Error; err != nil {
				panic(err)
			}

		}

	}

	return replicaSets
}

func (c *ClusterAllocator) removeUnneededMembers(tx *gorm.DB, r *ReplicaSet) {
	for persistence, count := range c.effectiveMemberCount(tx, r) {
		c.removeUnneededMembersByPersistence(tx, r, persistence, count)
	}
}

func slavePersistence(s *Slave) persistence {
	switch s.PersistentStorage {
	case true:
		return Persistent
	default:
		return Volatile
	}
}

func (c *ClusterAllocator) removeUnneededMembersByPersistence(tx *gorm.DB, r *ReplicaSet, p persistence, initialCount uint) {

	var configuredMemberCount uint
	if p == Persistent {
		configuredMemberCount = r.PersistentMemberCount
	} else if p == Volatile {
		configuredMemberCount = r.VolatileMemberCount
	}

	// Destroy any Mongod running on disabled slaves (no specific priority)
	for initialCount > configuredMemberCount {
		for _, m := range r.Mongods {

			if m.ParentSlave.ConfiguredState == SlaveStateDisabled &&
				slavePersistence(m.ParentSlave) == p {

				c.destroyMongod(tx, m)
				initialCount--
			}
		}
	}

	// Remove superfluous Mongods on busiest slaves first
	removalPQ := c.pqMongods(r.Mongods, p)
	for initialCount > configuredMemberCount {
		// Destroy any Mongod (lower priority)
		m := removalPQ.PopMongodOnBusiestSlave()

		if m == nil {
			break
		}

		// destroy
		c.destroyMongod(tx, m)
		initialCount--

	}

}

func (c *ClusterAllocator) destroyMongod(tx *gorm.DB, m *Mongod) {

	// Set the desired execution state to disabled

	m.DesiredState.ExecutionState = MongodExecutionStateDestroyed
	if err := tx.Model(&m.DesiredState).Update("execution_state", MongodExecutionStateDestroyed); err != nil {
		panic(err)
	}

	// TODO MongodMatchStatus

}

func (c *ClusterAllocator) effectiveMemberCount(tx *gorm.DB, r *ReplicaSet) memberCountTuple {

	var res memberCountTuple

	for _, m := range r.Mongods {

		if m.ObservedState.ExecutionState == MongodExecutionStateRunning &&
			m.DesiredState.ExecutionState == MongodExecutionStateRunning {
			if m.ParentSlave.PersistentStorage {
				res[Persistent]++
			} else {
				res[Volatile]++
			}
		}
	}

	return res
}

func (c *ClusterAllocator) addMembers(tx *gorm.DB, replicaSets []*ReplicaSet) {

	for _, persistence := range []persistence{Volatile, Persistent} {

		// build prioritization datastructures
		// will only return items that match current persistence and actually need more members

		pqReplicaSets := c.pqReplicaSets(replicaSets, persistence)

		for r := pqReplicaSets.Pop(); r != nil; {

			pqRiskGroups := c.pqRiskGroups(tx, r, persistence)

			if s := pqRiskGroups.PopSlaveInNonconflictingRiskGroup(); s != nil {

				// spawn new Mongod m on s and add it to r.Mongods
				// compute MongodState for m and set the DesiredState variable
				_ = c.spawnMongodOnSlave(tx, s, r)
				// TODO send DesiredReplicaSetConstraintStatus

				pqReplicaSets.PushIfDegraded(r)

			} else {

				// TODO send DesiredReplicaSetConstraintStatus
				panic("not implemented")

			}
		}

	}
}

func (c *ClusterAllocator) spawnMongodOnSlave(tx *gorm.DB, s *Slave, r *ReplicaSet) *Mongod {

	// Get a port number, validates expected invariant that there's a free port as a side effect
	portNumber, err := c.slaveNextMongodPort(tx, s)
	if err != nil {
		panic(err)
	}

	m := &Mongod{
		Port:        portNumber,
		ReplSetName: r.Name,
		ParentSlave: s,
		ReplicaSet:  r,
		DesiredState: MongodState{ // TODO verify this nested initialization works with gorm
			IsShardingConfigServer: r.ConfigureAsShardingConfigServer,
			ExecutionState:         MongodExecutionStateRunning,
		},
	}

	if err := tx.Create(&m).Error; err != nil {
		panic(err)
	}

	return m

}

func (c *ClusterAllocator) slaveNextMongodPort(tx *gorm.DB, s *Slave) (portNumber PortNumber, err error) {

	var mongods []*Mongod

	if err = tx.Model(s).Related(&mongods).Error; err != nil {
		return PortNumber(0), err
	}

	maxMongodCount := slaveMaxNumberOfMongods(s)
	if len(mongods) >= int(maxMongodCount) {
		return PortNumber(0), fmt.Errorf("slave '%s' is full or is running more than maximum of '%d' Mongods", s.Hostname, maxMongodCount)
	}

	if len(mongods) <= 0 {
		return s.MongodPortRangeBegin, nil
	}

	portsUsed := make([]bool, maxMongodCount)
	for _, m := range mongods {
		portsUsed[m.Port-s.MongodPortRangeBegin] = true
	}
	for i := PortNumber(0); i < maxMongodCount; i++ {
		if !portsUsed[i] {
			return s.MongodPortRangeBegin + i, nil
		}
	}

	panic("algorithm invariant violated: this code should not be reached")
	return PortNumber(0), nil
}

func slaveMaxNumberOfMongods(s *Slave) PortNumber {
	res := s.MongodPortRangeEnd - s.MongodPortRangeBegin + PortNumber(1)
	if res <= 0 {
		panic("datastructure invariant violated: the range of Mongod ports for a slave must be sized greater than 0")
	}
	return res
}

func (c *ClusterAllocator) alreadyAddedMemberCount(tx *gorm.DB, r *ReplicaSet) memberCountTuple {

	var res memberCountTuple

	for _, m := range r.Mongods {

		if m.ParentSlave.ConfiguredState != SlaveStateDisabled &&
			m.DesiredState.ExecutionState != MongodExecutionStateNotRunning &&
			m.DesiredState.ExecutionState != MongodExecutionStateDestroyed {
			if m.ParentSlave.PersistentStorage {
				res[Persistent]++
			} else {
				res[Volatile]++
			}
		}

	}

	return res
}

func slaveUsage(s *Slave) (runningMongods, maxMongods uint) {
	return uint(len(s.Mongods)), uint(slaveMaxNumberOfMongods(s))
}

func slaveBusyRate(s *Slave) float64 {
	runningMongods, maxMongods := slaveUsage(s)
	return float64(runningMongods) / float64(maxMongods)
}
<|endoftext|>


<commit_before>// Copyright 2015 Yahoo Inc.
// Licensed under the terms of the Apache version 2.0 license. See LICENSE file for terms.

package main

import (
	"bufio"
	"github.com/ardielle/ardielle-go/rdl"
	"log"
	"strings"
	"text/template"
)

type javaClientGenerator struct {
	registry rdl.TypeRegistry
	schema   *rdl.Schema
	name     string
	writer   *bufio.Writer
	err      error
	banner   string
	ns       string
	base     string
}

// GenerateJavaClient generates the client code to talk to the server
func GenerateJavaClient(banner string, schema *rdl.Schema, outdir string, ns string, base string, options []string) error {
	reg := rdl.NewTypeRegistry(schema)
	packageDir, err := javaGenerationDir(outdir, schema, ns)
	if err != nil {
		return err
	}

	cName := javaGenerationStringOptionSet(options, "clientclass")
	if cName == "" {
		cName = capitalize(string(schema.Name))
	}

	out, file, _, err := outputWriter(packageDir, cName, "Client.java")
	if err != nil {
		return err
	}
	gen := &javaClientGenerator{reg, schema, cName, out, nil, banner, ns, base}
	gen.processTemplate(javaClientTemplate)
	out.Flush()
	file.Close()
	if gen.err != nil {
		return gen.err
	}

	//ResourceException - the throawable wrapper for alternate return types
	out, file, _, err = outputWriter(packageDir, "ResourceException", ".java")
	if err != nil {
		return err
	}
	err = javaGenerateResourceException(banner, schema, out, ns)
	out.Flush()
	file.Close()
	if err != nil {
		return err
	}

	//ResourceError - the default data object for an error
	out, file, _, err = outputWriter(packageDir, "ResourceError", ".java")
	if err != nil {
		return err
	}
	err = javaGenerateResourceError(banner, schema, out, ns)
	out.Flush()
	file.Close()
	return err
}

func (gen *javaClientGenerator) processTemplate(templateSource string) error {
	commentFun := func(s string) string {
		return formatComment(s, 0, 80)
	}
	funcMap := template.FuncMap{
		"header": func() string { return javaGenerationHeader(gen.banner) },
		"package": func() string {
			s := javaGenerationPackage(gen.schema, gen.ns)
			if s == "" {
				return s
			}
			return "package " + s + ";\n"
		},
		"comment":    commentFun,
		"methodSig":  func(r *rdl.Resource) string { return gen.clientMethodSignature(r) },
		"methodBody": func(r *rdl.Resource) string { return gen.clientMethodBody(r) },
		"name":       func() string { return gen.name },
		"cName":      func() string { return capitalize(gen.name) },
		"lName":      func() string { return uncapitalize(gen.name) },
	}
	t := template.Must(template.New(gen.name).Funcs(funcMap).Parse(templateSource))
	return t.Execute(gen.writer, gen.schema)
}

func (gen *javaClientGenerator) resourcePath(r *rdl.Resource) string {
	path := r.Path
	i := strings.Index(path, "?")
	if i >= 0 {
		path = path[0:i]
	}
	return path
}

const javaClientTemplate = `{{header}}
{{package}}
import com.yahoo.rdl.*;
import javax.ws.rs.client.*;
import javax.ws.rs.*;
import javax.ws.rs.core.*;
import javax.net.ssl.HostnameVerifier;

public class {{cName}}Client {
    Client client;
    WebTarget base;
    String credsHeader;
    String credsToken;

    public {{cName}}Client(String url) {
        client = ClientBuilder.newClient();
        base = client.target(url);
    }

    public {{cName}}Client(String url, HostnameVerifier hostnameVerifier) {
        client = ClientBuilder.newBuilder()
            .hostnameVerifier(hostnameVerifier)
            .build();
        base = client.target(url);
    }

    public void close() {
        client.close();
    }

    public {{cName}}Client setProperty(String name, Object value) {
        client = client.property(name, value);
        base = client.target(base.getUri().toString());
        return this;
    }

    public {{cName}}Client addCredentials(String header, String token) {
        credsHeader = header;
        credsToken = token;
        return this;
    }
{{range .Resources}}
    {{methodSig .}} {
        {{methodBody .}}
    }
{{end}}
}
`

func (gen *javaClientGenerator) clientMethodSignature(r *rdl.Resource) string {
	reg := gen.registry
	returnType := javaType(reg, r.Type, false, "", "")
	methName, params := javaMethodName(reg, r)
	sparams := ""
	if len(params) > 0 {
		sparams = strings.Join(params, ", ")
	}
	if len(r.Outputs) > 0 {
		if sparams == "" {
			sparams = "java.util.Map<String, java.util.List<String>> headers"
		} else {
			sparams = sparams + ", java.util.Map<String, java.util.List<String>> headers"
		}
	}
	return "public " + returnType + " " + methName + "(" + sparams + ")"
}

func (gen *javaClientGenerator) clientMethodBody(r *rdl.Resource) string {
	reg := gen.registry
	returnType := javaType(reg, r.Type, false, "", "")
	path := r.Path
	s := "WebTarget target = base.path(\"" + path + "\")"
	entityName := ""
	q := ""
	h := ""
	for _, in := range r.Inputs {
		iname := javaName(in.Name)
		if in.PathParam {
			s += "\n            .resolveTemplate(\"" + iname + "\", " + iname + ")"
		} else if in.QueryParam != "" {
			q += "\n        if (" + iname + " != null) {"
			q += "\n            target = target.queryParam(\"" + in.QueryParam + "\", " + iname + ");"
			q += "\n        }"
		} else if in.Header != "" {
			h += "\n        if (" + iname + " != null) {"
			h += "\n            invocationBuilder = invocationBuilder.header(\"" + in.Header + "\", " + iname + ");"
			h += "\n        }"
		} else { //the entity
			entityName = iname
		}
	}
	s += ";"
	if q != "" {
		s += q
	}
	s += "\n        Invocation.Builder invocationBuilder = target.request(\"application/json\");"
	if r.Auth != nil {
		if r.Auth.Authenticate || (r.Auth.Action != "" && r.Auth.Resource != "") {
			s += "\n        if (credsHeader != null) {"
			s += "\n            invocationBuilder = invocationBuilder.header(credsHeader, credsToken);"
			s += "\n        }"
		} else {
			log.Println("*** Badly formed auth spec in resource input:", r)
		}
	}
	if h != "" {
		s += h
	}
	s += "\n"
	switch r.Method {
	case "PUT", "POST", "PATCH":
		if entityName == "" {
			s += "        Response response = invocationBuilder." + strings.ToLower(r.Method) + "(javax.ws.rs.client.Entity.entity(null, \"application/json\"));\n"
		} else {
			s += "        Response response = invocationBuilder." + strings.ToLower(r.Method) + "(javax.ws.rs.client.Entity.entity(" + entityName + ", \"application/json\"));\n"
		}
	default:
		s += "        Response response = invocationBuilder." + strings.ToLower(r.Method) + "();\n"
	}
	s += "        int code = response.getStatus();\n"
	s += "        switch (code) {\n"

	//loop for all expected results
	var expected []string
	expected = append(expected, rdl.StatusCode(r.Expected))
	couldBeNoContent := "NO_CONTENT" == r.Expected
	couldBeNotModified := "NOT_MODIFIED" == r.Expected
	noContent := couldBeNoContent && r.Alternatives == nil
	for _, e := range r.Alternatives {
		if "NO_CONTENT" == e {
			couldBeNoContent = true
		}
		if "NOT_MODIFIED" == e {
			couldBeNotModified = true
		}
		expected = append(expected, rdl.StatusCode(e))
	}
	for _, expCode := range expected {
		s += "        case " + expCode + ":\n"
	}
	if len(r.Outputs) > 0 {
		s += "            if (headers != null) {\n"
		for _, out := range r.Outputs {
			s += "                headers.put(\"" + string(out.Name) + "\", java.util.Arrays.asList((String) response.getHeaders().getFirst(\"" + out.Header + "\")));\n"
		}
		s += "            }\n"
	}
	if noContent {
		s += "            return null;\n"
	} else {
		if couldBeNoContent || couldBeNotModified {
			s += "            if (" + gen.responseCondition(couldBeNoContent, couldBeNotModified) + ") {\n"
			s += "                return null;\n"
			s += "            }\n"
		}
		s += "            return response.readEntity(" + returnType + ".class);\n"
	}
	s += "        default:\n"
	if r.Exceptions != nil {
		s += "            throw new ResourceException(code, response.readEntity(ResourceError.class));\n"
	} else {
		s += "            throw new ResourceException(code, response.readEntity(Object.class));\n"
	}
	s += "        }\n"
	return s
}

func (gen *javaClientGenerator) responseCondition(noContent, notModified bool) string {
	var s string
	if noContent && notModified {
		s += "code == " + rdl.StatusCode("NO_CONTENT") + " || code == " + rdl.StatusCode("NOT_MODIFIED")
	} else if noContent {
		s += "code == " + rdl.StatusCode("NO_CONTENT")
	} else {
		s += "code == " + rdl.StatusCode("NOT_MODIFIED")
	}
	return s
}
<commit_msg>Make java client compatible with setting cookie<commit_after>// Copyright 2015 Yahoo Inc.
// Licensed under the terms of the Apache version 2.0 license. See LICENSE file for terms.

package main

import (
	"bufio"
	"github.com/ardielle/ardielle-go/rdl"
	"log"
	"strings"
	"text/template"
)

type javaClientGenerator struct {
	registry rdl.TypeRegistry
	schema   *rdl.Schema
	name     string
	writer   *bufio.Writer
	err      error
	banner   string
	ns       string
	base     string
}

// GenerateJavaClient generates the client code to talk to the server
func GenerateJavaClient(banner string, schema *rdl.Schema, outdir string, ns string, base string, options []string) error {
	reg := rdl.NewTypeRegistry(schema)
	packageDir, err := javaGenerationDir(outdir, schema, ns)
	if err != nil {
		return err
	}

	cName := javaGenerationStringOptionSet(options, "clientclass")
	if cName == "" {
		cName = capitalize(string(schema.Name))
	}

	out, file, _, err := outputWriter(packageDir, cName, "Client.java")
	if err != nil {
		return err
	}
	gen := &javaClientGenerator{reg, schema, cName, out, nil, banner, ns, base}
	gen.processTemplate(javaClientTemplate)
	out.Flush()
	file.Close()
	if gen.err != nil {
		return gen.err
	}

	//ResourceException - the throawable wrapper for alternate return types
	out, file, _, err = outputWriter(packageDir, "ResourceException", ".java")
	if err != nil {
		return err
	}
	err = javaGenerateResourceException(banner, schema, out, ns)
	out.Flush()
	file.Close()
	if err != nil {
		return err
	}

	//ResourceError - the default data object for an error
	out, file, _, err = outputWriter(packageDir, "ResourceError", ".java")
	if err != nil {
		return err
	}
	err = javaGenerateResourceError(banner, schema, out, ns)
	out.Flush()
	file.Close()
	return err
}

func (gen *javaClientGenerator) processTemplate(templateSource string) error {
	commentFun := func(s string) string {
		return formatComment(s, 0, 80)
	}
	funcMap := template.FuncMap{
		"header": func() string { return javaGenerationHeader(gen.banner) },
		"package": func() string {
			s := javaGenerationPackage(gen.schema, gen.ns)
			if s == "" {
				return s
			}
			return "package " + s + ";\n"
		},
		"comment":    commentFun,
		"methodSig":  func(r *rdl.Resource) string { return gen.clientMethodSignature(r) },
		"methodBody": func(r *rdl.Resource) string { return gen.clientMethodBody(r) },
		"name":       func() string { return gen.name },
		"cName":      func() string { return capitalize(gen.name) },
		"lName":      func() string { return uncapitalize(gen.name) },
	}
	t := template.Must(template.New(gen.name).Funcs(funcMap).Parse(templateSource))
	return t.Execute(gen.writer, gen.schema)
}

func (gen *javaClientGenerator) resourcePath(r *rdl.Resource) string {
	path := r.Path
	i := strings.Index(path, "?")
	if i >= 0 {
		path = path[0:i]
	}
	return path
}

const javaClientTemplate = `{{header}}
{{package}}
import com.yahoo.rdl.*;
import javax.ws.rs.client.*;
import javax.ws.rs.*;
import javax.ws.rs.core.*;
import javax.net.ssl.HostnameVerifier;

public class {{cName}}Client {
    Client client;
    WebTarget base;
    String credsHeader;
    String credsToken;

    public {{cName}}Client(String url) {
        client = ClientBuilder.newClient();
        base = client.target(url);
    }

    public {{cName}}Client(String url, HostnameVerifier hostnameVerifier) {
        client = ClientBuilder.newBuilder()
            .hostnameVerifier(hostnameVerifier)
            .build();
        base = client.target(url);
    }

    public void close() {
        client.close();
    }

    public {{cName}}Client setProperty(String name, Object value) {
        client = client.property(name, value);
        base = client.target(base.getUri().toString());
        return this;
    }

    public {{cName}}Client addCredentials(String header, String token) {
        credsHeader = header;
        credsToken = token;
        return this;
    }
{{range .Resources}}
    {{methodSig .}} {
        {{methodBody .}}
    }
{{end}}
}
`

func (gen *javaClientGenerator) clientMethodSignature(r *rdl.Resource) string {
	reg := gen.registry
	returnType := javaType(reg, r.Type, false, "", "")
	methName, params := javaMethodName(reg, r)
	sparams := ""
	if len(params) > 0 {
		sparams = strings.Join(params, ", ")
	}
	if len(r.Outputs) > 0 {
		if sparams == "" {
			sparams = "java.util.Map<String, java.util.List<String>> headers"
		} else {
			sparams = sparams + ", java.util.Map<String, java.util.List<String>> headers"
		}
	}
	return "public " + returnType + " " + methName + "(" + sparams + ")"
}

func (gen *javaClientGenerator) clientMethodBody(r *rdl.Resource) string {
	reg := gen.registry
	returnType := javaType(reg, r.Type, false, "", "")
	path := r.Path
	s := "WebTarget target = base.path(\"" + path + "\")"
	entityName := ""
	q := ""
	h := ""
	for _, in := range r.Inputs {
		iname := javaName(in.Name)
		if in.PathParam {
			s += "\n            .resolveTemplate(\"" + iname + "\", " + iname + ")"
		} else if in.QueryParam != "" {
			q += "\n        if (" + iname + " != null) {"
			q += "\n            target = target.queryParam(\"" + in.QueryParam + "\", " + iname + ");"
			q += "\n        }"
		} else if in.Header != "" {
			h += "\n        if (" + iname + " != null) {"
			h += "\n            invocationBuilder = invocationBuilder.header(\"" + in.Header + "\", " + iname + ");"
			h += "\n        }"
		} else { //the entity
			entityName = iname
		}
	}
	s += ";"
	if q != "" {
		s += q
	}
	s += "\n        Invocation.Builder invocationBuilder = target.request(\"application/json\");"
	if r.Auth != nil {
		if r.Auth.Authenticate || (r.Auth.Action != "" && r.Auth.Resource != "") {
			s += "\n        if (credsHeader != null) {"
			s += "\n            invocationBuilder = credsHeader.startsWith(\"Cookie.\") ? invocationBuilder.cookie(credsHeader.substring(7),"
			s += "\n                credsToken) : invocationBuilder.header(credsHeader, credsToken);"
			s += "\n        }"
		} else {
			log.Println("*** Badly formed auth spec in resource input:", r)
		}
	}
	if h != "" {
		s += h
	}
	s += "\n"
	switch r.Method {
	case "PUT", "POST", "PATCH":
		if entityName == "" {
			s += "        Response response = invocationBuilder." + strings.ToLower(r.Method) + "(javax.ws.rs.client.Entity.entity(null, \"application/json\"));\n"
		} else {
			s += "        Response response = invocationBuilder." + strings.ToLower(r.Method) + "(javax.ws.rs.client.Entity.entity(" + entityName + ", \"application/json\"));\n"
		}
	default:
		s += "        Response response = invocationBuilder." + strings.ToLower(r.Method) + "();\n"
	}
	s += "        int code = response.getStatus();\n"
	s += "        switch (code) {\n"

	//loop for all expected results
	var expected []string
	expected = append(expected, rdl.StatusCode(r.Expected))
	couldBeNoContent := "NO_CONTENT" == r.Expected
	couldBeNotModified := "NOT_MODIFIED" == r.Expected
	noContent := couldBeNoContent && r.Alternatives == nil
	for _, e := range r.Alternatives {
		if "NO_CONTENT" == e {
			couldBeNoContent = true
		}
		if "NOT_MODIFIED" == e {
			couldBeNotModified = true
		}
		expected = append(expected, rdl.StatusCode(e))
	}
	for _, expCode := range expected {
		s += "        case " + expCode + ":\n"
	}
	if len(r.Outputs) > 0 {
		s += "            if (headers != null) {\n"
		for _, out := range r.Outputs {
			s += "                headers.put(\"" + string(out.Name) + "\", java.util.Arrays.asList((String) response.getHeaders().getFirst(\"" + out.Header + "\")));\n"
		}
		s += "            }\n"
	}
	if noContent {
		s += "            return null;\n"
	} else {
		if couldBeNoContent || couldBeNotModified {
			s += "            if (" + gen.responseCondition(couldBeNoContent, couldBeNotModified) + ") {\n"
			s += "                return null;\n"
			s += "            }\n"
		}
		s += "            return response.readEntity(" + returnType + ".class);\n"
	}
	s += "        default:\n"
	if r.Exceptions != nil {
		s += "            throw new ResourceException(code, response.readEntity(ResourceError.class));\n"
	} else {
		s += "            throw new ResourceException(code, response.readEntity(Object.class));\n"
	}
	s += "        }\n"
	return s
}

func (gen *javaClientGenerator) responseCondition(noContent, notModified bool) string {
	var s string
	if noContent && notModified {
		s += "code == " + rdl.StatusCode("NO_CONTENT") + " || code == " + rdl.StatusCode("NOT_MODIFIED")
	} else if noContent {
		s += "code == " + rdl.StatusCode("NO_CONTENT")
	} else {
		s += "code == " + rdl.StatusCode("NOT_MODIFIED")
	}
	return s
}
<|endoftext|>


<commit_before>package main

import (
	"strings"

	"github.com/codegangsta/cli"
	sdk "github.com/dysolution/espsdk"
)

var batchTypes = string(strings.Join(sdk.Batch{}.ValidTypes(), " OR "))

// A Batch wraps the verbs provided by the ESP API for Submission Batches.
type Batch struct{ context *cli.Context }

// Index requests a list of all Submission Batches belonging to the user.
func (b Batch) Index() sdk.BatchListContainer { return sdk.Batch{}.Index(&client) }

// Get requests the metadata for a specific Submission Batch.
func (b Batch) Get() sdk.Createable { return sdk.Get(batchPath(b.context), &client) }

// Create adds a new Submission Batch.
func (b Batch) Create() sdk.Createable { return sdk.Create(Batches, b.build(), &client) }

// Update changes fields for an existing Submission Batch.
func (b Batch) Update() sdk.Createable { return batch(b.id()).Update(&client, b.buildUpdate()) }

// Delete destroys a specific Submission Batch.
func (b Batch) Delete() { batch(b.id()).Delete(&client) }

// Unmarshal attempts to deserialize the provided JSON payload into a SubmissionBatch object.
func (b Batch) Unmarshal(payload []byte) sdk.DeserializedObject {
	return sdk.Unmarshal(payload)
}

func (b Batch) id() int { return getBatchID(b.context) }

func (b Batch) build() sdk.Batch {
	return sdk.Batch{
		BriefID:               b.context.String("brief-id"),
		EventID:               b.context.String("event-id"),
		Note:                  b.context.String("note"),
		AssignmentID:          b.context.String("assignment-id"),
		SaveExtractedMetadata: b.context.Bool("save-extracted-metadata"),
		SubmissionName:        b.context.String("submission-name"),
		SubmissionType:        b.context.String("submission-type"),
	}
}

func (b Batch) buildUpdate() sdk.BatchUpdate {
	return sdk.BatchUpdate{b.build()}
}
<commit_msg>use sdk.Delete()<commit_after>package main

import (
	"strings"

	"github.com/codegangsta/cli"
	sdk "github.com/dysolution/espsdk"
)

var batchTypes = string(strings.Join(sdk.Batch{}.ValidTypes(), " OR "))

// A Batch wraps the verbs provided by the ESP API for Submission Batches.
type Batch struct{ context *cli.Context }

// Index requests a list of all Submission Batches belonging to the user.
func (b Batch) Index() sdk.BatchListContainer { return sdk.Batch{}.Index(&client) }

// Get requests the metadata for a specific Submission Batch.
func (b Batch) Get() sdk.Createable { return sdk.Get(batchPath(b.context), &client) }

// Create adds a new Submission Batch.
func (b Batch) Create() sdk.Createable { return sdk.Create(Batches, b.build(), &client) }

// Update changes fields for an existing Submission Batch.
func (b Batch) Update() sdk.Createable { return batch(b.id()).Update(&client, b.buildUpdate()) }

// Delete destroys a specific Submission Batch.
func (b Batch) Delete() { sdk.Delete(b.path(), &client) }

// Unmarshal attempts to deserialize the provided JSON payload into a SubmissionBatch object.
func (b Batch) Unmarshal(payload []byte) sdk.DeserializedObject {
	return sdk.Unmarshal(payload)
}

func (b Batch) id() int      { return getBatchID(b.context) }
func (b Batch) path() string { return sdk.Batch{ID: b.id()}.Path() }

func (b Batch) build() sdk.Batch {
	return sdk.Batch{
		BriefID:               b.context.String("brief-id"),
		EventID:               b.context.String("event-id"),
		Note:                  b.context.String("note"),
		AssignmentID:          b.context.String("assignment-id"),
		SaveExtractedMetadata: b.context.Bool("save-extracted-metadata"),
		SubmissionName:        b.context.String("submission-name"),
		SubmissionType:        b.context.String("submission-type"),
	}
}

func (b Batch) buildUpdate() sdk.BatchUpdate {
	return sdk.BatchUpdate{b.build()}
}
<|endoftext|>


<commit_before>package async

import (
	"context"
	"fmt"
	"sync"
	"time"
)

var (
	one      sync.Once
	input    chan *cmd
	output   chan *batchCmd
	groupDic map[string]*group
	groupMux sync.RWMutex
)

type CacheProvider interface {
	Put(string, interface{}, interface{}) error
	Get(string, interface{}) (interface{}, error)
}

func init() {
	groupDic = make(map[string]*group)
	input = make(chan *cmd, 128)
	output = make(chan *batchCmd, 128)
}

type group struct {
	name      string
	batchSize int
	method    func(...interface{}) (map[interface{}]interface{}, error)
	cmdDic    map[interface{}][]*cmd
	keyDic    map[interface{}]bool
	cache     CacheProvider
}

type cmd struct {
	group    string
	key      interface{}
	count    int
	callback chan interface{}
	forced   bool
}

func (c *cmd) output(res interface{}) {
	c.callback <- res
	c.count--
	if c.count == 0 {
		close(c.callback)
	}
}

type batchCmd struct {
	group  string
	keys   []interface{}
	err    error
	result map[interface{}]interface{}
}

func RegisterGroup(
	name string,
	batchSize int,
	method func(...interface{}) (map[interface{}]interface{}, error),
	cache CacheProvider) error {

	groupMux.RLock()
	_, ok := groupDic[name]
	groupMux.RUnlock()
	if ok {
		return fmt.Errorf("duplicate group:%s", name)
	}

	groupMux.RLock()
	defer groupMux.RUlock()
	_, ok := groupDic[name]
	if ok {
		return nil
	}
	groupDic[name] = &group{
		name:      name,
		batchSize: batchSize,
		method:    method,
		cmdDic:    make(map[interface{}][]*cmd),
		keyDic:    make(map[interface{}]bool),
		cache:     cache,
	}
	one.Do(func() {
		go runloop()
	})
	return nil
}

func doing(ctx context.Context, b *batchCmd, method func(...interface{}) (map[interface{}]interface{}, error)) {
	res, err := Safety(func() (interface{}, error) {
		return method(b.keys...)
	})
	if err != nil {
		b.err = err
	} else {
		b.result = res.(map[interface{}]interface{})
	}
	output <- b
}

func runloop() {
	ctx := context.Background()
	timer := time.NewTimer(10 * time.Millisecond)
	for {
		select {
		case c := <-input:
			{
				g, ok := groupDic[c.group]
				if false == ok {
					c.output(fmt.Errorf("invalid found:%s", c.group))
					if c.count > 0 {
						close(c.callback)
					}
					continue
				}
				var keys []interface{}
				if c.count > 1 {
					keys = c.key.([]interface{})
				} else {
					keys = []interface{}{c.key}
				}

				for index, _ := range keys {
					key := keys[index]
					// cache provider
					if g.cache != nil && c.forced == false {
						res, err := g.cache.Get(g.name, key)
						if res != nil && err == nil {
							c.output(res)
							continue
						}
					}
					target, ok := g.cmdDic[key]
					if ok {
						g.cmdDic[key] = append(target, c)
					} else {
						g.cmdDic[key] = []*cmd{c}
						g.keyDic[key] = true
					}

					if len(g.keyDic) >= g.batchSize {
						b := &batchCmd{group: g.name}
						b.keys = make([]interface{}, len(g.keyDic))
						index := 0
						for k, _ := range g.keyDic {
							b.keys[index] = k
							index = index + 1
						}
						g.keyDic = make(map[interface{}]bool)
						go doing(ctx, b, g.method)
					}
				}
			}
		case b := <-output:
			{
				g, _ := groupDic[b.group]
				if b.err != nil {
					for _, key := range b.keys {
						target, ok := g.cmdDic[key]
						if ok {
							for index, _ := range target {
								c := target[index]
								c.output(b.err)
							}
						}
						delete(g.cmdDic, key)
					}
				} else {
					for _, key := range b.keys {
						res, ok := b.result[key]
						if false == ok {
							res = fmt.Errorf("invalid key:%s", key)
						} else {
							// cache provider
							if g.cache != nil && res != nil {
								g.cache.Put(g.name, key, res)
							}
						}
						target, ok := g.cmdDic[key]
						if ok {
							for index, _ := range target {
								c := target[index]
								c.output(res)
							}
						}
						delete(g.cmdDic, key)
					}
				}
			}
		case <-timer.C:
			{
				for _, g := range groupDic {
					if len(g.keyDic) > 0 {
						b := &batchCmd{group: g.name}
						b.keys = make([]interface{}, len(g.keyDic))
						index := 0
						for k, _ := range g.keyDic {
							b.keys[index] = k
							index = index + 1
						}
						g.keyDic = make(map[interface{}]bool)
						go doing(ctx, b, g.method)
					}
				}
				timer.Reset(10 * time.Millisecond)
			}
		}
	}
}

func Get(group string, key interface{}) (interface{}, error) {
	return exec(group, false, key)
}

func ForceGet(group string, key interface{}) (interface{}, error) {
	return exec(group, true, key)
}

func Exec(group string, req interface{}) (interface{}, error) {
	return exec(group, true, req)
}

func exec(group string, forced bool, key interface{}) (interface{}, error) {
	c := &cmd{
		group:    group,
		key:      key,
		count:    1,
		callback: make(chan interface{}, 1),
		forced:   forced,
	}
	input <- c
	res := <-c.callback
	switch res.(type) {
	case error:
		return nil, res.(error)
	default:
		return res, nil
	}
}

func MGet(group string, keys ...interface{}) []interface{} {
	return mexec(group, false, keys...)
}

func ForceMGet(group string, keys ...interface{}) []interface{} {
	return mexec(group, true, keys...)
}

func MExec(group string, reqs ...interface{}) (interface{}, error) {
	return mexec(group, true, reqs...)
}

func mexec(group string, forced bool, keys ...interface{}) []interface{} {
	c := &cmd{
		group:    group,
		key:      keys,
		count:    len(keys),
		callback: make(chan interface{}, len(keys)),
		forced:   forced,
	}
	input <- c
	results := make([]interface{}, 0)
	for result := range c.callback {
		results = append(results, result)
	}
	return results
}
<commit_msg>rollback<commit_after>package async

import (
	"context"
	"fmt"
	"sync"
	"time"
)

var (
	one      sync.Once
	input    chan *cmd
	output   chan *batchCmd
	groupDic map[string]*group
	groupMux sync.RWMutex
)

type CacheProvider interface {
	Put(string, interface{}, interface{}) error
	Get(string, interface{}) (interface{}, error)
}

func init() {
	groupDic = make(map[string]*group)
	input = make(chan *cmd, 128)
	output = make(chan *batchCmd, 128)
}

type group struct {
	name      string
	batchSize int
	method    func(...interface{}) (map[interface{}]interface{}, error)
	cmdDic    map[interface{}][]*cmd
	keyDic    map[interface{}]bool
	cache     CacheProvider
}

type cmd struct {
	group    string
	key      interface{}
	count    int
	callback chan interface{}
	forced   bool
}

func (c *cmd) output(res interface{}) {
	c.callback <- res
	c.count--
	if c.count == 0 {
		close(c.callback)
	}
}

type batchCmd struct {
	group  string
	keys   []interface{}
	err    error
	result map[interface{}]interface{}
}

func RegisterGroup(
	name string,
	batchSize int,
	method func(...interface{}) (map[interface{}]interface{}, error),
	cache CacheProvider) error {

	groupMux.RLock()
	_, ok := groupDic[name]
	groupMux.RUnlock()
	if ok {
		return fmt.Errorf("duplicate group:%s", name)
	}

	groupMux.RLock()
	defer groupMux.RUlock()
	_, ok := groupDic[name]
	if ok {
		return nil
	}
	groupDic[name] = &group{
		name:      name,
		batchSize: batchSize,
		method:    method,
		cmdDic:    make(map[interface{}][]*cmd),
		keyDic:    make(map[interface{}]bool),
		cache:     cache,
	}
	one.Do(func() {
		go runloop()
	})
	return nil
}

func doing(ctx context.Context, b *batchCmd, method func(...interface{}) (map[interface{}]interface{}, error)) {
	res, err := Safety(func() (interface{}, error) {
		return method(b.keys...)
	})
	if err != nil {
		b.err = err
	} else {
		b.result = res.(map[interface{}]interface{})
	}
	output <- b
}

func runloop() {
	ctx := context.Background()
	timer := time.NewTimer(10 * time.Millisecond)
	for {
		select {
		case c := <-input:
			{
				g, ok := groupDic[c.group]
				if false == ok {
					c.output(fmt.Errorf("invalid found:%s", c.group))
					if c.count > 0 {
						close(c.callback)
					}
					continue
				}
				var keys []interface{}
				if c.count > 1 {
					keys = c.key.([]interface{})
				} else {
					keys = []interface{}{c.key}
				}

				for index, _ := range keys {
					key := keys[index]
					// cache provider
					if g.cache != nil && c.forced == false {
						res, err := g.cache.Get(g.name, key)
						if res != nil && err == nil {
							c.output(res)
							continue
						}
					}
					target, ok := g.cmdDic[key]
					if ok {
						g.cmdDic[key] = append(target, c)
					} else {
						g.cmdDic[key] = []*cmd{c}
						g.keyDic[key] = true
					}

					if len(g.keyDic) >= g.batchSize {
						b := &batchCmd{group: g.name}
						b.keys = make([]interface{}, len(g.keyDic))
						index := 0
						for k, _ := range g.keyDic {
							b.keys[index] = k
							index = index + 1
						}
						g.keyDic = make(map[interface{}]bool)
						go doing(ctx, b, g.method)
					}
				}
			}
		case b := <-output:
			{
				g, _ := groupDic[b.group]
				if b.err != nil {
					for _, key := range b.keys {
						target, ok := g.cmdDic[key]
						if ok {
							for index, _ := range target {
								c := target[index]
								c.output(b.err)
							}
						}
						delete(g.cmdDic, key)
					}
				} else {
					for _, key := range b.keys {
						res, ok := b.result[key]
						if false == ok {
							res = fmt.Errorf("invalid key:%s", key)
						} else {
							// cache provider
							if g.cache != nil && res != nil {
								g.cache.Put(g.name, key, res)
							}
						}
						target, ok := g.cmdDic[key]
						if ok {
							for index, _ := range target {
								c := target[index]
								c.output(res)
							}
						}
						delete(g.cmdDic, key)
					}
				}
			}
		case <-timer.C:
			{
				for _, g := range groupDic {
					if len(g.keyDic) > 0 {
						b := &batchCmd{group: g.name}
						b.keys = make([]interface{}, len(g.keyDic))
						index := 0
						for k, _ := range g.keyDic {
							b.keys[index] = k
							index = index + 1
						}
						g.keyDic = make(map[interface{}]bool)
						go doing(ctx, b, g.method)
					}
				}
				timer.Reset(10 * time.Millisecond)
			}
		}
	}
}

func Get(group string, key interface{}) (interface{}, error) {
	return get(group, false, key)
}

func ForceGet(group string, key interface{}) (interface{}, error) {
	return get(group, true, key)
}

func get(group string, forced bool, key interface{}) (interface{}, error) {
	c := &cmd{
		group:    group,
		key:      key,
		count:    1,
		callback: make(chan interface{}, 1),
		forced:   forced,
	}
	input <- c
	res := <-c.callback
	switch res.(type) {
	case error:
		return nil, res.(error)
	default:
		return res, nil
	}
}

func MGet(group string, keys ...interface{}) []interface{} {
	return mget(group, false, keys...)
}

func ForceMGet(group string, keys ...interface{}) []interface{} {
	return mget(group, true, keys...)
}

func mget(group string, forced bool, keys ...interface{}) []interface{} {
	c := &cmd{
		group:    group,
		key:      keys,
		count:    len(keys),
		callback: make(chan interface{}, len(keys)),
		forced:   forced,
	}
	input <- c
	results := make([]interface{}, 0)
	for result := range c.callback {
		results = append(results, result)
	}
	return results
}
<|endoftext|>