<commit_before>// On-disk mutex protecting a resource
//
// A lock is represented on disk by a directory of a particular name,
// containing an information file.  Taking a lock is done by renaming a
// temporary directory into place.  We use temporary directories because for
// all filesystems we believe that exactly one attempt to claim the lock will
// succeed and the others will fail.
package fslock

import (
	"bytes"
	"crypto/rand"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"regexp"
	"time"

	"launchpad.net/juju-core/log"
)

const (
	nameRegexp      = "^[a-z]+[a-z0-9.-]*$"
	heldFilename    = "held"
	messageFilename = "message"
)

var (
	ErrLockNotHeld = errors.New("lock not held")
	ErrTimeout     = errors.New("lock timeout exceeded")

	validName = regexp.MustCompile(nameRegexp)

	lockWaitDelay = 1 * time.Second
)

type Lock struct {
	name   string
	parent string
	nonce  []byte
}

func generateNonce() ([]byte, error) {
	nonce := make([]byte, 20)
	if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
		return nil, err
	}
	return nonce, nil
}

// NewLock returns a new lock with the given name within the given lock
// directory, without acquiring it. The lock name must match the regular
// expression `^[a-z]+[a-z0-9.-]*`.
func NewLock(lockDir, name string) (*Lock, error) {
	if !validName.MatchString(name) {
		return nil, fmt.Errorf("Invalid lock name %q.  Names must match %q", name, nameRegexp)
	}
	nonce, err := generateNonce()
	if err != nil {
		return nil, err
	}
	lock := &Lock{
		name:   name,
		parent: lockDir,
		nonce:  nonce,
	}
	// Ensure the parent exists.
	if err := os.MkdirAll(lock.parent, 0755); err != nil {
		return nil, err
	}
	return lock, nil
}

func (lock *Lock) lockDir() string {
	return path.Join(lock.parent, lock.name)
}

func (lock *Lock) heldFile() string {
	return path.Join(lock.lockDir(), "held")
}

func (lock *Lock) messageFile() string {
	return path.Join(lock.lockDir(), "message")
}

// If message is set, it will write the message to the lock directory as the
// lock is taken.
func (lock *Lock) acquire(message string) (bool, error) {
	// If the lockDir exists, then the lock is held by someone else.
	_, err := os.Stat(lock.lockDir())
	if err == nil {
		return false, nil
	}
	if !os.IsNotExist(err) {
		return false, err
	}
	// Create a temporary directory (in the parent dir), and then move it to
	// the right name.  Using the same directory to make sure the directories
	// are on the same filesystem.  Use a directory name starting with "." as
	// it isn't a valid lock name.
	tempLockName := fmt.Sprintf(".%x", lock.nonce)
	tempDirName, err := ioutil.TempDir(lock.parent, tempLockName)
	if err != nil {
		return false, err // this shouldn't really fail...
	}
	// write nonce into the temp dir
	err = ioutil.WriteFile(path.Join(tempDirName, heldFilename), lock.nonce, 0755)
	if err != nil {
		return false, err
	}
	if message != "" {
		err = ioutil.WriteFile(path.Join(tempDirName, messageFilename), []byte(message), 0755)
		if err != nil {
			return false, err
		}
	}
	// Now move the temp directory to the lock directory.
	err = os.Rename(tempDirName, lock.lockDir())
	if err != nil {
		// Any error on rename means we failed.
		// Beaten to it, clean up temporary directory.
		os.RemoveAll(tempDirName)
		return false, nil
	}
	// We now have the lock.
	return true, nil
}

// Lock blocks until it is able to acquire the lock.  Since we are dealing
// with sharing and locking using the filesystem, it is good behaviour to
// provide a message that is saved with the lock.  This is output in debugging
// information, and can be queried by any other Lock dealing with the same
// lock name and lock directory.
func (lock *Lock) Lock(message string) error {
	var heldMessage = ""
	for {
		acquired, err := lock.acquire(message)
		if err != nil {
			return err
		}
		if acquired {
			return nil
		}
		currMessage := lock.Message()
		if currMessage != heldMessage {
			log.Infof("Attempt Lock failed %q, %s, currently held: %s", lock.name, message, currMessage)
			heldMessage = currMessage
		}
		time.Sleep(lockWaitDelay)
	}
	panic("unreachable")
}

// LockWithTimeout tries to acquire the lock. If it cannot acquire the lock
// within the given duration, it returns ErrTimeout.  See `Lock` for
// information about the message.
func (lock *Lock) LockWithTimeout(duration time.Duration, message string) error {
	deadline := time.Now().Add(duration)
	for {
		acquired, err := lock.acquire(message)
		if err != nil {
			return err
		}
		if acquired {
			return nil
		}
		if time.Now().After(deadline) {
			return ErrTimeout
		}
		time.Sleep(lockWaitDelay)
	}
	panic("unreachable")
}

// IsHeld returns whether the lock is currently held by the receiver.
func (lock *Lock) IsLockHeld() bool {
	heldNonce, err := ioutil.ReadFile(lock.heldFile())
	if err != nil {
		return false
	}
	return bytes.Equal(heldNonce, lock.nonce)
}

// Unlock releases a held lock.  If the lock is not held ErrLockNotHeld is
// returned.
func (lock *Lock) Unlock() error {
	if !lock.IsLockHeld() {
		return ErrLockNotHeld
	}
	// To ensure reasonable unlocking, we should rename to a temp name, and delete that.
	tempLockName := fmt.Sprintf(".%s.%x", lock.name, lock.nonce)
	tempDirName := path.Join(lock.parent, tempLockName)
	// Now move the lock directory to the temp directory to release the lock.
	if err := os.Rename(lock.lockDir(), tempDirName); err != nil {
		return err
	}
	// And now cleanup.
	return os.RemoveAll(tempDirName)
}

// IsLocked returns true if the lock is currently held by anyone.
func (lock *Lock) IsLocked() bool {
	_, err := os.Stat(lock.heldFile())
	return err == nil
}

// BreakLock forcably breaks the lock that is currently being held.
func (lock *Lock) BreakLock() error {
	return os.RemoveAll(lock.lockDir())
}

// Message returns the saved message, or the empty string if there is no
// saved message.
func (lock *Lock) Message() string {
	message, err := ioutil.ReadFile(lock.messageFile())
	if err != nil {
		return ""
	}
	return string(message)
}
<commit_msg>Take rog's idea of refactoring the internals.<commit_after>// On-disk mutex protecting a resource
//
// A lock is represented on disk by a directory of a particular name,
// containing an information file.  Taking a lock is done by renaming a
// temporary directory into place.  We use temporary directories because for
// all filesystems we believe that exactly one attempt to claim the lock will
// succeed and the others will fail.
package fslock

import (
	"bytes"
	"crypto/rand"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"regexp"
	"time"

	"launchpad.net/juju-core/log"
)

const (
	nameRegexp      = "^[a-z]+[a-z0-9.-]*$"
	heldFilename    = "held"
	messageFilename = "message"
)

var (
	ErrLockNotHeld = errors.New("lock not held")
	ErrTimeout     = errors.New("lock timeout exceeded")

	validName = regexp.MustCompile(nameRegexp)

	lockWaitDelay = 1 * time.Second
)

type Lock struct {
	name   string
	parent string
	nonce  []byte
}

func generateNonce() ([]byte, error) {
	nonce := make([]byte, 20)
	if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
		return nil, err
	}
	return nonce, nil
}

// NewLock returns a new lock with the given name within the given lock
// directory, without acquiring it. The lock name must match the regular
// expression `^[a-z]+[a-z0-9.-]*`.
func NewLock(lockDir, name string) (*Lock, error) {
	if !validName.MatchString(name) {
		return nil, fmt.Errorf("Invalid lock name %q.  Names must match %q", name, nameRegexp)
	}
	nonce, err := generateNonce()
	if err != nil {
		return nil, err
	}
	lock := &Lock{
		name:   name,
		parent: lockDir,
		nonce:  nonce,
	}
	// Ensure the parent exists.
	if err := os.MkdirAll(lock.parent, 0755); err != nil {
		return nil, err
	}
	return lock, nil
}

func (lock *Lock) lockDir() string {
	return path.Join(lock.parent, lock.name)
}

func (lock *Lock) heldFile() string {
	return path.Join(lock.lockDir(), "held")
}

func (lock *Lock) messageFile() string {
	return path.Join(lock.lockDir(), "message")
}

// If message is set, it will write the message to the lock directory as the
// lock is taken.
func (lock *Lock) acquire(message string) (bool, error) {
	// If the lockDir exists, then the lock is held by someone else.
	_, err := os.Stat(lock.lockDir())
	if err == nil {
		return false, nil
	}
	if !os.IsNotExist(err) {
		return false, err
	}
	// Create a temporary directory (in the parent dir), and then move it to
	// the right name.  Using the same directory to make sure the directories
	// are on the same filesystem.  Use a directory name starting with "." as
	// it isn't a valid lock name.
	tempLockName := fmt.Sprintf(".%x", lock.nonce)
	tempDirName, err := ioutil.TempDir(lock.parent, tempLockName)
	if err != nil {
		return false, err // this shouldn't really fail...
	}
	// write nonce into the temp dir
	err = ioutil.WriteFile(path.Join(tempDirName, heldFilename), lock.nonce, 0755)
	if err != nil {
		return false, err
	}
	if message != "" {
		err = ioutil.WriteFile(path.Join(tempDirName, messageFilename), []byte(message), 0755)
		if err != nil {
			return false, err
		}
	}
	// Now move the temp directory to the lock directory.
	err = os.Rename(tempDirName, lock.lockDir())
	if err != nil {
		// Any error on rename means we failed.
		// Beaten to it, clean up temporary directory.
		os.RemoveAll(tempDirName)
		return false, nil
	}
	// We now have the lock.
	return true, nil
}

// lockWithDeadline tries to acquire the lock. If the deadline is
// non-zero and it cannot acquire the lock before then,
// it returns ErrTimeout.
func (lock *Lock) lockWithDeadline(deadline time.Time, message string) error {
	var heldMessage = ""
	for {
		acquired, err := lock.acquire(message)
		if err != nil {
			return err
		}
		if acquired {
			return nil
		}
		if !deadline.IsZero() && time.Now().After(deadline) {
			return ErrTimeout
		}
		currMessage := lock.Message()
		if currMessage != heldMessage {
			log.Infof("Attempt Lock failed %q, %s, currently held: %s", lock.name, message, currMessage)
			heldMessage = currMessage
		}
		time.Sleep(lockWaitDelay)
	}
	panic("unreachable")
}

// Lock blocks until it is able to acquire the lock.  Since we are dealing
// with sharing and locking using the filesystem, it is good behaviour to
// provide a message that is saved with the lock.  This is output in debugging
// information, and can be queried by any other Lock dealing with the same
// lock name and lock directory.
func (lock *Lock) Lock(message string) error {
	return lock.lockWithDeadline(time.Time{}, message)
}

// LockWithTimeout tries to acquire the lock. If it cannot acquire the lock
// within the given duration, it returns ErrTimeout.  See `Lock` for
// information about the message.
func (lock *Lock) LockWithTimeout(duration time.Duration, message string) error {
	return lock.lockWithDeadline(time.Now().Add(duration), message)
}

// IsHeld returns whether the lock is currently held by the receiver.
func (lock *Lock) IsLockHeld() bool {
	heldNonce, err := ioutil.ReadFile(lock.heldFile())
	if err != nil {
		return false
	}
	return bytes.Equal(heldNonce, lock.nonce)
}

// Unlock releases a held lock.  If the lock is not held ErrLockNotHeld is
// returned.
func (lock *Lock) Unlock() error {
	if !lock.IsLockHeld() {
		return ErrLockNotHeld
	}
	// To ensure reasonable unlocking, we should rename to a temp name, and delete that.
	tempLockName := fmt.Sprintf(".%s.%x", lock.name, lock.nonce)
	tempDirName := path.Join(lock.parent, tempLockName)
	// Now move the lock directory to the temp directory to release the lock.
	if err := os.Rename(lock.lockDir(), tempDirName); err != nil {
		return err
	}
	// And now cleanup.
	return os.RemoveAll(tempDirName)
}

// IsLocked returns true if the lock is currently held by anyone.
func (lock *Lock) IsLocked() bool {
	_, err := os.Stat(lock.heldFile())
	return err == nil
}

// BreakLock forcably breaks the lock that is currently being held.
func (lock *Lock) BreakLock() error {
	return os.RemoveAll(lock.lockDir())
}

// Message returns the saved message, or the empty string if there is no
// saved message.
func (lock *Lock) Message() string {
	message, err := ioutil.ReadFile(lock.messageFile())
	if err != nil {
		return ""
	}
	return string(message)
}
<|endoftext|>


<commit_before>package harness

// This file handles the app code introspection.
// It catalogs the controllers, their methods, and their arguments.

import (
	"go/ast"
	"go/parser"
	"go/scanner"
	"go/token"
	"log"
	"os"
	"github.com/robfig/revel"
	"strings"
	"unicode"
)

type ControllerSpec struct {
	PackageName string
	StructName  string
	ImportPath  string
	MethodSpecs []*MethodSpec

	// Used internally to identify controllers that indirectly embed *rev.Controller.
	embeddedTypes []*embeddedTypeName
}

// This is a description of a call to c.Render(..)
// It documents the argument names used, in order to propagate them to RenderArgs.
type renderCall struct {
	Line  int
	Names []string
}

type MethodSpec struct {
	Name        string        // Name of the method, e.g. "Index"
	Args        []*MethodArg  // Argument descriptors
	RenderCalls []*renderCall // Descriptions of Render() invocations from this Method.
}

type MethodArg struct {
	Name       string // Name of the argument.
	TypeName   string // The name of the type, e.g. "int", "*pkg.UserType"
	ImportPath string // If the arg is of an imported type, this is the import path.
}

type embeddedTypeName struct {
	PackageName, StructName string
}

// Maps a controller simple name (e.g. "Login") to the methods for which it is a
// receiver.
type methodMap map[string][]*MethodSpec

// Parse the app directory and return a list of the controller types found.
// Returns a CompileError if the parsing fails.
func ScanControllers(path string) (specs []*ControllerSpec, compileError *rev.Error) {
	// Parse files within the path.
	var pkgs map[string]*ast.Package
	fset := token.NewFileSet()
	pkgs, err := parser.ParseDir(fset, path, func(f os.FileInfo) bool { return !f.IsDir() }, 0)
	if err != nil {
		if errList, ok := err.(scanner.ErrorList); ok {
			var pos token.Position = errList[0].Pos
			return nil, &rev.Error{
				SourceType:  ".go source",
				Title:       "Go Compilation Error",
				Path:        pos.Filename,
				Description: errList[0].Msg,
				Line:        pos.Line,
				Column:      pos.Column,
				SourceLines: rev.MustReadLines(pos.Filename),
			}
		}
		ast.Print(nil, err)
		log.Fatalf("Failed to parse dir: %s", err)
	}

	// For each package... (often only "controllers")
	for _, pkg := range pkgs {
		var structSpecs []*ControllerSpec
		methodSpecs := make(methodMap)

		// For each source file in the package...
		for _, file := range pkg.Files {

			// Imports maps the package key to the full import path.
			// e.g. import "sample/app/models" => "models": "sample/app/models"
			imports := map[string]string{}

			// For each declaration in the source file...
			for _, decl := range file.Decls {

				// Match and add both structs and methods
				addImports(imports, decl)
				structSpecs = appendStruct(structSpecs, pkg, decl)
				appendMethod(fset, methodSpecs, decl, pkg.Name, imports)
			}
		}

		// Filter the struct specs to just the ones that embed rev.Controller.
		structSpecs = filterControllers(structSpecs)

		// Add the method specs to them.
		for _, spec := range structSpecs {
			spec.MethodSpecs = methodSpecs[spec.StructName]
		}

		// Add the prepared ControllerSpecs to the list.
		specs = append(specs, structSpecs...)
	}
	return
}

func addImports(imports map[string]string, decl ast.Decl) {
	genDecl, ok := decl.(*ast.GenDecl)
	if !ok {
		return
	}

	if genDecl.Tok != token.IMPORT {
		return
	}

	for _, spec := range genDecl.Specs {
		importSpec := spec.(*ast.ImportSpec)
		quotedPath := importSpec.Path.Value           // e.g. "\"sample/app/models\""
		fullPath := quotedPath[1 : len(quotedPath)-1] // Remove the quotes
		key := fullPath
		if lastSlash := strings.LastIndex(fullPath, "/"); lastSlash != -1 {
			key = fullPath[lastSlash+1:]
		}
		imports[key] = fullPath
	}
}

// If this Decl is a struct type definition, it is summarized and added to specs.
// Else, specs is returned unchanged.
func appendStruct(specs []*ControllerSpec, pkg *ast.Package, decl ast.Decl) []*ControllerSpec {
	// Filter out non-Struct type declarations.
	genDecl, ok := decl.(*ast.GenDecl)
	if !ok {
		return specs
	}

	if genDecl.Tok != token.TYPE {
		return specs
	}

	if len(genDecl.Specs) != 1 {
		rev.LOG.Printf("Surprising: Decl does not have 1 Spec: %v", genDecl)
		return specs
	}

	spec := genDecl.Specs[0].(*ast.TypeSpec)
	structType, ok := spec.Type.(*ast.StructType)
	if !ok {
		return specs
	}

	// At this point we know it's a type declaration for a struct.
	// Fill in the rest of the info by diving into the fields.
	// Add it provisionally to the Controller list -- it's later filtered using field info.
	controllerSpec := &ControllerSpec{
		PackageName: pkg.Name,
		StructName:  spec.Name.Name,
		ImportPath:  rev.ImportPath + "/app/" + pkg.Name,
	}

	for _, field := range structType.Fields.List {
		// If field.Names is set, it's not an embedded type.
		if field.Names != nil {
			continue
		}

		// A direct "sub-type" has an ast.Field as either:
		//   Ident { "AppController" }
		//   SelectorExpr { "rev", "Controller" }
		// Additionally, that can be wrapped by StarExprs.
		fieldType := field.Type
		pkgName, typeName := func() (string, string) {
			// Drill through any StarExprs.
			for {
				if starExpr, ok := fieldType.(*ast.StarExpr); ok {
					fieldType = starExpr.X
					continue
				}
				break
			}

			// If the embedded type is in the same package, it's an Ident.
			if ident, ok := fieldType.(*ast.Ident); ok {
				return pkg.Name, ident.Name
			}

			if selectorExpr, ok := fieldType.(*ast.SelectorExpr); ok {
				if pkgIdent, ok := selectorExpr.X.(*ast.Ident); ok {
					return pkgIdent.Name, selectorExpr.Sel.Name
				}
			}

			return "", ""
		}()

		// If a typename wasn't found, skip it.
		if typeName == "" {
			continue
		}

		controllerSpec.embeddedTypes = append(controllerSpec.embeddedTypes, &embeddedTypeName{
			PackageName: pkgName,
			StructName:  typeName,
		})
	}

	return append(specs, controllerSpec)
}

// If decl is a Method declaration, it is summarized and added to the array
// underneath its receiver type.
// e.g. "Login" => {MethodSpec, MethodSpec, ..}
func appendMethod(fset *token.FileSet, mm methodMap, decl ast.Decl, pkgName string, imports map[string]string) {
	// Func declaration?
	funcDecl, ok := decl.(*ast.FuncDecl)
	if !ok {
		return
	}

	// Have a receiver?
	if funcDecl.Recv == nil {
		return
	}

	// Is it public?
	if !unicode.IsUpper([]rune(funcDecl.Name.Name)[0]) {
		return
	}

	// Does it return a rev.Result?
	if funcDecl.Type.Results == nil || len(funcDecl.Type.Results.List) != 1 {
		return
	}
	selExpr, ok := funcDecl.Type.Results.List[0].Type.(*ast.SelectorExpr)
	if !ok {
		return
	}
	if pkgIdent, ok := selExpr.X.(*ast.Ident); !ok || pkgIdent.Name != "rev" {
		return
	}
	if selExpr.Sel.Name != "Result" {
		return
	}

	// Get the receiver type, "dereferencing" it if necessary
	var recvTypeName string
	var recvType ast.Expr = funcDecl.Recv.List[0].Type
	if recvStarType, ok := recvType.(*ast.StarExpr); ok {
		recvTypeName = recvStarType.X.(*ast.Ident).Name
	} else {
		recvTypeName = recvType.(*ast.Ident).Name
	}

	method := &MethodSpec{
		Name: funcDecl.Name.Name,
	}

	// Add a description of the arguments to the method.
	for _, field := range funcDecl.Type.Params.List {
		for _, name := range field.Names {
			typeName := ExprName(field.Type)
			importPath := ""
			dotIndex := strings.Index(typeName, ".")
			isExported := unicode.IsUpper([]rune(typeName)[0])
			if dotIndex == -1 && isExported {
				typeName = pkgName + "." + typeName
			} else if dotIndex != -1 {
				// The type comes from may come from an imported package.
				argPkgName := typeName[:dotIndex]
				if importPath, ok = imports[argPkgName]; !ok {
					log.Println("Failed to find import for arg of type:", typeName)
				}
			}

			method.Args = append(method.Args, &MethodArg{
				Name:       name.Name,
				TypeName:   typeName,
				ImportPath: importPath,
			})
		}
	}

	// Add a description of the calls to Render from the method.
	// Inspect every node (e.g. always return true).
	method.RenderCalls = []*renderCall{}
	ast.Inspect(funcDecl.Body, func(node ast.Node) bool {
		// Is it a function call?
		callExpr, ok := node.(*ast.CallExpr)
		if !ok {
			return true
		}

		// Is it calling (*Controller).Render?
		selExpr, ok := callExpr.Fun.(*ast.SelectorExpr)
		if !ok {
			return true
		}

		// The type of the receiver is not easily available, so just store every
		// call to any method called Render.
		if selExpr.Sel.Name != "Render" {
			return true
		}

		// Add this call's args to the renderArgs.
		pos := fset.Position(callExpr.Rparen)
		renderCall := &renderCall{
			Line:  pos.Line,
			Names: []string{},
		}
		for _, arg := range callExpr.Args {
			argIdent, ok := arg.(*ast.Ident)
			if !ok {
				log.Println("Unnamed argument to Render call:", pos)
				continue
			}
			renderCall.Names = append(renderCall.Names, argIdent.Name)
		}
		method.RenderCalls = append(method.RenderCalls, renderCall)
		return true
	})

	mm[recvTypeName] = append(mm[recvTypeName], method)
}

func (s *ControllerSpec) SimpleName() string {
	return s.PackageName + "." + s.StructName
}

func (s *embeddedTypeName) SimpleName() string {
	return s.PackageName + "." + s.StructName
}

// Remove any types that do not (directly or indirectly) embed *rev.Controller.
func filterControllers(specs []*ControllerSpec) (filtered []*ControllerSpec) {
	// Do a search in the "embedded type graph", starting with rev.Controller.
	nodeQueue := []string{"rev.Controller"}
	for len(nodeQueue) > 0 {
		controllerSimpleName := nodeQueue[0]
		nodeQueue = nodeQueue[1:]
		for _, spec := range specs {
			if rev.ContainsString(nodeQueue, spec.SimpleName()) {
				continue // Already added
			}

			// Look through the embedded types to see if the current type is among them.
			for _, embeddedType := range spec.embeddedTypes {

				// If so, add this type's simple name to the nodeQueue, and its spec to
				// the filtered list.
				if controllerSimpleName == embeddedType.SimpleName() {
					nodeQueue = append(nodeQueue, spec.SimpleName())
					filtered = append(filtered, spec)
					break
				}
			}
		}
	}
	return
}

// This returns the syntactic expression for referencing this type in Go.
// One complexity is that package-local types have to be fully-qualified.
// For example, if the type is "Hello", then it really means "pkg.Hello".
func ExprName(expr ast.Expr) string {
	switch t := expr.(type) {
	case *ast.Ident:
		return t.Name
	case *ast.SelectorExpr:
		return ExprName(t.X) + "." + ExprName(t.Sel)
	case *ast.StarExpr:
		return "*" + ExprName(t.X)
	default:
		ast.Print(nil, expr)
		panic("Failed to generate name for field.")
	}
	return ""
}
<commit_msg>Ignore dot-files when building go<commit_after>package harness

// This file handles the app code introspection.
// It catalogs the controllers, their methods, and their arguments.

import (
	"go/ast"
	"go/parser"
	"go/scanner"
	"go/token"
	"log"
	"os"
	"github.com/robfig/revel"
	"strings"
	"unicode"
)

type ControllerSpec struct {
	PackageName string
	StructName  string
	ImportPath  string
	MethodSpecs []*MethodSpec

	// Used internally to identify controllers that indirectly embed *rev.Controller.
	embeddedTypes []*embeddedTypeName
}

// This is a description of a call to c.Render(..)
// It documents the argument names used, in order to propagate them to RenderArgs.
type renderCall struct {
	Line  int
	Names []string
}

type MethodSpec struct {
	Name        string        // Name of the method, e.g. "Index"
	Args        []*MethodArg  // Argument descriptors
	RenderCalls []*renderCall // Descriptions of Render() invocations from this Method.
}

type MethodArg struct {
	Name       string // Name of the argument.
	TypeName   string // The name of the type, e.g. "int", "*pkg.UserType"
	ImportPath string // If the arg is of an imported type, this is the import path.
}

type embeddedTypeName struct {
	PackageName, StructName string
}

// Maps a controller simple name (e.g. "Login") to the methods for which it is a
// receiver.
type methodMap map[string][]*MethodSpec

// Parse the app directory and return a list of the controller types found.
// Returns a CompileError if the parsing fails.
func ScanControllers(path string) (specs []*ControllerSpec, compileError *rev.Error) {
	// Parse files within the path.
	var pkgs map[string]*ast.Package
	fset := token.NewFileSet()
	pkgs, err := parser.ParseDir(fset, path, func(f os.FileInfo) bool {
		return !f.IsDir() && !strings.HasPrefix(f.Name(), ".")
	}, 0)
	if err != nil {
		if errList, ok := err.(scanner.ErrorList); ok {
			var pos token.Position = errList[0].Pos
			return nil, &rev.Error{
				SourceType:  ".go source",
				Title:       "Go Compilation Error",
				Path:        pos.Filename,
				Description: errList[0].Msg,
				Line:        pos.Line,
				Column:      pos.Column,
				SourceLines: rev.MustReadLines(pos.Filename),
			}
		}
		ast.Print(nil, err)
		log.Fatalf("Failed to parse dir: %s", err)
	}

	// For each package... (often only "controllers")
	for _, pkg := range pkgs {
		var structSpecs []*ControllerSpec
		methodSpecs := make(methodMap)

		// For each source file in the package...
		for _, file := range pkg.Files {

			// Imports maps the package key to the full import path.
			// e.g. import "sample/app/models" => "models": "sample/app/models"
			imports := map[string]string{}

			// For each declaration in the source file...
			for _, decl := range file.Decls {

				// Match and add both structs and methods
				addImports(imports, decl)
				structSpecs = appendStruct(structSpecs, pkg, decl)
				appendMethod(fset, methodSpecs, decl, pkg.Name, imports)
			}
		}

		// Filter the struct specs to just the ones that embed rev.Controller.
		structSpecs = filterControllers(structSpecs)

		// Add the method specs to them.
		for _, spec := range structSpecs {
			spec.MethodSpecs = methodSpecs[spec.StructName]
		}

		// Add the prepared ControllerSpecs to the list.
		specs = append(specs, structSpecs...)
	}
	return
}

func addImports(imports map[string]string, decl ast.Decl) {
	genDecl, ok := decl.(*ast.GenDecl)
	if !ok {
		return
	}

	if genDecl.Tok != token.IMPORT {
		return
	}

	for _, spec := range genDecl.Specs {
		importSpec := spec.(*ast.ImportSpec)
		quotedPath := importSpec.Path.Value           // e.g. "\"sample/app/models\""
		fullPath := quotedPath[1 : len(quotedPath)-1] // Remove the quotes
		key := fullPath
		if lastSlash := strings.LastIndex(fullPath, "/"); lastSlash != -1 {
			key = fullPath[lastSlash+1:]
		}
		imports[key] = fullPath
	}
}

// If this Decl is a struct type definition, it is summarized and added to specs.
// Else, specs is returned unchanged.
func appendStruct(specs []*ControllerSpec, pkg *ast.Package, decl ast.Decl) []*ControllerSpec {
	// Filter out non-Struct type declarations.
	genDecl, ok := decl.(*ast.GenDecl)
	if !ok {
		return specs
	}

	if genDecl.Tok != token.TYPE {
		return specs
	}

	if len(genDecl.Specs) != 1 {
		rev.LOG.Printf("Surprising: Decl does not have 1 Spec: %v", genDecl)
		return specs
	}

	spec := genDecl.Specs[0].(*ast.TypeSpec)
	structType, ok := spec.Type.(*ast.StructType)
	if !ok {
		return specs
	}

	// At this point we know it's a type declaration for a struct.
	// Fill in the rest of the info by diving into the fields.
	// Add it provisionally to the Controller list -- it's later filtered using field info.
	controllerSpec := &ControllerSpec{
		PackageName: pkg.Name,
		StructName:  spec.Name.Name,
		ImportPath:  rev.ImportPath + "/app/" + pkg.Name,
	}

	for _, field := range structType.Fields.List {
		// If field.Names is set, it's not an embedded type.
		if field.Names != nil {
			continue
		}

		// A direct "sub-type" has an ast.Field as either:
		//   Ident { "AppController" }
		//   SelectorExpr { "rev", "Controller" }
		// Additionally, that can be wrapped by StarExprs.
		fieldType := field.Type
		pkgName, typeName := func() (string, string) {
			// Drill through any StarExprs.
			for {
				if starExpr, ok := fieldType.(*ast.StarExpr); ok {
					fieldType = starExpr.X
					continue
				}
				break
			}

			// If the embedded type is in the same package, it's an Ident.
			if ident, ok := fieldType.(*ast.Ident); ok {
				return pkg.Name, ident.Name
			}

			if selectorExpr, ok := fieldType.(*ast.SelectorExpr); ok {
				if pkgIdent, ok := selectorExpr.X.(*ast.Ident); ok {
					return pkgIdent.Name, selectorExpr.Sel.Name
				}
			}

			return "", ""
		}()

		// If a typename wasn't found, skip it.
		if typeName == "" {
			continue
		}

		controllerSpec.embeddedTypes = append(controllerSpec.embeddedTypes, &embeddedTypeName{
			PackageName: pkgName,
			StructName:  typeName,
		})
	}

	return append(specs, controllerSpec)
}

// If decl is a Method declaration, it is summarized and added to the array
// underneath its receiver type.
// e.g. "Login" => {MethodSpec, MethodSpec, ..}
func appendMethod(fset *token.FileSet, mm methodMap, decl ast.Decl, pkgName string, imports map[string]string) {
	// Func declaration?
	funcDecl, ok := decl.(*ast.FuncDecl)
	if !ok {
		return
	}

	// Have a receiver?
	if funcDecl.Recv == nil {
		return
	}

	// Is it public?
	if !unicode.IsUpper([]rune(funcDecl.Name.Name)[0]) {
		return
	}

	// Does it return a rev.Result?
	if funcDecl.Type.Results == nil || len(funcDecl.Type.Results.List) != 1 {
		return
	}
	selExpr, ok := funcDecl.Type.Results.List[0].Type.(*ast.SelectorExpr)
	if !ok {
		return
	}
	if pkgIdent, ok := selExpr.X.(*ast.Ident); !ok || pkgIdent.Name != "rev" {
		return
	}
	if selExpr.Sel.Name != "Result" {
		return
	}

	// Get the receiver type, "dereferencing" it if necessary
	var recvTypeName string
	var recvType ast.Expr = funcDecl.Recv.List[0].Type
	if recvStarType, ok := recvType.(*ast.StarExpr); ok {
		recvTypeName = recvStarType.X.(*ast.Ident).Name
	} else {
		recvTypeName = recvType.(*ast.Ident).Name
	}

	method := &MethodSpec{
		Name: funcDecl.Name.Name,
	}

	// Add a description of the arguments to the method.
	for _, field := range funcDecl.Type.Params.List {
		for _, name := range field.Names {
			typeName := ExprName(field.Type)
			importPath := ""
			dotIndex := strings.Index(typeName, ".")
			isExported := unicode.IsUpper([]rune(typeName)[0])
			if dotIndex == -1 && isExported {
				typeName = pkgName + "." + typeName
			} else if dotIndex != -1 {
				// The type comes from may come from an imported package.
				argPkgName := typeName[:dotIndex]
				if importPath, ok = imports[argPkgName]; !ok {
					log.Println("Failed to find import for arg of type:", typeName)
				}
			}

			method.Args = append(method.Args, &MethodArg{
				Name:       name.Name,
				TypeName:   typeName,
				ImportPath: importPath,
			})
		}
	}

	// Add a description of the calls to Render from the method.
	// Inspect every node (e.g. always return true).
	method.RenderCalls = []*renderCall{}
	ast.Inspect(funcDecl.Body, func(node ast.Node) bool {
		// Is it a function call?
		callExpr, ok := node.(*ast.CallExpr)
		if !ok {
			return true
		}

		// Is it calling (*Controller).Render?
		selExpr, ok := callExpr.Fun.(*ast.SelectorExpr)
		if !ok {
			return true
		}

		// The type of the receiver is not easily available, so just store every
		// call to any method called Render.
		if selExpr.Sel.Name != "Render" {
			return true
		}

		// Add this call's args to the renderArgs.
		pos := fset.Position(callExpr.Rparen)
		renderCall := &renderCall{
			Line:  pos.Line,
			Names: []string{},
		}
		for _, arg := range callExpr.Args {
			argIdent, ok := arg.(*ast.Ident)
			if !ok {
				log.Println("Unnamed argument to Render call:", pos)
				continue
			}
			renderCall.Names = append(renderCall.Names, argIdent.Name)
		}
		method.RenderCalls = append(method.RenderCalls, renderCall)
		return true
	})

	mm[recvTypeName] = append(mm[recvTypeName], method)
}

func (s *ControllerSpec) SimpleName() string {
	return s.PackageName + "." + s.StructName
}

func (s *embeddedTypeName) SimpleName() string {
	return s.PackageName + "." + s.StructName
}

// Remove any types that do not (directly or indirectly) embed *rev.Controller.
func filterControllers(specs []*ControllerSpec) (filtered []*ControllerSpec) {
	// Do a search in the "embedded type graph", starting with rev.Controller.
	nodeQueue := []string{"rev.Controller"}
	for len(nodeQueue) > 0 {
		controllerSimpleName := nodeQueue[0]
		nodeQueue = nodeQueue[1:]
		for _, spec := range specs {
			if rev.ContainsString(nodeQueue, spec.SimpleName()) {
				continue // Already added
			}

			// Look through the embedded types to see if the current type is among them.
			for _, embeddedType := range spec.embeddedTypes {

				// If so, add this type's simple name to the nodeQueue, and its spec to
				// the filtered list.
				if controllerSimpleName == embeddedType.SimpleName() {
					nodeQueue = append(nodeQueue, spec.SimpleName())
					filtered = append(filtered, spec)
					break
				}
			}
		}
	}
	return
}

// This returns the syntactic expression for referencing this type in Go.
// One complexity is that package-local types have to be fully-qualified.
// For example, if the type is "Hello", then it really means "pkg.Hello".
func ExprName(expr ast.Expr) string {
	switch t := expr.(type) {
	case *ast.Ident:
		return t.Name
	case *ast.SelectorExpr:
		return ExprName(t.X) + "." + ExprName(t.Sel)
	case *ast.StarExpr:
		return "*" + ExprName(t.X)
	default:
		ast.Print(nil, expr)
		panic("Failed to generate name for field.")
	}
	return ""
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"
	"time"

	"github.com/hashicorp/terraform/helper/diff"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
	"github.com/mitchellh/goamz/ec2"
)

func resource_aws_subnet_create(
	s *terraform.InstanceState,
	d *terraform.InstanceDiff,
	meta interface{}) (*terraform.InstanceState, error) {
	p := meta.(*ResourceProvider)
	ec2conn := p.ec2conn

	// Merge the diff so that we have all the proper attributes
	s = s.MergeDiff(d)

	// Create the Subnet
	createOpts := &ec2.CreateSubnet{
		AvailabilityZone: s.Attributes["availability_zone"],
		CidrBlock:        s.Attributes["cidr_block"],
		VpcId:            s.Attributes["vpc_id"],
	}
	log.Printf("[DEBUG] Subnet create config: %#v", createOpts)
	resp, err := ec2conn.CreateSubnet(createOpts)
	if err != nil {
		return nil, fmt.Errorf("Error creating subnet: %s", err)
	}

	// Get the ID and store it
	subnet := &resp.Subnet
	s.ID = subnet.SubnetId
	log.Printf("[INFO] Subnet ID: %s", s.ID)

	// Wait for the Subnet to become available
	log.Printf(
		"[DEBUG] Waiting for subnet (%s) to become available",
		s.ID)
	stateConf := &resource.StateChangeConf{
		Pending: []string{"pending"},
		Target:  "available",
		Refresh: SubnetStateRefreshFunc(ec2conn, s.ID),
		Timeout: 10 * time.Minute,
	}
	subnetRaw, err := stateConf.WaitForState()
	if err != nil {
		return s, fmt.Errorf(
			"Error waiting for subnet (%s) to become available: %s",
			s.ID, err)
	}

	// Map public ip on launch must be set in another API call
	if attr := s.Attributes["map_public_ip_on_launch"]; attr == "true" {
		modifyOpts := &ec2.ModifySubnetAttribute{
			SubnetId:            s.ID,
			MapPublicIpOnLaunch: true,
		}
		log.Printf("[DEBUG] Subnet modify attributes: %#v", modifyOpts)
		_, err := ec2conn.ModifySubnetAttribute(modifyOpts)
		if err != nil {
			return nil, fmt.Errorf("Error modify subnet attributes: %s", err)
		}
	}

	// Update our attributes and return
	return resource_aws_subnet_update_state(s, subnetRaw.(*ec2.Subnet))
}

func resource_aws_subnet_update(
	s *terraform.InstanceState,
	d *terraform.InstanceDiff,
	meta interface{}) (*terraform.InstanceState, error) {
	// This should never be called because we have no update-able
	// attributes
	panic("Update for subnet is not supported")
}

func resource_aws_subnet_destroy(
	s *terraform.InstanceState,
	meta interface{}) error {
	p := meta.(*ResourceProvider)
	ec2conn := p.ec2conn

	log.Printf("[INFO] Deleting Subnet: %s", s.ID)
	f := func() error {
		_, err := ec2conn.DeleteSubnet(s.ID)
		return err
	}
	if err := resource.Retry(10*time.Second, f); err != nil {
		ec2err, ok := err.(*ec2.Error)
		if ok && ec2err.Code == "InvalidSubnetID.NotFound" {
			return nil
		}

		return fmt.Errorf("Error deleting subnet: %s", err)
	}

	// Wait for the Subnet to actually delete
	log.Printf("[DEBUG] Waiting for subnet (%s) to delete", s.ID)
	stateConf := &resource.StateChangeConf{
		Pending: []string{"available", "pending"},
		Target:  "",
		Refresh: SubnetStateRefreshFunc(ec2conn, s.ID),
		Timeout: 10 * time.Minute,
	}
	if _, err := stateConf.WaitForState(); err != nil {
		return fmt.Errorf(
			"Error waiting for subnet (%s) to destroy: %s",
			s.ID, err)
	}

	return nil
}

func resource_aws_subnet_refresh(
	s *terraform.InstanceState,
	meta interface{}) (*terraform.InstanceState, error) {
	p := meta.(*ResourceProvider)
	ec2conn := p.ec2conn

	subnetRaw, _, err := SubnetStateRefreshFunc(ec2conn, s.ID)()
	if err != nil {
		return s, err
	}
	if subnetRaw == nil {
		return nil, nil
	}

	subnet := subnetRaw.(*ec2.Subnet)
	return resource_aws_subnet_update_state(s, subnet)
}

func resource_aws_subnet_diff(
	s *terraform.InstanceState,
	c *terraform.ResourceConfig,
	meta interface{}) (*terraform.InstanceDiff, error) {
	b := &diff.ResourceBuilder{
		Attrs: map[string]diff.AttrType{
			"availability_zone":       diff.AttrTypeCreate,
			"cidr_block":              diff.AttrTypeCreate,
			"vpc_id":                  diff.AttrTypeCreate,
			"map_public_ip_on_launch": diff.AttrTypeCreate,
		},

		ComputedAttrs: []string{
			"availability_zone",
		},
	}

	return b.Diff(s, c)
}

func resource_aws_subnet_update_state(
	s *terraform.InstanceState,
	subnet *ec2.Subnet) (*terraform.InstanceState, error) {
	s.Attributes["availability_zone"] = subnet.AvailabilityZone
	s.Attributes["cidr_block"] = subnet.CidrBlock
	s.Attributes["vpc_id"] = subnet.VpcId
	if subnet.MapPublicIpOnLaunch {
		s.Attributes["map_public_ip_on_launch"] = "true"
	}

	return s, nil
}

// SubnetStateRefreshFunc returns a resource.StateRefreshFunc that is used to watch
// a Subnet.
func SubnetStateRefreshFunc(conn *ec2.EC2, id string) resource.StateRefreshFunc {
	return func() (interface{}, string, error) {
		resp, err := conn.DescribeSubnets([]string{id}, ec2.NewFilter())
		if err != nil {
			if ec2err, ok := err.(*ec2.Error); ok && ec2err.Code == "InvalidSubnetID.NotFound" {
				resp = nil
			} else {
				log.Printf("Error on SubnetStateRefresh: %s", err)
				return nil, "", err
			}
		}

		if resp == nil {
			// Sometimes AWS just has consistency issues and doesn't see
			// our instance yet. Return an empty state.
			return nil, "", nil
		}

		subnet := &resp.Subnets[0]
		return subnet, subnet.State, nil
	}
}
<commit_msg>providers/aws: retry destroying subnet for some time [GH-449]<commit_after>package aws

import (
	"fmt"
	"log"
	"time"

	"github.com/hashicorp/terraform/helper/diff"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
	"github.com/mitchellh/goamz/ec2"
)

func resource_aws_subnet_create(
	s *terraform.InstanceState,
	d *terraform.InstanceDiff,
	meta interface{}) (*terraform.InstanceState, error) {
	p := meta.(*ResourceProvider)
	ec2conn := p.ec2conn

	// Merge the diff so that we have all the proper attributes
	s = s.MergeDiff(d)

	// Create the Subnet
	createOpts := &ec2.CreateSubnet{
		AvailabilityZone: s.Attributes["availability_zone"],
		CidrBlock:        s.Attributes["cidr_block"],
		VpcId:            s.Attributes["vpc_id"],
	}
	log.Printf("[DEBUG] Subnet create config: %#v", createOpts)
	resp, err := ec2conn.CreateSubnet(createOpts)
	if err != nil {
		return nil, fmt.Errorf("Error creating subnet: %s", err)
	}

	// Get the ID and store it
	subnet := &resp.Subnet
	s.ID = subnet.SubnetId
	log.Printf("[INFO] Subnet ID: %s", s.ID)

	// Wait for the Subnet to become available
	log.Printf(
		"[DEBUG] Waiting for subnet (%s) to become available",
		s.ID)
	stateConf := &resource.StateChangeConf{
		Pending: []string{"pending"},
		Target:  "available",
		Refresh: SubnetStateRefreshFunc(ec2conn, s.ID),
		Timeout: 10 * time.Minute,
	}
	subnetRaw, err := stateConf.WaitForState()
	if err != nil {
		return s, fmt.Errorf(
			"Error waiting for subnet (%s) to become available: %s",
			s.ID, err)
	}

	// Map public ip on launch must be set in another API call
	if attr := s.Attributes["map_public_ip_on_launch"]; attr == "true" {
		modifyOpts := &ec2.ModifySubnetAttribute{
			SubnetId:            s.ID,
			MapPublicIpOnLaunch: true,
		}
		log.Printf("[DEBUG] Subnet modify attributes: %#v", modifyOpts)
		_, err := ec2conn.ModifySubnetAttribute(modifyOpts)
		if err != nil {
			return nil, fmt.Errorf("Error modify subnet attributes: %s", err)
		}
	}

	// Update our attributes and return
	return resource_aws_subnet_update_state(s, subnetRaw.(*ec2.Subnet))
}

func resource_aws_subnet_update(
	s *terraform.InstanceState,
	d *terraform.InstanceDiff,
	meta interface{}) (*terraform.InstanceState, error) {
	// This should never be called because we have no update-able
	// attributes
	panic("Update for subnet is not supported")
}

func resource_aws_subnet_destroy(
	s *terraform.InstanceState,
	meta interface{}) error {
	p := meta.(*ResourceProvider)
	ec2conn := p.ec2conn

	log.Printf("[INFO] Deleting Subnet: %s", s.ID)
	return resource.Retry(5*time.Minute, func() error {
		_, err := ec2conn.DeleteSubnet(s.ID)
		if err != nil {
			ec2err, ok := err.(*ec2.Error)
			if !ok {
				return err
			}

			switch ec2err.Code {
			case "InvalidSubnetID.NotFound":
				return nil
			case "DependencyViolation":
				return err // retry
			default:
				return resource.RetryError{err}
			}
		}

		return fmt.Errorf("Error deleting subnet: %s", err)
	})

	// Wait for the Subnet to actually delete
	log.Printf("[DEBUG] Waiting for subnet (%s) to delete", s.ID)
	stateConf := &resource.StateChangeConf{
		Pending: []string{"available", "pending"},
		Target:  "",
		Refresh: SubnetStateRefreshFunc(ec2conn, s.ID),
		Timeout: 10 * time.Minute,
	}
	if _, err := stateConf.WaitForState(); err != nil {
		return fmt.Errorf(
			"Error waiting for subnet (%s) to destroy: %s",
			s.ID, err)
	}

	return nil
}

func resource_aws_subnet_refresh(
	s *terraform.InstanceState,
	meta interface{}) (*terraform.InstanceState, error) {
	p := meta.(*ResourceProvider)
	ec2conn := p.ec2conn

	subnetRaw, _, err := SubnetStateRefreshFunc(ec2conn, s.ID)()
	if err != nil {
		return s, err
	}
	if subnetRaw == nil {
		return nil, nil
	}

	subnet := subnetRaw.(*ec2.Subnet)
	return resource_aws_subnet_update_state(s, subnet)
}

func resource_aws_subnet_diff(
	s *terraform.InstanceState,
	c *terraform.ResourceConfig,
	meta interface{}) (*terraform.InstanceDiff, error) {
	b := &diff.ResourceBuilder{
		Attrs: map[string]diff.AttrType{
			"availability_zone":       diff.AttrTypeCreate,
			"cidr_block":              diff.AttrTypeCreate,
			"vpc_id":                  diff.AttrTypeCreate,
			"map_public_ip_on_launch": diff.AttrTypeCreate,
		},

		ComputedAttrs: []string{
			"availability_zone",
		},
	}

	return b.Diff(s, c)
}

func resource_aws_subnet_update_state(
	s *terraform.InstanceState,
	subnet *ec2.Subnet) (*terraform.InstanceState, error) {
	s.Attributes["availability_zone"] = subnet.AvailabilityZone
	s.Attributes["cidr_block"] = subnet.CidrBlock
	s.Attributes["vpc_id"] = subnet.VpcId
	if subnet.MapPublicIpOnLaunch {
		s.Attributes["map_public_ip_on_launch"] = "true"
	}

	return s, nil
}

// SubnetStateRefreshFunc returns a resource.StateRefreshFunc that is used to watch
// a Subnet.
func SubnetStateRefreshFunc(conn *ec2.EC2, id string) resource.StateRefreshFunc {
	return func() (interface{}, string, error) {
		resp, err := conn.DescribeSubnets([]string{id}, ec2.NewFilter())
		if err != nil {
			if ec2err, ok := err.(*ec2.Error); ok && ec2err.Code == "InvalidSubnetID.NotFound" {
				resp = nil
			} else {
				log.Printf("Error on SubnetStateRefresh: %s", err)
				return nil, "", err
			}
		}

		if resp == nil {
			// Sometimes AWS just has consistency issues and doesn't see
			// our instance yet. Return an empty state.
			return nil, "", nil
		}

		subnet := &resp.Subnets[0]
		return subnet, subnet.State, nil
	}
}
<|endoftext|>


<commit_before>package integration_test

import (
	"fmt"
	"net/http"
	"os"

	"code.cloudfoundry.org/lager"

	uuid "github.com/nu7hatch/gouuid"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	. "github.com/concourse/atc/cessna"
	"github.com/concourse/baggageclaim"

	bclient "github.com/concourse/baggageclaim/client"

	"testing"

	"archive/tar"
	"bytes"
	"io"

	"code.cloudfoundry.org/garden/gardenfakes"
	gserver "code.cloudfoundry.org/garden/server"
	"code.cloudfoundry.org/lager/lagertest"
	"github.com/concourse/atc/cessna/cessnafakes"
	"github.com/concourse/baggageclaim/baggageclaimfakes"
)

var (
	skipped bool

	testBaseResource Resource
	worker           Worker
	baseResourceType BaseResourceType
	workerIp         string
	tarPath          string
	tarURL           string

	found bool

	logger lager.Logger

	fakeWorker             *cessnafakes.FakeWorker
	fakeGardenClient       *gardenfakes.FakeClient
	fakeBaggageClaimClient *baggageclaimfakes.FakeClient
)

var _ = BeforeSuite(func() {
	_, found = os.LookupEnv("RUN_CESSNA_TESTS")
	if !found {
		skipped = true
	}
})

var _ = AfterSuite(func() {
	if skipped || workerIp == "" {
		return
	}

	worker = NewWorker(fmt.Sprintf("%s:7777", workerIp), fmt.Sprintf("http://%s:7788", workerIp))

	containers, err := worker.GardenClient().Containers(nil)
	Expect(err).NotTo(HaveOccurred())

	for _, container := range containers {
		err = worker.GardenClient().Destroy(container.Handle())
		if err != nil {
			// once garden fixes container grace timeout to be indefinite we can remove this check
			if err != gserver.ErrConcurrentDestroy {
				Expect(err).NotTo(HaveOccurred())
			}
		}
	}

	volumes, err := worker.BaggageClaimClient().ListVolumes(logger, nil)
	Expect(err).NotTo(HaveOccurred())

	for _, volume := range volumes {
		err = volume.Destroy()
		Expect(err).NotTo(HaveOccurred())
	}
})

var _ = BeforeEach(func() {
	if skipped {
		Skip("$RUN_CESSNA_TESTS not set; skipping")
	}

	fakeWorker = new(cessnafakes.FakeWorker)
	fakeGardenClient = new(gardenfakes.FakeClient)
	fakeBaggageClaimClient = new(baggageclaimfakes.FakeClient)

	fakeWorker.BaggageClaimClientReturns(fakeBaggageClaimClient)
	fakeWorker.GardenClientReturns(fakeGardenClient)

	workerIp, found = os.LookupEnv("WORKER_IP")
	Expect(found).To(BeTrue(), "Must set WORKER_IP")

	tarPath, found = os.LookupEnv("ROOTFS_TAR_PATH")
	Expect(found).To(BeTrue(), "Must set ROOTFS_TAR_PATH")

	tarURL, found = os.LookupEnv("ROOTFS_TAR_URL")
	Expect(found).To(BeTrue(), "Must set ROOTFS_TAR_URL")

	worker = NewWorker(fmt.Sprintf("%s:7777", workerIp), fmt.Sprintf("http://%s:7788", workerIp))
	logger = lagertest.NewTestLogger("logger-test")
})

func TestResource(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Cessna Integration Suite")
}

func createBaseResourceVolume(r io.Reader) (string, error) {
	baggageclaimClient := bclient.New(fmt.Sprintf("http://%s:7788", workerIp), http.DefaultTransport)

	volumeSpec := baggageclaim.VolumeSpec{
		Strategy:   baggageclaim.EmptyStrategy{},
		Privileged: true,
	}

	handle, err := uuid.NewV4()
	if err != nil {
		return "", err
	}

	volume, err := baggageclaimClient.CreateVolume(
		lager.NewLogger("create-volume-for-base-resource"),
		handle.String(),
		volumeSpec,
	)

	if err != nil {
		return "", err
	}

	err = volume.StreamIn("/", r)
	if err != nil {
		return "", err
	}

	return volume.Path(), nil
}

func NewResourceContainer(check string, in string, out string) ResourceContainer {
	return ResourceContainer{
		Check:         check,
		In:            in,
		Out:           out,
		RootFSTarPath: tarPath,
	}
}

type ResourceContainer struct {
	Check         string
	In            string
	Out           string
	RootFSTarPath string
}

func (r ResourceContainer) RootFSify() (io.Reader, error) {
	f, err := os.Open(r.RootFSTarPath)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	buffer := new(bytes.Buffer)

	t := tar.NewWriter(buffer)
	rootFS := tar.NewReader(f)

	err = t.WriteHeader(&tar.Header{
		Name: "./opt/resource/check",
		Mode: 0755,
		Size: int64(len(r.Check)),
	})
	if err != nil {
		return nil, err
	}
	_, err = t.Write([]byte(r.Check))
	if err != nil {
		return nil, err
	}

	err = t.WriteHeader(&tar.Header{
		Name: "./opt/resource/in",
		Mode: 0755,
		Size: int64(len(r.In)),
	})
	if err != nil {
		return nil, err
	}
	_, err = t.Write([]byte(r.In))
	if err != nil {
		return nil, err
	}

	err = t.WriteHeader(&tar.Header{
		Name: "./opt/resource/out",
		Mode: 0755,
		Size: int64(len(r.Out)),
	})
	if err != nil {
		return nil, err
	}
	_, err = t.Write([]byte(r.Out))
	if err != nil {
		return nil, err
	}

	for {
		header, err := rootFS.Next()
		if err != nil {
			if err == io.EOF {
				break
			}
			return nil, err
		}

		err = t.WriteHeader(header)
		if err != nil {
			return nil, err
		}

		_, err = io.Copy(t, rootFS)
		if err != nil {
			return nil, err
		}
	}

	err = t.Close()
	if err != nil {
		return nil, err
	}

	return bytes.NewBuffer(buffer.Bytes()), nil
}
<commit_msg>ignore garden.ContainerNotFound error when deleting containers in cessna<commit_after>package integration_test

import (
	"fmt"
	"net/http"
	"os"

	"code.cloudfoundry.org/garden"
	"code.cloudfoundry.org/lager"

	uuid "github.com/nu7hatch/gouuid"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	. "github.com/concourse/atc/cessna"
	"github.com/concourse/baggageclaim"

	bclient "github.com/concourse/baggageclaim/client"

	"testing"

	"archive/tar"
	"bytes"
	"io"

	"code.cloudfoundry.org/garden/gardenfakes"
	gserver "code.cloudfoundry.org/garden/server"
	"code.cloudfoundry.org/lager/lagertest"
	"github.com/concourse/atc/cessna/cessnafakes"
	"github.com/concourse/baggageclaim/baggageclaimfakes"
)

var (
	skipped bool

	testBaseResource Resource
	worker           Worker
	baseResourceType BaseResourceType
	workerIp         string
	tarPath          string
	tarURL           string

	found bool

	logger lager.Logger

	fakeWorker             *cessnafakes.FakeWorker
	fakeGardenClient       *gardenfakes.FakeClient
	fakeBaggageClaimClient *baggageclaimfakes.FakeClient
)

var _ = BeforeSuite(func() {
	_, found = os.LookupEnv("RUN_CESSNA_TESTS")
	if !found {
		skipped = true
	}
})

var _ = AfterSuite(func() {
	if skipped || workerIp == "" {
		return
	}

	worker = NewWorker(fmt.Sprintf("%s:7777", workerIp), fmt.Sprintf("http://%s:7788", workerIp))

	containers, err := worker.GardenClient().Containers(nil)
	Expect(err).NotTo(HaveOccurred())

	for _, container := range containers {
		err = worker.GardenClient().Destroy(container.Handle())
		if err != nil {
			// once garden fixes container grace timeout to be indefinite we can remove this check
			if _, ok := err.(garden.ContainerNotFoundError); !ok {
				if err != gserver.ErrConcurrentDestroy {
					Expect(err).NotTo(HaveOccurred())
				}
			}
		}
	}

	volumes, err := worker.BaggageClaimClient().ListVolumes(logger, nil)
	Expect(err).NotTo(HaveOccurred())

	for _, volume := range volumes {
		err = volume.Destroy()
		Expect(err).NotTo(HaveOccurred())
	}
})

var _ = BeforeEach(func() {
	if skipped {
		Skip("$RUN_CESSNA_TESTS not set; skipping")
	}

	fakeWorker = new(cessnafakes.FakeWorker)
	fakeGardenClient = new(gardenfakes.FakeClient)
	fakeBaggageClaimClient = new(baggageclaimfakes.FakeClient)

	fakeWorker.BaggageClaimClientReturns(fakeBaggageClaimClient)
	fakeWorker.GardenClientReturns(fakeGardenClient)

	workerIp, found = os.LookupEnv("WORKER_IP")
	Expect(found).To(BeTrue(), "Must set WORKER_IP")

	tarPath, found = os.LookupEnv("ROOTFS_TAR_PATH")
	Expect(found).To(BeTrue(), "Must set ROOTFS_TAR_PATH")

	tarURL, found = os.LookupEnv("ROOTFS_TAR_URL")
	Expect(found).To(BeTrue(), "Must set ROOTFS_TAR_URL")

	worker = NewWorker(fmt.Sprintf("%s:7777", workerIp), fmt.Sprintf("http://%s:7788", workerIp))
	logger = lagertest.NewTestLogger("logger-test")
})

func TestResource(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Cessna Integration Suite")
}

func createBaseResourceVolume(r io.Reader) (string, error) {
	baggageclaimClient := bclient.New(fmt.Sprintf("http://%s:7788", workerIp), http.DefaultTransport)

	volumeSpec := baggageclaim.VolumeSpec{
		Strategy:   baggageclaim.EmptyStrategy{},
		Privileged: true,
	}

	handle, err := uuid.NewV4()
	if err != nil {
		return "", err
	}

	volume, err := baggageclaimClient.CreateVolume(
		lager.NewLogger("create-volume-for-base-resource"),
		handle.String(),
		volumeSpec,
	)

	if err != nil {
		return "", err
	}

	err = volume.StreamIn("/", r)
	if err != nil {
		return "", err
	}

	return volume.Path(), nil
}

func NewResourceContainer(check string, in string, out string) ResourceContainer {
	return ResourceContainer{
		Check:         check,
		In:            in,
		Out:           out,
		RootFSTarPath: tarPath,
	}
}

type ResourceContainer struct {
	Check         string
	In            string
	Out           string
	RootFSTarPath string
}

func (r ResourceContainer) RootFSify() (io.Reader, error) {
	f, err := os.Open(r.RootFSTarPath)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	buffer := new(bytes.Buffer)

	t := tar.NewWriter(buffer)
	rootFS := tar.NewReader(f)

	err = t.WriteHeader(&tar.Header{
		Name: "./opt/resource/check",
		Mode: 0755,
		Size: int64(len(r.Check)),
	})
	if err != nil {
		return nil, err
	}
	_, err = t.Write([]byte(r.Check))
	if err != nil {
		return nil, err
	}

	err = t.WriteHeader(&tar.Header{
		Name: "./opt/resource/in",
		Mode: 0755,
		Size: int64(len(r.In)),
	})
	if err != nil {
		return nil, err
	}
	_, err = t.Write([]byte(r.In))
	if err != nil {
		return nil, err
	}

	err = t.WriteHeader(&tar.Header{
		Name: "./opt/resource/out",
		Mode: 0755,
		Size: int64(len(r.Out)),
	})
	if err != nil {
		return nil, err
	}
	_, err = t.Write([]byte(r.Out))
	if err != nil {
		return nil, err
	}

	for {
		header, err := rootFS.Next()
		if err != nil {
			if err == io.EOF {
				break
			}
			return nil, err
		}

		err = t.WriteHeader(header)
		if err != nil {
			return nil, err
		}

		_, err = io.Copy(t, rootFS)
		if err != nil {
			return nil, err
		}
	}

	err = t.Close()
	if err != nil {
		return nil, err
	}

	return bytes.NewBuffer(buffer.Bytes()), nil
}
<|endoftext|>


<commit_before>package api

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"

	"github.com/tikasan/eventory/define"
	"github.com/tikasan/eventory/formater"
	"github.com/tikasan/eventory/model"
)

func NewInserter(rawurl string, rawapi int, token string) *Inserter {
	return &Inserter{
		Url:   rawurl,
		Api:   rawapi,
		Token: token,
	}
}

type Inserter struct {
	Url      string
	RespByte []byte
	Api      int
	Token    string
}

func (i *Inserter) Get() (events []model.Event, err error) {

	req, err := http.NewRequest("GET", i.Url, nil)
	if err != nil {
		fmt.Fprint(os.Stderr, err)
		return
	}
	if i.Token != "" {
		req.Header.Set("Authorization", i.Token)
	}

	client := new(http.Client)
	resp, err := client.Do(req)
	defer resp.Body.Close()

	respByte, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Fprint(os.Stderr, err)
		return
	}

	if i.Api == define.ATDN {
		var at model.At
		err = json.Unmarshal(respByte, &at)
		if err != nil {
			fmt.Fprint(os.Stderr, err)
			return events, nil
		}
		e := new(model.Event)
		events = make([]model.Event, len(at.Events))
		for i, v := range at.Events {
			formater.CopyStruct(v.Event, e)
			events[i] = *e
			events[i].ApiId = define.ATDN
		}

	} else if i.Api == define.CONNPASS {
		var cp model.Cp
		err := json.Unmarshal(respByte, &cp)
		if err != nil {
			fmt.Fprint(os.Stderr, err)
			return events, nil
		}

		e := new(model.Event)
		events = make([]model.Event, len(cp.Events))
		for i, v := range cp.Events {
			formater.CopyStruct(v, e)
			events[i] = *e
			events[i].ApiId = define.CONNPASS
		}

	} else if i.Api == define.DOORKEEPER {

		var dk []model.Dk
		err := json.Unmarshal(respByte, &dk)
		if err != nil {
			fmt.Fprint(os.Stderr, err)
			return events, nil
		}

		e := new(model.Event)
		events = make([]model.Event, len(dk))
		for i, v := range dk {
			formater.CopyStruct(v.Event, e)
			events[i] = *e
			events[i].ApiId = define.DOORKEEPER
		}

	} else {

		return events, errors.New("未知のAPIがセットされています。")

	}
	return events, nil
}
<commit_msg>hack: APIパッケージをテストしやすいようにコードを関数にまとめた。まだ改善の必要あり。<commit_after>package api

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"

	"github.com/tikasan/eventory/define"
	"github.com/tikasan/eventory/formater"
	"github.com/tikasan/eventory/model"
)

func NewInserter(rawurl string, rawapi int, token string) *Inserter {
	return &Inserter{
		Url:   rawurl,
		Api:   rawapi,
		Token: token,
	}
}

type Inserter struct {
	Url   string
	Api   int
	Token string
}

func (i *Inserter) sendQuery() (respByte []byte) {
	req, err := http.NewRequest("GET", i.Url, nil)
	if err != nil {
		fmt.Fprint(os.Stderr, err)
		return
	}
	if i.Token != "" {
		req.Header.Set("Authorization", i.Token)
	}

	client := new(http.Client)
	resp, err := client.Do(req)
	defer resp.Body.Close()

	respByte, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Fprint(os.Stderr, err)
		return
	}
	return respByte
}

func atdnJsonParse(respByte []byte) (events []model.Event, err error) {
	var at model.At
	err = json.Unmarshal(respByte, &at)
	if err != nil {
		fmt.Fprint(os.Stderr, err)
		return events, nil
	}
	e := new(model.Event)
	events = make([]model.Event, len(at.Events))
	for i, v := range at.Events {
		formater.CopyStruct(v.Event, e)
		events[i] = *e
		events[i].ApiId = define.ATDN
	}
	return events, nil
}

func connpassJsonParse(respByte []byte) (events []model.Event, err error) {
	var cp model.Cp
	err = json.Unmarshal(respByte, &cp)
	if err != nil {
		fmt.Fprint(os.Stderr, err)
		return events, nil
	}

	e := new(model.Event)
	events = make([]model.Event, len(cp.Events))
	for i, v := range cp.Events {
		formater.CopyStruct(v, e)
		events[i] = *e
		events[i].ApiId = define.CONNPASS
	}
	return events, nil
}

func doorkeeperJsonParse(respByte []byte) (events []model.Event, err error) {
	var dk []model.Dk
	err = json.Unmarshal(respByte, &dk)
	if err != nil {
		fmt.Fprint(os.Stderr, err)
		return events, nil
	}

	e := new(model.Event)
	events = make([]model.Event, len(dk))
	for i, v := range dk {
		formater.CopyStruct(v.Event, e)
		events[i] = *e
		events[i].ApiId = define.DOORKEEPER
	}
	return events, nil
}

func (i *Inserter) Get() (events []model.Event, err error) {

	respByte := i.sendQuery()

	if i.Api == define.ATDN {
		return atdnJsonParse(respByte)
	} else if i.Api == define.CONNPASS {
		return connpassJsonParse(respByte)
	} else if i.Api == define.DOORKEEPER {
		return doorkeeperJsonParse(respByte)
	}
	return events, errors.New("未知のAPIがセットされています。")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"strings"
	"sync"
	"time"

	maps "github.com/ashwanthkumar/golang-utils/maps"
	sets "github.com/ashwanthkumar/golang-utils/sets"
	marathon "github.com/gambol99/go-marathon"
	"github.com/rcrowley/go-metrics"
)

const (
	CheckSubscriptionLabel = "alerts.checks.subscribe"
	SubscribeAllChecks     = "all"
)

type CheckStatus uint8

const (
	Pass     = CheckStatus(99)
	Warning  = CheckStatus(2)
	Critical = CheckStatus(1)
)

var CheckLevels = [...]CheckStatus{Warning, Critical}

type AppChecker struct {
	Client        marathon.Marathon
	RunWaitGroup  sync.WaitGroup
	CheckInterval time.Duration
	stopChannel   chan bool
	Checks        []Checker
	AlertsChannel chan AppCheck
	// Snooze the entire system for some Time
	// Useful if we don't want to SPAM the notifications
	// when doing maintenance of mesos cluster
	// TODO - Enable this feature via API endpoint
	IsSnoozed  bool
	SnoozedAt  time.Time
	SnoozedFor time.Duration
}

func (a *AppChecker) Start() {
	fmt.Println("Starting App Checker...")
	a.RunWaitGroup.Add(1)
	a.stopChannel = make(chan bool)
	a.AlertsChannel = make(chan AppCheck)

	a.IsSnoozed = false

	go a.run()
	fmt.Println("App Checker Started.")
	fmt.Printf("App Checker - Checking the status of all the apps every %v\n", a.CheckInterval)
}

func (a *AppChecker) Stop() {
	fmt.Println("Stopping App Checker...")
	close(a.stopChannel)
	a.RunWaitGroup.Done()
}

func (a *AppChecker) run() {
	running := true
	for running {
		select {
		case <-time.After(a.CheckInterval):
			err := a.processChecks()
			if err != nil {
				log.Fatalf("Unexpected error - %v\n", err)
			}
		case <-a.stopChannel:
			metrics.GetOrRegisterCounter("apps-checker-stopped", DebugMetricsRegistry).Inc(int64(1))
			running = false
		}
		time.Sleep(1 * time.Second)
	}
}

func (a *AppChecker) processChecks() error {
	var apps *marathon.Applications
	var err error
	metrics.GetOrRegisterTimer("marathon-all-apps-response-time", nil).Time(func() {
		apps, err = a.Client.Applications(nil)
	})
	metrics.GetOrRegisterCounter("apps-checker-marathon-all-apps-api", DebugMetricsRegistry).Inc(int64(1))
	if err != nil {
		return err
	}
	for _, app := range apps.Apps {
		checksSubscribed := sets.FromSlice(
			strings.Split(maps.GetString(app.Labels, CheckSubscriptionLabel, SubscribeAllChecks),
				","))
		for _, check := range a.Checks {
			if checksSubscribed.Contains(check.Name()) || checksSubscribed.Contains(SubscribeAllChecks) {
				result := check.Check(app)
				a.AlertsChannel <- result
				metrics.GetOrRegisterCounter("apps-checker-alerts-sent", DebugMetricsRegistry).Inc(int64(1))
				metrics.GetOrRegisterCounter("apps-checker-check-"+check.Name(), DebugMetricsRegistry).Inc(int64(1))
				metrics.GetOrRegisterCounter("apps-checker-app-"+app.ID, DebugMetricsRegistry).Inc(int64(1))
				metrics.GetOrRegisterCounter("apps-checker-"+app.ID+"-"+check.Name(), DebugMetricsRegistry).Inc(int64(1))
			}
		}
	}

	return nil
}
<commit_msg>int64 doesn't need a wrapper in golang. TIL.<commit_after>package main

import (
	"fmt"
	"log"
	"strings"
	"sync"
	"time"

	maps "github.com/ashwanthkumar/golang-utils/maps"
	sets "github.com/ashwanthkumar/golang-utils/sets"
	marathon "github.com/gambol99/go-marathon"
	"github.com/rcrowley/go-metrics"
)

const (
	CheckSubscriptionLabel = "alerts.checks.subscribe"
	SubscribeAllChecks     = "all"
)

type CheckStatus uint8

const (
	Pass     = CheckStatus(99)
	Warning  = CheckStatus(2)
	Critical = CheckStatus(1)
)

var CheckLevels = [...]CheckStatus{Warning, Critical}

type AppChecker struct {
	Client        marathon.Marathon
	RunWaitGroup  sync.WaitGroup
	CheckInterval time.Duration
	stopChannel   chan bool
	Checks        []Checker
	AlertsChannel chan AppCheck
	// Snooze the entire system for some Time
	// Useful if we don't want to SPAM the notifications
	// when doing maintenance of mesos cluster
	// TODO - Enable this feature via API endpoint
	IsSnoozed  bool
	SnoozedAt  time.Time
	SnoozedFor time.Duration
}

func (a *AppChecker) Start() {
	fmt.Println("Starting App Checker...")
	a.RunWaitGroup.Add(1)
	a.stopChannel = make(chan bool)
	a.AlertsChannel = make(chan AppCheck)

	a.IsSnoozed = false

	go a.run()
	fmt.Println("App Checker Started.")
	fmt.Printf("App Checker - Checking the status of all the apps every %v\n", a.CheckInterval)
}

func (a *AppChecker) Stop() {
	fmt.Println("Stopping App Checker...")
	close(a.stopChannel)
	a.RunWaitGroup.Done()
}

func (a *AppChecker) run() {
	running := true
	for running {
		select {
		case <-time.After(a.CheckInterval):
			err := a.processChecks()
			if err != nil {
				log.Fatalf("Unexpected error - %v\n", err)
			}
		case <-a.stopChannel:
			metrics.GetOrRegisterCounter("apps-checker-stopped", DebugMetricsRegistry).Inc(1)
			running = false
		}
		time.Sleep(1 * time.Second)
	}
}

func (a *AppChecker) processChecks() error {
	var apps *marathon.Applications
	var err error
	metrics.GetOrRegisterTimer("marathon-all-apps-response-time", nil).Time(func() {
		apps, err = a.Client.Applications(nil)
	})
	metrics.GetOrRegisterCounter("apps-checker-marathon-all-apps-api", DebugMetricsRegistry).Inc(1)
	if err != nil {
		return err
	}
	for _, app := range apps.Apps {
		checksSubscribed := sets.FromSlice(
			strings.Split(maps.GetString(app.Labels, CheckSubscriptionLabel, SubscribeAllChecks),
				","))
		for _, check := range a.Checks {
			if checksSubscribed.Contains(check.Name()) || checksSubscribed.Contains(SubscribeAllChecks) {
				result := check.Check(app)
				a.AlertsChannel <- result
				metrics.GetOrRegisterCounter("apps-checker-alerts-sent", DebugMetricsRegistry).Inc(1)
				metrics.GetOrRegisterCounter("apps-checker-check-"+check.Name(), DebugMetricsRegistry).Inc(1)
				metrics.GetOrRegisterCounter("apps-checker-app-"+app.ID, DebugMetricsRegistry).Inc(1)
				metrics.GetOrRegisterCounter("apps-checker-"+app.ID+"-"+check.Name(), DebugMetricsRegistry).Inc(1)
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package mapstruct

import (
	"encoding/json"
)

// MapToStruct maps the map instance into the struct instance using JSON marshal logic
func MapToStruct(m []interface{}, val interface{}) error {
	tmp, err := json.Marshal(m)
	if err != nil {
		return err
	}
	err = json.Unmarshal(tmp, val)
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>Implement SliceToInterfaceSlice and SliceToInterfaceMap to handle collections abstractly<commit_after>package mapstruct

import (
	"encoding/json"
	"errors"
	"fmt"
	"reflect"
)

// MapToStruct maps the map instance into the struct instance using JSON marshal logic
func MapToStruct(m []interface{}, val interface{}) error {
	tmp, err := json.Marshal(m)
	if err != nil {
		return err
	}
	err = json.Unmarshal(tmp, val)
	if err != nil {
		return err
	}
	return nil
}

// SliceToInterfaceSlice creates a []interface{} from a slice of concrete type
func SliceToInterfaceSlice(slice interface{}) ([]interface{}, error) {
	v := reflect.ValueOf(slice)

	if v.Kind() != reflect.Slice && v.Kind() != reflect.Array {
		return nil, errors.New("given argument is neither a slice nor an array")
	}

	size := v.Len()
	result := make([]interface{}, size)
	for i := 0; i < size; i++ {
		item := v.Index(i)

		itemForField := item
		for itemForField.Kind() == reflect.Ptr {
			itemForField = itemForField.Elem()
		}
		if !itemForField.IsValid() {
			return nil, fmt.Errorf("the actual item indexed %d in given slice is not valid", i)
		}
		if itemForField.Kind() != reflect.Struct {
			return nil, errors.New("given slice isn't a slice of struct")
		}

		if !item.CanInterface() {
			return nil, fmt.Errorf("the original item indexed %d in given slice cannot interface", i)
		}
		result[i] = item.Interface()
	}
	return result, nil
}

// SliceToInterfaceMap creates a map[interface{}]interface{} from a slice of a struct with key defined in the struct
// It doesn't modify the raw type of elements like strpping pointer or something
func SliceToInterfaceMap(structSlice interface{}, keyFieldName string) (map[interface{}]interface{}, error) {
	v := reflect.ValueOf(structSlice)

	if v.Kind() != reflect.Slice && v.Kind() != reflect.Array {
		return nil, errors.New("given argument is neither a slice nor an array")
	}
	size := v.Len()
	result := make(map[interface{}]interface{}, size)
	for i := 0; i < size; i++ {
		item := v.Index(i)

		itemForField := item
		for itemForField.Kind() == reflect.Ptr {
			itemForField = itemForField.Elem()
		}
		if !itemForField.IsValid() {
			return nil, fmt.Errorf("the actual item indexed %d in given slice is not valid", i)
		}
		if itemForField.Kind() != reflect.Struct {
			return nil, errors.New("given slice isn't a slice of struct")
		}
		field := itemForField.FieldByName(keyFieldName)
		if !field.IsValid() {
			return nil, fmt.Errorf("the actual struct in given slice doesn't have a field named '%s'", keyFieldName)
		}

		if !item.CanInterface() {
			return nil, fmt.Errorf("the original item indexed %d in given slice cannot interface", i)
		}
		result[field.Interface()] = item.Interface()
	}
	return result, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2018 Pilosa Corp. All rights reserved.
//
// This file is part of Pilosa Enterprise Edition.
//
// Pilosa Enterprise Edition is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Pilosa Enterprise Edition is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with Pilosa Enterprise Edition.  If not, see <http://www.gnu.org/licenses/>.

package b

import (
	"io"

	"github.com/pilosa/pilosa/roaring"
)

func cmp(a, b uint64) int {
	return int(a - b)
}

type BTreeContainers struct {
	tree *Tree

	lastKey       uint64
	lastContainer *roaring.Container
}

func NewBTreeContainers() *BTreeContainers {
	return &BTreeContainers{
		tree: TreeNew(cmp),
	}
}

func NewBTreeBitmap(a ...uint64) *roaring.Bitmap {
	b := &roaring.Bitmap{
		Containers: NewBTreeContainers(),
	}
	b.Add(a...)
	return b
}

func (btc *BTreeContainers) Get(key uint64) *roaring.Container {
	// Check the last* cache for same container.
	if key == btc.lastKey && btc.lastContainer != nil {
		return btc.lastContainer
	}

	var c *roaring.Container
	el, ok := btc.tree.Get(key)
	if ok {
		c = el
		btc.lastKey = key
		btc.lastContainer = c
	}
	return c
}

func (btc *BTreeContainers) Put(key uint64, c *roaring.Container) {
	// If a mapped container is added to the tree, reset the
	// lastContainer cache so that the cache is not pointing
	// at a read-only mmap.
	if c.Mapped() {
		btc.lastContainer = nil
	}
	btc.tree.Set(key, c)
}

func (u updater) update(oldV *roaring.Container, exists bool) (*roaring.Container, bool) {
	// update the existing container
	if exists {
		oldV.Update(u.containerType, u.n, u.mapped)
		return oldV, false
	}
	cont := roaring.NewContainer()
	cont.Update(u.containerType, u.n, u.mapped)
	return cont, true
}

// this struct is added to prevent the closure locals from being escaped out to the heap
type updater struct {
	key           uint64
	containerType byte
	n             int
	mapped        bool
}

func (btc *BTreeContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) {
	a := updater{key, containerType, n, mapped}
	btc.tree.Put(key, a.update)
}

func (btc *BTreeContainers) Remove(key uint64) {
	btc.tree.Delete(key)
}

func (btc *BTreeContainers) GetOrCreate(key uint64) *roaring.Container {
	// Check the last* cache for same container.
	if key == btc.lastKey && btc.lastContainer != nil {
		return btc.lastContainer
	}

	btc.lastKey = key
	v, ok := btc.tree.Get(key)
	if !ok {
		cont := roaring.NewContainer()
		btc.tree.Set(key, cont)
		btc.lastContainer = cont
		return cont
	}

	btc.lastContainer = v
	return btc.lastContainer
}

func (btc *BTreeContainers) Count() (n uint64) {
	e, _ := btc.tree.Seek(0)
	_, c, err := e.Next()
	for err != io.EOF {
		n += uint64(c.N())
		_, c, err = e.Next()
	}
	return
}

func (btc *BTreeContainers) Clone() roaring.Containers {
	nbtc := NewBTreeContainers()

	itr, err := btc.tree.SeekFirst()
	if err == io.EOF {
		return nbtc
	}
	for {
		k, v, err := itr.Next()
		if err == io.EOF {
			break
		}
		nbtc.tree.Set(k, v.Clone())
	}
	return nbtc
}

func (btc *BTreeContainers) Last() (key uint64, c *roaring.Container) {
	if btc.tree.Len() == 0 {
		return 0, nil
	}
	k, v := btc.tree.Last()
	return k, v
}

func (btc *BTreeContainers) Size() int {
	return btc.tree.Len()
}

func (btc *BTreeContainers) Reset() {
	btc.tree = TreeNew(cmp)
}

func (btc *BTreeContainers) Iterator(key uint64) (citer roaring.ContainerIterator, found bool) {
	e, ok := btc.tree.Seek(key)
	if ok {
		found = true
	}

	return &BTCIterator{
		e: e,
	}, found
}

type BTCIterator struct {
	e   *Enumerator
	key uint64
	val *roaring.Container
}

func (i *BTCIterator) Next() bool {

	k, v, err := i.e.Next()
	if err == io.EOF {
		return false
	}
	i.key = k
	i.val = v
	return true
}

func (i *BTCIterator) Value() (uint64, *roaring.Container) {
	if i.val == nil {
		return 0, nil
	}
	return i.key, i.val
}
<commit_msg>fixed reset method on btree plugin<commit_after>// Copyright (c) 2018 Pilosa Corp. All rights reserved.
//
// This file is part of Pilosa Enterprise Edition.
//
// Pilosa Enterprise Edition is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Pilosa Enterprise Edition is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with Pilosa Enterprise Edition.  If not, see <http://www.gnu.org/licenses/>.

package b

import (
	"io"

	"github.com/pilosa/pilosa/roaring"
)

func cmp(a, b uint64) int {
	return int(a - b)
}

type BTreeContainers struct {
	tree *Tree

	lastKey       uint64
	lastContainer *roaring.Container
}

func NewBTreeContainers() *BTreeContainers {
	return &BTreeContainers{
		tree: TreeNew(cmp),
	}
}

func NewBTreeBitmap(a ...uint64) *roaring.Bitmap {
	b := &roaring.Bitmap{
		Containers: NewBTreeContainers(),
	}
	b.Add(a...)
	return b
}

func (btc *BTreeContainers) Get(key uint64) *roaring.Container {
	// Check the last* cache for same container.
	if key == btc.lastKey && btc.lastContainer != nil {
		return btc.lastContainer
	}

	var c *roaring.Container
	el, ok := btc.tree.Get(key)
	if ok {
		c = el
		btc.lastKey = key
		btc.lastContainer = c
	}
	return c
}

func (btc *BTreeContainers) Put(key uint64, c *roaring.Container) {
	// If a mapped container is added to the tree, reset the
	// lastContainer cache so that the cache is not pointing
	// at a read-only mmap.
	if c.Mapped() {
		btc.lastContainer = nil
	}
	btc.tree.Set(key, c)
}

func (u updater) update(oldV *roaring.Container, exists bool) (*roaring.Container, bool) {
	// update the existing container
	if exists {
		oldV.Update(u.containerType, u.n, u.mapped)
		return oldV, false
	}
	cont := roaring.NewContainer()
	cont.Update(u.containerType, u.n, u.mapped)
	return cont, true
}

// this struct is added to prevent the closure locals from being escaped out to the heap
type updater struct {
	key           uint64
	containerType byte
	n             int
	mapped        bool
}

func (btc *BTreeContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) {
	a := updater{key, containerType, n, mapped}
	btc.tree.Put(key, a.update)
}

func (btc *BTreeContainers) Remove(key uint64) {
	btc.tree.Delete(key)
}

func (btc *BTreeContainers) GetOrCreate(key uint64) *roaring.Container {
	// Check the last* cache for same container.
	if key == btc.lastKey && btc.lastContainer != nil {
		return btc.lastContainer
	}

	btc.lastKey = key
	v, ok := btc.tree.Get(key)
	if !ok {
		cont := roaring.NewContainer()
		btc.tree.Set(key, cont)
		btc.lastContainer = cont
		return cont
	}

	btc.lastContainer = v
	return btc.lastContainer
}

func (btc *BTreeContainers) Count() (n uint64) {
	e, _ := btc.tree.Seek(0)
	_, c, err := e.Next()
	for err != io.EOF {
		n += uint64(c.N())
		_, c, err = e.Next()
	}
	return
}

func (btc *BTreeContainers) Clone() roaring.Containers {
	nbtc := NewBTreeContainers()

	itr, err := btc.tree.SeekFirst()
	if err == io.EOF {
		return nbtc
	}
	for {
		k, v, err := itr.Next()
		if err == io.EOF {
			break
		}
		nbtc.tree.Set(k, v.Clone())
	}
	return nbtc
}

func (btc *BTreeContainers) Last() (key uint64, c *roaring.Container) {
	if btc.tree.Len() == 0 {
		return 0, nil
	}
	k, v := btc.tree.Last()
	return k, v
}

func (btc *BTreeContainers) Size() int {
	return btc.tree.Len()
}

func (btc *BTreeContainers) Reset() {
	btc.tree = TreeNew(cmp)
	btc.lastKey = 0
	btc.lastContainer = nil
}

func (btc *BTreeContainers) Iterator(key uint64) (citer roaring.ContainerIterator, found bool) {
	e, ok := btc.tree.Seek(key)
	if ok {
		found = true
	}

	return &BTCIterator{
		e: e,
	}, found
}

type BTCIterator struct {
	e   *Enumerator
	key uint64
	val *roaring.Container
}

func (i *BTCIterator) Next() bool {

	k, v, err := i.e.Next()
	if err == io.EOF {
		return false
	}
	i.key = k
	i.val = v
	return true
}

func (i *BTCIterator) Value() (uint64, *roaring.Container) {
	if i.val == nil {
		return 0, nil
	}
	return i.key, i.val
}
<|endoftext|>