<commit_before>package structs_test

import (
	"encoding/json"
	"fmt"
	"github.com/fatih/structs"
	"github.com/mitchellh/mapstructure"
	"github.com/stretchr/testify/assert"
	"os"
	"strings"
	"testing"
)

type Book struct {
	Title  string
	Author string
	Year   int
}

const jsonData string = `{
	"name": "Enrique Vila-Matas",
	"nationality": "Spain",
	"books": [{
		"title": "El mal de Montano",
		"year": 2002
	}, {
		"title": "Paris no se acaba nunca"
	}, {
		"title": "Doctor Pasavento",
		"year": 2005,
		"publisher": "Anagrama"
	}, {
		"title": "Dublinesca",
		"year": null
	}]
}`

func Test(t *testing.T) {
	assert.Nil(t, nil)

	book := Book{"El mal de Montano", "E. Vila-Matas", 2002}
	assert.Equal(t, "E. Vila-Matas", book.Author)

	bookUpdated1 := Book{}
	getUpdatedBook(book, &bookUpdated1, `{"author": "Enrique Vila-Matas"}`)
	assert.Equal(t, "Enrique Vila-Matas", bookUpdated1.Author)

	bookUpdated2 := Book{}
	getUpdatedBook(bookUpdated1, &bookUpdated2, `{"publisher": "Anagrama"}`)
	assert.Equal(t, bookUpdated2, bookUpdated1)

	bookUpdated3 := Book{}
	getUpdatedBook(bookUpdated1, &bookUpdated3, `{"year": null}`)
	assert.Equal(t, bookUpdated3.Title, bookUpdated1.Title)
	assert.Equal(t, bookUpdated3.Author, bookUpdated1.Author)
	assert.Equal(t, bookUpdated3.Year, 0)
}

func getUpdatedBook(orig Book, dest *Book, changeJson string) {
	origMap := structs.Map(orig)
	fmt.Println("---")
	json.NewEncoder(os.Stdout).Encode(origMap)

	var changeMap map[string]interface{}
	json.Unmarshal([]byte(changeJson), &changeMap)
	json.NewEncoder(os.Stdout).Encode(changeMap)

	destMap := make(map[string]interface{})
	for key, value := range origMap {
		key := strings.ToLower(key)
		destMap[key] = value
	}
	for key, value := range changeMap {
		key := strings.ToLower(key)
		destMap[key] = value
	}

	mapstructure.Decode(destMap, dest)
	json.NewEncoder(os.Stdout).Encode(dest)
}
<commit_msg>updated<commit_after>package structs_test

import (
	"encoding/json"
	"fmt"
	"github.com/fatih/structs"
	"github.com/mitchellh/mapstructure"
	"github.com/stretchr/testify/assert"
	"os"
	"strings"
	"testing"
)

type Book struct {
	Title  string
	Author string
	Year   int
}

const jsonData string = `{
	"name": "Enrique Vila-Matas",
	"nationality": "Spain",
	"books": [{
		"title": "El mal de Montano",
		"year": 2002
	}, {
		"title": "Paris no se acaba nunca"
	}, {
		"title": "Doctor Pasavento",
		"year": 2005,
		"publisher": "Anagrama"
	}, {
		"title": "Dublinesca",
		"year": null
	}]
}`

func Test(t *testing.T) {
	assert.Nil(t, nil)

	book := Book{"El mal de Montano", "E. Vila-Matas", 2002}
	assert.Equal(t, "E. Vila-Matas", book.Author)

	bookUpdated1 := Book{}
	getUpdatedStruct(book, &bookUpdated1, `{"author": "Enrique Vila-Matas"}`)
	assert.Equal(t, "Enrique Vila-Matas", bookUpdated1.Author)

	bookUpdated2 := Book{}
	getUpdatedStruct(bookUpdated1, &bookUpdated2, `{"publisher": "Anagrama"}`)
	assert.Equal(t, bookUpdated2, bookUpdated1)

	bookUpdated3 := Book{}
	getUpdatedStruct(bookUpdated1, &bookUpdated3, `{"year": null}`)
	assert.Equal(t, bookUpdated3.Title, bookUpdated1.Title)
	assert.Equal(t, bookUpdated3.Author, bookUpdated1.Author)
	assert.Equal(t, bookUpdated3.Year, 0)

	bookUpdated4 := Book{}
	getUpdatedStruct(bookUpdated1, &bookUpdated4, `{"title": null}`)
	assert.Equal(t, bookUpdated4.Title, "")
	assert.Equal(t, bookUpdated4.Author, bookUpdated1.Author)
	assert.Equal(t, bookUpdated4.Year, bookUpdated1.Year)
}

func getUpdatedStruct(orig interface{}, dest interface{}, changeJson string) {
	origMap := structs.Map(orig)
	fmt.Println("---")
	json.NewEncoder(os.Stdout).Encode(origMap)

	var changeMap map[string]interface{}
	json.Unmarshal([]byte(changeJson), &changeMap)
	json.NewEncoder(os.Stdout).Encode(changeMap)

	destMap := make(map[string]interface{})
	for key, value := range origMap {
		key := strings.ToLower(key)
		destMap[key] = value
	}
	for key, value := range changeMap {
		key := strings.ToLower(key)
		destMap[key] = value
	}

	mapstructure.Decode(destMap, dest)
	json.NewEncoder(os.Stdout).Encode(dest)
}
<|endoftext|>


<commit_before>package vm_pool

import (
	"database/sql"
	"os"
	"path/filepath"
	"time"

	bosherr "github.com/cloudfoundry/bosh-utils/errors"
	boshlog "github.com/cloudfoundry/bosh-utils/logger"
	_ "github.com/mattn/go-sqlite3"
	common "github.com/maximilien/bosh-softlayer-cpi/common"
)

var (
	SQLITE_DB_FOLDER    string
	SQLITE_DB_FILE      string
	SQLITE_DB_FILE_PATH string
	DB_RETRY_INTERVAL   = 1 * time.Second
	DB_RETRY_TIMEOUT    = 30 * time.Second
)

func InitVMPoolDB(retryTimeout time.Duration, retryInterval time.Duration, logger boshlog.Logger) error {

	SQLITE_DB_FOLDER = common.GetOSEnvVariable("SQLITE_DB_FOLDER", "/var/vcap/store/director/")
	SQLITE_DB_FILE = common.GetOSEnvVariable("SQLITE_DB_FILE", "vm_pool.sqlite")
	SQLITE_DB_FILE_PATH = filepath.Join(SQLITE_DB_FOLDER, SQLITE_DB_FILE)

	err := os.MkdirAll(SQLITE_DB_FOLDER, os.ModePerm)
	if err != nil {
		return bosherr.WrapError(err, "Failed to make directory: "+SQLITE_DB_FOLDER)
	}

	db, err := OpenDB(SQLITE_DB_FILE_PATH)
	defer db.Close()
	if err != nil {
		return bosherr.WrapError(err, "Opening DB")
	}

	sqlStmt := `CREATE TABLE IF NOT EXISTS vms (id int not null primary key, name varchar(32), in_use varchar(32),
										  public_ip varchar(32), private_ip varchar(32), root_pwd varchar(32),
										  image_id varchar(64),
										  agent_id varchar(32),
										  timestamp timestamp)`
	err = exec(db, sqlStmt, retryTimeout, retryInterval, logger)
	if err != nil {
		return bosherr.WrapError(err, "Failed to execute sql statement: "+sqlStmt)
	}

	return nil
}

func OpenDB(dbPath string) (*sql.DB, error) {

	db, err := sql.Open("sqlite3", dbPath)
	if err != nil {
		return nil, bosherr.WrapError(err, "Failed to open VM Pool DB")
	}

	return db, nil
}
<commit_msg>Increase timeout<commit_after>package vm_pool

import (
	"database/sql"
	"os"
	"path/filepath"
	"time"

	bosherr "github.com/cloudfoundry/bosh-utils/errors"
	boshlog "github.com/cloudfoundry/bosh-utils/logger"
	_ "github.com/mattn/go-sqlite3"
	common "github.com/maximilien/bosh-softlayer-cpi/common"
)

var (
	SQLITE_DB_FOLDER    string
	SQLITE_DB_FILE      string
	SQLITE_DB_FILE_PATH string
	DB_RETRY_INTERVAL   = 3 * time.Second
	DB_RETRY_TIMEOUT    = 60 * time.Second
)

func InitVMPoolDB(retryTimeout time.Duration, retryInterval time.Duration, logger boshlog.Logger) error {

	SQLITE_DB_FOLDER = common.GetOSEnvVariable("SQLITE_DB_FOLDER", "/var/vcap/store/director/")
	SQLITE_DB_FILE = common.GetOSEnvVariable("SQLITE_DB_FILE", "vm_pool.sqlite")
	SQLITE_DB_FILE_PATH = filepath.Join(SQLITE_DB_FOLDER, SQLITE_DB_FILE)

	err := os.MkdirAll(SQLITE_DB_FOLDER, os.ModePerm)
	if err != nil {
		return bosherr.WrapError(err, "Failed to make directory: "+SQLITE_DB_FOLDER)
	}

	db, err := OpenDB(SQLITE_DB_FILE_PATH)
	defer db.Close()
	if err != nil {
		return bosherr.WrapError(err, "Opening DB")
	}

	sqlStmt := `CREATE TABLE IF NOT EXISTS vms (id int not null primary key, name varchar(32), in_use varchar(32),
										  public_ip varchar(32), private_ip varchar(32), root_pwd varchar(32),
										  image_id varchar(64),
										  agent_id varchar(32),
										  timestamp timestamp)`
	err = exec(db, sqlStmt, retryTimeout, retryInterval, logger)
	if err != nil {
		return bosherr.WrapError(err, "Failed to execute sql statement: "+sqlStmt)
	}

	return nil
}

func OpenDB(dbPath string) (*sql.DB, error) {

	db, err := sql.Open("sqlite3", dbPath)
	if err != nil {
		return nil, bosherr.WrapError(err, "Failed to open VM Pool DB")
	}

	return db, nil
}
<|endoftext|>


<commit_before>package gntagger

import (
	"fmt"
	"io/ioutil"
	"log"
	"math"

	"github.com/gnames/gnfinder"
	"github.com/gnames/gntagger/annotation"
)

// Names is an object that keeps output of a name finder and the path where to
// save this data on disk
type Names struct {
	// Path to json file with names
	Path string
	// Data is a gnfinder output
	Data gnfinder.Output
}

// NewNames uses a name finder or existing information to return Names structure
// generated from a text
func NewNames(text *Text, gnt *GnTagger) *Names {
	dict :=
		gnfinder.LoadDictionary()

	opts := []gnfinder.Opt{gnfinder.WithBayesThreshold(gnt.OddsLow)}
	if gnt.Bayes {
		opts = append(opts, gnfinder.WithBayes)
	}

	data := gnfinder.FindNames(text.Processed, &dict, opts...)

	for i := range data.Names {
		n := &data.Names[i]
		if n.Odds != 0.0 && n.Odds < gnt.OddsHigh {
			n.Annotation = annotation.Doubtful.String()
		}
	}
	return &Names{Data: data, Path: text.FilePath(NamesFile)}
}

// Save writes current state of names to file
func (n *Names) Save() error {
	json := n.Data.ToJSON()
	return ioutil.WriteFile(n.Path, json, 0644)
}

// NameStrings composes text to show in terminal gui
func NameStrings(n *gnfinder.Name, current bool, i int,
	total int) ([]string, error) {
	name := make([]string, 4)
	nameString := n.Name
	if current {
		nameString = fmt.Sprintf("\033[33;40;1m%s\033[0m", nameString)
	}
	name[0] = fmt.Sprintf("    %d/%d", i+1, total)
	name[1] = n.Type
	if n.Odds != 0.0 {
		name[1] = fmt.Sprintf("%s (Score: %0.2f)", name[1], math.Log10(n.Odds))
	}
	name[2] = fmt.Sprintf("Name: %s", nameString)
	ann, err := annotation.NewAnnotation(n.Annotation)
	if err != nil {
		return nil, err
	}
	name[3] = ann.Format()
	return name, nil
}

// NamesFromJSON creates gntagger's name structure from a finder output
func NamesFromJSON(path string) *Names {
	o := gnfinder.Output{}
	b, err := ioutil.ReadFile(path)
	if err != nil {
		log.Panicln(err)
	}
	o.FromJSON(b)
	return &Names{path, o}
}

// GetCurrentName returns currently selected name
func (n *Names) GetCurrentName() *gnfinder.Name {
	return &n.Data.Names[n.Data.Meta.CurrentName]
}

// UpdateAnnotations takes an annotation updates the current name with it.
// If needed, it propagates annotations further down the 'unseen' list.
func (n *Names) UpdateAnnotations(newAnnot annotation.Annotation, edge int,
	gnt *GnTagger) error {
	var (
		err      error
		oldAnnot annotation.Annotation
	)

	name := n.GetCurrentName()
	oldAnnot, err = annotation.NewAnnotation(name.Annotation)
	if err != nil {
		return err
	}

	name.Annotation = newAnnot.String()

	notAtEdge := n.Data.Meta.CurrentName < edge-3
	notAcceptedOrRejected := !newAnnot.In(annotation.NotName, annotation.Accepted)

	if notAtEdge || notAcceptedOrRejected {
		return nil
	}
	return tryToChangeNamesForward(n, oldAnnot, newAnnot, edge, gnt)
}

func tryToChangeNamesForward(n *Names, oldAnnot annotation.Annotation,
	newAnnot annotation.Annotation, edge int, gnt *GnTagger) error {
	for i := n.Data.Meta.CurrentName + 1; i < len(n.Data.Names); i++ {
		name := &n.Data.Names[i]
		sameName := n.GetCurrentName().Name == name.Name
		if !sameName {
			continue
		}

		if oldAnnot.In(annotation.NotAssigned, annotation.Doubtful) {
			name.Annotation = newAnnot.String()
		} else {
			nameAnnot, err := annotation.NewAnnotation(name.Annotation)
			if err != nil {
				return err
			}
			unmarkNames(name, nameAnnot, gnt)
		}
	}
	return nil
}

func unmarkNames(name *gnfinder.Name, a annotation.Annotation, gnt *GnTagger) {
	if IsDoubtful(name, gnt) {
		name.Annotation = annotation.Doubtful.String()
	} else {
		name.Annotation = annotation.NotAssigned.String()
	}
}

func IsDoubtful(n *gnfinder.Name, gnt *GnTagger) bool {
	return n.Odds != 0 && n.Odds < gnt.OddsHigh
}
<commit_msg>compatibility with gnfinder's c183679<commit_after>package gntagger

import (
	"fmt"
	"io/ioutil"
	"log"
	"math"

	"github.com/gnames/gnfinder"
	"github.com/gnames/gnfinder/dict"
	"github.com/gnames/gnfinder/util"
	"github.com/gnames/gntagger/annotation"
)

// Names is an object that keeps output of a name finder and the path where to
// save this data on disk
type Names struct {
	// Path to json file with names
	Path string
	// Data is a gnfinder output
	Data gnfinder.Output
}

// NewNames uses a name finder or existing information to return Names structure
// generated from a text
func NewNames(text *Text, gnt *GnTagger) *Names {
	dict :=
		dict.LoadDictionary()

	opts := []util.Opt{util.WithBayesThreshold(gnt.OddsLow)}
	if gnt.Bayes {
		opts = append(opts, util.WithBayes(true))
	}

	data := gnfinder.FindNames(text.Processed, &dict, opts...)

	for i := range data.Names {
		n := &data.Names[i]
		if n.Odds != 0.0 && n.Odds < gnt.OddsHigh {
			n.Annotation = annotation.Doubtful.String()
		}
	}
	return &Names{Data: data, Path: text.FilePath(NamesFile)}
}

// Save writes current state of names to file
func (n *Names) Save() error {
	json := n.Data.ToJSON()
	return ioutil.WriteFile(n.Path, json, 0644)
}

// NameStrings composes text to show in terminal gui
func NameStrings(n *gnfinder.Name, current bool, i int,
	total int) ([]string, error) {
	name := make([]string, 4)
	nameString := n.Name
	if current {
		nameString = fmt.Sprintf("\033[33;40;1m%s\033[0m", nameString)
	}
	name[0] = fmt.Sprintf("    %d/%d", i+1, total)
	name[1] = n.Type
	if n.Odds != 0.0 {
		name[1] = fmt.Sprintf("%s (Score: %0.2f)", name[1], math.Log10(n.Odds))
	}
	name[2] = fmt.Sprintf("Name: %s", nameString)
	ann, err := annotation.NewAnnotation(n.Annotation)
	if err != nil {
		return nil, err
	}
	name[3] = ann.Format()
	return name, nil
}

// NamesFromJSON creates gntagger's name structure from a finder output
func NamesFromJSON(path string) *Names {
	o := gnfinder.Output{}
	b, err := ioutil.ReadFile(path)
	if err != nil {
		log.Panicln(err)
	}
	o.FromJSON(b)
	return &Names{path, o}
}

// GetCurrentName returns currently selected name
func (n *Names) GetCurrentName() *gnfinder.Name {
	return &n.Data.Names[n.Data.Meta.CurrentName]
}

// UpdateAnnotations takes an annotation updates the current name with it.
// If needed, it propagates annotations further down the 'unseen' list.
func (n *Names) UpdateAnnotations(newAnnot annotation.Annotation, edge int,
	gnt *GnTagger) error {
	var (
		err      error
		oldAnnot annotation.Annotation
	)

	name := n.GetCurrentName()
	oldAnnot, err = annotation.NewAnnotation(name.Annotation)
	if err != nil {
		return err
	}

	name.Annotation = newAnnot.String()

	notAtEdge := n.Data.Meta.CurrentName < edge-3
	notAcceptedOrRejected := !newAnnot.In(annotation.NotName, annotation.Accepted)

	if notAtEdge || notAcceptedOrRejected {
		return nil
	}
	return tryToChangeNamesForward(n, oldAnnot, newAnnot, edge, gnt)
}

func tryToChangeNamesForward(n *Names, oldAnnot annotation.Annotation,
	newAnnot annotation.Annotation, edge int, gnt *GnTagger) error {
	for i := n.Data.Meta.CurrentName + 1; i < len(n.Data.Names); i++ {
		name := &n.Data.Names[i]
		sameName := n.GetCurrentName().Name == name.Name
		if !sameName {
			continue
		}

		if oldAnnot.In(annotation.NotAssigned, annotation.Doubtful) {
			name.Annotation = newAnnot.String()
		} else {
			nameAnnot, err := annotation.NewAnnotation(name.Annotation)
			if err != nil {
				return err
			}
			unmarkNames(name, nameAnnot, gnt)
		}
	}
	return nil
}

func unmarkNames(name *gnfinder.Name, a annotation.Annotation, gnt *GnTagger) {
	if IsDoubtful(name, gnt) {
		name.Annotation = annotation.Doubtful.String()
	} else {
		name.Annotation = annotation.NotAssigned.String()
	}
}

func IsDoubtful(n *gnfinder.Name, gnt *GnTagger) bool {
	return n.Odds != 0 && n.Odds < gnt.OddsHigh
}
<|endoftext|>


<commit_before>package fuse

// Written with a look to http://ptspts.blogspot.com/2009/11/fuse-protocol-tutorial-for-linux-26.html

import (
	"fmt"
	"net"
	"os"
	"path"
	"syscall"
	"unsafe"
)

type FileSystem interface {
}

type MountPoint struct {
	mountPoint string
	f *os.File
}

// Mount create a fuse fs on the specified mount point.
func Mount(mountPoint string, fs FileSystem) (m *MountPoint, err os.Error) {
	local, remote, err := net.Socketpair("unixgram")
	if err != nil {
		return
	}

	defer local.Close()
	defer remote.Close()

	fmt.Printf("Mount: 20\n")
	mountPoint = path.Clean(mountPoint)
	if !path.Rooted(mountPoint) {
		cwd, err := os.Getwd()
		if err != nil {
			return
		}
		mountPoint = path.Clean(path.Join(cwd, mountPoint))
	}
	fmt.Printf("Mount: 40\n")
	pid, err := os.ForkExec("/bin/fusermount",
			[]string { "/bin/fusermount", mountPoint },
			[]string { "_FUSE_COMMFD=3" },
			"",
			[]*os.File { nil, nil, os.Stderr, remote.File() })
	if err != nil {
		return
	}
	w, err := os.Wait(pid, 0)
	if err != nil {
		return
	}
	if w.ExitStatus() != 0 {
		return nil, os.NewError(fmt.Sprintf("fusermount exited with code %d\n", w.ExitStatus()))
	}
	fmt.Printf("Mount: 100\n")
	f, err := getFuseConn(local)
	fmt.Printf("Mount: 110\n")
	if err != nil {
		return
	}
	m = &MountPoint { mountPoint, f }
	fmt.Printf("I'm here!!\n")
	return
}

func (m *MountPoint) Unmount() (err os.Error) {
	if m == nil {
		return nil
	}
	pid, err := os.ForkExec("/bin/fusermount",
			[]string { "/bin/fusermount", "-u", m.mountPoint },
			nil,
			"",
			[]*os.File { nil, nil, os.Stderr })
	if err != nil {
		return
	}
	w, err := os.Wait(pid, 0)
	if err != nil {
		return
	}
	if w.ExitStatus() != 0 {
		return os.NewError(fmt.Sprintf("fusermount exited with code %d\n", w.ExitStatus()))
	}
	m.f.Close()
	return
}

func recvmsg(fd int, msg *msghdr, flags int) (n int, errno int) {
	n1, _, e1 := syscall.Syscall(syscall.SYS_RECVMSG, uintptr(fd), uintptr(unsafe.Pointer(msg)), uintptr(flags))
	n = int(n1)
	errno = int(e1)
	return
}

type msghdr struct {
	Name       uintptr
	Namelen    uintptr
	Iov        uintptr
	Iovlen     uintptr
	Control    uintptr
	Controllen uintptr
	Flags      uintptr
}

type iovec struct {
	Base uintptr
	Len  uintptr
}

func Recvmsg(fd int, msg *msghdr, flags int) (n int, err os.Error) {
	fmt.Printf("Recvmsg, 0\n")
	n, errno := recvmsg(fd, msg, flags)
	fmt.Printf("Recvmsg, 10\n")
	if errno != 0 {
		err = os.NewSyscallError("recvmsg", errno)
	}
	return
}

func getFuseConn(local net.Conn) (f * os.File, err os.Error) {
	var msg msghdr
	var iov iovec
	base := make([]int32, 256)
	control := make([]int32, 256)

	iov.Base = uintptr(unsafe.Pointer(&base[0]))
	iov.Len = uintptr(len(base) * 4)
	msg.Iov = uintptr(unsafe.Pointer(&iov))
	msg.Iovlen = 1
	msg.Control = uintptr(unsafe.Pointer(&control[0]))
	msg.Controllen = uintptr(len(control) * 4)

	_, err = Recvmsg(local.File().Fd(), &msg, 0)
	fmt.Printf("getFuseConn: 100\n")
	if err != nil {
		return
	}
	fmt.Printf("getFuseConn: 110\n")

	length := control[0]
	fmt.Printf("getFuseConn: 120\n")
	typ := control[2] // syscall.Cmsghdr.Type
	fd := control[4]
	fmt.Printf("getFuseConn: 130\n")
	if typ != 1 {
		err = os.NewError(fmt.Sprintf("getFuseConn: recvmsg returned wrong control type: %d", typ))
		return
	}
	if length < 20 {
		err = os.NewError(fmt.Sprintf("getFuseConn: too short control message. Length: %d", length))
		return
	}

	if (fd < 0) {
		err = os.NewError(fmt.Sprintf("getFuseConn: fd < 0: %d", fd))
		return
	}

	fmt.Printf("getFuseConn: 180\n")
	f = os.NewFile(int(fd), "fuse-conn")
	return
}

<commit_msg>Use syscall.Iovec. Test PASS<commit_after>package fuse

// Written with a look to http://ptspts.blogspot.com/2009/11/fuse-protocol-tutorial-for-linux-26.html

import (
	"fmt"
	"net"
	"os"
	"path"
	"syscall"
	"unsafe"
)

type FileSystem interface {
}

type MountPoint struct {
	mountPoint string
	f *os.File
}

// Mount create a fuse fs on the specified mount point.
func Mount(mountPoint string, fs FileSystem) (m *MountPoint, err os.Error) {
	local, remote, err := net.Socketpair("unixgram")
	if err != nil {
		return
	}

	defer local.Close()
	defer remote.Close()

	fmt.Printf("Mount: 20\n")
	mountPoint = path.Clean(mountPoint)
	if !path.Rooted(mountPoint) {
		cwd, err := os.Getwd()
		if err != nil {
			return
		}
		mountPoint = path.Clean(path.Join(cwd, mountPoint))
	}
	fmt.Printf("Mount: 40\n")
	pid, err := os.ForkExec("/bin/fusermount",
			[]string { "/bin/fusermount", mountPoint },
			[]string { "_FUSE_COMMFD=3" },
			"",
			[]*os.File { nil, nil, os.Stderr, remote.File() })
	if err != nil {
		return
	}
	w, err := os.Wait(pid, 0)
	if err != nil {
		return
	}
	if w.ExitStatus() != 0 {
		return nil, os.NewError(fmt.Sprintf("fusermount exited with code %d\n", w.ExitStatus()))
	}
	fmt.Printf("Mount: 100\n")
	f, err := getFuseConn(local)
	fmt.Printf("Mount: 110\n")
	if err != nil {
		return
	}
	m = &MountPoint { mountPoint, f }
	fmt.Printf("I'm here!!\n")
	return
}

func (m *MountPoint) Unmount() (err os.Error) {
	if m == nil {
		return nil
	}
	pid, err := os.ForkExec("/bin/fusermount",
			[]string { "/bin/fusermount", "-u", m.mountPoint },
			nil,
			"",
			[]*os.File { nil, nil, os.Stderr })
	if err != nil {
		return
	}
	w, err := os.Wait(pid, 0)
	if err != nil {
		return
	}
	if w.ExitStatus() != 0 {
		return os.NewError(fmt.Sprintf("fusermount exited with code %d\n", w.ExitStatus()))
	}
	m.f.Close()
	return
}

func recvmsg(fd int, msg *msghdr, flags int) (n int, errno int) {
	n1, _, e1 := syscall.Syscall(syscall.SYS_RECVMSG, uintptr(fd), uintptr(unsafe.Pointer(msg)), uintptr(flags))
	n = int(n1)
	errno = int(e1)
	return
}

type msghdr struct {
	Name       uintptr
	Namelen    uintptr
	Iov        uintptr
	Iovlen     uintptr
	Control    uintptr
	Controllen uintptr
	Flags      uintptr
}

func Recvmsg(fd int, msg *msghdr, flags int) (n int, err os.Error) {
	fmt.Printf("Recvmsg, 0\n")
	n, errno := recvmsg(fd, msg, flags)
	fmt.Printf("Recvmsg, 10\n")
	if errno != 0 {
		err = os.NewSyscallError("recvmsg", errno)
	}
	return
}

func getFuseConn(local net.Conn) (f * os.File, err os.Error) {
	var msg msghdr
	var iov syscall.Iovec
	base := make([]int32, 256)
	control := make([]int32, 256)

	iov.Base = (*byte)(unsafe.Pointer(&base[0]))
	iov.Len = uint64(len(base) * 4)
	msg.Iov = uintptr(unsafe.Pointer(&iov))
	msg.Iovlen = 1
	msg.Control = uintptr(unsafe.Pointer(&control[0]))
	msg.Controllen = uintptr(len(control) * 4)

	_, err = Recvmsg(local.File().Fd(), &msg, 0)
	fmt.Printf("getFuseConn: 100\n")
	if err != nil {
		return
	}
	fmt.Printf("getFuseConn: 110\n")

	length := control[0]
	fmt.Printf("getFuseConn: 120\n")
	typ := control[2] // syscall.Cmsghdr.Type
	fd := control[4]
	fmt.Printf("getFuseConn: 130\n")
	if typ != 1 {
		err = os.NewError(fmt.Sprintf("getFuseConn: recvmsg returned wrong control type: %d", typ))
		return
	}
	if length < 20 {
		err = os.NewError(fmt.Sprintf("getFuseConn: too short control message. Length: %d", length))
		return
	}

	if (fd < 0) {
		err = os.NewError(fmt.Sprintf("getFuseConn: fd < 0: %d", fd))
		return
	}
	fmt.Printf("fd: %d\n", fd)

	fmt.Printf("getFuseConn: 180\n")
	f = os.NewFile(int(fd), "fuse-conn")
	return
}

<|endoftext|>


<commit_before>package e2etest

import (
	"context"
	"fmt"
	"testing"

	"github.com/hashicorp/go-version"
	tfjson "github.com/hashicorp/terraform-json"
	"github.com/zclconf/go-cty/cty"

	"github.com/hashicorp/terraform-exec/tfexec"
)

var (
	providersSchemaJSONMinVersion = version.Must(version.NewVersion("0.12.0"))
)

func TestProvidersSchema(t *testing.T) {
	for i, c := range []struct {
		fixtureDir string
		expected   func(*version.Version) *tfjson.ProviderSchemas
	}{
		{
			"basic", func(tfv *version.Version) *tfjson.ProviderSchemas {
				var providerSchema *tfjson.ProviderSchemas

				if tfv.GreaterThanOrEqual(version.Must(version.NewVersion("0.15.0"))) {
					providerSchema = &tfjson.ProviderSchemas{
						FormatVersion: "0.2",
						Schemas: map[string]*tfjson.ProviderSchema{
							"registry.terraform.io/hashicorp/null": {
								ConfigSchema: &tfjson.Schema{
									Version: 0,
									Block: &tfjson.SchemaBlock{
										DescriptionKind: "plain",
									},
								},
								ResourceSchemas: map[string]*tfjson.Schema{
									"null_resource": {
										Version: 0,
										Block: &tfjson.SchemaBlock{
											DescriptionKind: "markdown",
											Description:     "The `null_resource` resource implements the standard resource lifecycle but takes no further action.\n\nThe `triggers` argument allows specifying an arbitrary set of values that, when changed, will cause the resource to be replaced.",
											Attributes: map[string]*tfjson.SchemaAttribute{
												"id": {
													AttributeType:   cty.String,
													Optional:        false,
													Computed:        true,
													DescriptionKind: "markdown",
													Description:     "This is set to a random value at create time.",
												},
												"triggers": {
													AttributeType:   cty.Map(cty.String),
													Optional:        true,
													DescriptionKind: "markdown",
													Description:     "A map of arbitrary strings that, when changed, will force the null resource to be replaced, re-running any associated provisioners.",
												},
											},
										},
									},
								},
								DataSourceSchemas: map[string]*tfjson.Schema{
									"null_data_source": {
										Version: 0,
										Block: &tfjson.SchemaBlock{
											DescriptionKind: "markdown",
											Description: `The ` + "`null_data_source`" + ` data source implements the standard data source lifecycle but does not
interact with any external APIs.

Historically, the ` + "`null_data_source`" + ` was typically used to construct intermediate values to re-use elsewhere in configuration. The
same can now be achieved using [locals](https://www.terraform.io/docs/language/values/locals.html).
`,
											Deprecated: true,
											Attributes: map[string]*tfjson.SchemaAttribute{
												"has_computed_default": {
													AttributeType:   cty.String,
													Optional:        true,
													Computed:        true,
													DescriptionKind: "markdown",
													Description:     "If set, its literal value will be stored and returned. If not, its value defaults to `\"default\"`. This argument exists primarily for testing and has little practical use.",
												},
												"id": {
													AttributeType:   cty.String,
													Optional:        false,
													Computed:        true,
													DescriptionKind: "markdown",
													Description:     "This attribute is only present for some legacy compatibility issues and should not be used. It will be removed in a future version.",
													Deprecated:      true,
												},
												"inputs": {
													AttributeType:   cty.Map(cty.String),
													Optional:        true,
													DescriptionKind: "markdown",
													Description:     "A map of arbitrary strings that is copied into the `outputs` attribute, and accessible directly for interpolation.",
												},
												"outputs": {
													AttributeType:   cty.Map(cty.String),
													Computed:        true,
													DescriptionKind: "markdown",
													Description:     "After the data source is \"read\", a copy of the `inputs` map.",
												},
												"random": {
													AttributeType:   cty.String,
													Computed:        true,
													DescriptionKind: "markdown",
													Description:     "A random value. This is primarily for testing and has little practical use; prefer the [hashicorp/random provider](https://registry.terraform.io/providers/hashicorp/random) for more practical random number use-cases.",
												},
											},
										},
									},
								},
							},
						},
					}
				} else if tfv.GreaterThanOrEqual(version.Must(version.NewVersion("0.13.0"))) {
					providerSchema = &tfjson.ProviderSchemas{
						FormatVersion: "0.1",
						Schemas: map[string]*tfjson.ProviderSchema{
							"registry.terraform.io/hashicorp/null": {
								ConfigSchema: &tfjson.Schema{
									Version: 0,
									Block: &tfjson.SchemaBlock{
										DescriptionKind: "plain",
									},
								},
								ResourceSchemas: map[string]*tfjson.Schema{
									"null_resource": {
										Version: 0,
										Block: &tfjson.SchemaBlock{
											DescriptionKind: "markdown",
											Description:     "The `null_resource` resource implements the standard resource lifecycle but takes no further action.\n\nThe `triggers` argument allows specifying an arbitrary set of values that, when changed, will cause the resource to be replaced.",
											Attributes: map[string]*tfjson.SchemaAttribute{
												"id": {
													AttributeType:   cty.String,
													Optional:        false,
													Computed:        true,
													DescriptionKind: "markdown",
													Description:     "This is set to a random value at create time.",
												},
												"triggers": {
													AttributeType:   cty.Map(cty.String),
													Optional:        true,
													DescriptionKind: "markdown",
													Description:     "A map of arbitrary strings that, when changed, will force the null resource to be replaced, re-running any associated provisioners.",
												},
											},
										},
									},
								},
								DataSourceSchemas: map[string]*tfjson.Schema{
									"null_data_source": {
										Version: 0,
										Block: &tfjson.SchemaBlock{
											DescriptionKind: "markdown",
											Description: `The ` + "`null_data_source`" + ` data source implements the standard data source lifecycle but does not
interact with any external APIs.

Historically, the ` + "`null_data_source`" + ` was typically used to construct intermediate values to re-use elsewhere in configuration. The
same can now be achieved using [locals](https://www.terraform.io/docs/language/values/locals.html).
`,
											Deprecated: true,
											Attributes: map[string]*tfjson.SchemaAttribute{
												"has_computed_default": {
													AttributeType:   cty.String,
													Optional:        true,
													Computed:        true,
													DescriptionKind: "markdown",
													Description:     "If set, its literal value will be stored and returned. If not, its value defaults to `\"default\"`. This argument exists primarily for testing and has little practical use.",
												},
												"id": {
													AttributeType:   cty.String,
													Optional:        false,
													Computed:        true,
													DescriptionKind: "markdown",
													Description:     "This attribute is only present for some legacy compatibility issues and should not be used. It will be removed in a future version.",
													Deprecated:      true,
												},
												"inputs": {
													AttributeType:   cty.Map(cty.String),
													Optional:        true,
													DescriptionKind: "markdown",
													Description:     "A map of arbitrary strings that is copied into the `outputs` attribute, and accessible directly for interpolation.",
												},
												"outputs": {
													AttributeType:   cty.Map(cty.String),
													Computed:        true,
													DescriptionKind: "markdown",
													Description:     "After the data source is \"read\", a copy of the `inputs` map.",
												},
												"random": {
													AttributeType:   cty.String,
													Computed:        true,
													DescriptionKind: "markdown",
													Description:     "A random value. This is primarily for testing and has little practical use; prefer the [hashicorp/random provider](https://registry.terraform.io/providers/hashicorp/random) for more practical random number use-cases.",
												},
											},
										},
									},
								},
							},
						},
					}
				} else {
					providerSchema = &tfjson.ProviderSchemas{
						FormatVersion: "0.1",
						Schemas: map[string]*tfjson.ProviderSchema{
							"null": {
								ConfigSchema: &tfjson.Schema{
									Version: 0,
									Block:   &tfjson.SchemaBlock{},
								},
								ResourceSchemas: map[string]*tfjson.Schema{
									"null_resource": {
										Version: 0,
										Block: &tfjson.SchemaBlock{
											DescriptionKind: "",
											Attributes: map[string]*tfjson.SchemaAttribute{
												"id": {
													AttributeType: cty.String,
													Optional:      false,
													Computed:      true,
													Description:   "This is set to a random value at create time.",
												},
												"triggers": {
													AttributeType:   cty.Map(cty.String),
													Optional:        true,
													DescriptionKind: "",
													Description:     "A map of arbitrary strings that, when changed, will force the null resource to be replaced, re-running any associated provisioners.",
												},
											},
										},
									},
								},
								DataSourceSchemas: map[string]*tfjson.Schema{
									"null_data_source": {
										Version: 0,
										Block: &tfjson.SchemaBlock{
											DescriptionKind: "",
											Description:     "",
											Attributes: map[string]*tfjson.SchemaAttribute{
												"has_computed_default": {
													AttributeType:   cty.String,
													Optional:        true,
													Computed:        true,
													DescriptionKind: "",
													Description:     "If set, its literal value will be stored and returned. If not, its value defaults to `\"default\"`. This argument exists primarily for testing and has little practical use.",
												},
												"id": {
													AttributeType:   cty.String,
													Optional:        false,
													Computed:        true,
													DescriptionKind: "",
													Description:     "This attribute is only present for some legacy compatibility issues and should not be used. It will be removed in a future version.",
												},
												"inputs": {
													AttributeType:   cty.Map(cty.String),
													Optional:        true,
													DescriptionKind: "",
													Description:     "A map of arbitrary strings that is copied into the `outputs` attribute, and accessible directly for interpolation.",
												},
												"outputs": {
													AttributeType:   cty.Map(cty.String),
													Computed:        true,
													DescriptionKind: "",
													Description:     "After the data source is \"read\", a copy of the `inputs` map.",
												},
												"random": {
													AttributeType:   cty.String,
													Computed:        true,
													DescriptionKind: "",
													Description:     "A random value. This is primarily for testing and has little practical use; prefer the [hashicorp/random provider](https://registry.terraform.io/providers/hashicorp/random) for more practical random number use-cases.",
												},
											},
										},
									},
								},
							},
						},
					}
				}

				return providerSchema
			},
		},
		{
			"empty_with_tf_file", func(tfv *version.Version) *tfjson.ProviderSchemas {
				if tfv.GreaterThanOrEqual(version.Must(version.NewVersion("0.15.0"))) {
					return &tfjson.ProviderSchemas{
						FormatVersion: "0.2",
						Schemas:       nil,
					}
				}

				return &tfjson.ProviderSchemas{
					FormatVersion: "0.1",
					Schemas:       nil,
				}
			},
		},
	} {
		c := c
		t.Run(fmt.Sprintf("%d %s", i, c.fixtureDir), func(t *testing.T) {
			runTest(t, c.fixtureDir, func(t *testing.T, tfv *version.Version, tf *tfexec.Terraform) {
				if tfv.LessThan(providersSchemaJSONMinVersion) {
					t.Skip("providers schema -json was added in 0.12")
				}

				err := tf.Init(context.Background())
				if err != nil {
					t.Fatalf("error running Init in test directory: %s", err)
				}

				schemas, err := tf.ProvidersSchema(context.Background())
				if err != nil {
					t.Fatalf("error running ProvidersSchema in test directory: %s", err)
				}

				expected := c.expected(tfv)

				if diff := diffSchema(expected, schemas); diff != "" {
					t.Fatalf("mismatch (-want +got):\n%s", diff)
				}
			})
		})
	}

}

func TestProvidersSchema_versionMismatch(t *testing.T) {
	t.Skip("TODO! add version mismatch test for 0.11 as -json was added in 0.12 (I think)")
}
<commit_msg>hack: fix provider schema test in 0.15 prereleases<commit_after>package e2etest

import (
	"context"
	"fmt"
	"testing"

	"github.com/hashicorp/go-version"
	tfjson "github.com/hashicorp/terraform-json"
	"github.com/zclconf/go-cty/cty"

	"github.com/hashicorp/terraform-exec/tfexec"
)

var (
	providersSchemaJSONMinVersion = version.Must(version.NewVersion("0.12.0"))
)

func TestProvidersSchema(t *testing.T) {
	for i, c := range []struct {
		fixtureDir string
		expected   func(*version.Version) *tfjson.ProviderSchemas
	}{
		{
			"basic", func(tfv *version.Version) *tfjson.ProviderSchemas {
				var providerSchema *tfjson.ProviderSchemas

				if tfv.GreaterThanOrEqual(version.Must(version.NewVersion("0.15.0"))) || tfv.Equal(version.Must(version.NewVersion("0.15.0-beta2"))) || tfv.Equal(version.Must(version.NewVersion("0.15.0-dev"))) {
					providerSchema = &tfjson.ProviderSchemas{
						FormatVersion: "0.2",
						Schemas: map[string]*tfjson.ProviderSchema{
							"registry.terraform.io/hashicorp/null": {
								ConfigSchema: &tfjson.Schema{
									Version: 0,
									Block: &tfjson.SchemaBlock{
										DescriptionKind: "plain",
									},
								},
								ResourceSchemas: map[string]*tfjson.Schema{
									"null_resource": {
										Version: 0,
										Block: &tfjson.SchemaBlock{
											DescriptionKind: "markdown",
											Description:     "The `null_resource` resource implements the standard resource lifecycle but takes no further action.\n\nThe `triggers` argument allows specifying an arbitrary set of values that, when changed, will cause the resource to be replaced.",
											Attributes: map[string]*tfjson.SchemaAttribute{
												"id": {
													AttributeType:   cty.String,
													Optional:        false,
													Computed:        true,
													DescriptionKind: "markdown",
													Description:     "This is set to a random value at create time.",
												},
												"triggers": {
													AttributeType:   cty.Map(cty.String),
													Optional:        true,
													DescriptionKind: "markdown",
													Description:     "A map of arbitrary strings that, when changed, will force the null resource to be replaced, re-running any associated provisioners.",
												},
											},
										},
									},
								},
								DataSourceSchemas: map[string]*tfjson.Schema{
									"null_data_source": {
										Version: 0,
										Block: &tfjson.SchemaBlock{
											DescriptionKind: "markdown",
											Description: `The ` + "`null_data_source`" + ` data source implements the standard data source lifecycle but does not
interact with any external APIs.

Historically, the ` + "`null_data_source`" + ` was typically used to construct intermediate values to re-use elsewhere in configuration. The
same can now be achieved using [locals](https://www.terraform.io/docs/language/values/locals.html).
`,
											Deprecated: true,
											Attributes: map[string]*tfjson.SchemaAttribute{
												"has_computed_default": {
													AttributeType:   cty.String,
													Optional:        true,
													Computed:        true,
													DescriptionKind: "markdown",
													Description:     "If set, its literal value will be stored and returned. If not, its value defaults to `\"default\"`. This argument exists primarily for testing and has little practical use.",
												},
												"id": {
													AttributeType:   cty.String,
													Optional:        false,
													Computed:        true,
													DescriptionKind: "markdown",
													Description:     "This attribute is only present for some legacy compatibility issues and should not be used. It will be removed in a future version.",
													Deprecated:      true,
												},
												"inputs": {
													AttributeType:   cty.Map(cty.String),
													Optional:        true,
													DescriptionKind: "markdown",
													Description:     "A map of arbitrary strings that is copied into the `outputs` attribute, and accessible directly for interpolation.",
												},
												"outputs": {
													AttributeType:   cty.Map(cty.String),
													Computed:        true,
													DescriptionKind: "markdown",
													Description:     "After the data source is \"read\", a copy of the `inputs` map.",
												},
												"random": {
													AttributeType:   cty.String,
													Computed:        true,
													DescriptionKind: "markdown",
													Description:     "A random value. This is primarily for testing and has little practical use; prefer the [hashicorp/random provider](https://registry.terraform.io/providers/hashicorp/random) for more practical random number use-cases.",
												},
											},
										},
									},
								},
							},
						},
					}
				} else if tfv.GreaterThanOrEqual(version.Must(version.NewVersion("0.13.0"))) {
					providerSchema = &tfjson.ProviderSchemas{
						FormatVersion: "0.1",
						Schemas: map[string]*tfjson.ProviderSchema{
							"registry.terraform.io/hashicorp/null": {
								ConfigSchema: &tfjson.Schema{
									Version: 0,
									Block: &tfjson.SchemaBlock{
										DescriptionKind: "plain",
									},
								},
								ResourceSchemas: map[string]*tfjson.Schema{
									"null_resource": {
										Version: 0,
										Block: &tfjson.SchemaBlock{
											DescriptionKind: "markdown",
											Description:     "The `null_resource` resource implements the standard resource lifecycle but takes no further action.\n\nThe `triggers` argument allows specifying an arbitrary set of values that, when changed, will cause the resource to be replaced.",
											Attributes: map[string]*tfjson.SchemaAttribute{
												"id": {
													AttributeType:   cty.String,
													Optional:        false,
													Computed:        true,
													DescriptionKind: "markdown",
													Description:     "This is set to a random value at create time.",
												},
												"triggers": {
													AttributeType:   cty.Map(cty.String),
													Optional:        true,
													DescriptionKind: "markdown",
													Description:     "A map of arbitrary strings that, when changed, will force the null resource to be replaced, re-running any associated provisioners.",
												},
											},
										},
									},
								},
								DataSourceSchemas: map[string]*tfjson.Schema{
									"null_data_source": {
										Version: 0,
										Block: &tfjson.SchemaBlock{
											DescriptionKind: "markdown",
											Description: `The ` + "`null_data_source`" + ` data source implements the standard data source lifecycle but does not
interact with any external APIs.

Historically, the ` + "`null_data_source`" + ` was typically used to construct intermediate values to re-use elsewhere in configuration. The
same can now be achieved using [locals](https://www.terraform.io/docs/language/values/locals.html).
`,
											Deprecated: true,
											Attributes: map[string]*tfjson.SchemaAttribute{
												"has_computed_default": {
													AttributeType:   cty.String,
													Optional:        true,
													Computed:        true,
													DescriptionKind: "markdown",
													Description:     "If set, its literal value will be stored and returned. If not, its value defaults to `\"default\"`. This argument exists primarily for testing and has little practical use.",
												},
												"id": {
													AttributeType:   cty.String,
													Optional:        false,
													Computed:        true,
													DescriptionKind: "markdown",
													Description:     "This attribute is only present for some legacy compatibility issues and should not be used. It will be removed in a future version.",
													Deprecated:      true,
												},
												"inputs": {
													AttributeType:   cty.Map(cty.String),
													Optional:        true,
													DescriptionKind: "markdown",
													Description:     "A map of arbitrary strings that is copied into the `outputs` attribute, and accessible directly for interpolation.",
												},
												"outputs": {
													AttributeType:   cty.Map(cty.String),
													Computed:        true,
													DescriptionKind: "markdown",
													Description:     "After the data source is \"read\", a copy of the `inputs` map.",
												},
												"random": {
													AttributeType:   cty.String,
													Computed:        true,
													DescriptionKind: "markdown",
													Description:     "A random value. This is primarily for testing and has little practical use; prefer the [hashicorp/random provider](https://registry.terraform.io/providers/hashicorp/random) for more practical random number use-cases.",
												},
											},
										},
									},
								},
							},
						},
					}
				} else {
					providerSchema = &tfjson.ProviderSchemas{
						FormatVersion: "0.1",
						Schemas: map[string]*tfjson.ProviderSchema{
							"null": {
								ConfigSchema: &tfjson.Schema{
									Version: 0,
									Block:   &tfjson.SchemaBlock{},
								},
								ResourceSchemas: map[string]*tfjson.Schema{
									"null_resource": {
										Version: 0,
										Block: &tfjson.SchemaBlock{
											DescriptionKind: "",
											Attributes: map[string]*tfjson.SchemaAttribute{
												"id": {
													AttributeType: cty.String,
													Optional:      false,
													Computed:      true,
													Description:   "This is set to a random value at create time.",
												},
												"triggers": {
													AttributeType:   cty.Map(cty.String),
													Optional:        true,
													DescriptionKind: "",
													Description:     "A map of arbitrary strings that, when changed, will force the null resource to be replaced, re-running any associated provisioners.",
												},
											},
										},
									},
								},
								DataSourceSchemas: map[string]*tfjson.Schema{
									"null_data_source": {
										Version: 0,
										Block: &tfjson.SchemaBlock{
											DescriptionKind: "",
											Description:     "",
											Attributes: map[string]*tfjson.SchemaAttribute{
												"has_computed_default": {
													AttributeType:   cty.String,
													Optional:        true,
													Computed:        true,
													DescriptionKind: "",
													Description:     "If set, its literal value will be stored and returned. If not, its value defaults to `\"default\"`. This argument exists primarily for testing and has little practical use.",
												},
												"id": {
													AttributeType:   cty.String,
													Optional:        false,
													Computed:        true,
													DescriptionKind: "",
													Description:     "This attribute is only present for some legacy compatibility issues and should not be used. It will be removed in a future version.",
												},
												"inputs": {
													AttributeType:   cty.Map(cty.String),
													Optional:        true,
													DescriptionKind: "",
													Description:     "A map of arbitrary strings that is copied into the `outputs` attribute, and accessible directly for interpolation.",
												},
												"outputs": {
													AttributeType:   cty.Map(cty.String),
													Computed:        true,
													DescriptionKind: "",
													Description:     "After the data source is \"read\", a copy of the `inputs` map.",
												},
												"random": {
													AttributeType:   cty.String,
													Computed:        true,
													DescriptionKind: "",
													Description:     "A random value. This is primarily for testing and has little practical use; prefer the [hashicorp/random provider](https://registry.terraform.io/providers/hashicorp/random) for more practical random number use-cases.",
												},
											},
										},
									},
								},
							},
						},
					}
				}

				return providerSchema
			},
		},
		{
			"empty_with_tf_file", func(tfv *version.Version) *tfjson.ProviderSchemas {
				if tfv.GreaterThanOrEqual(version.Must(version.NewVersion("0.15.0"))) {
					return &tfjson.ProviderSchemas{
						FormatVersion: "0.2",
						Schemas:       nil,
					}
				}

				return &tfjson.ProviderSchemas{
					FormatVersion: "0.1",
					Schemas:       nil,
				}
			},
		},
	} {
		c := c
		t.Run(fmt.Sprintf("%d %s", i, c.fixtureDir), func(t *testing.T) {
			runTest(t, c.fixtureDir, func(t *testing.T, tfv *version.Version, tf *tfexec.Terraform) {
				if tfv.LessThan(providersSchemaJSONMinVersion) {
					t.Skip("providers schema -json was added in 0.12")
				}

				err := tf.Init(context.Background())
				if err != nil {
					t.Fatalf("error running Init in test directory: %s", err)
				}

				schemas, err := tf.ProvidersSchema(context.Background())
				if err != nil {
					t.Fatalf("error running ProvidersSchema in test directory: %s", err)
				}

				expected := c.expected(tfv)

				if diff := diffSchema(expected, schemas); diff != "" {
					t.Fatalf("mismatch (-want +got):\n%s", diff)
				}
			})
		})
	}

}

func TestProvidersSchema_versionMismatch(t *testing.T) {
	t.Skip("TODO! add version mismatch test for 0.11 as -json was added in 0.12 (I think)")
}
<|endoftext|>


<commit_before>package vizzini_test

import (
	"fmt"
	"net/http"
	"time"

	. "github.com/pivotal-cf-experimental/vizzini/matchers"

	"github.com/cloudfoundry-incubator/receptor"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

func MakeGraceExit(baseURL string, status int) {
	//make sure Grace is up first
	Eventually(EndpointCurler(baseURL + "/env")).Should(Equal(http.StatusOK))

	//make Grace exit
	for i := 0; i < 3; i++ {
		url := fmt.Sprintf("%s/exit/%d", baseURL, status)
		resp, err := http.Post(url, "application/octet-stream", nil)
		Ω(err).ShouldNot(HaveOccurred())
		resp.Body.Close()
		if resp.StatusCode == http.StatusOK {
			return
		}
		time.Sleep(10 * time.Millisecond)
	}
	Fail("failed to make grace exit")
}

func TellGraceToDeleteFile(baseURL string, filename string) {
	url := fmt.Sprintf("%s/file/%s", baseURL, filename)
	req, err := http.NewRequest("DELETE", url, nil)
	Ω(err).ShouldNot(HaveOccurred())
	resp, err := http.DefaultClient.Do(req)
	Ω(err).ShouldNot(HaveOccurred())
	resp.Body.Close()
	Ω(resp.StatusCode).Should(Equal(http.StatusOK))
}

func DirectURL(guid string, index int) string {
	actualLRP, err := ActualGetter(guid, 0)()
	Ω(err).ShouldNot(HaveOccurred())
	Ω(actualLRP).ShouldNot(BeZero())
	return fmt.Sprintf("http://%s:%d", actualLRP.Address, actualLRP.Ports[0].HostPort)
}

var _ = Describe("Crashes", func() {
	var lrp receptor.DesiredLRPCreateRequest
	var url string

	BeforeEach(func() {
		url = fmt.Sprintf("http://%s", RouteForGuid(guid))
		lrp = DesiredLRPWithGuid(guid)
		lrp.Action = &models.RunAction{
			Path: "./grace",
			Env:  []models.EnvironmentVariable{{Name: "PORT", Value: "8080"}},
		}
		lrp.Monitor = nil
	})

	Describe("backoff behavior", func() {
		BeforeEach(func() {
			Ω(client.CreateDesiredLRP(lrp)).Should(Succeed())
			Eventually(EndpointCurler(url + "/env")).Should(Equal(http.StatusOK))
		})

		It("{SLOW} restarts the application immediately twice, and then starts backing it off, and updates the modification tag as it goes", func() {
			actualLRP, err := client.ActualLRPByProcessGuidAndIndex(guid, 0)
			Ω(err).ShouldNot(HaveOccurred())
			tag := actualLRP.ModificationTag

			By("immediately restarting #1")
			MakeGraceExit(url, 1)
			Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateRunning, 1))

			restartedActualLRP, err := client.ActualLRPByProcessGuidAndIndex(guid, 0)
			Ω(err).ShouldNot(HaveOccurred())
			Ω(restartedActualLRP.InstanceGuid).ShouldNot(Equal(actualLRP.InstanceGuid))
			Ω(restartedActualLRP.ModificationTag.Epoch).Should(Equal(tag.Epoch))
			Ω(restartedActualLRP.ModificationTag.Index).Should(BeNumerically(">", tag.Index))

			By("immediately restarting #2")
			MakeGraceExit(url, 1)
			Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateRunning, 2))

			By("eventually restarting #3 (slow)")
			MakeGraceExit(url, 1)
			Eventually(ActualGetter(guid, 0), ConvergerInterval).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateCrashed, 3))
			Consistently(ActualGetter(guid, 0), CrashRestartTimeout-5*time.Second).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateCrashed, 3))
			Eventually(ActualGetter(guid, 0), ConvergerInterval*2).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateRunning, 3))
			Eventually(EndpointCurler(url+"/env")).Should(Equal(http.StatusOK), "This can be removed when #89463754 lands")
		})

		It("deletes the crashed ActualLRP when scaling down", func() {
			By("immediately restarting #1")
			MakeGraceExit(url, 1)
			Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateRunning, 1))

			By("immediately restarting #2")
			MakeGraceExit(url, 1)
			Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateRunning, 2))

			By("eventually restarting #3")
			MakeGraceExit(url, 1)
			Eventually(ActualGetter(guid, 0), ConvergerInterval).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateCrashed, 3))

			By("deleting the DesiredLRP")
			Ω(client.DeleteDesiredLRP(guid)).Should(Succeed())
			Eventually(ActualByProcessGuidGetter(guid)).Should(BeEmpty())
		})
	})

	Describe("killing crashed applications", func() {
		BeforeEach(func() {
			Ω(client.CreateDesiredLRP(lrp)).Should(Succeed())
			Eventually(EndpointCurler(url + "/env")).Should(Equal(http.StatusOK))
		})

		It("should delete the Crashed ActualLRP succesfully", func() {
			By("immediately restarting #1")
			MakeGraceExit(url, 1)
			Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateRunning, 1))

			By("immediately restarting #2")
			MakeGraceExit(url, 1)
			Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateRunning, 2))

			By("eventually restarting #3")
			MakeGraceExit(url, 1)
			Eventually(ActualGetter(guid, 0), ConvergerInterval).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateCrashed, 3))

			Ω(client.KillActualLRPByProcessGuidAndIndex(guid, 0)).Should(Succeed())
			Eventually(ActualByProcessGuidGetter(guid)).Should(BeEmpty())
		})
	})

	Context("with no monitor action", func() {
		BeforeEach(func() {
			Ω(client.CreateDesiredLRP(lrp)).Should(Succeed())
			Eventually(EndpointCurler(url + "/env")).Should(Equal(http.StatusOK))
		})

		It("comes up as soon as the process starts", func() {
			Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithState(guid, 0, receptor.ActualLRPStateRunning))
		})

		Context("when the process dies with exit code 0", func() {
			BeforeEach(func() {
				MakeGraceExit(url, 0)
			})

			It("gets restarted immediately", func() {
				Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateRunning, 1))
				Eventually(EndpointCurler(url+"/env")).Should(Equal(http.StatusOK), "This can be removed when #89463754 lands")
			})
		})

		Context("when the process dies with exit code 1", func() {
			BeforeEach(func() {
				MakeGraceExit(url, 1)
			})

			It("gets restarted immediately", func() {
				Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateRunning, 1))
				Eventually(EndpointCurler(url+"/env")).Should(Equal(http.StatusOK), "This can be removed when #89463754 lands")
			})
		})
	})

	Context("with a monitor action", func() {
		Context("when the monitor eventually succeeds", func() {
			var directURL string
			var indirectURL string
			BeforeEach(func() {
				lrp.Action = &models.RunAction{
					Path: "./grace",
					Args: []string{"-upFile=up"},
					Env:  []models.EnvironmentVariable{{Name: "PORT", Value: "8080"}},
				}

				lrp.Monitor = &models.RunAction{
					Path: "cat",
					Args: []string{"/tmp/up"},
				}

				Ω(client.CreateDesiredLRP(lrp)).Should(Succeed())
				Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithState(guid, 0, receptor.ActualLRPStateRunning))
				Eventually(EndpointCurler(url + "/env")).Should(Equal(http.StatusOK))
				directURL = DirectURL(guid, 0)
				indirectURL = "http://" + RouteForGuid(guid)
			})

			It("enters the running state", func() {
				Ω(ActualGetter(guid, 0)()).Should(BeActualLRPWithState(guid, 0, receptor.ActualLRPStateRunning))
			})

			Context("when the process dies with exit code 0", func() {
				BeforeEach(func() {
					MakeGraceExit(indirectURL, 0)
				})

				It("does not get marked as crashed (may have daemonized)", func() {
					Consistently(ActualGetter(guid, 0), 3).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateRunning, 0))
				})
			})

			Context("when the process dies with exit code 0 and the monitor subsequently fails", func() {
				BeforeEach(func() {
					//tell grace to delete the file then exit, it's highly unlikely that the health check will run
					//between these two lines so the test should actually be covering the edge case in question
					TellGraceToDeleteFile(url, "up")
					MakeGraceExit(indirectURL, 0)
				})

				It("{SLOW} is marked as crashed", func() {
					Consistently(ActualGetter(guid, 0), 2).Should(BeActualLRPWithState(guid, 0, receptor.ActualLRPStateRunning), "Banking on the fact that the health check runs every thirty seconds and is unlikely to run immediately")
					Eventually(ActualGetter(guid, 0), HealthyCheckInterval+5*time.Second).Should(BeActualLRPWithCrashCount(guid, 0, 1))
				})
			})

			Context("when the process dies with exit code 1", func() {
				BeforeEach(func() {
					MakeGraceExit(indirectURL, 1)
				})

				It("is marked as crashed (immediately)", func() {
					Eventually(ActualGetter(guid, 0), HealthyCheckInterval/3).Should(BeActualLRPWithCrashCount(guid, 0, 1))
				})
			})

			//{LOCAL} because: this test attempts to communicate with the container *directly* to ensure the process has been torn down
			//this is not possible against a remote installation as it entails connecting directly into the VPC
			Context("{LOCAL} when the monitor subsequently fails", func() {
				BeforeEach(func() {
					TellGraceToDeleteFile(indirectURL, "up")
				})

				It("{SLOW} is marked as crashed (and reaped)", func() {
					httpClient := &http.Client{
						Timeout: time.Second,
					}

					By("first validate that we can connect to the container directly")
					_, err := httpClient.Get(directURL + "/env")
					Ω(err).ShouldNot(HaveOccurred())

					By("being marked as crashed")
					Eventually(ActualGetter(guid, 0), HealthyCheckInterval+5*time.Second).Should(BeActualLRPWithCrashCount(guid, 0, 1))

					By("tearing down the process -- this reaches out to the container's direct address and ensures we can't reach it")
					_, err = httpClient.Get(directURL + "/env")
					Ω(err).Should(HaveOccurred())
				})
			})
		})

		Context("when the monitor never succeeds", func() {
			JustBeforeEach(func() {
				lrp.Monitor = &models.RunAction{
					Path: "false",
				}

				Ω(client.CreateDesiredLRP(lrp)).Should(Succeed())
				Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithState(guid, 0, receptor.ActualLRPStateClaimed))
			})

			Context("when the process dies with exit code 0", func() {
				BeforeEach(func() {
					lrp.Action = &models.RunAction{
						Path: "./grace",
						Args: []string{"-exitAfter=2s", "-exitAfterCode=0"},
						Env:  []models.EnvironmentVariable{{Name: "PORT", Value: "8080"}},
					}
				})

				It("does not get marked as crash, as it has presumably daemonized and we are waiting on the health check", func() {
					Consistently(ActualGetter(guid, 0), 3).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateClaimed, 0))
				})
			})

			Context("when the process dies with exit code 1", func() {
				BeforeEach(func() {
					lrp.Action = &models.RunAction{
						Path: "./grace",
						Args: []string{"-exitAfter=2s", "-exitAfterCode=1"},
						Env:  []models.EnvironmentVariable{{Name: "PORT", Value: "8080"}},
					}
				})

				It("gets marked as crashed (immediately)", func() {
					Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithCrashCount(guid, 0, 1))
				})
			})

			Context("and there is a StartTimeout", func() {
				BeforeEach(func() {
					lrp.StartTimeout = 5
				})

				It("never enters the running state and is marked as crashed after the StartTimeout", func() {
					Consistently(ActualGetter(guid, 0), 3).Should(BeActualLRPWithState(guid, 0, receptor.ActualLRPStateClaimed))
					Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithCrashCount(guid, 0, 1))
				})
			})

			Context("and there is no start timeout", func() {
				BeforeEach(func() {
					lrp.StartTimeout = 0
				})

				It("never enters the running state, and never crashes", func() {
					Consistently(ActualGetter(guid, 0), 5).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateClaimed, 0))
				})
			})
		})
	})
})
<commit_msg>add crashes exit status test<commit_after>package vizzini_test

import (
	"fmt"
	"net/http"
	"time"

	. "github.com/pivotal-cf-experimental/vizzini/matchers"

	"github.com/cloudfoundry-incubator/receptor"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

func MakeGraceExit(baseURL string, status int) {
	//make sure Grace is up first
	Eventually(EndpointCurler(baseURL + "/env")).Should(Equal(http.StatusOK))

	//make Grace exit
	for i := 0; i < 3; i++ {
		url := fmt.Sprintf("%s/exit/%d", baseURL, status)
		resp, err := http.Post(url, "application/octet-stream", nil)
		Ω(err).ShouldNot(HaveOccurred())
		resp.Body.Close()
		if resp.StatusCode == http.StatusOK {
			return
		}
		time.Sleep(10 * time.Millisecond)
	}
	Fail("failed to make grace exit")
}

func TellGraceToDeleteFile(baseURL string, filename string) {
	url := fmt.Sprintf("%s/file/%s", baseURL, filename)
	req, err := http.NewRequest("DELETE", url, nil)
	Ω(err).ShouldNot(HaveOccurred())
	resp, err := http.DefaultClient.Do(req)
	Ω(err).ShouldNot(HaveOccurred())
	resp.Body.Close()
	Ω(resp.StatusCode).Should(Equal(http.StatusOK))
}

func DirectURL(guid string, index int) string {
	actualLRP, err := ActualGetter(guid, 0)()
	Ω(err).ShouldNot(HaveOccurred())
	Ω(actualLRP).ShouldNot(BeZero())
	return fmt.Sprintf("http://%s:%d", actualLRP.Address, actualLRP.Ports[0].HostPort)
}

var _ = Describe("Crashes", func() {
	var lrp receptor.DesiredLRPCreateRequest
	var url string

	BeforeEach(func() {
		url = fmt.Sprintf("http://%s", RouteForGuid(guid))
		lrp = DesiredLRPWithGuid(guid)
		lrp.Action = &models.RunAction{
			Path: "./grace",
			Env:  []models.EnvironmentVariable{{Name: "PORT", Value: "8080"}},
		}
		lrp.Monitor = nil
	})

	Describe("Annotating the Crash Reason", func() {
		BeforeEach(func() {
			Ω(client.CreateDesiredLRP(lrp)).Should(Succeed())
			Eventually(EndpointCurler(url + "/env")).Should(Equal(http.StatusOK))
		})

		It("adds the crash reason to the application", func() {
			MakeGraceExit(url, 17)
			Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateRunning, 1))
			actualLRP, err := client.ActualLRPByProcessGuidAndIndex(guid, 0)
			Ω(err).ShouldNot(HaveOccurred())
			Ω(actualLRP.CrashReason).Should(ContainSubstring("Exited with status 17"))
		})
	})

	Describe("backoff behavior", func() {
		BeforeEach(func() {
			Ω(client.CreateDesiredLRP(lrp)).Should(Succeed())
			Eventually(EndpointCurler(url + "/env")).Should(Equal(http.StatusOK))
		})

		It("{SLOW} restarts the application immediately twice, and then starts backing it off, and updates the modification tag as it goes", func() {
			actualLRP, err := client.ActualLRPByProcessGuidAndIndex(guid, 0)
			Ω(err).ShouldNot(HaveOccurred())
			tag := actualLRP.ModificationTag

			By("immediately restarting #1")
			MakeGraceExit(url, 1)
			Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateRunning, 1))

			restartedActualLRP, err := client.ActualLRPByProcessGuidAndIndex(guid, 0)
			Ω(err).ShouldNot(HaveOccurred())
			Ω(restartedActualLRP.InstanceGuid).ShouldNot(Equal(actualLRP.InstanceGuid))
			Ω(restartedActualLRP.ModificationTag.Epoch).Should(Equal(tag.Epoch))
			Ω(restartedActualLRP.ModificationTag.Index).Should(BeNumerically(">", tag.Index))

			By("immediately restarting #2")
			MakeGraceExit(url, 1)
			Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateRunning, 2))

			By("eventually restarting #3 (slow)")
			MakeGraceExit(url, 1)
			Eventually(ActualGetter(guid, 0), ConvergerInterval).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateCrashed, 3))
			Consistently(ActualGetter(guid, 0), CrashRestartTimeout-5*time.Second).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateCrashed, 3))
			Eventually(ActualGetter(guid, 0), ConvergerInterval*2).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateRunning, 3))
			Eventually(EndpointCurler(url+"/env")).Should(Equal(http.StatusOK), "This can be removed when #89463754 lands")
		})

		It("deletes the crashed ActualLRP when scaling down", func() {
			By("immediately restarting #1")
			MakeGraceExit(url, 1)
			Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateRunning, 1))

			By("immediately restarting #2")
			MakeGraceExit(url, 1)
			Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateRunning, 2))

			By("eventually restarting #3")
			MakeGraceExit(url, 1)
			Eventually(ActualGetter(guid, 0), ConvergerInterval).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateCrashed, 3))

			By("deleting the DesiredLRP")
			Ω(client.DeleteDesiredLRP(guid)).Should(Succeed())
			Eventually(ActualByProcessGuidGetter(guid)).Should(BeEmpty())
		})
	})

	Describe("killing crashed applications", func() {
		BeforeEach(func() {
			Ω(client.CreateDesiredLRP(lrp)).Should(Succeed())
			Eventually(EndpointCurler(url + "/env")).Should(Equal(http.StatusOK))
		})

		It("should delete the Crashed ActualLRP succesfully", func() {
			By("immediately restarting #1")
			MakeGraceExit(url, 1)
			Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateRunning, 1))

			By("immediately restarting #2")
			MakeGraceExit(url, 1)
			Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateRunning, 2))

			By("eventually restarting #3")
			MakeGraceExit(url, 1)
			Eventually(ActualGetter(guid, 0), ConvergerInterval).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateCrashed, 3))

			Ω(client.KillActualLRPByProcessGuidAndIndex(guid, 0)).Should(Succeed())
			Eventually(ActualByProcessGuidGetter(guid)).Should(BeEmpty())
		})
	})

	Context("with no monitor action", func() {
		BeforeEach(func() {
			Ω(client.CreateDesiredLRP(lrp)).Should(Succeed())
			Eventually(EndpointCurler(url + "/env")).Should(Equal(http.StatusOK))
		})

		It("comes up as soon as the process starts", func() {
			Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithState(guid, 0, receptor.ActualLRPStateRunning))
		})

		Context("when the process dies with exit code 0", func() {
			BeforeEach(func() {
				MakeGraceExit(url, 0)
			})

			It("gets restarted immediately", func() {
				Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateRunning, 1))
				Eventually(EndpointCurler(url+"/env")).Should(Equal(http.StatusOK), "This can be removed when #89463754 lands")
			})
		})

		Context("when the process dies with exit code 1", func() {
			BeforeEach(func() {
				MakeGraceExit(url, 1)
			})

			It("gets restarted immediately", func() {
				Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateRunning, 1))
				Eventually(EndpointCurler(url+"/env")).Should(Equal(http.StatusOK), "This can be removed when #89463754 lands")
			})
		})
	})

	Context("with a monitor action", func() {
		Context("when the monitor eventually succeeds", func() {
			var directURL string
			var indirectURL string
			BeforeEach(func() {
				lrp.Action = &models.RunAction{
					Path: "./grace",
					Args: []string{"-upFile=up"},
					Env:  []models.EnvironmentVariable{{Name: "PORT", Value: "8080"}},
				}

				lrp.Monitor = &models.RunAction{
					Path: "cat",
					Args: []string{"/tmp/up"},
				}

				Ω(client.CreateDesiredLRP(lrp)).Should(Succeed())
				Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithState(guid, 0, receptor.ActualLRPStateRunning))
				Eventually(EndpointCurler(url + "/env")).Should(Equal(http.StatusOK))
				directURL = DirectURL(guid, 0)
				indirectURL = "http://" + RouteForGuid(guid)
			})

			It("enters the running state", func() {
				Ω(ActualGetter(guid, 0)()).Should(BeActualLRPWithState(guid, 0, receptor.ActualLRPStateRunning))
			})

			Context("when the process dies with exit code 0", func() {
				BeforeEach(func() {
					MakeGraceExit(indirectURL, 0)
				})

				It("does not get marked as crashed (may have daemonized)", func() {
					Consistently(ActualGetter(guid, 0), 3).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateRunning, 0))
				})
			})

			Context("when the process dies with exit code 0 and the monitor subsequently fails", func() {
				BeforeEach(func() {
					//tell grace to delete the file then exit, it's highly unlikely that the health check will run
					//between these two lines so the test should actually be covering the edge case in question
					TellGraceToDeleteFile(url, "up")
					MakeGraceExit(indirectURL, 0)
				})

				It("{SLOW} is marked as crashed", func() {
					Consistently(ActualGetter(guid, 0), 2).Should(BeActualLRPWithState(guid, 0, receptor.ActualLRPStateRunning), "Banking on the fact that the health check runs every thirty seconds and is unlikely to run immediately")
					Eventually(ActualGetter(guid, 0), HealthyCheckInterval+5*time.Second).Should(BeActualLRPWithCrashCount(guid, 0, 1))
				})
			})

			Context("when the process dies with exit code 1", func() {
				BeforeEach(func() {
					MakeGraceExit(indirectURL, 1)
				})

				It("is marked as crashed (immediately)", func() {
					Eventually(ActualGetter(guid, 0), HealthyCheckInterval/3).Should(BeActualLRPWithCrashCount(guid, 0, 1))
				})
			})

			//{LOCAL} because: this test attempts to communicate with the container *directly* to ensure the process has been torn down
			//this is not possible against a remote installation as it entails connecting directly into the VPC
			Context("{LOCAL} when the monitor subsequently fails", func() {
				BeforeEach(func() {
					TellGraceToDeleteFile(indirectURL, "up")
				})

				It("{SLOW} is marked as crashed (and reaped)", func() {
					httpClient := &http.Client{
						Timeout: time.Second,
					}

					By("first validate that we can connect to the container directly")
					_, err := httpClient.Get(directURL + "/env")
					Ω(err).ShouldNot(HaveOccurred())

					By("being marked as crashed")
					Eventually(ActualGetter(guid, 0), HealthyCheckInterval+5*time.Second).Should(BeActualLRPWithCrashCount(guid, 0, 1))

					By("tearing down the process -- this reaches out to the container's direct address and ensures we can't reach it")
					_, err = httpClient.Get(directURL + "/env")
					Ω(err).Should(HaveOccurred())
				})
			})
		})

		Context("when the monitor never succeeds", func() {
			JustBeforeEach(func() {
				lrp.Monitor = &models.RunAction{
					Path: "false",
				}

				Ω(client.CreateDesiredLRP(lrp)).Should(Succeed())
				Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithState(guid, 0, receptor.ActualLRPStateClaimed))
			})

			Context("when the process dies with exit code 0", func() {
				BeforeEach(func() {
					lrp.Action = &models.RunAction{
						Path: "./grace",
						Args: []string{"-exitAfter=2s", "-exitAfterCode=0"},
						Env:  []models.EnvironmentVariable{{Name: "PORT", Value: "8080"}},
					}
				})

				It("does not get marked as crash, as it has presumably daemonized and we are waiting on the health check", func() {
					Consistently(ActualGetter(guid, 0), 3).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateClaimed, 0))
				})
			})

			Context("when the process dies with exit code 1", func() {
				BeforeEach(func() {
					lrp.Action = &models.RunAction{
						Path: "./grace",
						Args: []string{"-exitAfter=2s", "-exitAfterCode=1"},
						Env:  []models.EnvironmentVariable{{Name: "PORT", Value: "8080"}},
					}
				})

				It("gets marked as crashed (immediately)", func() {
					Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithCrashCount(guid, 0, 1))
				})
			})

			Context("and there is a StartTimeout", func() {
				BeforeEach(func() {
					lrp.StartTimeout = 5
				})

				It("never enters the running state and is marked as crashed after the StartTimeout", func() {
					Consistently(ActualGetter(guid, 0), 3).Should(BeActualLRPWithState(guid, 0, receptor.ActualLRPStateClaimed))
					Eventually(ActualGetter(guid, 0)).Should(BeActualLRPWithCrashCount(guid, 0, 1))
				})
			})

			Context("and there is no start timeout", func() {
				BeforeEach(func() {
					lrp.StartTimeout = 0
				})

				It("never enters the running state, and never crashes", func() {
					Consistently(ActualGetter(guid, 0), 5).Should(BeActualLRPWithStateAndCrashCount(guid, 0, receptor.ActualLRPStateClaimed, 0))
				})
			})
		})
	})
})
<|endoftext|>


<commit_before>package core

import (
	"io"
	"io/ioutil"
	"log"
	"os"
)

var (
	Trace   *log.Logger
	Info    *log.Logger
	Warning *log.Logger
	Error   *log.Logger
)
var logFile *os.File

type DashboardLogWriter struct {
}

func (w *DashboardLogWriter) Write(p []byte) (n int, err error) {
	Dispatch("log", p[:len(p)])
	return len(p), nil
}

func initLogging() {
	dash := new(DashboardLogWriter)
	var err error

	logFile, err = os.OpenFile("nebuleuse.log", os.O_RDWR|os.O_APPEND, os.ModeAppend)
	if err != nil {
		panic(err)
	}

	Trace = log.New(ioutil.Discard,
		"TRACE: ",
		log.Ldate|log.Ltime|log.Lshortfile)

	Info = log.New(io.MultiWriter(os.Stdout, dash, logFile),
		"INFO: ",
		log.Ldate|log.Ltime|log.Lshortfile)

	Warning = log.New(io.MultiWriter(os.Stdout, dash, logFile),
		"WARNING: ",
		log.Ldate|log.Ltime|log.Lshortfile)

	Error = log.New(io.MultiWriter(os.Stderr, dash, logFile),
		"ERROR: ",
		log.Ldate|log.Ltime|log.Lshortfile)
}

func GetPastLogs(size int64) string {
	fi, err := logFile.Stat()
	position := fi.Size() - size
	if position < 0 {
		position = 0
		size = fi.Size()
	}

	_, err = logFile.Seek(position, 0)
	if err != nil {
		Error.Println("Could not seek file", err)
		return ""
	}

	var buffer []byte
	buffer = make([]byte, size)
	_, err = logFile.Read(buffer)

	if err != nil {
		Error.Println(err)
		return ""
	}
	return string(buffer)
}
<commit_msg>Changed Standard log prefix, opt and output to match existing logging<commit_after>package core

import (
	"io"
	"io/ioutil"
	"log"
	"os"
)

var (
	Trace   *log.Logger
	Info    *log.Logger
	Warning *log.Logger
	Error   *log.Logger
)
var logFile *os.File

type DashboardLogWriter struct {
}

func (w *DashboardLogWriter) Write(p []byte) (n int, err error) {
	Dispatch("log", p[:len(p)])
	return len(p), nil
}

func initLogging() {
	dash := new(DashboardLogWriter)
	var err error

	logFile, err = os.OpenFile("nebuleuse.log", os.O_RDWR|os.O_APPEND, os.ModeAppend)
	if err != nil {
		panic(err)
	}

	normalOut := io.MultiWriter(os.Stdout, dash, logFile)
	errOut := io.MultiWriter(os.Stderr, dash, logFile)

	Trace = log.New(ioutil.Discard,
		"TRACE: ",
		log.Ldate|log.Ltime|log.Lshortfile)

	Info = log.New(normalOut,
		"INFO: ",
		log.Ldate|log.Ltime|log.Lshortfile)

	Warning = log.New(normalOut,
		"WARNING: ",
		log.Ldate|log.Ltime|log.Lshortfile)

	Error = log.New(errOut,
		"ERROR: ",
		log.Ldate|log.Ltime|log.Lshortfile)

	log.SetOutput(errOut)
	log.SetPrefix("ERROR: ")
	log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)
}

func GetPastLogs(size int64) string {
	fi, err := logFile.Stat()
	position := fi.Size() - size
	if position < 0 {
		position = 0
		size = fi.Size()
	}

	_, err = logFile.Seek(position, 0)
	if err != nil {
		Error.Println("Could not seek file", err)
		return ""
	}

	var buffer []byte
	buffer = make([]byte, size)
	_, err = logFile.Read(buffer)

	if err != nil {
		Error.Println(err)
		return ""
	}
	return string(buffer)
}
<|endoftext|>


<commit_before>package core

// transfer2go data core module, request implementation
// Copyright (c) 2017 - Valentin Kuznetsov <vkuznet@gmail.com>

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"mime/multipart"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	"github.com/vkuznet/transfer2go/utils"
)

// AgentStatus data type
type AgentStatus struct {
	Url       string            `json:"url"`      // agent url
	Name      string            `json:"name"`     // agent name or alias
	TimeStamp int64             `json:"ts"`       // time stamp
	Catalog   string            `json:"catalog"`  // underlying TFC catalog
	Protocol  string            `json:"protocol"` // underlying transfer protocol
	Backend   string            `json:"backend"`  // underlying transfer backend
	Tool      string            `json:"tool"`     // underlying transfer tool, e.g. xrdcp
	ToolOpts  string            `json:"toolopts"` // options for backend tool
	Agents    map[string]string `json:"agents"`   // list of known agents
	Addrs     []string          `json:"addrs"`    // list of all IP addresses
	Metrics   map[string]int64  `json:"metrics"`  // agent metrics
}

// Processor is an object who process' given task
// The logic of the Processor should be implemented.
type Processor struct {
}

// Request interface defines a task process
type Request interface {
	Process(*TransferRequest) error
}

// RequestFunc is a function type that implements the Request interface
type RequestFunc func(*TransferRequest) error

// Decorator wraps a request with extra behavior
type Decorator func(Request) Request

// DefaultProcessor is a default processor instance
var DefaultProcessor = &Processor{}

// String provides string representation of given agent status
func (a *AgentStatus) String() string {
	return fmt.Sprintf("<Agent name=%s url=%s catalog=%s protocol=%s backend=%s tool=%s toolOpts=%s agents=%v addrs=%v metrics(%v)>", a.Name, a.Url, a.Catalog, a.Protocol, a.Backend, a.Tool, a.ToolOpts, a.Agents, a.Addrs, a.Metrics)
}

// Process defines execution process for a given task
func (e *Processor) Process(t *TransferRequest) error {
	return nil
}

// Process is a method of TransferRequest
func (f RequestFunc) Process(t *TransferRequest) error {
	return f(t)
}

// filleTransferRequest creates HTTP request to transfer a given file name
// https://matt.aimonetti.net/posts/2013/07/01/golang-multipart-file-upload-example/
func fileTransferRequest(c CatalogEntry, tr *TransferRequest) (*http.Request, error) {
	file, err := os.Open(c.Pfn)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	part, err := writer.CreateFormFile("data", filepath.Base(c.Pfn))
	if err != nil {
		return nil, err
	}
	_, err = io.Copy(part, file)
	err = writer.Close()
	if err != nil {
		return nil, err
	}

	url := fmt.Sprintf("%s/upload", tr.DstUrl)
	req, err := http.NewRequest("POST", url, body)
	req.Header.Set("Content-Type", writer.FormDataContentType())
	req.Header.Set("Pfn", c.Pfn)
	req.Header.Set("Lfn", c.Lfn)
	req.Header.Set("Bytes", fmt.Sprintf("%d", c.Bytes))
	req.Header.Set("Hash", c.Hash)
	req.Header.Set("Src", tr.SrcAlias)
	req.Header.Set("Dst", tr.DstAlias)
	return req, err
}

// helper function to perform transfer via HTTP protocol
func httpTransfer(c CatalogEntry, t *TransferRequest) (string, error) {
	// create file transfer request
	request, err := fileTransferRequest(c, t)
	if err != nil {
		return "", err
	}
	client := utils.HttpClient()
	resp, err := client.Do(request)
	defer resp.Body.Close()

	var r CatalogEntry
	err = json.NewDecoder(resp.Body).Decode(&r)
	if err != nil {
		return "", err
	}
	return r.Pfn, nil
}

// Transfer returns a Decorator that performs request transfers
func Transfer() Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			log.Println("Request Transfer", t.String())
			records := TFC.Records(*t)
			if len(records) == 0 {
				// file does not exists in TFC, nothing to do, return immediately
				log.Printf("WARNING %v does match anything in TFC of this agent\n", t)
				return r.Process(t)
			}
			// obtain information about source and destination agents
			url := fmt.Sprintf("%s/status", t.DstUrl)
			resp := utils.FetchResponse(url, []byte{})
			if resp.Error != nil {
				return resp.Error
			}
			var dstAgent AgentStatus
			err := json.Unmarshal(resp.Data, &dstAgent)
			if err != nil {
				return err
			}
			url = fmt.Sprintf("%s/status", t.SrcUrl)
			resp = utils.FetchResponse(url, []byte{})
			if resp.Error != nil {
				return resp.Error
			}
			var srcAgent AgentStatus
			err = json.Unmarshal(resp.Data, &srcAgent)
			if err != nil {
				return err
			}

			// TODO: I need to implement bulk transfer for all files in found records
			// so far I loop over them individually and transfer one by one
			var trRecords []CatalogEntry // list of successfully transferred records
			for _, rec := range records {

				time0 := time.Now().Unix()

				AgentMetrics.Bytes.Inc(rec.Bytes)

				// if protocol is not given use default one: HTTP
				var rpfn string // remote PFN
				if srcAgent.Protocol == "" || srcAgent.Protocol == "http" {
					log.Println("Transfer via HTTP protocol to", dstAgent.String())
					rpfn, err = httpTransfer(rec, t)
					if err != nil {
						log.Println("ERROR Transfer", rec.String(), t.String(), err)
						continue // if we fail on single record we continue with others
					}
				} else {
					// construct remote PFN by using destination agent backend and record LFN
					rpfn = fmt.Sprintf("%s%s", dstAgent.Backend, rec.Lfn)
					// perform transfer with the help of backend tool
					cmd := exec.Command(srcAgent.Tool, srcAgent.ToolOpts, rec.Pfn, rpfn)
					if srcAgent.ToolOpts == "" {
						cmd = exec.Command(srcAgent.Tool, rec.Pfn, rpfn)
					}
					log.Println("Transfer command", cmd)
					err = cmd.Run()
					if err != nil {
						log.Println("ERROR Transfer", srcAgent.Tool, srcAgent.ToolOpts, rec.Pfn, rpfn, err)
						continue // if we fail on single record we continue with others
					}
				}
				r := CatalogEntry{Dataset: rec.Dataset, Block: rec.Block, Lfn: rec.Lfn, Pfn: rpfn, Bytes: rec.Bytes, Hash: rec.Hash, TransferTime: (time.Now().Unix() - time0), Timestamp: time.Now().Unix()}
				trRecords = append(trRecords, r)

				// record how much we transferred
				AgentMetrics.TotalBytes.Inc(r.Bytes) // keep growing
				AgentMetrics.Total.Inc(1)            // keep growing
				AgentMetrics.Bytes.Dec(rec.Bytes)    // decrement since we're done

			}
			// Add entry for remote TFC after transfer is completed
			url = fmt.Sprintf("%s/tfc", t.DstUrl)
			d, e := json.Marshal(trRecords)
			if e != nil {
				return e
			}
			resp = utils.FetchResponse(url, d) // POST request
			if resp.Error != nil {
				return resp.Error
			}

			return r.Process(t)
		})
	}
}

// Logging returns a Decorator that logs client requests
func Logging(l *log.Logger) Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			l.Println("TransferRequest", t)
			return r.Process(t)
		})
	}
}

// Pause returns a Decorator that pauses request for a given time interval
func Pause(interval time.Duration) Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			if interval > 0 {
				log.Println("TransferRequest", t, "is paused by", interval)
				time.Sleep(interval)
			}
			return r.Process(t)
		})
	}
}

// Tracer returns a Decorator that traces given request
func Tracer() Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			log.Println("Trace", t)
			return r.Process(t)
		})
	}
}

// Decorate decorates a Request r with all given Decorators
func Decorate(r Request, ds ...Decorator) Request {
	decorated := r
	for _, decorate := range ds {
		decorated = decorate(decorated)
	}
	return decorated
}
<commit_msg>Changes in request.go<commit_after>package core

// transfer2go data core module, request implementation
// Copyright (c) 2017 - Valentin Kuznetsov <vkuznet@gmail.com>

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	log "github.com/sirupsen/logrus"
	"github.com/vkuznet/transfer2go/utils"
)

// AgentStatus data type
type AgentStatus struct {
	Url       string            `json:"url"`      // agent url
	Name      string            `json:"name"`     // agent name or alias
	TimeStamp int64             `json:"ts"`       // time stamp
	Catalog   string            `json:"catalog"`  // underlying TFC catalog
	Protocol  string            `json:"protocol"` // underlying transfer protocol
	Backend   string            `json:"backend"`  // underlying transfer backend
	Tool      string            `json:"tool"`     // underlying transfer tool, e.g. xrdcp
	ToolOpts  string            `json:"toolopts"` // options for backend tool
	Agents    map[string]string `json:"agents"`   // list of known agents
	Addrs     []string          `json:"addrs"`    // list of all IP addresses
	Metrics   map[string]int64  `json:"metrics"`  // agent metrics
}

// Processor is an object who process' given task
// The logic of the Processor should be implemented.
type Processor struct {
}

// Request interface defines a task process
type Request interface {
	Process(*TransferRequest) error
}

// RequestFunc is a function type that implements the Request interface
type RequestFunc func(*TransferRequest) error

// Decorator wraps a request with extra behavior
type Decorator func(Request) Request

// DefaultProcessor is a default processor instance
var DefaultProcessor = &Processor{}

// String provides string representation of given agent status
func (a *AgentStatus) String() string {
	return fmt.Sprintf("<Agent name=%s url=%s catalog=%s protocol=%s backend=%s tool=%s toolOpts=%s agents=%v addrs=%v metrics(%v)>", a.Name, a.Url, a.Catalog, a.Protocol, a.Backend, a.Tool, a.ToolOpts, a.Agents, a.Addrs, a.Metrics)
}

// Process defines execution process for a given task
func (e *Processor) Process(t *TransferRequest) error {
	return nil
}

// Process is a method of TransferRequest
func (f RequestFunc) Process(t *TransferRequest) error {
	return f(t)
}

// filleTransferRequest creates HTTP request to transfer a given file name
// https://matt.aimonetti.net/posts/2013/07/01/golang-multipart-file-upload-example/
func fileTransferRequest(c CatalogEntry, tr *TransferRequest) (*http.Request, error) {
	file, err := os.Open(c.Pfn)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	part, err := writer.CreateFormFile("data", filepath.Base(c.Pfn))
	if err != nil {
		return nil, err
	}
	_, err = io.Copy(part, file)
	err = writer.Close()
	if err != nil {
		return nil, err
	}

	url := fmt.Sprintf("%s/upload", tr.DstUrl)
	req, err := http.NewRequest("POST", url, body)
	req.Header.Set("Content-Type", writer.FormDataContentType())
	req.Header.Set("Pfn", c.Pfn)
	req.Header.Set("Lfn", c.Lfn)
	req.Header.Set("Bytes", fmt.Sprintf("%d", c.Bytes))
	req.Header.Set("Hash", c.Hash)
	req.Header.Set("Src", tr.SrcAlias)
	req.Header.Set("Dst", tr.DstAlias)
	return req, err
}

// helper function to perform transfer via HTTP protocol
func httpTransfer(c CatalogEntry, t *TransferRequest) (string, error) {
	// create file transfer request
	request, err := fileTransferRequest(c, t)
	if err != nil {
		return "", err
	}
	client := utils.HttpClient()
	resp, err := client.Do(request)
	defer resp.Body.Close()

	var r CatalogEntry
	err = json.NewDecoder(resp.Body).Decode(&r)
	if err != nil {
		return "", err
	}
	return r.Pfn, nil
}

// Transfer returns a Decorator that performs request transfers
func Transfer() Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			log.WithFields(log.Fields{
				"Request": t.String(),
			}).Println("Request Transfer", t.String())
			records := TFC.Records(*t)
			if len(records) == 0 {
				// file does not exists in TFC, nothing to do, return immediately
				log.WithFields(log.Fields{
					"TransferRequest": t,
				}).Warn("Does match anything in TFC of this agent\n", t)
				return r.Process(t)
			}
			// obtain information about source and destination agents
			url := fmt.Sprintf("%s/status", t.DstUrl)
			resp := utils.FetchResponse(url, []byte{})
			if resp.Error != nil {
				return resp.Error
			}
			var dstAgent AgentStatus
			err := json.Unmarshal(resp.Data, &dstAgent)
			if err != nil {
				return err
			}
			url = fmt.Sprintf("%s/status", t.SrcUrl)
			resp = utils.FetchResponse(url, []byte{})
			if resp.Error != nil {
				return resp.Error
			}
			var srcAgent AgentStatus
			err = json.Unmarshal(resp.Data, &srcAgent)
			if err != nil {
				return err
			}

			// TODO: I need to implement bulk transfer for all files in found records
			// so far I loop over them individually and transfer one by one
			var trRecords []CatalogEntry // list of successfully transferred records
			for _, rec := range records {

				time0 := time.Now().Unix()

				AgentMetrics.Bytes.Inc(rec.Bytes)

				// if protocol is not given use default one: HTTP
				var rpfn string // remote PFN
				if srcAgent.Protocol == "" || srcAgent.Protocol == "http" {
					log.WithFields(log.Fields{
						"dstAgent": dstAgent.String(),
					}).Println("Transfer via HTTP protocol to", dstAgent.String())
					rpfn, err = httpTransfer(rec, t)
					if err != nil {
						log.WithFields(log.Fields{
							"TransferRequest": t.String(),
							"Record":          rec.String(),
							"Err":             err,
						}).Error("Transfer", rec.String(), t.String(), err)
						continue // if we fail on single record we continue with others
					}
				} else {
					// construct remote PFN by using destination agent backend and record LFN
					rpfn = fmt.Sprintf("%s%s", dstAgent.Backend, rec.Lfn)
					// perform transfer with the help of backend tool
					cmd := exec.Command(srcAgent.Tool, srcAgent.ToolOpts, rec.Pfn, rpfn)
					if srcAgent.ToolOpts == "" {
						cmd = exec.Command(srcAgent.Tool, rec.Pfn, rpfn)
					}
					log.WithFields(log.Fields{
						"Command": cmd,
					}).Println("Transfer command")
					err = cmd.Run()
					if err != nil {
						log.WithFields(log.Fields{
							"Tool":         srcAgent.Tool,
							"Tool options": srcAgent.ToolOpts,
							"PFN":          rec.Pfn,
							"Remote PFN":   rpfn,
							"Err":          err,
						}).Error("Transfer")
						continue // if we fail on single record we continue with others
					}
				}
				r := CatalogEntry{Dataset: rec.Dataset, Block: rec.Block, Lfn: rec.Lfn, Pfn: rpfn, Bytes: rec.Bytes, Hash: rec.Hash, TransferTime: (time.Now().Unix() - time0), Timestamp: time.Now().Unix()}
				trRecords = append(trRecords, r)

				// record how much we transferred
				AgentMetrics.TotalBytes.Inc(r.Bytes) // keep growing
				AgentMetrics.Total.Inc(1)            // keep growing
				AgentMetrics.Bytes.Dec(rec.Bytes)    // decrement since we're done

			}
			// Add entry for remote TFC after transfer is completed
			url = fmt.Sprintf("%s/tfc", t.DstUrl)
			d, e := json.Marshal(trRecords)
			if e != nil {
				return e
			}
			resp = utils.FetchResponse(url, d) // POST request
			if resp.Error != nil {
				return resp.Error
			}

			return r.Process(t)
		})
	}
}

// Logging returns a Decorator that logs client requests
func Logging(l *log.Logger) Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			l.Println("TransferRequest", t)
			return r.Process(t)
		})
	}
}

// Pause returns a Decorator that pauses request for a given time interval
func Pause(interval time.Duration) Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			if interval > 0 {
				log.WithFields(log.Fields{
					"Request":  t,
					"Interval": interval,
				}).Println("TransferRequest is paused by")
				time.Sleep(interval)
			}
			return r.Process(t)
		})
	}
}

// Tracer returns a Decorator that traces given request
func Tracer() Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			log.WithFields(log.Fields{
				"TransferRequest": t,
			}).Println("Trace")
			return r.Process(t)
		})
	}
}

// Decorate decorates a Request r with all given Decorators
func Decorate(r Request, ds ...Decorator) Request {
	decorated := r
	for _, decorate := range ds {
		decorated = decorate(decorated)
	}
	return decorated
}
<|endoftext|>