<commit_before>package fs

import (
	"errors"
	"fmt"
	"log"
	"os"
	"sync"
	"syscall"

	"bazil.org/bazil/cas/blobs"
	wirecas "bazil.org/bazil/cas/wire"
	"bazil.org/bazil/db"
	"bazil.org/bazil/fs/inodes"
	"bazil.org/bazil/fs/snap"
	wiresnap "bazil.org/bazil/fs/snap/wire"
	"bazil.org/bazil/fs/wire"
	"bazil.org/bazil/util/env"
	"bazil.org/fuse"
	"bazil.org/fuse/fs"
	"github.com/golang/protobuf/proto"
	"golang.org/x/net/context"
)

type dir struct {
	fs.NodeRef

	inode  uint64
	parent *dir
	fs     *Volume

	// mu protects the fields below.
	//
	// If multiple dir.mu instances need to be locked at the same
	// time, the locks must be taken in topologically sorted
	// order, parent first.
	//
	// As there can be only one db.Update at a time, those calls
	// must be considered as lock operations too. To avoid lock
	// ordering related deadlocks, never hold mu while calling
	// db.Update.
	mu sync.Mutex

	name string

	// each in-memory child, so we can return the same node on
	// multiple Lookups and know what to do on .save()
	//
	// each child also stores its own name; if the value in the child
	// is an empty string, that means the child has been unlinked
	active map[string]node
}

func newDir(filesys *Volume, inode uint64, parent *dir, name string) *dir {
	d := &dir{
		inode:  inode,
		name:   name,
		parent: parent,
		fs:     filesys,
		active: make(map[string]node),
	}
	return d
}

var _ = node(&dir{})
var _ = fs.Node(&dir{})
var _ = fs.NodeCreater(&dir{})
var _ = fs.NodeForgetter(&dir{})
var _ = fs.NodeMkdirer(&dir{})
var _ = fs.NodeRemover(&dir{})
var _ = fs.NodeRenamer(&dir{})
var _ = fs.NodeStringLookuper(&dir{})
var _ = fs.HandleReadDirAller(&dir{})

func (d *dir) setName(name string) {
	d.mu.Lock()
	defer d.mu.Unlock()
	d.name = name
}

func (d *dir) Attr(ctx context.Context, a *fuse.Attr) error {
	a.Inode = d.inode
	a.Mode = os.ModeDir | 0755
	a.Uid = env.MyUID
	a.Gid = env.MyGID
	return nil
}

func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
	if d.inode == 1 && name == ".snap" {
		return &listSnaps{
			fs: d.fs,
		}, nil
	}

	d.mu.Lock()
	defer d.mu.Unlock()

	if child, ok := d.active[name]; ok {
		return child, nil
	}

	var de *wire.Dirent
	lookup := func(tx *db.Tx) error {
		var err error
		de, err = d.fs.bucket(tx).Dirs().Get(d.inode, name)
		if err != nil {
			return err
		}
		return nil
	}
	if err := d.fs.db.View(lookup); err != nil {
		return nil, err
	}
	child, err := d.reviveNode(de, name)
	if err != nil {
		return nil, fmt.Errorf("dirent node unmarshal problem: %v", err)
	}
	d.active[name] = child
	return child, nil
}

func unmarshalDirent(buf []byte) (*wire.Dirent, error) {
	var de wire.Dirent
	if err := proto.Unmarshal(buf, &de); err != nil {
		return nil, err
	}
	return &de, nil
}

func (d *dir) reviveDir(de *wire.Dirent, name string) (*dir, error) {
	if de.Dir == nil {
		return nil, fmt.Errorf("tried to revive non-directory as directory: %v", de)
	}
	child := newDir(d.fs, de.Inode, d, name)
	return child, nil
}

func (d *dir) reviveNode(de *wire.Dirent, name string) (node, error) {
	switch {
	case de.Dir != nil:
		return d.reviveDir(de, name)

	case de.File != nil:
		manifest, err := de.File.Manifest.ToBlob("file")
		if err != nil {
			return nil, err
		}
		blob, err := blobs.Open(d.fs.chunkStore, manifest)
		if err != nil {
			return nil, err
		}
		child := &file{
			inode:  de.Inode,
			name:   name,
			parent: d,
			blob:   blob,
		}
		return child, nil
	}

	return nil, fmt.Errorf("dirent unknown type: %v", de)
}

func (d *dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
	d.mu.Lock()
	defer d.mu.Unlock()

	var entries []fuse.Dirent
	readDirAll := func(tx *db.Tx) error {
		c := d.fs.bucket(tx).Dirs().List(d.inode)
		for item := c.First(); item != nil; item = c.Next() {
			var de wire.Dirent
			if err := item.Unmarshal(&de); err != nil {
				return fmt.Errorf("readdir error: %v", err)
			}
			fde := de.GetFUSEDirent(item.Name())
			entries = append(entries, fde)
		}
		return nil
	}
	err := d.fs.db.View(readDirAll)
	return entries, err
}

// saveInternal persists entry name in dir to the database.
//
// uses no mutable state of d, and hence does not need to lock d.mu.
func (d *dir) saveInternal(tx *db.Tx, name string, n node) error {
	de, err := n.marshal()
	if err != nil {
		return fmt.Errorf("node save error: %v", err)
	}
	if err := d.fs.bucket(tx).Dirs().Put(d.inode, name, de); err != nil {
		return fmt.Errorf("dirent save error: %v", err)
	}
	return nil
}

func (d *dir) marshal() (*wire.Dirent, error) {
	de := &wire.Dirent{
		Inode: d.inode,
	}
	de.Dir = &wire.Dir{}
	return de, nil
}

func (d *dir) save(tx *db.Tx, name string, n node) error {
	if name == "" {
		// unlinked
		return nil
	}

	d.mu.Lock()
	defer d.mu.Unlock()
	return d.saveInternal(tx, name, n)
}

const debugCreateExisting = true

func (d *dir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) {
	// TODO check for duplicate name

	switch req.Mode & os.ModeType {
	case 0:
		var child node
		createFile := func(tx *db.Tx) error {
			bucket := d.fs.bucket(tx).InodeBucket()
			if bucket == nil {
				return errors.New("inode bucket is missing")
			}
			inode, err := inodes.Allocate(bucket)
			if err != nil {
				return err
			}

			manifest := blobs.EmptyManifest("file")
			blob, err := blobs.Open(d.fs.chunkStore, manifest)
			if err != nil {
				return fmt.Errorf("blob open problem: %v", err)
			}
			child = &file{
				inode:  inode,
				name:   req.Name,
				parent: d,
				blob:   blob,
			}
			if err := d.saveInternal(tx, req.Name, child); err != nil {
				return err
			}
			return nil
		}
		if err := d.fs.db.Update(createFile); err != nil {
			return nil, nil, err
		}

		d.mu.Lock()
		defer d.mu.Unlock()
		if debugCreateExisting {
			if n, ok := d.active[req.Name]; ok {
				log.Printf("asked to create with existing node: %q %#v", req.Name, n)
				n.setName("")
			}
		}
		d.active[req.Name] = child
		return child, child, nil
	default:
		return nil, nil, fuse.EPERM
	}
}

const debugActiveChildren = true

func (d *dir) forgetChild(name string, child node) {
	if name == "" {
		// unlinked
		return
	}

	d.mu.Lock()
	defer d.mu.Unlock()

	if debugActiveChildren {
		have, ok := d.active[name]
		switch {
		case !ok:
			log.Printf("asked to forget non-active child: %q %#v", name, child)
		case have != child:
			log.Printf("asked to forget wrong child: %q %#v", name, child)
		}
	}
	delete(d.active, name)
}

func (d *dir) Forget() {
	if d.parent == nil {
		// root dir, don't keep track
		return
	}

	d.mu.Lock()
	name := d.name
	d.mu.Unlock()

	d.parent.forgetChild(name, d)
}

const debugMkdirExisting = true

func (d *dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) {
	// TODO handle req.Mode

	var child node
	mkdir := func(tx *db.Tx) error {
		bucket := d.fs.bucket(tx)
		inode, err := inodes.Allocate(bucket.InodeBucket())
		if err != nil {
			return err
		}
		child = newDir(d.fs, inode, d, req.Name)
		if err := d.saveInternal(tx, req.Name, child); err != nil {
			return err
		}
		if _, err := bucket.Clock().Create(d.inode, req.Name, d.fs.dirtyEpoch()); err != nil {
			return err
		}
		return nil
	}
	if err := d.fs.db.Update(mkdir); err != nil {
		if err == inodes.ErrOutOfInodes {
			return nil, fuse.Errno(syscall.ENOSPC)
		}
		return nil, err
	}

	d.mu.Lock()
	defer d.mu.Unlock()
	if debugMkdirExisting {
		if n, ok := d.active[req.Name]; ok {
			log.Printf("asked to mkdir with existing node: %q %#v", req.Name, n)
			n.setName("")
		}
	}
	d.active[req.Name] = child
	return child, nil
}

func (d *dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {
	remove := func(tx *db.Tx) error {
		if err := d.fs.bucket(tx).Dirs().Delete(d.inode, req.Name); err != nil {
			return err
		}

		// TODO free inode
		return nil
	}
	if err := d.fs.db.Update(remove); err != nil {
		return err
	}

	d.mu.Lock()
	defer d.mu.Unlock()
	if node, ok := d.active[req.Name]; ok {
		delete(d.active, req.Name)
		node.setName("")
	}
	return nil
}

func (d *dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDir fs.Node) error {
	// if you ever change this, also guard against renaming into
	// special directories like .snap; check type of newDir is *dir
	//
	// also worry about deadlocks
	if newDir != d {
		return fuse.Errno(syscall.EXDEV)
	}

	rename := func(tx *db.Tx) error {
		// TODO don't need to load from db if req.OldName is in active.
		// instead, save active state if we have it; call .save() not this
		// kludge
		//
		// TODO don't need to load from db if req.NewName is in active
		loser, err := d.fs.bucket(tx).Dirs().Rename(d.inode, req.OldName, req.NewName)
		if err != nil {
			return err
		}

		if loser != nil {
			// TODO free loser inode
		}
		return nil
	}
	if err := d.fs.db.Update(rename); err != nil {
		return err
	}

	d.mu.Lock()
	defer d.mu.Unlock()

	// tell overwritten node it's unlinked
	if n, ok := d.active[req.NewName]; ok {
		n.setName("")
	}

	// if the source inode is active, record its new name
	if nodeOld, ok := d.active[req.OldName]; ok {
		nodeOld.setName(req.NewName)
		delete(d.active, req.OldName)
		d.active[req.NewName] = nodeOld
	}

	return nil
}

// snapshot records a snapshot of the directory and stores it in wde
func (d *dir) snapshot(ctx context.Context, tx *db.Tx) (*wiresnap.Dirent, error) {
	// NOT HOLDING THE LOCK, accessing database snapshot ONLY

	// TODO move bucket lookup to caller?
	bucket := d.fs.bucket(tx)

	manifest := blobs.EmptyManifest("dir")
	blob, err := blobs.Open(d.fs.chunkStore, manifest)
	if err != nil {
		return nil, err
	}
	w := snap.NewWriter(blob)

	c := bucket.Dirs().List(d.inode)
	for item := c.First(); item != nil; item = c.Next() {
		var de wire.Dirent
		if err := item.Unmarshal(&de); err != nil {
			return nil, err
		}
		var sde *wiresnap.Dirent
		switch {
		case de.File != nil:
			// TODO d.reviveNode would do blobs.Open and that's a bit
			// too much work; rework the apis
			sde = &wiresnap.Dirent{
				File: &wiresnap.File{
					Manifest: de.File.Manifest,
				},
			}
		case de.Dir != nil:
			child, err := d.reviveDir(&de, item.Name())
			if err != nil {
				return nil, err
			}
			sde, err = child.snapshot(ctx, tx)
			if err != nil {
				return nil, err
			}
		default:
			return nil, errors.New("TODO")
		}
		sde.Name = item.Name()
		err = w.Add(sde)
		if err != nil {
			return nil, err
		}
	}

	manifest, err = blob.Save()
	if err != nil {
		return nil, err
	}
	msg := wiresnap.Dirent{
		Dir: &wiresnap.Dir{
			Manifest: wirecas.FromBlob(manifest),
			Align:    w.Align(),
		},
	}
	return &msg, nil
}
<commit_msg>fs: Record logical time of file Create<commit_after>package fs

import (
	"errors"
	"fmt"
	"log"
	"os"
	"sync"
	"syscall"

	"bazil.org/bazil/cas/blobs"
	wirecas "bazil.org/bazil/cas/wire"
	"bazil.org/bazil/db"
	"bazil.org/bazil/fs/inodes"
	"bazil.org/bazil/fs/snap"
	wiresnap "bazil.org/bazil/fs/snap/wire"
	"bazil.org/bazil/fs/wire"
	"bazil.org/bazil/util/env"
	"bazil.org/fuse"
	"bazil.org/fuse/fs"
	"github.com/golang/protobuf/proto"
	"golang.org/x/net/context"
)

type dir struct {
	fs.NodeRef

	inode  uint64
	parent *dir
	fs     *Volume

	// mu protects the fields below.
	//
	// If multiple dir.mu instances need to be locked at the same
	// time, the locks must be taken in topologically sorted
	// order, parent first.
	//
	// As there can be only one db.Update at a time, those calls
	// must be considered as lock operations too. To avoid lock
	// ordering related deadlocks, never hold mu while calling
	// db.Update.
	mu sync.Mutex

	name string

	// each in-memory child, so we can return the same node on
	// multiple Lookups and know what to do on .save()
	//
	// each child also stores its own name; if the value in the child
	// is an empty string, that means the child has been unlinked
	active map[string]node
}

func newDir(filesys *Volume, inode uint64, parent *dir, name string) *dir {
	d := &dir{
		inode:  inode,
		name:   name,
		parent: parent,
		fs:     filesys,
		active: make(map[string]node),
	}
	return d
}

var _ = node(&dir{})
var _ = fs.Node(&dir{})
var _ = fs.NodeCreater(&dir{})
var _ = fs.NodeForgetter(&dir{})
var _ = fs.NodeMkdirer(&dir{})
var _ = fs.NodeRemover(&dir{})
var _ = fs.NodeRenamer(&dir{})
var _ = fs.NodeStringLookuper(&dir{})
var _ = fs.HandleReadDirAller(&dir{})

func (d *dir) setName(name string) {
	d.mu.Lock()
	defer d.mu.Unlock()
	d.name = name
}

func (d *dir) Attr(ctx context.Context, a *fuse.Attr) error {
	a.Inode = d.inode
	a.Mode = os.ModeDir | 0755
	a.Uid = env.MyUID
	a.Gid = env.MyGID
	return nil
}

func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
	if d.inode == 1 && name == ".snap" {
		return &listSnaps{
			fs: d.fs,
		}, nil
	}

	d.mu.Lock()
	defer d.mu.Unlock()

	if child, ok := d.active[name]; ok {
		return child, nil
	}

	var de *wire.Dirent
	lookup := func(tx *db.Tx) error {
		var err error
		de, err = d.fs.bucket(tx).Dirs().Get(d.inode, name)
		if err != nil {
			return err
		}
		return nil
	}
	if err := d.fs.db.View(lookup); err != nil {
		return nil, err
	}
	child, err := d.reviveNode(de, name)
	if err != nil {
		return nil, fmt.Errorf("dirent node unmarshal problem: %v", err)
	}
	d.active[name] = child
	return child, nil
}

func unmarshalDirent(buf []byte) (*wire.Dirent, error) {
	var de wire.Dirent
	if err := proto.Unmarshal(buf, &de); err != nil {
		return nil, err
	}
	return &de, nil
}

func (d *dir) reviveDir(de *wire.Dirent, name string) (*dir, error) {
	if de.Dir == nil {
		return nil, fmt.Errorf("tried to revive non-directory as directory: %v", de)
	}
	child := newDir(d.fs, de.Inode, d, name)
	return child, nil
}

func (d *dir) reviveNode(de *wire.Dirent, name string) (node, error) {
	switch {
	case de.Dir != nil:
		return d.reviveDir(de, name)

	case de.File != nil:
		manifest, err := de.File.Manifest.ToBlob("file")
		if err != nil {
			return nil, err
		}
		blob, err := blobs.Open(d.fs.chunkStore, manifest)
		if err != nil {
			return nil, err
		}
		child := &file{
			inode:  de.Inode,
			name:   name,
			parent: d,
			blob:   blob,
		}
		return child, nil
	}

	return nil, fmt.Errorf("dirent unknown type: %v", de)
}

func (d *dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
	d.mu.Lock()
	defer d.mu.Unlock()

	var entries []fuse.Dirent
	readDirAll := func(tx *db.Tx) error {
		c := d.fs.bucket(tx).Dirs().List(d.inode)
		for item := c.First(); item != nil; item = c.Next() {
			var de wire.Dirent
			if err := item.Unmarshal(&de); err != nil {
				return fmt.Errorf("readdir error: %v", err)
			}
			fde := de.GetFUSEDirent(item.Name())
			entries = append(entries, fde)
		}
		return nil
	}
	err := d.fs.db.View(readDirAll)
	return entries, err
}

// saveInternal persists entry name in dir to the database.
//
// uses no mutable state of d, and hence does not need to lock d.mu.
func (d *dir) saveInternal(tx *db.Tx, name string, n node) error {
	de, err := n.marshal()
	if err != nil {
		return fmt.Errorf("node save error: %v", err)
	}
	if err := d.fs.bucket(tx).Dirs().Put(d.inode, name, de); err != nil {
		return fmt.Errorf("dirent save error: %v", err)
	}
	return nil
}

func (d *dir) marshal() (*wire.Dirent, error) {
	de := &wire.Dirent{
		Inode: d.inode,
	}
	de.Dir = &wire.Dir{}
	return de, nil
}

func (d *dir) save(tx *db.Tx, name string, n node) error {
	if name == "" {
		// unlinked
		return nil
	}

	d.mu.Lock()
	defer d.mu.Unlock()
	return d.saveInternal(tx, name, n)
}

const debugCreateExisting = true

func (d *dir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) {
	// TODO check for duplicate name

	switch req.Mode & os.ModeType {
	case 0:
		var child node
		createFile := func(tx *db.Tx) error {
			bucket := d.fs.bucket(tx)
			inode, err := inodes.Allocate(bucket.InodeBucket())
			if err != nil {
				return err
			}

			manifest := blobs.EmptyManifest("file")
			blob, err := blobs.Open(d.fs.chunkStore, manifest)
			if err != nil {
				return fmt.Errorf("blob open problem: %v", err)
			}
			child = &file{
				inode:  inode,
				name:   req.Name,
				parent: d,
				blob:   blob,
			}
			if err := d.saveInternal(tx, req.Name, child); err != nil {
				return err
			}
			if _, err := bucket.Clock().Create(d.inode, req.Name, d.fs.dirtyEpoch()); err != nil {
				return err
			}
			return nil
		}
		if err := d.fs.db.Update(createFile); err != nil {
			return nil, nil, err
		}

		d.mu.Lock()
		defer d.mu.Unlock()
		if debugCreateExisting {
			if n, ok := d.active[req.Name]; ok {
				log.Printf("asked to create with existing node: %q %#v", req.Name, n)
				n.setName("")
			}
		}
		d.active[req.Name] = child
		return child, child, nil
	default:
		return nil, nil, fuse.EPERM
	}
}

const debugActiveChildren = true

func (d *dir) forgetChild(name string, child node) {
	if name == "" {
		// unlinked
		return
	}

	d.mu.Lock()
	defer d.mu.Unlock()

	if debugActiveChildren {
		have, ok := d.active[name]
		switch {
		case !ok:
			log.Printf("asked to forget non-active child: %q %#v", name, child)
		case have != child:
			log.Printf("asked to forget wrong child: %q %#v", name, child)
		}
	}
	delete(d.active, name)
}

func (d *dir) Forget() {
	if d.parent == nil {
		// root dir, don't keep track
		return
	}

	d.mu.Lock()
	name := d.name
	d.mu.Unlock()

	d.parent.forgetChild(name, d)
}

const debugMkdirExisting = true

func (d *dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) {
	// TODO handle req.Mode

	var child node
	mkdir := func(tx *db.Tx) error {
		bucket := d.fs.bucket(tx)
		inode, err := inodes.Allocate(bucket.InodeBucket())
		if err != nil {
			return err
		}
		child = newDir(d.fs, inode, d, req.Name)
		if err := d.saveInternal(tx, req.Name, child); err != nil {
			return err
		}
		if _, err := bucket.Clock().Create(d.inode, req.Name, d.fs.dirtyEpoch()); err != nil {
			return err
		}
		return nil
	}
	if err := d.fs.db.Update(mkdir); err != nil {
		if err == inodes.ErrOutOfInodes {
			return nil, fuse.Errno(syscall.ENOSPC)
		}
		return nil, err
	}

	d.mu.Lock()
	defer d.mu.Unlock()
	if debugMkdirExisting {
		if n, ok := d.active[req.Name]; ok {
			log.Printf("asked to mkdir with existing node: %q %#v", req.Name, n)
			n.setName("")
		}
	}
	d.active[req.Name] = child
	return child, nil
}

func (d *dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {
	remove := func(tx *db.Tx) error {
		if err := d.fs.bucket(tx).Dirs().Delete(d.inode, req.Name); err != nil {
			return err
		}

		// TODO free inode
		return nil
	}
	if err := d.fs.db.Update(remove); err != nil {
		return err
	}

	d.mu.Lock()
	defer d.mu.Unlock()
	if node, ok := d.active[req.Name]; ok {
		delete(d.active, req.Name)
		node.setName("")
	}
	return nil
}

func (d *dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDir fs.Node) error {
	// if you ever change this, also guard against renaming into
	// special directories like .snap; check type of newDir is *dir
	//
	// also worry about deadlocks
	if newDir != d {
		return fuse.Errno(syscall.EXDEV)
	}

	rename := func(tx *db.Tx) error {
		// TODO don't need to load from db if req.OldName is in active.
		// instead, save active state if we have it; call .save() not this
		// kludge
		//
		// TODO don't need to load from db if req.NewName is in active
		loser, err := d.fs.bucket(tx).Dirs().Rename(d.inode, req.OldName, req.NewName)
		if err != nil {
			return err
		}

		if loser != nil {
			// TODO free loser inode
		}
		return nil
	}
	if err := d.fs.db.Update(rename); err != nil {
		return err
	}

	d.mu.Lock()
	defer d.mu.Unlock()

	// tell overwritten node it's unlinked
	if n, ok := d.active[req.NewName]; ok {
		n.setName("")
	}

	// if the source inode is active, record its new name
	if nodeOld, ok := d.active[req.OldName]; ok {
		nodeOld.setName(req.NewName)
		delete(d.active, req.OldName)
		d.active[req.NewName] = nodeOld
	}

	return nil
}

// snapshot records a snapshot of the directory and stores it in wde
func (d *dir) snapshot(ctx context.Context, tx *db.Tx) (*wiresnap.Dirent, error) {
	// NOT HOLDING THE LOCK, accessing database snapshot ONLY

	// TODO move bucket lookup to caller?
	bucket := d.fs.bucket(tx)

	manifest := blobs.EmptyManifest("dir")
	blob, err := blobs.Open(d.fs.chunkStore, manifest)
	if err != nil {
		return nil, err
	}
	w := snap.NewWriter(blob)

	c := bucket.Dirs().List(d.inode)
	for item := c.First(); item != nil; item = c.Next() {
		var de wire.Dirent
		if err := item.Unmarshal(&de); err != nil {
			return nil, err
		}
		var sde *wiresnap.Dirent
		switch {
		case de.File != nil:
			// TODO d.reviveNode would do blobs.Open and that's a bit
			// too much work; rework the apis
			sde = &wiresnap.Dirent{
				File: &wiresnap.File{
					Manifest: de.File.Manifest,
				},
			}
		case de.Dir != nil:
			child, err := d.reviveDir(&de, item.Name())
			if err != nil {
				return nil, err
			}
			sde, err = child.snapshot(ctx, tx)
			if err != nil {
				return nil, err
			}
		default:
			return nil, errors.New("TODO")
		}
		sde.Name = item.Name()
		err = w.Add(sde)
		if err != nil {
			return nil, err
		}
	}

	manifest, err = blob.Save()
	if err != nil {
		return nil, err
	}
	msg := wiresnap.Dirent{
		Dir: &wiresnap.Dir{
			Manifest: wirecas.FromBlob(manifest),
			Align:    w.Align(),
		},
	}
	return &msg, nil
}
<|endoftext|>


<commit_before>package fs

import (
	"errors"
	"os"
	"path"
	"path/filepath"
	"time"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/koding/fuseklient/transport"
)

var (
	ErrNotADir  = errors.New("Node is a not directory.")
	ErrNotAFile = errors.New("Node is a not file.")
)

type entry struct {
	Offset fuseops.DirOffset
	Name   string
	Type   fuseutil.DirentType
	Mode   os.FileMode
	Size   uint64
	Time   time.Time
}

type Dir struct {
	// Node is generic structure that contains commonality between File and Dir.
	*Inode

	// NodeIDGen is responsible for generating ids for newly created nodes.
	NodeIDGen *NodeIDGen

	////// Node#RWLock protects the fields below.

	// Entries contains list of files and directories that belong to this Node.
	// This list of empty if Node is a file.
	Entries []fuseutil.Dirent

	// EntriesList contains list of files and directories that belong to this Node
	// mapped by entry name for easy lookup. This list of empty if Node is a file.
	EntriesList map[string]Node
}

func NewDir(n *Inode, idGen *NodeIDGen) *Dir {
	return &Dir{
		Inode:       n,
		NodeIDGen:   idGen,
		Entries:     []fuseutil.Dirent{},
		EntriesList: map[string]Node{},
	}
}

func (d *Dir) ReadEntries(offset fuseops.DirOffset) ([]fuseutil.Dirent, error) {
	d.RLock()
	var entries = d.Entries
	d.RUnlock()

	if len(entries) == 0 {
		if err := d.updateEntriesFromRemote(); err != nil {
			return nil, err
		}
		entries = d.Entries
	}

	if offset > fuseops.DirOffset(len(entries)) {
		return nil, fuse.EIO
	}

	var liveEntries []fuseutil.Dirent
	for _, e := range entries[offset:] {
		if e.Type != fuseutil.DT_Unknown {
			liveEntries = append(liveEntries, e)
		}
	}

	return liveEntries, nil
}

func (d *Dir) FindEntry(name string) (Node, error) {
	return d.findEntry(name)
}

func (d *Dir) FindEntryFile(name string) (*File, error) {
	n, err := d.findEntry(name)
	if err != nil {
		return nil, err
	}

	f, ok := n.(*File)
	if !ok {
		return nil, ErrNotAFile
	}

	return f, nil
}

func (d *Dir) FindEntryDir(name string) (*Dir, error) {
	n, err := d.findEntry(name)
	if err != nil {
		return nil, err
	}

	d, ok := n.(*Dir)
	if !ok {
		return nil, ErrNotADir
	}

	return d, nil
}

func (d *Dir) CreateEntryDir(name string, mode os.FileMode) (*Dir, error) {
	if _, err := d.findEntry(name); err != fuse.ENOENT {
		return nil, fuse.EEXIST
	}

	req := struct {
		Path      string
		Recursive bool
	}{
		Path:      filepath.Join(d.RemotePath, name),
		Recursive: true,
	}
	var res bool
	if err := d.Trip("fs.createDirectory", req, &res); err != nil {
		return nil, err
	}

	e := &entry{
		Name: name,
		Type: fuseutil.DT_Directory,
		Mode: d.Attrs.Mode,
	}

	child, err := d.initializeChild(e)
	if err != nil {
		return nil, err
	}

	dir, ok := child.(*Dir)
	if !ok {
		return nil, ErrNotADir
	}

	dir.Attrs.Mode = mode

	return dir, nil
}

func (d *Dir) CreateEntryFile(name string, mode os.FileMode) (*File, error) {
	if _, err := d.findEntry(name); err != fuse.ENOENT {
		return nil, fuse.EEXIST
	}

	e := &entry{
		Name: name,
		Type: fuseutil.DT_File,
		Mode: d.Attrs.Mode,
	}

	child, err := d.initializeChild(e)
	if err != nil {
		return nil, err
	}

	file, ok := child.(*File)
	if !ok {
		return nil, ErrNotAFile
	}

	file.Attrs.Mode = mode

	if err := file.Create(); err != nil {
		return file, err
	}

	return file, nil
}

func (d *Dir) MoveEntry(oldName, newName string, newDir *Dir) error {
	child, err := d.findEntry(oldName)
	if err != nil {
		return err
	}

	if err := d.removeChild(oldName); err != nil {
		return err
	}

	req := struct{ OldPath, NewPath string }{
		OldPath: filepath.Join(d.RemotePath, oldName),
		NewPath: filepath.Join(newDir.RemotePath, newName),
	}
	var res bool

	if err := d.Trip("fs.rename", req, res); err != nil {
		return err
	}

	e := &entry{
		Name: newName,
		Type: child.GetType(),
		Mode: child.GetAttrs().Mode,
	}

	_, err = newDir.initializeChild(e)

	return err
}

func (d *Dir) RemoveEntry(name string) (Node, error) {
	entry, err := d.findEntry(name)
	if err != nil {
		return nil, err
	}

	req := struct {
		Path      string
		Recursive bool
	}{
		Path:      path.Join(d.RemotePath, name),
		Recursive: true,
	}
	var res bool
	if err := d.Trip("fs.remove", req, &res); err != nil {
		return nil, err
	}

	if err := d.removeChild(name); err != nil {
		return nil, err
	}

	return entry, nil
}

func (d *Dir) GetType() fuseutil.DirentType {
	return fuseutil.DT_Directory
}

func (d *Dir) IsADir() bool {
	return true
}

///// Helpers

func (d *Dir) findEntry(name string) (Node, error) {
	d.RLock()
	defer d.RUnlock()

	child, ok := d.EntriesList[name]
	if !ok {
		return nil, fuse.ENOENT
	}

	return child, nil
}

func (d *Dir) updateEntriesFromRemote() error {
	d.Lock()
	defer d.Unlock()

	entries, err := d.getEntriesFromRemote()
	if err != nil {
		return err
	}

	d.Entries = []fuseutil.Dirent{}
	d.EntriesList = map[string]Node{}

	for _, e := range entries {
		if _, err := d.initializeChild(e); err != nil {
			return err
		}
	}

	return nil
}

func (d *Dir) getEntriesFromRemote() ([]*entry, error) {
	req := struct{ Path string }{d.RemotePath}
	res := transport.FsReadDirectoryRes{}
	if err := d.Trip("fs.readDirectory", req, &res); err != nil {
		return nil, err
	}

	var entries []*entry
	for _, file := range res.Files {
		var fileType fuseutil.DirentType = fuseutil.DT_File
		if file.IsDir {
			fileType = fuseutil.DT_Directory
		}

		e := &entry{
			Name: file.Name,
			Type: fileType,
			Mode: file.Mode,
			Size: uint64(file.Size),
			Time: file.Time,
		}

		entries = append(entries, e)
	}

	return entries, nil
}

func (d *Dir) initializeChild(e *entry) (Node, error) {
	var t = e.Time
	if t.IsZero() {
		t = time.Now()
	}

	attrs := fuseops.InodeAttributes{
		Size:   e.Size,
		Uid:    d.Attrs.Uid,
		Gid:    d.Attrs.Gid,
		Mode:   e.Mode,
		Atime:  t,
		Mtime:  t,
		Ctime:  t,
		Crtime: t,
	}

	n := NewInode(d, e.Name)
	n.Attrs = attrs

	dirEntry := fuseutil.Dirent{
		Offset: fuseops.DirOffset(len(d.Entries)) + 1, // offset is 1 indexed
		Inode:  n.ID,
		Name:   e.Name,
		Type:   e.Type,
	}

	d.Entries = append(d.Entries, dirEntry)

	var dt Node
	switch e.Type {
	case fuseutil.DT_Directory:
		dt = NewDir(n, d.NodeIDGen)
	case fuseutil.DT_File:
		dt = NewFile(n)
	default:
	}

	d.EntriesList[e.Name] = dt

	return dt, nil
}

func (d *Dir) removeChild(name string) error {
	d.Lock()
	defer d.Unlock()

	listEntry, ok := d.EntriesList[name]
	if !ok {
		return fuse.ENOENT
	}

	listEntry.Forget()

	for index, mapEntry := range d.Entries {
		if mapEntry.Name == name {
			mapEntry.Type = fuseutil.DT_Unknown
			d.Entries[index] = mapEntry
		}
	}

	return nil
}
<commit_msg>delete entry from Dir#EntriesList when it's deleted<commit_after>package fs

import (
	"errors"
	"os"
	"path"
	"path/filepath"
	"time"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/koding/fuseklient/transport"
)

var (
	ErrNotADir  = errors.New("Node is a not directory.")
	ErrNotAFile = errors.New("Node is a not file.")
)

type entry struct {
	Offset fuseops.DirOffset
	Name   string
	Type   fuseutil.DirentType
	Mode   os.FileMode
	Size   uint64
	Time   time.Time
}

type Dir struct {
	// Node is generic structure that contains commonality between File and Dir.
	*Inode

	// NodeIDGen is responsible for generating ids for newly created nodes.
	NodeIDGen *NodeIDGen

	////// Node#RWLock protects the fields below.

	// Entries contains list of files and directories that belong to this Node.
	// This list of empty if Node is a file.
	Entries []fuseutil.Dirent

	// EntriesList contains list of files and directories that belong to this Node
	// mapped by entry name for easy lookup. This list of empty if Node is a file.
	EntriesList map[string]Node
}

func NewDir(n *Inode, idGen *NodeIDGen) *Dir {
	return &Dir{
		Inode:       n,
		NodeIDGen:   idGen,
		Entries:     []fuseutil.Dirent{},
		EntriesList: map[string]Node{},
	}
}

func (d *Dir) ReadEntries(offset fuseops.DirOffset) ([]fuseutil.Dirent, error) {
	d.RLock()
	var entries = d.Entries
	d.RUnlock()

	if len(entries) == 0 {
		if err := d.updateEntriesFromRemote(); err != nil {
			return nil, err
		}
		entries = d.Entries
	}

	if offset > fuseops.DirOffset(len(entries)) {
		return nil, fuse.EIO
	}

	var liveEntries []fuseutil.Dirent
	for _, e := range entries[offset:] {
		if e.Type != fuseutil.DT_Unknown {
			liveEntries = append(liveEntries, e)
		}
	}

	return liveEntries, nil
}

func (d *Dir) FindEntry(name string) (Node, error) {
	return d.findEntry(name)
}

func (d *Dir) FindEntryFile(name string) (*File, error) {
	n, err := d.findEntry(name)
	if err != nil {
		return nil, err
	}

	f, ok := n.(*File)
	if !ok {
		return nil, ErrNotAFile
	}

	return f, nil
}

func (d *Dir) FindEntryDir(name string) (*Dir, error) {
	n, err := d.findEntry(name)
	if err != nil {
		return nil, err
	}

	d, ok := n.(*Dir)
	if !ok {
		return nil, ErrNotADir
	}

	return d, nil
}

func (d *Dir) CreateEntryDir(name string, mode os.FileMode) (*Dir, error) {
	if _, err := d.findEntry(name); err != fuse.ENOENT {
		return nil, fuse.EEXIST
	}

	req := struct {
		Path      string
		Recursive bool
	}{
		Path:      filepath.Join(d.RemotePath, name),
		Recursive: true,
	}
	var res bool
	if err := d.Trip("fs.createDirectory", req, &res); err != nil {
		return nil, err
	}

	e := &entry{
		Name: name,
		Type: fuseutil.DT_Directory,
		Mode: d.Attrs.Mode,
	}

	child, err := d.initializeChild(e)
	if err != nil {
		return nil, err
	}

	dir, ok := child.(*Dir)
	if !ok {
		return nil, ErrNotADir
	}

	dir.Attrs.Mode = mode

	return dir, nil
}

func (d *Dir) CreateEntryFile(name string, mode os.FileMode) (*File, error) {
	if _, err := d.findEntry(name); err != fuse.ENOENT {
		return nil, fuse.EEXIST
	}

	e := &entry{
		Name: name,
		Type: fuseutil.DT_File,
		Mode: d.Attrs.Mode,
	}

	child, err := d.initializeChild(e)
	if err != nil {
		return nil, err
	}

	file, ok := child.(*File)
	if !ok {
		return nil, ErrNotAFile
	}

	file.Attrs.Mode = mode

	if err := file.Create(); err != nil {
		return file, err
	}

	return file, nil
}

func (d *Dir) MoveEntry(oldName, newName string, newDir *Dir) error {
	child, err := d.findEntry(oldName)
	if err != nil {
		return err
	}

	if err := d.removeChild(oldName); err != nil {
		return err
	}

	req := struct{ OldPath, NewPath string }{
		OldPath: filepath.Join(d.RemotePath, oldName),
		NewPath: filepath.Join(newDir.RemotePath, newName),
	}
	var res bool

	if err := d.Trip("fs.rename", req, res); err != nil {
		return err
	}

	e := &entry{
		Name: newName,
		Type: child.GetType(),
		Mode: child.GetAttrs().Mode,
	}

	_, err = newDir.initializeChild(e)

	return err
}

func (d *Dir) RemoveEntry(name string) (Node, error) {
	entry, err := d.findEntry(name)
	if err != nil {
		return nil, err
	}

	req := struct {
		Path      string
		Recursive bool
	}{
		Path:      path.Join(d.RemotePath, name),
		Recursive: true,
	}
	var res bool
	if err := d.Trip("fs.remove", req, &res); err != nil {
		return nil, err
	}

	if err := d.removeChild(name); err != nil {
		return nil, err
	}

	return entry, nil
}

func (d *Dir) GetType() fuseutil.DirentType {
	return fuseutil.DT_Directory
}

func (d *Dir) IsADir() bool {
	return true
}

///// Helpers

func (d *Dir) findEntry(name string) (Node, error) {
	d.RLock()
	defer d.RUnlock()

	child, ok := d.EntriesList[name]
	if !ok {
		return nil, fuse.ENOENT
	}

	return child, nil
}

func (d *Dir) updateEntriesFromRemote() error {
	d.Lock()
	defer d.Unlock()

	entries, err := d.getEntriesFromRemote()
	if err != nil {
		return err
	}

	d.Entries = []fuseutil.Dirent{}
	d.EntriesList = map[string]Node{}

	for _, e := range entries {
		if _, err := d.initializeChild(e); err != nil {
			return err
		}
	}

	return nil
}

func (d *Dir) getEntriesFromRemote() ([]*entry, error) {
	req := struct{ Path string }{d.RemotePath}
	res := transport.FsReadDirectoryRes{}
	if err := d.Trip("fs.readDirectory", req, &res); err != nil {
		return nil, err
	}

	var entries []*entry
	for _, file := range res.Files {
		var fileType fuseutil.DirentType = fuseutil.DT_File
		if file.IsDir {
			fileType = fuseutil.DT_Directory
		}

		e := &entry{
			Name: file.Name,
			Type: fileType,
			Mode: file.Mode,
			Size: uint64(file.Size),
			Time: file.Time,
		}

		entries = append(entries, e)
	}

	return entries, nil
}

func (d *Dir) initializeChild(e *entry) (Node, error) {
	var t = e.Time
	if t.IsZero() {
		t = time.Now()
	}

	attrs := fuseops.InodeAttributes{
		Size:   e.Size,
		Uid:    d.Attrs.Uid,
		Gid:    d.Attrs.Gid,
		Mode:   e.Mode,
		Atime:  t,
		Mtime:  t,
		Ctime:  t,
		Crtime: t,
	}

	n := NewInode(d, e.Name)
	n.Attrs = attrs

	dirEntry := fuseutil.Dirent{
		Offset: fuseops.DirOffset(len(d.Entries)) + 1, // offset is 1 indexed
		Inode:  n.ID,
		Name:   e.Name,
		Type:   e.Type,
	}

	d.Entries = append(d.Entries, dirEntry)

	var dt Node
	switch e.Type {
	case fuseutil.DT_Directory:
		dt = NewDir(n, d.NodeIDGen)
	case fuseutil.DT_File:
		dt = NewFile(n)
	default:
	}

	d.EntriesList[e.Name] = dt

	return dt, nil
}

func (d *Dir) removeChild(name string) error {
	d.Lock()
	defer d.Unlock()

	listEntry, ok := d.EntriesList[name]
	if !ok {
		return fuse.ENOENT
	}

	listEntry.Forget()

	delete(d.EntriesList, name)

	for index, mapEntry := range d.Entries {
		if mapEntry.Name == name {
			mapEntry.Type = fuseutil.DT_Unknown
			d.Entries[index] = mapEntry
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package job

import (
	"fmt"
	"os"
)

var tmpDirPattern = "/tmp/pdfcombiner/%s/"

func (j *Job) LocalPath() string {
	return j.workingDirectory + "combined.pdf"
}

// Make a randomized temporary directory to work in.
// Panics if it can't be created, since we can't continue.
func (j *Job) mkTmpDir() {
	j.workingDirectory = fmt.Sprintf(tmpDirPattern, j.Id())
	err := os.MkdirAll(j.workingDirectory, 0777)
	if err != nil {
		panic(err)
	}
	return
}

// Delete the tmp directory for this job.
func (j *Job) Cleanup() {
	if j.workingDirectory != "" && os.Getenv("DEBUG") == "" {
		os.RemoveAll(j.workingDirectory)
	}
}

// Get the absolute paths to the completed docs.
func (j *Job) ComponentPaths() (paths []string) {
	paths = make([]string, len(j.DocList))
	for idx, doc := range j.DocList {
		paths[idx] = doc.LocalPath()
	}
	return
}

// For every doc that failed for whatever reason, replace it with a page
// containing an error message.
func (j *Job) CleanupBrokenDocs() {
	for i, doc := range j.DocList {
		if !j.wasDownloaded(doc) {
			j.DocList[i].writeErrorDocument()
		}
	}
}

// Determine if the given Doc is present in the list of downloaded documents.
// In other words, find out if something went wrong.
func (j Job) wasDownloaded(inputDoc Document) bool {
	for _, doc := range j.Downloaded {
		if inputDoc.Key == doc.Key {
			return true
		}
	}
	return false
}
<commit_msg>Add a time-suffix to the temp directory<commit_after>package job

import (
	"fmt"
	"os"
	"time"
	"strconv"
)

var tmpDirPattern = "/tmp/pdfcombiner/%s.%s/"

func (j *Job) LocalPath() string {
	return j.workingDirectory + "combined.pdf"
}

// Make a temporary directory to work in based on the job dir plus the current
// time.  Panics if it can't be created, since we can't continue.
func (j *Job) mkTmpDir() {
	timeSuffix := strconv.FormatInt(time.Now().UnixNano(),10)[8:14]
	j.workingDirectory = fmt.Sprintf(tmpDirPattern, j.Id(), timeSuffix)
	err := os.MkdirAll(j.workingDirectory, 0777)
	if err != nil {
		panic(err)
	}
	return
}

// Delete the tmp directory for this job.
func (j *Job) Cleanup() {
	if j.workingDirectory != "" && os.Getenv("DEBUG") == "" {
		os.RemoveAll(j.workingDirectory)
	}
}

// Get the absolute paths to the completed docs.
func (j *Job) ComponentPaths() (paths []string) {
	paths = make([]string, len(j.DocList))
	for idx, doc := range j.DocList {
		paths[idx] = doc.LocalPath()
	}
	return
}

// For every doc that failed for whatever reason, replace it with a page
// containing an error message.
func (j *Job) CleanupBrokenDocs() {
	for i, doc := range j.DocList {
		if !j.wasDownloaded(doc) {
			j.DocList[i].writeErrorDocument()
		}
	}
}

// Determine if the given Doc is present in the list of downloaded documents.
// In other words, find out if something went wrong.
func (j Job) wasDownloaded(inputDoc Document) bool {
	for _, doc := range j.Downloaded {
		if inputDoc.Key == doc.Key {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2013,2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package backups_test

import (
	"bytes"
	"io/ioutil"
	"time"

	"github.com/juju/errors"
	jc "github.com/juju/testing/checkers"
	"github.com/juju/utils/set"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/state/backups"
	"github.com/juju/juju/state/backups/db"
	"github.com/juju/juju/state/backups/files"
	"github.com/juju/juju/state/backups/metadata"
	backupstesting "github.com/juju/juju/state/backups/testing"
)

type backupsSuite struct {
	backupstesting.BaseSuite

	api backups.Backups
}

var _ = gc.Suite(&backupsSuite{}) // Register the suite.

func (s *backupsSuite) SetUpTest(c *gc.C) {
	s.BaseSuite.SetUpTest(c)

	s.api = backups.NewBackups(s.Storage)
}

func (s *backupsSuite) setStored(id string) *time.Time {
	s.Storage.ID = id
	s.Storage.Meta = backupstesting.NewMetadataStarted(id, "")
	stored := time.Now().UTC()
	s.Storage.Meta.SetStored(&stored)
	return &stored
}

func (s *backupsSuite) checkFailure(c *gc.C, expected string) {
	paths := files.Paths{DataDir: "/var/lib/juju"}
	connInfo := db.ConnInfo{"a", "b", "c"}
	targets := set.NewStrings("juju", "admin")
	dbInfo := db.Info{connInfo, &targets}
	origin := metadata.NewOrigin("<env ID>", "<machine ID>", "<hostname>")
	_, err := s.api.Create(paths, dbInfo, *origin, "some notes")

	c.Check(err, gc.ErrorMatches, expected)
}

func (s *backupsSuite) TestNewBackups(c *gc.C) {
	api := backups.NewBackups(s.Storage)

	c.Check(api, gc.NotNil)
}

func (s *backupsSuite) TestCreateOkay(c *gc.C) {

	// Patch the internals.
	archiveFile := ioutil.NopCloser(bytes.NewBufferString("<compressed tarball>"))
	result := backups.NewTestCreateResult(archiveFile, 10, "<checksum>")
	received, testCreate := backups.NewTestCreate(result)
	s.PatchValue(backups.RunCreate, testCreate)

	rootDir := "<was never set>"
	s.PatchValue(backups.GetFilesToBackUp, func(root string, paths files.Paths) ([]string, error) {
		rootDir = root
		return []string{"<some file>"}, nil
	})

	var receivedDBInfo *db.Info
	s.PatchValue(backups.GetDBDumper, func(info db.Info) (db.Dumper, error) {
		receivedDBInfo = &info
		return nil, nil
	})

	stored := s.setStored("spam")

	// Run the backup.
	paths := files.Paths{DataDir: "/var/lib/juju"}
	connInfo := db.ConnInfo{"a", "b", "c"}
	targets := set.NewStrings("juju", "admin")
	dbInfo := db.Info{connInfo, &targets}
	origin := metadata.NewOrigin("<env ID>", "<machine ID>", "<hostname>")
	meta, err := s.api.Create(paths, dbInfo, *origin, "some notes")

	// Test the call values.
	s.Storage.CheckCalled(c, "spam", meta, archiveFile, "Add", "Metadata")
	filesToBackUp, _ := backups.ExposeCreateArgs(received)
	c.Check(filesToBackUp, jc.SameContents, []string{"<some file>"})

	err = receivedDBInfo.Validate()
	c.Assert(err, gc.IsNil)
	c.Check(receivedDBInfo.Address, gc.Equals, "a")
	c.Check(receivedDBInfo.Username, gc.Equals, "b")
	c.Check(receivedDBInfo.Password, gc.Equals, "c")
	c.Check(*receivedDBInfo.Targets, gc.DeepEquals, targets)

	c.Check(rootDir, gc.Equals, "")

	// Check the resulting metadata.
	c.Check(meta, gc.Equals, s.Storage.MetaArg)
	c.Check(meta.ID(), gc.Equals, "spam")
	c.Check(meta.Size(), gc.Equals, int64(10))
	c.Check(meta.Checksum(), gc.Equals, "<checksum>")
	c.Check(meta.Stored().Unix(), gc.Equals, stored.Unix())
	c.Check(meta.Origin.Environment, gc.Equals, "<env ID>")
	c.Check(meta.Origin.Machine, gc.Equals, "<machine ID>")
	c.Check(meta.Origin.Hostname, gc.Equals, "<hostname>")
	c.Check(meta.Notes, gc.Equals, "some notes")

	// Check the file storage.
	s.Storage.Meta = meta
	s.Storage.File = archiveFile
	storedMeta, storedFile, err := s.Storage.Get(meta.ID())
	c.Check(err, gc.IsNil)
	c.Check(storedMeta, gc.DeepEquals, meta)
	data, err := ioutil.ReadAll(storedFile)
	c.Assert(err, gc.IsNil)
	c.Check(string(data), gc.Equals, "<compressed tarball>")
}

func (s *backupsSuite) TestCreateFailToListFiles(c *gc.C) {
	s.PatchValue(backups.GetFilesToBackUp, func(root string, paths files.Paths) ([]string, error) {
		return nil, errors.New("failed!")
	})

	s.checkFailure(c, "while listing files to back up: failed!")
}

func (s *backupsSuite) TestCreateFailToCreate(c *gc.C) {
	s.PatchValue(backups.GetFilesToBackUp, func(root string, paths files.Paths) ([]string, error) {
		return []string{}, nil
	})
	s.PatchValue(backups.RunCreate, backups.NewTestCreateFailure("failed!"))

	s.checkFailure(c, "while creating backup archive: failed!")
}

func (s *backupsSuite) TestCreateFailToFinishMeta(c *gc.C) {
	s.PatchValue(backups.GetFilesToBackUp, func(root string, paths files.Paths) ([]string, error) {
		return []string{}, nil
	})
	_, testCreate := backups.NewTestCreate(nil)
	s.PatchValue(backups.RunCreate, testCreate)
	s.PatchValue(backups.FinishMeta, backups.NewTestMetaFinisher("failed!"))

	s.checkFailure(c, "while updating metadata: failed!")
}

func (s *backupsSuite) TestCreateFailToStoreArchive(c *gc.C) {
	s.PatchValue(backups.GetFilesToBackUp, func(root string, paths files.Paths) ([]string, error) {
		return []string{}, nil
	})
	_, testCreate := backups.NewTestCreate(nil)
	s.PatchValue(backups.RunCreate, testCreate)
	s.PatchValue(backups.FinishMeta, backups.NewTestMetaFinisher(""))
	s.PatchValue(backups.StoreArchiveRef, backups.NewTestArchiveStorer("failed!"))

	s.checkFailure(c, "while storing backup archive: failed!")
}

func (s *backupsSuite) TestStoreArchive(c *gc.C) {
	stored := s.setStored("spam")

	meta := backupstesting.NewMetadataStarted("", "")
	c.Assert(meta.ID(), gc.Equals, "")
	c.Assert(meta.Stored(), gc.IsNil)
	archive := &bytes.Buffer{}
	err := backups.StoreArchive(s.Storage, meta, archive)
	c.Assert(err, gc.IsNil)

	s.Storage.CheckCalled(c, "spam", meta, archive, "Add", "Metadata")
	c.Assert(meta.ID(), gc.Equals, "spam")
	c.Assert(meta.Stored(), jc.DeepEquals, stored)
}
<commit_msg>Fix tests.<commit_after>// Copyright 2013,2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package backups_test

import (
	"bytes"
	"io/ioutil"
	"time"

	"github.com/juju/errors"
	jc "github.com/juju/testing/checkers"
	"github.com/juju/utils/set"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/state/backups"
	"github.com/juju/juju/state/backups/db"
	"github.com/juju/juju/state/backups/files"
	"github.com/juju/juju/state/backups/metadata"
	backupstesting "github.com/juju/juju/state/backups/testing"
)

type backupsSuite struct {
	backupstesting.BaseSuite

	api backups.Backups
}

var _ = gc.Suite(&backupsSuite{}) // Register the suite.

func (s *backupsSuite) SetUpTest(c *gc.C) {
	s.BaseSuite.SetUpTest(c)

	s.api = backups.NewBackups(s.Storage)
}

func (s *backupsSuite) setStored(id string) *time.Time {
	s.Storage.ID = id
	s.Storage.Meta = backupstesting.NewMetadataStarted(id, "")
	stored := time.Now().UTC()
	s.Storage.Meta.SetStored(&stored)
	return &stored
}

type fakeDumper struct{}

func (*fakeDumper) Dump(dumpDir string) error {
	return nil
}

func (s *backupsSuite) checkFailure(c *gc.C, expected string) {
	s.PatchValue(backups.GetDBDumper, func(db.Info) (db.Dumper, error) {
		return &fakeDumper{}, nil
	})

	paths := files.Paths{DataDir: "/var/lib/juju"}
	connInfo := db.ConnInfo{"a", "b", "c"}
	targets := set.NewStrings("juju", "admin")
	dbInfo := db.Info{connInfo, &targets}
	origin := metadata.NewOrigin("<env ID>", "<machine ID>", "<hostname>")
	_, err := s.api.Create(paths, dbInfo, *origin, "some notes")

	c.Check(err, gc.ErrorMatches, expected)
}

func (s *backupsSuite) TestNewBackups(c *gc.C) {
	api := backups.NewBackups(s.Storage)

	c.Check(api, gc.NotNil)
}

func (s *backupsSuite) TestCreateOkay(c *gc.C) {

	// Patch the internals.
	archiveFile := ioutil.NopCloser(bytes.NewBufferString("<compressed tarball>"))
	result := backups.NewTestCreateResult(archiveFile, 10, "<checksum>")
	received, testCreate := backups.NewTestCreate(result)
	s.PatchValue(backups.RunCreate, testCreate)

	rootDir := "<was never set>"
	s.PatchValue(backups.GetFilesToBackUp, func(root string, paths files.Paths) ([]string, error) {
		rootDir = root
		return []string{"<some file>"}, nil
	})

	var receivedDBInfo *db.Info
	s.PatchValue(backups.GetDBDumper, func(info db.Info) (db.Dumper, error) {
		receivedDBInfo = &info
		return nil, nil
	})

	stored := s.setStored("spam")

	// Run the backup.
	paths := files.Paths{DataDir: "/var/lib/juju"}
	connInfo := db.ConnInfo{"a", "b", "c"}
	targets := set.NewStrings("juju", "admin")
	dbInfo := db.Info{connInfo, &targets}
	origin := metadata.NewOrigin("<env ID>", "<machine ID>", "<hostname>")
	meta, err := s.api.Create(paths, dbInfo, *origin, "some notes")

	// Test the call values.
	s.Storage.CheckCalled(c, "spam", meta, archiveFile, "Add", "Metadata")
	filesToBackUp, _ := backups.ExposeCreateArgs(received)
	c.Check(filesToBackUp, jc.SameContents, []string{"<some file>"})

	err = receivedDBInfo.Validate()
	c.Assert(err, gc.IsNil)
	c.Check(receivedDBInfo.Address, gc.Equals, "a")
	c.Check(receivedDBInfo.Username, gc.Equals, "b")
	c.Check(receivedDBInfo.Password, gc.Equals, "c")
	c.Check(*receivedDBInfo.Targets, gc.DeepEquals, targets)

	c.Check(rootDir, gc.Equals, "")

	// Check the resulting metadata.
	c.Check(meta, gc.Equals, s.Storage.MetaArg)
	c.Check(meta.ID(), gc.Equals, "spam")
	c.Check(meta.Size(), gc.Equals, int64(10))
	c.Check(meta.Checksum(), gc.Equals, "<checksum>")
	c.Check(meta.Stored().Unix(), gc.Equals, stored.Unix())
	c.Check(meta.Origin.Environment, gc.Equals, "<env ID>")
	c.Check(meta.Origin.Machine, gc.Equals, "<machine ID>")
	c.Check(meta.Origin.Hostname, gc.Equals, "<hostname>")
	c.Check(meta.Notes, gc.Equals, "some notes")

	// Check the file storage.
	s.Storage.Meta = meta
	s.Storage.File = archiveFile
	storedMeta, storedFile, err := s.Storage.Get(meta.ID())
	c.Check(err, gc.IsNil)
	c.Check(storedMeta, gc.DeepEquals, meta)
	data, err := ioutil.ReadAll(storedFile)
	c.Assert(err, gc.IsNil)
	c.Check(string(data), gc.Equals, "<compressed tarball>")
}

func (s *backupsSuite) TestCreateFailToListFiles(c *gc.C) {
	s.PatchValue(backups.GetFilesToBackUp, func(root string, paths files.Paths) ([]string, error) {
		return nil, errors.New("failed!")
	})

	s.checkFailure(c, "while listing files to back up: failed!")
}

func (s *backupsSuite) TestCreateFailToCreate(c *gc.C) {
	s.PatchValue(backups.GetFilesToBackUp, func(root string, paths files.Paths) ([]string, error) {
		return []string{}, nil
	})
	s.PatchValue(backups.RunCreate, backups.NewTestCreateFailure("failed!"))

	s.checkFailure(c, "while creating backup archive: failed!")
}

func (s *backupsSuite) TestCreateFailToFinishMeta(c *gc.C) {
	s.PatchValue(backups.GetFilesToBackUp, func(root string, paths files.Paths) ([]string, error) {
		return []string{}, nil
	})
	_, testCreate := backups.NewTestCreate(nil)
	s.PatchValue(backups.RunCreate, testCreate)
	s.PatchValue(backups.FinishMeta, backups.NewTestMetaFinisher("failed!"))

	s.checkFailure(c, "while updating metadata: failed!")
}

func (s *backupsSuite) TestCreateFailToStoreArchive(c *gc.C) {
	s.PatchValue(backups.GetFilesToBackUp, func(root string, paths files.Paths) ([]string, error) {
		return []string{}, nil
	})
	_, testCreate := backups.NewTestCreate(nil)
	s.PatchValue(backups.RunCreate, testCreate)
	s.PatchValue(backups.FinishMeta, backups.NewTestMetaFinisher(""))
	s.PatchValue(backups.StoreArchiveRef, backups.NewTestArchiveStorer("failed!"))

	s.checkFailure(c, "while storing backup archive: failed!")
}

func (s *backupsSuite) TestStoreArchive(c *gc.C) {
	stored := s.setStored("spam")

	meta := backupstesting.NewMetadataStarted("", "")
	c.Assert(meta.ID(), gc.Equals, "")
	c.Assert(meta.Stored(), gc.IsNil)
	archive := &bytes.Buffer{}
	err := backups.StoreArchive(s.Storage, meta, archive)
	c.Assert(err, gc.IsNil)

	s.Storage.CheckCalled(c, "spam", meta, archive, "Add", "Metadata")
	c.Assert(meta.ID(), gc.Equals, "spam")
	c.Assert(meta.Stored(), jc.DeepEquals, stored)
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package testing

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"io"

	"github.com/juju/errors"

	"github.com/juju/juju/state/backups/metadata"
)

// File represents a file during testing.
type File struct {
	// Name is the path to which the file will be identified in the archive.
	Name string
	// Content is the data that will be written to the archive for the file.
	Content string
}

// Add the file to the tar archive.
func (f *File) AddToArchive(archive *tar.Writer) error {
	hdr := &tar.Header{
		Name: f.Name,
		Size: int64(len(f.Content)),
	}
	if err := archive.WriteHeader(hdr); err != nil {
		return errors.Trace(err)
	}
	if _, err := archive.Write([]byte(f.Content)); err != nil {
		return errors.Trace(err)
	}
	return nil
}

// NewArchive returns a new archive file containing the files.
func NewArchive(meta *metadata.Metadata, files, dump []File) (*bytes.Buffer, error) {
	var rootFile bytes.Buffer
	if err := writeToTar(&rootFile, files); err != nil {
		return nil, errors.Trace(err)
	}

	metaFile, err := meta.AsJSONBuffer()
	if err != nil {
		return nil, errors.Trace(err)
	}

	topfiles := append(dump,
		File{
			Name:    "juju-backup/root.tar",
			Content: rootFile.String(),
		},
		File{
			Name:    "juju-backup/metadata.json",
			Content: metaFile.(*bytes.Buffer).String(),
		},
	)

	var arFile bytes.Buffer
	compressed := gzip.NewWriter(&arFile)
	defer compressed.Close()
	if err := writeToTar(compressed, topfiles); err != nil {
		return nil, errors.Trace(err)
	}

	return &arFile, nil
}

// NewArchiveBasic returns a new archive file with a few files provided.
func NewArchiveBasic(meta *metadata.Metadata) (*bytes.Buffer, error) {
	files := []File{
		File{
			Name:    "var/lib/juju/tools/1.21-alpha2.1-trusty-amd64/jujud",
			Content: "<binary data>",
		},
		File{
			Name:    "var/lib/juju/system-identity",
			Content: "<ssh key>",
		},
	}
	dump := []File{
		File{
			Name:    "juju-backup/dump/juju/machines.bson",
			Content: "<db dump>",
		},
		File{
			Name:    "juju-backup/dump/oplog.bson",
			Content: "<db dump>",
		},
	}

	arFile, err := NewArchive(meta, files, dump)
	if err != nil {
		return nil, errors.Trace(err)
	}
	return arFile, nil
}

func writeToTar(archive io.Writer, files []File) error {
	tarw := tar.NewWriter(archive)
	defer tarw.Close()

	for _, file := range files {
		if err := file.AddToArchive(tarw); err != nil {
			return errors.Trace(err)
		}
	}
	return nil
}
<commit_msg>Clarify the content place-holders.<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package testing

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"io"

	"github.com/juju/errors"

	"github.com/juju/juju/state/backups/metadata"
)

// File represents a file during testing.
type File struct {
	// Name is the path to which the file will be identified in the archive.
	Name string
	// Content is the data that will be written to the archive for the file.
	Content string
}

// Add the file to the tar archive.
func (f *File) AddToArchive(archive *tar.Writer) error {
	hdr := &tar.Header{
		Name: f.Name,
		Size: int64(len(f.Content)),
	}
	if err := archive.WriteHeader(hdr); err != nil {
		return errors.Trace(err)
	}
	if _, err := archive.Write([]byte(f.Content)); err != nil {
		return errors.Trace(err)
	}
	return nil
}

// NewArchive returns a new archive file containing the files.
func NewArchive(meta *metadata.Metadata, files, dump []File) (*bytes.Buffer, error) {
	var rootFile bytes.Buffer
	if err := writeToTar(&rootFile, files); err != nil {
		return nil, errors.Trace(err)
	}

	metaFile, err := meta.AsJSONBuffer()
	if err != nil {
		return nil, errors.Trace(err)
	}

	topfiles := append(dump,
		File{
			Name:    "juju-backup/root.tar",
			Content: rootFile.String(),
		},
		File{
			Name:    "juju-backup/metadata.json",
			Content: metaFile.(*bytes.Buffer).String(),
		},
	)

	var arFile bytes.Buffer
	compressed := gzip.NewWriter(&arFile)
	defer compressed.Close()
	if err := writeToTar(compressed, topfiles); err != nil {
		return nil, errors.Trace(err)
	}

	return &arFile, nil
}

// NewArchiveBasic returns a new archive file with a few files provided.
func NewArchiveBasic(meta *metadata.Metadata) (*bytes.Buffer, error) {
	files := []File{
		File{
			Name:    "var/lib/juju/tools/1.21-alpha2.1-trusty-amd64/jujud",
			Content: "<some binary data goes here>",
		},
		File{
			Name:    "var/lib/juju/system-identity",
			Content: "<an ssh key goes here>",
		},
	}
	dump := []File{
		File{
			Name:    "juju-backup/dump/juju/machines.bson",
			Content: "<BSON data goes here>",
		},
		File{
			Name:    "juju-backup/dump/oplog.bson",
			Content: "<BSON data goes here>",
		},
	}

	arFile, err := NewArchive(meta, files, dump)
	if err != nil {
		return nil, errors.Trace(err)
	}
	return arFile, nil
}

func writeToTar(archive io.Writer, files []File) error {
	tarw := tar.NewWriter(archive)
	defer tarw.Close()

	for _, file := range files {
		if err := file.AddToArchive(tarw); err != nil {
			return errors.Trace(err)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

/*
   Copyright 2012-2013 Ask Bjørn Hansen

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

import (
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"os/signal"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"strings"
	"time"
)

// VERSION is the current version of GeoDNS
var VERSION string = "2.4.4"
var buildTime string
var gitVersion string

var serverID string
var serverIP string
var serverGroups []string

var timeStarted = time.Now()

var (
	flagconfig      = flag.String("config", "./dns/", "directory of zone files")
	flagcheckconfig = flag.Bool("checkconfig", false, "check configuration and exit")
	flagidentifier  = flag.String("identifier", "", "identifier (hostname, pop name or similar)")
	flaginter       = flag.String("interface", "*", "set the listener address")
	flagport        = flag.String("port", "53", "default port number")
	flaghttp        = flag.String("http", ":8053", "http listen address (:8053)")
	flaglog         = flag.Bool("log", false, "be more verbose")
	flagcpus        = flag.Int("cpus", 1, "Set the maximum number of CPUs to use")

	flagShowVersion = flag.Bool("version", false, "Show dnsconfig version")

	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
	memprofile = flag.String("memprofile", "", "write memory profile to this file")
)

func init() {
	if len(gitVersion) > 0 {
		VERSION = VERSION + "/" + gitVersion
	}

	log.SetPrefix("geodns ")
	log.SetFlags(log.Lmicroseconds | log.Lshortfile)
}

func main() {
	flag.Parse()

	if *flagShowVersion {
		fmt.Println("geodns", VERSION, buildTime)
		os.Exit(0)
	}

	if len(*flagidentifier) > 0 {
		ids := strings.Split(*flagidentifier, ",")
		serverID = ids[0]
		if len(ids) > 1 {
			serverGroups = ids[1:]
		}
	}

	configFileName := filepath.Clean(*flagconfig + "/geodns.conf")

	if *flagcheckconfig {
		dirName := *flagconfig

		err := configReader(configFileName)
		if err != nil {
			log.Println("Errors reading config", err)
			os.Exit(2)
		}

		Zones := make(Zones)
		setupPgeodnsZone(Zones)
		err = zonesReadDir(dirName, Zones)
		if err != nil {
			log.Println("Errors reading zones", err)
			os.Exit(2)
		}
		return
	}

	if *flagcpus == 0 {
		runtime.GOMAXPROCS(runtime.NumCPU())
	} else {
		runtime.GOMAXPROCS(*flagcpus)
	}

	log.Printf("Starting geodns %s\n", VERSION)

	if *cpuprofile != "" {
		prof, err := os.Create(*cpuprofile)
		if err != nil {
			panic(err.Error())
		}

		pprof.StartCPUProfile(prof)
		defer func() {
			log.Println("closing file")
			prof.Close()
		}()
		defer func() {
			log.Println("stopping profile")
			pprof.StopCPUProfile()
		}()
	}

	go configWatcher(configFileName)

	metrics := NewMetrics()
	go metrics.Updater()

	go statHatPoster()

	if *flaginter == "*" {
		addrs, _ := net.InterfaceAddrs()
		ips := make([]string, 0)
		for _, addr := range addrs {
			ip, _, err := net.ParseCIDR(addr.String())
			if err != nil {
				continue
			}
			if !(ip.IsLoopback() || ip.IsGlobalUnicast()) {
				continue
			}
			ips = append(ips, ip.String())
		}
		*flaginter = strings.Join(ips, ",")
	}

	inter := getInterfaces()

	Zones := make(Zones)

	go monitor(Zones)
	go Zones.statHatPoster()

	setupPgeodnsZone(Zones)

	dirName := *flagconfig
	go zonesReader(dirName, Zones)

	for _, host := range inter {
		go listenAndServe(host)
	}

	terminate := make(chan os.Signal)
	signal.Notify(terminate, os.Interrupt)

	<-terminate
	log.Printf("geodns: signal received, stopping")

	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.WriteHeapProfile(f)
		f.Close()
	}

}
<commit_msg>Sample memory in smaller chunks when using memory profiling<commit_after>package main

/*
   Copyright 2012-2013 Ask Bjørn Hansen

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

import (
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"os/signal"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"strings"
	"time"
)

// VERSION is the current version of GeoDNS
var VERSION string = "2.4.4"
var buildTime string
var gitVersion string

var serverID string
var serverIP string
var serverGroups []string

var timeStarted = time.Now()

var (
	flagconfig      = flag.String("config", "./dns/", "directory of zone files")
	flagcheckconfig = flag.Bool("checkconfig", false, "check configuration and exit")
	flagidentifier  = flag.String("identifier", "", "identifier (hostname, pop name or similar)")
	flaginter       = flag.String("interface", "*", "set the listener address")
	flagport        = flag.String("port", "53", "default port number")
	flaghttp        = flag.String("http", ":8053", "http listen address (:8053)")
	flaglog         = flag.Bool("log", false, "be more verbose")
	flagcpus        = flag.Int("cpus", 1, "Set the maximum number of CPUs to use")

	flagShowVersion = flag.Bool("version", false, "Show dnsconfig version")

	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
	memprofile = flag.String("memprofile", "", "write memory profile to this file")
)

func init() {
	if len(gitVersion) > 0 {
		VERSION = VERSION + "/" + gitVersion
	}

	if *memprofile != "" {
		runtime.MemProfileRate = 1024
	}

	log.SetPrefix("geodns ")
	log.SetFlags(log.Lmicroseconds | log.Lshortfile)
}

func main() {
	flag.Parse()

	if *flagShowVersion {
		fmt.Println("geodns", VERSION, buildTime)
		os.Exit(0)
	}

	if len(*flagidentifier) > 0 {
		ids := strings.Split(*flagidentifier, ",")
		serverID = ids[0]
		if len(ids) > 1 {
			serverGroups = ids[1:]
		}
	}

	configFileName := filepath.Clean(*flagconfig + "/geodns.conf")

	if *flagcheckconfig {
		dirName := *flagconfig

		err := configReader(configFileName)
		if err != nil {
			log.Println("Errors reading config", err)
			os.Exit(2)
		}

		Zones := make(Zones)
		setupPgeodnsZone(Zones)
		err = zonesReadDir(dirName, Zones)
		if err != nil {
			log.Println("Errors reading zones", err)
			os.Exit(2)
		}
		return
	}

	if *flagcpus == 0 {
		runtime.GOMAXPROCS(runtime.NumCPU())
	} else {
		runtime.GOMAXPROCS(*flagcpus)
	}

	log.Printf("Starting geodns %s\n", VERSION)

	if *cpuprofile != "" {
		prof, err := os.Create(*cpuprofile)
		if err != nil {
			panic(err.Error())
		}

		pprof.StartCPUProfile(prof)
		defer func() {
			log.Println("closing file")
			prof.Close()
		}()
		defer func() {
			log.Println("stopping profile")
			pprof.StopCPUProfile()
		}()
	}

	go configWatcher(configFileName)

	metrics := NewMetrics()
	go metrics.Updater()

	go statHatPoster()

	if *flaginter == "*" {
		addrs, _ := net.InterfaceAddrs()
		ips := make([]string, 0)
		for _, addr := range addrs {
			ip, _, err := net.ParseCIDR(addr.String())
			if err != nil {
				continue
			}
			if !(ip.IsLoopback() || ip.IsGlobalUnicast()) {
				continue
			}
			ips = append(ips, ip.String())
		}
		*flaginter = strings.Join(ips, ",")
	}

	inter := getInterfaces()

	Zones := make(Zones)

	go monitor(Zones)
	go Zones.statHatPoster()

	setupPgeodnsZone(Zones)

	dirName := *flagconfig
	go zonesReader(dirName, Zones)

	for _, host := range inter {
		go listenAndServe(host)
	}

	terminate := make(chan os.Signal)
	signal.Notify(terminate, os.Interrupt)

	<-terminate
	log.Printf("geodns: signal received, stopping")

	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.WriteHeapProfile(f)
		f.Close()
	}

}
<|endoftext|>


<commit_before>package resource

import (
	"errors"
	"io"
	"net/url"
	"os"

	"code.google.com/p/goauth2/oauth"

	"github.com/zachgersh/go-github/github"
)

//go:generate counterfeiter . GitHub

type GitHub interface {
	ListReleases() ([]github.RepositoryRelease, error)
	GetReleaseByTag(tag string) (*github.RepositoryRelease, error)
	GetRelease(id int) (*github.RepositoryRelease, error)
	CreateRelease(release github.RepositoryRelease) (*github.RepositoryRelease, error)
	UpdateRelease(release github.RepositoryRelease) (*github.RepositoryRelease, error)

	ListReleaseAssets(release github.RepositoryRelease) ([]github.ReleaseAsset, error)
	UploadReleaseAsset(release github.RepositoryRelease, name string, file *os.File) error
	DeleteReleaseAsset(asset github.ReleaseAsset) error
	DownloadReleaseAsset(asset github.ReleaseAsset) (io.ReadCloser, error)

	GetTarballLink(tag string) (*url.URL, error)
	GetZipballLink(tag string) (*url.URL, error)
}

type GitHubClient struct {
	client *github.Client

	user       string
	repository string
}

func NewGitHubClient(source Source) (*GitHubClient, error) {
	transport := &oauth.Transport{
		Token: &oauth.Token{
			AccessToken: source.AccessToken,
		},
	}

	var client *github.Client

	if transport.Token.AccessToken == "" {
		client = github.NewClient(nil)
	} else {
		client = github.NewClient(transport.Client())
	}

	if source.GitHubAPIURL != "" {
		var err error
		client.BaseURL, err = url.Parse(source.GitHubAPIURL)
		if err != nil {
			return nil, err
		}
	}

	return &GitHubClient{
		client:     client,
		user:       source.User,
		repository: source.Repository,
	}, nil
}

func (g *GitHubClient) ListReleases() ([]github.RepositoryRelease, error) {
	releases, res, err := g.client.Repositories.ListReleases(g.user, g.repository, nil)
	if err != nil {
		return []github.RepositoryRelease{}, err
	}

	err = res.Body.Close()
	if err != nil {
		return nil, err
	}

	return releases, nil
}

func (g *GitHubClient) GetReleaseByTag(tag string) (*github.RepositoryRelease, error) {
	release, res, err := g.client.Repositories.GetReleaseByTag(g.user, g.repository, tag)
	if err != nil {
		return &github.RepositoryRelease{}, nil
	}

	err = res.Body.Close()
	if err != nil {
		return nil, err
	}

	return release, nil
}

func (g *GitHubClient) GetRelease(id int) (*github.RepositoryRelease, error) {
	release, res, err := g.client.Repositories.GetRelease(g.user, g.repository, id)
	if err != nil {
		return &github.RepositoryRelease{}, nil
	}

	err = res.Body.Close()
	if err != nil {
		return nil, err
	}

	return release, nil
}

func (g *GitHubClient) CreateRelease(release github.RepositoryRelease) (*github.RepositoryRelease, error) {
	createdRelease, res, err := g.client.Repositories.CreateRelease(g.user, g.repository, &release)
	if err != nil {
		return &github.RepositoryRelease{}, err
	}

	err = res.Body.Close()
	if err != nil {
		return nil, err
	}

	return createdRelease, nil
}

func (g *GitHubClient) UpdateRelease(release github.RepositoryRelease) (*github.RepositoryRelease, error) {
	if release.ID == nil {
		return nil, errors.New("release did not have an ID: has it been saved yet?")
	}

	updatedRelease, res, err := g.client.Repositories.EditRelease(g.user, g.repository, *release.ID, &release)
	if err != nil {
		return &github.RepositoryRelease{}, err
	}

	err = res.Body.Close()
	if err != nil {
		return nil, err
	}

	return updatedRelease, nil
}

func (g *GitHubClient) ListReleaseAssets(release github.RepositoryRelease) ([]github.ReleaseAsset, error) {
	assets, res, err := g.client.Repositories.ListReleaseAssets(g.user, g.repository, *release.ID, nil)
	if err != nil {
		return []github.ReleaseAsset{}, nil
	}

	err = res.Body.Close()
	if err != nil {
		return nil, err
	}

	return assets, nil
}

func (g *GitHubClient) UploadReleaseAsset(release github.RepositoryRelease, name string, file *os.File) error {
	_, res, err := g.client.Repositories.UploadReleaseAsset(
		g.user,
		g.repository,
		*release.ID,
		&github.UploadOptions{
			Name: name,
		},
		file,
	)
	if err != nil {
		return err
	}

	return res.Body.Close()
}

func (g *GitHubClient) DeleteReleaseAsset(asset github.ReleaseAsset) error {
	res, err := g.client.Repositories.DeleteReleaseAsset(g.user, g.repository, *asset.ID)
	if err != nil {
		return err
	}

	return res.Body.Close()
}

func (g *GitHubClient) DownloadReleaseAsset(asset github.ReleaseAsset) (io.ReadCloser, error) {
	res, err := g.client.Repositories.DownloadReleaseAsset(g.user, g.repository, *asset.ID)
	if err != nil {
		return nil, err
	}

	return res, err
}

func (g *GitHubClient) GetTarballLink(tag string) (*url.URL, error) {
	opt := &github.RepositoryContentGetOptions{Ref: tag}
	u, res, err := g.client.Repositories.GetArchiveLink(g.user, g.repository, github.Tarball, opt)
	if err != nil {
		return nil, err
	}
	res.Body.Close()
	return u, nil
}

func (g *GitHubClient) GetZipballLink(tag string) (*url.URL, error) {
	opt := &github.RepositoryContentGetOptions{Ref: tag}
	u, res, err := g.client.Repositories.GetArchiveLink(g.user, g.repository, github.Zipball, opt)
	if err != nil {
		return nil, err
	}
	res.Body.Close()
	return u, nil
}
<commit_msg>return errors in cases where there may be errors<commit_after>package resource

import (
	"errors"
	"io"
	"net/url"
	"os"

	"code.google.com/p/goauth2/oauth"

	"github.com/zachgersh/go-github/github"
)

//go:generate counterfeiter . GitHub

type GitHub interface {
	ListReleases() ([]github.RepositoryRelease, error)
	GetReleaseByTag(tag string) (*github.RepositoryRelease, error)
	GetRelease(id int) (*github.RepositoryRelease, error)
	CreateRelease(release github.RepositoryRelease) (*github.RepositoryRelease, error)
	UpdateRelease(release github.RepositoryRelease) (*github.RepositoryRelease, error)

	ListReleaseAssets(release github.RepositoryRelease) ([]github.ReleaseAsset, error)
	UploadReleaseAsset(release github.RepositoryRelease, name string, file *os.File) error
	DeleteReleaseAsset(asset github.ReleaseAsset) error
	DownloadReleaseAsset(asset github.ReleaseAsset) (io.ReadCloser, error)

	GetTarballLink(tag string) (*url.URL, error)
	GetZipballLink(tag string) (*url.URL, error)
}

type GitHubClient struct {
	client *github.Client

	user       string
	repository string
}

func NewGitHubClient(source Source) (*GitHubClient, error) {
	transport := &oauth.Transport{
		Token: &oauth.Token{
			AccessToken: source.AccessToken,
		},
	}

	var client *github.Client

	if transport.Token.AccessToken == "" {
		client = github.NewClient(nil)
	} else {
		client = github.NewClient(transport.Client())
	}

	if source.GitHubAPIURL != "" {
		var err error
		client.BaseURL, err = url.Parse(source.GitHubAPIURL)
		if err != nil {
			return nil, err
		}
	}

	return &GitHubClient{
		client:     client,
		user:       source.User,
		repository: source.Repository,
	}, nil
}

func (g *GitHubClient) ListReleases() ([]github.RepositoryRelease, error) {
	releases, res, err := g.client.Repositories.ListReleases(g.user, g.repository, nil)
	if err != nil {
		return []github.RepositoryRelease{}, err
	}

	err = res.Body.Close()
	if err != nil {
		return nil, err
	}

	return releases, nil
}

func (g *GitHubClient) GetReleaseByTag(tag string) (*github.RepositoryRelease, error) {
	release, res, err := g.client.Repositories.GetReleaseByTag(g.user, g.repository, tag)
	if err != nil {
		return &github.RepositoryRelease{}, err
	}

	err = res.Body.Close()
	if err != nil {
		return nil, err
	}

	return release, nil
}

func (g *GitHubClient) GetRelease(id int) (*github.RepositoryRelease, error) {
	release, res, err := g.client.Repositories.GetRelease(g.user, g.repository, id)
	if err != nil {
		return &github.RepositoryRelease{}, err
	}

	err = res.Body.Close()
	if err != nil {
		return nil, err
	}

	return release, nil
}

func (g *GitHubClient) CreateRelease(release github.RepositoryRelease) (*github.RepositoryRelease, error) {
	createdRelease, res, err := g.client.Repositories.CreateRelease(g.user, g.repository, &release)
	if err != nil {
		return &github.RepositoryRelease{}, err
	}

	err = res.Body.Close()
	if err != nil {
		return nil, err
	}

	return createdRelease, nil
}

func (g *GitHubClient) UpdateRelease(release github.RepositoryRelease) (*github.RepositoryRelease, error) {
	if release.ID == nil {
		return nil, errors.New("release did not have an ID: has it been saved yet?")
	}

	updatedRelease, res, err := g.client.Repositories.EditRelease(g.user, g.repository, *release.ID, &release)
	if err != nil {
		return &github.RepositoryRelease{}, err
	}

	err = res.Body.Close()
	if err != nil {
		return nil, err
	}

	return updatedRelease, nil
}

func (g *GitHubClient) ListReleaseAssets(release github.RepositoryRelease) ([]github.ReleaseAsset, error) {
	assets, res, err := g.client.Repositories.ListReleaseAssets(g.user, g.repository, *release.ID, nil)
	if err != nil {
		return []github.ReleaseAsset{}, err
	}

	err = res.Body.Close()
	if err != nil {
		return nil, err
	}

	return assets, nil
}

func (g *GitHubClient) UploadReleaseAsset(release github.RepositoryRelease, name string, file *os.File) error {
	_, res, err := g.client.Repositories.UploadReleaseAsset(
		g.user,
		g.repository,
		*release.ID,
		&github.UploadOptions{
			Name: name,
		},
		file,
	)
	if err != nil {
		return err
	}

	return res.Body.Close()
}

func (g *GitHubClient) DeleteReleaseAsset(asset github.ReleaseAsset) error {
	res, err := g.client.Repositories.DeleteReleaseAsset(g.user, g.repository, *asset.ID)
	if err != nil {
		return err
	}

	return res.Body.Close()
}

func (g *GitHubClient) DownloadReleaseAsset(asset github.ReleaseAsset) (io.ReadCloser, error) {
	res, err := g.client.Repositories.DownloadReleaseAsset(g.user, g.repository, *asset.ID)
	if err != nil {
		return nil, err
	}

	return res, err
}

func (g *GitHubClient) GetTarballLink(tag string) (*url.URL, error) {
	opt := &github.RepositoryContentGetOptions{Ref: tag}
	u, res, err := g.client.Repositories.GetArchiveLink(g.user, g.repository, github.Tarball, opt)
	if err != nil {
		return nil, err
	}
	res.Body.Close()
	return u, nil
}

func (g *GitHubClient) GetZipballLink(tag string) (*url.URL, error) {
	opt := &github.RepositoryContentGetOptions{Ref: tag}
	u, res, err := g.client.Repositories.GetArchiveLink(g.user, g.repository, github.Zipball, opt)
	if err != nil {
		return nil, err
	}
	res.Body.Close()
	return u, nil
}
<|endoftext|>


<commit_before>package modify

import (
	oas3 "github.com/getkin/kin-openapi/openapi3"
)

// RemoveOperationsSecurity removes the security property
// for all operations. It is useful when building a spec
// to get individual specs to validate before setting the
// correct security property.
func RemoveOperationsSecurity(spec *oas3.Swagger) {
	VisitOperations(spec, func(op *oas3.Operation) {
		if op == nil {
			return
		}
		op.Security = &oas3.SecurityRequirements{}
	})
}
<commit_msg>add openapi3/modify add spec and operation API key security<commit_after>package modify

import (
	"errors"
	"fmt"
	"strings"

	oas3 "github.com/getkin/kin-openapi/openapi3"
	"github.com/grokify/gotilla/type/stringsutil"
)

const SecuritySchemeDefaultNameApiKeyAuth = "ApiKeyAuth"

// AddAPIKey adds an API Key definition to the spec.
// https://swagger.io/docs/specification/authentication/api-keys/
func SecuritySchemeAddDefinitionApiKey(spec *oas3.Swagger, schemeName, location, name string) error {
	schemeName = strings.TrimSpace(schemeName)
	location = strings.TrimSpace(location)
	name = strings.TrimSpace(name)
	if len(schemeName) == 0 {
		schemeName = SecuritySchemeDefaultNameApiKeyAuth
	}
	if len(location) == 0 {
		return errors.New("API Key Security Scheme Location cannot be empty. Must be one of: [\"header\", \"query\", \"cookie\"]")
	} else if location != "header" && location != "query" && location != "cookie" {
		return fmt.Errorf("API Key Security Scheme Invalid Location [%s], must be one of: [\"header\", \"query\", \"cookie\"]", location)
	}
	if len(name) == 0 {
		return errors.New("API Key Security Scheme name cannot be empty.")
	}
	if spec.Components.SecuritySchemes == nil {
		spec.Components.SecuritySchemes = map[string]*oas3.SecuritySchemeRef{}
	}
	spec.Components.SecuritySchemes[schemeName] = &oas3.SecuritySchemeRef{
		Value: &oas3.SecurityScheme{
			Type: "apiKey",
			In:   location,
			Name: name,
		},
	}
	return nil
}

func SecuritySchemeAddDefinitionOperations(spec *oas3.Swagger, tags []string, keyName string) {
	keyName = strings.TrimSpace(keyName)
	if len(keyName) == 0 {
		keyName = SecuritySchemeDefaultNameApiKeyAuth
	}
	tagsMap := map[string]int{}
	for _, tagName := range tags {
		tagName = strings.TrimSpace(tagName)
		tagsMap[tagName] = 1
	}
	VisitOperations(spec, func(op *oas3.Operation) {
		if op == nil || !SliceIntersectionExists(tagsMap, op.Tags) {
			return
		}
		if op.Security == nil {
			op.Security = &oas3.SecurityRequirements{}
		}
		secreq := oas3.SecurityRequirement{}
		secreq[keyName] = []string{}
		*op.Security = append(*op.Security, secreq)
	})
}

func SliceIntersection(haystack map[string]int, needles []string, unique bool) []string {
	if unique {
		needles = stringsutil.SliceCondenseSpace(needles, true, false)
	}
	matches := []string{}
	for _, needle := range needles {
		if _, ok := haystack[needle]; ok {
			matches = append(matches, needle)
		}
	}
	return matches
}

func SliceIntersectionExists(haystack map[string]int, needles []string) bool {
	for _, needle := range needles {
		if _, ok := haystack[needle]; ok {
			return true
		}
	}
	return false
}

// RemoveOperationsSecurity removes the security property
// for all operations. It is useful when building a spec
// to get individual specs to validate before setting the
// correct security property.
func RemoveOperationsSecurity(spec *oas3.Swagger) {
	VisitOperations(spec, func(op *oas3.Operation) {
		if op == nil {
			return
		}
		op.Security = &oas3.SecurityRequirements{}
	})
}
<|endoftext|>


<commit_before>package main

import (
  "net/http"
  "log"
  "io/ioutil"
  "encoding/json"
  "os/exec"
  "flag"
  "strconv"
  "crypto/md5"
  "text/template"
  "bytes"
  "gopkg.in/fsnotify.v1"
)

// A map of repos => (a map of branches => (a list of actions))
// Todo: Idiomatic way of doing this?
type Config map[string]map[string][]string

type HookMsg struct {
  CanonicalUrl string `json:"canon_url"`

  Commits []struct {
    Branch string
  }

  Repository struct {
    AbsoluteUrl string `json:"absolute_url"`
  }
}

type Commit struct {
  Repository string
  Branch string
}

type Job struct {
  Commit Commit
  Action string
}

var listenPort = flag.Int("port", 8080, "portnumber to listen on")
var configFile = flag.String("config", "golive.json", "the configfile to read")
var verbose = flag.Bool("v", false, "print more output")

var jobTemplates = make(map[[16]byte]template.Template)
var config Config

func main() {
	flag.Parse()

  parseConfig(*configFile)
  watchConfig(*configFile)

  msgs := make(chan HookMsg, 100)
  commits := make(chan Commit, 100)
  jobs := make(chan Job, 100)
  actions := make(chan string, 100)

  go hookWrangler(msgs, commits)
  go commitWrangler(commits, jobs, config)
  go jobWrangler(jobs, actions)
  go actionRunner(actions)

  http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
    payload := r.FormValue("payload")

    var hookmsg HookMsg
    if err := json.Unmarshal([]byte(payload), &hookmsg); err != nil {
      http.Error(w, "Could not decode json", 500)
      log.Fatal(err)
    }

    if *verbose {
      log.Print("Received commit: ", hookmsg)
    }

    msgs <- hookmsg
  })

  log.Fatal(http.ListenAndServe(":" + strconv.Itoa(*listenPort), nil))
}

func watchConfig(configFile string) {
  watcher, err := fsnotify.NewWatcher()
  if err != nil {
    log.Fatal(err)
  }

  log.Print("Watching config file: ", configFile)

  defer watcher.Close()

	done := make(chan bool)

  go func() {
    for {
      select {
        case event := <-watcher.Events:
          if event.Name == "" {
            continue
          }

          if *verbose {
            log.Println(event.Name, event.Op, event)
          }

          // The watching doesn't seem to follow renames, which happen i.e.
          // during Vim editing, so set up a new one
          if event.Op & fsnotify.Rename == fsnotify.Rename {
            addWatcher(watcher, configFile)
          }

          log.Print("Relading config file ", configFile)

          parseConfig(configFile)

        case err := <-watcher.Errors:
          if err == nil {
            continue
          }

          done <- true
          log.Fatal("Error when watching config file: ", err)
      }
    }
  }()

  addWatcher(watcher, configFile)

  <- done
}

func addWatcher(watcher *fsnotify.Watcher, file string) {
  err := watcher.Add(file)
  if err != nil {
    log.Fatal(err)
  }
}

func parseConfig(configFile string) {
    config_raw, err := ioutil.ReadFile(configFile)
    if err != nil {
      log.Fatal(err)
    }

    var newConfig Config
    json.Unmarshal(config_raw, &newConfig)

    config = newConfig

    if *verbose {
      log.Print("Loaded config: ", config)
    }
}

func hookWrangler(msgs <-chan HookMsg, results chan<- Commit) {
  for msg := range msgs {
    repository := msg.CanonicalUrl + msg.Repository.AbsoluteUrl

    for _, commit := range msg.Commits {
      results <- Commit{ repository, commit.Branch}
    }
  }
}

func commitWrangler(commits <-chan Commit, results chan<- Job, config Config) {
  for commit := range commits {
    if commit.Branch == "" || commit.Repository == "" {
      continue
    }

    if branches, ok := config[commit.Repository]; ok {
      if actions, ok := branches[commit.Branch]; ok {
        for _, action := range actions {
          results <- Job{commit, string(action)}
        }
      }
    }
  }
}

func jobWrangler(jobs <-chan Job, actions chan<- string) {
  for job := range jobs {
    if *verbose {
      log.Print("Running job: ", job)
    }

    hash := md5.Sum([]byte(job.Action))

    if _, ok := jobTemplates[hash]; !ok {
      t, err := template.New(string(hash[:])).Parse(job.Action)
      if err != nil {
        log.Fatal("Could not compile template: ", job.Action, " - ", err)
      }

      jobTemplates[hash] = *t
    }

    t := jobTemplates[hash]

    var buff bytes.Buffer
    (&t).Execute(&buff, job.Commit)
    s := buff.String()

    actions <- s
  }
}

func actionRunner(actions <-chan string) {
  for action := range actions {
    command := exec.Command("bash", "-c", action)

    command.Run()
  }
}
<commit_msg>Fix blocking watcher bug<commit_after>package main

import (
  "net/http"
  "log"
  "io/ioutil"
  "encoding/json"
  "os/exec"
  "flag"
  "strconv"
  "crypto/md5"
  "text/template"
  "bytes"
  "gopkg.in/fsnotify.v1"
)

// A map of repos => (a map of branches => (a list of actions))
// Todo: Idiomatic way of doing this?
type Config map[string]map[string][]string

type HookMsg struct {
  CanonicalUrl string `json:"canon_url"`

  Commits []struct {
    Branch string
  }

  Repository struct {
    AbsoluteUrl string `json:"absolute_url"`
  }
}

type Commit struct {
  Repository string
  Branch string
}

type Job struct {
  Commit Commit
  Action string
}

var listenPort = flag.Int("port", 8080, "portnumber to listen on")
var configFile = flag.String("config", "golive.json", "the configfile to read")
var verbose = flag.Bool("v", false, "print more output")

var jobTemplates = make(map[[16]byte]template.Template)
var config Config

func main() {
	flag.Parse()

  parseConfig(*configFile)
  go watchConfig(*configFile)

  msgs := make(chan HookMsg, 100)
  commits := make(chan Commit, 100)
  jobs := make(chan Job, 100)
  actions := make(chan string, 100)

  go hookWrangler(msgs, commits)
  go commitWrangler(commits, jobs, config)
  go jobWrangler(jobs, actions)
  go actionRunner(actions)

  log.Print("Starting golive server")

  http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
    log.Print("Received request")

    payload := r.FormValue("payload")

    var hookmsg HookMsg
    if err := json.Unmarshal([]byte(payload), &hookmsg); err != nil {
      http.Error(w, "Could not decode json", 500)
      log.Fatal(err)
    }

    if *verbose {
      log.Print("Received commit: ", hookmsg)
    }

    msgs <- hookmsg
  })

  log.Fatal(http.ListenAndServe(":" + strconv.Itoa(*listenPort), nil))
}

func watchConfig(configFile string) {
  watcher, err := fsnotify.NewWatcher()
  if err != nil {
    log.Fatal(err)
  }

  log.Print("Watching config file: ", configFile)

  defer watcher.Close()

	done := make(chan bool)

  go func() {
    for {
      select {
        case event := <-watcher.Events:
          if event.Name == "" {
            continue
          }

          if *verbose {
            log.Println(event.Name, event.Op, event)
          }

          // The watching doesn't seem to follow renames, which happen i.e.
          // during Vim editing, so set up a new one
          if event.Op & fsnotify.Rename == fsnotify.Rename {
            addWatcher(watcher, configFile)
          }

          log.Print("Relading config file ", configFile)

          parseConfig(configFile)

        case err := <-watcher.Errors:
          if err == nil {
            continue
          }

          done <- true
          log.Fatal("Error when watching config file: ", err)
      }
    }
  }()

  addWatcher(watcher, configFile)

  <- done
}

func addWatcher(watcher *fsnotify.Watcher, file string) {
  err := watcher.Add(file)
  if err != nil {
    log.Fatal(err)
  }
}

func parseConfig(configFile string) {
    config_raw, err := ioutil.ReadFile(configFile)
    if err != nil {
      log.Fatal(err)
    }

    var newConfig Config
    json.Unmarshal(config_raw, &newConfig)

    config = newConfig

    if *verbose {
      log.Print("Loaded config: ", config)
    }
}

func hookWrangler(msgs <-chan HookMsg, results chan<- Commit) {
  for msg := range msgs {
    repository := msg.CanonicalUrl + msg.Repository.AbsoluteUrl

    for _, commit := range msg.Commits {
      results <- Commit{ repository, commit.Branch}
    }
  }
}

func commitWrangler(commits <-chan Commit, results chan<- Job, config Config) {
  for commit := range commits {
    if commit.Branch == "" || commit.Repository == "" {
      continue
    }

    if branches, ok := config[commit.Repository]; ok {
      if actions, ok := branches[commit.Branch]; ok {
        for _, action := range actions {
          results <- Job{commit, string(action)}
        }
      }
    }
  }
}

func jobWrangler(jobs <-chan Job, actions chan<- string) {
  for job := range jobs {
    if *verbose {
      log.Print("Running job: ", job)
    }

    hash := md5.Sum([]byte(job.Action))

    if _, ok := jobTemplates[hash]; !ok {
      t, err := template.New(string(hash[:])).Parse(job.Action)
      if err != nil {
        log.Fatal("Could not compile template: ", job.Action, " - ", err)
      }

      jobTemplates[hash] = *t
    }

    t := jobTemplates[hash]

    var buff bytes.Buffer
    (&t).Execute(&buff, job.Commit)
    s := buff.String()

    actions <- s
  }
}

func actionRunner(actions <-chan string) {
  for action := range actions {
    command := exec.Command("bash", "-c", action)

    command.Run()
  }
}
<|endoftext|>


<commit_before>/*
Package gomake provides a makefile-like syntax to define rules and their
dependencies.

gomake is designed to be imported and used to create a main package with rules
to make your program. We can start off by writing a Gomakefile that will build
itself:

	package main

	import (
		"fmt"
		"os"
		"os/exec"

		"github.com/hinshun/gomake"
	)

	func main() {
		gomakefile := gomake.NewGomakefile()

		gomakeItself := gomakefile.AddRule("gomake", "Builds gomake", nil, func() error {
			build := exec.Command("go", "build")
			build.Stdout = os.Stdout
			build.Stderr = os.Stderr

			return build.Run()
		})

		// Sets the default target
		gomakefile.Targets[""] = gomakeItself

		err := gomake.Gomake(gomakefile).Run(os.Args)
		if err != nil {
			fmt.Println(err)
			os.Exit(-1)
		}
	}
*/
package gomake

import (
	"sort"

	"github.com/hinshun/gomake/pkg/cli"
)

const (
	// Version is the current version of gomake.
	Version = "0.1.0"
)

// Gomake creates a cli app for the given Gomakefile.
func Gomake(gomakefile *Gomakefile) *cli.App {
	app := &cli.App{
		Name:    "gomake",
		Usage:   "Makefile for gophers",
		Version: Version,
		Action: func(ctx *cli.Context) error {
			_, ok := gomakefile.Targets[""]
			if !ok {
				return nil
			}

			results := gomakefile.Make("")
			return HandleResults(results)
		},
	}

	for gomakeTarget, rule := range gomakefile.Targets {
		// Skip default
		if gomakeTarget == "" {
			continue
		}

		// Create closure around target for command
		target := gomakeTarget
		command := &cli.Command{
			Name:        target,
			Description: rule.Description,
			Action: func(ctx *cli.Context) error {
				results := gomakefile.Make(target)
				return HandleResults(results)
			},
		}

		app.Commands = append(app.Commands, command)
	}

	sort.Sort(app.Commands)
	return app
}
<commit_msg>Update package documentation<commit_after>/*
Package gomake provides a makefile-like syntax to define rules and their
dependencies.

gomake is designed to be imported and used to create a main package with rules
to make your program. We can start off by writing a Gomakefile that will build
itself:

	package main

	import (
		"fmt"
		"os"
		"os/exec"

		"github.com/hinshun/gomake"
	)

	func main() {
		gomakefile := gomake.NewGomakefile()

		gomakeItself := gomakefile.AddRule("itself", "Builds gomake", nil, func() error {
			build := exec.Command("go", "build")
			build.Stdout = os.Stdout
			build.Stderr = os.Stderr

			return build.Run()
		})

		// Sets the default target
		gomakefile.Targets[""] = gomakeItself

		err := gomake.Gomake(gomakefile).Run(os.Args)
		if err != nil {
			fmt.Println(err)
			os.Exit(-1)
		}
	}
*/
package gomake

import (
	"sort"

	"github.com/hinshun/gomake/pkg/cli"
)

const (
	// Version is the current version of gomake.
	Version = "0.1.0"
)

// Gomake creates a cli app for the given Gomakefile.
func Gomake(gomakefile *Gomakefile) *cli.App {
	app := &cli.App{
		Name:    "gomake",
		Usage:   "Makefile for gophers",
		Version: Version,
		Action: func(ctx *cli.Context) error {
			_, ok := gomakefile.Targets[""]
			if !ok {
				return nil
			}

			results := gomakefile.Make("")
			return HandleResults(results)
		},
	}

	for gomakeTarget, rule := range gomakefile.Targets {
		// Skip default
		if gomakeTarget == "" {
			continue
		}

		// Create closure around target for command
		target := gomakeTarget
		command := &cli.Command{
			Name:        target,
			Description: rule.Description,
			Action: func(ctx *cli.Context) error {
				results := gomakefile.Make(target)
				return HandleResults(results)
			},
		}

		app.Commands = append(app.Commands, command)
	}

	sort.Sort(app.Commands)
	return app
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	flags "github.com/jessevdk/go-flags"
	"github.com/monochromegane/terminal"
	"github.com/ongaeshi/gomilk/search/option"
	"github.com/ongaeshi/gomilk/search"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
)

const version = "0.1.0"

var opts option.Option

func init() {
	if cpu := runtime.NumCPU(); cpu == 1 {
		runtime.GOMAXPROCS(2)
	} else {
		runtime.GOMAXPROCS(cpu)
	}
}

func main() {

	parser := flags.NewParser(&opts, flags.Default)
	parser.Name = "gomilk"
	parser.Usage = "[OPTIONS] PATTERN [PATH]"

	args, err := parser.Parse()
	if err != nil {
		os.Exit(1)
	}

	if opts.Version {
		fmt.Printf("%s\n", version)
		os.Exit(0)
	}

	if len(args) == 0 && opts.FilesWithRegexp == "" {
		parser.WriteHelp(os.Stdout)
		os.Exit(1)
	}

	if len(args) >= 2 {
		fmt.Println("AND search does not currently support")
		os.Exit(1)
	}

	var root = "."

	if opts.Directory != "" {
		root = strings.TrimRight(opts.Directory, "\"")
		_, err := os.Lstat(root)
		if err != nil {
			fmt.Fprintf(os.Stderr, "%s\n", err)
			os.Exit(1)
		}
	}

	opts.Proc = runtime.NumCPU()

	if !terminal.IsTerminal(os.Stdout) {
		opts.NoColor = true
		opts.NoGroup = true
	}

	if opts.Context > 0 {
		opts.Before = opts.Context
		opts.After = opts.Context
	}

	if opts.Context > 0 {
		opts.Before = opts.Context
		opts.After = opts.Context
	}

	pattern := ""
	if len(args) > 0 {
		pattern = args[0]
	}

	if opts.Update {
		prevDir, _ := filepath.Abs(".")
		os.Chdir(root)

		cmd := exec.Command("milk", "update")
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr

		err := cmd.Run()

		if err != nil {
			log.Fatal(err)
		}

		os.Chdir(prevDir)
	}

	searcher := search.Searcher{root, pattern, &opts}
	err = searcher.Search()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s\n", err)
		os.Exit(1)
	}

}
<commit_msg>M-x gofmt<commit_after>package main

import (
	"fmt"
	flags "github.com/jessevdk/go-flags"
	"github.com/monochromegane/terminal"
	"github.com/ongaeshi/gomilk/search"
	"github.com/ongaeshi/gomilk/search/option"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
)

const version = "0.1.0"

var opts option.Option

func init() {
	if cpu := runtime.NumCPU(); cpu == 1 {
		runtime.GOMAXPROCS(2)
	} else {
		runtime.GOMAXPROCS(cpu)
	}
}

func main() {

	parser := flags.NewParser(&opts, flags.Default)
	parser.Name = "gomilk"
	parser.Usage = "[OPTIONS] PATTERN [PATH]"

	args, err := parser.Parse()
	if err != nil {
		os.Exit(1)
	}

	if opts.Version {
		fmt.Printf("%s\n", version)
		os.Exit(0)
	}

	if len(args) == 0 && opts.FilesWithRegexp == "" {
		parser.WriteHelp(os.Stdout)
		os.Exit(1)
	}

	if len(args) >= 2 {
		fmt.Println("AND search does not currently support")
		os.Exit(1)
	}

	var root = "."

	if opts.Directory != "" {
		root = strings.TrimRight(opts.Directory, "\"")
		_, err := os.Lstat(root)
		if err != nil {
			fmt.Fprintf(os.Stderr, "%s\n", err)
			os.Exit(1)
		}
	}

	opts.Proc = runtime.NumCPU()

	if !terminal.IsTerminal(os.Stdout) {
		opts.NoColor = true
		opts.NoGroup = true
	}

	if opts.Context > 0 {
		opts.Before = opts.Context
		opts.After = opts.Context
	}

	if opts.Context > 0 {
		opts.Before = opts.Context
		opts.After = opts.Context
	}

	pattern := ""
	if len(args) > 0 {
		pattern = args[0]
	}

	if opts.Update {
		prevDir, _ := filepath.Abs(".")
		os.Chdir(root)

		cmd := exec.Command("milk", "update")
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr

		err := cmd.Run()

		if err != nil {
			log.Fatal(err)
		}

		os.Chdir(prevDir)
	}

	searcher := search.Searcher{root, pattern, &opts}
	err = searcher.Search()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s\n", err)
		os.Exit(1)
	}

}
<|endoftext|>