<commit_before>package btrfs

import "sort"

/*
#cgo LDFLAGS: -lbtrfs

#include <stddef.h>
#include <btrfs/ioctl.h>
#include "btrfs.h"

// Required because Go has struct casting rules for negative numbers
const __u64 u64_BTRFS_LAST_FREE_OBJECTID = (__u64)BTRFS_LAST_FREE_OBJECTID;
const __u64 negative_one = (__u64)-1;

static char* get_name_btrfs_ioctl_vol_args_v2(struct btrfs_ioctl_vol_args_v2* btrfs_struct) {
	return btrfs_struct->name;
}
*/
import "C"

import (
	"fmt"
	"os"
	"path/filepath"
	"syscall"
	"unsafe"

	"github.com/pkg/errors"
)

// IsSubvolume returns nil if the path is a valid subvolume. An error is
// returned if the path does not exist or the path is not a valid subvolume.
func IsSubvolume(path string) error {
	fi, err := os.Lstat(path)
	if err != nil {
		return err
	}

	if err := isFileInfoSubvol(fi); err != nil {
		return err
	}

	var statfs syscall.Statfs_t
	if err := syscall.Statfs(path, &statfs); err != nil {
		return err
	}

	return isStatfsSubvol(&statfs)
}

// SubvolInfo returns information about the subvolume at the provided path.
func SubvolInfo(path string) (info Info, err error) {
	path, err = filepath.EvalSymlinks(path)
	if err != nil {
		return info, err
	}

	fp, err := openSubvolDir(path)
	if err != nil {
		return info, err
	}
	defer fp.Close()

	id, err := subvolID(fp.Fd())
	if err != nil {
		return info, err
	}

	subvolsByID, err := subvolMap(path)
	if err != nil {
		return info, err
	}

	if info, ok := subvolsByID[id]; ok {
		return info, nil
	}

	return info, errors.Errorf("%q not found", path)
}

func subvolMap(path string) (map[uint64]Info, error) {
	fp, err := openSubvolDir(path)
	if err != nil {
		return nil, err
	}
	defer fp.Close()

	var args C.struct_btrfs_ioctl_search_args

	args.key.tree_id = C.BTRFS_ROOT_TREE_OBJECTID
	args.key.min_type = C.BTRFS_ROOT_ITEM_KEY
	args.key.max_type = C.BTRFS_ROOT_BACKREF_KEY
	args.key.min_objectid = C.BTRFS_FS_TREE_OBJECTID
	args.key.max_objectid = C.u64_BTRFS_LAST_FREE_OBJECTID
	args.key.max_offset = C.negative_one
	args.key.max_transid = C.negative_one

	subvolsByID := map[uint64]Info{}

	for {
		args.key.nr_items = 4096
		if err := ioctl(fp.Fd(), C.BTRFS_IOC_TREE_SEARCH, uintptr(unsafe.Pointer(&args))); err != nil {
			return nil, err
		}

		if args.key.nr_items == 0 {
			break
		}

		var (
			sh     C.struct_btrfs_ioctl_search_header
			shSize = unsafe.Sizeof(sh)
			buf    = (*[1<<31 - 1]byte)(unsafe.Pointer(&args.buf[0]))[:C.BTRFS_SEARCH_ARGS_BUFSIZE]
		)

		for i := 0; i < int(args.key.nr_items); i++ {
			sh = (*(*C.struct_btrfs_ioctl_search_header)(unsafe.Pointer(&buf[0])))
			buf = buf[shSize:]

			info := subvolsByID[uint64(sh.objectid)]
			info.ID = uint64(sh.objectid)

			if sh._type == C.BTRFS_ROOT_BACKREF_KEY {
				rr := (*(*C.struct_btrfs_root_ref)(unsafe.Pointer(&buf[0])))

				// This branch processes the backrefs from the root object. We
				// get an entry of the objectid, with name, but the parent is
				// the offset.

				nname := C.btrfs_stack_root_ref_name_len(&rr)
				name := string(buf[C.sizeof_struct_btrfs_root_ref : C.sizeof_struct_btrfs_root_ref+uintptr(nname)])

				info.ID = uint64(sh.objectid)
				info.ParentID = uint64(sh.offset)
				info.Name = name
				info.DirID = uint64(C.btrfs_stack_root_ref_dirid(&rr))

				subvolsByID[uint64(sh.objectid)] = info
			} else if sh._type == C.BTRFS_ROOT_ITEM_KEY &&
				(sh.objectid >= C.BTRFS_ROOT_ITEM_KEY ||
					sh.objectid == C.BTRFS_FS_TREE_OBJECTID) {

				var (
					ri  = (*C.struct_btrfs_root_item)(unsafe.Pointer(&buf[0]))
					gri C.struct_gosafe_btrfs_root_item
				)

				C.unpack_root_item(&gri, ri)

				if gri.flags&C.BTRFS_ROOT_SUBVOL_RDONLY != 0 {
					info.Readonly = true
				}

				// in this case, the offset is the actual offset.
				info.Offset = uint64(sh.offset)

				info.UUID = uuidString(&gri.uuid)
				info.ParentUUID = uuidString(&gri.parent_uuid)
				info.ReceivedUUID = uuidString(&gri.received_uuid)

				info.Generation = uint64(gri.gen)
				info.OriginalGeneration = uint64(gri.ogen)

				subvolsByID[uint64(sh.objectid)] = info
			}

			args.key.min_objectid = sh.objectid
			args.key.min_offset = sh.offset
			args.key.min_type = sh._type //  this is very questionable.

			buf = buf[sh.len:]
		}

		args.key.min_offset++
		if args.key.min_offset == 0 {
			args.key.min_type++
		} else {
			continue
		}

		if args.key.min_type > C.BTRFS_ROOT_BACKREF_KEY {
			args.key.min_type = C.BTRFS_ROOT_ITEM_KEY
			args.key.min_objectid++
		} else {
			continue
		}

		if args.key.min_objectid > args.key.max_objectid {
			break
		}
	}

	mnt, err := findMountPoint(path)
	if err != nil {
		return nil, err
	}

	for _, sv := range subvolsByID {
		path := sv.Name
		parentID := sv.ParentID

		for parentID != 0 {
			parent, ok := subvolsByID[parentID]
			if !ok {
				break
			}

			parentID = parent.ParentID
			path = filepath.Join(parent.Name, path)
		}

		sv.Path = filepath.Join(mnt, path)
	}
	return subvolsByID, nil
}

// SubvolList will return the information for all subvolumes corresponding to
// the provided path.
func SubvolList(path string) ([]Info, error) {
	subvolsByID, err := subvolMap(path)
	if err != nil {
		return nil, err
	}

	subvols := make([]Info, 0, len(subvolsByID))
	for _, sv := range subvolsByID {
		subvols = append(subvols, sv)
	}

	sort.Sort(infosByID(subvols))

	return subvols, nil
}

// SubvolCreate creates a subvolume at the provided path.
func SubvolCreate(path string) error {
	dir, name := filepath.Split(path)

	fp, err := os.Open(dir)
	if err != nil {
		return err
	}
	defer fp.Close()

	var args C.struct_btrfs_ioctl_vol_args
	args.fd = C.__s64(fp.Fd())

	if len(name) > C.BTRFS_PATH_NAME_MAX {
		return errors.Errorf("%q too long for subvolume", name)
	}
	nameptr := (*[1<<31 - 1]byte)(unsafe.Pointer(&args.name[0]))
	copy(nameptr[:C.BTRFS_PATH_NAME_MAX], []byte(name))

	if err := ioctl(fp.Fd(), C.BTRFS_IOC_SUBVOL_CREATE, uintptr(unsafe.Pointer(&args))); err != nil {
		return errors.Wrap(err, "btrfs subvolume create failed")
	}

	return nil
}

// SubvolSnapshot creates a snapshot in dst from src. If readonly is true, the
// snapshot will be readonly.
func SubvolSnapshot(dst, src string, readonly bool) error {
	dstdir, dstname := filepath.Split(dst)

	dstfp, err := openSubvolDir(dstdir)
	if err != nil {
		return errors.Wrapf(err, "opening snapshot desination subvolume failed")
	}
	defer dstfp.Close()

	srcfp, err := openSubvolDir(src)
	if err != nil {
		return errors.Wrapf(err, "opening snapshot source subvolume failed")
	}
	defer srcfp.Close()

	// dstdir is the ioctl arg, wile srcdir gets set on the args
	var args C.struct_btrfs_ioctl_vol_args_v2
	args.fd = C.__s64(srcfp.Fd())
	name := C.get_name_btrfs_ioctl_vol_args_v2(&args)

	if len(dstname) > C.BTRFS_SUBVOL_NAME_MAX {
		return errors.Errorf("%q too long for subvolume", dstname)
	}

	nameptr := (*[1<<31 - 1]byte)(unsafe.Pointer(name))
	copy(nameptr[:C.BTRFS_SUBVOL_NAME_MAX], []byte(dstname))

	if readonly {
		args.flags |= C.BTRFS_SUBVOL_RDONLY
	}

	if err := ioctl(dstfp.Fd(), C.BTRFS_IOC_SNAP_CREATE_V2, uintptr(unsafe.Pointer(&args))); err != nil {
		return errors.Wrapf(err, "snapshot create failed")
	}

	return nil
}

// SubvolDelete deletes the subvolumes under the given path.
func SubvolDelete(path string) error {
	fmt.Println("delete", path)
	dir, name := filepath.Split(path)
	fp, err := openSubvolDir(dir)
	if err != nil {
		return errors.Wrapf(err, "failed opening %v", path)
	}
	defer fp.Close()

	// remove child subvolumes
	if err := filepath.Walk(path, func(p string, fi os.FileInfo, err error) error {
		if err != nil {
			if os.IsNotExist(err) || p == path {
				return nil
			}

			return errors.Wrapf(err, "failed walking subvolume %v", p)
		}

		if !fi.IsDir() {
			return nil // just ignore it!
		}

		if p == path {
			return nil
		}

		if err := isFileInfoSubvol(fi); err != nil {
			return nil
		}

		if err := SubvolDelete(p); err != nil {
			return errors.Wrapf(err, "recursive delete of %v failed", p)
		}

		return filepath.SkipDir // children get walked by call above.
	}); err != nil {
		return err
	}

	var args C.struct_btrfs_ioctl_vol_args
	if len(name) > C.BTRFS_SUBVOL_NAME_MAX {
		return errors.Errorf("%q too long for subvolume", name)
	}

	nameptr := (*[1<<31 - 1]byte)(unsafe.Pointer(&args.name[0]))
	copy(nameptr[:C.BTRFS_SUBVOL_NAME_MAX], []byte(name))

	if err := ioctl(fp.Fd(), C.BTRFS_IOC_SNAP_DESTROY, uintptr(unsafe.Pointer(&args))); err != nil {
		return errors.Wrapf(err, "failed removing subvolume %v", path)
	}

	return nil
}

func openSubvolDir(path string) (*os.File, error) {
	fp, err := os.Open(path)
	if err != nil {
		return nil, errors.Wrapf(err, "opening %v as subvolume failed", path)
	}

	return fp, nil
}

func isStatfsSubvol(statfs *syscall.Statfs_t) error {
	if int64(statfs.Type) != int64(C.BTRFS_SUPER_MAGIC) {
		return errors.Errorf("not a btrfs filesystem")
	}

	return nil
}

func isFileInfoSubvol(fi os.FileInfo) error {
	if !fi.IsDir() {
		errors.Errorf("must be a directory")
	}

	stat := fi.Sys().(*syscall.Stat_t)

	if stat.Ino != C.BTRFS_FIRST_FREE_OBJECTID {
		return errors.Errorf("incorrect inode type")
	}

	return nil
}
<commit_msg>Remove unneeded link and import<commit_after>package btrfs

import "sort"

/*
#include <stddef.h>
#include <btrfs/ioctl.h>
#include "btrfs.h"

// Required because Go has struct casting rules for negative numbers
const __u64 u64_BTRFS_LAST_FREE_OBJECTID = (__u64)BTRFS_LAST_FREE_OBJECTID;
const __u64 negative_one = (__u64)-1;

static char* get_name_btrfs_ioctl_vol_args_v2(struct btrfs_ioctl_vol_args_v2* btrfs_struct) {
	return btrfs_struct->name;
}
*/
import "C"

import (
	"os"
	"path/filepath"
	"syscall"
	"unsafe"

	"github.com/pkg/errors"
)

// IsSubvolume returns nil if the path is a valid subvolume. An error is
// returned if the path does not exist or the path is not a valid subvolume.
func IsSubvolume(path string) error {
	fi, err := os.Lstat(path)
	if err != nil {
		return err
	}

	if err := isFileInfoSubvol(fi); err != nil {
		return err
	}

	var statfs syscall.Statfs_t
	if err := syscall.Statfs(path, &statfs); err != nil {
		return err
	}

	return isStatfsSubvol(&statfs)
}

// SubvolInfo returns information about the subvolume at the provided path.
func SubvolInfo(path string) (info Info, err error) {
	path, err = filepath.EvalSymlinks(path)
	if err != nil {
		return info, err
	}

	fp, err := openSubvolDir(path)
	if err != nil {
		return info, err
	}
	defer fp.Close()

	id, err := subvolID(fp.Fd())
	if err != nil {
		return info, err
	}

	subvolsByID, err := subvolMap(path)
	if err != nil {
		return info, err
	}

	if info, ok := subvolsByID[id]; ok {
		return info, nil
	}

	return info, errors.Errorf("%q not found", path)
}

func subvolMap(path string) (map[uint64]Info, error) {
	fp, err := openSubvolDir(path)
	if err != nil {
		return nil, err
	}
	defer fp.Close()

	var args C.struct_btrfs_ioctl_search_args

	args.key.tree_id = C.BTRFS_ROOT_TREE_OBJECTID
	args.key.min_type = C.BTRFS_ROOT_ITEM_KEY
	args.key.max_type = C.BTRFS_ROOT_BACKREF_KEY
	args.key.min_objectid = C.BTRFS_FS_TREE_OBJECTID
	args.key.max_objectid = C.u64_BTRFS_LAST_FREE_OBJECTID
	args.key.max_offset = C.negative_one
	args.key.max_transid = C.negative_one

	subvolsByID := map[uint64]Info{}

	for {
		args.key.nr_items = 4096
		if err := ioctl(fp.Fd(), C.BTRFS_IOC_TREE_SEARCH, uintptr(unsafe.Pointer(&args))); err != nil {
			return nil, err
		}

		if args.key.nr_items == 0 {
			break
		}

		var (
			sh     C.struct_btrfs_ioctl_search_header
			shSize = unsafe.Sizeof(sh)
			buf    = (*[1<<31 - 1]byte)(unsafe.Pointer(&args.buf[0]))[:C.BTRFS_SEARCH_ARGS_BUFSIZE]
		)

		for i := 0; i < int(args.key.nr_items); i++ {
			sh = (*(*C.struct_btrfs_ioctl_search_header)(unsafe.Pointer(&buf[0])))
			buf = buf[shSize:]

			info := subvolsByID[uint64(sh.objectid)]
			info.ID = uint64(sh.objectid)

			if sh._type == C.BTRFS_ROOT_BACKREF_KEY {
				rr := (*(*C.struct_btrfs_root_ref)(unsafe.Pointer(&buf[0])))

				// This branch processes the backrefs from the root object. We
				// get an entry of the objectid, with name, but the parent is
				// the offset.

				nname := C.btrfs_stack_root_ref_name_len(&rr)
				name := string(buf[C.sizeof_struct_btrfs_root_ref : C.sizeof_struct_btrfs_root_ref+uintptr(nname)])

				info.ID = uint64(sh.objectid)
				info.ParentID = uint64(sh.offset)
				info.Name = name
				info.DirID = uint64(C.btrfs_stack_root_ref_dirid(&rr))

				subvolsByID[uint64(sh.objectid)] = info
			} else if sh._type == C.BTRFS_ROOT_ITEM_KEY &&
				(sh.objectid >= C.BTRFS_ROOT_ITEM_KEY ||
					sh.objectid == C.BTRFS_FS_TREE_OBJECTID) {

				var (
					ri  = (*C.struct_btrfs_root_item)(unsafe.Pointer(&buf[0]))
					gri C.struct_gosafe_btrfs_root_item
				)

				C.unpack_root_item(&gri, ri)

				if gri.flags&C.BTRFS_ROOT_SUBVOL_RDONLY != 0 {
					info.Readonly = true
				}

				// in this case, the offset is the actual offset.
				info.Offset = uint64(sh.offset)

				info.UUID = uuidString(&gri.uuid)
				info.ParentUUID = uuidString(&gri.parent_uuid)
				info.ReceivedUUID = uuidString(&gri.received_uuid)

				info.Generation = uint64(gri.gen)
				info.OriginalGeneration = uint64(gri.ogen)

				subvolsByID[uint64(sh.objectid)] = info
			}

			args.key.min_objectid = sh.objectid
			args.key.min_offset = sh.offset
			args.key.min_type = sh._type //  this is very questionable.

			buf = buf[sh.len:]
		}

		args.key.min_offset++
		if args.key.min_offset == 0 {
			args.key.min_type++
		} else {
			continue
		}

		if args.key.min_type > C.BTRFS_ROOT_BACKREF_KEY {
			args.key.min_type = C.BTRFS_ROOT_ITEM_KEY
			args.key.min_objectid++
		} else {
			continue
		}

		if args.key.min_objectid > args.key.max_objectid {
			break
		}
	}

	mnt, err := findMountPoint(path)
	if err != nil {
		return nil, err
	}

	for _, sv := range subvolsByID {
		path := sv.Name
		parentID := sv.ParentID

		for parentID != 0 {
			parent, ok := subvolsByID[parentID]
			if !ok {
				break
			}

			parentID = parent.ParentID
			path = filepath.Join(parent.Name, path)
		}

		sv.Path = filepath.Join(mnt, path)
	}
	return subvolsByID, nil
}

// SubvolList will return the information for all subvolumes corresponding to
// the provided path.
func SubvolList(path string) ([]Info, error) {
	subvolsByID, err := subvolMap(path)
	if err != nil {
		return nil, err
	}

	subvols := make([]Info, 0, len(subvolsByID))
	for _, sv := range subvolsByID {
		subvols = append(subvols, sv)
	}

	sort.Sort(infosByID(subvols))

	return subvols, nil
}

// SubvolCreate creates a subvolume at the provided path.
func SubvolCreate(path string) error {
	dir, name := filepath.Split(path)

	fp, err := os.Open(dir)
	if err != nil {
		return err
	}
	defer fp.Close()

	var args C.struct_btrfs_ioctl_vol_args
	args.fd = C.__s64(fp.Fd())

	if len(name) > C.BTRFS_PATH_NAME_MAX {
		return errors.Errorf("%q too long for subvolume", name)
	}
	nameptr := (*[1<<31 - 1]byte)(unsafe.Pointer(&args.name[0]))
	copy(nameptr[:C.BTRFS_PATH_NAME_MAX], []byte(name))

	if err := ioctl(fp.Fd(), C.BTRFS_IOC_SUBVOL_CREATE, uintptr(unsafe.Pointer(&args))); err != nil {
		return errors.Wrap(err, "btrfs subvolume create failed")
	}

	return nil
}

// SubvolSnapshot creates a snapshot in dst from src. If readonly is true, the
// snapshot will be readonly.
func SubvolSnapshot(dst, src string, readonly bool) error {
	dstdir, dstname := filepath.Split(dst)

	dstfp, err := openSubvolDir(dstdir)
	if err != nil {
		return errors.Wrapf(err, "opening snapshot desination subvolume failed")
	}
	defer dstfp.Close()

	srcfp, err := openSubvolDir(src)
	if err != nil {
		return errors.Wrapf(err, "opening snapshot source subvolume failed")
	}
	defer srcfp.Close()

	// dstdir is the ioctl arg, wile srcdir gets set on the args
	var args C.struct_btrfs_ioctl_vol_args_v2
	args.fd = C.__s64(srcfp.Fd())
	name := C.get_name_btrfs_ioctl_vol_args_v2(&args)

	if len(dstname) > C.BTRFS_SUBVOL_NAME_MAX {
		return errors.Errorf("%q too long for subvolume", dstname)
	}

	nameptr := (*[1<<31 - 1]byte)(unsafe.Pointer(name))
	copy(nameptr[:C.BTRFS_SUBVOL_NAME_MAX], []byte(dstname))

	if readonly {
		args.flags |= C.BTRFS_SUBVOL_RDONLY
	}

	if err := ioctl(dstfp.Fd(), C.BTRFS_IOC_SNAP_CREATE_V2, uintptr(unsafe.Pointer(&args))); err != nil {
		return errors.Wrapf(err, "snapshot create failed")
	}

	return nil
}

// SubvolDelete deletes the subvolumes under the given path.
func SubvolDelete(path string) error {
	dir, name := filepath.Split(path)
	fp, err := openSubvolDir(dir)
	if err != nil {
		return errors.Wrapf(err, "failed opening %v", path)
	}
	defer fp.Close()

	// remove child subvolumes
	if err := filepath.Walk(path, func(p string, fi os.FileInfo, err error) error {
		if err != nil {
			if os.IsNotExist(err) || p == path {
				return nil
			}

			return errors.Wrapf(err, "failed walking subvolume %v", p)
		}

		if !fi.IsDir() {
			return nil // just ignore it!
		}

		if p == path {
			return nil
		}

		if err := isFileInfoSubvol(fi); err != nil {
			return nil
		}

		if err := SubvolDelete(p); err != nil {
			return errors.Wrapf(err, "recursive delete of %v failed", p)
		}

		return filepath.SkipDir // children get walked by call above.
	}); err != nil {
		return err
	}

	var args C.struct_btrfs_ioctl_vol_args
	if len(name) > C.BTRFS_SUBVOL_NAME_MAX {
		return errors.Errorf("%q too long for subvolume", name)
	}

	nameptr := (*[1<<31 - 1]byte)(unsafe.Pointer(&args.name[0]))
	copy(nameptr[:C.BTRFS_SUBVOL_NAME_MAX], []byte(name))

	if err := ioctl(fp.Fd(), C.BTRFS_IOC_SNAP_DESTROY, uintptr(unsafe.Pointer(&args))); err != nil {
		return errors.Wrapf(err, "failed removing subvolume %v", path)
	}

	return nil
}

func openSubvolDir(path string) (*os.File, error) {
	fp, err := os.Open(path)
	if err != nil {
		return nil, errors.Wrapf(err, "opening %v as subvolume failed", path)
	}

	return fp, nil
}

func isStatfsSubvol(statfs *syscall.Statfs_t) error {
	if int64(statfs.Type) != int64(C.BTRFS_SUPER_MAGIC) {
		return errors.Errorf("not a btrfs filesystem")
	}

	return nil
}

func isFileInfoSubvol(fi os.FileInfo) error {
	if !fi.IsDir() {
		errors.Errorf("must be a directory")
	}

	stat := fi.Sys().(*syscall.Stat_t)

	if stat.Ino != C.BTRFS_FIRST_FREE_OBJECTID {
		return errors.Errorf("incorrect inode type")
	}

	return nil
}
<|endoftext|>


<commit_before>package urknall

import (
	"bytes"
	"fmt"
	"path/filepath"
	"strings"

	"github.com/dynport/urknall/cmd"
	"github.com/dynport/urknall/pubsub"
)

// A shortcut for rendering the given template to the given target.
func Run(target Target, tpl Template) (e error) {
	return (&Build{Target: target, Template: tpl}).Run()
}

// A shortcut for rendering the given template to the given target, without
// actually executing any commands. This is quite helpful to see which commands
// would be exeucted in the target's current state.
func DryRun(target Target, tpl Template) (e error) {
	return (&Build{Target: target, Template: tpl, DryRun: true}).Run()
}

// A build is the glue between a target and template. It contains the basic
// parameters required for actually doing something.
type Build struct {
	Target            // Where to run the build.
	Template          // What to actually build.
	DryRun   bool     // Should the commands actually be executed.
	Env      []string // Environment variables in the form `KEY=VALUE` set for each command.
}

// This will render the build's template into a package and run all its tasks.
func (b *Build) Run() error {
	pkg, e := renderTemplate(b.Template)
	if e != nil {
		return e
	}
	return pkg.Build(b)
}

func (build *Build) prepare() error {
	if build.User() == "" {
		return fmt.Errorf("User not set")
	}
	cmd, e := build.prepareCommand(fmt.Sprintf(`{ grep "^%s:" /etc/group | grep %s; } && [ -d /var/lib/urknall ]`, ukGROUP, build.User()))
	if e != nil {
		return e
	}
	if e := cmd.Run(); e != nil {
		// If user is missing the group, create group (if necessary), add user and restart ssh connection.
		cmds := []string{
			fmt.Sprintf(`{ grep -e '^%[1]s:' /etc/group > /dev/null || { groupadd %[1]s; }; }`, ukGROUP),
			fmt.Sprintf(`{ [ -d %[1]s ] || { mkdir -p -m 2775 %[1]s && chgrp %[2]s %[1]s; }; }`, ukCACHEDIR, ukGROUP),
			fmt.Sprintf("usermod -a -G %s %s", ukGROUP, build.User()),
		}

		cmd, e = build.prepareCommand(strings.Join(cmds, " && "))
		if e != nil {
			return e
		}
		out := &bytes.Buffer{}
		err := &bytes.Buffer{}
		cmd.SetStderr(err)
		cmd.SetStdout(out)
		if e := cmd.Run(); e != nil {
			return fmt.Errorf("failed to initiate user %q for provisioning: %s, out=%q err=%q", build.User(), e, out.String(), err.String())
		}
		return build.Reset()
	}
	return nil
}

func (build *Build) buildTask(tsk *task, ct checksumTree) (e error) {
	commands, e := tsk.Commands()
	if e != nil {
		return e
	}
	cacheKey := tsk.name
	if cacheKey == "" {
		return fmt.Errorf("CacheKey must not be empty")
	}
	checksumDir := fmt.Sprintf(ukCACHEDIR+"/%s", tsk.name)

	var found bool
	var checksumHash map[string]struct{}
	if checksumHash, found = ct[cacheKey]; !found {
		ct[cacheKey] = map[string]struct{}{}
		checksumHash = ct[cacheKey]

		// Create checksum dir and set group bit (all new files will inherit the directory's group). This allows for
		// different users (being part of that group) to create, modify and delete the contained checksum and log files.
		createChecksumDirCmd := fmt.Sprintf("mkdir -m2775 -p %s", checksumDir)

		cmd, e := build.prepareCommand(createChecksumDirCmd)
		if e != nil {
			return e
		}
		err := &bytes.Buffer{}

		cmd.SetStderr(err)

		if e := cmd.Run(); e != nil {
			return fmt.Errorf(err.String() + ": " + e.Error())
		}
	}

	for _, command := range commands {
		logMsg := command.Shell()
		if logger, ok := command.(cmd.Logger); ok {
			logMsg = logger.Logging()
		}

		checksum, e := commandChecksum(command)
		if e != nil {
			return e
		}
		m := &pubsub.Message{Key: pubsub.MessageRunlistsProvisionTask, TaskChecksum: checksum, Message: logMsg, Hostname: build.hostname(), RunlistName: cacheKey}
		if _, found := checksumHash[checksum]; found { // Task is cached.
			m.ExecStatus = pubsub.StatusCached
			m.Publish("finished")
			delete(checksumHash, checksum) // Delete checksums of cached tasks from hash.
			continue
		}

		if len(checksumHash) > 0 { // All remaining checksums are invalid, as something changed.
			if e = build.cleanUpRemainingCachedEntries(checksumDir, checksumHash); e != nil {
				return e
			}
			checksumHash = make(map[string]struct{})
		}
		m.ExecStatus = pubsub.StatusExecStart
		if build.DryRun {
			m.Publish("executed")
		} else {
			m.Publish("started")
			e = executeCommand(command, build, checksumDir, tsk.name)
			m.Error = e
			m.ExecStatus = pubsub.StatusExecFinished
			m.Publish("finished")
		}
		if e != nil {
			return e
		}
	}

	return nil
}

type checksumTree map[string]map[string]struct{}

func (build *Build) buildChecksumTree() (ct checksumTree, e error) {
	ct = checksumTree{}

	cmd, e := build.prepareCommand(fmt.Sprintf(`[ -d %[1]s ] && find %[1]s -type f -name \*.done`, ukCACHEDIR))
	if e != nil {
		return nil, e
	}
	out := &bytes.Buffer{}
	err := &bytes.Buffer{}

	cmd.SetStdout(out)
	cmd.SetStderr(err)

	if e := cmd.Run(); e != nil {
		return nil, fmt.Errorf("%s: out=%s err=%s", e.Error(), out.String(), err.String())
	}
	for _, line := range strings.Split(out.String(), "\n") {
		line = strings.TrimSpace(line)
		if line == "" {
			continue
		}

		pkgname := filepath.Dir(strings.TrimPrefix(line, ukCACHEDIR+"/"))
		checksum := strings.TrimSuffix(filepath.Base(line), ".done")
		if len(checksum) != 64 {
			return nil, fmt.Errorf("invalid checksum %q found for package %q", checksum, pkgname)
		}
		if _, found := ct[pkgname]; !found {
			ct[pkgname] = map[string]struct{}{}
		}
		ct[pkgname][checksum] = struct{}{}
	}

	return ct, nil
}

func (build *Build) cleanUpRemainingCachedEntries(checksumDir string, checksumHash map[string]struct{}) (e error) {
	invalidCacheEntries := make([]string, 0, len(checksumHash))
	for k, _ := range checksumHash {
		invalidCacheEntries = append(invalidCacheEntries, fmt.Sprintf("%s.done", k))
	}
	if build.DryRun {
		(&pubsub.Message{Key: pubsub.MessageCleanupCacheEntries, InvalidatedCacheEntries: invalidCacheEntries, Hostname: build.hostname()}).Publish(".dryrun")
	} else {
		cmd := fmt.Sprintf("cd %s && rm -f *.failed %s", checksumDir, strings.Join(invalidCacheEntries, " "))
		m := &pubsub.Message{Key: pubsub.MessageUrknallInternal, Hostname: build.hostname()}
		m.Publish("started")

		c, e := build.prepareCommand(cmd)
		if e != nil {
			return e
		}
		if e := c.Run(); e != nil {
			return e
		}
		//m.sshResult = result
		m.Publish("finished")
	}
	return nil
}

func (build *Build) prepareCommand(cmd string) (cmd.ExecCommand, error) {
	var sudo string
	if build.User() != "root" {
		sudo = "sudo "
	}
	cmd = fmt.Sprintf(sudo+"sh -x -e <<EOC\n%s\nEOC\n", cmd)
	return build.Command(cmd)
}

func (build *Build) hostname() string {
	if s, ok := build.Target.(fmt.Stringer); ok {
		return s.String()
	}
	return "MISSING"
}
<commit_msg>improved here docs<commit_after>package urknall

import (
	"bytes"
	"fmt"
	"path/filepath"
	"strings"

	"github.com/dynport/urknall/cmd"
	"github.com/dynport/urknall/pubsub"
)

// A shortcut for rendering the given template to the given target.
func Run(target Target, tpl Template) (e error) {
	return (&Build{Target: target, Template: tpl}).Run()
}

// A shortcut for rendering the given template to the given target, without
// actually executing any commands. This is quite helpful to see which commands
// would be exeucted in the target's current state.
func DryRun(target Target, tpl Template) (e error) {
	return (&Build{Target: target, Template: tpl, DryRun: true}).Run()
}

// A build is the glue between a target and template. It contains the basic
// parameters required for actually doing something.
type Build struct {
	Target            // Where to run the build.
	Template          // What to actually build.
	DryRun   bool     // Should the commands actually be executed.
	Env      []string // Environment variables in the form `KEY=VALUE` set for each command.
}

// This will render the build's template into a package and run all its tasks.
func (b *Build) Run() error {
	pkg, e := renderTemplate(b.Template)
	if e != nil {
		return e
	}
	return pkg.Build(b)
}

func (build *Build) prepare() error {
	if build.User() == "" {
		return fmt.Errorf("User not set")
	}
	cmd, e := build.prepareCommand(fmt.Sprintf(`{ grep "^%s:" /etc/group | grep %s; } && [ -d /var/lib/urknall ]`, ukGROUP, build.User()))
	if e != nil {
		return e
	}
	if e := cmd.Run(); e != nil {
		// If user is missing the group, create group (if necessary), add user and restart ssh connection.
		cmds := []string{
			fmt.Sprintf(`{ grep -e '^%[1]s:' /etc/group > /dev/null || { groupadd %[1]s; }; }`, ukGROUP),
			fmt.Sprintf(`{ [ -d %[1]s ] || { mkdir -p -m 2775 %[1]s && chgrp %[2]s %[1]s; }; }`, ukCACHEDIR, ukGROUP),
			fmt.Sprintf("usermod -a -G %s %s", ukGROUP, build.User()),
		}

		cmd, e = build.prepareCommand(strings.Join(cmds, " && "))
		if e != nil {
			return e
		}
		out := &bytes.Buffer{}
		err := &bytes.Buffer{}
		cmd.SetStderr(err)
		cmd.SetStdout(out)
		if e := cmd.Run(); e != nil {
			return fmt.Errorf("failed to initiate user %q for provisioning: %s, out=%q err=%q", build.User(), e, out.String(), err.String())
		}
		return build.Reset()
	}
	return nil
}

func (build *Build) buildTask(tsk *task, ct checksumTree) (e error) {
	commands, e := tsk.Commands()
	if e != nil {
		return e
	}
	cacheKey := tsk.name
	if cacheKey == "" {
		return fmt.Errorf("CacheKey must not be empty")
	}
	checksumDir := fmt.Sprintf(ukCACHEDIR+"/%s", tsk.name)

	var found bool
	var checksumHash map[string]struct{}
	if checksumHash, found = ct[cacheKey]; !found {
		ct[cacheKey] = map[string]struct{}{}
		checksumHash = ct[cacheKey]

		// Create checksum dir and set group bit (all new files will inherit the directory's group). This allows for
		// different users (being part of that group) to create, modify and delete the contained checksum and log files.
		createChecksumDirCmd := fmt.Sprintf("mkdir -m2775 -p %s", checksumDir)

		cmd, e := build.prepareCommand(createChecksumDirCmd)
		if e != nil {
			return e
		}
		err := &bytes.Buffer{}

		cmd.SetStderr(err)

		if e := cmd.Run(); e != nil {
			return fmt.Errorf(err.String() + ": " + e.Error())
		}
	}

	for _, command := range commands {
		logMsg := command.Shell()
		if logger, ok := command.(cmd.Logger); ok {
			logMsg = logger.Logging()
		}

		checksum, e := commandChecksum(command)
		if e != nil {
			return e
		}
		m := &pubsub.Message{Key: pubsub.MessageRunlistsProvisionTask, TaskChecksum: checksum, Message: logMsg, Hostname: build.hostname(), RunlistName: cacheKey}
		if _, found := checksumHash[checksum]; found { // Task is cached.
			m.ExecStatus = pubsub.StatusCached
			m.Publish("finished")
			delete(checksumHash, checksum) // Delete checksums of cached tasks from hash.
			continue
		}

		if len(checksumHash) > 0 { // All remaining checksums are invalid, as something changed.
			if e = build.cleanUpRemainingCachedEntries(checksumDir, checksumHash); e != nil {
				return e
			}
			checksumHash = make(map[string]struct{})
		}
		m.ExecStatus = pubsub.StatusExecStart
		if build.DryRun {
			m.Publish("executed")
		} else {
			m.Publish("started")
			e = executeCommand(command, build, checksumDir, tsk.name)
			m.Error = e
			m.ExecStatus = pubsub.StatusExecFinished
			m.Publish("finished")
		}
		if e != nil {
			return e
		}
	}

	return nil
}

type checksumTree map[string]map[string]struct{}

func (build *Build) buildChecksumTree() (ct checksumTree, e error) {
	ct = checksumTree{}

	cmd, e := build.prepareCommand(fmt.Sprintf(`[ -d %[1]s ] && find %[1]s -type f -name \*.done`, ukCACHEDIR))
	if e != nil {
		return nil, e
	}
	out := &bytes.Buffer{}
	err := &bytes.Buffer{}

	cmd.SetStdout(out)
	cmd.SetStderr(err)

	if e := cmd.Run(); e != nil {
		return nil, fmt.Errorf("%s: out=%s err=%s", e.Error(), out.String(), err.String())
	}
	for _, line := range strings.Split(out.String(), "\n") {
		line = strings.TrimSpace(line)
		if line == "" {
			continue
		}

		pkgname := filepath.Dir(strings.TrimPrefix(line, ukCACHEDIR+"/"))
		checksum := strings.TrimSuffix(filepath.Base(line), ".done")
		if len(checksum) != 64 {
			return nil, fmt.Errorf("invalid checksum %q found for package %q", checksum, pkgname)
		}
		if _, found := ct[pkgname]; !found {
			ct[pkgname] = map[string]struct{}{}
		}
		ct[pkgname][checksum] = struct{}{}
	}

	return ct, nil
}

func (build *Build) cleanUpRemainingCachedEntries(checksumDir string, checksumHash map[string]struct{}) (e error) {
	invalidCacheEntries := make([]string, 0, len(checksumHash))
	for k, _ := range checksumHash {
		invalidCacheEntries = append(invalidCacheEntries, fmt.Sprintf("%s.done", k))
	}
	if build.DryRun {
		(&pubsub.Message{Key: pubsub.MessageCleanupCacheEntries, InvalidatedCacheEntries: invalidCacheEntries, Hostname: build.hostname()}).Publish(".dryrun")
	} else {
		cmd := fmt.Sprintf("cd %s && rm -f *.failed %s", checksumDir, strings.Join(invalidCacheEntries, " "))
		m := &pubsub.Message{Key: pubsub.MessageUrknallInternal, Hostname: build.hostname()}
		m.Publish("started")

		c, e := build.prepareCommand(cmd)
		if e != nil {
			return e
		}
		if e := c.Run(); e != nil {
			return e
		}
		//m.sshResult = result
		m.Publish("finished")
	}
	return nil
}

func (build *Build) prepareCommand(cmd string) (cmd.ExecCommand, error) {
	var sudo string
	if build.User() != "root" {
		sudo = "sudo "
	}
	cmd = fmt.Sprintf(sudo+"sh -x -e <<\"EOC\"\n%s\nEOC\n", cmd)
	return build.Command(cmd)
}

func (build *Build) hostname() string {
	if s, ok := build.Target.(fmt.Stringer); ok {
		return s.String()
	}
	return "MISSING"
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// The ct_server binary runs the CT personality.
package main

import (
	"context"
	"flag"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/etcd/clientv3"
	etcdnaming "github.com/coreos/etcd/clientv3/naming"
	"github.com/golang/glog"
	"github.com/google/certificate-transparency-go/trillian/ctfe"
	"github.com/google/certificate-transparency-go/trillian/util"
	"github.com/google/trillian"
	"github.com/google/trillian/crypto/keys"
	"github.com/google/trillian/monitoring/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"google.golang.org/grpc"
	"google.golang.org/grpc/naming"
)

// Global flags that affect all log instances.
var (
	httpEndpoint       = flag.String("http_endpoint", "localhost:6962", "Endpoint for HTTP (host:port)")
	metricsEndpoint    = flag.String("metrics_endpoint", "localhost:6963", "Endpoint for serving metrics; if left empty, metrics will be visible on --http_endpoint")
	rpcBackendFlag     = flag.String("log_rpc_server", "localhost:8090", "Backend specification; comma-separated list or etcd service name (if --etcd_servers specified)")
	rpcDeadlineFlag    = flag.Duration("rpc_deadline", time.Second*10, "Deadline for backend RPC requests")
	logConfigFlag      = flag.String("log_config", "", "File holding log config in JSON")
	maxGetEntriesFlag  = flag.Int64("max_get_entries", 0, "Max number of entries we allow in a get-entries request (0=>use default 1000)")
	etcdServers        = flag.String("etcd_servers", "", "A comma-separated list of etcd servers")
	etcdHTTPService    = flag.String("etcd_http_service", "trillian-ctfe-http", "Service name to announce our HTTP endpoint under")
	etcdMetricsService = flag.String("etcd_metrics_service", "trillian-ctfe-metrics-http", "Service name to announce our HTTP metrics endpoint under")
)

func main() {
	flag.Parse()
	ctx := context.Background()

	if *maxGetEntriesFlag > 0 {
		ctfe.MaxGetEntriesAllowed = *maxGetEntriesFlag
	}

	// Get log config from file before we start.
	cfg, err := ctfe.LogConfigFromFile(*logConfigFlag)
	if err != nil {
		glog.Exitf("Failed to read log config: %v", err)
	}

	glog.CopyStandardLogTo("WARNING")
	glog.Info("**** CT HTTP Server Starting ****")

	metricsAt := *metricsEndpoint
	if metricsAt == "" {
		metricsAt = *httpEndpoint
	}

	// TODO(Martin2112): Support TLS and other stuff for RPC client and http server, this is just to
	// get started. Uses a blocking connection so we don't start serving before we're connected
	// to backend.
	var res naming.Resolver
	if len(*etcdServers) > 0 {
		// Use etcd to provide endpoint resolution.
		cfg := clientv3.Config{Endpoints: strings.Split(*etcdServers, ","), DialTimeout: 5 * time.Second}
		client, err := clientv3.New(cfg)
		if err != nil {
			glog.Exitf("Failed to connect to etcd at %v: %v", *etcdServers, err)
		}
		etcdRes := &etcdnaming.GRPCResolver{Client: client}
		res = etcdRes

		// Also announce ourselves.
		updateHTTP := naming.Update{Op: naming.Add, Addr: *httpEndpoint}
		updateMetrics := naming.Update{Op: naming.Add, Addr: metricsAt}
		glog.Infof("Announcing our presence in %v with %+v", *etcdHTTPService, updateHTTP)
		etcdRes.Update(ctx, *etcdHTTPService, updateHTTP)
		glog.Infof("Announcing our presence in %v with %+v", *etcdMetricsService, updateMetrics)
		etcdRes.Update(ctx, *etcdMetricsService, updateMetrics)

		byeHTTP := naming.Update{Op: naming.Delete, Addr: *httpEndpoint}
		byeMetrics := naming.Update{Op: naming.Delete, Addr: metricsAt}
		defer func() {
			glog.Infof("Removing our presence in %v with %+v", *etcdHTTPService, byeHTTP)
			etcdRes.Update(ctx, *etcdHTTPService, byeHTTP)
			glog.Infof("Removing our presence in %v with %+v", *etcdMetricsService, byeMetrics)
			etcdRes.Update(ctx, *etcdMetricsService, byeMetrics)
		}()
	} else {
		// Use a fixed endpoint resolution that just returns the addresses configured on the command line.
		res = util.FixedBackendResolver{}
	}
	bal := grpc.RoundRobin(res)
	conn, err := grpc.Dial(*rpcBackendFlag, grpc.WithInsecure(), grpc.WithBlock(), grpc.WithBalancer(bal))
	if err != nil {
		glog.Exitf("Could not connect to rpc server: %v", err)
	}
	defer conn.Close()
	client := trillian.NewTrillianLogClient(conn)

	sf := &keys.DefaultSignerFactory{}

	for _, c := range cfg {
		handlers, err := ctfe.SetUpInstance(ctx, client, c, sf, *rpcDeadlineFlag, prometheus.MetricFactory{})
		if err != nil {
			glog.Exitf("Failed to set up log instance for %+v: %v", cfg, err)
		}
		for path, handler := range *handlers {
			http.Handle(path, handler)
		}
	}
	if metricsAt != *httpEndpoint {
		// Run a separate handler for metrics.
		go func() {
			mux := http.NewServeMux()
			mux.Handle("/metrics", promhttp.Handler())
			metricsServer := http.Server{Addr: metricsAt, Handler: mux}
			err := metricsServer.ListenAndServe()
			glog.Warningf("Metrics server exited: %v", err)
		}()
	} else {
		// Handle metrics on the DefaultServeMux.
		http.Handle("/metrics", promhttp.Handler())
	}

	// Bring up the HTTP server and serve until we get a signal not to.
	go awaitSignal(func() {
		os.Exit(1)
	})
	server := http.Server{Addr: *httpEndpoint, Handler: nil}
	err = server.ListenAndServe()
	glog.Warningf("Server exited: %v", err)
	glog.Flush()
}

// awaitSignal waits for standard termination signals, then runs the given
// function; it should be run as a separate goroutine.
func awaitSignal(doneFn func()) {
	// Arrange notification for the standard set of signals used to terminate a server
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)

	// Now block main and wait for a signal
	sig := <-sigs
	glog.Warningf("Signal received: %v", sig)
	glog.Flush()

	doneFn()
}
<commit_msg>trillian/ctfe: update STH on a ticker<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// The ct_server binary runs the CT personality.
package main

import (
	"context"
	"flag"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/etcd/clientv3"
	etcdnaming "github.com/coreos/etcd/clientv3/naming"
	"github.com/golang/glog"
	"github.com/google/certificate-transparency-go/trillian/ctfe"
	"github.com/google/certificate-transparency-go/trillian/ctfe/configpb"
	"github.com/google/certificate-transparency-go/trillian/util"
	"github.com/google/trillian"
	"github.com/google/trillian/crypto/keys"
	"github.com/google/trillian/monitoring/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"google.golang.org/grpc"
	"google.golang.org/grpc/naming"
)

// Global flags that affect all log instances.
var (
	httpEndpoint       = flag.String("http_endpoint", "localhost:6962", "Endpoint for HTTP (host:port)")
	metricsEndpoint    = flag.String("metrics_endpoint", "localhost:6963", "Endpoint for serving metrics; if left empty, metrics will be visible on --http_endpoint")
	rpcBackendFlag     = flag.String("log_rpc_server", "localhost:8090", "Backend specification; comma-separated list or etcd service name (if --etcd_servers specified)")
	rpcDeadlineFlag    = flag.Duration("rpc_deadline", time.Second*10, "Deadline for backend RPC requests")
	getSTHInterval     = flag.Duration("get_sth_interval", time.Second*180, "Interval between internal get-sth operations (0 to disable)")
	logConfigFlag      = flag.String("log_config", "", "File holding log config in JSON")
	maxGetEntriesFlag  = flag.Int64("max_get_entries", 0, "Max number of entries we allow in a get-entries request (0=>use default 1000)")
	etcdServers        = flag.String("etcd_servers", "", "A comma-separated list of etcd servers")
	etcdHTTPService    = flag.String("etcd_http_service", "trillian-ctfe-http", "Service name to announce our HTTP endpoint under")
	etcdMetricsService = flag.String("etcd_metrics_service", "trillian-ctfe-metrics-http", "Service name to announce our HTTP metrics endpoint under")
)

func main() {
	flag.Parse()
	ctx := context.Background()

	if *maxGetEntriesFlag > 0 {
		ctfe.MaxGetEntriesAllowed = *maxGetEntriesFlag
	}

	// Get log config from file before we start.
	cfg, err := ctfe.LogConfigFromFile(*logConfigFlag)
	if err != nil {
		glog.Exitf("Failed to read log config: %v", err)
	}

	glog.CopyStandardLogTo("WARNING")
	glog.Info("**** CT HTTP Server Starting ****")

	metricsAt := *metricsEndpoint
	if metricsAt == "" {
		metricsAt = *httpEndpoint
	}

	// TODO(Martin2112): Support TLS and other stuff for RPC client and http server, this is just to
	// get started. Uses a blocking connection so we don't start serving before we're connected
	// to backend.
	var res naming.Resolver
	if len(*etcdServers) > 0 {
		// Use etcd to provide endpoint resolution.
		cfg := clientv3.Config{Endpoints: strings.Split(*etcdServers, ","), DialTimeout: 5 * time.Second}
		client, err := clientv3.New(cfg)
		if err != nil {
			glog.Exitf("Failed to connect to etcd at %v: %v", *etcdServers, err)
		}
		etcdRes := &etcdnaming.GRPCResolver{Client: client}
		res = etcdRes

		// Also announce ourselves.
		updateHTTP := naming.Update{Op: naming.Add, Addr: *httpEndpoint}
		updateMetrics := naming.Update{Op: naming.Add, Addr: metricsAt}
		glog.Infof("Announcing our presence in %v with %+v", *etcdHTTPService, updateHTTP)
		etcdRes.Update(ctx, *etcdHTTPService, updateHTTP)
		glog.Infof("Announcing our presence in %v with %+v", *etcdMetricsService, updateMetrics)
		etcdRes.Update(ctx, *etcdMetricsService, updateMetrics)

		byeHTTP := naming.Update{Op: naming.Delete, Addr: *httpEndpoint}
		byeMetrics := naming.Update{Op: naming.Delete, Addr: metricsAt}
		defer func() {
			glog.Infof("Removing our presence in %v with %+v", *etcdHTTPService, byeHTTP)
			etcdRes.Update(ctx, *etcdHTTPService, byeHTTP)
			glog.Infof("Removing our presence in %v with %+v", *etcdMetricsService, byeMetrics)
			etcdRes.Update(ctx, *etcdMetricsService, byeMetrics)
		}()
	} else {
		// Use a fixed endpoint resolution that just returns the addresses configured on the command line.
		res = util.FixedBackendResolver{}
	}
	bal := grpc.RoundRobin(res)
	conn, err := grpc.Dial(*rpcBackendFlag, grpc.WithInsecure(), grpc.WithBlock(), grpc.WithBalancer(bal))
	if err != nil {
		glog.Exitf("Could not connect to rpc server: %v", err)
	}
	defer conn.Close()
	client := trillian.NewTrillianLogClient(conn)

	sf := &keys.DefaultSignerFactory{}

	for _, c := range cfg {
		handlers, err := ctfe.SetUpInstance(ctx, client, c, sf, *rpcDeadlineFlag, prometheus.MetricFactory{})
		if err != nil {
			glog.Exitf("Failed to set up log instance for %+v: %v", cfg, err)
		}
		for path, handler := range *handlers {
			http.Handle(path, handler)
		}
	}
	if metricsAt != *httpEndpoint {
		// Run a separate handler for metrics.
		go func() {
			mux := http.NewServeMux()
			mux.Handle("/metrics", promhttp.Handler())
			metricsServer := http.Server{Addr: metricsAt, Handler: mux}
			err := metricsServer.ListenAndServe()
			glog.Warningf("Metrics server exited: %v", err)
		}()
	} else {
		// Handle metrics on the DefaultServeMux.
		http.Handle("/metrics", promhttp.Handler())
	}

	if *getSTHInterval > 0 {
		// Regularly update the internal STH for each log so our metrics stay up-to-date with any tree head
		// changes that are not triggered by us.
		for _, c := range cfg {
			ticker := time.NewTicker(*getSTHInterval)
			go func(c *configpb.LogConfig) {
				glog.Infof("start internal get-sth operations on log %v (%d)", c.Prefix, c.LogId)
				for t := range ticker.C {
					glog.V(1).Infof("tick at %v: force internal get-sth for log %v (%d)", t, c.Prefix, c.LogId)
					if _, err := ctfe.GetTreeHead(ctx, client, c.LogId, c.Prefix); err != nil {
						glog.Warningf("failed to retrieve tree head for log %v (%d): %v", c.Prefix, c.LogId, err)
					}
				}
			}(c)
		}
	}

	// Bring up the HTTP server and serve until we get a signal not to.
	go awaitSignal(func() {
		os.Exit(1)
	})
	server := http.Server{Addr: *httpEndpoint, Handler: nil}
	err = server.ListenAndServe()
	glog.Warningf("Server exited: %v", err)
	glog.Flush()
}

// awaitSignal waits for standard termination signals, then runs the given
// function; it should be run as a separate goroutine.
func awaitSignal(doneFn func()) {
	// Arrange notification for the standard set of signals used to terminate a server
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)

	// Now block main and wait for a signal
	sig := <-sigs
	glog.Warningf("Signal received: %v", sig)
	glog.Flush()

	doneFn()
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package evictions

import (
	"fmt"
	"net/http/httptest"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"k8s.io/api/core/v1"
	"k8s.io/api/policy/v1beta1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/informers"
	clientset "k8s.io/client-go/kubernetes"
	restclient "k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"
	"k8s.io/kubernetes/pkg/controller/disruption"
	"k8s.io/kubernetes/test/integration/framework"
	"reflect"
)

const (
	numOfEvictions = 10
)

// TestConcurrentEvictionRequests is to make sure pod disruption budgets (PDB) controller is able to
// handle concurrent eviction requests. Original issue:#37605
func TestConcurrentEvictionRequests(t *testing.T) {
	podNameFormat := "test-pod-%d"

	s, closeFn, rm, informers, clientSet := rmSetup(t)
	defer closeFn()

	ns := framework.CreateTestingNamespace("concurrent-eviction-requests", s, t)
	defer framework.DeleteTestingNamespace(ns, s, t)

	stopCh := make(chan struct{})
	informers.Start(stopCh)
	go rm.Run(stopCh)
	defer close(stopCh)

	config := restclient.Config{Host: s.URL}
	clientSet, err := clientset.NewForConfig(&config)
	if err != nil {
		t.Fatalf("Failed to create clientset: %v", err)
	}

	var gracePeriodSeconds int64 = 30
	deleteOption := &metav1.DeleteOptions{
		GracePeriodSeconds: &gracePeriodSeconds,
	}

	// Generate numOfEvictions pods to evict
	for i := 0; i < numOfEvictions; i++ {
		podName := fmt.Sprintf(podNameFormat, i)
		pod := newPod(podName)

		if _, err := clientSet.CoreV1().Pods(ns.Name).Create(pod); err != nil {
			t.Errorf("Failed to create pod: %v", err)
		}

		addPodConditionReady(pod)
		if _, err := clientSet.CoreV1().Pods(ns.Name).UpdateStatus(pod); err != nil {
			t.Fatal(err)
		}
	}

	waitToObservePods(t, informers.Core().V1().Pods().Informer(), numOfEvictions)

	pdb := newPDB()
	if _, err := clientSet.Policy().PodDisruptionBudgets(ns.Name).Create(pdb); err != nil {
		t.Errorf("Failed to create PodDisruptionBudget: %v", err)
	}

	waitPDBStable(t, clientSet, numOfEvictions, ns.Name, pdb.Name)

	var numberPodsEvicted uint32 = 0
	errCh := make(chan error, 3*numOfEvictions)
	var wg sync.WaitGroup
	// spawn numOfEvictions goroutines to concurrently evict the pods
	for i := 0; i < numOfEvictions; i++ {
		wg.Add(1)
		go func(id int, errCh chan error) {
			defer wg.Done()
			podName := fmt.Sprintf(podNameFormat, id)
			eviction := newEviction(ns.Name, podName, deleteOption)

			err := wait.PollImmediate(5*time.Second, 60*time.Second, func() (bool, error) {
				e := clientSet.Policy().Evictions(ns.Name).Evict(eviction)
				switch {
				case errors.IsTooManyRequests(e):
					return false, nil
				case errors.IsConflict(e):
					return false, fmt.Errorf("Unexpected Conflict (409) error caused by failing to handle concurrent PDB updates: %v", e)
				case e == nil:
					return true, nil
				default:
					return false, e
				}
			})

			if err != nil {
				errCh <- err
				// should not return here otherwise we would leak the pod
			}

			_, err = clientSet.CoreV1().Pods(ns.Name).Get(podName, metav1.GetOptions{})
			switch {
			case errors.IsNotFound(err):
				atomic.AddUint32(&numberPodsEvicted, 1)
				// pod was evicted and deleted so return from goroutine immediately
				return
			case err == nil:
				// this shouldn't happen if the pod was evicted successfully
				errCh <- fmt.Errorf("Pod %q is expected to be evicted", podName)
			default:
				errCh <- err
			}

			// delete pod which still exists due to error
			e := clientSet.CoreV1().Pods(ns.Name).Delete(podName, deleteOption)
			if e != nil {
				errCh <- e
			}

		}(i, errCh)
	}

	wg.Wait()

	close(errCh)
	var errList []error
	if err := clientSet.Policy().PodDisruptionBudgets(ns.Name).Delete(pdb.Name, deleteOption); err != nil {
		errList = append(errList, fmt.Errorf("Failed to delete PodDisruptionBudget: %v", err))
	}
	for err := range errCh {
		errList = append(errList, err)
	}
	if len(errList) > 0 {
		t.Fatal(utilerrors.NewAggregate(errList))
	}

	if atomic.LoadUint32(&numberPodsEvicted) != numOfEvictions {
		t.Fatalf("fewer number of successful evictions than expected : %d", numberPodsEvicted)
	}
}

// TestTerminalPodEviction ensures that PDB is not checked for terminal pods.
func TestTerminalPodEviction(t *testing.T) {
	s, closeFn, rm, informers, clientSet := rmSetup(t)
	defer closeFn()

	ns := framework.CreateTestingNamespace("terminalpod-eviction", s, t)
	defer framework.DeleteTestingNamespace(ns, s, t)

	stopCh := make(chan struct{})
	informers.Start(stopCh)
	go rm.Run(stopCh)
	defer close(stopCh)

	config := restclient.Config{Host: s.URL}
	clientSet, err := clientset.NewForConfig(&config)
	if err != nil {
		t.Fatalf("Failed to create clientset: %v", err)
	}

	var gracePeriodSeconds int64 = 30
	deleteOption := &metav1.DeleteOptions{
		GracePeriodSeconds: &gracePeriodSeconds,
	}
	pod := newPod("test-terminal-pod1")
	if _, err := clientSet.CoreV1().Pods(ns.Name).Create(pod); err != nil {
		t.Errorf("Failed to create pod: %v", err)
	}
	addPodConditionSucceeded(pod)
	if _, err := clientSet.CoreV1().Pods(ns.Name).UpdateStatus(pod); err != nil {
		t.Fatal(err)
	}

	waitToObservePods(t, informers.Core().V1().Pods().Informer(), 1)

	pdb := newPDB()
	if _, err := clientSet.Policy().PodDisruptionBudgets(ns.Name).Create(pdb); err != nil {
		t.Errorf("Failed to create PodDisruptionBudget: %v", err)
	}

	pdbList, err := clientSet.Policy().PodDisruptionBudgets(ns.Name).List(metav1.ListOptions{})
	if err != nil {
		t.Fatalf("Error while listing pod disruption budget")
	}
	oldPdb := pdbList.Items[0]
	eviction := newEviction(ns.Name, pod.Name, deleteOption)
	err = wait.PollImmediate(5*time.Second, 60*time.Second, func() (bool, error) {
		e := clientSet.Policy().Evictions(ns.Name).Evict(eviction)
		switch {
		case errors.IsTooManyRequests(e):
			return false, nil
		case errors.IsConflict(e):
			return false, fmt.Errorf("Unexpected Conflict (409) error caused by failing to handle concurrent PDB updates: %v", e)
		case e == nil:
			return true, nil
		default:
			return false, e
		}
	})
	if err != nil {
		t.Fatalf("Eviction of pod failed %v", err)
	}
	pdbList, err = clientSet.Policy().PodDisruptionBudgets(ns.Name).List(metav1.ListOptions{})
	if err != nil {
		t.Fatalf("Error while listing pod disruption budget")
	}
	newPdb := pdbList.Items[0]
	// We shouldn't see an update in pod disruption budget status' generation number as we are evicting terminal pods without checking for pod disruption.
	if !reflect.DeepEqual(newPdb.Status.ObservedGeneration, oldPdb.Status.ObservedGeneration) {
		t.Fatalf("Expected the pdb generation to be of same value %v but got %v", newPdb.Status.ObservedGeneration, oldPdb.Status.ObservedGeneration)
	}

	if err := clientSet.Policy().PodDisruptionBudgets(ns.Name).Delete(pdb.Name, deleteOption); err != nil {
		t.Fatalf("Failed to delete pod disruption budget")
	}
}

func newPod(podName string) *v1.Pod {
	return &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:   podName,
			Labels: map[string]string{"app": "test-evictions"},
		},
		Spec: v1.PodSpec{
			Containers: []v1.Container{
				{
					Name:  "fake-name",
					Image: "fakeimage",
				},
			},
		},
	}
}

func addPodConditionSucceeded(pod *v1.Pod) {
	pod.Status = v1.PodStatus{
		Phase: v1.PodSucceeded,
		Conditions: []v1.PodCondition{
			{
				Type:   v1.PodReady,
				Status: v1.ConditionTrue,
			},
		},
	}
}

func addPodConditionReady(pod *v1.Pod) {
	pod.Status = v1.PodStatus{
		Phase: v1.PodRunning,
		Conditions: []v1.PodCondition{
			{
				Type:   v1.PodReady,
				Status: v1.ConditionTrue,
			},
		},
	}
}

func newPDB() *v1beta1.PodDisruptionBudget {
	return &v1beta1.PodDisruptionBudget{
		ObjectMeta: metav1.ObjectMeta{
			Name: "test-pdb",
		},
		Spec: v1beta1.PodDisruptionBudgetSpec{
			MinAvailable: &intstr.IntOrString{
				Type:   intstr.Int,
				IntVal: 0,
			},
			Selector: &metav1.LabelSelector{
				MatchLabels: map[string]string{"app": "test-evictions"},
			},
		},
	}
}

func newEviction(ns, evictionName string, deleteOption *metav1.DeleteOptions) *v1beta1.Eviction {
	return &v1beta1.Eviction{
		TypeMeta: metav1.TypeMeta{
			APIVersion: "Policy/v1beta1",
			Kind:       "Eviction",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name:      evictionName,
			Namespace: ns,
		},
		DeleteOptions: deleteOption,
	}
}

func rmSetup(t *testing.T) (*httptest.Server, framework.CloseFunc, *disruption.DisruptionController, informers.SharedInformerFactory, clientset.Interface) {
	masterConfig := framework.NewIntegrationTestMasterConfig()
	_, s, closeFn := framework.RunAMaster(masterConfig)

	config := restclient.Config{Host: s.URL}
	clientSet, err := clientset.NewForConfig(&config)
	if err != nil {
		t.Fatalf("Error in create clientset: %v", err)
	}
	resyncPeriod := 12 * time.Hour
	informers := informers.NewSharedInformerFactory(clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "pdb-informers")), resyncPeriod)

	rm := disruption.NewDisruptionController(
		informers.Core().V1().Pods(),
		informers.Policy().V1beta1().PodDisruptionBudgets(),
		informers.Core().V1().ReplicationControllers(),
		informers.Apps().V1().ReplicaSets(),
		informers.Apps().V1().Deployments(),
		informers.Apps().V1().StatefulSets(),
		clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "disruption-controller")),
	)
	return s, closeFn, rm, informers, clientSet
}

// wait for the podInformer to observe the pods. Call this function before
// running the RS controller to prevent the rc manager from creating new pods
// rather than adopting the existing ones.
func waitToObservePods(t *testing.T, podInformer cache.SharedIndexInformer, podNum int) {
	if err := wait.PollImmediate(2*time.Second, 60*time.Second, func() (bool, error) {
		objects := podInformer.GetIndexer().List()
		if len(objects) == podNum {
			return true, nil
		}
		return false, nil
	}); err != nil {
		t.Fatal(err)
	}
}

func waitPDBStable(t *testing.T, clientSet clientset.Interface, podNum int32, ns, pdbName string) {
	if err := wait.PollImmediate(2*time.Second, 60*time.Second, func() (bool, error) {
		pdb, err := clientSet.Policy().PodDisruptionBudgets(ns).Get(pdbName, metav1.GetOptions{})
		if err != nil {
			return false, err
		}
		if pdb.Status.CurrentHealthy != podNum {
			return false, nil
		}
		return true, nil
	}); err != nil {
		t.Fatal(err)
	}
}
<commit_msg>Fix flaking evictions test<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package evictions

import (
	"fmt"
	"net/http/httptest"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"k8s.io/api/core/v1"
	"k8s.io/api/policy/v1beta1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/informers"
	clientset "k8s.io/client-go/kubernetes"
	restclient "k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"
	"k8s.io/kubernetes/pkg/controller/disruption"
	"k8s.io/kubernetes/test/integration/framework"
	"reflect"
)

const (
	numOfEvictions = 10
)

// TestConcurrentEvictionRequests is to make sure pod disruption budgets (PDB) controller is able to
// handle concurrent eviction requests. Original issue:#37605
func TestConcurrentEvictionRequests(t *testing.T) {
	podNameFormat := "test-pod-%d"

	s, closeFn, rm, informers, clientSet := rmSetup(t)
	defer closeFn()

	ns := framework.CreateTestingNamespace("concurrent-eviction-requests", s, t)
	defer framework.DeleteTestingNamespace(ns, s, t)

	stopCh := make(chan struct{})
	informers.Start(stopCh)
	go rm.Run(stopCh)
	defer close(stopCh)

	config := restclient.Config{Host: s.URL}
	clientSet, err := clientset.NewForConfig(&config)
	if err != nil {
		t.Fatalf("Failed to create clientset: %v", err)
	}

	var gracePeriodSeconds int64 = 30
	deleteOption := &metav1.DeleteOptions{
		GracePeriodSeconds: &gracePeriodSeconds,
	}

	// Generate numOfEvictions pods to evict
	for i := 0; i < numOfEvictions; i++ {
		podName := fmt.Sprintf(podNameFormat, i)
		pod := newPod(podName)

		if _, err := clientSet.CoreV1().Pods(ns.Name).Create(pod); err != nil {
			t.Errorf("Failed to create pod: %v", err)
		}

		addPodConditionReady(pod)
		if _, err := clientSet.CoreV1().Pods(ns.Name).UpdateStatus(pod); err != nil {
			t.Fatal(err)
		}
	}

	waitToObservePods(t, informers.Core().V1().Pods().Informer(), numOfEvictions, v1.PodRunning)

	pdb := newPDB()
	if _, err := clientSet.Policy().PodDisruptionBudgets(ns.Name).Create(pdb); err != nil {
		t.Errorf("Failed to create PodDisruptionBudget: %v", err)
	}

	waitPDBStable(t, clientSet, numOfEvictions, ns.Name, pdb.Name)

	var numberPodsEvicted uint32 = 0
	errCh := make(chan error, 3*numOfEvictions)
	var wg sync.WaitGroup
	// spawn numOfEvictions goroutines to concurrently evict the pods
	for i := 0; i < numOfEvictions; i++ {
		wg.Add(1)
		go func(id int, errCh chan error) {
			defer wg.Done()
			podName := fmt.Sprintf(podNameFormat, id)
			eviction := newEviction(ns.Name, podName, deleteOption)

			err := wait.PollImmediate(5*time.Second, 60*time.Second, func() (bool, error) {
				e := clientSet.Policy().Evictions(ns.Name).Evict(eviction)
				switch {
				case errors.IsTooManyRequests(e):
					return false, nil
				case errors.IsConflict(e):
					return false, fmt.Errorf("Unexpected Conflict (409) error caused by failing to handle concurrent PDB updates: %v", e)
				case e == nil:
					return true, nil
				default:
					return false, e
				}
			})

			if err != nil {
				errCh <- err
				// should not return here otherwise we would leak the pod
			}

			_, err = clientSet.CoreV1().Pods(ns.Name).Get(podName, metav1.GetOptions{})
			switch {
			case errors.IsNotFound(err):
				atomic.AddUint32(&numberPodsEvicted, 1)
				// pod was evicted and deleted so return from goroutine immediately
				return
			case err == nil:
				// this shouldn't happen if the pod was evicted successfully
				errCh <- fmt.Errorf("Pod %q is expected to be evicted", podName)
			default:
				errCh <- err
			}

			// delete pod which still exists due to error
			e := clientSet.CoreV1().Pods(ns.Name).Delete(podName, deleteOption)
			if e != nil {
				errCh <- e
			}

		}(i, errCh)
	}

	wg.Wait()

	close(errCh)
	var errList []error
	if err := clientSet.Policy().PodDisruptionBudgets(ns.Name).Delete(pdb.Name, deleteOption); err != nil {
		errList = append(errList, fmt.Errorf("Failed to delete PodDisruptionBudget: %v", err))
	}
	for err := range errCh {
		errList = append(errList, err)
	}
	if len(errList) > 0 {
		t.Fatal(utilerrors.NewAggregate(errList))
	}

	if atomic.LoadUint32(&numberPodsEvicted) != numOfEvictions {
		t.Fatalf("fewer number of successful evictions than expected : %d", numberPodsEvicted)
	}
}

// TestTerminalPodEviction ensures that PDB is not checked for terminal pods.
func TestTerminalPodEviction(t *testing.T) {
	s, closeFn, rm, informers, clientSet := rmSetup(t)
	defer closeFn()

	ns := framework.CreateTestingNamespace("terminalpod-eviction", s, t)
	defer framework.DeleteTestingNamespace(ns, s, t)

	stopCh := make(chan struct{})
	informers.Start(stopCh)
	go rm.Run(stopCh)
	defer close(stopCh)

	config := restclient.Config{Host: s.URL}
	clientSet, err := clientset.NewForConfig(&config)
	if err != nil {
		t.Fatalf("Failed to create clientset: %v", err)
	}

	var gracePeriodSeconds int64 = 30
	deleteOption := &metav1.DeleteOptions{
		GracePeriodSeconds: &gracePeriodSeconds,
	}
	pod := newPod("test-terminal-pod1")
	if _, err := clientSet.CoreV1().Pods(ns.Name).Create(pod); err != nil {
		t.Errorf("Failed to create pod: %v", err)
	}

	addPodConditionSucceeded(pod)
	if _, err := clientSet.CoreV1().Pods(ns.Name).UpdateStatus(pod); err != nil {
		t.Fatal(err)
	}

	waitToObservePods(t, informers.Core().V1().Pods().Informer(), 1, v1.PodSucceeded)

	pdb := newPDB()
	if _, err := clientSet.Policy().PodDisruptionBudgets(ns.Name).Create(pdb); err != nil {
		t.Errorf("Failed to create PodDisruptionBudget: %v", err)
	}

	waitPDBStable(t, clientSet, 1, ns.Name, pdb.Name)

	pdbList, err := clientSet.Policy().PodDisruptionBudgets(ns.Name).List(metav1.ListOptions{})
	if err != nil {
		t.Fatalf("Error while listing pod disruption budget")
	}
	oldPdb := pdbList.Items[0]
	eviction := newEviction(ns.Name, pod.Name, deleteOption)
	err = wait.PollImmediate(5*time.Second, 60*time.Second, func() (bool, error) {
		e := clientSet.Policy().Evictions(ns.Name).Evict(eviction)
		switch {
		case errors.IsTooManyRequests(e):
			return false, nil
		case errors.IsConflict(e):
			return false, fmt.Errorf("Unexpected Conflict (409) error caused by failing to handle concurrent PDB updates: %v", e)
		case e == nil:
			return true, nil
		default:
			return false, e
		}
	})
	if err != nil {
		t.Fatalf("Eviction of pod failed %v", err)
	}
	pdbList, err = clientSet.Policy().PodDisruptionBudgets(ns.Name).List(metav1.ListOptions{})
	if err != nil {
		t.Fatalf("Error while listing pod disruption budget")
	}
	newPdb := pdbList.Items[0]
	// We shouldn't see an update in pod disruption budget status' generation number as we are evicting terminal pods without checking for pod disruption.
	if !reflect.DeepEqual(newPdb.Status.ObservedGeneration, oldPdb.Status.ObservedGeneration) {
		t.Fatalf("Expected the pdb generation to be of same value %v but got %v", newPdb.Status.ObservedGeneration, oldPdb.Status.ObservedGeneration)
	}

	if err := clientSet.Policy().PodDisruptionBudgets(ns.Name).Delete(pdb.Name, deleteOption); err != nil {
		t.Fatalf("Failed to delete pod disruption budget")
	}
}

func newPod(podName string) *v1.Pod {
	return &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:   podName,
			Labels: map[string]string{"app": "test-evictions"},
		},
		Spec: v1.PodSpec{
			Containers: []v1.Container{
				{
					Name:  "fake-name",
					Image: "fakeimage",
				},
			},
		},
	}
}

func addPodConditionSucceeded(pod *v1.Pod) {
	pod.Status = v1.PodStatus{
		Phase: v1.PodSucceeded,
		Conditions: []v1.PodCondition{
			{
				Type:   v1.PodReady,
				Status: v1.ConditionTrue,
			},
		},
	}
}

func addPodConditionReady(pod *v1.Pod) {
	pod.Status = v1.PodStatus{
		Phase: v1.PodRunning,
		Conditions: []v1.PodCondition{
			{
				Type:   v1.PodReady,
				Status: v1.ConditionTrue,
			},
		},
	}
}

func newPDB() *v1beta1.PodDisruptionBudget {
	return &v1beta1.PodDisruptionBudget{
		ObjectMeta: metav1.ObjectMeta{
			Name: "test-pdb",
		},
		Spec: v1beta1.PodDisruptionBudgetSpec{
			MinAvailable: &intstr.IntOrString{
				Type:   intstr.Int,
				IntVal: 0,
			},
			Selector: &metav1.LabelSelector{
				MatchLabels: map[string]string{"app": "test-evictions"},
			},
		},
	}
}

func newEviction(ns, evictionName string, deleteOption *metav1.DeleteOptions) *v1beta1.Eviction {
	return &v1beta1.Eviction{
		TypeMeta: metav1.TypeMeta{
			APIVersion: "Policy/v1beta1",
			Kind:       "Eviction",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name:      evictionName,
			Namespace: ns,
		},
		DeleteOptions: deleteOption,
	}
}

func rmSetup(t *testing.T) (*httptest.Server, framework.CloseFunc, *disruption.DisruptionController, informers.SharedInformerFactory, clientset.Interface) {
	masterConfig := framework.NewIntegrationTestMasterConfig()
	_, s, closeFn := framework.RunAMaster(masterConfig)

	config := restclient.Config{Host: s.URL}
	clientSet, err := clientset.NewForConfig(&config)
	if err != nil {
		t.Fatalf("Error in create clientset: %v", err)
	}
	resyncPeriod := 12 * time.Hour
	informers := informers.NewSharedInformerFactory(clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "pdb-informers")), resyncPeriod)

	rm := disruption.NewDisruptionController(
		informers.Core().V1().Pods(),
		informers.Policy().V1beta1().PodDisruptionBudgets(),
		informers.Core().V1().ReplicationControllers(),
		informers.Apps().V1().ReplicaSets(),
		informers.Apps().V1().Deployments(),
		informers.Apps().V1().StatefulSets(),
		clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "disruption-controller")),
	)
	return s, closeFn, rm, informers, clientSet
}

// wait for the podInformer to observe the pods. Call this function before
// running the RS controller to prevent the rc manager from creating new pods
// rather than adopting the existing ones.
func waitToObservePods(t *testing.T, podInformer cache.SharedIndexInformer, podNum int, phase v1.PodPhase) {
	if err := wait.PollImmediate(2*time.Second, 60*time.Second, func() (bool, error) {
		objects := podInformer.GetIndexer().List()
		if len(objects) != podNum {
			return false, nil
		}
		for _, obj := range objects {
			pod := obj.(*v1.Pod)
			if pod.Status.Phase != phase {
				return false, nil
			}
		}
		return true, nil
	}); err != nil {
		t.Fatal(err)
	}
}

func waitPDBStable(t *testing.T, clientSet clientset.Interface, podNum int32, ns, pdbName string) {
	if err := wait.PollImmediate(2*time.Second, 60*time.Second, func() (bool, error) {
		pdb, err := clientSet.Policy().PodDisruptionBudgets(ns).Get(pdbName, metav1.GetOptions{})
		if err != nil {
			return false, err
		}
		if pdb.Status.CurrentHealthy != podNum {
			return false, nil
		}
		return true, nil
	}); err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package webdriver_test

import (
	"fmt"
	"strings"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/sclevine/agouti"
	"github.com/sclevine/agouti/api"
	. "github.com/sclevine/agouti/matchers"
)

var _ = Describe("gRPC-Web Unit Tests", func() {
	var page *agouti.Page

	BeforeEach(func() {
		var err error
		page, err = agoutiDriver.NewPage()
		Expect(err).NotTo(HaveOccurred())
	})

	AfterEach(func() {
		Expect(page.Destroy()).NotTo(HaveOccurred())
	})

	It("should pass", func() {
		By("loading the test page", func() {
			Expect(page.Navigate("https://localhost:10000")).NotTo(HaveOccurred())
			Expect(page).To(HaveURL("https://localhost:10000/"))
		})

		By("finding the number of failures", func() {
			Eventually(page.FirstByClass("failed")).Should(BeFound())
			failures, err := page.FindByID("qunit-testresult").FindByClass("failed").Text()
			Expect(err).NotTo(HaveOccurred())
			if failures == "0" {
				return
			}

			// We have at least one failure - lets compile an error message
			Eventually(page.FindByID(
				"qunit-tests",
			).AllByClass(
				"fail",
			).AllByClass(
				"fail",
			)).Should(BeFound())
			messages := page.FindByID(
				"qunit-tests",
			).AllByClass(
				"fail",
			).AllByClass(
				"fail",
			)
			elements, err := messages.Elements()
			Expect(err).NotTo(HaveOccurred())
			var errMsgs []string
			for _, element := range elements {
				// Get error summary
				msg, err := element.GetElement(api.Selector{
					Using: "css selector",
					Value: ".test-message",
				})
				Expect(err).NotTo(HaveOccurred())
				errText, err := msg.GetText()
				Expect(err).NotTo(HaveOccurred())

				// Get diff
				expected, err := element.GetElements(api.Selector{
					Using: "css selector",
					Value: "del",
				})
				Expect(err).NotTo(HaveOccurred())
				var expectedText string
				if len(expected) > 0 {
					expectedText, err = expected[0].GetText()
					Expect(err).NotTo(HaveOccurred())
				}
				actual, err := element.GetElements(api.Selector{
					Using: "css selector",
					Value: "ins",
				})
				Expect(err).NotTo(HaveOccurred())
				var actualText string
				if len(actual) > 0 {
					actualText, err = actual[0].GetText()
					Expect(err).NotTo(HaveOccurred())
				}

				errMsg := errText
				if expectedText != "" && actualText != "" {
					errMsg = fmt.Sprintf(
						"%s\n\tExpected: %q\n\tActual: %q",
						errText,
						strings.TrimSuffix(expectedText, " "),
						strings.TrimSuffix(actualText, " "),
					)
				}
				errMsgs = append(errMsgs, errMsg)
			}

			// Prints each error
			Fail(strings.Join(errMsgs, "\n-----------------------------------\n"))
		})
	})
})
<commit_msg>Capitalise By-closure descriptions<commit_after>package webdriver_test

import (
	"fmt"
	"strings"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/sclevine/agouti"
	"github.com/sclevine/agouti/api"
	. "github.com/sclevine/agouti/matchers"
)

var _ = Describe("gRPC-Web Unit Tests", func() {
	var page *agouti.Page

	BeforeEach(func() {
		var err error
		page, err = agoutiDriver.NewPage()
		Expect(err).NotTo(HaveOccurred())
	})

	AfterEach(func() {
		Expect(page.Destroy()).NotTo(HaveOccurred())
	})

	It("should pass", func() {
		By("Loading the test page", func() {
			Expect(page.Navigate("https://localhost:10000")).NotTo(HaveOccurred())
			Expect(page).To(HaveURL("https://localhost:10000/"))
		})

		By("Finding the number of failures", func() {
			Eventually(page.FirstByClass("failed")).Should(BeFound())
			Eventually(page.FindByID("qunit-testresult").FindByClass("failed")).Should(BeFound())
			failures, err := page.FindByID("qunit-testresult").FindByClass("failed").Text()
			Expect(err).NotTo(HaveOccurred())
			if failures == "0" {
				return
			}

			// We have at least one failure - lets compile an error message
			Eventually(page.FindByID(
				"qunit-tests",
			).AllByClass(
				"fail",
			).AllByClass(
				"fail",
			)).Should(BeFound())
			messages := page.FindByID(
				"qunit-tests",
			).AllByClass(
				"fail",
			).AllByClass(
				"fail",
			)
			elements, err := messages.Elements()
			Expect(err).NotTo(HaveOccurred())
			var errMsgs []string
			for _, element := range elements {
				// Get error summary
				msg, err := element.GetElement(api.Selector{
					Using: "css selector",
					Value: ".test-message",
				})
				Expect(err).NotTo(HaveOccurred())
				errText, err := msg.GetText()
				Expect(err).NotTo(HaveOccurred())

				// Get diff
				expected, err := element.GetElements(api.Selector{
					Using: "css selector",
					Value: "del",
				})
				Expect(err).NotTo(HaveOccurred())
				var expectedText string
				if len(expected) > 0 {
					expectedText, err = expected[0].GetText()
					Expect(err).NotTo(HaveOccurred())
				}
				actual, err := element.GetElements(api.Selector{
					Using: "css selector",
					Value: "ins",
				})
				Expect(err).NotTo(HaveOccurred())
				var actualText string
				if len(actual) > 0 {
					actualText, err = actual[0].GetText()
					Expect(err).NotTo(HaveOccurred())
				}

				errMsg := errText
				if expectedText != "" && actualText != "" {
					errMsg = fmt.Sprintf(
						"%s\n\tExpected: %q\n\tActual: %q",
						errText,
						strings.TrimSuffix(expectedText, " "),
						strings.TrimSuffix(actualText, " "),
					)
				}
				errMsgs = append(errMsgs, errMsg)
			}

			// Prints each error
			Fail(strings.Join(errMsgs, "\n-----------------------------------\n"))
		})
	})
})
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package version

var (
	// Major is the current major version of master branch.
	Major = 0
	// Minor is the current minor version of master branch.
	Minor = 2
	// Patch is the curernt patched version of the master branch.
	Patch = 2
	// Release is the current release level of the master branch. Valid values
	// are dev (developement unreleased), rcX (release candidate with current
	// iteration), stable (indicates a final released version).
	Release = "dev"
)
<commit_msg>Bump the version to 0.3.0-dev since there are too many functionality changes<commit_after>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package version

var (
	// Major is the current major version of master branch.
	Major = 0
	// Minor is the current minor version of master branch.
	Minor = 3
	// Patch is the curernt patched version of the master branch.
	Patch = 0
	// Release is the current release level of the master branch. Valid values
	// are dev (developement unreleased), rcX (release candidate with current
	// iteration), stable (indicates a final released version).
	Release = "dev"
)
<|endoftext|>


<commit_before>package main

// Open url in browser:
// http://localhost:14000/app

import (
	"fmt"
	"github.com/RangelReale/osin"
	"github.com/RangelReale/osin/example"
	"net/http"
	"net/url"
)

func main() {
	cfg := osin.NewServerConfig()
	cfg.AllowGetAccessRequest = true
	cfg.AllowClientSecretInParams = true

	server := osin.NewServer(cfg, example.NewTestStorage())

	// Authorization code endpoint
	http.HandleFunc("/authorize", func(w http.ResponseWriter, r *http.Request) {
		resp := server.NewResponse()
		defer resp.Close()

		if ar := server.HandleAuthorizeRequest(resp, r); ar != nil {
			if !example.HandleLoginPage(ar, w, r) {
				return
			}
			ar.Authorized = true
			server.FinishAuthorizeRequest(resp, r, ar)
		}
		if resp.IsError && resp.InternalError != nil {
			fmt.Printf("ERROR: %s\n", resp.InternalError)
		}
		osin.OutputJSON(resp, w, r)
	})

	// Access token endpoint
	http.HandleFunc("/token", func(w http.ResponseWriter, r *http.Request) {
		resp := server.NewResponse()
		defer resp.Close()

		if ar := server.HandleAccessRequest(resp, r); ar != nil {
			ar.Authorized = true
			server.FinishAccessRequest(resp, r, ar)
		}
		if resp.IsError && resp.InternalError != nil {
			fmt.Printf("ERROR: %s\n", resp.InternalError)
		}
		osin.OutputJSON(resp, w, r)
	})

	// Information endpoint
	http.HandleFunc("/info", func(w http.ResponseWriter, r *http.Request) {
		resp := server.NewResponse()
		defer resp.Close()

		if ir := server.HandleInfoRequest(resp, r); ir != nil {
			server.FinishInfoRequest(resp, r, ir)
		}
		osin.OutputJSON(resp, w, r)
	})

	// Application home endpoint
	http.HandleFunc("/app", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(""))
		w.Write([]byte(fmt.Sprintf("Login
", url.QueryEscape("http://localhost:14000/appauth/code"))))
		w.Write([]byte(""))
	})

	// Application destination - CODE
	http.HandleFunc("/appauth/code", func(w http.ResponseWriter, r *http.Request) {
		r.ParseForm()

		code := r.Form.Get("code")

		w.Write([]byte(""))
		w.Write([]byte("APP AUTH - CODE
"))
		defer w.Write([]byte(""))

		if code == "" {
			w.Write([]byte("Nothing to do"))
			return
		}

		jr := make(map[string]interface{})

		// build access code url
		aurl := fmt.Sprintf("/token?grant_type=authorization_code&client_id=1234&client_secret=aabbccdd&state=xyz&redirect_uri=%s&code=%s",
			url.QueryEscape("http://localhost:14000/appauth/code"), url.QueryEscape(code))

		// if parse, download and parse json
		if r.Form.Get("doparse") == "1" {
			err := example.DownloadAccessToken(fmt.Sprintf("http://localhost:14000%s", aurl),
				&osin.BasicAuth{"1234", "aabbccdd"}, jr)
			if err != nil {
				w.Write([]byte(err.Error()))
				w.Write([]byte("
"))
			}
		}

		// show json error
		if erd, ok := jr["error"]; ok {
			w.Write([]byte(fmt.Sprintf("ERROR: %s
\n", erd)))
		}

		// show json access token
		if at, ok := jr["access_token"]; ok {
			w.Write([]byte(fmt.Sprintf("ACCESS TOKEN: %s
\n", at)))
		}

		w.Write([]byte(fmt.Sprintf("FULL RESULT: %+v
\n", jr)))

		// output links
		w.Write([]byte(fmt.Sprintf("Goto Token URL
", aurl)))

		cururl := *r.URL
		curq := cururl.Query()
		curq.Add("doparse", "1")
		cururl.RawQuery = curq.Encode()
		w.Write([]byte(fmt.Sprintf("Download Token
", cururl.String())))
	})

	http.ListenAndServe(":14000", nil)
}
<commit_msg>Update simple.go<commit_after>package main

// Open url in browser:
// http://localhost:14000/app

import (
	"fmt"
	"github.com/RangelReale/osin"
	"github.com/RangelReale/osin/example"
	"net/http"
	"net/url"
	"github.com/ory-am/common/env"
)

func main() {
	cfg := osin.NewServerConfig()
	cfg.AllowGetAccessRequest = true
	cfg.AllowClientSecretInParams = true

	server := osin.NewServer(cfg, example.NewTestStorage())

	// Authorization code endpoint
	http.HandleFunc("/authorize", func(w http.ResponseWriter, r *http.Request) {
		resp := server.NewResponse()
		defer resp.Close()

		if ar := server.HandleAuthorizeRequest(resp, r); ar != nil {
			if !example.HandleLoginPage(ar, w, r) {
				return
			}
			ar.Authorized = true
			server.FinishAuthorizeRequest(resp, r, ar)
		}
		if resp.IsError && resp.InternalError != nil {
			fmt.Printf("ERROR: %s\n", resp.InternalError)
		}
		osin.OutputJSON(resp, w, r)
	})

	// Access token endpoint
	http.HandleFunc("/token", func(w http.ResponseWriter, r *http.Request) {
		resp := server.NewResponse()
		defer resp.Close()

		if ar := server.HandleAccessRequest(resp, r); ar != nil {
			ar.Authorized = true
			server.FinishAccessRequest(resp, r, ar)
		}
		if resp.IsError && resp.InternalError != nil {
			fmt.Printf("ERROR: %s\n", resp.InternalError)
		}
		osin.OutputJSON(resp, w, r)
	})

	// Information endpoint
	http.HandleFunc("/info", func(w http.ResponseWriter, r *http.Request) {
		resp := server.NewResponse()
		defer resp.Close()

		if ir := server.HandleInfoRequest(resp, r); ir != nil {
			server.FinishInfoRequest(resp, r, ir)
		}
		osin.OutputJSON(resp, w, r)
	})

	// Application home endpoint
	http.HandleFunc("/app", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(""))
		w.Write([]byte(fmt.Sprintf("Login
", url.QueryEscape("http://localhost:14000/appauth/code"))))
		w.Write([]byte(""))
	})

	// Application destination - CODE
	http.HandleFunc("/appauth/code", func(w http.ResponseWriter, r *http.Request) {
		r.ParseForm()

		code := r.Form.Get("code")

		w.Write([]byte(""))
		w.Write([]byte("APP AUTH - CODE
"))
		defer w.Write([]byte(""))

		if code == "" {
			w.Write([]byte("Nothing to do"))
			return
		}

		jr := make(map[string]interface{})

		// build access code url
		aurl := fmt.Sprintf("/token?grant_type=authorization_code&client_id=1234&client_secret=aabbccdd&state=xyz&redirect_uri=%s&code=%s",
			url.QueryEscape("http://localhost:14000/appauth/code"), url.QueryEscape(code))

		// if parse, download and parse json
		if r.Form.Get("doparse") == "1" {
			err := example.DownloadAccessToken(fmt.Sprintf("http://localhost:14000%s", aurl),
				&osin.BasicAuth{"1234", "aabbccdd"}, jr)
			if err != nil {
				w.Write([]byte(err.Error()))
				w.Write([]byte("
"))
			}
		}

		// show json error
		if erd, ok := jr["error"]; ok {
			w.Write([]byte(fmt.Sprintf("ERROR: %s
\n", erd)))
		}

		// show json access token
		if at, ok := jr["access_token"]; ok {
			w.Write([]byte(fmt.Sprintf("ACCESS TOKEN: %s
\n", at)))
		}

		w.Write([]byte(fmt.Sprintf("FULL RESULT: %+v
\n", jr)))

		// output links
		w.Write([]byte(fmt.Sprintf("Goto Token URL
", aurl)))

		cururl := *r.URL
		curq := cururl.Query()
		curq.Add("doparse", "1")
		cururl.RawQuery = curq.Encode()
		w.Write([]byte(fmt.Sprintf("Download Token
", cururl.String())))
	})

	port := env.Getenv("PORT", "14000")
	http.ListenAndServe(":" + port, nil)
}
<|endoftext|>


<commit_before>package duktape_test

import (
	"errors"
	"fmt"
	"reflect"
	"unsafe"

	"gopkg.in/olebedev/go-duktape.v3"
)

func ExampleContext_LoadFunction() {
// Example of serialization duktape function to a byte slice; deserialization from byte slice to duktape
// function and eval in another duktape context.
//
// Usecase: get byte slice from duktape function (to possibly later save it to the file, send by network and etc.)
// Please, keep in mind restrictions for a deserialized duktape function from bytecode:
// https://github.com/svaarala/duktape/blob/master/doc/bytecode.rst
//
//Excerpts:
//	When to use bytecode dump/load
//		There are two main motivations for using bytecode dump/load:
//			-	Performance
//			-	Obfuscation

	// Parenthesis is necessary.
	js := "(function dump_from() { return 'It\\'s alive!'; })"

	ctxSerialize := duktape.New()

	// Compile js to duktape function and put it on the context stack
	ctxSerialize.EvalLstring(js, len(js))

	// Transmute duktape function on the top of the context stack into serializable duktape bytecode
	ctxSerialize.DumpFunction()

	// void *duk_get_buffer(duk_context *ctx, duk_idx_t idx, duk_size_t *out_size);
	// typedef size_t duk_size_t;
	// duk_size_t equivalent in Go is uint, however slice len is of a type int
	// and I don't think that the limit of max(int) can be reached so for the sake of simplicity
	// lets assume that bytecode size is always can be contained in int
	var sz int
	rawmem := ctxSerialize.GetBuffer(-1, &sz)
	// Check for null is necessary because duk_get_buffer can return NULL.
	if uintptr(rawmem) == uintptr(0) {
		return nil, errors.New("Can't interpret bytecode dump as a valid, non-empty buffer.")
	}
	rawmemslice := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(rawmem), Len: sz, Cap: sz}))

	// Creating another slice for bytecode is necessary because rawmemslice pointing at memory that belongs to
	// current context. That memory will be freed during execution of DestroyHeap().
	bytecode := make([]byte, sz)
	copy(bytecode, rawmemslice)

	// To prevent memory leaks, don't forget to clean up after
	// yourself when you're done using a context.
	ctxSerialize.DestroyHeap()

	ctxDeserialize := duktape.New()

	//creating buffer on the context stack
	rawmem := ctxDeserialize.PushBuffer(len(bc), false)
	if uintptr(rawmem) == uintptr(0) {
		return "", errors.New("Can't push buffer to the context stack.")
	}

	//copying bytecode into the created buffer
	rawmemslice := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(rawmem), Len: len(bc), Cap: len(bc)}))
	copy(rawmemslice, bc)

	// Transmute duktape bytecode into duktape function
	ctxDeserialize.LoadFunction()

	// Call the function on top of a stack, example function doesn't have arguments.
	ctxDeserialize.Call(0)

	//example function return value is string
	retval := ctxDeserialize.GetString(-1)

	// To prevent memory leaks, don't forget to clean up after
	// yourself when you're done using a context.
	ctxDeserialize.DestroyHeap()

	fmt.Println(retval)

    // Output:
    // It's alive!
}<commit_msg>make example applicable as a test<commit_after>package duktape_test

import (
	"fmt"
	"reflect"
	"unsafe"
	"log"

	"gopkg.in/olebedev/go-duktape.v3"
)

func ExampleContext_LoadFunction() {
	// Example of serialization duktape function to a byte slice; deserialization from byte slice to duktape
	// function and eval in another duktape context.
	//
	// Usecase: get byte slice from duktape function (to possibly later save it to the file, send by network and etc.)
	// Please, keep in mind restrictions for a deserialized duktape function from bytecode:
	// https://github.com/svaarala/duktape/blob/master/doc/bytecode.rst
	//
	//Excerpts:
	//	When to use bytecode dump/load
	//		There are two main motivations for using bytecode dump/load:
	//			-	Performance
	//			-	Obfuscation

	// Parenthesis is necessary.
	js := "(function dump_from() { return 'It\\'s alive!'; })"

	ctxSerialize := duktape.New()

	// Compile js to duktape function and put it on the context stack
	ctxSerialize.EvalLstring(js, len(js))

	// Transmute duktape function on the top of the context stack into serializable duktape bytecode
	ctxSerialize.DumpFunction()

	// void *duk_get_buffer(duk_context *ctx, duk_idx_t idx, duk_size_t *out_size);
	// typedef size_t duk_size_t;
	// duk_size_t equivalent in Go is uint, however slice len is of a type int
	// and I don't think that the limit of max(int) can be reached so for the sake of simplicity
	// lets assume that bytecode size is always can be contained in int
	var sz int
	rawmem := ctxSerialize.GetBuffer(-1, &sz)
	// Check for null is necessary because duk_get_buffer can return NULL.
	if uintptr(rawmem) == uintptr(0) {
		log.Fatalf("Can't interpret bytecode dump as a valid, non-empty buffer.")
	}
	rawmemslice := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(rawmem), Len: sz, Cap: sz}))

	// Creating another slice for bytecode is necessary because rawmemslice pointing at memory that belongs to
	// current context. That memory will be freed during execution of DestroyHeap().
	bytecode := make([]byte, sz)
	copy(bytecode, rawmemslice)

	// To prevent memory leaks, don't forget to clean up after
	// yourself when you're done using a context.
	ctxSerialize.DestroyHeap()

	ctxDeserialize := duktape.New()

	//creating buffer on the context stack
	rawmem = ctxDeserialize.PushBuffer(len(bytecode), false)
	if uintptr(rawmem) == uintptr(0) {
		log.Fatalf("Can't push buffer to the context stack.")
	}

	//copying bytecode into the created buffer
	rawmemslice = *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(rawmem), Len: len(bytecode), Cap: len(bytecode)}))
	copy(rawmemslice, bytecode)

	// Transmute duktape bytecode into duktape function
	ctxDeserialize.LoadFunction()

	// Call the function on top of a stack, example function doesn't have arguments.
	ctxDeserialize.Call(0)

	//example function return value is string
	retval := ctxDeserialize.GetString(-1)

	// To prevent memory leaks, don't forget to clean up after
	// yourself when you're done using a context.
	ctxDeserialize.DestroyHeap()

	fmt.Println(retval)

	// Output:
	// It's alive!
}<|endoftext|>