<commit_before>// +build linux

package btrfs

/*
#include <stdlib.h>
#include <dirent.h>
#include <btrfs/ioctl.h>
#include <btrfs/ctree.h>
*/
import "C"

import (
	"fmt"
	"os"
	"path"
	"path/filepath"
	"syscall"
	"unsafe"

	"github.com/docker/docker/daemon/graphdriver"
	"github.com/docker/docker/pkg/idtools"
	"github.com/docker/docker/pkg/mount"
	"github.com/opencontainers/runc/libcontainer/label"
)

func init() {
	graphdriver.Register("btrfs", Init)
}

// Init returns a new BTRFS driver.
// An error is returned if BTRFS is not supported.
func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) {

	fsMagic, err := graphdriver.GetFSMagic(home)
	if err != nil {
		return nil, err
	}

	if fsMagic != graphdriver.FsMagicBtrfs {
		return nil, graphdriver.ErrPrerequisites
	}

	rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps)
	if err != nil {
		return nil, err
	}
	if err := idtools.MkdirAllAs(home, 0700, rootUID, rootGID); err != nil {
		return nil, err
	}

	if err := mount.MakePrivate(home); err != nil {
		return nil, err
	}

	driver := &Driver{
		home:    home,
		uidMaps: uidMaps,
		gidMaps: gidMaps,
	}

	return graphdriver.NewNaiveDiffDriver(driver, uidMaps, gidMaps), nil
}

// Driver contains information about the filesystem mounted.
type Driver struct {
	//root of the file system
	home    string
	uidMaps []idtools.IDMap
	gidMaps []idtools.IDMap
}

// String prints the name of the driver (btrfs).
func (d *Driver) String() string {
	return "btrfs"
}

// Status returns current driver information in a two dimensional string array.
// Output contains "Build Version" and "Library Version" of the btrfs libraries used.
// Version information can be used to check compatibility with your kernel.
func (d *Driver) Status() [][2]string {
	status := [][2]string{}
	if bv := btrfsBuildVersion(); bv != "-" {
		status = append(status, [2]string{"Build Version", bv})
	}
	if lv := btrfsLibVersion(); lv != -1 {
		status = append(status, [2]string{"Library Version", fmt.Sprintf("%d", lv)})
	}
	return status
}

// GetMetadata returns empty metadata for this driver.
func (d *Driver) GetMetadata(id string) (map[string]string, error) {
	return nil, nil
}

// Cleanup unmounts the home directory.
func (d *Driver) Cleanup() error {
	return mount.Unmount(d.home)
}

func free(p *C.char) {
	C.free(unsafe.Pointer(p))
}

func openDir(path string) (*C.DIR, error) {
	Cpath := C.CString(path)
	defer free(Cpath)

	dir := C.opendir(Cpath)
	if dir == nil {
		return nil, fmt.Errorf("Can't open dir")
	}
	return dir, nil
}

func closeDir(dir *C.DIR) {
	if dir != nil {
		C.closedir(dir)
	}
}

func getDirFd(dir *C.DIR) uintptr {
	return uintptr(C.dirfd(dir))
}

func subvolCreate(path, name string) error {
	dir, err := openDir(path)
	if err != nil {
		return err
	}
	defer closeDir(dir)

	var args C.struct_btrfs_ioctl_vol_args
	for i, c := range []byte(name) {
		args.name[i] = C.char(c)
	}

	_, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_SUBVOL_CREATE,
		uintptr(unsafe.Pointer(&args)))
	if errno != 0 {
		return fmt.Errorf("Failed to create btrfs subvolume: %v", errno.Error())
	}
	return nil
}

func subvolSnapshot(src, dest, name string) error {
	srcDir, err := openDir(src)
	if err != nil {
		return err
	}
	defer closeDir(srcDir)

	destDir, err := openDir(dest)
	if err != nil {
		return err
	}
	defer closeDir(destDir)

	var args C.struct_btrfs_ioctl_vol_args_v2
	args.fd = C.__s64(getDirFd(srcDir))
	for i, c := range []byte(name) {
		args.name[i] = C.char(c)
	}

	_, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(destDir), C.BTRFS_IOC_SNAP_CREATE_V2,
		uintptr(unsafe.Pointer(&args)))
	if errno != 0 {
		return fmt.Errorf("Failed to create btrfs snapshot: %v", errno.Error())
	}
	return nil
}

func isSubvolume(p string) (bool, error) {
	var bufStat syscall.Stat_t
	if err := syscall.Lstat(p, &bufStat); err != nil {
		return false, err
	}

	// return true if it is a btrfs subvolume
	return bufStat.Ino == C.BTRFS_FIRST_FREE_OBJECTID, nil
}

func subvolDelete(dirpath, name string) error {
	dir, err := openDir(dirpath)
	if err != nil {
		return err
	}
	defer closeDir(dir)
	fullPath := path.Join(dirpath, name)

	var args C.struct_btrfs_ioctl_vol_args

	// walk the btrfs subvolumes
	walkSubvolumes := func(p string, f os.FileInfo, err error) error {
		if err != nil {
			if os.IsNotExist(err) && p != fullPath {
				// missing most likely because the path was a subvolume that got removed in the previous iteration
				// since it's gone anyway, we don't care
				return nil
			}
			return fmt.Errorf("error walking subvolumes: %v", err)
		}
		// we want to check children only so skip itself
		// it will be removed after the filepath walk anyways
		if f.IsDir() && p != fullPath {
			sv, err := isSubvolume(p)
			if err != nil {
				return fmt.Errorf("Failed to test if %s is a btrfs subvolume: %v", p, err)
			}
			if sv {
				if err := subvolDelete(path.Dir(p), f.Name()); err != nil {
					return fmt.Errorf("Failed to destroy btrfs child subvolume (%s) of parent (%s): %v", p, dirpath, err)
				}
			}
		}
		return nil
	}
	if err := filepath.Walk(path.Join(dirpath, name), walkSubvolumes); err != nil {
		return fmt.Errorf("Recursively walking subvolumes for %s failed: %v", dirpath, err)
	}

	// all subvolumes have been removed
	// now remove the one originally passed in
	for i, c := range []byte(name) {
		args.name[i] = C.char(c)
	}
	_, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_SNAP_DESTROY,
		uintptr(unsafe.Pointer(&args)))
	if errno != 0 {
		return fmt.Errorf("Failed to destroy btrfs snapshot %s for %s: %v", dirpath, name, errno.Error())
	}
	return nil
}

func (d *Driver) subvolumesDir() string {
	return path.Join(d.home, "subvolumes")
}

func (d *Driver) subvolumesDirID(id string) string {
	return path.Join(d.subvolumesDir(), id)
}

// Create the filesystem with given id.
func (d *Driver) Create(id, parent, mountLabel string) error {
	subvolumes := path.Join(d.home, "subvolumes")
	rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
	if err != nil {
		return err
	}
	if err := idtools.MkdirAllAs(subvolumes, 0700, rootUID, rootGID); err != nil {
		return err
	}
	if parent == "" {
		if err := subvolCreate(subvolumes, id); err != nil {
			return err
		}
	} else {
		parentDir, err := d.Get(parent, "")
		if err != nil {
			return err
		}
		if err := subvolSnapshot(parentDir, subvolumes, id); err != nil {
			return err
		}
	}

	// if we have a remapped root (user namespaces enabled), change the created snapshot
	// dir ownership to match
	if rootUID != 0 || rootGID != 0 {
		if err := os.Chown(path.Join(subvolumes, id), rootUID, rootGID); err != nil {
			return err
		}
	}

	return label.Relabel(path.Join(subvolumes, id), mountLabel, false)
}

// Remove the filesystem with given id.
func (d *Driver) Remove(id string) error {
	dir := d.subvolumesDirID(id)
	if _, err := os.Stat(dir); err != nil {
		return err
	}
	if err := subvolDelete(d.subvolumesDir(), id); err != nil {
		return err
	}
	if err := os.RemoveAll(dir); err != nil && !os.IsNotExist(err) {
		return err
	}
	return nil
}

// Get the requested filesystem id.
func (d *Driver) Get(id, mountLabel string) (string, error) {
	dir := d.subvolumesDirID(id)
	st, err := os.Stat(dir)
	if err != nil {
		return "", err
	}

	if !st.IsDir() {
		return "", fmt.Errorf("%s: not a directory", dir)
	}

	return dir, nil
}

// Put is not implemented for BTRFS as there is no cleanup required for the id.
func (d *Driver) Put(id string) error {
	// Get() creates no runtime resources (like e.g. mounts)
	// so this doesn't need to do anything.
	return nil
}

// Exists checks if the id exists in the filesystem.
func (d *Driver) Exists(id string) bool {
	dir := d.subvolumesDirID(id)
	_, err := os.Stat(dir)
	return err == nil
}
<commit_msg>Graphdriver/btrfs: Avoid using single d.Get()<commit_after>// +build linux

package btrfs

/*
#include <stdlib.h>
#include <dirent.h>
#include <btrfs/ioctl.h>
#include <btrfs/ctree.h>
*/
import "C"

import (
	"fmt"
	"os"
	"path"
	"path/filepath"
	"syscall"
	"unsafe"

	"github.com/docker/docker/daemon/graphdriver"
	"github.com/docker/docker/pkg/idtools"
	"github.com/docker/docker/pkg/mount"
	"github.com/opencontainers/runc/libcontainer/label"
)

func init() {
	graphdriver.Register("btrfs", Init)
}

// Init returns a new BTRFS driver.
// An error is returned if BTRFS is not supported.
func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) {

	fsMagic, err := graphdriver.GetFSMagic(home)
	if err != nil {
		return nil, err
	}

	if fsMagic != graphdriver.FsMagicBtrfs {
		return nil, graphdriver.ErrPrerequisites
	}

	rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps)
	if err != nil {
		return nil, err
	}
	if err := idtools.MkdirAllAs(home, 0700, rootUID, rootGID); err != nil {
		return nil, err
	}

	if err := mount.MakePrivate(home); err != nil {
		return nil, err
	}

	driver := &Driver{
		home:    home,
		uidMaps: uidMaps,
		gidMaps: gidMaps,
	}

	return graphdriver.NewNaiveDiffDriver(driver, uidMaps, gidMaps), nil
}

// Driver contains information about the filesystem mounted.
type Driver struct {
	//root of the file system
	home    string
	uidMaps []idtools.IDMap
	gidMaps []idtools.IDMap
}

// String prints the name of the driver (btrfs).
func (d *Driver) String() string {
	return "btrfs"
}

// Status returns current driver information in a two dimensional string array.
// Output contains "Build Version" and "Library Version" of the btrfs libraries used.
// Version information can be used to check compatibility with your kernel.
func (d *Driver) Status() [][2]string {
	status := [][2]string{}
	if bv := btrfsBuildVersion(); bv != "-" {
		status = append(status, [2]string{"Build Version", bv})
	}
	if lv := btrfsLibVersion(); lv != -1 {
		status = append(status, [2]string{"Library Version", fmt.Sprintf("%d", lv)})
	}
	return status
}

// GetMetadata returns empty metadata for this driver.
func (d *Driver) GetMetadata(id string) (map[string]string, error) {
	return nil, nil
}

// Cleanup unmounts the home directory.
func (d *Driver) Cleanup() error {
	return mount.Unmount(d.home)
}

func free(p *C.char) {
	C.free(unsafe.Pointer(p))
}

func openDir(path string) (*C.DIR, error) {
	Cpath := C.CString(path)
	defer free(Cpath)

	dir := C.opendir(Cpath)
	if dir == nil {
		return nil, fmt.Errorf("Can't open dir")
	}
	return dir, nil
}

func closeDir(dir *C.DIR) {
	if dir != nil {
		C.closedir(dir)
	}
}

func getDirFd(dir *C.DIR) uintptr {
	return uintptr(C.dirfd(dir))
}

func subvolCreate(path, name string) error {
	dir, err := openDir(path)
	if err != nil {
		return err
	}
	defer closeDir(dir)

	var args C.struct_btrfs_ioctl_vol_args
	for i, c := range []byte(name) {
		args.name[i] = C.char(c)
	}

	_, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_SUBVOL_CREATE,
		uintptr(unsafe.Pointer(&args)))
	if errno != 0 {
		return fmt.Errorf("Failed to create btrfs subvolume: %v", errno.Error())
	}
	return nil
}

func subvolSnapshot(src, dest, name string) error {
	srcDir, err := openDir(src)
	if err != nil {
		return err
	}
	defer closeDir(srcDir)

	destDir, err := openDir(dest)
	if err != nil {
		return err
	}
	defer closeDir(destDir)

	var args C.struct_btrfs_ioctl_vol_args_v2
	args.fd = C.__s64(getDirFd(srcDir))
	for i, c := range []byte(name) {
		args.name[i] = C.char(c)
	}

	_, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(destDir), C.BTRFS_IOC_SNAP_CREATE_V2,
		uintptr(unsafe.Pointer(&args)))
	if errno != 0 {
		return fmt.Errorf("Failed to create btrfs snapshot: %v", errno.Error())
	}
	return nil
}

func isSubvolume(p string) (bool, error) {
	var bufStat syscall.Stat_t
	if err := syscall.Lstat(p, &bufStat); err != nil {
		return false, err
	}

	// return true if it is a btrfs subvolume
	return bufStat.Ino == C.BTRFS_FIRST_FREE_OBJECTID, nil
}

func subvolDelete(dirpath, name string) error {
	dir, err := openDir(dirpath)
	if err != nil {
		return err
	}
	defer closeDir(dir)
	fullPath := path.Join(dirpath, name)

	var args C.struct_btrfs_ioctl_vol_args

	// walk the btrfs subvolumes
	walkSubvolumes := func(p string, f os.FileInfo, err error) error {
		if err != nil {
			if os.IsNotExist(err) && p != fullPath {
				// missing most likely because the path was a subvolume that got removed in the previous iteration
				// since it's gone anyway, we don't care
				return nil
			}
			return fmt.Errorf("error walking subvolumes: %v", err)
		}
		// we want to check children only so skip itself
		// it will be removed after the filepath walk anyways
		if f.IsDir() && p != fullPath {
			sv, err := isSubvolume(p)
			if err != nil {
				return fmt.Errorf("Failed to test if %s is a btrfs subvolume: %v", p, err)
			}
			if sv {
				if err := subvolDelete(path.Dir(p), f.Name()); err != nil {
					return fmt.Errorf("Failed to destroy btrfs child subvolume (%s) of parent (%s): %v", p, dirpath, err)
				}
			}
		}
		return nil
	}
	if err := filepath.Walk(path.Join(dirpath, name), walkSubvolumes); err != nil {
		return fmt.Errorf("Recursively walking subvolumes for %s failed: %v", dirpath, err)
	}

	// all subvolumes have been removed
	// now remove the one originally passed in
	for i, c := range []byte(name) {
		args.name[i] = C.char(c)
	}
	_, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_SNAP_DESTROY,
		uintptr(unsafe.Pointer(&args)))
	if errno != 0 {
		return fmt.Errorf("Failed to destroy btrfs snapshot %s for %s: %v", dirpath, name, errno.Error())
	}
	return nil
}

func (d *Driver) subvolumesDir() string {
	return path.Join(d.home, "subvolumes")
}

func (d *Driver) subvolumesDirID(id string) string {
	return path.Join(d.subvolumesDir(), id)
}

// Create the filesystem with given id.
func (d *Driver) Create(id, parent, mountLabel string) error {
	subvolumes := path.Join(d.home, "subvolumes")
	rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
	if err != nil {
		return err
	}
	if err := idtools.MkdirAllAs(subvolumes, 0700, rootUID, rootGID); err != nil {
		return err
	}
	if parent == "" {
		if err := subvolCreate(subvolumes, id); err != nil {
			return err
		}
	} else {
		parentDir := d.subvolumesDirID(parent)
		st, err := os.Stat(parentDir)
		if err != nil {
			return err
		}
		if !st.IsDir() {
			return fmt.Errorf("%s: not a direcotory", parentDir)
		}
		if err := subvolSnapshot(parentDir, subvolumes, id); err != nil {
			return err
		}
	}

	// if we have a remapped root (user namespaces enabled), change the created snapshot
	// dir ownership to match
	if rootUID != 0 || rootGID != 0 {
		if err := os.Chown(path.Join(subvolumes, id), rootUID, rootGID); err != nil {
			return err
		}
	}

	return label.Relabel(path.Join(subvolumes, id), mountLabel, false)
}

// Remove the filesystem with given id.
func (d *Driver) Remove(id string) error {
	dir := d.subvolumesDirID(id)
	if _, err := os.Stat(dir); err != nil {
		return err
	}
	if err := subvolDelete(d.subvolumesDir(), id); err != nil {
		return err
	}
	if err := os.RemoveAll(dir); err != nil && !os.IsNotExist(err) {
		return err
	}
	return nil
}

// Get the requested filesystem id.
func (d *Driver) Get(id, mountLabel string) (string, error) {
	dir := d.subvolumesDirID(id)
	st, err := os.Stat(dir)
	if err != nil {
		return "", err
	}

	if !st.IsDir() {
		return "", fmt.Errorf("%s: not a directory", dir)
	}

	return dir, nil
}

// Put is not implemented for BTRFS as there is no cleanup required for the id.
func (d *Driver) Put(id string) error {
	// Get() creates no runtime resources (like e.g. mounts)
	// so this doesn't need to do anything.
	return nil
}

// Exists checks if the id exists in the filesystem.
func (d *Driver) Exists(id string) bool {
	dir := d.subvolumesDirID(id)
	_, err := os.Stat(dir)
	return err == nil
}
<|endoftext|>


<commit_before>package config

import (
	"fmt"
	"reflect"
	"strings"
)

type ServiceConfig map[string]interface{}

func (sc ServiceConfig) Assign(to interface{}) error {
	t := reflect.TypeOf(to)
	v := reflect.ValueOf(to)
	fc := t.NumField()
	for i := 0; i < fc; i++ {
		tf := t.Field(i)
		if tf.Anonymous {
			continue
		}
		name := tf.Name
		upName := strings.ToUpper(name)
		if name[0] != upName[0] {
			continue
		}
		tag, ok := tf.Tag.Lookup("servconf")
		if !ok {
			continue
		}
		vf := v.Field(i)
		sc.handleField(tag, &vf)

	}

	return nil
}

func (sc ServiceConfig) handleField(tag string, vf *reflect.Value) error {
	tagVals := strings.Split(tag, ",")
	tvc := len(tagVals)
	if tvc < 1 {
		return fmt.Errorf("TODO implement error type for invalid struct fields")
	}

	fName := strings.TrimSpace(tagVals[0])
	if fName == "" {
		return fmt.Errorf("TODO implement error type for empty name field")
	}

	value, ok := sc[fName]

	required := false

	if tvc >= 2 {
		required = tagVals[1] == "required"
	}

	if !ok {
		if required {
			return fmt.Errorf("TODO implement error type for missing required field")
		}
	} else {
		if err := assignValueToField(value, vf, required); err != nil {
			return err
		}

	}

	return nil
}

func assignValueToField(value interface{}, vf *reflect.Value, required bool) error {
	if !vf.CanSet() {
		if required {
			return fmt.Errorf("TODO implement error type for having a filed that can not be set required")
		}
		return nil
	}

	vsc, ok := value.(ServiceConfig)
	if ok {
		//TODO figure out if this mess works
		iface := vf.Interface()
		err := vsc.Assign(&iface)
		if err != nil {
			return err
		}
		value = iface
	}

	valType := reflect.TypeOf(value)
	valVal := reflect.ValueOf(value)
	t := vf.Type()

	if valType.AssignableTo(t) {
		vf.Set(valVal)
	} else if valType.ConvertibleTo(t) {
		converted := valVal.Convert(t)
		vf.Set(converted)
	}
	return nil
}
<commit_msg>Determine if a field is exported by checking reflect.Value.CanSet() on it<commit_after>package config

import (
	"fmt"
	"reflect"
	"strings"
)

type ServiceConfig map[string]interface{}

func (sc ServiceConfig) Assign(to interface{}) error {
	t := reflect.TypeOf(to)
	v := reflect.ValueOf(to)
	fc := t.NumField()
	for i := 0; i < fc; i++ {
		tf := t.Field(i)
		if tf.Anonymous {
			continue
		}

		if !v.CanSet() {
			continue
		}
		tag, ok := tf.Tag.Lookup("servconf")
		if !ok {
			continue
		}
		vf := v.Field(i)
		sc.handleField(tag, &vf)

	}

	return nil
}

func (sc ServiceConfig) handleField(tag string, vf *reflect.Value) error {
	tagVals := strings.Split(tag, ",")
	tvc := len(tagVals)
	if tvc < 1 {
		return fmt.Errorf("TODO implement error type for invalid struct fields")
	}

	fName := strings.TrimSpace(tagVals[0])
	if fName == "" {
		return fmt.Errorf("TODO implement error type for empty name field")
	}

	value, ok := sc[fName]

	required := false

	if tvc >= 2 {
		required = tagVals[1] == "required"
	}

	if !ok {
		if required {
			return fmt.Errorf("TODO implement error type for missing required field")
		}
	} else {
		if err := assignValueToField(value, vf, required); err != nil {
			return err
		}

	}

	return nil
}

func assignValueToField(value interface{}, vf *reflect.Value, required bool) error {
	vsc, ok := value.(ServiceConfig)
	if ok {
		//TODO figure out if this mess works
		iface := vf.Interface()
		err := vsc.Assign(&iface)
		if err != nil {
			return err
		}
		value = iface
	}

	valType := reflect.TypeOf(value)
	valVal := reflect.ValueOf(value)
	t := vf.Type()

	if valType.AssignableTo(t) {
		vf.Set(valVal)
	} else if valType.ConvertibleTo(t) {
		converted := valVal.Convert(t)
		vf.Set(converted)
	}
	return nil
}
<|endoftext|>


<commit_before>package tsdb

import (
	"github.com/syndtr/goleveldb/leveldb"
)

// Batch is a write batch.
type Batch struct {
	// DB handle
	db *DB
	// LevelDB batch
	lb *leveldb.Batch
}

// NewBatch creates a write batch on a DB instance.
func NewBatch(db *DB) *Batch {
	b := new(Batch)
	b.db = db
	b.lb = new(leveldb.Batch)
	return b
}

// Add Put operation to the write batch.
func (b *Batch) Put(name string, t uint64, v float64) {
	//
}
<commit_msg>Add batch put<commit_after>package tsdb

import (
	"github.com/syndtr/goleveldb/leveldb"
)

// Batch is a write batch.
type Batch struct {
	// DB handle
	db *DB
	// LevelDB batch
	b *leveldb.Batch
}

// NewBatch creates a write batch on a DB instance.
func NewBatch(db *DB) *Batch {
	b := new(Batch)
	b.db = db
	b.b = new(leveldb.Batch)
	return b
}

// Reset batch.
func (b *Batch) Reset() {
	b.b.Reset()
}

// Flush write batch to db.
func (b *Batch) Flush() {
	b.db.db.Write(b.b, nil)
}

// Put stamp with value and score to db.
func (b *Batch) PutStamp(name string, stamp uint64, value float64, score float64) {
	key := encodeStampKey(name, stamp)
	val := encodeStampValue(value, score)
	b.b.Put([]byte(key), []byte(val))
}

// Put name with trend and stamp.
func (b *Batch) PutName(name string, trend float64, stamp uint64) {
	key := encodeNameKey(name)
	val := encodeNameValue(trend, stamp)
	b.b.Put([]byte(key), []byte(val))
}
<|endoftext|>


<commit_before>package main

import (
  "bytes"
  "compress/zlib"
  "encoding/binary"
  "encoding/json"
  "errors"
  "fmt"
  "io"
  "log"
  "math/rand"
  "os"
  "time"
)

const (
  default_publisher_hostname string = "localhost.localdomain"
  keepalive_timeout          time.Duration = 900 * time.Second
  max_pending_payloads       int = 100
)

type PendingPayload struct {
  next       *PendingPayload
  nonce      string
  events     []*FileEvent
  num_events int
  ack_events int
  payload_start int
  payload    []byte
}

type Publisher struct {
  config           *NetworkConfig
  hostname         string
  transport        Transport
  can_send         <-chan int
  pending_ping     bool
  pending_payloads map[string]*PendingPayload
  first_payload    *PendingPayload
  last_payload     *PendingPayload
  num_payloads     int
}

func (p *Publisher) Init() error {
  var err error

  p.hostname, err = os.Hostname()
  if err != nil {
    log.Printf("Failed to determine the FQDN; using localhost.localdomain.\n")
    p.hostname = default_publisher_hostname
  }

  // Set up the selected transport (currently only TLS)
  if p.transport, err = CreateTransportTls(p.config); err != nil {
    return err
  }

  p.pending_payloads = make(map[string]*PendingPayload)

  return nil
}

func (p *Publisher) Publish(input <-chan []*FileEvent, registrar_chan chan<- []RegistrarEvent) {
  var input_toggle <-chan []*FileEvent
  var retry_payload *PendingPayload
  var err error

  // TODO(driskell): Make the idle timeout configurable like the network timeout is?
  timer := time.NewTimer(keepalive_timeout)

  // TODO: We should still obey network timeout if we've sent events and not yet received response
  //       as its the quickest way to detect a connection problem after idle

  for {
    p.transport.Connect()
    p.can_send = p.transport.CanSend()
    input_toggle = nil

  SelectLoop:
    for {
      // TODO: implement shutdown select
      select {
      case <-p.can_send:
        if retry_payload != nil {
          var buffer bytes.Buffer

          // Do we need to regenerate the payload? Remember to account for ACK we have but not yet sent to registrar due to out-of-order receive
          if retry_payload.payload == nil {
            if err = p.bufferJdatData(&buffer, retry_payload.events[retry_payload.ack_events:], retry_payload.nonce); err != nil {
              break SelectLoop
            }

            retry_payload.payload = buffer.Bytes()
            retry_payload.payload_start = retry_payload.ack_events
          }

          // Send the payload again
          if err = p.transport.Write("JDAT", retry_payload.payload); err != nil {
            break SelectLoop
          }

          retry_payload = retry_payload.next
          break
        }

        // No pending payloads, enable event wait
        input_toggle = input

        // Continue loop so we don't reset the ping timer - we've not performed any activity just yet
        continue
      case events := <-input_toggle:
        // Send JDAT
        if err = p.sendJdat(events); err != nil {
          break SelectLoop
        }

        // Wait for send signal again
        input_toggle = nil

        if p.num_payloads >= max_pending_payloads {
          // Too many pending payloads, disable send temporarily
          p.can_send = nil
        }
      case data := <-p.transport.Read():
        var signature, message []byte

        // Error? Or data?
        switch data.(type) {
          case error:
            err = data.(error)
            break SelectLoop
          default:
            signature = data.([][]byte)[0]
            message = data.([][]byte)[1]
        }

        switch {
        case bytes.Compare(signature, []byte("PONG")) == 0:
          if err = p.processPong(message); err != nil {
            break SelectLoop
          }
        case bytes.Compare(signature, []byte("ACKN")) == 0:
          if err = p.processAck(message, registrar_chan); err != nil {
            break SelectLoop
          }
        default:
          err = errors.New(fmt.Sprintf("Unknown message received: % X", signature))
          break SelectLoop
        }
      case <-timer.C:
        log.Printf("<-timer.C")
        // If we haven't received a PONG yet this is a timeout
        if p.pending_ping {
          err = errors.New("Server did not respond to PING")
          break SelectLoop
        }

        // If the send buffer is full, we should have been receiving ACK by now...
        if input_toggle == nil {
          err = errors.New("Server stopped responding")
          break SelectLoop
        }

        // Send a ping and expect a pong back (eventually)
        // If we receive an ACK first, that's fine we'll reset timer
        // But after those ACKs we should get a PONG
        if err = p.transport.Write("PING", nil); err != nil {
          break SelectLoop
        }

        // We may have just filled the send buffer
        input_toggle = nil
      } /* select */

      // Reset the timer
      timer.Reset(keepalive_timeout)
    } /* loop forever, break to reconnect */

    // Disconnect and retry payloads
    log.Printf("Transport error, will reconnect: %s\n", err)
    p.transport.Disconnect()
    retry_payload = p.first_payload

    time.Sleep(p.config.reconnect)
  } /* Publish: for loop, break to shutdown */
} // Publish

func (p *Publisher) generateNonce() string {
  // This could maybe be made a bit more efficient
  nonce := make([]byte, 16)
  for i := 0; i < 16; i++ {
    nonce[i] = byte(rand.Intn(255))
  }
  return string(nonce)
}

func (p *Publisher) sendJdat(events []*FileEvent) (err error) {
  var buffer bytes.Buffer

  // Calculate a nonce
  nonce := p.generateNonce()
  for {
    if _, found := p.pending_payloads[nonce]; !found {
      break
    }
    // Collision - generate again - should be extremely rare
    nonce = p.generateNonce()
  }

  // Generate the data first
  if err = p.bufferJdatData(&buffer, events, nonce); err != nil {
    return
  }

  // Save pending payload until we receive ack, and discard buffer
  payload := &PendingPayload{events: events, nonce: nonce, num_events: len(events), payload_start: 0, payload: buffer.Bytes()}
  p.pending_payloads[nonce] = payload
  if p.first_payload == nil {
    p.first_payload = payload
  } else {
    p.last_payload.next = payload
  }
  p.last_payload = payload
  p.num_payloads++

  return p.transport.Write("JDAT", payload.payload)
}

func (p *Publisher) bufferJdatData(output io.Writer, events []*FileEvent, nonce string) (err error) {
  // Begin with the nonce
  if _, err = output.Write([]byte(nonce)); err != nil {
    return
  }

  var compressor *zlib.Writer
  if compressor, err = zlib.NewWriterLevel(output, 3); err != nil {
    return
  }

  // Append all the events
  for _, event := range events {
    // Add host field
    event.Event["host"] = p.hostname
    if err = p.bufferJdatDataEvent(compressor, event); err != nil {
      return
    }
  }

  compressor.Close()

  return nil
}

func (p *Publisher) bufferJdatDataEvent(output io.Writer, event *FileEvent) (err error) {
  var value []byte
  value, err = json.Marshal(event.Event)
  if err != nil {
    log.Printf("JSON event encoding error: %s\n", err)

    if err = binary.Write(output, binary.BigEndian, 2); err != nil {
      return
    }
    if _, err = output.Write([]byte("{}")); err != nil {
      return
    }

    return
  }

  if err = binary.Write(output, binary.BigEndian, uint32(len(value))); err != nil {
    return
  }
  if _, err = output.Write(value); err != nil {
    return
  }

  return nil
}

func (p *Publisher) processPong(message []byte) error {
  if len(message) != 0 {
    return errors.New(fmt.Sprintf("PONG message overflow (%d)", len(message)))
  }

  // Were we pending a ping?
  if !p.pending_ping {
    return errors.New("Unexpected PONG received")
  }

  p.pending_ping = false
  return nil
}

func (p *Publisher) processAck(message []byte, registrar_chan chan<- []RegistrarEvent) (err error) {
  if len(message) != 20 {
    err = errors.New(fmt.Sprintf("ACKN message corruption (%d)", len(message)))
    return
  }

  // Read the nonce and sequence number acked
  nonce, sequence := string(message[:16]), binary.BigEndian.Uint32(message[16:20])

  // Grab the payload the ACK corresponds to by using nonce
  payload, found := p.pending_payloads[nonce]
  if !found {
    // Don't fail here in case we had temporary issues and resend a payload, only for us to receive duplicate ACKN
    return
  }

  // Full ACK?
  // TODO: Protocol error if sequence is too large?
  if int(sequence) >= payload.num_events - payload.payload_start {
    // No more events left for this payload, free the payload memory
    payload.ack_events = len(payload.events)
    payload.payload = nil
    delete(p.pending_payloads, nonce)
  } else {
    // Only process the ACK if something was actually processed
    if int(sequence) > payload.num_events - payload.ack_events {
      payload.ack_events = int(sequence) + payload.payload_start
      // If we need to resend, we'll need to regenerate payload, so free that memory early
      payload.payload = nil
    }
  }

  // We potentially receive out-of-order ACKs due to payloads distributed across servers
  // This is where we enforce ordering again to ensure registrar receives ACK in order
  if payload == p.first_payload {
    for payload.ack_events != 0 {
      if payload.ack_events == len(payload.events) {
        registrar_chan <- []RegistrarEvent{&EventsEvent{Events: payload.events}}
        payload = payload.next
        p.first_payload = payload
        p.num_payloads--

        // Resume sending if we stopped due to excessive pending payload count
        if p.can_send == nil {
          p.can_send = p.transport.CanSend()
        }
      } else {
        registrar_chan <- []RegistrarEvent{&EventsEvent{Events: payload.events[:payload.ack_events]}}
        payload.events = payload.events[payload.ack_events:]
        payload.num_events = len(payload.events)
        payload.ack_events = 0
        payload.payload_start = 0
      }

      if payload == nil {
        break
      }
    }
  }

  return
}
<commit_msg>Handle connection failure<commit_after>package main

import (
  "bytes"
  "compress/zlib"
  "encoding/binary"
  "encoding/json"
  "errors"
  "fmt"
  "io"
  "log"
  "math/rand"
  "os"
  "time"
)

const (
  default_publisher_hostname string = "localhost.localdomain"
  keepalive_timeout          time.Duration = 900 * time.Second
  max_pending_payloads       int = 100
)

type PendingPayload struct {
  next       *PendingPayload
  nonce      string
  events     []*FileEvent
  num_events int
  ack_events int
  payload_start int
  payload    []byte
}

type Publisher struct {
  config           *NetworkConfig
  hostname         string
  transport        Transport
  can_send         <-chan int
  pending_ping     bool
  pending_payloads map[string]*PendingPayload
  first_payload    *PendingPayload
  last_payload     *PendingPayload
  num_payloads     int
}

func (p *Publisher) Init() error {
  var err error

  p.hostname, err = os.Hostname()
  if err != nil {
    log.Printf("Failed to determine the FQDN; using localhost.localdomain.\n")
    p.hostname = default_publisher_hostname
  }

  // Set up the selected transport (currently only TLS)
  if p.transport, err = CreateTransportZmq(p.config); err != nil {
    return err
  }

  p.pending_payloads = make(map[string]*PendingPayload)

  return nil
}

func (p *Publisher) Publish(input <-chan []*FileEvent, registrar_chan chan<- []RegistrarEvent) {
  var input_toggle <-chan []*FileEvent
  var retry_payload *PendingPayload
  var err error

  // TODO(driskell): Make the idle timeout configurable like the network timeout is?
  timer := time.NewTimer(keepalive_timeout)

  // TODO: We should still obey network timeout if we've sent events and not yet received response
  //       as its the quickest way to detect a connection problem after idle

  for {
    if err = p.transport.Connect(); err != nil {
      log.Printf("Connect attempt failed: %s\n", err)
      // TODO: implement shutdown select
      time.Sleep(p.config.reconnect)
      continue
    }
    p.can_send = p.transport.CanSend()
    input_toggle = nil

  SelectLoop:
    for {
      // TODO: implement shutdown select
      select {
      case <-p.can_send:
        if retry_payload != nil {
          var buffer bytes.Buffer

          // Do we need to regenerate the payload? Remember to account for ACK we have but not yet sent to registrar due to out-of-order receive
          if retry_payload.payload == nil {
            if err = p.bufferJdatData(&buffer, retry_payload.events[retry_payload.ack_events:], retry_payload.nonce); err != nil {
              break SelectLoop
            }

            retry_payload.payload = buffer.Bytes()
            retry_payload.payload_start = retry_payload.ack_events
          }

          // Send the payload again
          if err = p.transport.Write("JDAT", retry_payload.payload); err != nil {
            break SelectLoop
          }

          retry_payload = retry_payload.next
          break
        }

        // No pending payloads, enable event wait
        input_toggle = input

        // Continue loop so we don't reset the ping timer - we've not performed any activity just yet
        continue
      case events := <-input_toggle:
        // Send JDAT
        if err = p.sendJdat(events); err != nil {
          break SelectLoop
        }

        // Wait for send signal again
        input_toggle = nil

        if p.num_payloads >= max_pending_payloads {
          // Too many pending payloads, disable send temporarily
          p.can_send = nil
        }
      case data := <-p.transport.Read():
        var signature, message []byte

        // Error? Or data?
        switch data.(type) {
          case error:
            err = data.(error)
            break SelectLoop
          default:
            signature = data.([][]byte)[0]
            message = data.([][]byte)[1]
        }

        switch {
        case bytes.Compare(signature, []byte("PONG")) == 0:
          if err = p.processPong(message); err != nil {
            break SelectLoop
          }
        case bytes.Compare(signature, []byte("ACKN")) == 0:
          if err = p.processAck(message, registrar_chan); err != nil {
            break SelectLoop
          }
        default:
          err = errors.New(fmt.Sprintf("Unknown message received: % X", signature))
          break SelectLoop
        }
      case <-timer.C:
        log.Printf("<-timer.C")
        // If we haven't received a PONG yet this is a timeout
        if p.pending_ping {
          err = errors.New("Server did not respond to PING")
          break SelectLoop
        }

        // If the send buffer is full, we should have been receiving ACK by now...
        if input_toggle == nil {
          err = errors.New("Server stopped responding")
          break SelectLoop
        }

        // Send a ping and expect a pong back (eventually)
        // If we receive an ACK first, that's fine we'll reset timer
        // But after those ACKs we should get a PONG
        if err = p.transport.Write("PING", nil); err != nil {
          break SelectLoop
        }

        // We may have just filled the send buffer
        input_toggle = nil
      } /* select */

      // Reset the timer
      timer.Reset(keepalive_timeout)
    } /* loop forever, break to reconnect */

    // Disconnect and retry payloads
    log.Printf("Transport error, will reconnect: %s\n", err)
    p.transport.Disconnect()
    retry_payload = p.first_payload

    time.Sleep(p.config.reconnect)
  } /* Publish: for loop, break to shutdown */
} // Publish

func (p *Publisher) generateNonce() string {
  // This could maybe be made a bit more efficient
  nonce := make([]byte, 16)
  for i := 0; i < 16; i++ {
    nonce[i] = byte(rand.Intn(255))
  }
  return string(nonce)
}

func (p *Publisher) sendJdat(events []*FileEvent) (err error) {
  var buffer bytes.Buffer

  // Calculate a nonce
  nonce := p.generateNonce()
  for {
    if _, found := p.pending_payloads[nonce]; !found {
      break
    }
    // Collision - generate again - should be extremely rare
    nonce = p.generateNonce()
  }

  // Generate the data first
  if err = p.bufferJdatData(&buffer, events, nonce); err != nil {
    return
  }

  // Save pending payload until we receive ack, and discard buffer
  payload := &PendingPayload{events: events, nonce: nonce, num_events: len(events), payload_start: 0, payload: buffer.Bytes()}
  p.pending_payloads[nonce] = payload
  if p.first_payload == nil {
    p.first_payload = payload
  } else {
    p.last_payload.next = payload
  }
  p.last_payload = payload
  p.num_payloads++

  return p.transport.Write("JDAT", payload.payload)
}

func (p *Publisher) bufferJdatData(output io.Writer, events []*FileEvent, nonce string) (err error) {
  // Begin with the nonce
  if _, err = output.Write([]byte(nonce)); err != nil {
    return
  }

  var compressor *zlib.Writer
  if compressor, err = zlib.NewWriterLevel(output, 3); err != nil {
    return
  }

  // Append all the events
  for _, event := range events {
    // Add host field
    event.Event["host"] = p.hostname
    if err = p.bufferJdatDataEvent(compressor, event); err != nil {
      return
    }
  }

  compressor.Close()

  return nil
}

func (p *Publisher) bufferJdatDataEvent(output io.Writer, event *FileEvent) (err error) {
  var value []byte
  value, err = json.Marshal(event.Event)
  if err != nil {
    log.Printf("JSON event encoding error: %s\n", err)

    if err = binary.Write(output, binary.BigEndian, 2); err != nil {
      return
    }
    if _, err = output.Write([]byte("{}")); err != nil {
      return
    }

    return
  }

  if err = binary.Write(output, binary.BigEndian, uint32(len(value))); err != nil {
    return
  }
  if _, err = output.Write(value); err != nil {
    return
  }

  return nil
}

func (p *Publisher) processPong(message []byte) error {
  if len(message) != 0 {
    return errors.New(fmt.Sprintf("PONG message overflow (%d)", len(message)))
  }

  // Were we pending a ping?
  if !p.pending_ping {
    return errors.New("Unexpected PONG received")
  }

  p.pending_ping = false
  return nil
}

func (p *Publisher) processAck(message []byte, registrar_chan chan<- []RegistrarEvent) (err error) {
  if len(message) != 20 {
    err = errors.New(fmt.Sprintf("ACKN message corruption (%d)", len(message)))
    return
  }

  // Read the nonce and sequence number acked
  nonce, sequence := string(message[:16]), binary.BigEndian.Uint32(message[16:20])

  // Grab the payload the ACK corresponds to by using nonce
  payload, found := p.pending_payloads[nonce]
  if !found {
    // Don't fail here in case we had temporary issues and resend a payload, only for us to receive duplicate ACKN
    return
  }

  // Full ACK?
  // TODO: Protocol error if sequence is too large?
  if int(sequence) >= payload.num_events - payload.payload_start {
    // No more events left for this payload, free the payload memory
    payload.ack_events = len(payload.events)
    payload.payload = nil
    delete(p.pending_payloads, nonce)
  } else {
    // Only process the ACK if something was actually processed
    if int(sequence) > payload.num_events - payload.ack_events {
      payload.ack_events = int(sequence) + payload.payload_start
      // If we need to resend, we'll need to regenerate payload, so free that memory early
      payload.payload = nil
    }
  }

  // We potentially receive out-of-order ACKs due to payloads distributed across servers
  // This is where we enforce ordering again to ensure registrar receives ACK in order
  if payload == p.first_payload {
    for payload.ack_events != 0 {
      if payload.ack_events == len(payload.events) {
        registrar_chan <- []RegistrarEvent{&EventsEvent{Events: payload.events}}
        payload = payload.next
        p.first_payload = payload
        p.num_payloads--

        // Resume sending if we stopped due to excessive pending payload count
        if p.can_send == nil {
          p.can_send = p.transport.CanSend()
        }
      } else {
        registrar_chan <- []RegistrarEvent{&EventsEvent{Events: payload.events[:payload.ack_events]}}
        payload.events = payload.events[payload.ack_events:]
        payload.num_events = len(payload.events)
        payload.ack_events = 0
        payload.payload_start = 0
      }

      if payload == nil {
        break
      }
    }
  }

  return
}
<|endoftext|>


<commit_before>package integration_test

import (
	"os/exec"
	"path/filepath"
	"time"

	"github.com/cloudfoundry/libbuildpack/cutlass"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("CF NodeJS Buildpack", func() {
	var app *cutlass.App
	AfterEach(func() {
		if app != nil {
			app.Destroy()
		}
		app = nil
	})

	Describe("nodeJS versions", func() {
		Context("when specifying a range for the nodeJS version in the package.json", func() {
			BeforeEach(func() {
				app = cutlass.New(Fixtures("node_version_range"))
			})

			It("resolves to a nodeJS version successfully", func() {
				PushAppAndConfirm(app)

				Eventually(app.Stdout.String).Should(MatchRegexp("Installing node 10\\.\\d+\\.\\d+"))
				Expect(app.GetBody("/")).To(ContainSubstring("Hello, World!"))
			})
		})

		Context("when specifying a version 6 for the nodeJS version in the package.json", func() {
			BeforeEach(func() {
				app = cutlass.New(Fixtures("node_version_6"))
			})

			It("resolves to a nodeJS version successfully", func() {
				PushAppAndConfirm(app)

				Eventually(app.Stdout.String).Should(MatchRegexp("Installing node 10\\.\\d+\\.\\d+"))
				Expect(app.GetBody("/")).To(ContainSubstring("Hello, World!"))

				if ApiHasTask() {
					By("running a task", func() {
						By("can find node in the container", func() {
							command := exec.Command("cf", "run-task", app.Name, "echo \"RUNNING A TASK: $(node --version)\"")
							_, err := command.Output()
							Expect(err).To(BeNil())

							Eventually(func() string {
								return app.Stdout.String()
							}, "30s").Should(MatchRegexp("RUNNING A TASK: v10\\.\\d+\\.\\d+"))
						})
					})
				}
			})
		})

		Context("when not specifying a nodeJS version in the package.json", func() {
			BeforeEach(func() {
				app = cutlass.New(Fixtures("without_node_version"))
			})

			It("resolves to the stable nodeJS version successfully", func() {
				PushAppAndConfirm(app)

				Eventually(app.Stdout.String).Should(MatchRegexp("Installing node 10\\.\\d+\\.\\d+"))
				Expect(app.GetBody("/")).To(ContainSubstring("Hello, World!"))
			})
		})

		Context("with an unreleased nodejs version", func() {
			BeforeEach(func() {
				app = cutlass.New(Fixtures("unreleased_node_version"))
			})

			It("displays a nice error message and gracefully fails", func() {
				Expect(app.Push()).ToNot(BeNil())

				Eventually(app.Stdout.String, 2*time.Second).Should(ContainSubstring("Unable to install node: no match found for 9000.0.0"))
				Expect(app.ConfirmBuildpack(buildpackVersion)).To(Succeed())
			})
		})

		Context("with an unsupported, but released, nodejs version", func() {
			BeforeEach(func() {
				app = cutlass.New(Fixtures("unsupported_node_version"))
			})

			It("displays a nice error messages and gracefully fails", func() {
				Expect(app.Push()).ToNot(BeNil())

				Eventually(app.Stdout.String, 2*time.Second).Should(ContainSubstring("Unable to install node: no match found for 4.1.1"))
				Expect(app.ConfirmBuildpack(buildpackVersion)).To(Succeed())
			})
		})
	})

	Context("with no Procfile and OPTIMIZE_MEMORY=true", func() {
		BeforeEach(func() {
			app = cutlass.New(Fixtures("simple_app"))
			app.SetEnv("OPTIMIZE_MEMORY", "true")
		})

		It("is running with autosized max_old_space_size", func() {
			PushAppAndConfirm(app)

			Expect(app.GetBody("/")).To(ContainSubstring("NodeOptions: --max_old_space_size=96"))
		})
	})

	Context("with no Procfile and OPTIMIZE_MEMORY is unset", func() {
		BeforeEach(func() {
			app = cutlass.New(Fixtures("simple_app"))
		})

		It("is not running with autosized max_old_space_size", func() {
			PushAppAndConfirm(app)

			Expect(app.GetBody("/")).To(ContainSubstring("NodeOptions: undefined"))
		})

		Context("a nvmrc file that takes precedence over package.json", func() {
			BeforeEach(func() {
				app = cutlass.New(Fixtures("simple_app_with_nvmrc"))
			})

			It("deploys", func() {
				PushAppAndConfirm(app)

				Expect(app.GetBody("/")).To(ContainSubstring("NodeOptions: undefined"))
			})
		})
	})

	Describe("Vendored Dependencies", func() {
		Context("with an app that has vendored dependencies", func() {
			It("deploys", func() {
				app = cutlass.New(Fixtures("vendored_dependencies"))
				app.SetEnv("BP_DEBUG", "true")
				PushAppAndConfirm(app)

				By("does not output protip that recommends user vendors dependencies", func() {
					Expect(app.Stdout.String()).ToNot(MatchRegexp("PRO TIP:(.*) It is recommended to vendor the application's Node.js dependencies"))
				})

				By("not changing the app directory during staging", func() {
					Expect(app).To(HaveUnchangedAppDir())
				})

				if !cutlass.Cached {
					By("with an uncached buildpack", func() {
						By("successfully deploys and includes the dependencies", func() {
							Expect(app.GetBody("/")).To(ContainSubstring("0000000005"))
							Eventually(app.Stdout.String).Should(ContainSubstring("Download [https://"))
						})
					})
				}

				if cutlass.Cached {
					By("with a cached buildpack", func() {
						By("deploys without hitting the internet", func() {
							Expect(app.GetBody("/")).To(ContainSubstring("0000000005"))
							Eventually(app.Stdout.String).Should(ContainSubstring("Copy [/tmp/buildpacks/"))
						})
					})
				}
			})

			AssertNoInternetTraffic("vendored_dependencies")
		})

		Context("Vendored Depencencies with node module binaries", func() {
			BeforeEach(func() {
				if !ApiSupportsSymlinks() {
					Skip("Requires api symlink support")
				}
			})

			It("deploys", func() {
				app = cutlass.New(Fixtures("vendored_dependencies_with_binaries"))
				app.SetEnv("BP_DEBUG", "true")
				PushAppAndConfirm(app)
			})
		})

		Context("with an app with a yarn.lock and vendored dependencies", func() {
			BeforeEach(func() {
				app = cutlass.New(Fixtures("with_yarn_vendored"))
				app.SetEnv("BP_DEBUG", "true")
			})

			It("deploys without hitting the internet", func() {
				PushAppAndConfirm(app)

				Expect(filepath.Join(app.Path, "node_modules")).To(BeADirectory())
				Eventually(app.Stdout.String).Should(ContainSubstring("Running yarn in offline mode"))
				Expect(app.GetBody("/microtime")).To(MatchRegexp("native time: \\d+\\.\\d+"))
			})

			AssertNoInternetTraffic("with_yarn_vendored")
		})

		Context("with an incomplete node_modules directory", func() {
			BeforeEach(func() {
				app = cutlass.New(Fixtures("incomplete_node_modules"))
			})

			It("downloads missing dependencies from package.json", func() {
				PushAppAndConfirm(app)
				Expect(filepath.Join(app.Path, "node_modules")).To(BeADirectory())
				Expect(filepath.Join(app.Path, "node_modules", "hashish")).ToNot(BeADirectory())
				Expect(app.Files("app/node_modules")).To(ContainElement("app/node_modules/hashish"))
				Expect(app.Files("app/node_modules")).To(ContainElement("app/node_modules/express"))
			})
		})
		Context("with an incomplete package.json", func() {
			BeforeEach(func() {
				app = cutlass.New(Fixtures("incomplete_package_json"))
			})

			It("overwrites the vendored modules not listed in package.json", func() {
				PushAppAndConfirm(app)
				Expect(app.Files("app/node_modules")).To(ContainElement("app/node_modules/leftpad"))
				Expect(app.Files("app/node_modules")).NotTo(ContainElement("app/node_modules/hashish"))
				Expect(app.Files("app/node_modules")).NotTo(ContainElement("app/node_modules/traverse"))
			})
		})
	})

	Describe("No Vendored Dependencies", func() {
		Context("with an app with no vendored dependencies", func() {
			BeforeEach(func() {
				app = cutlass.New(Fixtures("no_vendored_dependencies"))
				app.SetEnv("BP_DEBUG", "true")
			})

			It("successfully deploys and vendors the dependencies", func() {
				PushAppAndConfirm(app)

				Expect(filepath.Join(app.Path, "node_modules")).ToNot(BeADirectory())

				depsDir, err := app.GetBody("/deps_dir")
				Expect(err).ToNot(HaveOccurred())
				Expect(app.Files(depsDir)).To(ContainElement(depsDir + "/0/node_modules"))

				Expect(app.GetBody("/")).To(ContainSubstring("Hello, World!"))

				By("outputs protip that recommends user vendors dependencies", func() {
					Eventually(app.Stdout.String).Should(MatchRegexp("PRO TIP:(.*) It is recommended to vendor the application's Node.js dependencies"))
				})
			})

			AssertUsesProxyDuringStagingIfPresent("no_vendored_dependencies")
		})
		Context("with an app with a yarn.lock file", func() {
			BeforeEach(func() {
				app = cutlass.New(Fixtures("with_yarn"))
				app.SetEnv("BP_DEBUG", "true")
			})

			It("successfully deploys and vendors the dependencies via yarn", func() {
				PushAppAndConfirm(app)

				Expect(filepath.Join(app.Path, "node_modules")).ToNot(BeADirectory())
				By("not changing the app directory during staging", func() {
					Expect(app).To(HaveUnchangedAppDir())
				})

				Eventually(app.Stdout.String).Should(ContainSubstring("Running yarn in online mode"))

				depsDir, err := app.GetBody("/deps_dir")
				Expect(err).ToNot(HaveOccurred())
				Expect(app.Files(depsDir)).To(ContainElement(depsDir + "/0/node_modules"))

				Expect(app.GetBody("/")).To(ContainSubstring("Hello, World!"))
			})

			AssertUsesProxyDuringStagingIfPresent("with_yarn")
		})

		Context("with an app with an out of date yarn.lock", func() {
			BeforeEach(func() {
				app = cutlass.New(Fixtures("out_of_date_yarn_lock"))
			})

			It("warns that yarn.lock is out of date", func() {
				PushAppAndConfirm(app)
				Eventually(app.Stdout.String).Should(ContainSubstring("yarn.lock is outdated"))
			})
		})

		Context("with an app with pre and post scripts", func() {
			BeforeEach(func() {
				app = cutlass.New(Fixtures("pre_post_commands"))
			})

			It("runs the scripts through npm run", func() {
				PushAppAndConfirm(app)
				Eventually(app.Stdout.String).Should(ContainSubstring("Running heroku-prebuild (npm)"))
				Eventually(app.Stdout.String).Should(ContainSubstring("Running heroku-postbuild (npm)"))
				Expect(app.GetBody("/")).To(ContainSubstring("Text: Hello Buildpacks Team"))
				Expect(app.GetBody("/")).To(ContainSubstring("Text: Goodbye Buildpacks Team"))
			})

			It("runs the postinstall script in the app directory", func() {
				PushAppAndConfirm(app)
				Eventually(app.Stdout.String, 2*time.Second).Should(ContainSubstring("Current dir: /tmp/app"))
			})
		})

		Context("with an app using node-pre-gyp and yarn", func() {
			BeforeEach(func() {
				app = cutlass.New(Fixtures("yarn_pre_gyp"))
			})

			It("successfully deploys", func() {
				PushAppAndConfirm(app)
				Expect(app.GetBody("/")).To(ContainSubstring("Hello, World!"))

			})
		})

	})

	Describe("NODE_HOME and NODE_ENV", func() {
		BeforeEach(func() {
			if !cutlass.Cached {
				Skip("running uncached tests")
			}
			app = cutlass.New(Fixtures("logenv"))
		})

		It("sets the NODE_HOME to correct value", func() {
			PushAppAndConfirm(app)
			Eventually(app.Stdout.String).Should(MatchRegexp("NODE_HOME=\\S*/0/node"))

			body, err := app.GetBody("/")
			Expect(err).To(BeNil())
			Expect(body).To(MatchRegexp(`"NODE_HOME":"[^"]*/0/node"`))
			Expect(body).To(ContainSubstring(`"NODE_ENV":"production"`))
			Expect(body).To(ContainSubstring(`"MEMORY_AVAILABLE":"128"`))
		})
	})

	Describe("System CA Store", func() {
		BeforeEach(func() {
			app = cutlass.New(Fixtures("use-openssl-ca"))
			app.SetEnv("SSL_CERT_FILE", "cert.pem")
		})
		It("uses the system CA store (or env)", func() {
			PushAppAndConfirm(app)
			Expect(app.GetBody("/")).To(ContainSubstring("Response over self signed https"))
		})
	})
})
<commit_msg>Tests: Default expected version is 14<commit_after>package integration_test

import (
	"os/exec"
	"path/filepath"
	"time"

	"github.com/cloudfoundry/libbuildpack/cutlass"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("CF NodeJS Buildpack", func() {
	var app *cutlass.App
	AfterEach(func() {
		if app != nil {
			app.Destroy()
		}
		app = nil
	})

	Describe("nodeJS versions", func() {
		Context("when specifying a range for the nodeJS version in the package.json", func() {
			BeforeEach(func() {
				app = cutlass.New(Fixtures("node_version_range"))
			})

			It("resolves to a nodeJS version successfully", func() {
				PushAppAndConfirm(app)

				Eventually(app.Stdout.String).Should(MatchRegexp("Installing node 14\\.\\d+\\.\\d+"))
				Expect(app.GetBody("/")).To(ContainSubstring("Hello, World!"))
			})
		})

		Context("when specifying a version 6 for the nodeJS version in the package.json", func() {
			BeforeEach(func() {
				app = cutlass.New(Fixtures("node_version_6"))
			})

			It("resolves to a nodeJS version successfully", func() {
				PushAppAndConfirm(app)

				Eventually(app.Stdout.String).Should(MatchRegexp("Installing node 14\\.\\d+\\.\\d+"))
				Expect(app.GetBody("/")).To(ContainSubstring("Hello, World!"))

				if ApiHasTask() {
					By("running a task", func() {
						By("can find node in the container", func() {
							command := exec.Command("cf", "run-task", app.Name, "echo \"RUNNING A TASK: $(node --version)\"")
							_, err := command.Output()
							Expect(err).To(BeNil())

							Eventually(func() string {
								return app.Stdout.String()
							}, "30s").Should(MatchRegexp("RUNNING A TASK: v10\\.\\d+\\.\\d+"))
						})
					})
				}
			})
		})

		Context("when not specifying a nodeJS version in the package.json", func() {
			BeforeEach(func() {
				app = cutlass.New(Fixtures("without_node_version"))
			})

			It("resolves to the stable nodeJS version successfully", func() {
				PushAppAndConfirm(app)

				Eventually(app.Stdout.String).Should(MatchRegexp("Installing node 14\\.\\d+\\.\\d+"))
				Expect(app.GetBody("/")).To(ContainSubstring("Hello, World!"))
			})
		})

		Context("with an unreleased nodejs version", func() {
			BeforeEach(func() {
				app = cutlass.New(Fixtures("unreleased_node_version"))
			})

			It("displays a nice error message and gracefully fails", func() {
				Expect(app.Push()).ToNot(BeNil())

				Eventually(app.Stdout.String, 2*time.Second).Should(ContainSubstring("Unable to install node: no match found for 9000.0.0"))
				Expect(app.ConfirmBuildpack(buildpackVersion)).To(Succeed())
			})
		})

		Context("with an unsupported, but released, nodejs version", func() {
			BeforeEach(func() {
				app = cutlass.New(Fixtures("unsupported_node_version"))
			})

			It("displays a nice error messages and gracefully fails", func() {
				Expect(app.Push()).ToNot(BeNil())

				Eventually(app.Stdout.String, 2*time.Second).Should(ContainSubstring("Unable to install node: no match found for 4.1.1"))
				Expect(app.ConfirmBuildpack(buildpackVersion)).To(Succeed())
			})
		})
	})

	Context("with no Procfile and OPTIMIZE_MEMORY=true", func() {
		BeforeEach(func() {
			app = cutlass.New(Fixtures("simple_app"))
			app.SetEnv("OPTIMIZE_MEMORY", "true")
		})

		It("is running with autosized max_old_space_size", func() {
			PushAppAndConfirm(app)

			Expect(app.GetBody("/")).To(ContainSubstring("NodeOptions: --max_old_space_size=96"))
		})
	})

	Context("with no Procfile and OPTIMIZE_MEMORY is unset", func() {
		BeforeEach(func() {
			app = cutlass.New(Fixtures("simple_app"))
		})

		It("is not running with autosized max_old_space_size", func() {
			PushAppAndConfirm(app)

			Expect(app.GetBody("/")).To(ContainSubstring("NodeOptions: undefined"))
		})

		Context("a nvmrc file that takes precedence over package.json", func() {
			BeforeEach(func() {
				app = cutlass.New(Fixtures("simple_app_with_nvmrc"))
			})

			It("deploys", func() {
				PushAppAndConfirm(app)

				Expect(app.GetBody("/")).To(ContainSubstring("NodeOptions: undefined"))
			})
		})
	})

	Describe("Vendored Dependencies", func() {
		Context("with an app that has vendored dependencies", func() {
			It("deploys", func() {
				app = cutlass.New(Fixtures("vendored_dependencies"))
				app.SetEnv("BP_DEBUG", "true")
				PushAppAndConfirm(app)

				By("does not output protip that recommends user vendors dependencies", func() {
					Expect(app.Stdout.String()).ToNot(MatchRegexp("PRO TIP:(.*) It is recommended to vendor the application's Node.js dependencies"))
				})

				By("not changing the app directory during staging", func() {
					Expect(app).To(HaveUnchangedAppDir())
				})

				if !cutlass.Cached {
					By("with an uncached buildpack", func() {
						By("successfully deploys and includes the dependencies", func() {
							Expect(app.GetBody("/")).To(ContainSubstring("0000000005"))
							Eventually(app.Stdout.String).Should(ContainSubstring("Download [https://"))
						})
					})
				}

				if cutlass.Cached {
					By("with a cached buildpack", func() {
						By("deploys without hitting the internet", func() {
							Expect(app.GetBody("/")).To(ContainSubstring("0000000005"))
							Eventually(app.Stdout.String).Should(ContainSubstring("Copy [/tmp/buildpacks/"))
						})
					})
				}
			})

			AssertNoInternetTraffic("vendored_dependencies")
		})

		Context("Vendored Depencencies with node module binaries", func() {
			BeforeEach(func() {
				if !ApiSupportsSymlinks() {
					Skip("Requires api symlink support")
				}
			})

			It("deploys", func() {
				app = cutlass.New(Fixtures("vendored_dependencies_with_binaries"))
				app.SetEnv("BP_DEBUG", "true")
				PushAppAndConfirm(app)
			})
		})

		Context("with an app with a yarn.lock and vendored dependencies", func() {
			BeforeEach(func() {
				app = cutlass.New(Fixtures("with_yarn_vendored"))
				app.SetEnv("BP_DEBUG", "true")
			})

			It("deploys without hitting the internet", func() {
				PushAppAndConfirm(app)

				Expect(filepath.Join(app.Path, "node_modules")).To(BeADirectory())
				Eventually(app.Stdout.String).Should(ContainSubstring("Running yarn in offline mode"))
				Expect(app.GetBody("/microtime")).To(MatchRegexp("native time: \\d+\\.\\d+"))
			})

			AssertNoInternetTraffic("with_yarn_vendored")
		})

		Context("with an incomplete node_modules directory", func() {
			BeforeEach(func() {
				app = cutlass.New(Fixtures("incomplete_node_modules"))
			})

			It("downloads missing dependencies from package.json", func() {
				PushAppAndConfirm(app)
				Expect(filepath.Join(app.Path, "node_modules")).To(BeADirectory())
				Expect(filepath.Join(app.Path, "node_modules", "hashish")).ToNot(BeADirectory())
				Expect(app.Files("app/node_modules")).To(ContainElement("app/node_modules/hashish"))
				Expect(app.Files("app/node_modules")).To(ContainElement("app/node_modules/express"))
			})
		})
		Context("with an incomplete package.json", func() {
			BeforeEach(func() {
				app = cutlass.New(Fixtures("incomplete_package_json"))
			})

			It("overwrites the vendored modules not listed in package.json", func() {
				PushAppAndConfirm(app)
				Expect(app.Files("app/node_modules")).To(ContainElement("app/node_modules/leftpad"))
				Expect(app.Files("app/node_modules")).NotTo(ContainElement("app/node_modules/hashish"))
				Expect(app.Files("app/node_modules")).NotTo(ContainElement("app/node_modules/traverse"))
			})
		})
	})

	Describe("No Vendored Dependencies", func() {
		Context("with an app with no vendored dependencies", func() {
			BeforeEach(func() {
				app = cutlass.New(Fixtures("no_vendored_dependencies"))
				app.SetEnv("BP_DEBUG", "true")
			})

			It("successfully deploys and vendors the dependencies", func() {
				PushAppAndConfirm(app)

				Expect(filepath.Join(app.Path, "node_modules")).ToNot(BeADirectory())

				depsDir, err := app.GetBody("/deps_dir")
				Expect(err).ToNot(HaveOccurred())
				Expect(app.Files(depsDir)).To(ContainElement(depsDir + "/0/node_modules"))

				Expect(app.GetBody("/")).To(ContainSubstring("Hello, World!"))

				By("outputs protip that recommends user vendors dependencies", func() {
					Eventually(app.Stdout.String).Should(MatchRegexp("PRO TIP:(.*) It is recommended to vendor the application's Node.js dependencies"))
				})
			})

			AssertUsesProxyDuringStagingIfPresent("no_vendored_dependencies")
		})
		Context("with an app with a yarn.lock file", func() {
			BeforeEach(func() {
				app = cutlass.New(Fixtures("with_yarn"))
				app.SetEnv("BP_DEBUG", "true")
			})

			It("successfully deploys and vendors the dependencies via yarn", func() {
				PushAppAndConfirm(app)

				Expect(filepath.Join(app.Path, "node_modules")).ToNot(BeADirectory())
				By("not changing the app directory during staging", func() {
					Expect(app).To(HaveUnchangedAppDir())
				})

				Eventually(app.Stdout.String).Should(ContainSubstring("Running yarn in online mode"))

				depsDir, err := app.GetBody("/deps_dir")
				Expect(err).ToNot(HaveOccurred())
				Expect(app.Files(depsDir)).To(ContainElement(depsDir + "/0/node_modules"))

				Expect(app.GetBody("/")).To(ContainSubstring("Hello, World!"))
			})

			AssertUsesProxyDuringStagingIfPresent("with_yarn")
		})

		Context("with an app with an out of date yarn.lock", func() {
			BeforeEach(func() {
				app = cutlass.New(Fixtures("out_of_date_yarn_lock"))
			})

			It("warns that yarn.lock is out of date", func() {
				PushAppAndConfirm(app)
				Eventually(app.Stdout.String).Should(ContainSubstring("yarn.lock is outdated"))
			})
		})

		Context("with an app with pre and post scripts", func() {
			BeforeEach(func() {
				app = cutlass.New(Fixtures("pre_post_commands"))
			})

			It("runs the scripts through npm run", func() {
				PushAppAndConfirm(app)
				Eventually(app.Stdout.String).Should(ContainSubstring("Running heroku-prebuild (npm)"))
				Eventually(app.Stdout.String).Should(ContainSubstring("Running heroku-postbuild (npm)"))
				Expect(app.GetBody("/")).To(ContainSubstring("Text: Hello Buildpacks Team"))
				Expect(app.GetBody("/")).To(ContainSubstring("Text: Goodbye Buildpacks Team"))
			})

			It("runs the postinstall script in the app directory", func() {
				PushAppAndConfirm(app)
				Eventually(app.Stdout.String, 2*time.Second).Should(ContainSubstring("Current dir: /tmp/app"))
			})
		})

		Context("with an app using node-pre-gyp and yarn", func() {
			BeforeEach(func() {
				app = cutlass.New(Fixtures("yarn_pre_gyp"))
			})

			It("successfully deploys", func() {
				PushAppAndConfirm(app)
				Expect(app.GetBody("/")).To(ContainSubstring("Hello, World!"))

			})
		})

	})

	Describe("NODE_HOME and NODE_ENV", func() {
		BeforeEach(func() {
			if !cutlass.Cached {
				Skip("running uncached tests")
			}
			app = cutlass.New(Fixtures("logenv"))
		})

		It("sets the NODE_HOME to correct value", func() {
			PushAppAndConfirm(app)
			Eventually(app.Stdout.String).Should(MatchRegexp("NODE_HOME=\\S*/0/node"))

			body, err := app.GetBody("/")
			Expect(err).To(BeNil())
			Expect(body).To(MatchRegexp(`"NODE_HOME":"[^"]*/0/node"`))
			Expect(body).To(ContainSubstring(`"NODE_ENV":"production"`))
			Expect(body).To(ContainSubstring(`"MEMORY_AVAILABLE":"128"`))
		})
	})

	Describe("System CA Store", func() {
		BeforeEach(func() {
			app = cutlass.New(Fixtures("use-openssl-ca"))
			app.SetEnv("SSL_CERT_FILE", "cert.pem")
		})
		It("uses the system CA store (or env)", func() {
			PushAppAndConfirm(app)
			Expect(app.GetBody("/")).To(ContainSubstring("Response over self signed https"))
		})
	})
})
<|endoftext|>


<commit_before>package cbfsclient

import (
	"fmt"
	"time"
)

// Representation of a storage node.
type StorageNode struct {
	Addr      string
	AddrRaw   string    `json:"addr_raw"`
	Started   time.Time `json:"starttime"`
	HBTime    time.Time `json:"hbtime"`
	BindAddr  string
	FrameBind string
	HBAgeStr  string `json:"hbage_str"`
	Used      int64
	Free      int64
	Size      int64
	UptimeStr string `json:"uptime_str"`
	Version   string
}

func (a StorageNode) BlobURL(h string) string {
	return a.URLFor("/.cbfs/blob/" + h)
}

func (a StorageNode) URLFor(h string) string {
	if h[0] != '/' {
		h = "/" + h
	}
	return fmt.Sprintf("http://%s%s", a.Addr, h)
}

// Get the information about the nodes in a cluster.
func (c *Client) Nodes() (map[string]StorageNode, error) {
	var err error
	if c.nodes == nil {
		c.nodes = map[string]StorageNode{}
		err = getJsonData(c.URLFor("/.cbfs/nodes/"), &c.nodes)
	}
	return c.nodes, err
}
<commit_msg>Added RandomNode<commit_after>package cbfsclient

import (
	"fmt"
	"math/rand"
	"time"
)

// Representation of a storage node.
type StorageNode struct {
	Addr      string
	AddrRaw   string    `json:"addr_raw"`
	Started   time.Time `json:"starttime"`
	HBTime    time.Time `json:"hbtime"`
	BindAddr  string
	FrameBind string
	HBAgeStr  string `json:"hbage_str"`
	Used      int64
	Free      int64
	Size      int64
	UptimeStr string `json:"uptime_str"`
	Version   string
}

func (a StorageNode) BlobURL(h string) string {
	return a.URLFor("/.cbfs/blob/" + h)
}

func (a StorageNode) URLFor(h string) string {
	if h[0] != '/' {
		h = "/" + h
	}
	return fmt.Sprintf("http://%s%s", a.Addr, h)
}

// Get the information about the nodes in a cluster.
func (c *Client) Nodes() (map[string]StorageNode, error) {
	var err error
	if c.nodes == nil {
		c.nodes = map[string]StorageNode{}
		err = getJsonData(c.URLFor("/.cbfs/nodes/"), &c.nodes)
	}
	return c.nodes, err
}

func (c *Client) RandomNode() (string, StorageNode, error) {
	nodeMap, err := c.Nodes()
	if err != nil {
		return "", StorageNode{}, err
	}

	nodes := make([]string, 0, len(nodeMap))
	for k := range nodeMap {
		nodes = append(nodes, k)
	}

	name := nodes[rand.Intn(len(nodes))]

	return name, nodeMap[name], nil
}
<|endoftext|>


<commit_before>package ics

import (
	"bytes"
	"io"
	"reflect"
	"testing"
)

type typeTest struct {
	Params map[string]string
	Data   string
	Input  interface {
		Decode(map[string]string, string) error
	}
	Match interface {
		Encode(io.Writer)
	}
	Output string
	Error  error
}

func testType(t *testing.T, tests []typeTest) {
	var buf bytes.Buffer
	for n, test := range tests {
		err := test.Input.Decode(test.Params, test.Data)
		if err != test.Error {
			if !reflect.DeepEqual(err, test.Error) {
				t.Errorf("test %d: expecting error %s, got %s", n+1, test.Error, err)
				continue
			}
		}
		if !reflect.DeepEqual(test.Input, test.Match) {
			t.Errorf("test %d: input does not match expected", n+1)
			continue
		}
		test.Match.Encode(&buf)
		if str := buf.String(); str != test.Output {
			t.Errorf("test %d: expecting output string %q, got %q", n+1, test.Output, str)
		}
		buf.Reset()
	}
}

func TestBinary(t *testing.T) {
	testType(t, []typeTest{
		{
			Input: &Binary{},
			Match: &Binary{},
			Error: ErrInvalidEncoding,
		},
		{
			Params: map[string]string{"ENCODING": "BASE64"},
			Data:   "MTIzNDU=",
			Input:  &Binary{},
			Match:  &Binary{'1', '2', '3', '4', '5'},
			Output: "MTIzNDU=",
		},
	})
}

func TestBoolean(t *testing.T) {
	tr := new(Boolean)
	fa := new(Boolean)
	*tr = true
	testType(t, []typeTest{
		{
			Data:   "False",
			Input:  fa,
			Match:  fa,
			Output: "FALSE",
		},
		{
			Data:   "true",
			Input:  tr,
			Match:  tr,
			Output: "TRUE",
		},
		{
			Data:   "HotDog",
			Input:  fa,
			Match:  fa,
			Output: "FALSE",
			Error:  ErrInvalidBoolean,
		},
	})
}

func TestDate(t *testing.T) {

}
<commit_msg>added tests for Date<commit_after>package ics

import (
	"bytes"
	"io"
	"reflect"
	"testing"
	"time"
)

type typeTest struct {
	Params map[string]string
	Data   string
	Input  interface {
		Decode(map[string]string, string) error
	}
	Match interface {
		Encode(io.Writer)
	}
	Output string
	Error  error
}

func testType(t *testing.T, tests []typeTest) {
	var buf bytes.Buffer
	for n, test := range tests {
		err := test.Input.Decode(test.Params, test.Data)
		if err != test.Error {
			if !reflect.DeepEqual(err, test.Error) {
				t.Errorf("test %d: expecting error %s, got %s", n+1, test.Error, err)
				continue
			}
		}
		if !reflect.DeepEqual(test.Input, test.Match) {
			t.Errorf("test %d: input does not match expected", n+1)
			continue
		}
		test.Match.Encode(&buf)
		if str := buf.String(); str != test.Output {
			t.Errorf("test %d: expecting output string %q, got %q", n+1, test.Output, str)
		}
		buf.Reset()
	}
}

func TestBinary(t *testing.T) {
	testType(t, []typeTest{
		{
			Input: &Binary{},
			Match: &Binary{},
			Error: ErrInvalidEncoding,
		},
		{
			Params: map[string]string{"ENCODING": "BASE64"},
			Data:   "MTIzNDU=",
			Input:  &Binary{},
			Match:  &Binary{'1', '2', '3', '4', '5'},
			Output: "MTIzNDU=",
		},
	})
}

func TestBoolean(t *testing.T) {
	tr := new(Boolean)
	fa := new(Boolean)
	*tr = true
	testType(t, []typeTest{
		{
			Data:   "False",
			Input:  fa,
			Match:  fa,
			Output: "FALSE",
		},
		{
			Data:   "true",
			Input:  tr,
			Match:  tr,
			Output: "TRUE",
		},
		{
			Data:   "HotDog",
			Input:  fa,
			Match:  fa,
			Output: "FALSE",
			Error:  ErrInvalidBoolean,
		},
	})
}

func TestDate(t *testing.T) {
	testType(t, []typeTest{
		{
			Data:   "20011225",
			Input:  &Date{},
			Match:  &Date{time.Date(2001, 12, 25, 0, 0, 0, 0, time.UTC)},
			Output: "20011225",
		},
		{
			Data:   "20081111",
			Input:  &Date{},
			Match:  &Date{time.Date(2008, 11, 11, 0, 0, 0, 0, time.UTC)},
			Output: "20081111",
		},
	})
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/rand"
	"fmt"
	"image"
	"image/gif"
	"io"
	"log"
	"net/http"
	"strconv"
	"time"

	"github.com/avct/uasurfer"

	"./database"

	"encoding/hex"

	influx "github.com/influxdata/influxdb/client/v2"
)

// VersionString converts a uasurfer.Version into a string
func VersionString(v uasurfer.Version) string {
	if v.Major == 0 {
		return ""
	}

	return strconv.Itoa(v.Major) + "." + strconv.Itoa(v.Minor) + "." + strconv.Itoa(v.Patch)
}

var b bytes.Buffer
var err = gif.Encode(&b, image.NewAlpha(image.Rect(0, 0, 1, 1)), nil)

// OnePixelGIF - The data for a one pixel transparent GIF
var OnePixelGIF = b.Bytes()

func handler(w http.ResponseWriter, r *http.Request) {

	tags := make(map[string]string)

	var fpt string

	// TODO: set a limit on arguments
	for key, vals := range r.URL.Query() {
		log.Printf("%s: %s\n", key, vals[0])

		if key != "fpt" {
			tags[key] = vals[0]
		} else {
			fpt = vals[0]
		}
	}

	ua := r.Header.Get("User-Agent")

	if ua != "" {
		parsedUa := uasurfer.Parse(ua)

		tags["browser"] = parsedUa.Browser.Name.String()
		tags["browser_ver"] = VersionString(parsedUa.Browser.Version)

		if parsedUa.Browser.Version.Major != 0 {
			tags["browser_major"] = strconv.Itoa(parsedUa.Browser.Version.Major)
		}

		tags["os"] = parsedUa.OS.Name.String()

		tags["os_ver"] = VersionString(parsedUa.OS.Version)

		if parsedUa.OS.Version.Major != 0 {
			// OS X versions are weird
			if parsedUa.OS.Name == uasurfer.OSMacOSX {
				tags["os_major"] = strconv.Itoa(parsedUa.OS.Version.Minor)
			} else {
				tags["os_major"] = strconv.Itoa(parsedUa.OS.Version.Major)
			}
		}

		tags["device_type"] = parsedUa.DeviceType.String()

		log.Printf("%v\n", parsedUa.Browser.Name)
	}

	// TODO: break URL into components

	connection := database.Connect("test", "test", "http://xenial.dev:8086")
	points, err := influx.NewBatchPoints(influx.BatchPointsConfig{
		Database: "test",
	})

	if err != nil {
		log.Fatal(err)
	}

	pt, _ := influx.NewPoint("hello", tags, map[string]interface{}{
		"fpt": fpt,
	})

	points.AddPoint(pt)

	log.Println(connection.Ping(time.Second))
	err = connection.Write(points)

	if err != nil {
		log.Fatal(err)
	}

	w.Header().Add("Content-Type", "image/gif")
	io.Copy(w, bytes.NewReader(OnePixelGIF))
}

func randomFingerprint(w http.ResponseWriter, r *http.Request) {
	bytes := make([]byte, 8)

	_, err := rand.Read(bytes)

	if err != nil {
		r.Response.StatusCode = 500
	} else {
		fmt.Fprintf(w, "%s", hex.EncodeToString(bytes))
	}
}

func main() {
	http.HandleFunc("/aurora", handler)
	http.HandleFunc("/fp", randomFingerprint)
	http.ListenAndServe(":3030", nil)
}
<commit_msg>Added IP tracking<commit_after>package main

import (
	"bytes"
	"crypto/rand"
	"fmt"
	"image"
	"image/gif"
	"io"
	"log"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/avct/uasurfer"

	"./database"

	"encoding/hex"

	influx "github.com/influxdata/influxdb/client/v2"
)

// VersionString converts a uasurfer.Version into a string
func VersionString(v uasurfer.Version) string {
	if v.Major == 0 {
		return ""
	}

	return strconv.Itoa(v.Major) + "." + strconv.Itoa(v.Minor) + "." + strconv.Itoa(v.Patch)
}

var b bytes.Buffer
var err = gif.Encode(&b, image.NewAlpha(image.Rect(0, 0, 1, 1)), nil)

// OnePixelGIF - The data for a one pixel transparent GIF
var OnePixelGIF = b.Bytes()

func handler(w http.ResponseWriter, r *http.Request) {

	tags := make(map[string]string)

	var fpt string

	// TODO: set a limit on arguments
	for key, vals := range r.URL.Query() {
		log.Printf("%s: %s\n", key, vals[0])

		if key != "fpt" {
			tags[key] = vals[0]
		} else {
			fpt = vals[0]
		}
	}

	ua := r.Header.Get("User-Agent")

	if ua != "" {
		parsedUa := uasurfer.Parse(ua)

		tags["browser"] = parsedUa.Browser.Name.String()
		tags["browser_ver"] = VersionString(parsedUa.Browser.Version)

		if parsedUa.Browser.Version.Major != 0 {
			tags["browser_major"] = strconv.Itoa(parsedUa.Browser.Version.Major)
		}

		tags["os"] = parsedUa.OS.Name.String()

		tags["os_ver"] = VersionString(parsedUa.OS.Version)

		if parsedUa.OS.Version.Major != 0 {
			// OS X versions are weird
			if parsedUa.OS.Name == uasurfer.OSMacOSX {
				tags["os_major"] = strconv.Itoa(parsedUa.OS.Version.Minor)
			} else {
				tags["os_major"] = strconv.Itoa(parsedUa.OS.Version.Major)
			}
		}

		tags["device_type"] = parsedUa.DeviceType.String()

		log.Printf("%v\n", parsedUa.Browser.Name)
	}

	// Remove port from IP
	ip := r.RemoteAddr[0:strings.LastIndex(r.RemoteAddr, ":")]

	// Remove square brackets from IPv6
	if ip[0] == '[' {
		ip = ip[1:(len(ip) - 1)]
	}

	tags["ip"] = ip

	// TODO: break URL into components

	connection := database.Connect("test", "test", "http://xenial.dev:8086")
	points, err := influx.NewBatchPoints(influx.BatchPointsConfig{
		Database: "test",
	})

	if err != nil {
		log.Fatal(err)
	}

	pt, _ := influx.NewPoint("hello", tags, map[string]interface{}{
		"fpt": fpt,
	})

	points.AddPoint(pt)

	log.Println(connection.Ping(time.Second))
	err = connection.Write(points)

	if err != nil {
		log.Fatal(err)
	}

	w.Header().Add("Content-Type", "image/gif")
	io.Copy(w, bytes.NewReader(OnePixelGIF))
}

func randomFingerprint(w http.ResponseWriter, r *http.Request) {
	bytes := make([]byte, 8)

	_, err := rand.Read(bytes)

	if err != nil {
		r.Response.StatusCode = 500
	} else {
		fmt.Fprintf(w, "%s", hex.EncodeToString(bytes))
	}
}

func main() {
	http.HandleFunc("/aurora", handler)
	http.HandleFunc("/fp", randomFingerprint)
	http.ListenAndServe(":3030", nil)
}
<|endoftext|>


<commit_before>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package clientv3 implements the official Go etcd client for v3.
//
// Create client using `clientv3.New`:
//
//	// expect dial time-out on ipv4 blackhole
//	_, err := clientv3.New(clientv3.Config{
//		Endpoints:   []string{"http://254.0.0.1:12345"},
//		DialTimeout: 2 * time.Second
//	})
//
//	// etcd clientv3 >= v3.2.10, grpc/grpc-go >= v1.7.3
//	if err == context.DeadlineExceeded {
//		// handle errors
//	}
//
//	// etcd clientv3 <= v3.2.9, grpc/grpc-go <= v1.2.1
//	if err == grpc.ErrClientConnTimeout {
//		// handle errors
//	}
//
//	cli, err := clientv3.New(clientv3.Config{
//		Endpoints:   []string{"localhost:2379", "localhost:22379", "localhost:32379"},
//		DialTimeout: 5 * time.Second,
//	})
//	if err != nil {
//		// handle error!
//	}
//	defer cli.Close()
//
// Make sure to close the client after using it. If the client is not closed, the
// connection will have leaky goroutines.
//
// To specify a client request timeout, wrap the context with context.WithTimeout:
//
//	ctx, cancel := context.WithTimeout(context.Background(), timeout)
//	resp, err := kvc.Put(ctx, "sample_key", "sample_value")
//	cancel()
//	if err != nil {
//	    // handle error!
//	}
//	// use the response
//
// The Client has internal state (watchers and leases), so Clients should be reused instead of created as needed.
// Clients are safe for concurrent use by multiple goroutines.
//
// etcd client returns 3 types of errors:
//
//	1. context error: canceled or deadline exceeded.
//  2. gRPC status error: e.g. when clock drifts in server-side before client's context deadline exceeded.
//	3. gRPC error: see https://github.com/coreos/etcd/blob/master/etcdserver/api/v3rpc/rpctypes/error.go
//
// Here is the example code to handle client errors:
//
//	resp, err := kvc.Put(ctx, "", "")
//	if err != nil {
//		if err == context.Canceled {
//			// ctx is canceled by another routine
//		} else if err == context.DeadlineExceeded {
//			// ctx is attached with a deadline and it exceeded
//		} else if ev, ok := status.FromError(err); ok {
//			code := ev.Code()
//			if code == codes.DeadlineExceeded {
//				// server-side context might have timed-out first (due to clock skew)
//				// while original client-side context is not timed-out yet
//			}
//		} else if verr, ok := err.(*v3rpc.ErrEmptyKey); ok {
//			// process (verr.Errors)
//		} else {
//			// bad cluster endpoints, which are not etcd servers
//		}
//	}
//
//	go func() { cli.Close() }()
//	_, err := kvc.Get(ctx, "a")
//	if err != nil {
//		if err == context.Canceled {
//			// grpc balancer calls 'Get' with an inflight client.Close
//		} else if err == grpc.ErrClientConnClosing {
//			// grpc balancer calls 'Get' after client.Close.
//		}
//	}
//
package clientv3
<commit_msg>clientv3: fix indentation in doc.go<commit_after>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package clientv3 implements the official Go etcd client for v3.
//
// Create client using `clientv3.New`:
//
//	// expect dial time-out on ipv4 blackhole
//	_, err := clientv3.New(clientv3.Config{
//		Endpoints:   []string{"http://254.0.0.1:12345"},
//		DialTimeout: 2 * time.Second
//	})
//
//	// etcd clientv3 >= v3.2.10, grpc/grpc-go >= v1.7.3
//	if err == context.DeadlineExceeded {
//		// handle errors
//	}
//
//	// etcd clientv3 <= v3.2.9, grpc/grpc-go <= v1.2.1
//	if err == grpc.ErrClientConnTimeout {
//		// handle errors
//	}
//
//	cli, err := clientv3.New(clientv3.Config{
//		Endpoints:   []string{"localhost:2379", "localhost:22379", "localhost:32379"},
//		DialTimeout: 5 * time.Second,
//	})
//	if err != nil {
//		// handle error!
//	}
//	defer cli.Close()
//
// Make sure to close the client after using it. If the client is not closed, the
// connection will have leaky goroutines.
//
// To specify a client request timeout, wrap the context with context.WithTimeout:
//
//	ctx, cancel := context.WithTimeout(context.Background(), timeout)
//	resp, err := kvc.Put(ctx, "sample_key", "sample_value")
//	cancel()
//	if err != nil {
//	    // handle error!
//	}
//	// use the response
//
// The Client has internal state (watchers and leases), so Clients should be reused instead of created as needed.
// Clients are safe for concurrent use by multiple goroutines.
//
// etcd client returns 3 types of errors:
//
//  1. context error: canceled or deadline exceeded.
//  2. gRPC status error: e.g. when clock drifts in server-side before client's context deadline exceeded.
//  3. gRPC error: see https://github.com/coreos/etcd/blob/master/etcdserver/api/v3rpc/rpctypes/error.go
//
// Here is the example code to handle client errors:
//
//	resp, err := kvc.Put(ctx, "", "")
//	if err != nil {
//		if err == context.Canceled {
//			// ctx is canceled by another routine
//		} else if err == context.DeadlineExceeded {
//			// ctx is attached with a deadline and it exceeded
//		} else if ev, ok := status.FromError(err); ok {
//			code := ev.Code()
//			if code == codes.DeadlineExceeded {
//				// server-side context might have timed-out first (due to clock skew)
//				// while original client-side context is not timed-out yet
//			}
//		} else if verr, ok := err.(*v3rpc.ErrEmptyKey); ok {
//			// process (verr.Errors)
//		} else {
//			// bad cluster endpoints, which are not etcd servers
//		}
//	}
//
//	go func() { cli.Close() }()
//	_, err := kvc.Get(ctx, "a")
//	if err != nil {
//		if err == context.Canceled {
//			// grpc balancer calls 'Get' with an inflight client.Close
//		} else if err == grpc.ErrClientConnClosing {
//			// grpc balancer calls 'Get' after client.Close.
//		}
//	}
//
package clientv3
<|endoftext|>


<commit_before>package deploystack

import (
	"context"
	"fmt"
	"strings"
	"sync"

	"google.golang.org/api/cloudbilling/v1"
	"google.golang.org/api/cloudresourcemanager/v1"
)

var cloudbillingService *cloudbilling.APIService

func getCloudbillingService() (*cloudbilling.APIService, error) {
	if cloudbillingService != nil {
		return cloudbillingService, nil
	}

	ctx := context.Background()
	svc, err := cloudbilling.NewService(ctx, opts)
	if err != nil {
		return nil, err
	}

	cloudbillingService = svc

	return svc, nil
}

// billingAccounts gets a list of the billing accounts a user has access to
func billingAccounts() ([]*cloudbilling.BillingAccount, error) {
	resp := []*cloudbilling.BillingAccount{}
	svc, err := getCloudbillingService()
	if err != nil {
		return resp, err
	}

	results, err := svc.BillingAccounts.List().Do()
	if err != nil {
		return resp, err
	}

	return results.BillingAccounts, nil
}

// BillingAccountProjectAttach will enable billing in a given project
func BillingAccountProjectAttach(project, account string) error {
	retries := 10
	svc, err := getCloudbillingService()
	if err != nil {
		return err
	}

	ba := fmt.Sprintf("billingAccounts/%s", account)
	proj := fmt.Sprintf("projects/%s", project)

	cfg := cloudbilling.ProjectBillingInfo{
		BillingAccountName: ba,
	}

	var looperr error
	for i := 0; i < retries; i++ {
		_, looperr = svc.Projects.UpdateBillingInfo(proj, &cfg).Do()
		if looperr == nil {
			return nil
		}
		if strings.Contains(looperr.Error(), "User is not authorized to get billing info") {
			continue
		}
	}

	if strings.Contains(looperr.Error(), "Request contains an invalid argument") {
		return ErrorBillingInvalidAccount
	}

	if strings.Contains(looperr.Error(), "Not a valid billing account") {
		return ErrorBillingInvalidAccount
	}

	if strings.Contains(looperr.Error(), "The caller does not have permission") {
		return ErrorBillingNoPermission
	}

	return looperr
}

func getBillingForProjects(p []*cloudresourcemanager.Project) ([]projectWithBilling, error) {
	res := []projectWithBilling{}

	svc, err := getCloudbillingService()
	if err != nil {
		return res, err
	}
	var wg sync.WaitGroup
	wg.Add(len(p))

	for _, v := range p {
		go func(p *cloudresourcemanager.Project) {
			defer wg.Done()
			if p.LifecycleState == "ACTIVE" && p.Name != "" {
				proj := fmt.Sprintf("projects/%s", p.ProjectId)
				tmp, err := svc.Projects.GetBillingInfo(proj).Do()
				if err != nil {
					if strings.Contains(err.Error(), "The caller does not have permission") {
						fmt.Printf("project: %+v\n", p)
						return
					}

					fmt.Printf("error: %s\n", err)
					return
				}

				pwb := projectWithBilling{p.Name, tmp.BillingEnabled}
				res = append(res, pwb)
			}
		}(v)
	}
	wg.Wait()

	return res, nil
}
<commit_msg>Fixing a bug<commit_after>package deploystack

import (
	"context"
	"fmt"
	"math/rand"
	"strings"
	"sync"
	"time"

	"google.golang.org/api/cloudbilling/v1"
	"google.golang.org/api/cloudresourcemanager/v1"
)

var cloudbillingService *cloudbilling.APIService

func getCloudbillingService() (*cloudbilling.APIService, error) {
	if cloudbillingService != nil {
		return cloudbillingService, nil
	}

	ctx := context.Background()
	svc, err := cloudbilling.NewService(ctx, opts)
	if err != nil {
		return nil, err
	}

	cloudbillingService = svc

	return svc, nil
}

// billingAccounts gets a list of the billing accounts a user has access to
func billingAccounts() ([]*cloudbilling.BillingAccount, error) {
	resp := []*cloudbilling.BillingAccount{}
	svc, err := getCloudbillingService()
	if err != nil {
		return resp, err
	}

	results, err := svc.BillingAccounts.List().Do()
	if err != nil {
		return resp, err
	}

	return results.BillingAccounts, nil
}

// BillingAccountProjectAttach will enable billing in a given project
func BillingAccountProjectAttach(project, account string) error {
	retries := 10
	svc, err := getCloudbillingService()
	if err != nil {
		return err
	}

	ba := fmt.Sprintf("billingAccounts/%s", account)
	proj := fmt.Sprintf("projects/%s", project)

	cfg := cloudbilling.ProjectBillingInfo{
		BillingAccountName: ba,
	}

	var looperr error
	for i := 0; i < retries; i++ {
		_, looperr = svc.Projects.UpdateBillingInfo(proj, &cfg).Do()
		if looperr == nil {
			return nil
		}
		if strings.Contains(looperr.Error(), "User is not authorized to get billing info") {
			continue
		}
	}

	if strings.Contains(looperr.Error(), "Request contains an invalid argument") {
		return ErrorBillingInvalidAccount
	}

	if strings.Contains(looperr.Error(), "Not a valid billing account") {
		return ErrorBillingInvalidAccount
	}

	if strings.Contains(looperr.Error(), "The caller does not have permission") {
		return ErrorBillingNoPermission
	}

	return looperr
}

func getBillingForProjects(p []*cloudresourcemanager.Project) ([]projectWithBilling, error) {
	res := []projectWithBilling{}

	svc, err := getCloudbillingService()
	if err != nil {
		return res, err
	}
	var wg sync.WaitGroup
	wg.Add(len(p))

	for _, v := range p {
		go func(p *cloudresourcemanager.Project) {
			defer wg.Done()
			// Getting random quota errors when somebody had too many projects.
			// sleeping randoming for a second fixed it.
			// I don't think these requests can be fixed by batching.
			sleepRandom()
			if p.LifecycleState == "ACTIVE" && p.Name != "" {
				proj := fmt.Sprintf("projects/%s", p.ProjectId)
				tmp, err := svc.Projects.GetBillingInfo(proj).Do()
				if err != nil {
					if strings.Contains(err.Error(), "The caller does not have permission") {
						fmt.Printf("project: %+v\n", p)
						return
					}

					fmt.Printf("error getting billing information: %s\n", err)
					return
				}

				pwb := projectWithBilling{p.Name, tmp.BillingEnabled}
				res = append(res, pwb)
			}
		}(v)
	}
	wg.Wait()

	return res, nil
}

func randomInRange(min, max int) int {
	rand.Seed(time.Now().UnixNano())
	return rand.Intn(max-min+1) + min
}

func sleepRandom() {
	d := time.Second * time.Duration(randomInRange(0, 1))
	time.Sleep(d)
}
<|endoftext|>


<commit_before>package cluster

import (
	"fmt"
	"os"
	"os/exec"
	"path"
	"runtime"
	"sort"
	"testing"
	"time"

	"github.com/Shopify/sarama"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("PartitionSlice", func() {

	It("should sort correctly", func() {
		p1 := Partition{Addr: "host1:9093", ID: 1}
		p2 := Partition{Addr: "host1:9092", ID: 2}
		p3 := Partition{Addr: "host2:9092", ID: 3}
		p4 := Partition{Addr: "host3:9091", ID: 4}
		p5 := Partition{Addr: "host2:9093", ID: 5}
		p6 := Partition{Addr: "host1:9092", ID: 6}

		slice := PartitionSlice{p1, p2, p3, p4, p5, p6}
		sort.Sort(slice)
		Expect(slice).To(BeEquivalentTo(PartitionSlice{p2, p6, p1, p3, p5, p4}))
	})

})

var _ = Describe("GUID", func() {

	BeforeEach(func() {
		cGUID.hostname = "testhost"
		cGUID.pid = 20100
	})

	AfterEach(func() {
		cGUID.inc = 0
	})

	It("should create GUIDs", func() {
		cGUID.inc = 0xffffffff
		Expect(NewGUIDAt("prefix", time.Unix(1313131313, 0))).To(Equal("prefix-testhost-20100-1313131313-0"))
		Expect(NewGUIDAt("prefix", time.Unix(1414141414, 0))).To(Equal("prefix-testhost-20100-1414141414-1"))
	})

	It("should increment correctly", func() {
		cGUID.inc = 0xffffffff - 1
		Expect(NewGUIDAt("prefix", time.Unix(1313131313, 0))).To(Equal("prefix-testhost-20100-1313131313-4294967295"))
		Expect(NewGUIDAt("prefix", time.Unix(1313131313, 0))).To(Equal("prefix-testhost-20100-1313131313-0"))
	})
})

/*********************************************************************
 * TEST HOOK
 *********************************************************************/

const (
	t_KAFKA_VERSION = "kafka_2.10-0.8.1.1"
	t_CLIENT        = "sarama-cluster-client"
	t_TOPIC         = "sarama-cluster-topic"
	t_GROUP         = "sarama-cluster-group"
	t_DIR           = "/tmp/sarama-cluster-test"
)

var _ = BeforeSuite(func() {
	runner := testDir(t_KAFKA_VERSION, "bin", "kafka-run-class.sh")
	testState.zookeeper = exec.Command(runner, "-name", "zookeeper", "org.apache.zookeeper.server.ZooKeeperServerMain", testDir("zookeeper.properties"))
	testState.kafka = exec.Command(runner, "-name", "kafkaServer", "kafka.Kafka", testDir("server.properties"))
	testState.kafka.Env = []string{"KAFKA_HEAP_OPTS=-Xmx1G -Xms1G"}

	// Create Dir
	Expect(os.MkdirAll(t_DIR, 0775)).NotTo(HaveOccurred())

	// Start ZK
	Expect(testState.zookeeper.Start()).NotTo(HaveOccurred())
	Eventually(func() *os.Process {
		return testState.zookeeper.Process
	}).ShouldNot(BeNil())

	// Start Kafka
	Expect(testState.kafka.Start()).NotTo(HaveOccurred())
	Eventually(func() *os.Process {
		return testState.kafka.Process
	}).ShouldNot(BeNil())

	// Create partition
	time.Sleep(3 * time.Second)
	err := exec.Command(testDir(t_KAFKA_VERSION, "bin", "kafka-topics.sh"),
		"--create",
		"--topic", t_TOPIC,
		"--zookeeper", "127.0.0.1:22181",
		"--partitions", "12",
		"--replication-factor", "1",
	).Run()
	Expect(err).NotTo(HaveOccurred())
	time.Sleep(3 * time.Second)

	// Seed messages
	Expect(seedMessages(10000)).NotTo(HaveOccurred())
})

var _ = AfterSuite(func() {
	if testState.kafka != nil {
		testState.kafka.Process.Kill()
	}
	if testState.zookeeper != nil {
		testState.zookeeper.Process.Kill()
	}
	Expect(os.RemoveAll(t_DIR)).NotTo(HaveOccurred())
})

func TestSuite(t *testing.T) {
	RegisterFailHandler(Fail)
	BeforeEach(func() {
		testState.notifier = &mockNotifier{messages: make([]string, 0)}
	})
	RunSpecs(t, "sarama/cluster")
}

/*******************************************************************
 * TEST HELPERS
 *******************************************************************/

var testState struct {
	kafka, zookeeper *exec.Cmd
	notifier         *mockNotifier
}

func newClient() (*sarama.Client, error) {
	return sarama.NewClient(t_CLIENT, []string{"127.0.0.1:29092"}, sarama.NewClientConfig())
}

func testDir(tokens ...string) string {
	_, filename, _, _ := runtime.Caller(1)
	tokens = append([]string{path.Dir(filename), "test"}, tokens...)
	return path.Join(tokens...)
}

func testConsumerConfig() *sarama.ConsumerConfig {
	return sarama.NewConsumerConfig()
}

func seedMessages(count int) error {
	client, err := newClient()
	if err != nil {
		return err
	}
	defer client.Close()

	producer, err := sarama.NewSimpleProducer(client, t_TOPIC, nil)
	if err != nil {
		return err
	}
	defer producer.Close()

	for i := 0; i < count; i++ {
		err := producer.SendMessage(nil, sarama.StringEncoder(fmt.Sprintf("PLAINDATA-%08d", i)))
		if err != nil {
			return err
		}
	}
	return nil
}

type mockNotifier struct{ messages []string }

func (n *mockNotifier) RebalanceStart(cg *ConsumerGroup) {
	n.messages = append(n.messages, fmt.Sprintf("rebalance start %s", cg.Name()))
}
func (n *mockNotifier) RebalanceOK(cg *ConsumerGroup) {
	n.messages = append(n.messages, fmt.Sprintf("rebalance ok %s", cg.Name()))
}
func (n *mockNotifier) RebalanceError(cg *ConsumerGroup, err error) {
	n.messages = append(n.messages, fmt.Sprintf("rebalance error %s: %s", cg.Name(), err.Error()))
}
<commit_msg>More reliable tests<commit_after>package cluster

import (
	"fmt"
	"os"
	"os/exec"
	"path"
	"runtime"
	"sort"
	"testing"
	"time"

	"github.com/Shopify/sarama"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("PartitionSlice", func() {

	It("should sort correctly", func() {
		p1 := Partition{Addr: "host1:9093", ID: 1}
		p2 := Partition{Addr: "host1:9092", ID: 2}
		p3 := Partition{Addr: "host2:9092", ID: 3}
		p4 := Partition{Addr: "host3:9091", ID: 4}
		p5 := Partition{Addr: "host2:9093", ID: 5}
		p6 := Partition{Addr: "host1:9092", ID: 6}

		slice := PartitionSlice{p1, p2, p3, p4, p5, p6}
		sort.Sort(slice)
		Expect(slice).To(BeEquivalentTo(PartitionSlice{p2, p6, p1, p3, p5, p4}))
	})

})

var _ = Describe("GUID", func() {

	BeforeEach(func() {
		cGUID.hostname = "testhost"
		cGUID.pid = 20100
	})

	AfterEach(func() {
		cGUID.inc = 0
	})

	It("should create GUIDs", func() {
		cGUID.inc = 0xffffffff
		Expect(NewGUIDAt("prefix", time.Unix(1313131313, 0))).To(Equal("prefix-testhost-20100-1313131313-0"))
		Expect(NewGUIDAt("prefix", time.Unix(1414141414, 0))).To(Equal("prefix-testhost-20100-1414141414-1"))
	})

	It("should increment correctly", func() {
		cGUID.inc = 0xffffffff - 1
		Expect(NewGUIDAt("prefix", time.Unix(1313131313, 0))).To(Equal("prefix-testhost-20100-1313131313-4294967295"))
		Expect(NewGUIDAt("prefix", time.Unix(1313131313, 0))).To(Equal("prefix-testhost-20100-1313131313-0"))
	})
})

/*********************************************************************
 * TEST HOOK
 *********************************************************************/

const (
	t_KAFKA_VERSION = "kafka_2.10-0.8.1.1"
	t_CLIENT        = "sarama-cluster-client"
	t_TOPIC         = "sarama-cluster-topic"
	t_GROUP         = "sarama-cluster-group"
	t_DIR           = "/tmp/sarama-cluster-test"
)

var _ = BeforeSuite(func() {
	runner := testDir(t_KAFKA_VERSION, "bin", "kafka-run-class.sh")
	testState.zookeeper = exec.Command(runner, "-name", "zookeeper", "org.apache.zookeeper.server.ZooKeeperServerMain", testDir("zookeeper.properties"))
	testState.kafka = exec.Command(runner, "-name", "kafkaServer", "kafka.Kafka", testDir("server.properties"))
	testState.kafka.Env = []string{"KAFKA_HEAP_OPTS=-Xmx1G -Xms1G"}

	// Create Dir
	Expect(os.MkdirAll(t_DIR, 0775)).NotTo(HaveOccurred())

	// Start ZK
	Expect(testState.zookeeper.Start()).NotTo(HaveOccurred())
	Eventually(func() *os.Process {
		return testState.zookeeper.Process
	}).ShouldNot(BeNil())

	// Start Kafka
	Expect(testState.kafka.Start()).NotTo(HaveOccurred())
	Eventually(func() *os.Process {
		return testState.kafka.Process
	}).ShouldNot(BeNil())

	// Create partition
	time.Sleep(1 * time.Second)
	err := exec.Command(testDir(t_KAFKA_VERSION, "bin", "kafka-topics.sh"),
		"--create",
		"--topic", t_TOPIC,
		"--zookeeper", "127.0.0.1:22181",
		"--partitions", "12",
		"--replication-factor", "1",
	).Run()
	Expect(err).NotTo(HaveOccurred())

	// Create and wait for client
	client, err := newClient()
	Expect(err).NotTo(HaveOccurred())
	defer client.Close()

	Eventually(func() error {
		_, err := client.Partitions(t_TOPIC)
		return err
	}).ShouldNot(HaveOccurred(), "10s")

	// Seed messages
	Expect(seedMessages(client, 10000)).NotTo(HaveOccurred())
})

var _ = AfterSuite(func() {
	if testState.kafka != nil {
		testState.kafka.Process.Kill()
	}
	if testState.zookeeper != nil {
		testState.zookeeper.Process.Kill()
	}
	Expect(os.RemoveAll(t_DIR)).NotTo(HaveOccurred())
})

func TestSuite(t *testing.T) {
	RegisterFailHandler(Fail)
	BeforeEach(func() {
		testState.notifier = &mockNotifier{messages: make([]string, 0)}
	})
	RunSpecs(t, "sarama/cluster")
}

/*******************************************************************
 * TEST HELPERS
 *******************************************************************/

var testState struct {
	kafka, zookeeper *exec.Cmd
	notifier         *mockNotifier
}

func newClient() (*sarama.Client, error) {
	return sarama.NewClient(t_CLIENT, []string{"127.0.0.1:29092"}, sarama.NewClientConfig())
}

func testDir(tokens ...string) string {
	_, filename, _, _ := runtime.Caller(1)
	tokens = append([]string{path.Dir(filename), "test"}, tokens...)
	return path.Join(tokens...)
}

func testConsumerConfig() *sarama.ConsumerConfig {
	return sarama.NewConsumerConfig()
}

func seedMessages(client *sarama.Client, count int) error {
	producer, err := sarama.NewSimpleProducer(client, t_TOPIC, nil)
	if err != nil {
		return err
	}
	defer producer.Close()

	for i := 0; i < count; i++ {
		err := producer.SendMessage(nil, sarama.StringEncoder(fmt.Sprintf("PLAINDATA-%08d", i)))
		if err != nil {
			return err
		}
	}
	return nil
}

type mockNotifier struct{ messages []string }

func (n *mockNotifier) RebalanceStart(cg *ConsumerGroup) {
	n.messages = append(n.messages, fmt.Sprintf("rebalance start %s", cg.Name()))
}
func (n *mockNotifier) RebalanceOK(cg *ConsumerGroup) {
	n.messages = append(n.messages, fmt.Sprintf("rebalance ok %s", cg.Name()))
}
func (n *mockNotifier) RebalanceError(cg *ConsumerGroup, err error) {
	n.messages = append(n.messages, fmt.Sprintf("rebalance error %s: %s", cg.Name(), err.Error()))
}
<|endoftext|>