<commit_before>package fap

/*
#cgo LDFLAGS: -lfap
#include <stdlib.h>
#include <string.h>
#include <fap.h>

char* str_at(char **lst, int idx) {
    return lst[idx];
}
char* new_c_str(uint size) {
    return (char*) malloc( size*sizeof(char) );
}
fap_packet_type_t* packet_type(fap_packet_t *p) {
    return p->type;
}
*/
import "C"

import (
	"errors"
	"log"
	"time"
	"unsafe"
)

const (
	UNKNOWN          = 0
	LOCATION         = C.fapLOCATION
	OBJECT           = C.fapOBJECT
	ITEM             = C.fapITEM
	MICE             = C.fapMICE
	NMEA             = C.fapNMEA
	WX               = C.fapWX
	MESSAGE          = C.fapMESSAGE
	CAPABILITIES     = C.fapCAPABILITIES
	STATUS           = C.fapSTATUS
	TELEMETRY        = C.fapTELEMETRY
	TELEMETRYMESSAGE = C.fapTELEMETRY_MESSAGE
	DXSPOT           = C.fapDX_SPOT
	EXPERIMENTAL     = C.fapEXPERIMENTAL
)

const (
	POSUNKNOWN      = 0
	POSCOMPRESSED   = C.fapPOS_COMPRESSED
	POSUNCOMPRESSED = C.fapPOS_UNCOMPRESSED
	POSMICE         = C.fapPOS_MICE
	POSNMEA         = C.fapPOS_NMEA
)

func init() {
	C.fap_init()
}

func Cleanup() {
	C.fap_cleanup()
}

func ParseAprs(input string, isAX25 bool) (*FapPacket, error) {
	c_input := C.CString(input)
	defer C.free(unsafe.Pointer(c_input))

	c_len := C.uint(C.strlen(c_input))

	var c_isAX25 C.short
	if isAX25 {
		c_isAX25 = 1
	}

	c_fapPacket := C.fap_parseaprs(c_input, c_len, c_isAX25)
	defer C.fap_free(c_fapPacket)

	if c_fapPacket == nil {
		log.Fatal("fap_parseaprs returned nil. Is libfap initialized?")
	}

	fapPacket, err := c_fapPacket.goFapPacket()

	return fapPacket, err
}

func Distance(lon0, lat0, lon1, lat1 float64) float64 {
	c_dist := C.fap_distance(
		C.double(lon0), C.double(lat0),
		C.double(lon1), C.double(lat1),
	)

	return float64(c_dist)
}

func Direction(lon0, lat0, lon1, lat1 float64) float64 {
	c_dir := C.fap_direction(
		C.double(lon0), C.double(lat0),
		C.double(lon1), C.double(lat1),
	)

	return float64(c_dir)
}

func MicEMbitsToMessage(mbits string) string {
	if mbits == "" {
		log.Fatal("MicEMbitsToMessage() called with empty string")
	}

	buffer := C.new_c_str(60)
	defer C.free(unsafe.Pointer(buffer))

	C.fap_mice_mbits_to_message(C.CString(mbits), buffer)

	return C.GoString(buffer)
}

func (c *_Ctype_fap_packet_t) goFapPacket() (*FapPacket, error) {
	err := c.error()

	packet := FapPacket{
		// error_code (removed)
		// type -> PacketType (set below)

		OrigPacket: goString(c.orig_packet),
		// orig_packet_len (removed)

		Header: goString(c.header),
		Body:   goString(c.body),
		// body_len (removed)

		SrcCallsign: goString(c.src_callsign),
		DstCallsign: goString(c.dst_callsign),
		// path (set below)
		// path_len (removed)

		Latitude:      goFloat64(c.latitude),
		Longitude:     goFloat64(c.longitude),
		PosResolution: goFloat64(c.pos_resolution),
		PosAmbiguity:  goUnsignedInt(c.pos_ambiguity),
		DaoDatumByte:  byte(c.dao_datum_byte), // 0x00 = undef
		Altitude:      goFloat64(c.altitude),
		Course:        goUnsignedInt(c.course),
		Speed:         goFloat64(c.speed),

		SymbolTable: byte(c.symbol_table), // 0x00 = undef
		SymbolCode:  byte(c.symbol_code),  // 0x00 = undef

		Messaging:   goBool(c.messaging),
		Destination: goString(c.destination),
		Message:     goString(c.message),
		MessageAck:  goString(c.message_ack),
		MessageNack: goString(c.message_nack),
		MessageId:   goString(c.message_id),

		// comment (set below)
		// comment_len (removed)

		ObjectOrItemName: goString(c.object_or_item_name),
		Alive:            goBool(c.alive),
		GpsFixStatus:     goBool(c.gps_fix_status),
		RadioRange:       goUnsignedInt(c.radio_range),
		Phg:              goString(c.phg),

		// timestamp (set below)
		NmeaChecksumOk: goBool(c.nmea_checksum_ok),

		// wx_report (TODO)
		// telemetry (TODO)

		Messagebits: goString(c.messagebits),

		// status (set below)

		// capabilities (TODO)
		// capabilities_len (removed)
	}

	if C.packet_type(c) != nil {
		packet.PacketType = uint(*C.packet_type(c))
	}
	if c.format != nil {
		packet.Format = uint(*c.format)
	}
	if c.status != nil {
		packet.Status = C.GoStringN(c.status, C.int(c.status_len))
	}
	if c.comment != nil {
		packet.Comment = C.GoStringN(c.comment, C.int(c.comment_len))
	}
	if c.timestamp != nil {
		packet.Timestamp = time.Unix(int64(*c.timestamp), 0)
	}

	// Get path
	packet.Path = make([]string, int(c.path_len))
	for i := 0; i < int(c.path_len); i++ {
		packet.Path[i] = goString(C.str_at(c.path, C.int(i)))
	}

	return &packet, err
}

func (c *_Ctype_fap_packet_t) error() error {
	if c.error_code == nil {
		return nil
	}

	buffer := C.new_c_str(64)
	defer C.free(unsafe.Pointer(buffer))

	C.fap_explain_error(*c.error_code, buffer)

	return errors.New(C.GoString(buffer))
}
<commit_msg>Add some documentation<commit_after>package fap

/*
#cgo LDFLAGS: -lfap
#include <stdlib.h>
#include <string.h>
#include <fap.h>

char* str_at(char **lst, int idx) {
    return lst[idx];
}
char* new_c_str(uint size) {
    return (char*) malloc( size*sizeof(char) );
}
fap_packet_type_t* packet_type(fap_packet_t *p) {
    return p->type;
}
*/
import "C"

import (
	"errors"
	"log"
	"time"
	"unsafe"
)

const (
	UNKNOWN          = 0
	LOCATION         = C.fapLOCATION
	OBJECT           = C.fapOBJECT
	ITEM             = C.fapITEM
	MICE             = C.fapMICE
	NMEA             = C.fapNMEA
	WX               = C.fapWX
	MESSAGE          = C.fapMESSAGE
	CAPABILITIES     = C.fapCAPABILITIES
	STATUS           = C.fapSTATUS
	TELEMETRY        = C.fapTELEMETRY
	TELEMETRYMESSAGE = C.fapTELEMETRY_MESSAGE
	DXSPOT           = C.fapDX_SPOT
	EXPERIMENTAL     = C.fapEXPERIMENTAL
)

const (
	POSUNKNOWN      = 0
	POSCOMPRESSED   = C.fapPOS_COMPRESSED
	POSUNCOMPRESSED = C.fapPOS_UNCOMPRESSED
	POSMICE         = C.fapPOS_MICE
	POSNMEA         = C.fapPOS_NMEA
)

func init() {
	C.fap_init()
}

// Cleanup should be called when done using this package.
func Cleanup() {
	C.fap_cleanup()
}

// ParseAprs is the main parser method. Parses content of input
// string. Setting isAX25 to true source callsign and path
// elements are checked to be strictly compatible with AX.25
// specs so that theycan be sent into AX.25 network. Destination
// callsign is always checked this way.
func ParseAprs(input string, isAX25 bool) (*FapPacket, error) {
	c_input := C.CString(input)
	defer C.free(unsafe.Pointer(c_input))

	c_len := C.uint(C.strlen(c_input))

	var c_isAX25 C.short
	if isAX25 {
		c_isAX25 = 1
	}

	c_fapPacket := C.fap_parseaprs(c_input, c_len, c_isAX25)
	defer C.fap_free(c_fapPacket)

	if c_fapPacket == nil {
		log.Fatal("fap_parseaprs returned nil. Is libfap initialized?")
	}

	fapPacket, err := c_fapPacket.goFapPacket()

	return fapPacket, err
}

// Calculates distance between given locations,
// returning the the distance in kilometers.
func Distance(lon0, lat0, lon1, lat1 float64) float64 {
	c_dist := C.fap_distance(
		C.double(lon0), C.double(lat0),
		C.double(lon1), C.double(lat1),
	)

	return float64(c_dist)
}

// Calculates direction from first to second location.
func Direction(lon0, lat0, lon1, lat1 float64) float64 {
	c_dir := C.fap_direction(
		C.double(lon0), C.double(lat0),
		C.double(lon1), C.double(lat1),
	)

	return float64(c_dir)
}

// MicEMbitsToMessage converts mic-e message bits (three numbers 0-2)
// to a textual message.
func MicEMbitsToMessage(mbits string) string {
	if mbits == "" {
		log.Fatal("MicEMbitsToMessage() called with empty string")
	}

	buffer := C.new_c_str(60)
	defer C.free(unsafe.Pointer(buffer))

	C.fap_mice_mbits_to_message(C.CString(mbits), buffer)

	return C.GoString(buffer)
}

func (c *_Ctype_fap_packet_t) goFapPacket() (*FapPacket, error) {
	err := c.error()

	packet := FapPacket{
		// error_code (removed)
		// type -> PacketType (set below)

		OrigPacket: goString(c.orig_packet),
		// orig_packet_len (removed)

		Header: goString(c.header),
		Body:   goString(c.body),
		// body_len (removed)

		SrcCallsign: goString(c.src_callsign),
		DstCallsign: goString(c.dst_callsign),
		// path (set below)
		// path_len (removed)

		Latitude:      goFloat64(c.latitude),
		Longitude:     goFloat64(c.longitude),
		PosResolution: goFloat64(c.pos_resolution),
		PosAmbiguity:  goUnsignedInt(c.pos_ambiguity),
		DaoDatumByte:  byte(c.dao_datum_byte), // 0x00 = undef
		Altitude:      goFloat64(c.altitude),
		Course:        goUnsignedInt(c.course),
		Speed:         goFloat64(c.speed),

		SymbolTable: byte(c.symbol_table), // 0x00 = undef
		SymbolCode:  byte(c.symbol_code),  // 0x00 = undef

		Messaging:   goBool(c.messaging),
		Destination: goString(c.destination),
		Message:     goString(c.message),
		MessageAck:  goString(c.message_ack),
		MessageNack: goString(c.message_nack),
		MessageId:   goString(c.message_id),

		// comment (set below)
		// comment_len (removed)

		ObjectOrItemName: goString(c.object_or_item_name),
		Alive:            goBool(c.alive),
		GpsFixStatus:     goBool(c.gps_fix_status),
		RadioRange:       goUnsignedInt(c.radio_range),
		Phg:              goString(c.phg),

		// timestamp (set below)
		NmeaChecksumOk: goBool(c.nmea_checksum_ok),

		// wx_report (TODO)
		// telemetry (TODO)

		Messagebits: goString(c.messagebits),

		// status (set below)

		// capabilities (TODO)
		// capabilities_len (removed)
	}

	if C.packet_type(c) != nil {
		packet.PacketType = uint(*C.packet_type(c))
	}
	if c.format != nil {
		packet.Format = uint(*c.format)
	}
	if c.status != nil {
		packet.Status = C.GoStringN(c.status, C.int(c.status_len))
	}
	if c.comment != nil {
		packet.Comment = C.GoStringN(c.comment, C.int(c.comment_len))
	}
	if c.timestamp != nil {
		packet.Timestamp = time.Unix(int64(*c.timestamp), 0)
	}

	// Get path
	packet.Path = make([]string, int(c.path_len))
	for i := 0; i < int(c.path_len); i++ {
		packet.Path[i] = goString(C.str_at(c.path, C.int(i)))
	}

	return &packet, err
}

func (c *_Ctype_fap_packet_t) error() error {
	if c.error_code == nil {
		return nil
	}

	buffer := C.new_c_str(64)
	defer C.free(unsafe.Pointer(buffer))

	C.fap_explain_error(*c.error_code, buffer)

	return errors.New(C.GoString(buffer))
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package syncbaselib

import (
	"fmt"
	"os"
	"runtime/pprof"

	"v.io/v23"
	"v.io/v23/context"
	"v.io/v23/options"
	"v.io/v23/rpc"
	"v.io/x/ref/lib/security/securityflag"
	"v.io/x/ref/services/syncbase/server"
	"v.io/x/ref/services/syncbase/vsync"
)

// Serve starts the Syncbase server. Returns rpc.Server and rpc.Dispatcher for
// use in the Mojo bindings, along with a cleanup function.
func Serve(ctx *context.T, opts Opts) (rpc.Server, rpc.Dispatcher, func()) {
	// Note: Adding the "runtime/pprof" import does not significantly increase the
	// binary size (only ~4500 bytes), so it seems okay to expose the option to
	// profile.
	if opts.CpuProfile != "" {
		f, err := os.Create(opts.CpuProfile)
		if err != nil {
			ctx.Fatal("Unable to create the cpu profile file: ", err)
		}
		defer f.Close()
		if err = pprof.StartCPUProfile(f); err != nil {
			ctx.Fatal("StartCPUProfile failed: ", err)
		}
		defer pprof.StopCPUProfile()
	}

	perms, err := securityflag.PermissionsFromFlag()
	if err != nil {
		ctx.Fatal("securityflag.PermissionsFromFlag() failed: ", err)
	}
	if perms != nil {
		ctx.Infof("Read permissions from command line flag: %v", server.PermsString(ctx, perms))
	}
	service, err := server.NewService(ctx, server.ServiceOptions{
		Perms:           perms,
		RootDir:         opts.RootDir,
		Engine:          opts.Engine,
		SkipPublishInNh: opts.SkipPublishInNh,
		DevMode:         opts.DevMode,
	})
	if err != nil {
		ctx.Fatal("server.NewService() failed: ", err)
	}
	d := server.NewDispatcher(service)

	// Publish the service in the mount table.
	ctx, cancel := context.WithCancel(ctx)
	ctx, s, err := v23.WithNewDispatchingServer(ctx, opts.Name, d, options.ChannelTimeout(vsync.NeighborConnectionTimeout))
	if err != nil {
		ctx.Fatal("v23.WithNewDispatchingServer() failed: ", err)
	}

	// Publish syncgroups and such in the various places that they should be
	// published.
	// TODO(sadovsky): Improve comments (and perhaps method name) for AddNames.
	// It's not just publishing more names in the default mount table, and under
	// certain configurations it also publishes to the neighborhood.
	if err := service.AddNames(ctx, s); err != nil {
		ctx.Fatal("AddNames failed: ", err)
	}

	// Print mount name and endpoint.
	if opts.Name != "" {
		ctx.Info("Mounted at: ", opts.Name)
	}
	if eps := s.Status().Endpoints; len(eps) > 0 {
		// Integration tests wait for this to be printed before trying to access the
		// service.
		fmt.Printf("ENDPOINT=%s\n", eps[0].Name())
	}

	cleanup := func() {
		cancel()
		<-s.Closed()
		service.Close()
	}

	return s, d, cleanup
}
<commit_msg>synbase: Make server and clients in syncbase share connections.<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package syncbaselib

import (
	"fmt"
	"os"
	"runtime/pprof"

	"v.io/v23"
	"v.io/v23/context"
	"v.io/v23/options"
	"v.io/v23/rpc"
	"v.io/x/ref/lib/dispatcher"
	"v.io/x/ref/lib/security/securityflag"
	"v.io/x/ref/services/syncbase/server"
	"v.io/x/ref/services/syncbase/vsync"
)

// Serve starts the Syncbase server. Returns rpc.Server and rpc.Dispatcher for
// use in the Mojo bindings, along with a cleanup function.
func Serve(ctx *context.T, opts Opts) (rpc.Server, rpc.Dispatcher, func()) {
	// Note: Adding the "runtime/pprof" import does not significantly increase the
	// binary size (only ~4500 bytes), so it seems okay to expose the option to
	// profile.
	if opts.CpuProfile != "" {
		f, err := os.Create(opts.CpuProfile)
		if err != nil {
			ctx.Fatal("Unable to create the cpu profile file: ", err)
		}
		defer f.Close()
		if err = pprof.StartCPUProfile(f); err != nil {
			ctx.Fatal("StartCPUProfile failed: ", err)
		}
		defer pprof.StopCPUProfile()
	}

	// Create the rpc server before the service so that connections are shared between
	// clients in the service and the rpc server. (i.e. connections are shared if the
	// context returned from WithNewDispatchingServer is used for client calls).
	d := dispatcher.NewDispatcherWrapper()
	ctx, cancel := context.WithCancel(ctx)
	ctx, s, err := v23.WithNewDispatchingServer(ctx, opts.Name, d, options.ChannelTimeout(vsync.NeighborConnectionTimeout))
	if err != nil {
		ctx.Fatal("v23.WithNewDispatchingServer() failed: ", err)
	}

	perms, err := securityflag.PermissionsFromFlag()
	if err != nil {
		ctx.Fatal("securityflag.PermissionsFromFlag() failed: ", err)
	}
	if perms != nil {
		ctx.Infof("Read permissions from command line flag: %v", server.PermsString(ctx, perms))
	}
	service, err := server.NewService(ctx, server.ServiceOptions{
		Perms:           perms,
		RootDir:         opts.RootDir,
		Engine:          opts.Engine,
		SkipPublishInNh: opts.SkipPublishInNh,
		DevMode:         opts.DevMode,
	})
	if err != nil {
		ctx.Fatal("server.NewService() failed: ", err)
	}

	// Set the dispatcher in the dispatcher wrapper for the server to start responding
	// to incoming rpcs.
	d.SetDispatcher(server.NewDispatcher(service))

	// Publish syncgroups and such in the various places that they should be
	// published.
	// TODO(sadovsky): Improve comments (and perhaps method name) for AddNames.
	// It's not just publishing more names in the default mount table, and under
	// certain configurations it also publishes to the neighborhood.
	if err := service.AddNames(ctx, s); err != nil {
		ctx.Fatal("AddNames failed: ", err)
	}

	// Print mount name and endpoint.
	if opts.Name != "" {
		ctx.Info("Mounted at: ", opts.Name)
	}
	if eps := s.Status().Endpoints; len(eps) > 0 {
		// Integration tests wait for this to be printed before trying to access the
		// service.
		fmt.Printf("ENDPOINT=%s\n", eps[0].Name())
	}

	cleanup := func() {
		cancel()
		<-s.Closed()
		service.Close()
	}

	return s, d, cleanup
}
<|endoftext|>


<commit_before>package disk

import (
	"path/filepath"
	"strconv"

	"github.com/cloudfoundry/bosh-cpi-go/apiv1"
	bosherr "github.com/cloudfoundry/bosh-utils/errors"
	boshlog "github.com/cloudfoundry/bosh-utils/logger"
	boshsys "github.com/cloudfoundry/bosh-utils/system"
	boshuuid "github.com/cloudfoundry/bosh-utils/uuid"
)

type FSFactory struct {
	dirPath string

	fs        boshsys.FileSystem
	uuidGen   boshuuid.Generator
	cmdRunner boshsys.CmdRunner

	logTag string
	logger boshlog.Logger
}

func NewFSFactory(
	dirPath string,
	fs boshsys.FileSystem,
	uuidGen boshuuid.Generator,
	cmdRunner boshsys.CmdRunner,
	logger boshlog.Logger,
) FSFactory {
	return FSFactory{
		dirPath: dirPath,

		fs:        fs,
		uuidGen:   uuidGen,
		cmdRunner: cmdRunner,

		logTag: "disk.FSFactory",
		logger: logger,
	}
}

func (f FSFactory) Create(size int) (Disk, error) {
	f.logger.Debug(f.logTag, "Creating disk of size '%d'", size)

	id, err := f.uuidGen.Generate()
	if err != nil {
		return nil, bosherr.WrapError(err, "Generating disk id")
	}

	diskPath := filepath.Join(f.dirPath, id)

	err = f.fs.WriteFile(diskPath, []byte{})
	if err != nil {
		return nil, bosherr.WrapError(err, "Creating empty disk")
	}

	sizeStr := strconv.Itoa(size) + "MB"

	_, _, _, err = f.cmdRunner.RunCommand("truncate", "-s", sizeStr, diskPath)
	if err != nil {
		f.cleanUpFile(diskPath)
		return nil, bosherr.WrapErrorf(err, "Resizing disk to '%s'", sizeStr)
	}

	_, _, _, err = f.cmdRunner.RunCommand("/sbin/mkfs", "-t", "ext4", "-F", diskPath)
	if err != nil {
		f.cleanUpFile(diskPath)
		return nil, bosherr.WrapErrorf(err, "Building disk filesystem '%s'", diskPath)
	}

	return NewFSDisk(apiv1.NewDiskCID(id), diskPath, f.fs, f.logger), nil
}

func (f FSFactory) Find(id apiv1.DiskCID) (Disk, error) {
	return NewFSDisk(id, filepath.Join(f.dirPath, id.AsString()), f.fs, f.logger), nil
}

func (f FSFactory) cleanUpFile(path string) {
	err := f.fs.RemoveAll(path)
	if err != nil {
		f.logger.Error(f.logTag, "Failed deleting file '%s': %s", path, err.Error())
	}
}
<commit_msg>update to correct disk calcuation<commit_after>package disk

import (
	"path/filepath"
	"strconv"

	"github.com/cloudfoundry/bosh-cpi-go/apiv1"
	bosherr "github.com/cloudfoundry/bosh-utils/errors"
	boshlog "github.com/cloudfoundry/bosh-utils/logger"
	boshsys "github.com/cloudfoundry/bosh-utils/system"
	boshuuid "github.com/cloudfoundry/bosh-utils/uuid"
)

type FSFactory struct {
	dirPath string

	fs        boshsys.FileSystem
	uuidGen   boshuuid.Generator
	cmdRunner boshsys.CmdRunner

	logTag string
	logger boshlog.Logger
}

func NewFSFactory(
	dirPath string,
	fs boshsys.FileSystem,
	uuidGen boshuuid.Generator,
	cmdRunner boshsys.CmdRunner,
	logger boshlog.Logger,
) FSFactory {
	return FSFactory{
		dirPath: dirPath,

		fs:        fs,
		uuidGen:   uuidGen,
		cmdRunner: cmdRunner,

		logTag: "disk.FSFactory",
		logger: logger,
	}
}

func (f FSFactory) Create(size int) (Disk, error) {
	f.logger.Debug(f.logTag, "Creating disk of size '%d'", size)

	id, err := f.uuidGen.Generate()
	if err != nil {
		return nil, bosherr.WrapError(err, "Generating disk id")
	}

	diskPath := filepath.Join(f.dirPath, id)

	err = f.fs.WriteFile(diskPath, []byte{})
	if err != nil {
		return nil, bosherr.WrapError(err, "Creating empty disk")
	}

	sizeStr := strconv.Itoa(size) + "M"

	_, _, _, err = f.cmdRunner.RunCommand("truncate", "-s", sizeStr, diskPath)
	if err != nil {
		f.cleanUpFile(diskPath)
		return nil, bosherr.WrapErrorf(err, "Resizing disk to '%s'", sizeStr)
	}

	_, _, _, err = f.cmdRunner.RunCommand("/sbin/mkfs", "-t", "ext4", "-F", diskPath)
	if err != nil {
		f.cleanUpFile(diskPath)
		return nil, bosherr.WrapErrorf(err, "Building disk filesystem '%s'", diskPath)
	}

	return NewFSDisk(apiv1.NewDiskCID(id), diskPath, f.fs, f.logger), nil
}

func (f FSFactory) Find(id apiv1.DiskCID) (Disk, error) {
	return NewFSDisk(id, filepath.Join(f.dirPath, id.AsString()), f.fs, f.logger), nil
}

func (f FSFactory) cleanUpFile(path string) {
	err := f.fs.RemoveAll(path)
	if err != nil {
		f.logger.Error(f.logTag, "Failed deleting file '%s': %s", path, err.Error())
	}
}
<|endoftext|>


<commit_before>package arangodb

import (
	"context"
	"crypto/tls"
	"fmt"
	"strings"
	"time"

	"github.com/arangodb/go-driver"
	"github.com/arangodb/go-driver/http"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func init() {
	filer.Stores = append(filer.Stores, &ArangodbStore{})
}

type ArangodbStore struct {
	connect    driver.Connection
	client     driver.Client
	database   driver.Database
	collection driver.Collection

	databaseName string
}

type Model struct {
	Key       string `json:"_key"`
	Directory string `json:"directory,omitempty"`
	Name      string `json:"name,omitempty"`
	Bucket    string `json:"bucket,omitempty"`
	Ttl       string `json:"ttl,omitempty"`

	//arangodb does not support binary blobs
	//we encode byte slice into uint64 slice
	//see helpers.go
	Meta []uint64 `json:"meta"`
}

func (store *ArangodbStore) GetName() string {
	return "arangodb"
}

func (store *ArangodbStore) Initialize(configuration util.Configuration, prefix string) (err error) {
	store.databaseName = configuration.GetString(prefix + "db_name")
	return store.connection(configuration.GetStringSlice(prefix+"servers"),
		configuration.GetString(prefix+"user"),
		configuration.GetString(prefix+"pass"),
		configuration.GetBool(prefix+"insecure_skip_verify"),
	)
}

func (store *ArangodbStore) connection(uris []string, user string, pass string, insecure bool) (err error) {
	ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)

	store.connect, err = http.NewConnection(http.ConnectionConfig{
		Endpoints: uris,
		TLSConfig: &tls.Config{
			InsecureSkipVerify: insecure,
		},
	})
	if err != nil {
		return err
	}
	store.client, err = driver.NewClient(driver.ClientConfig{
		Connection:     store.connect,
		Authentication: driver.BasicAuthentication(user, pass),
	})
	if err != nil {
		return err
	}
	ok, err := store.client.DatabaseExists(ctx, store.databaseName)
	if err != nil {
		return err
	}
	if ok {
		store.database, err = store.client.Database(ctx, store.databaseName)
	} else {
		store.database, err = store.client.CreateDatabase(ctx, store.databaseName, &driver.CreateDatabaseOptions{})
	}
	if err != nil {
		return err
	}

	coll_name := "files"
	ok, err = store.database.CollectionExists(ctx, coll_name)
	if err != nil {
		return err
	}
	if ok {
		store.collection, err = store.database.Collection(ctx, coll_name)
	} else {
		store.collection, err = store.database.CreateCollection(ctx, coll_name, &driver.CreateCollectionOptions{})
	}
	if err != nil {
		return err
	}

	// ensure indices

	if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"directory", "name"},
		&driver.EnsurePersistentIndexOptions{
			Name:   "directory_name_multi",
			Unique: true,
		}); err != nil {
		return err
	}
	if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"directory"},
		&driver.EnsurePersistentIndexOptions{Name: "IDX_directory"}); err != nil {
		return err
	}

	if _, _, err = store.collection.EnsureTTLIndex(ctx, "ttl", 1,
		&driver.EnsureTTLIndexOptions{Name: "IDX_TTL"}); err != nil {
		return err
	}

	if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"name"}, &driver.EnsurePersistentIndexOptions{
		Name: "IDX_name",
	}); err != nil {
		return err
	}
	if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"bucket"}, &driver.EnsurePersistentIndexOptions{
		Name:   "IDX_bucket",
		Sparse: true, //sparse index, to locate files of bucket
	}); err != nil {
		return err
	}
	return err
}

type key int

const (
	transactionKey key = 0
)

func (store *ArangodbStore) BeginTransaction(ctx context.Context) (context.Context, error) {
	txn, err := store.database.BeginTransaction(ctx, driver.TransactionCollections{
		Exclusive: []string{"files"},
	}, &driver.BeginTransactionOptions{})
	if err != nil {
		return nil, err
	}

	return context.WithValue(ctx, transactionKey, txn), nil
}

func (store *ArangodbStore) CommitTransaction(ctx context.Context) error {
	val := ctx.Value(transactionKey)
	cast, ok := val.(driver.TransactionID)
	if !ok {
		return fmt.Errorf("txn cast fail %s:", val)
	}
	err := store.database.CommitTransaction(ctx, cast, &driver.CommitTransactionOptions{})
	if err != nil {
		return err
	}
	return nil
}

func (store *ArangodbStore) RollbackTransaction(ctx context.Context) error {
	val := ctx.Value(transactionKey)
	cast, ok := val.(driver.TransactionID)
	if !ok {
		return fmt.Errorf("txn cast fail %s:", val)
	}
	err := store.database.AbortTransaction(ctx, cast, &driver.AbortTransactionOptions{})
	if err != nil {
		return err
	}
	return nil
}

func (store *ArangodbStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
	dir, name := entry.FullPath.DirAndName()
	meta, err := entry.EncodeAttributesAndChunks()
	if err != nil {
		return fmt.Errorf("encode %s: %s", entry.FullPath, err)
	}

	if len(entry.Chunks) > 50 {
		meta = util.MaybeGzipData(meta)
	}
	bucket, _ := extractBucket(entry.FullPath)
	model := &Model{
		Key:       hashString(string(entry.FullPath)),
		Directory: dir,
		Name:      name,
		Meta:      bytesToArray(meta),
		Bucket:    bucket,
	}
	if entry.TtlSec > 0 {
		model.Ttl = time.Now().Add(time.Second * time.Duration(entry.TtlSec)).Format(time.RFC3339)
	} else {
		model.Ttl = ""
	}
	_, err = store.collection.CreateDocument(ctx, model)
	if driver.IsConflict(err) {
		return store.UpdateEntry(ctx, entry)
	}
	if err != nil {
		return fmt.Errorf("InsertEntry %s: %v", entry.FullPath, err)
	}

	return nil

}

func extractBucket(fullpath util.FullPath) (string, string) {
	if !strings.HasPrefix(string(fullpath), "/buckets/") {
		return "", string(fullpath)
	}
	bucketAndObjectKey := string(fullpath)[len("/buckets/"):]
	t := strings.Index(bucketAndObjectKey, "/")
	bucket := bucketAndObjectKey
	shortPath := "/"
	if t > 0 {
		bucket = bucketAndObjectKey[:t]
		shortPath = string(util.FullPath(bucketAndObjectKey[t:]))
	}
	return bucket, shortPath
}

func (store *ArangodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
	dir, name := entry.FullPath.DirAndName()
	meta, err := entry.EncodeAttributesAndChunks()
	if err != nil {
		return fmt.Errorf("encode %s: %s", entry.FullPath, err)
	}

	if len(entry.Chunks) > 50 {
		meta = util.MaybeGzipData(meta)
	}
	model := &Model{
		Key:       hashString(string(entry.FullPath)),
		Directory: dir,
		Name:      name,
		Meta:      bytesToArray(meta),
	}
	if entry.TtlSec > 0 {
		model.Ttl = time.Now().Add(time.Duration(entry.TtlSec) * time.Second).Format(time.RFC3339)
	} else {
		model.Ttl = "none"
	}

	_, err = store.collection.UpdateDocument(ctx, model.Key, model)

	if err != nil {
		return fmt.Errorf("UpdateEntry %s: %v", entry.FullPath, err)
	}

	return nil
}

func (store *ArangodbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
	var data Model
	_, err = store.collection.ReadDocument(ctx, hashString(string(fullpath)), &data)
	if driver.IsNotFound(err) {
		return nil, filer_pb.ErrNotFound
	}
	if err != nil {
		glog.Errorf("find %s: %v", fullpath, err)
		return nil, filer_pb.ErrNotFound
	}
	if len(data.Meta) == 0 {
		return nil, filer_pb.ErrNotFound
	}
	entry = &filer.Entry{
		FullPath: fullpath,
	}
	err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(arrayToBytes(data.Meta)))
	if err != nil {
		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
	}

	return entry, nil
}

func (store *ArangodbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) error {
	_, err := store.collection.RemoveDocument(ctx, hashString(string(fullpath)))
	if err != nil && !driver.IsNotFound(err) {
		glog.Errorf("find %s: %v", fullpath, err)
		return fmt.Errorf("delete %s : %v", fullpath, err)
	}
	return nil
}

func (store *ArangodbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error {
	var query string
	query = query + fmt.Sprintf(`filter starts_with(d.directory, "%s") remove d._key in files`,
		strings.Join(strings.Split(string(fullpath), "/"), ","),
		string(fullpath),
	)
	cur, err := store.database.Query(ctx, query, nil)
	if err != nil {
		return fmt.Errorf("delete %s : %v", fullpath, err)
	}
	defer cur.Close()
	return nil
}

func (store *ArangodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
	// if no prefix, then dont use index
	if prefix == "" {
		return store.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, eachEntryFunc)
	}
	eq := ""
	if includeStartFile {
		eq = "filter d.name >= \"" + startFileName + "\""
	} else {
		eq = "filter d.name > \"" + startFileName + "\""
	}
	query := fmt.Sprintf(`
for d in files
filter d.directory == @dir
filter starts_with(d.name, @prefix)
%s
sort d.name asc
limit %d
return d`, eq, limit)
	cur, err := store.database.Query(ctx, query, map[string]interface{}{"dir": dirPath, "prefix": prefix})
	if err != nil {
		return lastFileName, fmt.Errorf("failed to list directory entries: find error: %w", err)
	}
	defer cur.Close()
	for cur.HasMore() {
		var data Model
		_, err = cur.ReadDocument(ctx, &data)
		if err != nil {
			break
		}
		entry := &filer.Entry{
			FullPath: util.NewFullPath(data.Directory, data.Name),
		}
		lastFileName = data.Name
		converted := arrayToBytes(data.Meta)
		if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(converted)); decodeErr != nil {
			err = decodeErr
			glog.V(0).Infof("list %s : %v", entry.FullPath, err)
			break
		}

		if !eachEntryFunc(entry) {
			break
		}

	}
	return lastFileName, err
}

func (store *ArangodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
	eq := ""
	if includeStartFile {
		eq = "filter d.name >= \"" + startFileName + "\""
	} else {
		eq = "filter d.name > \"" + startFileName + "\""
	}
	query := fmt.Sprintf(`
for d in files
filter d.directory == "%s"
%s
sort d.name asc
limit %d
return d`, string(dirPath), eq, limit)
	cur, err := store.database.Query(ctx, query, nil)
	if err != nil {
		return lastFileName, fmt.Errorf("failed to list directory entries: find error: %w", err)
	}
	defer cur.Close()
	for cur.HasMore() {
		var data Model
		_, err = cur.ReadDocument(ctx, &data)
		if err != nil {
			break
		}
		entry := &filer.Entry{
			FullPath: util.NewFullPath(string(dirPath), data.Name),
		}
		lastFileName = data.Name
		converted := arrayToBytes(data.Meta)
		if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(converted)); decodeErr != nil {
			err = decodeErr
			glog.V(0).Infof("list %s : %v", entry.FullPath, err)
			break
		}

		if !eachEntryFunc(entry) {
			break
		}

	}
	return lastFileName, err
}

func (store *ArangodbStore) Shutdown() {
}
<commit_msg>put in delete folder children query<commit_after>package arangodb

import (
	"context"
	"crypto/tls"
	"fmt"
	"strings"
	"time"

	"github.com/arangodb/go-driver"
	"github.com/arangodb/go-driver/http"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func init() {
	filer.Stores = append(filer.Stores, &ArangodbStore{})
}

type ArangodbStore struct {
	connect    driver.Connection
	client     driver.Client
	database   driver.Database
	collection driver.Collection

	databaseName string
}

type Model struct {
	Key       string `json:"_key"`
	Directory string `json:"directory,omitempty"`
	Name      string `json:"name,omitempty"`
	Bucket    string `json:"bucket,omitempty"`
	Ttl       string `json:"ttl,omitempty"`

	//arangodb does not support binary blobs
	//we encode byte slice into uint64 slice
	//see helpers.go
	Meta []uint64 `json:"meta"`
}

func (store *ArangodbStore) GetName() string {
	return "arangodb"
}

func (store *ArangodbStore) Initialize(configuration util.Configuration, prefix string) (err error) {
	store.databaseName = configuration.GetString(prefix + "db_name")
	return store.connection(configuration.GetStringSlice(prefix+"servers"),
		configuration.GetString(prefix+"user"),
		configuration.GetString(prefix+"pass"),
		configuration.GetBool(prefix+"insecure_skip_verify"),
	)
}

func (store *ArangodbStore) connection(uris []string, user string, pass string, insecure bool) (err error) {
	ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)

	store.connect, err = http.NewConnection(http.ConnectionConfig{
		Endpoints: uris,
		TLSConfig: &tls.Config{
			InsecureSkipVerify: insecure,
		},
	})
	if err != nil {
		return err
	}
	store.client, err = driver.NewClient(driver.ClientConfig{
		Connection:     store.connect,
		Authentication: driver.BasicAuthentication(user, pass),
	})
	if err != nil {
		return err
	}
	ok, err := store.client.DatabaseExists(ctx, store.databaseName)
	if err != nil {
		return err
	}
	if ok {
		store.database, err = store.client.Database(ctx, store.databaseName)
	} else {
		store.database, err = store.client.CreateDatabase(ctx, store.databaseName, &driver.CreateDatabaseOptions{})
	}
	if err != nil {
		return err
	}

	coll_name := "files"
	ok, err = store.database.CollectionExists(ctx, coll_name)
	if err != nil {
		return err
	}
	if ok {
		store.collection, err = store.database.Collection(ctx, coll_name)
	} else {
		store.collection, err = store.database.CreateCollection(ctx, coll_name, &driver.CreateCollectionOptions{})
	}
	if err != nil {
		return err
	}

	// ensure indices

	if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"directory", "name"},
		&driver.EnsurePersistentIndexOptions{
			Name:   "directory_name_multi",
			Unique: true,
		}); err != nil {
		return err
	}
	if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"directory"},
		&driver.EnsurePersistentIndexOptions{Name: "IDX_directory"}); err != nil {
		return err
	}

	if _, _, err = store.collection.EnsureTTLIndex(ctx, "ttl", 1,
		&driver.EnsureTTLIndexOptions{Name: "IDX_TTL"}); err != nil {
		return err
	}

	if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"name"}, &driver.EnsurePersistentIndexOptions{
		Name: "IDX_name",
	}); err != nil {
		return err
	}
	if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"bucket"}, &driver.EnsurePersistentIndexOptions{
		Name:   "IDX_bucket",
		Sparse: true, //sparse index, to locate files of bucket
	}); err != nil {
		return err
	}
	return err
}

type key int

const (
	transactionKey key = 0
)

func (store *ArangodbStore) BeginTransaction(ctx context.Context) (context.Context, error) {
	txn, err := store.database.BeginTransaction(ctx, driver.TransactionCollections{
		Exclusive: []string{"files"},
	}, &driver.BeginTransactionOptions{})
	if err != nil {
		return nil, err
	}

	return context.WithValue(ctx, transactionKey, txn), nil
}

func (store *ArangodbStore) CommitTransaction(ctx context.Context) error {
	val := ctx.Value(transactionKey)
	cast, ok := val.(driver.TransactionID)
	if !ok {
		return fmt.Errorf("txn cast fail %s:", val)
	}
	err := store.database.CommitTransaction(ctx, cast, &driver.CommitTransactionOptions{})
	if err != nil {
		return err
	}
	return nil
}

func (store *ArangodbStore) RollbackTransaction(ctx context.Context) error {
	val := ctx.Value(transactionKey)
	cast, ok := val.(driver.TransactionID)
	if !ok {
		return fmt.Errorf("txn cast fail %s:", val)
	}
	err := store.database.AbortTransaction(ctx, cast, &driver.AbortTransactionOptions{})
	if err != nil {
		return err
	}
	return nil
}

func (store *ArangodbStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
	dir, name := entry.FullPath.DirAndName()
	meta, err := entry.EncodeAttributesAndChunks()
	if err != nil {
		return fmt.Errorf("encode %s: %s", entry.FullPath, err)
	}

	if len(entry.Chunks) > 50 {
		meta = util.MaybeGzipData(meta)
	}
	bucket, _ := extractBucket(entry.FullPath)
	model := &Model{
		Key:       hashString(string(entry.FullPath)),
		Directory: dir,
		Name:      name,
		Meta:      bytesToArray(meta),
		Bucket:    bucket,
	}
	if entry.TtlSec > 0 {
		model.Ttl = time.Now().Add(time.Second * time.Duration(entry.TtlSec)).Format(time.RFC3339)
	} else {
		model.Ttl = ""
	}
	_, err = store.collection.CreateDocument(ctx, model)
	if driver.IsConflict(err) {
		return store.UpdateEntry(ctx, entry)
	}
	if err != nil {
		return fmt.Errorf("InsertEntry %s: %v", entry.FullPath, err)
	}

	return nil

}

func extractBucket(fullpath util.FullPath) (string, string) {
	if !strings.HasPrefix(string(fullpath), "/buckets/") {
		return "", string(fullpath)
	}
	bucketAndObjectKey := string(fullpath)[len("/buckets/"):]
	t := strings.Index(bucketAndObjectKey, "/")
	bucket := bucketAndObjectKey
	shortPath := "/"
	if t > 0 {
		bucket = bucketAndObjectKey[:t]
		shortPath = string(util.FullPath(bucketAndObjectKey[t:]))
	}
	return bucket, shortPath
}

func (store *ArangodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
	dir, name := entry.FullPath.DirAndName()
	meta, err := entry.EncodeAttributesAndChunks()
	if err != nil {
		return fmt.Errorf("encode %s: %s", entry.FullPath, err)
	}

	if len(entry.Chunks) > 50 {
		meta = util.MaybeGzipData(meta)
	}
	model := &Model{
		Key:       hashString(string(entry.FullPath)),
		Directory: dir,
		Name:      name,
		Meta:      bytesToArray(meta),
	}
	if entry.TtlSec > 0 {
		model.Ttl = time.Now().Add(time.Duration(entry.TtlSec) * time.Second).Format(time.RFC3339)
	} else {
		model.Ttl = "none"
	}

	_, err = store.collection.UpdateDocument(ctx, model.Key, model)

	if err != nil {
		return fmt.Errorf("UpdateEntry %s: %v", entry.FullPath, err)
	}

	return nil
}

func (store *ArangodbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
	var data Model
	_, err = store.collection.ReadDocument(ctx, hashString(string(fullpath)), &data)
	if driver.IsNotFound(err) {
		return nil, filer_pb.ErrNotFound
	}
	if err != nil {
		glog.Errorf("find %s: %v", fullpath, err)
		return nil, filer_pb.ErrNotFound
	}
	if len(data.Meta) == 0 {
		return nil, filer_pb.ErrNotFound
	}
	entry = &filer.Entry{
		FullPath: fullpath,
	}
	err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(arrayToBytes(data.Meta)))
	if err != nil {
		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
	}

	return entry, nil
}

func (store *ArangodbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) error {
	_, err := store.collection.RemoveDocument(ctx, hashString(string(fullpath)))
	if err != nil && !driver.IsNotFound(err) {
		glog.Errorf("find %s: %v", fullpath, err)
		return fmt.Errorf("delete %s : %v", fullpath, err)
	}
	return nil
}

func (store *ArangodbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error {
	var query string
	query = query + fmt.Sprintf(`
	for d in files
	filter starts_with(d.directory, "%s/")  || d.directory == "%s"
	remove d._key in files`,
		strings.Join(strings.Split(string(fullpath), "/"), ","),
		string(fullpath),
	)
	cur, err := store.database.Query(ctx, query, nil)
	if err != nil {
		return fmt.Errorf("delete %s : %v", fullpath, err)
	}
	defer cur.Close()
	return nil
}

func (store *ArangodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
	// if no prefix, then dont use index
	if prefix == "" {
		return store.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, eachEntryFunc)
	}
	eq := ""
	if includeStartFile {
		eq = "filter d.name >= \"" + startFileName + "\""
	} else {
		eq = "filter d.name > \"" + startFileName + "\""
	}
	query := fmt.Sprintf(`
for d in files
filter d.directory == @dir
filter starts_with(d.name, @prefix)
%s
sort d.name asc
limit %d
return d`, eq, limit)
	cur, err := store.database.Query(ctx, query, map[string]interface{}{"dir": dirPath, "prefix": prefix})
	if err != nil {
		return lastFileName, fmt.Errorf("failed to list directory entries: find error: %w", err)
	}
	defer cur.Close()
	for cur.HasMore() {
		var data Model
		_, err = cur.ReadDocument(ctx, &data)
		if err != nil {
			break
		}
		entry := &filer.Entry{
			FullPath: util.NewFullPath(data.Directory, data.Name),
		}
		lastFileName = data.Name
		converted := arrayToBytes(data.Meta)
		if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(converted)); decodeErr != nil {
			err = decodeErr
			glog.V(0).Infof("list %s : %v", entry.FullPath, err)
			break
		}

		if !eachEntryFunc(entry) {
			break
		}

	}
	return lastFileName, err
}

func (store *ArangodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
	eq := ""
	if includeStartFile {
		eq = "filter d.name >= \"" + startFileName + "\""
	} else {
		eq = "filter d.name > \"" + startFileName + "\""
	}
	query := fmt.Sprintf(`
for d in files
filter d.directory == "%s"
%s
sort d.name asc
limit %d
return d`, string(dirPath), eq, limit)
	cur, err := store.database.Query(ctx, query, nil)
	if err != nil {
		return lastFileName, fmt.Errorf("failed to list directory entries: find error: %w", err)
	}
	defer cur.Close()
	for cur.HasMore() {
		var data Model
		_, err = cur.ReadDocument(ctx, &data)
		if err != nil {
			break
		}
		entry := &filer.Entry{
			FullPath: util.NewFullPath(string(dirPath), data.Name),
		}
		lastFileName = data.Name
		converted := arrayToBytes(data.Meta)
		if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(converted)); decodeErr != nil {
			err = decodeErr
			glog.V(0).Infof("list %s : %v", entry.FullPath, err)
			break
		}

		if !eachEntryFunc(entry) {
			break
		}

	}
	return lastFileName, err
}

func (store *ArangodbStore) Shutdown() {
}
<|endoftext|>


<commit_before>package aero

import (
	"encoding/json"
	"io/ioutil"
)

// Configuration represents the data in your config.json file.
type Configuration struct {
	Domain    string               `json:"domain"`
	Title     string               `json:"title"`
	Fonts     []string             `json:"fonts"`
	Styles    []string             `json:"styles"`
	Scripts   ScriptsConfiguration `json:"scripts"`
	Push      []string             `json:"push"`
	Manifest  Manifest             `json:"manifest"`
	GZip      bool                 `json:"gzip"`
	GZipCache bool                 `json:"gzipCache"`
	Ports     PortConfiguration    `json:"ports"`
}

// ScriptsConfiguration lets you configure your main entry script.
type ScriptsConfiguration struct {
	// Entry point for scripts
	Main string `json:"main"`
}

// Manifest represents a web manifest
type Manifest struct {
	Name            string         `json:"name"`
	ShortName       string         `json:"short_name"`
	Icons           []ManifestIcon `json:"icons,omitempty"`
	StartURL        string         `json:"start_url"`
	Display         string         `json:"display"`
	Lang            string         `json:"lang,omitempty"`
	ThemeColor      string         `json:"theme_color,omitempty"`
	BackgroundColor string         `json:"background_color,omitempty"`
	GCMSenderID     string         `json:"gcm_sender_id,omitempty"`
}

// ManifestIcon represents a single icon in the web manifest.
type ManifestIcon struct {
	Source string `json:"src"`
	Sizes  string `json:"sizes"`
}

// PortConfiguration lets you configure the ports that Aero will listen on.
type PortConfiguration struct {
	HTTP  int `json:"http"`
	HTTPS int `json:"https"`
}

// Reset resets all fields to the default configuration.
func (config *Configuration) Reset() {
	config.GZip = true
	config.GZipCache = true
	config.Ports.HTTP = 4000
	config.Ports.HTTPS = 4001
	config.Manifest.StartURL = "/"
	config.Manifest.Display = "standalone"
	config.Manifest.Lang = "en"
	config.Manifest.ShortName = "Untitled"
	config.Title = "Untitled site"
}

// LoadConfig loads the application configuration from the file system.
func LoadConfig(path string) (*Configuration, error) {
	data, err := ioutil.ReadFile(path)

	if err != nil {
		return nil, err
	}

	config := &Configuration{}
	config.Reset()

	jsonDecodeError := json.Unmarshal(data, config)

	if jsonDecodeError != nil {
		return nil, jsonDecodeError
	}

	if config.Manifest.Name == "" {
		config.Manifest.Name = config.Title
	}

	if config.Manifest.ShortName == "" {
		config.Manifest.ShortName = config.Title
	}

	return config, nil
}
<commit_msg>Added type to manifest icons<commit_after>package aero

import (
	"encoding/json"
	"io/ioutil"
)

// Configuration represents the data in your config.json file.
type Configuration struct {
	Domain    string               `json:"domain"`
	Title     string               `json:"title"`
	Fonts     []string             `json:"fonts"`
	Styles    []string             `json:"styles"`
	Scripts   ScriptsConfiguration `json:"scripts"`
	Push      []string             `json:"push"`
	Manifest  Manifest             `json:"manifest"`
	GZip      bool                 `json:"gzip"`
	GZipCache bool                 `json:"gzipCache"`
	Ports     PortConfiguration    `json:"ports"`
}

// ScriptsConfiguration lets you configure your main entry script.
type ScriptsConfiguration struct {
	// Entry point for scripts
	Main string `json:"main"`
}

// Manifest represents a web manifest
type Manifest struct {
	Name            string         `json:"name"`
	ShortName       string         `json:"short_name"`
	Icons           []ManifestIcon `json:"icons,omitempty"`
	StartURL        string         `json:"start_url"`
	Display         string         `json:"display"`
	Lang            string         `json:"lang,omitempty"`
	ThemeColor      string         `json:"theme_color,omitempty"`
	BackgroundColor string         `json:"background_color,omitempty"`
	GCMSenderID     string         `json:"gcm_sender_id,omitempty"`
}

// ManifestIcon represents a single icon in the web manifest.
type ManifestIcon struct {
	Source string `json:"src"`
	Sizes  string `json:"sizes"`
	Type   string `json:"type"`
}

// PortConfiguration lets you configure the ports that Aero will listen on.
type PortConfiguration struct {
	HTTP  int `json:"http"`
	HTTPS int `json:"https"`
}

// Reset resets all fields to the default configuration.
func (config *Configuration) Reset() {
	config.GZip = true
	config.GZipCache = true
	config.Ports.HTTP = 4000
	config.Ports.HTTPS = 4001
	config.Manifest.StartURL = "/"
	config.Manifest.Display = "standalone"
	config.Manifest.Lang = "en"
	config.Manifest.ShortName = "Untitled"
	config.Title = "Untitled site"
}

// LoadConfig loads the application configuration from the file system.
func LoadConfig(path string) (*Configuration, error) {
	data, err := ioutil.ReadFile(path)

	if err != nil {
		return nil, err
	}

	config := &Configuration{}
	config.Reset()

	jsonDecodeError := json.Unmarshal(data, config)

	if jsonDecodeError != nil {
		return nil, jsonDecodeError
	}

	if config.Manifest.Name == "" {
		config.Manifest.Name = config.Title
	}

	if config.Manifest.ShortName == "" {
		config.Manifest.ShortName = config.Title
	}

	return config, nil
}
<|endoftext|>


<commit_before>package channel_test

import (
	"fmt"
	"sync"
	"testing"
	"veyron.io/wspr/veyron/services/wsprd/channel"
)

func TestChannelRpcs(t *testing.T) {
	var bHandler channel.MessageSender
	channelA := channel.NewChannel(func(msg channel.Message) {
		bHandler(msg)
	})
	channelB := channel.NewChannel(channelA.HandleMessage)
	bHandler = channelB.HandleMessage

	type testCase struct {
		SendChannel *channel.Channel
		RecvChannel *channel.Channel
		Type        string
		ReqVal      int
		RespVal     int
		Err         error
	}

	tests := []testCase{}

	expectedNumSuccessfulEachDirection := 128
	for i := 0; i < expectedNumSuccessfulEachDirection; i++ {
		tests = append(tests, testCase{channelA, channelB, fmt.Sprintf("Type%d", i), i, i + 1000, nil})
		tests = append(tests, testCase{channelB, channelA, "TypeB", -i, -i - 1000, nil})
	}

	expectedNumFailures := 1
	tests = append(tests, testCase{channelB, channelA, "Type3", 0, 0, fmt.Errorf("TestError")})

	expectedNumCalls := expectedNumSuccessfulEachDirection*2 + expectedNumFailures
	callCountLock := sync.Mutex{}
	numCalls := 0

	wg := sync.WaitGroup{}
	wg.Add(len(tests))
	for i, test := range tests {
		go func() {
			defer wg.Done()
			handler := func(v interface{}) (interface{}, error) {
				callCountLock.Lock()
				numCalls++
				callCountLock.Unlock()

				if test.ReqVal != v.(int) {
					t.Errorf("For test %d, expected request value was %q but got %q", i, test.ReqVal, v.(int))
				}
				return test.RespVal, test.Err
			}
			test.RecvChannel.RegisterRequestHandler(test.Type, handler)
			result, err := test.SendChannel.PerformRpc(test.Type, test.ReqVal)
			if test.Err != nil {
				if err == nil {
					t.Errorf("For test %d, expected an error but didn't get one", i)
				}
			} else {
				if err != nil {
					t.Errorf("For test %d, received unexpected error %v", i, err)
					return
				}
				if result.(int) != test.RespVal {
					t.Errorf("For test %d, expected response value was %q but got %q", i, test.RespVal, result.(int))
				}
			}
		}()
	}

	wg.Wait()

	if numCalls != expectedNumCalls {
		t.Errorf("Expected to receive %d rpcs, but only got %d", expectedNumCalls, numCalls)
	}
}
<commit_msg>TBR nacl/channel/race: Fix race in channel test<commit_after>package channel_test

import (
	"fmt"
	"sync"
	"testing"
	"veyron.io/wspr/veyron/services/wsprd/channel"
)

func TestChannelRpcs(t *testing.T) {
	// Two channels are used and different test send in different directions.
	var bHandler channel.MessageSender
	channelA := channel.NewChannel(func(msg channel.Message) {
		bHandler(msg)
	})
	channelB := channel.NewChannel(channelA.HandleMessage)
	bHandler = channelB.HandleMessage

	type testCase struct {
		SendChannel *channel.Channel
		RecvChannel *channel.Channel
		Type        string
		ReqVal      int
		RespVal     int
		Err         error
	}

	// The list of tests to run concurrently in goroutines.
	// Half of the tests are with different type keys to test multiple type keys.
	// Half of the tests use the same type key
	// One test returns an error.
	tests := []testCase{}
	const reusedTypeName string = "reusedTypeName"
	expectedNumSuccessfulEachDirection := 128
	for i := 0; i < expectedNumSuccessfulEachDirection; i++ {
		tests = append(tests, testCase{channelA, channelB, fmt.Sprintf("Type%d", i), i, i + 1000, nil})
		tests = append(tests, testCase{channelB, channelA, reusedTypeName, -i - 1, -i - 1001, nil})
	}
	expectedNumFailures := 1
	tests = append(tests, testCase{channelB, channelA, "Type3", 0, 0, fmt.Errorf("TestError")})
	expectedNumCalls := expectedNumSuccessfulEachDirection*2 + expectedNumFailures
	callCountLock := sync.Mutex{}
	numCalls := 0

	// reusedHandler handles requests to the same type name.
	reusedHandler := func(v interface{}) (interface{}, error) {
		callCountLock.Lock()
		numCalls++
		callCountLock.Unlock()

		return v.(int) - 1000, nil
	}

	wg := sync.WaitGroup{}
	wg.Add(len(tests))
	var testGoRoutine = func(i int, test testCase) {
		defer wg.Done()

		// Get the message handler. Either the reused handle or a unique handle for this
		// test, depending on the type name.
		var handler func(v interface{}) (interface{}, error)
		if test.Type == reusedTypeName {
			handler = reusedHandler
		} else {
			handler = func(v interface{}) (interface{}, error) {
				callCountLock.Lock()
				numCalls++
				callCountLock.Unlock()

				if test.ReqVal != v.(int) {
					t.Errorf("For test %d, expected request value was %d but got %d", i, test.ReqVal, v.(int))
				}
				return test.RespVal, test.Err
			}
		}
		test.RecvChannel.RegisterRequestHandler(test.Type, handler)

		// Perform the RPC.
		result, err := test.SendChannel.PerformRpc(test.Type, test.ReqVal)
		if test.Err != nil {
			if err == nil {
				t.Errorf("For test %d, expected an error but didn't get one", i)
			}
		} else {
			if err != nil {
				t.Errorf("For test %d, received unexpected error %v", i, err)
				return
			}
			if result.(int) != test.RespVal {
				t.Errorf("For test %d, expected response value was %d but got %d", i, test.RespVal, result.(int))
			}
		}
	}
	for i, test := range tests {
		go testGoRoutine(i, test)
	}

	wg.Wait()

	if numCalls != expectedNumCalls {
		t.Errorf("Expected to receive %d rpcs, but only got %d", expectedNumCalls, numCalls)
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package provisioner_test

import (
	"fmt"
	"path/filepath"
	"runtime"
	"time"

	"github.com/juju/errors"
	"github.com/juju/names"
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/agent"
	"github.com/juju/juju/cloudconfig/instancecfg"
	"github.com/juju/juju/constraints"
	"github.com/juju/juju/container"
	"github.com/juju/juju/container/kvm/mock"
	kvmtesting "github.com/juju/juju/container/kvm/testing"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/instance"
	instancetest "github.com/juju/juju/instance/testing"
	jujutesting "github.com/juju/juju/juju/testing"
	"github.com/juju/juju/state"
	coretesting "github.com/juju/juju/testing"
	coretools "github.com/juju/juju/tools"
	"github.com/juju/juju/version"
	"github.com/juju/juju/worker/provisioner"
)

type kvmSuite struct {
	kvmtesting.TestSuite
	events     chan mock.Event
	eventsDone chan struct{}
}

type kvmBrokerSuite struct {
	kvmSuite
	broker      environs.InstanceBroker
	agentConfig agent.Config
	api         *fakeAPI
}

var _ = gc.Suite(&kvmBrokerSuite{})

func (s *kvmSuite) SetUpTest(c *gc.C) {
	if runtime.GOOS == "windows" {
		c.Skip("Skipping kvm tests on windows")
	}
	s.TestSuite.SetUpTest(c)
	s.events = make(chan mock.Event)
	s.eventsDone = make(chan struct{})
	go func() {
		defer close(s.eventsDone)
		for event := range s.events {
			c.Output(3, fmt.Sprintf("kvm event: <%s, %s>", event.Action, event.InstanceId))
		}
	}()
	s.TestSuite.ContainerFactory.AddListener(s.events)
}

func (s *kvmSuite) TearDownTest(c *gc.C) {
	close(s.events)
	<-s.eventsDone
	s.TestSuite.TearDownTest(c)
}

func (s *kvmBrokerSuite) SetUpTest(c *gc.C) {
	if runtime.GOOS == "windows" {
		c.Skip("Skipping kvm tests on windows")
	}
	s.kvmSuite.SetUpTest(c)
	var err error
	s.agentConfig, err = agent.NewAgentConfig(
		agent.AgentConfigParams{
			DataDir:           "/not/used/here",
			Tag:               names.NewUnitTag("ubuntu/1"),
			UpgradedToVersion: version.Current.Number,
			Password:          "dummy-secret",
			Nonce:             "nonce",
			APIAddresses:      []string{"10.0.0.1:1234"},
			CACert:            coretesting.CACert,
			Environment:       coretesting.EnvironmentTag,
		})
	c.Assert(err, jc.ErrorIsNil)
	s.api = NewFakeAPI()
	managerConfig := container.ManagerConfig{container.ConfigName: "juju"}
	s.broker, err = provisioner.NewKvmBroker(s.api, s.agentConfig, managerConfig, false)
	c.Assert(err, jc.ErrorIsNil)
}

func (s *kvmBrokerSuite) startInstance(c *gc.C, machineId string) instance.Instance {
	machineNonce := "fake-nonce"
	stateInfo := jujutesting.FakeStateInfo(machineId)
	apiInfo := jujutesting.FakeAPIInfo(machineId)
	instanceConfig, err := instancecfg.NewInstanceConfig(machineId, machineNonce, "released", "quantal", true, nil, stateInfo, apiInfo)
	c.Assert(err, jc.ErrorIsNil)
	cons := constraints.Value{}
	possibleTools := coretools.List{&coretools.Tools{
		Version: version.MustParseBinary("2.3.4-quantal-amd64"),
		URL:     "http://tools.testing.invalid/2.3.4-quantal-amd64.tgz",
	}}
	result, err := s.broker.StartInstance(environs.StartInstanceParams{
		Constraints:    cons,
		Tools:          possibleTools,
		InstanceConfig: instanceConfig,
	})
	c.Assert(err, jc.ErrorIsNil)
	return result.Instance
}

func (s *kvmBrokerSuite) TestStopInstance(c *gc.C) {
	kvm0 := s.startInstance(c, "1/kvm/0")
	kvm1 := s.startInstance(c, "1/kvm/1")
	kvm2 := s.startInstance(c, "1/kvm/2")

	err := s.broker.StopInstances(kvm0.Id())
	c.Assert(err, jc.ErrorIsNil)
	s.assertInstances(c, kvm1, kvm2)
	c.Assert(s.kvmContainerDir(kvm0), jc.DoesNotExist)
	c.Assert(s.kvmRemovedContainerDir(kvm0), jc.IsDirectory)

	err = s.broker.StopInstances(kvm1.Id(), kvm2.Id())
	c.Assert(err, jc.ErrorIsNil)
	s.assertInstances(c)
}

func (s *kvmBrokerSuite) TestAllInstances(c *gc.C) {
	kvm0 := s.startInstance(c, "1/kvm/0")
	kvm1 := s.startInstance(c, "1/kvm/1")
	s.assertInstances(c, kvm0, kvm1)

	err := s.broker.StopInstances(kvm1.Id())
	c.Assert(err, jc.ErrorIsNil)
	kvm2 := s.startInstance(c, "1/kvm/2")
	s.assertInstances(c, kvm0, kvm2)
}

func (s *kvmBrokerSuite) assertInstances(c *gc.C, inst ...instance.Instance) {
	results, err := s.broker.AllInstances()
	c.Assert(err, jc.ErrorIsNil)
	instancetest.MatchInstances(c, results, inst...)
}

func (s *kvmBrokerSuite) kvmContainerDir(inst instance.Instance) string {
	return filepath.Join(s.ContainerDir, string(inst.Id()))
}

func (s *kvmBrokerSuite) kvmRemovedContainerDir(inst instance.Instance) string {
	return filepath.Join(s.RemovedDir, string(inst.Id()))
}

type kvmProvisionerSuite struct {
	CommonProvisionerSuite
	kvmSuite
	events chan mock.Event
}

var _ = gc.Suite(&kvmProvisionerSuite{})

func (s *kvmProvisionerSuite) SetUpSuite(c *gc.C) {
	if runtime.GOOS == "windows" {
		c.Skip("Skipping kvm tests on windows")
	}
	s.CommonProvisionerSuite.SetUpSuite(c)
	s.kvmSuite.SetUpSuite(c)
}

func (s *kvmProvisionerSuite) TearDownSuite(c *gc.C) {
	s.kvmSuite.TearDownSuite(c)
	s.CommonProvisionerSuite.TearDownSuite(c)
}

func (s *kvmProvisionerSuite) SetUpTest(c *gc.C) {
	s.CommonProvisionerSuite.SetUpTest(c)
	s.kvmSuite.SetUpTest(c)

	s.events = make(chan mock.Event, 25)
	s.ContainerFactory.AddListener(s.events)
}

func (s *kvmProvisionerSuite) nextEvent(c *gc.C) mock.Event {
	select {
	case event := <-s.events:
		return event
	case <-time.After(coretesting.LongWait):
		c.Fatalf("no event arrived")
	}
	panic("not reachable")
}

func (s *kvmProvisionerSuite) expectStarted(c *gc.C, machine *state.Machine) string {
	// This check in particular leads to tests just hanging
	// indefinitely quite often on i386.
	coretesting.SkipIfI386(c, "lp:1425569")

	s.State.StartSync()
	event := s.nextEvent(c)
	c.Assert(event.Action, gc.Equals, mock.Started)
	err := machine.Refresh()
	c.Assert(err, jc.ErrorIsNil)
	s.waitInstanceId(c, machine, instance.Id(event.InstanceId))
	return event.InstanceId
}

func (s *kvmProvisionerSuite) expectStopped(c *gc.C, instId string) {
	// This check in particular leads to tests just hanging
	// indefinitely quite often on i386.
	coretesting.SkipIfI386(c, "lp:1425569")

	s.State.StartSync()
	event := s.nextEvent(c)
	c.Assert(event.Action, gc.Equals, mock.Stopped)
	c.Assert(event.InstanceId, gc.Equals, instId)
}

func (s *kvmProvisionerSuite) expectNoEvents(c *gc.C) {
	select {
	case event := <-s.events:
		c.Fatalf("unexpected event %#v", event)
	case <-time.After(coretesting.ShortWait):
		return
	}
}

func (s *kvmProvisionerSuite) TearDownTest(c *gc.C) {
	close(s.events)
	s.kvmSuite.TearDownTest(c)
	s.CommonProvisionerSuite.TearDownTest(c)
}

func (s *kvmProvisionerSuite) newKvmProvisioner(c *gc.C) provisioner.Provisioner {
	machineTag := names.NewMachineTag("0")
	agentConfig := s.AgentConfigForTag(c, machineTag)
	managerConfig := container.ManagerConfig{container.ConfigName: "juju"}
	broker, err := provisioner.NewKvmBroker(s.provisioner, agentConfig, managerConfig, false)
	c.Assert(err, jc.ErrorIsNil)
	toolsFinder := (*provisioner.GetToolsFinder)(s.provisioner)
	return provisioner.NewContainerProvisioner(instance.KVM, s.provisioner, agentConfig, broker, toolsFinder)
}

func (s *kvmProvisionerSuite) TestProvisionerStartStop(c *gc.C) {
	p := s.newKvmProvisioner(c)
	c.Assert(p.Stop(), gc.IsNil)
}

func (s *kvmProvisionerSuite) TestDoesNotStartEnvironMachines(c *gc.C) {
	p := s.newKvmProvisioner(c)
	defer stop(c, p)

	// Check that an instance is not provisioned when the machine is created.
	_, err := s.State.AddMachine(coretesting.FakeDefaultSeries, state.JobHostUnits)
	c.Assert(err, jc.ErrorIsNil)

	s.expectNoEvents(c)
}

func (s *kvmProvisionerSuite) TestDoesNotHaveRetryWatcher(c *gc.C) {
	p := s.newKvmProvisioner(c)
	defer stop(c, p)

	w, err := provisioner.GetRetryWatcher(p)
	c.Assert(w, gc.IsNil)
	c.Assert(err, jc.Satisfies, errors.IsNotImplemented)
}

func (s *kvmProvisionerSuite) addContainer(c *gc.C) *state.Machine {
	template := state.MachineTemplate{
		Series: coretesting.FakeDefaultSeries,
		Jobs:   []state.MachineJob{state.JobHostUnits},
	}
	container, err := s.State.AddMachineInsideMachine(template, "0", instance.KVM)
	c.Assert(err, jc.ErrorIsNil)
	return container
}

func (s *kvmProvisionerSuite) TestContainerStartedAndStopped(c *gc.C) {
	coretesting.SkipIfI386(c, "lp:1425569")

	p := s.newKvmProvisioner(c)
	defer stop(c, p)

	container := s.addContainer(c)

	instId := s.expectStarted(c, container)

	// ...and removed, along with the machine, when the machine is Dead.
	c.Assert(container.EnsureDead(), gc.IsNil)
	s.expectStopped(c, instId)
	s.waitRemoved(c, container)
}

func (s *kvmProvisionerSuite) TestKVMProvisionerObservesConfigChanges(c *gc.C) {
	p := s.newKvmProvisioner(c)
	defer stop(c, p)
	s.assertProvisionerObservesConfigChanges(c, p)
}
<commit_msg>Test NetworkInfo for kvm-broker<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package provisioner_test

import (
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"net"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/juju/errors"
	"github.com/juju/names"
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/agent"
	"github.com/juju/juju/cloudconfig/instancecfg"
	"github.com/juju/juju/constraints"
	"github.com/juju/juju/container"
	"github.com/juju/juju/container/kvm/mock"
	kvmtesting "github.com/juju/juju/container/kvm/testing"
	containertesting "github.com/juju/juju/container/testing"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/feature"
	"github.com/juju/juju/instance"
	instancetest "github.com/juju/juju/instance/testing"
	"github.com/juju/juju/juju/arch"
	jujutesting "github.com/juju/juju/juju/testing"
	"github.com/juju/juju/network"
	"github.com/juju/juju/state"
	coretesting "github.com/juju/juju/testing"
	coretools "github.com/juju/juju/tools"
	"github.com/juju/juju/version"
	"github.com/juju/juju/worker/provisioner"
)

type kvmSuite struct {
	kvmtesting.TestSuite
	events     chan mock.Event
	eventsDone chan struct{}
}

type kvmBrokerSuite struct {
	kvmSuite
	broker      environs.InstanceBroker
	agentConfig agent.Config
	api         *fakeAPI
}

var _ = gc.Suite(&kvmBrokerSuite{})

func (s *kvmSuite) SetUpTest(c *gc.C) {
	if runtime.GOOS == "windows" {
		c.Skip("Skipping kvm tests on windows")
	}
	s.TestSuite.SetUpTest(c)
	s.events = make(chan mock.Event)
	s.eventsDone = make(chan struct{})
	go func() {
		defer close(s.eventsDone)
		for event := range s.events {
			c.Output(3, fmt.Sprintf("kvm event: <%s, %s>", event.Action, event.InstanceId))
		}
	}()
	s.TestSuite.ContainerFactory.AddListener(s.events)
}

func (s *kvmSuite) TearDownTest(c *gc.C) {
	close(s.events)
	<-s.eventsDone
	s.TestSuite.TearDownTest(c)
}

func (s *kvmBrokerSuite) SetUpTest(c *gc.C) {
	if runtime.GOOS == "windows" {
		c.Skip("Skipping kvm tests on windows")
	}
	s.kvmSuite.SetUpTest(c)
	var err error
	s.agentConfig, err = agent.NewAgentConfig(
		agent.AgentConfigParams{
			DataDir:           "/not/used/here",
			Tag:               names.NewUnitTag("ubuntu/1"),
			UpgradedToVersion: version.Current.Number,
			Password:          "dummy-secret",
			Nonce:             "nonce",
			APIAddresses:      []string{"10.0.0.1:1234"},
			CACert:            coretesting.CACert,
			Environment:       coretesting.EnvironmentTag,
		})
	c.Assert(err, jc.ErrorIsNil)
	s.api = NewFakeAPI()
	managerConfig := container.ManagerConfig{container.ConfigName: "juju"}
	s.broker, err = provisioner.NewKvmBroker(s.api, s.agentConfig, managerConfig, false)
	c.Assert(err, jc.ErrorIsNil)
}

func (s *kvmBrokerSuite) instanceConfig(c *gc.C, machineId string) *instancecfg.InstanceConfig {
	machineNonce := "fake-nonce"
	// To isolate the tests from the host's architecture, we override it here.
	s.PatchValue(&version.Current.Arch, arch.AMD64)
	stateInfo := jujutesting.FakeStateInfo(machineId)
	apiInfo := jujutesting.FakeAPIInfo(machineId)
	instanceConfig, err := instancecfg.NewInstanceConfig(machineId, machineNonce, "released", "quantal", true, nil, stateInfo, apiInfo)
	c.Assert(err, jc.ErrorIsNil)
	// Ensure the <rootfs>/etc/network path exists.
	containertesting.EnsureRootFSEtcNetwork(c, "juju-"+names.NewMachineTag(machineId).String())
	return instanceConfig
}

func (s *kvmBrokerSuite) startInstance(c *gc.C, machineId string) instance.Instance {
	instanceConfig := s.instanceConfig(c, machineId)
	cons := constraints.Value{}
	possibleTools := coretools.List{&coretools.Tools{
		Version: version.MustParseBinary("2.3.4-quantal-amd64"),
		URL:     "http://tools.testing.invalid/2.3.4-quantal-amd64.tgz",
	}}
	result, err := s.broker.StartInstance(environs.StartInstanceParams{
		Constraints:    cons,
		Tools:          possibleTools,
		InstanceConfig: instanceConfig,
	})
	c.Assert(err, jc.ErrorIsNil)
	return result.Instance
}

func (s *kvmBrokerSuite) TestStopInstance(c *gc.C) {
	kvm0 := s.startInstance(c, "1/kvm/0")
	kvm1 := s.startInstance(c, "1/kvm/1")
	kvm2 := s.startInstance(c, "1/kvm/2")

	err := s.broker.StopInstances(kvm0.Id())
	c.Assert(err, jc.ErrorIsNil)
	s.assertInstances(c, kvm1, kvm2)
	c.Assert(s.kvmContainerDir(kvm0), jc.DoesNotExist)
	c.Assert(s.kvmRemovedContainerDir(kvm0), jc.IsDirectory)

	err = s.broker.StopInstances(kvm1.Id(), kvm2.Id())
	c.Assert(err, jc.ErrorIsNil)
	s.assertInstances(c)
}

func (s *kvmBrokerSuite) TestAllInstances(c *gc.C) {
	kvm0 := s.startInstance(c, "1/kvm/0")
	kvm1 := s.startInstance(c, "1/kvm/1")
	s.assertInstances(c, kvm0, kvm1)

	err := s.broker.StopInstances(kvm1.Id())
	c.Assert(err, jc.ErrorIsNil)
	kvm2 := s.startInstance(c, "1/kvm/2")
	s.assertInstances(c, kvm0, kvm2)
}

func (s *kvmBrokerSuite) assertInstances(c *gc.C, inst ...instance.Instance) {
	results, err := s.broker.AllInstances()
	c.Assert(err, jc.ErrorIsNil)
	instancetest.MatchInstances(c, results, inst...)
}

func (s *kvmBrokerSuite) kvmContainerDir(inst instance.Instance) string {
	return filepath.Join(s.ContainerDir, string(inst.Id()))
}

func (s *kvmBrokerSuite) kvmRemovedContainerDir(inst instance.Instance) string {
	return filepath.Join(s.RemovedDir, string(inst.Id()))
}

func (s *kvmBrokerSuite) TestStartInstancePopulatesNetworkInfo(c *gc.C) {
	s.SetFeatureFlags(feature.AddressAllocation)
	s.PatchValue(provisioner.InterfaceAddrs, func(i *net.Interface) ([]net.Addr, error) {
		return []net.Addr{&fakeAddr{"0.1.2.1/24"}}, nil
	})
	fakeResolvConf := filepath.Join(c.MkDir(), "resolv.conf")
	err := ioutil.WriteFile(fakeResolvConf, []byte("nameserver ns1.dummy\n"), 0644)
	c.Assert(err, jc.ErrorIsNil)
	s.PatchValue(provisioner.ResolvConf, fakeResolvConf)

	instanceConfig := s.instanceConfig(c, "42")
	possibleTools := coretools.List{&coretools.Tools{
		Version: version.MustParseBinary("2.3.4-quantal-amd64"),
		URL:     "http://tools.testing.invalid/2.3.4-quantal-amd64.tgz",
	}}
	result, err := s.broker.StartInstance(environs.StartInstanceParams{
		Constraints:    constraints.Value{},
		Tools:          possibleTools,
		InstanceConfig: instanceConfig,
	})
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(result.NetworkInfo, gc.HasLen, 1)
	iface := result.NetworkInfo[0]
	macAddress := iface.MACAddress
	c.Assert(macAddress[:8], gc.Equals, provisioner.MACAddressTemplate[:8])
	remainder := strings.Replace(macAddress[8:], ":", "", 3)
	c.Assert(remainder, gc.HasLen, 6)
	_, err = hex.DecodeString(remainder)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(iface, jc.DeepEquals, network.InterfaceInfo{
		DeviceIndex:    0,
		CIDR:           "0.1.2.0/24",
		ConfigType:     network.ConfigStatic,
		InterfaceName:  "eth0", // generated from the device index.
		MACAddress:     macAddress,
		DNSServers:     network.NewAddresses("ns1.dummy"),
		Address:        network.NewAddress("0.1.2.3"),
		GatewayAddress: network.NewAddress("0.1.2.1"),
		NetworkName:    network.DefaultPrivate,
		ProviderId:     network.DefaultProviderId,
	})
}

type kvmProvisionerSuite struct {
	CommonProvisionerSuite
	kvmSuite
	events chan mock.Event
}

var _ = gc.Suite(&kvmProvisionerSuite{})

func (s *kvmProvisionerSuite) SetUpSuite(c *gc.C) {
	if runtime.GOOS == "windows" {
		c.Skip("Skipping kvm tests on windows")
	}
	s.CommonProvisionerSuite.SetUpSuite(c)
	s.kvmSuite.SetUpSuite(c)
}

func (s *kvmProvisionerSuite) TearDownSuite(c *gc.C) {
	s.kvmSuite.TearDownSuite(c)
	s.CommonProvisionerSuite.TearDownSuite(c)
}

func (s *kvmProvisionerSuite) SetUpTest(c *gc.C) {
	s.CommonProvisionerSuite.SetUpTest(c)
	s.kvmSuite.SetUpTest(c)

	s.events = make(chan mock.Event, 25)
	s.ContainerFactory.AddListener(s.events)
}

func (s *kvmProvisionerSuite) nextEvent(c *gc.C) mock.Event {
	select {
	case event := <-s.events:
		return event
	case <-time.After(coretesting.LongWait):
		c.Fatalf("no event arrived")
	}
	panic("not reachable")
}

func (s *kvmProvisionerSuite) expectStarted(c *gc.C, machine *state.Machine) string {
	// This check in particular leads to tests just hanging
	// indefinitely quite often on i386.
	coretesting.SkipIfI386(c, "lp:1425569")

	s.State.StartSync()
	event := s.nextEvent(c)
	c.Assert(event.Action, gc.Equals, mock.Started)
	err := machine.Refresh()
	c.Assert(err, jc.ErrorIsNil)
	s.waitInstanceId(c, machine, instance.Id(event.InstanceId))
	return event.InstanceId
}

func (s *kvmProvisionerSuite) expectStopped(c *gc.C, instId string) {
	// This check in particular leads to tests just hanging
	// indefinitely quite often on i386.
	coretesting.SkipIfI386(c, "lp:1425569")

	s.State.StartSync()
	event := s.nextEvent(c)
	c.Assert(event.Action, gc.Equals, mock.Stopped)
	c.Assert(event.InstanceId, gc.Equals, instId)
}

func (s *kvmProvisionerSuite) expectNoEvents(c *gc.C) {
	select {
	case event := <-s.events:
		c.Fatalf("unexpected event %#v", event)
	case <-time.After(coretesting.ShortWait):
		return
	}
}

func (s *kvmProvisionerSuite) TearDownTest(c *gc.C) {
	close(s.events)
	s.kvmSuite.TearDownTest(c)
	s.CommonProvisionerSuite.TearDownTest(c)
}

func (s *kvmProvisionerSuite) newKvmProvisioner(c *gc.C) provisioner.Provisioner {
	machineTag := names.NewMachineTag("0")
	agentConfig := s.AgentConfigForTag(c, machineTag)
	managerConfig := container.ManagerConfig{container.ConfigName: "juju"}
	broker, err := provisioner.NewKvmBroker(s.provisioner, agentConfig, managerConfig, false)
	c.Assert(err, jc.ErrorIsNil)
	toolsFinder := (*provisioner.GetToolsFinder)(s.provisioner)
	return provisioner.NewContainerProvisioner(instance.KVM, s.provisioner, agentConfig, broker, toolsFinder)
}

func (s *kvmProvisionerSuite) TestProvisionerStartStop(c *gc.C) {
	p := s.newKvmProvisioner(c)
	c.Assert(p.Stop(), gc.IsNil)
}

func (s *kvmProvisionerSuite) TestDoesNotStartEnvironMachines(c *gc.C) {
	p := s.newKvmProvisioner(c)
	defer stop(c, p)

	// Check that an instance is not provisioned when the machine is created.
	_, err := s.State.AddMachine(coretesting.FakeDefaultSeries, state.JobHostUnits)
	c.Assert(err, jc.ErrorIsNil)

	s.expectNoEvents(c)
}

func (s *kvmProvisionerSuite) TestDoesNotHaveRetryWatcher(c *gc.C) {
	p := s.newKvmProvisioner(c)
	defer stop(c, p)

	w, err := provisioner.GetRetryWatcher(p)
	c.Assert(w, gc.IsNil)
	c.Assert(err, jc.Satisfies, errors.IsNotImplemented)
}

func (s *kvmProvisionerSuite) addContainer(c *gc.C) *state.Machine {
	template := state.MachineTemplate{
		Series: coretesting.FakeDefaultSeries,
		Jobs:   []state.MachineJob{state.JobHostUnits},
	}
	container, err := s.State.AddMachineInsideMachine(template, "0", instance.KVM)
	c.Assert(err, jc.ErrorIsNil)
	return container
}

func (s *kvmProvisionerSuite) TestContainerStartedAndStopped(c *gc.C) {
	coretesting.SkipIfI386(c, "lp:1425569")

	p := s.newKvmProvisioner(c)
	defer stop(c, p)

	container := s.addContainer(c)

	instId := s.expectStarted(c, container)

	// ...and removed, along with the machine, when the machine is Dead.
	c.Assert(container.EnsureDead(), gc.IsNil)
	s.expectStopped(c, instId)
	s.waitRemoved(c, container)
}

func (s *kvmProvisionerSuite) TestKVMProvisionerObservesConfigChanges(c *gc.C) {
	p := s.newKvmProvisioner(c)
	defer stop(c, p)
	s.assertProvisionerObservesConfigChanges(c, p)
}
<|endoftext|>


<commit_before>package flv

import (
	"encoding/binary"
	"errors"
	"io"
	"os"
)

type File struct {
	file     *os.File
	name     string
	readOnly bool
	size     int64
}

type TagHeader struct {
	TagType   byte
	DataSize  uint32
	Timestamp uint32
}

func CreateFile(name string) (flvFile *File, err error) {
	var file *os.File
	// Create file
	if file, err = os.OpenFile(name, os.O_CREATE|os.O_WRONLY, 0666); err != nil {
		return
	}
	// Write flv header
	if _, err = file.Write(HEADER_BYTES); err != nil {
		file.Close()
		return
	}
	// Sync to disk
	if err = file.Sync(); err != nil {
		file.Close()
		return
	}

	flvFile = &File{
		file:     file,
		name:     name,
		readOnly: false,
	}
	return
}

func ReadAtLeast()

func OpenFile(name string) (flvFile *File, err error) {
	var file *os.File
	// Open file
	file, err = os.Open(name)
	if err != nil {
		return
	}

	var size int64
	if size, err = file.Seek(0, 2); err != nil {
		file.Close()
		return
	}
	if _, err = file.Seek(0, 0); err != nil {
		file.Close()
		return
	}

	flvFile = &File{
		file:     file,
		name:     name,
		readOnly: true,
		size:     size,
	}

	// Read flv header
	remain := len(HEADER_BYTES)
	flvHeader := make([]byte, remain)

	if _, err = io.ReadFull(file, flvHeader); err != nil {
		file.Close()
		return
	}
	if flvHeader[0] != 'F' ||
		flvHeader[1] != 'L' ||
		flvHeader[2] != 'V' {
		file.Close()
		return nil, errors.New("File format error")
	}

	return
}

func (flvFile *File) Close() {
	flvFile.file.Close()
}

// Data with audio header
func (flvFile *File) WriteAudioTag(data []byte, timestamp uint32) (err error) {
	return flvFile.WriteTag(data, AUDIO_TAG, timestamp)
}

// Data with video header
func (flvFile *File) WriteVideoTag(data []byte, timestamp uint32) (err error) {
	return flvFile.WriteTag(data, VIDEO_TAG, timestamp)
}

// Write tag
func (flvFile *File) WriteTag(data []byte, tagType byte, timestamp uint32) (err error) {
	// Write tag header
	if _, err = flvFile.file.Write([]byte{tagType}); err != nil {
		return
	}
	tmpBuf := make([]byte, 4)

	// Write tag size
	binary.BigEndian.PutUint32(tmpBuf, uint32(len(data)))
	if _, err = flvFile.file.Write(tmpBuf[1:]); err != nil {
		return
	}

	// Write timestamp
	binary.BigEndian.PutUint32(tmpBuf, timestamp)
	if _, err = flvFile.file.Write(tmpBuf[1:]); err != nil {
		return
	}
	if _, err = flvFile.file.Write(tmpBuf[:1]); err != nil {
		return
	}

	// Write stream ID
	if _, err = flvFile.file.Write([]byte{0, 0, 0}); err != nil {
		return
	}

	// Write data
	if _, err = flvFile.file.Write(data); err != nil {
		return
	}

	// Write previous tag size
	if err = binary.Write(flvFile.file, binary.BigEndian, uint32(len(data)+11)); err != nil {
		return
	}

	// Sync to disk
	if err = flvFile.file.Sync(); err != nil {
		return
	}
	return
}

func (flvFile *File) ReadTag() (header *TagHeader, data []byte, err error) {
	tmpBuf := make([]byte, 4)
	header = &TagHeader{}
	// Read tag header
	if _, err = io.ReadFull(flvFile.file, tmpBuf[3:]); err != nil {
		return
	}
	header.TagType = tmpBuf[3]

	// Read tag size
	if _, err = io.ReadFull(flvFile.file, tmpBuf[1:]); err != nil {
		return
	}
	header.DataSize = uint32(tmpBuf[1])<<16 | uint32(tmpBuf[2])<<8 | uint32(tmpBuf[3])

	// Read timestamp
	if _, err = io.ReadFull(flvFile.file, tmpBuf); err != nil {
		return
	}
	header.Timestamp = uint32(tmpBuf[3])<<32 + uint32(tmpBuf[0])<<16 + uint32(tmpBuf[1])<<8 + uint32(tmpBuf[2])

	// Read stream ID
	if _, err = io.ReadFull(flvFile.file, tmpBuf[1:]); err != nil {
		return
	}

	// Read data
	data = make([]byte, header.DataSize)
	if _, err = io.ReadFull(flvFile.file, data); err != nil {
		return
	}

	// Read previous tag size
	if _, err = io.ReadFull(flvFile.file, tmpBuf); err != nil {
		return
	}

	return
}

func (flvFile *File) IsFinished() bool {
	pos, err := flvFile.file.Seek(0, 1)
	return (err != nil) || (pos >= flvFile.size)
}
func (flvFile *File) LoopBack() {
	flvFile.file.Seek(int64(len(HEADER_BYTES)), 0)
}
<commit_msg>Remove some useless codes<commit_after>package flv

import (
	"encoding/binary"
	"errors"
	"io"
	"os"
)

type File struct {
	file     *os.File
	name     string
	readOnly bool
	size     int64
}

type TagHeader struct {
	TagType   byte
	DataSize  uint32
	Timestamp uint32
}

func CreateFile(name string) (flvFile *File, err error) {
	var file *os.File
	// Create file
	if file, err = os.OpenFile(name, os.O_CREATE|os.O_WRONLY, 0666); err != nil {
		return
	}
	// Write flv header
	if _, err = file.Write(HEADER_BYTES); err != nil {
		file.Close()
		return
	}
	// Sync to disk
	if err = file.Sync(); err != nil {
		file.Close()
		return
	}

	flvFile = &File{
		file:     file,
		name:     name,
		readOnly: false,
	}
	return
}

func OpenFile(name string) (flvFile *File, err error) {
	var file *os.File
	// Open file
	file, err = os.Open(name)
	if err != nil {
		return
	}

	var size int64
	if size, err = file.Seek(0, 2); err != nil {
		file.Close()
		return
	}
	if _, err = file.Seek(0, 0); err != nil {
		file.Close()
		return
	}

	flvFile = &File{
		file:     file,
		name:     name,
		readOnly: true,
		size:     size,
	}

	// Read flv header
	remain := len(HEADER_BYTES)
	flvHeader := make([]byte, remain)

	if _, err = io.ReadFull(file, flvHeader); err != nil {
		file.Close()
		return
	}
	if flvHeader[0] != 'F' ||
		flvHeader[1] != 'L' ||
		flvHeader[2] != 'V' {
		file.Close()
		return nil, errors.New("File format error")
	}

	return
}

func (flvFile *File) Close() {
	flvFile.file.Close()
}

// Data with audio header
func (flvFile *File) WriteAudioTag(data []byte, timestamp uint32) (err error) {
	return flvFile.WriteTag(data, AUDIO_TAG, timestamp)
}

// Data with video header
func (flvFile *File) WriteVideoTag(data []byte, timestamp uint32) (err error) {
	return flvFile.WriteTag(data, VIDEO_TAG, timestamp)
}

// Write tag
func (flvFile *File) WriteTag(data []byte, tagType byte, timestamp uint32) (err error) {
	// Write tag header
	if _, err = flvFile.file.Write([]byte{tagType}); err != nil {
		return
	}
	tmpBuf := make([]byte, 4)

	// Write tag size
	binary.BigEndian.PutUint32(tmpBuf, uint32(len(data)))
	if _, err = flvFile.file.Write(tmpBuf[1:]); err != nil {
		return
	}

	// Write timestamp
	binary.BigEndian.PutUint32(tmpBuf, timestamp)
	if _, err = flvFile.file.Write(tmpBuf[1:]); err != nil {
		return
	}
	if _, err = flvFile.file.Write(tmpBuf[:1]); err != nil {
		return
	}

	// Write stream ID
	if _, err = flvFile.file.Write([]byte{0, 0, 0}); err != nil {
		return
	}

	// Write data
	if _, err = flvFile.file.Write(data); err != nil {
		return
	}

	// Write previous tag size
	if err = binary.Write(flvFile.file, binary.BigEndian, uint32(len(data)+11)); err != nil {
		return
	}

	// Sync to disk
	if err = flvFile.file.Sync(); err != nil {
		return
	}
	return
}

func (flvFile *File) ReadTag() (header *TagHeader, data []byte, err error) {
	tmpBuf := make([]byte, 4)
	header = &TagHeader{}
	// Read tag header
	if _, err = io.ReadFull(flvFile.file, tmpBuf[3:]); err != nil {
		return
	}
	header.TagType = tmpBuf[3]

	// Read tag size
	if _, err = io.ReadFull(flvFile.file, tmpBuf[1:]); err != nil {
		return
	}
	header.DataSize = uint32(tmpBuf[1])<<16 | uint32(tmpBuf[2])<<8 | uint32(tmpBuf[3])

	// Read timestamp
	if _, err = io.ReadFull(flvFile.file, tmpBuf); err != nil {
		return
	}
	header.Timestamp = uint32(tmpBuf[3])<<32 + uint32(tmpBuf[0])<<16 + uint32(tmpBuf[1])<<8 + uint32(tmpBuf[2])

	// Read stream ID
	if _, err = io.ReadFull(flvFile.file, tmpBuf[1:]); err != nil {
		return
	}

	// Read data
	data = make([]byte, header.DataSize)
	if _, err = io.ReadFull(flvFile.file, data); err != nil {
		return
	}

	// Read previous tag size
	if _, err = io.ReadFull(flvFile.file, tmpBuf); err != nil {
		return
	}

	return
}

func (flvFile *File) IsFinished() bool {
	pos, err := flvFile.file.Seek(0, 1)
	return (err != nil) || (pos >= flvFile.size)
}
func (flvFile *File) LoopBack() {
	flvFile.file.Seek(int64(len(HEADER_BYTES)), 0)
}
<|endoftext|>


<commit_before>package creeper

import (
	"regexp"
	"strconv"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/moxar/arithmetic"
)

var (
	rx_funName = regexp.MustCompile(`^[a-z$][a-zA-Z]{0,15}`)
)

type Fun struct {
	Raw  string
	Node *Node

	Name   string
	Params []string

	Selection *goquery.Selection
	Result    string

	PrevFun *Fun
	NextFun *Fun
}

func (f *Fun) Append(s string) (*Fun, *Fun) {
	f.NextFun = ParseFun(f.Node, s)
	f.NextFun.PrevFun = f
	return f, f.NextFun
}

func PowerfulFind(s *goquery.Selection, q string) *goquery.Selection {
	rx_selectPseudoEq := regexp.MustCompile(`:eq\(\d+\)`)
	if rx_selectPseudoEq.MatchString(q) {
		rs := rx_selectPseudoEq.FindAllStringIndex(q, -1)
		sel := s
		for _, r := range rs {
			iStr := q[r[0]+4 : r[1]-1]
			i64, _ := strconv.ParseInt(iStr, 10, 32)
			i := int(i64)
			sq := q[:r[0]]
			q = strings.TrimSpace(q[r[1]:])
			sel = sel.Find(sq).Eq(i)
		}
		if len(q) > 0 {
			sel = sel.Find(q)
		}
		return sel
	} else {
		return s.Find(q)
	}
}

func (f *Fun) PageBody() (*goquery.Document, error) {
	body, err := f.Node.Page.Body()
	if err != nil {
		return nil, err
	}
	r := strings.NewReader(body)
	return goquery.NewDocumentFromReader(r)
}

func (f *Fun) InitSelector() error {
	if f.Node.IsArray || f.Node.IndentLen == 0 || f.Node.Page != nil {
		doc, err := f.PageBody()
		if err != nil {
			return err
		}
		bud := PowerfulFind(doc.Selection, f.Params[0])

		// overflow current page
		if len(bud.Nodes) > f.Node.Index {
			f.Selection = PowerfulFind(doc.Selection, f.Params[0]).Eq(f.Node.Index)
		} else {
			f.Node.Page.Inc()
			f.Node.Reset()
			f.InitSelector()
		}
	} else {
		_, err := f.Node.ParentNode.Fun.Invoke()
		if err != nil {
			return err
		}
		if len(f.Params) > 0 {
			f.Selection = PowerfulFind(f.Node.ParentNode.Fun.Selection, f.Params[0]).Eq(f.Node.Index)
		} else {
			f.Selection = f.Node.ParentNode.Fun.Selection.Eq(f.Node.Index)
		}
	}
	return nil
}

func (f *Fun) Invoke() (string, error) {
	var err error
	switch f.Name {
	case "$":
		err = f.InitSelector()
	case "attr":
		f.Result, _ = f.PrevFun.Selection.Attr(f.Params[0])
	case "text":
		f.Result = f.PrevFun.Selection.Text()
	case "html":
		f.Result, err = f.PrevFun.Selection.Html()
	case "outerHTML":
		f.Result, err = goquery.OuterHtml(f.PrevFun.Selection)
	case "style":
		f.Result, _ = f.PrevFun.Selection.Attr("style")
	case "href":
		f.Result, _ = f.PrevFun.Selection.Attr("href")
	case "src":
		f.Result, _ = f.PrevFun.Selection.Attr("src")
	case "class":
		f.Result, _ = f.PrevFun.Selection.Attr("class")
	case "id":
		f.Result, _ = f.PrevFun.Selection.Attr("id")
	case "calc":
		v, err := arithmetic.Parse(f.PrevFun.Result)
		if err != nil {
			return "", err
		}
		n, _ := arithmetic.ToFloat(v)
		prec := 2
		if len(f.Params) > 0 {
			i64, err := strconv.ParseInt(f.Params[0], 10, 32)
			if err != nil {
				return "", err
			}
			prec = int(i64)
		}
		f.Result = strconv.FormatFloat(n, 'g', prec, 64)
	case "expand":
		rx, err := regexp.Compile(f.Params[0])
		if err != nil {
			return "", err
		}
		src := f.PrevFun.Result
		dst := []byte{}
		m := rx.FindStringSubmatchIndex(src)
		s := rx.ExpandString(dst, f.Params[1], src, m)
		f.Result = string(s)
	case "match":
		rx, err := regexp.Compile(f.Params[0])
		if err != nil {
			return "", err
		}
		rs := rx.FindAllStringSubmatch(f.PrevFun.Result, -1)
		if len(rs) > 0 && len(rs[0]) > 1 {
			f.Result = rs[0][1]
		}
	}
	if err != nil {
		return "", err
	}
	if f.NextFun != nil {
		return f.NextFun.Invoke()
	} else {
		return f.Result, nil
	}
}

func ParseFun(n *Node, s string) *Fun {
	fun := new(Fun)
	fun.Node = n
	fun.Raw = s

	sa := rx_funName.FindAllString(s, -1)
	fun.Name = sa[0]
	ls := s[len(sa[0]):]
	ps := []string{}
	p, pl := parseParams(ls)
	for i := 0;; i++ {
		if v, e := p["$"+strconv.Itoa(i)]; e {
			ps = append(ps, v)
		} else {
			break
		}
	}
	if len(ps) > 0 {
		fun.Params = ps
	}
	ls = ls[pl+1:]
	if len(ls) > 0 {
		ls = ls[1:]
		fun.Append(ls)
	}

	return fun
}
<commit_msg>Optimize the selector function<commit_after>package creeper

import (
	"regexp"
	"strconv"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/moxar/arithmetic"
)

var (
	rx_funName = regexp.MustCompile(`^[a-z$][a-zA-Z]{0,15}`)
)

type Fun struct {
	Raw  string
	Node *Node

	Name   string
	Params []string

	Document *goquery.Document
	Selection *goquery.Selection
	Result    string

	PrevFun *Fun
	NextFun *Fun
}

func (f *Fun) Append(s string) (*Fun, *Fun) {
	f.NextFun = ParseFun(f.Node, s)
	f.NextFun.PrevFun = f
	return f, f.NextFun
}

func PowerfulFind(s *goquery.Selection, q string) *goquery.Selection {
	rx_selectPseudoEq := regexp.MustCompile(`:eq\(\d+\)`)
	if rx_selectPseudoEq.MatchString(q) {
		rs := rx_selectPseudoEq.FindAllStringIndex(q, -1)
		sel := s
		for _, r := range rs {
			iStr := q[r[0]+4 : r[1]-1]
			i64, _ := strconv.ParseInt(iStr, 10, 32)
			i := int(i64)
			sq := q[:r[0]]
			q = strings.TrimSpace(q[r[1]:])
			sel = sel.Find(sq).Eq(i)
		}
		if len(q) > 0 {
			sel = sel.Find(q)
		}
		return sel
	} else {
		return s.Find(q)
	}
}

func (f *Fun) PageBody() (*goquery.Document, error) {
	body, err := f.Node.Page.Body()
	if err != nil {
		return nil, err
	}
	r := strings.NewReader(body)
	return goquery.NewDocumentFromReader(r)
}

func (f *Fun) InitSelector(root bool) error {
	var baseSel *goquery.Selection

	if f.Node.Page != nil {
		doc, err := f.PageBody()
		if err != nil {
			return err
		}
		f.Document = doc
		baseSel = f.Document.Selection
	} else {
		f.Node.ParentNode.Fun.Invoke()
		baseSel = f.Node.ParentNode.Fun.Selection
	}

	if f.Node.IsArray {
		bundle := PowerfulFind(baseSel, f.Params[0])
		if len(bundle.Nodes) > f.Node.Index {
			f.Selection = PowerfulFind(baseSel, f.Params[0]).Eq(f.Node.Index)
		} else {
			// overflow current page
			f.Node.Page.Inc()
			f.Node.Reset()
			f.InitSelector(root)
		}
	} else {
		if len(f.Params) > 0 {
			f.Selection = PowerfulFind(baseSel, f.Params[0]).Eq(f.Node.Index)
		} else {
			f.Selection = baseSel.Eq(f.Node.Index)
		}
	}

	return nil
}

func (f *Fun) Invoke() (string, error) {
	var err error
	switch f.Name {
	case "$":
		err = f.InitSelector(false)
	case "attr":
		f.Result, _ = f.PrevFun.Selection.Attr(f.Params[0])
	case "text":
		f.Result = f.PrevFun.Selection.Text()
	case "html":
		f.Result, err = f.PrevFun.Selection.Html()
	case "outerHTML":
		f.Result, err = goquery.OuterHtml(f.PrevFun.Selection)
	case "style":
		f.Result, _ = f.PrevFun.Selection.Attr("style")
	case "href":
		f.Result, _ = f.PrevFun.Selection.Attr("href")
	case "src":
		f.Result, _ = f.PrevFun.Selection.Attr("src")
	case "class":
		f.Result, _ = f.PrevFun.Selection.Attr("class")
	case "id":
		f.Result, _ = f.PrevFun.Selection.Attr("id")
	case "calc":
		v, err := arithmetic.Parse(f.PrevFun.Result)
		if err != nil {
			return "", err
		}
		n, _ := arithmetic.ToFloat(v)
		prec := 2
		if len(f.Params) > 0 {
			i64, err := strconv.ParseInt(f.Params[0], 10, 32)
			if err != nil {
				return "", err
			}
			prec = int(i64)
		}
		f.Result = strconv.FormatFloat(n, 'g', prec, 64)
	case "expand":
		rx, err := regexp.Compile(f.Params[0])
		if err != nil {
			return "", err
		}
		src := f.PrevFun.Result
		dst := []byte{}
		m := rx.FindStringSubmatchIndex(src)
		s := rx.ExpandString(dst, f.Params[1], src, m)
		f.Result = string(s)
	case "match":
		rx, err := regexp.Compile(f.Params[0])
		if err != nil {
			return "", err
		}
		rs := rx.FindAllStringSubmatch(f.PrevFun.Result, -1)
		if len(rs) > 0 && len(rs[0]) > 1 {
			f.Result = rs[0][1]
		}
	}
	if err != nil {
		return "", err
	}
	if f.NextFun != nil {
		return f.NextFun.Invoke()
	} else {
		return f.Result, nil
	}
}

func ParseFun(n *Node, s string) *Fun {
	fun := new(Fun)
	fun.Node = n
	fun.Raw = s

	sa := rx_funName.FindAllString(s, -1)
	fun.Name = sa[0]
	ls := s[len(sa[0]):]
	ps := []string{}
	p, pl := parseParams(ls)
	for i := 0;; i++ {
		if v, e := p["$"+strconv.Itoa(i)]; e {
			ps = append(ps, v)
		} else {
			break
		}
	}
	if len(ps) > 0 {
		fun.Params = ps
	}
	ls = ls[pl+1:]
	if len(ls) > 0 {
		ls = ls[1:]
		fun.Append(ls)
	}

	return fun
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io"
	"os/exec"
	"regexp"
)

type Matcher struct {
	re *regexp.Regexp
}

func NewMatcher(expr string) (m *Matcher, err error) {
	m = &Matcher{}
	m.re, err = regexp.Compile(expr)
	if err != nil {
		return nil, err
	}
	return m, nil
}

func (m *Matcher) MatchString(s string) bool {
	return m.re.MatchString(s)
}

type Processor struct {
	cmd *exec.Cmd
}

func NewProcessor(name string, arg ...string) (p *Processor, err error) {
	if _, err = exec.LookPath(name); err != nil {
		return nil, err
	}
	p = &Processor{}
	p.cmd = exec.Command(name, arg...)
	return p, nil
}

func (p *Processor) Process(a []string) error {
	in, err := p.cmd.StdinPipe()
	if err != nil {
		return err
	}
	defer in.Close()

	out, err := p.cmd.StdoutPipe()
	if err != nil {
		return err
	}
	defer out.Close()

	if err = p.cmd.Start(); err != nil {
		return err
	}
	for _, s := range a {
		fmt.Fprintln(in, s)
	}
	if err = in.Close(); err != nil {
		return err
	}

	b := bufio.NewScanner(out)
	for i := 0; i < len(a) && b.Scan(); i++ {
		a[i] = b.Text()
	}
	return b.Err()
}

type Lines struct {
	lines          []string
	matchedLines   []string
	matchedIndexes map[int]bool
}

func LoadLines(r io.Reader, expr string) (l *Lines, err error) {
	m, err := NewMatcher(expr)
	if err != nil {
		return nil, err
	}
	l = &Lines{}
	l.matchedIndexes = make(map[int]bool)
	b := bufio.NewScanner(r)
	for i := 0; b.Scan(); i++ {
		line := b.Text()
		if m.MatchString(line) {
			l.matchedLines = append(l.matchedLines, line)
			l.matchedIndexes[i] = true
		}
		l.lines = append(l.lines, line)
	}
	return l, b.Err()
}
<commit_msg>Implement NewLines<commit_after>package main

import (
	"bufio"
	"fmt"
	"os/exec"
	"regexp"
)

type Matcher struct {
	re *regexp.Regexp
}

func NewMatcher(expr string) (m *Matcher, err error) {
	m = &Matcher{}
	m.re, err = regexp.Compile(expr)
	if err != nil {
		return nil, err
	}
	return m, nil
}

func (m *Matcher) MatchString(s string) bool {
	return m.re.MatchString(s)
}

type Processor struct {
	cmd *exec.Cmd
}

func NewProcessor(name string, arg ...string) (p *Processor, err error) {
	if _, err = exec.LookPath(name); err != nil {
		return nil, err
	}
	p = &Processor{}
	p.cmd = exec.Command(name, arg...)
	return p, nil
}

func (p *Processor) Process(a []string) error {
	in, err := p.cmd.StdinPipe()
	if err != nil {
		return err
	}
	defer in.Close()

	out, err := p.cmd.StdoutPipe()
	if err != nil {
		return err
	}
	defer out.Close()

	if err = p.cmd.Start(); err != nil {
		return err
	}
	for _, s := range a {
		fmt.Fprintln(in, s)
	}
	if err = in.Close(); err != nil {
		return err
	}

	b := bufio.NewScanner(out)
	for i := 0; i < len(a) && b.Scan(); i++ {
		a[i] = b.Text()
	}
	return b.Err()
}

type Lines struct {
	matcher        *Matcher
	processor      *Processor
	lines          []string
	matchedLines   []string
	matchedIndexes map[int]bool
}

func NewLines(m *Matcher, p *Processor) *Lines {
	return &Lines{
		matcher:        m,
		processor:      p,
		lines:          []string{},
		matchedLines:   []string{},
		matchedIndexes: make(map[int]bool),
	}
}
<|endoftext|>