<commit_before>package runner

import (
	"github.com/mesos/mesos-go/scheduler"
	"go.uber.org/zap"
)

func LogEvent(e *scheduler.Event, log *zap.Logger) {
	eventType := zap.String("type", e.Type.String())
	if msg := log.Check(zap.DebugLevel, "EVENT"); msg != nil {
		msg.Write(
			eventType,
			zap.Any("message", e),
		)
		return
	}
	switch *e.Type {
	case scheduler.Event_SUBSCRIBED:
		master := e.Subscribed.MasterInfo
		framework := e.Subscribed.FrameworkID
		log.Info(
			"EVENT",
			eventType,
			zap.Any("master", master.Address),
			zap.String("FrameworkID", framework.Value),
		)
	case scheduler.Event_UPDATE:
		log.Info(
			"EVENT",
			eventType,
			zap.String("data", string(e.Update.Status.Data)),
		)
	case scheduler.Event_OFFERS:
		log.Info(
			"EVENT",
			eventType,
			//zap.Any("offers", e.Offers.Offers),
		)
	default:
		log.Info(
			"EVENT",
			eventType,
		)
	}
}

func LogCall(c *scheduler.Call, log *zap.Logger) {
	callType := zap.String("type", c.Type.String())
	if msg := log.Check(zap.DebugLevel, "CALL"); msg != nil {
		msg.Write(
			callType,
			zap.Any("event", c),
		)
		return
	}
	switch *c.Type {
	case scheduler.Call_SUBSCRIBE:
		log.Info(
			"CALL",
			callType,
			zap.Any("framework", c.Subscribe.FrameworkInfo),
		)
	case scheduler.Call_ACCEPT:
		log.Info(
			"CALL",
			callType,
			zap.Any("operations", c.Accept.Operations),
			zap.Any("offers", c.Accept.OfferIDs),
		)
	default:
		log.Info(
			"CALL",
			callType,
		)
	}
}
<commit_msg>log message on update<commit_after>package runner

import (
	"github.com/mesos/mesos-go/scheduler"
	"go.uber.org/zap"
)

func LogEvent(e *scheduler.Event, log *zap.Logger) {
	eventType := zap.String("type", e.Type.String())
	if msg := log.Check(zap.DebugLevel, "EVENT"); msg != nil {
		msg.Write(
			eventType,
			zap.Any("message", e),
		)
		return
	}
	switch *e.Type {
	case scheduler.Event_SUBSCRIBED:
		master := e.Subscribed.MasterInfo
		framework := e.Subscribed.FrameworkID
		log.Info(
			"EVENT",
			eventType,
			zap.Any("master", master.Address),
			zap.String("FrameworkID", framework.Value),
		)
	case scheduler.Event_UPDATE:
		log.Info(
			"EVENT",
			eventType,
			//zap.String("data", string(e.Update.Status.Data)),
			zap.String("state", e.Update.Status.State.String()),
			zap.String("message", e.Update.Status.Message),
		)
	case scheduler.Event_OFFERS:
		log.Info(
			"EVENT",
			eventType,
			//zap.Any("offers", e.Offers.Offers),
		)
	default:
		log.Info(
			"EVENT",
			eventType,
		)
	}
}

func LogCall(c *scheduler.Call, log *zap.Logger) {
	callType := zap.String("type", c.Type.String())
	if msg := log.Check(zap.DebugLevel, "CALL"); msg != nil {
		msg.Write(
			callType,
			zap.Any("event", c),
		)
		return
	}
	switch *c.Type {
	case scheduler.Call_SUBSCRIBE:
		log.Info(
			"CALL",
			callType,
			zap.Any("framework", c.Subscribe.FrameworkInfo),
		)
	case scheduler.Call_ACCEPT:
		log.Info(
			"CALL",
			callType,
			zap.Any("operations", c.Accept.Operations),
			zap.Any("offers", c.Accept.OfferIDs),
		)
	default:
		log.Info(
			"CALL",
			callType,
		)
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package boot

import (
	"fmt"
	"path"
	"sort"
	"strconv"
	"strings"

	specs "github.com/opencontainers/runtime-spec/specs-go"
	"gvisor.dev/gvisor/pkg/abi/linux"
	"gvisor.dev/gvisor/pkg/fspath"
	"gvisor.dev/gvisor/pkg/sentry/devices/memdev"
	"gvisor.dev/gvisor/pkg/sentry/fs"
	devpts2 "gvisor.dev/gvisor/pkg/sentry/fsimpl/devpts"
	devtmpfsimpl "gvisor.dev/gvisor/pkg/sentry/fsimpl/devtmpfs"
	goferimpl "gvisor.dev/gvisor/pkg/sentry/fsimpl/gofer"
	procimpl "gvisor.dev/gvisor/pkg/sentry/fsimpl/proc"
	sysimpl "gvisor.dev/gvisor/pkg/sentry/fsimpl/sys"
	tmpfsimpl "gvisor.dev/gvisor/pkg/sentry/fsimpl/tmpfs"
	"gvisor.dev/gvisor/pkg/syserror"

	"gvisor.dev/gvisor/pkg/context"
	"gvisor.dev/gvisor/pkg/log"
	"gvisor.dev/gvisor/pkg/sentry/kernel"
	"gvisor.dev/gvisor/pkg/sentry/kernel/auth"
	"gvisor.dev/gvisor/pkg/sentry/vfs"
)

func registerFilesystems(ctx context.Context, vfsObj *vfs.VirtualFilesystem, creds *auth.Credentials) error {
	vfsObj.MustRegisterFilesystemType(devpts2.Name, &devpts2.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{
		AllowUserList: true,
		// TODO(b/29356795): Users may mount this once the terminals are in a
		//  usable state.
		AllowUserMount: false,
	})
	vfsObj.MustRegisterFilesystemType(devtmpfsimpl.Name, &devtmpfsimpl.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{
		AllowUserMount: true,
		AllowUserList:  true,
	})
	vfsObj.MustRegisterFilesystemType(goferimpl.Name, &goferimpl.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{
		AllowUserList: true,
	})
	vfsObj.MustRegisterFilesystemType(procimpl.Name, &procimpl.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{
		AllowUserMount: true,
		AllowUserList:  true,
	})
	vfsObj.MustRegisterFilesystemType(sysimpl.Name, &sysimpl.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{
		AllowUserMount: true,
		AllowUserList:  true,
	})
	vfsObj.MustRegisterFilesystemType(tmpfsimpl.Name, &tmpfsimpl.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{
		AllowUserMount: true,
		AllowUserList:  true,
	})

	// Setup files in devtmpfs.
	if err := memdev.Register(vfsObj); err != nil {
		return fmt.Errorf("registering memdev: %w", err)
	}
	a, err := devtmpfsimpl.NewAccessor(ctx, vfsObj, creds, devtmpfsimpl.Name)
	if err != nil {
		return fmt.Errorf("creating devtmpfs accessor: %w", err)
	}
	defer a.Release()

	if err := a.UserspaceInit(ctx); err != nil {
		return fmt.Errorf("initializing userspace: %w", err)
	}
	if err := memdev.CreateDevtmpfsFiles(ctx, a); err != nil {
		return fmt.Errorf("creating devtmpfs files: %w", err)
	}
	return nil
}

func setupContainerVFS2(ctx context.Context, conf *Config, mntr *containerMounter, procArgs *kernel.CreateProcessArgs) error {
	if err := mntr.k.VFS().Init(); err != nil {
		return fmt.Errorf("failed to initialize VFS: %w", err)
	}
	mns, err := mntr.setupVFS2(ctx, conf, procArgs)
	if err != nil {
		return fmt.Errorf("failed to setupFS: %w", err)
	}
	procArgs.MountNamespaceVFS2 = mns
	return setExecutablePathVFS2(ctx, procArgs)
}

func setExecutablePathVFS2(ctx context.Context, procArgs *kernel.CreateProcessArgs) error {
	exe := procArgs.Argv[0]

	// Absolute paths can be used directly.
	if path.IsAbs(exe) {
		procArgs.Filename = exe
		return nil
	}

	// Paths with '/' in them should be joined to the working directory, or
	// to the root if working directory is not set.
	if strings.IndexByte(exe, '/') > 0 {
		if !path.IsAbs(procArgs.WorkingDirectory) {
			return fmt.Errorf("working directory %q must be absolute", procArgs.WorkingDirectory)
		}
		procArgs.Filename = path.Join(procArgs.WorkingDirectory, exe)
		return nil
	}

	// Paths with a '/' are relative to the CWD.
	if strings.IndexByte(exe, '/') > 0 {
		procArgs.Filename = path.Join(procArgs.WorkingDirectory, exe)
		return nil
	}

	// Otherwise, We must lookup the name in the paths, starting from the
	// root directory.
	root := procArgs.MountNamespaceVFS2.Root()
	defer root.DecRef()

	paths := fs.GetPath(procArgs.Envv)
	creds := procArgs.Credentials

	for _, p := range paths {
		binPath := path.Join(p, exe)
		pop := &vfs.PathOperation{
			Root:               root,
			Start:              root,
			Path:               fspath.Parse(binPath),
			FollowFinalSymlink: true,
		}
		opts := &vfs.OpenOptions{
			FileExec: true,
			Flags:    linux.O_RDONLY,
		}
		dentry, err := root.Mount().Filesystem().VirtualFilesystem().OpenAt(ctx, creds, pop, opts)
		if err == syserror.ENOENT || err == syserror.EACCES {
			// Didn't find it here.
			continue
		}
		if err != nil {
			return err
		}
		dentry.DecRef()

		procArgs.Filename = binPath
		return nil
	}

	return fmt.Errorf("executable %q not found in $PATH=%q", exe, strings.Join(paths, ":"))
}

func (c *containerMounter) setupVFS2(ctx context.Context, conf *Config, procArgs *kernel.CreateProcessArgs) (*vfs.MountNamespace, error) {
	log.Infof("Configuring container's file system with VFS2")

	// Create context with root credentials to mount the filesystem (the current
	// user may not be privileged enough).
	rootCreds := auth.NewRootCredentials(procArgs.Credentials.UserNamespace)
	rootProcArgs := *procArgs
	rootProcArgs.WorkingDirectory = "/"
	rootProcArgs.Credentials = rootCreds
	rootProcArgs.Umask = 0022
	rootProcArgs.MaxSymlinkTraversals = linux.MaxSymlinkTraversals
	rootCtx := procArgs.NewContext(c.k)

	if err := registerFilesystems(rootCtx, c.k.VFS(), rootCreds); err != nil {
		return nil, fmt.Errorf("register filesystems: %w", err)
	}

	mns, err := c.createMountNamespaceVFS2(rootCtx, conf, rootCreds)
	if err != nil {
		return nil, fmt.Errorf("creating mount namespace: %w", err)
	}
	rootProcArgs.MountNamespaceVFS2 = mns

	// Mount submounts.
	if err := c.mountSubmountsVFS2(rootCtx, conf, mns, rootCreds); err != nil {
		return nil, fmt.Errorf("mounting submounts vfs2: %w", err)
	}
	return mns, nil
}

func (c *containerMounter) createMountNamespaceVFS2(ctx context.Context, conf *Config, creds *auth.Credentials) (*vfs.MountNamespace, error) {
	fd := c.fds.remove()
	opts := strings.Join(p9MountOptionsVFS2(fd, conf.FileAccess), ",")

	log.Infof("Mounting root over 9P, ioFD: %d", fd)
	mns, err := c.k.VFS().NewMountNamespace(ctx, creds, "", rootFsName, &vfs.GetFilesystemOptions{Data: opts})
	if err != nil {
		return nil, fmt.Errorf("setting up mount namespace: %w", err)
	}
	return mns, nil
}

func (c *containerMounter) mountSubmountsVFS2(ctx context.Context, conf *Config, mns *vfs.MountNamespace, creds *auth.Credentials) error {
	c.prepareMountsVFS2()

	for _, submount := range c.mounts {
		log.Debugf("Mounting %q to %q, type: %s, options: %s", submount.Source, submount.Destination, submount.Type, submount.Options)
		if err := c.mountSubmountVFS2(ctx, conf, mns, creds, &submount); err != nil {
			return err
		}
	}

	// TODO(gvisor.dev/issue/1487): implement mountTmp from fs.go.

	return c.checkDispenser()
}

func (c *containerMounter) prepareMountsVFS2() {
	// Sort the mounts so that we don't place children before parents.
	sort.Slice(c.mounts, func(i, j int) bool { return len(c.mounts[i].Destination) < len(c.mounts[j].Destination) })
}

// TODO(gvisor.dev/issue/1487): Implement submount options similar to the VFS1 version.
func (c *containerMounter) mountSubmountVFS2(ctx context.Context, conf *Config, mns *vfs.MountNamespace, creds *auth.Credentials, submount *specs.Mount) error {
	root := mns.Root()
	defer root.DecRef()
	target := &vfs.PathOperation{
		Root:  root,
		Start: root,
		Path:  fspath.Parse(submount.Destination),
	}

	fsName, options, useOverlay, err := c.getMountNameAndOptionsVFS2(conf, *submount)
	if err != nil {
		return fmt.Errorf("mountOptions failed: %w", err)
	}
	if fsName == "" {
		// Filesystem is not supported (e.g. cgroup), just skip it.
		return nil
	}

	if err := c.makeSyntheticMount(ctx, submount.Destination, root, creds); err != nil {
		return err
	}
	log.Debugf("directory exists or made directory for submount: %s", submount.Destination)

	opts := &vfs.MountOptions{
		GetFilesystemOptions: vfs.GetFilesystemOptions{
			Data: strings.Join(options, ","),
		},
		InternalMount: true,
	}

	// All writes go to upper, be paranoid and make lower readonly.
	opts.ReadOnly = useOverlay

	if err := c.k.VFS().MountAt(ctx, creds, "", target, fsName, opts); err != nil {
		return fmt.Errorf("failed to mount %q (type: %s): %w, opts: %v", submount.Destination, submount.Type, err, opts)
	}
	log.Infof("Mounted %q to %q type: %s, internal-options: %q", submount.Source, submount.Destination, submount.Type, opts)
	return nil
}

// getMountNameAndOptionsVFS2 retrieves the fsName, opts, and useOverlay values
// used for mounts.
func (c *containerMounter) getMountNameAndOptionsVFS2(conf *Config, m specs.Mount) (string, []string, bool, error) {
	var (
		fsName     string
		opts       []string
		useOverlay bool
	)

	switch m.Type {
	case devpts, devtmpfs, proc, sysfs:
		fsName = m.Type
	case nonefs:
		fsName = sysfs
	case tmpfs:
		fsName = m.Type

		var err error
		opts, err = parseAndFilterOptions(m.Options, tmpfsAllowedOptions...)
		if err != nil {
			return "", nil, false, err
		}

	case bind:
		fd := c.fds.remove()
		fsName = "9p"
		opts = p9MountOptionsVFS2(fd, c.getMountAccessType(m))
		// If configured, add overlay to all writable mounts.
		useOverlay = conf.Overlay && !mountFlags(m.Options).ReadOnly

	default:
		log.Warningf("ignoring unknown filesystem type %q", m.Type)
	}
	return fsName, opts, useOverlay, nil
}

// p9MountOptions creates a slice of options for a p9 mount.
// TODO(gvisor.dev/issue/1624): Remove this version once privateunixsocket is
// deleted, along with the rest of VFS1.
func p9MountOptionsVFS2(fd int, fa FileAccessType) []string {
	opts := []string{
		"trans=fd",
		"rfdno=" + strconv.Itoa(fd),
		"wfdno=" + strconv.Itoa(fd),
	}
	if fa == FileAccessShared {
		opts = append(opts, "cache=remote_revalidating")
	}
	return opts
}

func (c *containerMounter) makeSyntheticMount(ctx context.Context, currentPath string, root vfs.VirtualDentry, creds *auth.Credentials) error {
	target := &vfs.PathOperation{
		Root:  root,
		Start: root,
		Path:  fspath.Parse(currentPath),
	}

	_, err := c.k.VFS().StatAt(ctx, creds, target, &vfs.StatOptions{})
	switch {
	case err == syserror.ENOENT:
		if err := c.makeSyntheticMount(ctx, path.Dir(currentPath), root, creds); err != nil {
			return err
		}
		mkdirOpts := &vfs.MkdirOptions{Mode: 0777, ForSyntheticMountpoint: true}
		if err := c.k.VFS().MkdirAt(ctx, creds, target, mkdirOpts); err != nil {
			return fmt.Errorf("failed to makedir for mount %+v: %w", target, err)
		}
		return nil

	case err != nil:
		return fmt.Errorf("stat failed for mount %+v: %w", target, err)

	default:
		return nil
	}
}
<commit_msg>Adjust a few log messages<commit_after>// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package boot

import (
	"fmt"
	"path"
	"sort"
	"strconv"
	"strings"

	specs "github.com/opencontainers/runtime-spec/specs-go"
	"gvisor.dev/gvisor/pkg/abi/linux"
	"gvisor.dev/gvisor/pkg/fspath"
	"gvisor.dev/gvisor/pkg/sentry/devices/memdev"
	"gvisor.dev/gvisor/pkg/sentry/fs"
	devpts2 "gvisor.dev/gvisor/pkg/sentry/fsimpl/devpts"
	devtmpfsimpl "gvisor.dev/gvisor/pkg/sentry/fsimpl/devtmpfs"
	goferimpl "gvisor.dev/gvisor/pkg/sentry/fsimpl/gofer"
	procimpl "gvisor.dev/gvisor/pkg/sentry/fsimpl/proc"
	sysimpl "gvisor.dev/gvisor/pkg/sentry/fsimpl/sys"
	tmpfsimpl "gvisor.dev/gvisor/pkg/sentry/fsimpl/tmpfs"
	"gvisor.dev/gvisor/pkg/syserror"

	"gvisor.dev/gvisor/pkg/context"
	"gvisor.dev/gvisor/pkg/log"
	"gvisor.dev/gvisor/pkg/sentry/kernel"
	"gvisor.dev/gvisor/pkg/sentry/kernel/auth"
	"gvisor.dev/gvisor/pkg/sentry/vfs"
)

func registerFilesystems(ctx context.Context, vfsObj *vfs.VirtualFilesystem, creds *auth.Credentials) error {
	vfsObj.MustRegisterFilesystemType(devpts2.Name, &devpts2.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{
		AllowUserList: true,
		// TODO(b/29356795): Users may mount this once the terminals are in a
		//  usable state.
		AllowUserMount: false,
	})
	vfsObj.MustRegisterFilesystemType(devtmpfsimpl.Name, &devtmpfsimpl.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{
		AllowUserMount: true,
		AllowUserList:  true,
	})
	vfsObj.MustRegisterFilesystemType(goferimpl.Name, &goferimpl.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{
		AllowUserList: true,
	})
	vfsObj.MustRegisterFilesystemType(procimpl.Name, &procimpl.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{
		AllowUserMount: true,
		AllowUserList:  true,
	})
	vfsObj.MustRegisterFilesystemType(sysimpl.Name, &sysimpl.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{
		AllowUserMount: true,
		AllowUserList:  true,
	})
	vfsObj.MustRegisterFilesystemType(tmpfsimpl.Name, &tmpfsimpl.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{
		AllowUserMount: true,
		AllowUserList:  true,
	})

	// Setup files in devtmpfs.
	if err := memdev.Register(vfsObj); err != nil {
		return fmt.Errorf("registering memdev: %w", err)
	}
	a, err := devtmpfsimpl.NewAccessor(ctx, vfsObj, creds, devtmpfsimpl.Name)
	if err != nil {
		return fmt.Errorf("creating devtmpfs accessor: %w", err)
	}
	defer a.Release()

	if err := a.UserspaceInit(ctx); err != nil {
		return fmt.Errorf("initializing userspace: %w", err)
	}
	if err := memdev.CreateDevtmpfsFiles(ctx, a); err != nil {
		return fmt.Errorf("creating devtmpfs files: %w", err)
	}
	return nil
}

func setupContainerVFS2(ctx context.Context, conf *Config, mntr *containerMounter, procArgs *kernel.CreateProcessArgs) error {
	if err := mntr.k.VFS().Init(); err != nil {
		return fmt.Errorf("failed to initialize VFS: %w", err)
	}
	mns, err := mntr.setupVFS2(ctx, conf, procArgs)
	if err != nil {
		return fmt.Errorf("failed to setupFS: %w", err)
	}
	procArgs.MountNamespaceVFS2 = mns
	return setExecutablePathVFS2(ctx, procArgs)
}

func setExecutablePathVFS2(ctx context.Context, procArgs *kernel.CreateProcessArgs) error {
	exe := procArgs.Argv[0]

	// Absolute paths can be used directly.
	if path.IsAbs(exe) {
		procArgs.Filename = exe
		return nil
	}

	// Paths with '/' in them should be joined to the working directory, or
	// to the root if working directory is not set.
	if strings.IndexByte(exe, '/') > 0 {
		if !path.IsAbs(procArgs.WorkingDirectory) {
			return fmt.Errorf("working directory %q must be absolute", procArgs.WorkingDirectory)
		}
		procArgs.Filename = path.Join(procArgs.WorkingDirectory, exe)
		return nil
	}

	// Paths with a '/' are relative to the CWD.
	if strings.IndexByte(exe, '/') > 0 {
		procArgs.Filename = path.Join(procArgs.WorkingDirectory, exe)
		return nil
	}

	// Otherwise, We must lookup the name in the paths, starting from the
	// root directory.
	root := procArgs.MountNamespaceVFS2.Root()
	defer root.DecRef()

	paths := fs.GetPath(procArgs.Envv)
	creds := procArgs.Credentials

	for _, p := range paths {
		binPath := path.Join(p, exe)
		pop := &vfs.PathOperation{
			Root:               root,
			Start:              root,
			Path:               fspath.Parse(binPath),
			FollowFinalSymlink: true,
		}
		opts := &vfs.OpenOptions{
			FileExec: true,
			Flags:    linux.O_RDONLY,
		}
		dentry, err := root.Mount().Filesystem().VirtualFilesystem().OpenAt(ctx, creds, pop, opts)
		if err == syserror.ENOENT || err == syserror.EACCES {
			// Didn't find it here.
			continue
		}
		if err != nil {
			return err
		}
		dentry.DecRef()

		procArgs.Filename = binPath
		return nil
	}

	return fmt.Errorf("executable %q not found in $PATH=%q", exe, strings.Join(paths, ":"))
}

func (c *containerMounter) setupVFS2(ctx context.Context, conf *Config, procArgs *kernel.CreateProcessArgs) (*vfs.MountNamespace, error) {
	log.Infof("Configuring container's file system with VFS2")

	// Create context with root credentials to mount the filesystem (the current
	// user may not be privileged enough).
	rootCreds := auth.NewRootCredentials(procArgs.Credentials.UserNamespace)
	rootProcArgs := *procArgs
	rootProcArgs.WorkingDirectory = "/"
	rootProcArgs.Credentials = rootCreds
	rootProcArgs.Umask = 0022
	rootProcArgs.MaxSymlinkTraversals = linux.MaxSymlinkTraversals
	rootCtx := procArgs.NewContext(c.k)

	if err := registerFilesystems(rootCtx, c.k.VFS(), rootCreds); err != nil {
		return nil, fmt.Errorf("register filesystems: %w", err)
	}

	mns, err := c.createMountNamespaceVFS2(rootCtx, conf, rootCreds)
	if err != nil {
		return nil, fmt.Errorf("creating mount namespace: %w", err)
	}
	rootProcArgs.MountNamespaceVFS2 = mns

	// Mount submounts.
	if err := c.mountSubmountsVFS2(rootCtx, conf, mns, rootCreds); err != nil {
		return nil, fmt.Errorf("mounting submounts vfs2: %w", err)
	}
	return mns, nil
}

func (c *containerMounter) createMountNamespaceVFS2(ctx context.Context, conf *Config, creds *auth.Credentials) (*vfs.MountNamespace, error) {
	fd := c.fds.remove()
	opts := strings.Join(p9MountOptionsVFS2(fd, conf.FileAccess), ",")

	log.Infof("Mounting root over 9P, ioFD: %d", fd)
	mns, err := c.k.VFS().NewMountNamespace(ctx, creds, "", rootFsName, &vfs.GetFilesystemOptions{Data: opts})
	if err != nil {
		return nil, fmt.Errorf("setting up mount namespace: %w", err)
	}
	return mns, nil
}

func (c *containerMounter) mountSubmountsVFS2(ctx context.Context, conf *Config, mns *vfs.MountNamespace, creds *auth.Credentials) error {
	c.prepareMountsVFS2()

	for _, submount := range c.mounts {
		log.Debugf("Mounting %q to %q, type: %s, options: %s", submount.Source, submount.Destination, submount.Type, submount.Options)
		if err := c.mountSubmountVFS2(ctx, conf, mns, creds, &submount); err != nil {
			return err
		}
	}

	// TODO(gvisor.dev/issue/1487): implement mountTmp from fs.go.

	return c.checkDispenser()
}

func (c *containerMounter) prepareMountsVFS2() {
	// Sort the mounts so that we don't place children before parents.
	sort.Slice(c.mounts, func(i, j int) bool { return len(c.mounts[i].Destination) < len(c.mounts[j].Destination) })
}

// TODO(gvisor.dev/issue/1487): Implement submount options similar to the VFS1 version.
func (c *containerMounter) mountSubmountVFS2(ctx context.Context, conf *Config, mns *vfs.MountNamespace, creds *auth.Credentials, submount *specs.Mount) error {
	root := mns.Root()
	defer root.DecRef()
	target := &vfs.PathOperation{
		Root:  root,
		Start: root,
		Path:  fspath.Parse(submount.Destination),
	}

	fsName, options, useOverlay, err := c.getMountNameAndOptionsVFS2(conf, *submount)
	if err != nil {
		return fmt.Errorf("mountOptions failed: %w", err)
	}
	if fsName == "" {
		// Filesystem is not supported (e.g. cgroup), just skip it.
		return nil
	}

	if err := c.makeSyntheticMount(ctx, submount.Destination, root, creds); err != nil {
		return err
	}

	opts := &vfs.MountOptions{
		GetFilesystemOptions: vfs.GetFilesystemOptions{
			Data: strings.Join(options, ","),
		},
		InternalMount: true,
	}

	// All writes go to upper, be paranoid and make lower readonly.
	opts.ReadOnly = useOverlay

	if err := c.k.VFS().MountAt(ctx, creds, "", target, fsName, opts); err != nil {
		return fmt.Errorf("failed to mount %q (type: %s): %w, opts: %v", submount.Destination, submount.Type, err, opts)
	}
	log.Infof("Mounted %q to %q type: %s, internal-options: %q", submount.Source, submount.Destination, submount.Type, opts.GetFilesystemOptions.Data)
	return nil
}

// getMountNameAndOptionsVFS2 retrieves the fsName, opts, and useOverlay values
// used for mounts.
func (c *containerMounter) getMountNameAndOptionsVFS2(conf *Config, m specs.Mount) (string, []string, bool, error) {
	var (
		fsName     string
		opts       []string
		useOverlay bool
	)

	switch m.Type {
	case devpts, devtmpfs, proc, sysfs:
		fsName = m.Type
	case nonefs:
		fsName = sysfs
	case tmpfs:
		fsName = m.Type

		var err error
		opts, err = parseAndFilterOptions(m.Options, tmpfsAllowedOptions...)
		if err != nil {
			return "", nil, false, err
		}

	case bind:
		fd := c.fds.remove()
		fsName = "9p"
		opts = p9MountOptionsVFS2(fd, c.getMountAccessType(m))
		// If configured, add overlay to all writable mounts.
		useOverlay = conf.Overlay && !mountFlags(m.Options).ReadOnly

	default:
		log.Warningf("ignoring unknown filesystem type %q", m.Type)
	}
	return fsName, opts, useOverlay, nil
}

// p9MountOptions creates a slice of options for a p9 mount.
// TODO(gvisor.dev/issue/1624): Remove this version once privateunixsocket is
// deleted, along with the rest of VFS1.
func p9MountOptionsVFS2(fd int, fa FileAccessType) []string {
	opts := []string{
		"trans=fd",
		"rfdno=" + strconv.Itoa(fd),
		"wfdno=" + strconv.Itoa(fd),
	}
	if fa == FileAccessShared {
		opts = append(opts, "cache=remote_revalidating")
	}
	return opts
}

func (c *containerMounter) makeSyntheticMount(ctx context.Context, currentPath string, root vfs.VirtualDentry, creds *auth.Credentials) error {
	target := &vfs.PathOperation{
		Root:  root,
		Start: root,
		Path:  fspath.Parse(currentPath),
	}
	_, err := c.k.VFS().StatAt(ctx, creds, target, &vfs.StatOptions{})
	if err == nil {
		// Mount point exists, nothing else to do.
		return nil
	}
	if err != syserror.ENOENT {
		return fmt.Errorf("stat failed for %q during mount point creation: %w", currentPath, err)
	}

	// Recurse to ensure parent is created and then create the mount point.
	if err := c.makeSyntheticMount(ctx, path.Dir(currentPath), root, creds); err != nil {
		return err
	}
	log.Debugf("Creating dir %q for mount point", currentPath)
	mkdirOpts := &vfs.MkdirOptions{Mode: 0777, ForSyntheticMountpoint: true}
	if err := c.k.VFS().MkdirAt(ctx, creds, target, mkdirOpts); err != nil {
		return fmt.Errorf("failed to create directory %q for mount: %w", currentPath, err)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"os/user"
	"strings"

	log "github.com/Sirupsen/logrus"
)

// NewJob represents a new job
type NewJob struct {
	PBSJob PBSJob
	Args   []string
}

// CreateJob creates a new Job.
func (n *NewJob) CreateJob() (*Job, error) {

	// Get the user
	u, err := n.GetUser()
	if err != nil {
		return nil, err
	}

	// Get the image
	i, err := n.GetImage()
	if err != nil {
		return nil, err
	}

	// Get the shell
	s, err := n.GetShell()
	if err != nil {
		return nil, err
	}

	var m Mounts
	m, err = n.addMounts(m)
	if err != nil {
		return nil, err
	}

	e, err := n.GetEnv()
	if err != nil {
		return nil, err
	}

	c, err := n.GetCmd(i)
	if err != nil {
		return nil, err
	}

	job := &Job{
		User:   u,
		Cmd:    c,
		Shell:  s,
		Image:  i,
		Mounts: m,
		Env:    e,
	}

	bs, err := NewBootstrap(job)
	if err != nil {
		return nil, err
	}

	job.Mounts = job.Mounts.AddMount(bs.GetScriptPath(), "/bootstrap.sh", false)

	log.WithFields(log.Fields{
		"job": job,
	}).Info("Parsed job")

	return job, nil
}

// GetCmd get the command to run in the container
func (n *NewJob) GetCmd(i *Image) ([]string, error) {
	cmd := n.Args

	// Prepend the bootstrap script
	cmd = append([]string{"/bootstrap.sh"}, cmd...)

	// Prepend the init command
	if i.InitCmd != "" {
		cmd = append([]string{i.InitCmd}, cmd...)
	}

	return cmd, nil
}

// GetEnv get the current Environment
func (n *NewJob) GetEnv() ([]string, error) {
	e := os.Environ()
	return e, nil
}

// GetUser returns the User who submitted this job.
func (n *NewJob) GetUser() (*User, error) {
	u, err := user.Lookup(n.PBSJob.Origin.Logname)
	if err != nil {
		return nil, err
	}

	// TODO Go standard lib does not have a way to get primary groupname.
	// If Go starts supporting it this can be removed.
	cmd := exec.Command("id", "-g", "-n", u.Username)
	var out bytes.Buffer
	cmd.Stdout = &out
	err = cmd.Run()
	if err != nil {
		return nil, err
	}

	g := out.String()
	// Trim space and newline
	g = strings.TrimSpace(g)

	return &User{
		Username:  u.Username,
		UID:       u.Uid,
		Groupname: g,
		GID:       u.Gid,
		HomeDir:   u.HomeDir,
	}, nil
}

// GetShell returns the shell to use in the container.
func (n *NewJob) GetShell() (*Shell, error) {
	// If we are in interactive mode, don't tee.
	if n.PBSJob.Environment == "PBS_INTERACTIVE" {
		return &Shell{
			Stdout: os.Stdout,
			Stderr: os.Stderr,
			Stdin:  os.Stdin,
			TTY:    true,
		}, nil
	}

	return &Shell{
		Stdout: os.Stdout,
		Stderr: os.Stderr,
		Stdin:  os.Stdin,
		TTY:    false,
	}, nil
}

// GetImage returns a new Image.
func (n *NewJob) GetImage() (*Image, error) {

	if config.GetImageName() == "" && config.GetImageSource() == "" {
		return nil, fmt.Errorf("No image specifed.")
	}

	if config.GetImageName() != "" && config.GetImageSource() != "" {
		return nil, fmt.Errorf("Can not get image based on name and source.")
	}

	source := false

	if config.GetImageSource() != "" {
		source = true
	}

	return &Image{
		ImageName:   config.GetImageName(),
		ImageSource: config.GetImageSource(),
		Source:      source,
		Privileged:  config.GetImagePrivileged(),
		InitCmd:     config.GetImageInit(),
	}, nil
}

// getMounts Get the default mounts
func (n *NewJob) addMounts(m Mounts) (Mounts, error) {
	// Not interactive job.
	if n.PBSJob.Environment != "PBS_INTERACTIVE" {
		s := fmt.Sprintf("%v%v.SC", config.GetJobDir(), n.PBSJob.JobID)
		m = m.AddMount(s, s, false) // Read only
	}

	// Mount home
	if config.MountHome() {
		m = m.AddMount(n.PBSJob.Origin.Home, n.PBSJob.Origin.Home, true)
	}

	// Mount hosts file
	if config.MountHosts() {
		m = m.AddMount("/etc/hosts", "/etc/hosts", false) // Read only
	}

	// Mount cvmfs
	if config.GetCvmfs() != "" {
		m = m.AddMount(config.GetCvmfs(), "/cvmfs", false) // Read only
	}

	return m, nil
}
<commit_msg>add jobid to influxdb logs<commit_after>package main

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"os/user"
	"strings"

	log "github.com/Sirupsen/logrus"
)

// NewJob represents a new job
type NewJob struct {
	PBSJob PBSJob
	Args   []string
}

// CreateJob creates a new Job.
func (n *NewJob) CreateJob() (*Job, error) {

	// Get the user
	u, err := n.GetUser()
	if err != nil {
		return nil, err
	}

	// Get the image
	i, err := n.GetImage()
	if err != nil {
		return nil, err
	}

	// Get the shell
	s, err := n.GetShell()
	if err != nil {
		return nil, err
	}

	var m Mounts
	m, err = n.addMounts(m)
	if err != nil {
		return nil, err
	}

	e, err := n.GetEnv()
	if err != nil {
		return nil, err
	}

	c, err := n.GetCmd(i)
	if err != nil {
		return nil, err
	}

	job := &Job{
		ID:     n.PBSJob.JobID,
		User:   u,
		Cmd:    c,
		Shell:  s,
		Image:  i,
		Mounts: m,
		Env:    e,
	}

	bs, err := NewBootstrap(job)
	if err != nil {
		return nil, err
	}

	job.Mounts = job.Mounts.AddMount(bs.GetScriptPath(), "/bootstrap.sh", false)

	log.WithFields(log.Fields{
		"job": job,
	}).Info("Parsed job")

	return job, nil
}

// GetCmd get the command to run in the container
func (n *NewJob) GetCmd(i *Image) ([]string, error) {
	cmd := n.Args

	// Prepend the bootstrap script
	cmd = append([]string{"/bootstrap.sh"}, cmd...)

	// Prepend the init command
	if i.InitCmd != "" {
		cmd = append([]string{i.InitCmd}, cmd...)
	}

	return cmd, nil
}

// GetEnv get the current Environment
func (n *NewJob) GetEnv() ([]string, error) {
	e := os.Environ()
	return e, nil
}

// GetUser returns the User who submitted this job.
func (n *NewJob) GetUser() (*User, error) {
	u, err := user.Lookup(n.PBSJob.Origin.Logname)
	if err != nil {
		return nil, err
	}

	// TODO Go standard lib does not have a way to get primary groupname.
	// If Go starts supporting it this can be removed.
	cmd := exec.Command("id", "-g", "-n", u.Username)
	var out bytes.Buffer
	cmd.Stdout = &out
	err = cmd.Run()
	if err != nil {
		return nil, err
	}

	g := out.String()
	// Trim space and newline
	g = strings.TrimSpace(g)

	return &User{
		Username:  u.Username,
		UID:       u.Uid,
		Groupname: g,
		GID:       u.Gid,
		HomeDir:   u.HomeDir,
	}, nil
}

// GetShell returns the shell to use in the container.
func (n *NewJob) GetShell() (*Shell, error) {
	// If we are in interactive mode, don't tee.
	if n.PBSJob.Environment == "PBS_INTERACTIVE" {
		return &Shell{
			Stdout: os.Stdout,
			Stderr: os.Stderr,
			Stdin:  os.Stdin,
			TTY:    true,
		}, nil
	}

	return &Shell{
		Stdout: os.Stdout,
		Stderr: os.Stderr,
		Stdin:  os.Stdin,
		TTY:    false,
	}, nil
}

// GetImage returns a new Image.
func (n *NewJob) GetImage() (*Image, error) {

	if config.GetImageName() == "" && config.GetImageSource() == "" {
		return nil, fmt.Errorf("No image specifed.")
	}

	if config.GetImageName() != "" && config.GetImageSource() != "" {
		return nil, fmt.Errorf("Can not get image based on name and source.")
	}

	source := false

	if config.GetImageSource() != "" {
		source = true
	}

	return &Image{
		ImageName:   config.GetImageName(),
		ImageSource: config.GetImageSource(),
		Source:      source,
		Privileged:  config.GetImagePrivileged(),
		InitCmd:     config.GetImageInit(),
	}, nil
}

// getMounts Get the default mounts
func (n *NewJob) addMounts(m Mounts) (Mounts, error) {
	// Not interactive job.
	if n.PBSJob.Environment != "PBS_INTERACTIVE" {
		s := fmt.Sprintf("%v%v.SC", config.GetJobDir(), n.PBSJob.JobID)
		m = m.AddMount(s, s, false) // Read only
	}

	// Mount home
	if config.MountHome() {
		m = m.AddMount(n.PBSJob.Origin.Home, n.PBSJob.Origin.Home, true)
	}

	// Mount hosts file
	if config.MountHosts() {
		m = m.AddMount("/etc/hosts", "/etc/hosts", false) // Read only
	}

	// Mount cvmfs
	if config.GetCvmfs() != "" {
		m = m.AddMount(config.GetCvmfs(), "/cvmfs", false) // Read only
	}

	return m, nil
}
<|endoftext|>


<commit_before>package workers

import (
	"strings"
	"sync"
)

type manager struct {
	queue       string
	fetch       Fetcher
	job         jobFunc
	concurrency int
	workers     []*worker
	workersM    *sync.Mutex
	confirm     chan *Msg
	stop        chan bool
	exit        chan bool
	mids        *Middlewares
	*sync.WaitGroup
}

func (m *manager) start() {
	m.Add(1)
	go m.manage()
}

func (m *manager) prepare() {
	if !m.fetch.Closed() {
		m.fetch.Close()
	}
}

func (m *manager) quit() {
	Logger.Println("quitting queue", m.queueName(), "(waiting for", m.processing(), "/", len(m.workers), "workers).")
	m.prepare()

	for _, worker := range m.workers {
		worker.quit()
	}

	m.stop <- true
	<-m.exit

	m.Done()
}

func (m *manager) manage() {
	Logger.Println("processing queue", m.queueName(), "with", m.concurrency, "workers.")

	m.loadWorkers()

	go m.fetch.Fetch()

	for {
		select {
		case message := <-m.confirm:
			m.fetch.Acknowledge(message)
		case <-m.stop:
			m.exit <- true
			break
		}
	}
}

func (m *manager) loadWorkers() {
	m.workersM.Lock()
	for i := 0; i < m.concurrency; i++ {
		m.workers[i] = newWorker(m)
		m.workers[i].start()
	}
	m.workersM.Unlock()
}

func (m *manager) processing() (count int) {
	m.workersM.Lock()
	for _, worker := range m.workers {
		if worker.processing() {
			count++
		}
	}
	m.workersM.Unlock()
	return
}

func (m *manager) queueName() string {
	return strings.Replace(m.queue, "queue:", "", 1)
}

func newManager(queue string, job jobFunc, concurrency int, mids ...Action) *manager {
	var customMids *Middlewares
	if len(mids) == 0 {
		customMids = Middleware
	} else {
		customMids = NewMiddleware(Middleware.actions...)
		for _, m := range mids {
			customMids.Append(m)
		}
	}
	m := &manager{
		Config.Namespace + "queue:" + queue,
		nil,
		job,
		concurrency,
		make([]*worker, concurrency),
		&sync.Mutex{},
		make(chan *Msg),
		make(chan bool),
		make(chan bool),
		customMids,
		&sync.WaitGroup{},
	}

	m.fetch = Config.Fetch(m.queue)

	return m
}
<commit_msg>Lock list of workers when quitting to avoid data race<commit_after>package workers

import (
	"strings"
	"sync"
)

type manager struct {
	queue       string
	fetch       Fetcher
	job         jobFunc
	concurrency int
	workers     []*worker
	workersM    *sync.Mutex
	confirm     chan *Msg
	stop        chan bool
	exit        chan bool
	mids        *Middlewares
	*sync.WaitGroup
}

func (m *manager) start() {
	m.Add(1)
	go m.manage()
}

func (m *manager) prepare() {
	if !m.fetch.Closed() {
		m.fetch.Close()
	}
}

func (m *manager) quit() {
	Logger.Println("quitting queue", m.queueName(), "(waiting for", m.processing(), "/", len(m.workers), "workers).")
	m.prepare()

	m.workersM.Lock()
	for _, worker := range m.workers {
		worker.quit()
	}
	m.workersM.Unlock()

	m.stop <- true
	<-m.exit

	m.Done()
}

func (m *manager) manage() {
	Logger.Println("processing queue", m.queueName(), "with", m.concurrency, "workers.")

	m.loadWorkers()

	go m.fetch.Fetch()

	for {
		select {
		case message := <-m.confirm:
			m.fetch.Acknowledge(message)
		case <-m.stop:
			m.exit <- true
			break
		}
	}
}

func (m *manager) loadWorkers() {
	m.workersM.Lock()
	for i := 0; i < m.concurrency; i++ {
		m.workers[i] = newWorker(m)
		m.workers[i].start()
	}
	m.workersM.Unlock()
}

func (m *manager) processing() (count int) {
	m.workersM.Lock()
	for _, worker := range m.workers {
		if worker.processing() {
			count++
		}
	}
	m.workersM.Unlock()
	return
}

func (m *manager) queueName() string {
	return strings.Replace(m.queue, "queue:", "", 1)
}

func newManager(queue string, job jobFunc, concurrency int, mids ...Action) *manager {
	var customMids *Middlewares
	if len(mids) == 0 {
		customMids = Middleware
	} else {
		customMids = NewMiddleware(Middleware.actions...)
		for _, m := range mids {
			customMids.Append(m)
		}
	}
	m := &manager{
		Config.Namespace + "queue:" + queue,
		nil,
		job,
		concurrency,
		make([]*worker, concurrency),
		&sync.Mutex{},
		make(chan *Msg),
		make(chan bool),
		make(chan bool),
		customMids,
		&sync.WaitGroup{},
	}

	m.fetch = Config.Fetch(m.queue)

	return m
}
<|endoftext|>


<commit_before>package kurs

import(
	"net/http"
	"io/ioutil"
	"strings"
	"github.com/moovweb/gokogiri"
)

func processMandiri() Data{
	url := "http://www.bankmandiri.co.id/resource/kurs.asp?row=2"

	respon, _ := http.Get(url)
	page, _ := ioutil.ReadAll(respon.Body)
	document, _ := gokogiri.ParseHtml(page)
	defer document.Free()

	nilai := "1.00"
	var matauang string
	var kursjual string
	var kursbeli string
	kurs := make(map[string]Currency)

	str, _ := document.Search("//p[@class='catatan']");
	firstData := strings.Split(strings.Split(str[0].InnerHtml(), "
")[0], " ")
	lastUpdated := firstData[2] + " " + firstData[3] + " " + firstData[4] + " " + firstData[5] + " " + firstData[6]

	doc, _ :=document.Search("//table[@class='tbl-view']/tr")
	for i, tr := range doc{
		t := 0
		for td := tr.FirstChild(); td != nil; td = td.NextSibling(){
			teks := strings.TrimSpace(td.Content())
			if i > 0 && i <= 15 && len(teks) > 0{
				if t == 1{
					matauang = teks
				} else if t == 2{
					kursbeli = teks
				} else if t == 3{
					kursjual = teks
				}
				t += 1
				kurs[matauang] = Currency{Nilai:nilai,KursJual:kursjual,KursBeli:kursbeli}
			}
		}
	}

	response := Data{
		LastUpdated: lastUpdated,
		Currency: kurs,
	}

	return response
}
<commit_msg>do not process row without currency data<commit_after>package kurs

import(
	"net/http"
	"io/ioutil"
	"strings"
	"github.com/moovweb/gokogiri"
)

func processMandiri() Data{
	url := "http://www.bankmandiri.co.id/resource/kurs.asp?row=2"

	respon, _ := http.Get(url)
	page, _ := ioutil.ReadAll(respon.Body)
	document, _ := gokogiri.ParseHtml(page)
	defer document.Free()

	nilai := "1.00"
	var matauang string
	var kursjual string
	var kursbeli string
	kurs := make(map[string]Currency)

	str, _ := document.Search("//p[@class='catatan']");
	firstData := strings.Split(strings.Split(str[0].InnerHtml(), "
")[0], " ")
	lastUpdated := firstData[2] + " " + firstData[3] + " " + firstData[4] + " " + firstData[5] + " " + firstData[6]

	doc, _ :=document.Search("//table[@class='tbl-view']/tr")
	for i, tr := range doc{
		t := 0
		for td := tr.FirstChild(); td != nil; td = td.NextSibling(){
			teks := strings.TrimSpace(td.Content())
			if i > 0 && i <= 15 && len(teks) > 0{
				if t == 1{
					matauang = teks
				} else if t == 2{
					kursbeli = teks
				} else if t == 3{
					kursjual = teks
				}
				t += 1

				if kursjual != "" && kursbeli != "" {
					kurs[matauang] = Currency{Nilai:nilai,KursJual:kursjual,KursBeli:kursbeli}
				}
			}
		}
	}

	response := Data{
		LastUpdated: lastUpdated,
		Currency: kurs,
	}

	return response
}
<|endoftext|>


<commit_before>package captcha_test

import (
	"testing"
    "github.com/leocbs/2captcha/captcha"
)

const testKey = "testKey"

func TestShouldValidateEmptyKey(t *testing.T) {
    _,err := captcha.New("")
    if err == nil{
        t.Error("new captcha don't valid empty key")
    }
}

func TestShouldValidateEmptyBase64(t *testing.T) {
	twocaptcha,_ := captcha.New(testKey)
	_, err := twocaptcha.UploadBase64Image("")
	if err == nil{
        t.Error("new captcha don't valid empty base64 image")
    }
}

func TestShouldUploadBase64Image(t *testing.T) {
	twocaptcha,_ := captcha.New(testKey)
	_, err := twocaptcha.UploadBase64Image("dHdvY2FwdGNoYQ==")
	if err != nil{
        t.Errorf("upload response not OK: %s", err)
    }
}<commit_msg>mock http response to captcha ok<commit_after>package captcha_test

import (
	"github.com/leocbs/2captcha/captcha"
	"io"
	"net/http"
	"strconv"
	"strings"
	"testing"
)

const testKey = "testKey"

type Responder func(*http.Request) (*http.Response, error)

type MockTransport struct {
	responders  map[string]Responder
	noResponder Responder
}

var defaultTransport = NewMockTransport()

func NewMockTransport() *MockTransport {
	return &MockTransport{make(map[string]Responder), nil}
}

func (m *MockTransport) registerResponder(method, url string, responder Responder) {
	m.responders[method+" "+url] = responder
}

func newRespBodyFromString(body string) io.ReadCloser {
	return &dummyReadCloser{strings.NewReader(body)}
}
func (m *MockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	url := req.URL.String()

	// try and get a responder that matches the method and URL
	responder := m.responderForKey(req.Method + " " + url)

	// if we weren't able to find a responder and the URL contains a querystring
	// then we strip off the querystring and try again.
	if responder == nil && strings.Contains(url, "?") {
		responder = m.responderForKey(req.Method + " " + strings.Split(url, "?")[0])
	}

	// if we found a responder, call it
	if responder != nil {
		return responder(req)
	}

	return nil, nil
}
func (m *MockTransport) responderForKey(key string) Responder {
	for k, r := range m.responders {
		if k != key {
			continue
		}
		return r
	}
	return nil
}

// dummycloser
type dummyReadCloser struct {
	body io.ReadSeeker
}

func (d *dummyReadCloser) Read(p []byte) (n int, err error) {
	n, err = d.body.Read(p)
	if err == io.EOF {
		d.body.Seek(0, 0)
	}
	return n, err
}

func (d *dummyReadCloser) Close() error {
	return nil
}

//end

func setUp() {
	http.DefaultTransport = defaultTransport
}

func TestShouldValidateEmptyKey(t *testing.T) {
	_, err := captcha.New("")
	if err == nil {
		t.Error("new captcha don't valid empty key")
	}
}

func TestShouldValidateEmptyBase64(t *testing.T) {
	twocaptcha, _ := captcha.New(testKey)
	_, err := twocaptcha.UploadBase64Image("")
	if err == nil {
		t.Error("new captcha don't valid empty base64 image")
	}
}

func TestShouldUploadBase64Image(t *testing.T) {
	setUp()
	defaultTransport.registerResponder("POST", "http://2captcha.com/in.php",
		func(req *http.Request) (*http.Response, error) {
			return &http.Response{
				Status:     strconv.Itoa(500),
				StatusCode: 200,
				Body:       newRespBodyFromString("OK|captchaID"),
				Header:     http.Header{},
			}, nil

		})

	twocaptcha, _ := captcha.New(testKey)
	_, err := twocaptcha.UploadBase64Image("dHdvY2FwdGNoYQ==")
	if err != nil {
		t.Errorf("upload response not OK: %s", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"context"
	"encoding/binary"
	"flag"
	"fmt"
	"io"
	"log"
	"math"
	"net"
	"os"
	"time"

	"github.com/sirupsen/logrus"
)

func getTemps(buf *bufio.Reader, w io.Writer) {
	fmt.Println("frånluft") // Extract air
	Send(buf, w, []byte{0xff, 0x1e, 0xc8, 0x04, 0xb6, 0x04, 0x08, 0x00})

	fmt.Println("ute") // Outdoor air
	Send(buf, w, []byte{0xff, 0x1e, 0xc8, 0x04, 0xb6, 0x04, 0x04, 0xec})

	fmt.Println("avluft") // Exhaust air
	Send(buf, w, []byte{0xff, 0x1e, 0xc8, 0x04, 0xb6, 0x3b, 0x00, 0x95})

	fmt.Println("tilluft") // Supply air
	Send(buf, w, []byte{0xff, 0x1e, 0xc8, 0x04, 0xb6, 0x39, 0x02, 0xfa})

	fmt.Println("börvärde") // Set temperature
	Send(buf, w, []byte{0xff, 0x1e, 0xc8, 0x04, 0xb6, 0x04, 0x00, 0x08})
}

func main() {
	// test read escape 0x1b
	test := bytes.NewBuffer([]byte{0x3d, 0x5, 0x0, 0xf5, 0x1b, 0xe4, 0xaf, 0x41, 0x5, 0x3e})
	buff := bufio.NewReader(test)
	read(buff)
	os.Exit(0)
	data := []byte{0x3d, 0x5, 0x0, 0xff, 0x1b, 0x41, 0x92, 0x3e}
	value := data[3 : len(data)-2]
	if len(value) == 3 {
		value = append(value, 0)
		copy(value[1:], value)
		value[0] = 0x00
	}

	fmt.Println(decode(value))
	os.Exit(0)

	var dialer net.Dialer
	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
	defer cancel()

	conn, err := dialer.DialContext(ctx, "tcp", "192.168.13.57:26486")
	if err != nil {
		log.Fatalf("Failed to dial: %v", err)
	}
	defer conn.Close()
	buf := bufio.NewReader(conn)

	boolPtr := flag.Bool("t", false, "print temperatures")
	flag.Parse()

	if *boolPtr {
		getTemps(buf, conn)
		return
	}

	var a, b, c uint8
	for a = 0; a < 255; a++ {
		for b = 0; b < 255; b++ {
			for c = 0; c < 255; c++ {
				Send(buf, conn, []byte{0xff, 0x1e, 0xc8, 0x04, 0xb6, a, b, c})
			}
		}
	}
}

func compileMsg(b []byte) []byte {
	shouldEscape := []uint8{
		0x3c, // client start
		0x3d, // server response
		0x1b, // escape char
		0x3e, // last in msg
	}

	// calculate checksum
	var s byte
	for _, v := range b {
		s ^= v
	}

	escapeChecksum := false
	for _, b := range shouldEscape {
		if s == b {
			escapeChecksum = true
		}
	}

	//[]byte{0x3c, 0xff, 0x1e, 0xc8, 0x04, 0xb6, 0x03, 0x02, 0x0c, 0x96, 0x3e}
	msg := []byte{0x3c}

	for _, a := range b {
		escape := false
		for _, b := range shouldEscape {
			if a == b {
				escape = true
			}
		}

		if escape {
			msg = append(msg, 0x1b, a^0xff) // add escape and bit invert here
		} else {
			msg = append(msg, a)
		}
	}
	if escapeChecksum {
		msg = append(msg, 0x1b, s^0xff, 0x3e) // add escape and bit invert here
		return msg
	}
	msg = append(msg, []byte{s, 0x3e}...)
	return msg
}

func Send(buf *bufio.Reader, w io.Writer, data []byte) {
	sendMsg := compileMsg(data)
	fmt.Printf("send msg: %x\n", sendMsg)
	_, err := w.Write(sendMsg)
	if err != nil {
		logrus.Error(err)
	}

	read(buf)
}

func read(buf *bufio.Reader) {
	msg, err := buf.ReadBytes(0x3e)
	if err != nil {
		log.Fatalln(err)
	}

	// handle escape byte
	indexesToFlip := []int{}
	n := 0
	for _, v := range msg {
		if v == 0x1b {
			indexesToFlip = append(indexesToFlip, n)
			continue
		}
		msg[n] = v
		n++
	}
	msg = msg[:n]
	for _, v := range indexesToFlip {
		msg[v] ^= 0xff
	}

	fmt.Printf("hex:")
	for _, v := range msg {
		fmt.Printf("%#x ", v)
	}
	fmt.Println()

	if msg[1] != 0x05 && msg[2] != 0x00 {
		return
	}

	f, err := decode(msg[3 : len(msg)-2])
	if err != nil {
		return
	}
	fmt.Println("decode float:", f)
}

func decode(data []byte) (float64, error) {
	// this works OK according to protocol example:
	// Extract temperature 21,8
	// 0000   3d 05 00 d3 5e ae 41 67 3e
	if len(data) != 4 {
		return 0.0, fmt.Errorf("could not pase float from binary")
	}
	bits := binary.LittleEndian.Uint32(data)
	float := float64(math.Float32frombits(bits))
	return math.Round(float*100) / 100, nil
}
<commit_msg>exoline lab stuff<commit_after>package main

import (
	"bufio"
	"context"
	"encoding/binary"
	"flag"
	"fmt"
	"io"
	"log"
	"math"
	"net"
	"time"

	"github.com/sirupsen/logrus"
)

func getTemps(buf *bufio.Reader, w io.Writer) {
	fmt.Println("frånluft") // Extract air
	Send(buf, w, []byte{0xff, 0x1e, 0xc8, 0x04, 0xb6, 0x04, 0x08, 0x00})

	fmt.Println("ute") // Outdoor air
	Send(buf, w, []byte{0xff, 0x1e, 0xc8, 0x04, 0xb6, 0x04, 0x04, 0xec})

	fmt.Println("avluft") // Exhaust air
	Send(buf, w, []byte{0xff, 0x1e, 0xc8, 0x04, 0xb6, 0x3b, 0x00, 0x95})

	fmt.Println("tilluft") // Supply air
	Send(buf, w, []byte{0xff, 0x1e, 0xc8, 0x04, 0xb6, 0x39, 0x02, 0xfa})

	fmt.Println("börvärde") // Set temperature
	Send(buf, w, []byte{0xff, 0x1e, 0xc8, 0x04, 0xb6, 0x04, 0x00, 0x08})
}

func main() {
	// test read escape 0x1b
	// test := bytes.NewBuffer([]byte{0x3d, 0x5, 0x0, 0xf5, 0x1b, 0xe4, 0xaf, 0x41, 0x5, 0x3e})
	// buff := bufio.NewReader(test)
	// read(buff)
	// os.Exit(0)

	//data := []byte{0x3d, 0x5, 0x0, 0xff, 0x1b, 0x41, 0x92, 0x3e}
	//value := data[3 : len(data)-2]
	//if len(value) == 3 {
	//value = append(value, 0)
	//copy(value[1:], value)
	//value[0] = 0x00
	//}

	// fmt.Println(decodeInt(value))
	// os.Exit(0)

	var dialer net.Dialer
	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
	defer cancel()

	conn, err := dialer.DialContext(ctx, "tcp", "192.168.13.57:26486")
	if err != nil {
		log.Fatalf("Failed to dial: %v", err)
	}
	defer conn.Close()
	buf := bufio.NewReader(conn)

	boolPtr := flag.Bool("t", false, "print temperatures")
	flag.Parse()

	if *boolPtr {
		getTemps(buf, conn)
		return
	}

	var a, b, c uint8
	for a = 0; a < 60; a++ {
		for b = 0; b < 60; b++ {
			for c = 0; c < 60; c++ {
				Send(buf, conn, []byte{0xff, 0x1e, 0xc8, 0x04, 0xb6, a, b, c})
			}
		}
	}
}

func compileMsg(b []byte) []byte {
	shouldEscape := []uint8{
		0x3c, // client start
		0x3d, // server response
		0x1b, // escape char
		0x3e, // last in msg
	}

	// calculate checksum
	var s byte
	for _, v := range b {
		s ^= v
	}

	escapeChecksum := false
	for _, b := range shouldEscape {
		if s == b {
			escapeChecksum = true
		}
	}

	//[]byte{0x3c, 0xff, 0x1e, 0xc8, 0x04, 0xb6, 0x03, 0x02, 0x0c, 0x96, 0x3e}
	msg := []byte{0x3c}

	for _, a := range b {
		escape := false
		for _, b := range shouldEscape {
			if a == b {
				escape = true
			}
		}

		if escape {
			msg = append(msg, 0x1b, a^0xff) // add escape and bit invert here
		} else {
			msg = append(msg, a)
		}
	}
	if escapeChecksum {
		msg = append(msg, 0x1b, s^0xff, 0x3e) // add escape and bit invert here
		return msg
	}
	msg = append(msg, []byte{s, 0x3e}...)
	return msg
}

func Send(buf *bufio.Reader, w io.Writer, data []byte) {
	sendMsg := compileMsg(data)
	fmt.Printf("send msg: %x ", sendMsg)
	_, err := w.Write(sendMsg)
	if err != nil {
		logrus.Error(err)
	}

	read(buf)
}

func read(buf *bufio.Reader) {
	msg, err := buf.ReadBytes(0x3e)
	if err != nil {
		log.Fatalln(err)
	}

	// handle escape byte
	indexesToFlip := []int{}
	n := 0
	for _, v := range msg {
		if v == 0x1b {
			indexesToFlip = append(indexesToFlip, n)
			continue
		}
		msg[n] = v
		n++
	}
	msg = msg[:n]
	for _, v := range indexesToFlip {
		msg[v] ^= 0xff
	}

	fmt.Printf("resp:")
	for _, v := range msg {
		fmt.Printf("%#x ", v)
	}
	// fmt.Println()

	//if msg[1] != 0x05 && msg[2] != 0x00 {
	//return
	//}

	f, _ := decodeFloat(msg[3 : len(msg)-2])
	//if err != nil {
	//if err != nil {
	//return
	//}
	//return
	//}
	if f != 0.0 {
		fmt.Print(" decode float:", f)
	}
	i, _ := decodeIntLittleEndian(msg[3 : len(msg)-2])
	if i != 0 {
		fmt.Print(" decode int littleEndian:", i)
	}
	i2, _ := decodeIntBigEndian(msg[3 : len(msg)-2])
	if i2 != 0 {
		fmt.Print(" decode int bigEndian:", i2)
	}
	fmt.Println()
}

func decodeIntBigEndian(data []byte) (int, error) {
	if len(data) == 4 {
		bits := binary.BigEndian.Uint32(data)
		return int(bits), nil
	}
	if len(data) == 2 {
		bits := binary.BigEndian.Uint16(data)
		return int(bits), nil
	}
	return 0, fmt.Errorf("could not parse int from binary")
}

func decodeIntLittleEndian(data []byte) (int, error) {
	if len(data) == 4 {
		bits := binary.LittleEndian.Uint32(data)
		return int(bits), nil
	}
	if len(data) == 2 {
		bits := binary.LittleEndian.Uint16(data)
		return int(bits), nil
	}
	return 0, fmt.Errorf("could not parse int from binary")
}

func decodeFloat(data []byte) (float64, error) {
	// this works OK according to protocol example:
	// Extract temperature 21,8
	// 0000   3d 05 00 d3 5e ae 41 67 3e
	if len(data) != 4 {
		return 0.0, fmt.Errorf("could not pase float from binary")
	}
	bits := binary.LittleEndian.Uint32(data)
	float := float64(math.Float32frombits(bits))
	return math.Round(float*100) / 100, nil
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	"github.com/cayleygraph/cayley"
	"github.com/cayleygraph/cayley/graph"
	_ "github.com/aselus-hub/cayley/graph/sql"
	"github.com/cayleygraph/cayley/quad"
	"github.com/satori/go.uuid"
	"log"
	"math/rand"
	"sync"
	"time"
)

const insecureCdbPath = "postgresql://root@127.0.0.1:26257/"

const runTime = 60*time.Second
const batchSize = 15

type node struct {
	parent string
	child  string
	id     string
}

func generateAndSendAGraph(wg *sync.WaitGroup, sumChan chan uint64, quit chan struct{}) {
	defer wg.Done()
	var numProccessed uint64
	if store, err := InitCayley(insecureCdbPath + "cayley?sslmode=disable"); err != nil {
		fmt.Printf("Could not initialize cayley, %s\n", err.Error())
		return
	} else {
		defer store.Close()
		tx := cayley.NewTransaction()
		for {
			select {
			case <-quit:
				tx = commitTx(store, tx, true)
				sumChan <- numProccessed
				return
			default:
				nodes := generateGraph(store)
				for _, n := range nodes {
					n = n
					addNquadsForNode(n, tx)
					tx = commitTx(store, tx, false)
					numProccessed++
				}
			}
		}
	}
}

func generateGraph(store *cayley.Handle) []node {
	// create a lineage for connecting A through link to B, etc.
	numItems := 2 + rand.Intn(10)
	// permutations := rand.Perm(numItems)
	// create a list of nodes, with number of true items and number of links.
	nodes := make([]node, numItems, numItems)
	currentUUID := ""
	for index := range nodes {
		newUuid := uuid.NewV4().String()
		nodes[index] = node{parent: currentUUID, id: uuid.NewV4().String(), child: currentUUID}
		currentUUID = newUuid
	}
	return nodes
}

func addNquadsForNode(n node, tx *graph.Transaction)  {
	if n.parent != "" {
		tx.AddQuad(quad.Make(n.parent, "related_through", n.id, nil))
	}

	if n.child != "" {
		tx.AddQuad(quad.Make(n.id, "related_through", n.child, nil))
	}
}

func commitTx(store *cayley.Handle, tx *graph.Transaction, force bool) *graph.Transaction {
	if len(tx.Deltas) > batchSize || force {
		err := store.ApplyTransaction(tx)
		if err != nil {
			panic(err.Error())
		}
		return cayley.NewTransaction()
	} else {
		return tx
	}
}

const NumRoutines = 8

func main() {
	rand.Seed(time.Now().UTC().UnixNano())
	SetupCayleyInCdb()
	termination := sync.WaitGroup{}
	termination.Add(NumRoutines)
	var trueEnd time.Duration
	var totalSum uint64
	processedNodes := make(chan uint64, NumRoutines)
	quit := make(chan struct{})

	start := time.Now()
	for i := 0; i < NumRoutines; i++ {
		go generateAndSendAGraph(&termination, processedNodes, quit)
	}
	for time.Since(start) < runTime {
		time.Sleep(time.Millisecond * 10)
	}
	close(quit)
	termination.Wait()
	trueEnd = time.Since(start)

outerFor:
	for {
		select {
		case subSum := <-processedNodes:
			totalSum += subSum
		default:
			break outerFor
		}
	}

	fmt.Printf("time elapsed: %v\n", trueEnd)
	fmt.Printf("total processed: %v\n", totalSum)
	fmt.Printf("sets/second: %v\n", float64(totalSum) / trueEnd.Seconds())
}

func SetupCayleyInCdb() {
	db, err := sql.Open("postgres", insecureCdbPath+"system?sslmode=disable")
	if err != nil {
		log.Fatalf("Unable to initialize cockroach: %s", err.Error())
	}
	defer db.Close()
	_, err = db.Exec("CREATE DATABASE IF NOT EXISTS cayley")
	if err != nil {
		log.Fatalf("Unable to initialize cockroach: %s", err.Error())
	}

	if err = graph.InitQuadStore("sql", insecureCdbPath+"cayley?sslmode=disable", graph.Options{"flavor": "cockroach"}); err != nil {
		if err != graph.ErrDatabaseExists {
			panic(err.Error())
		}
	}

}

func InitCayley(pathToCockroachDB string) (store *cayley.Handle, err error) {
	store, err = cayley.NewGraph("sql", pathToCockroachDB, graph.Options{"flavor": "cockroach"})
	if err != nil {
		return
	}
	return
}
<commit_msg>updated to have cleaner exit/more matching machine pattern w/ 4 cores<commit_after>package main

import (
	"database/sql"
	"fmt"
	"github.com/cayleygraph/cayley"
	"github.com/cayleygraph/cayley/graph"
	_ "github.com/aselus-hub/cayley/graph/sql"
	"github.com/cayleygraph/cayley/quad"
	"github.com/satori/go.uuid"
	"log"
	"math/rand"
	"sync"
	"time"
)

const insecureCdbPath = "postgresql://root@127.0.0.1:26257/"

const runTime = 1*time.Second
const batchSize = 15

type node struct {
	parent string
	child  string
	id     string
}

func generateAndSendAGraph(wg *sync.WaitGroup, sumChan chan uint64, quit chan struct{}) {
	defer wg.Done()
	var numProccessed uint64
	if store, err := InitCayley(insecureCdbPath + "cayley?sslmode=disable"); err != nil {
		fmt.Printf("Could not initialize cayley, %s\n", err.Error())
		return
	} else {
		defer store.Close()
		tx := cayley.NewTransaction()
		for {
			select {
			case <-quit:
				tx = commitTx(store, tx, true)
				sumChan <- numProccessed
				return
			default:
				nodes := generateGraph(store)
				for _, n := range nodes {
					n = n
					addNquadsForNode(n, tx)
					tx = commitTx(store, tx, false)
					numProccessed++
				}
			}
		}
	}
}

func generateGraph(store *cayley.Handle) []node {
	// create a lineage for connecting A through link to B, etc.
	numItems := 2 + rand.Intn(10)
	// permutations := rand.Perm(numItems)
	// create a list of nodes, with number of true items and number of links.
	nodes := make([]node, numItems, numItems)
	currentUUID := ""
	for index := range nodes {
		newUuid := uuid.NewV4().String()
		nodes[index] = node{parent: currentUUID, id: uuid.NewV4().String(), child: currentUUID}
		currentUUID = newUuid
	}
	return nodes
}

func addNquadsForNode(n node, tx *graph.Transaction)  {
	if n.parent != "" {
		tx.AddQuad(quad.Make(n.parent, "related_through", n.id, nil))
	}

	if n.child != "" {
		tx.AddQuad(quad.Make(n.id, "related_through", n.child, nil))
	}
}

func commitTx(store *cayley.Handle, tx *graph.Transaction, force bool) *graph.Transaction {
	if len(tx.Deltas) > batchSize || force {
		err := store.ApplyTransaction(tx)
		if err != nil {
			panic(err.Error())
		}
		return cayley.NewTransaction()
	} else {
		return tx
	}
}

const NumRoutines = 4

func main() {
	rand.Seed(time.Now().UTC().UnixNano())
	SetupCayleyInCdb()
	termination := sync.WaitGroup{}
	termination.Add(NumRoutines)
	var trueEnd time.Duration
	var totalSum uint64
	processedNodes := make(chan uint64, NumRoutines)
	quit := make(chan struct{})

	start := time.Now()
	for i := 0; i < NumRoutines; i++ {
		go generateAndSendAGraph(&termination, processedNodes, quit)
	}
	for time.Since(start) < runTime {
		time.Sleep(time.Millisecond * 10)
	}
	close(quit)
	termination.Wait()
	trueEnd = time.Since(start)

outerFor:
	for {
		select {
		case subSum := <-processedNodes:
			totalSum += subSum
		default:
			break outerFor
		}
	}

	fmt.Printf("time elapsed: %v\n", trueEnd)
	fmt.Printf("total processed: %v\n", totalSum)
	fmt.Printf("sets/second: %v\n", float64(totalSum) / trueEnd.Seconds())
}

func SetupCayleyInCdb() {
	db, err := sql.Open("postgres", insecureCdbPath+"system?sslmode=disable")
	if err != nil {
		log.Fatalf("Unable to initialize cockroach: %s", err.Error())
	}
	defer db.Close()
	_, err = db.Exec("CREATE DATABASE IF NOT EXISTS cayley")
	if err != nil {
		log.Fatalf("Unable to initialize cockroach: %s", err.Error())
	}

	if err = graph.InitQuadStore("sql", insecureCdbPath+"cayley?sslmode=disable", graph.Options{"flavor": "cockroach"}); err != nil {
		if err != graph.ErrDatabaseExists {
			panic(err.Error())
		}
	}

}

func InitCayley(pathToCockroachDB string) (store *cayley.Handle, err error) {
	store, err = cayley.NewGraph("sql", pathToCockroachDB, graph.Options{"flavor": "cockroach"})
	if err != nil {
		return
	}
	return
}
<|endoftext|>


<commit_before>package device

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/ScriptRock/crypto/ssh"
)

type CiscoDevice struct {
	Username  string
	Password  string
	Enable    string
	name      string
	Hostname  string
	stdin     io.WriteCloser
	stdout    io.Reader
	session   *ssh.Session
	Echo      bool
	EnableLog bool
	Logdir    string
	Log       *os.File
	Prompt    string
}

func (d *CiscoDevice) Connect() error {
	config := &ssh.ClientConfig{
		User: d.Username,
		Auth: []ssh.AuthMethod{
			ssh.Password(d.Password),
		},
		Config: ssh.Config{
			Ciphers: ssh.AllSupportedCiphers(),
		},
	}
	client, err := ssh.Dial("tcp", d.Hostname+":22", config)
	if err != nil {
		return err
	}
	session, err := client.NewSession()
	if err != nil {
		return err
	}
	d.stdin, _ = session.StdinPipe()
	d.stdout, _ = session.StdoutPipe()
	d.Echo = true
	d.EnableLog = true
	modes := ssh.TerminalModes{
		ssh.ECHO:          0, // disable echoing
		ssh.OCRNL:         0,
		ssh.TTY_OP_ISPEED: 38400, // input speed = 14.4kbaud
		ssh.TTY_OP_OSPEED: 38400, // output speed = 14.4kbaud
	}
	session.RequestPty("vt100", 0, 2000, modes)
	session.Shell()
	if d.Logdir != "" {
		t := time.Now()
		d.Log, err = os.OpenFile(filepath.Join(d.Logdir, t.Format("200601021504")+"-"+d.Hostname), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			return err
		}
	}
	d.init()
	d.session = session
	return nil
}

func (d *CiscoDevice) Close() {
	d.session.Close()
}

func (d *CiscoDevice) Cmd(cmd string) (string, error) {
	bufstdout := bufio.NewReader(d.stdout)
	lines := strings.Split(cmd, "!")
	for _, line := range lines {
		io.WriteString(d.stdin, line+"\n")
		time.Sleep(time.Millisecond * 100)
	}
	output, err := d.readln(bufstdout)
	if err != nil {
		return "", err
	}
	output = strings.Replace(output, "\r", "", -1)
	if d.Echo == false {
		output = strings.Replace(output, lines[0], "", 1)
	}
	if d.Logdir != "" {
		return "", nil
	}
	return output, nil
}

func (d *CiscoDevice) init() {
	bufstdout := bufio.NewReader(d.stdout)
	io.WriteString(d.stdin, "enable\n")
	time.Sleep(time.Millisecond * 100)
	re := regexp.MustCompile("assword:")
	buf := make([]byte, 1000)
	loadStr := ""
	for {
		n, err := bufstdout.Read(buf)
		if err != nil {
			log.Fatal(err)
		}
		loadStr += string(buf[:n])
		if re.MatchString(loadStr) {
			io.WriteString(d.stdin, d.Enable+"\n")
			break
		} else {
			break
		}
	}
	d.Cmd("terminal length 0")
	d.Cmd("")
	prompt, _ := d.Cmd("")
	d.Prompt = strings.TrimSpace(prompt)
}

func (d *CiscoDevice) readln(r *bufio.Reader) (string, error) {
	var re *regexp.Regexp
	if d.Prompt == "" {
		re = regexp.MustCompile("[[:alnum:]]#.?$")
	} else {
		re = regexp.MustCompile(d.Prompt + ".?$")
	}
	buf := make([]byte, 10000)
	loadStr := ""
	for {
		n, err := r.Read(buf)
		if err != nil {
			return "", err
		}
		loadStr += string(buf[:n])
		// logging to file if necessary
		if d.Logdir != "" {
			if d.EnableLog {
				fmt.Fprint(d.Log, string(buf[:n]))
			}
		}
		if re.MatchString(string(buf[:n])) {
			break
		}
	}
	return loadStr, nil
}
<commit_msg>Fix for <conf t> not working because of other prompt<commit_after>package device

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/ScriptRock/crypto/ssh"
)

type CiscoDevice struct {
	Username  string
	Password  string
	Enable    string
	name      string
	Hostname  string
	stdin     io.WriteCloser
	stdout    io.Reader
	session   *ssh.Session
	Echo      bool
	EnableLog bool
	Logdir    string
	Log       *os.File
	Prompt    string
}

func (d *CiscoDevice) Connect() error {
	config := &ssh.ClientConfig{
		User: d.Username,
		Auth: []ssh.AuthMethod{
			ssh.Password(d.Password),
		},
		Config: ssh.Config{
			Ciphers: ssh.AllSupportedCiphers(),
		},
	}
	client, err := ssh.Dial("tcp", d.Hostname+":22", config)
	if err != nil {
		return err
	}
	session, err := client.NewSession()
	if err != nil {
		return err
	}
	d.stdin, _ = session.StdinPipe()
	d.stdout, _ = session.StdoutPipe()
	d.Echo = true
	d.EnableLog = true
	modes := ssh.TerminalModes{
		ssh.ECHO:          0, // disable echoing
		ssh.OCRNL:         0,
		ssh.TTY_OP_ISPEED: 38400, // input speed = 14.4kbaud
		ssh.TTY_OP_OSPEED: 38400, // output speed = 14.4kbaud
	}
	session.RequestPty("vt100", 0, 2000, modes)
	session.Shell()
	if d.Logdir != "" {
		t := time.Now()
		d.Log, err = os.OpenFile(filepath.Join(d.Logdir, t.Format("200601021504")+"-"+d.Hostname), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			return err
		}
	}
	d.init()
	d.session = session
	return nil
}

func (d *CiscoDevice) Close() {
	d.session.Close()
}

func (d *CiscoDevice) Cmd(cmd string) (string, error) {
	bufstdout := bufio.NewReader(d.stdout)
	lines := strings.Split(cmd, "!")
	for _, line := range lines {
		io.WriteString(d.stdin, line+"\n")
		time.Sleep(time.Millisecond * 100)
	}
	output, err := d.readln(bufstdout)
	if err != nil {
		return "", err
	}
	output = strings.Replace(output, "\r", "", -1)
	if d.Echo == false {
		output = strings.Replace(output, lines[0], "", 1)
	}
	if d.Logdir != "" {
		return "", nil
	}
	return output, nil
}

func (d *CiscoDevice) init() {
	bufstdout := bufio.NewReader(d.stdout)
	io.WriteString(d.stdin, "enable\n")
	time.Sleep(time.Millisecond * 100)
	re := regexp.MustCompile("assword:")
	buf := make([]byte, 1000)
	loadStr := ""
	for {
		n, err := bufstdout.Read(buf)
		if err != nil {
			log.Fatal(err)
		}
		loadStr += string(buf[:n])
		if re.MatchString(loadStr) {
			io.WriteString(d.stdin, d.Enable+"\n")
			break
		} else {
			break
		}
	}
	d.Cmd("terminal length 0")
	d.Cmd("")
	prompt, _ := d.Cmd("")
	d.Prompt = strings.TrimSpace(prompt)
	d.Prompt = strings.Replace(d.Prompt, "#", "", -1)
}

func (d *CiscoDevice) readln(r *bufio.Reader) (string, error) {
	var re *regexp.Regexp
	if d.Prompt == "" {
		re = regexp.MustCompile("[[:alnum:]]#.?$")
	} else {
		re = regexp.MustCompile(d.Prompt + ".*?#$")
	}
	buf := make([]byte, 10000)
	loadStr := ""
	for {
		n, err := r.Read(buf)
		if err != nil {
			return "", err
		}
		loadStr += string(buf[:n])
		// logging to file if necessary
		if d.Logdir != "" {
			if d.EnableLog {
				fmt.Fprint(d.Log, string(buf[:n]))
			}
		}
		if re.MatchString(string(buf[:n])) {
			break
		}
	}
	return loadStr, nil
}
<|endoftext|>


<commit_before>package main

/*

infra.tf
payload.json (json description of vm)
resource.json (status tracking)
terraform.tfstate ()

*/

import (
	"encoding/json"
	"fmt"
	"io"
	"strings"

	"io/ioutil"

	"bytes"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/perigee/terrant/app"
	"github.com/perigee/terrant/design"
)

const (
	terraBUCKETNAME     = "autoterrarepostate"
	terraDEFAULTREGION  = "us-east-1"
	terraTerraformState = "infra.tf"
	terraStatusFile     = "resource.json"
	terraAttributeFile  = "attributes.json"
)

type ResourceInfo struct {
	ResourceID string `json:resourceId`
	Status     string `json:status, string`
}

func downloadS3object(s3client *s3.S3, key string) ([]byte, error) {
	res, err := s3client.GetObject(&s3.GetObjectInput{
		Bucket: aws.String(terraBUCKETNAME),
		Key:    aws.String(key),
	})

	if err != nil {
		return nil, err
	}

	defer res.Body.Close()

	buf := bytes.NewBuffer(nil)

	if _, err := io.Copy(buf, res.Body); err != nil {
		return nil, err

	}

	return buf.Bytes(), nil
}

func uploadS3object(s3client *s3.S3, key string, body []byte) error {

	_, err := s3client.PutObject(&s3.PutObjectInput{
		Bucket: aws.String(terraBUCKETNAME),
		Key:    aws.String(key),
		Body:   bytes.NewReader(body),
	})

	if err != nil {
		fmt.Println(err.Error())
		return err
	}

	return nil
}

func s3KeyGen(ctx *app.CreateChefContext, filename string) string {
	okey := make([]string, 3)

	okey[0] = ctx.Request.Header.Get(design.SPACEID)
	okey[1] = ctx.Payload.Vmuid
	okey[2] = filename
	return strings.Join(okey, "/")
}

// FetchObject fetch the object
func FetchObject(ctx *app.CreateChefContext) ([]byte, error) {
	sess, err := session.NewSession()

	if err != nil {
		panic(err)
	}

	svc := s3.New(sess, &aws.Config{Region: aws.String(terraDEFAULTREGION)})

	/*  ===== DOWNLOAD/UPLOAD Tesitng
	resp, err := downloadS3object(svc, s3KeyGen(ctx, terraStatusFile))

	if err != nil {
		fmt.Println(err.Error())
		return nil, err
	}

	if err := uploadS3object(svc, s3KeyGen(ctx, "jun_tmp_file"), resp); err != nil {
		fmt.Println(err.Error())
		return nil, err
	}*/

	state, err := CheckJobStatus(svc, s3KeyGen(ctx, terraStatusFile))

	if err != nil {
		fmt.Println(err.Error())
		return nil, err
	}

	return []byte(state), nil
}

// CheckJobStatus verifies the status of current resource
func CheckJobStatus(s3client *s3.S3, key string) (string, error) {

	var resinfo ResourceInfo

	res, err := downloadS3object(s3client, key)

	if err != nil {
		fmt.Println(err.Error())
		return "", nil
	}

	if err := json.Unmarshal(res, &resinfo); err != nil {
		fmt.Println(err.Error())
		return "", nil
	}

	return resinfo.Status, nil
}

// UpdateTerraFile updates
func UpdateTerraFile(s3obj *s3.GetObjectOutput) (*s3.GetObjectOutput, error) {
	return nil, nil
}

// UploadObject upload the object on s3
func UploadObject() error {
	return nil
}

// JSON2File creates the file based on given json
/*
    if err := JSON2File(ctx.Payload.NodeAttributes, "atrb.json"); err != nil {
		panic(err)
	}
*/
func JSON2File(in *interface{}, filename string) error {
	jsonStr, err := json.Marshal(*in)
	if err != nil {
		fmt.Println(err.Error())
		return err
	}
	ioutil.WriteFile(filename, jsonStr, 0644)
	return nil
}
<commit_msg>using jeffail/gabs pkg<commit_after>package main

/*

infra.tf
payload.json (json description of vm)
resource.json (status tracking)
terraform.tfstate ()

*/

import (
	"encoding/json"
	"fmt"
	"io"
	"strings"

	"io/ioutil"

	"bytes"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/jeffail/gabs"
	"github.com/perigee/terrant/app"
	"github.com/perigee/terrant/design"
)

const (
	terraBUCKETNAME     = "autoterrarepostate"
	terraDEFAULTREGION  = "us-east-1"
	terraTerraformState = "infra.tf"
	terraStatusFile     = "resource.json"
	terraAttributeFile  = "attributes.json"
)

func downloadS3object(s3client *s3.S3, key string) ([]byte, error) {
	res, err := s3client.GetObject(&s3.GetObjectInput{
		Bucket: aws.String(terraBUCKETNAME),
		Key:    aws.String(key),
	})

	if err != nil {
		return nil, err
	}

	defer res.Body.Close()

	buf := bytes.NewBuffer(nil)

	if _, err := io.Copy(buf, res.Body); err != nil {
		return nil, err

	}

	return buf.Bytes(), nil
}

func uploadS3object(s3client *s3.S3, key string, body []byte) error {

	_, err := s3client.PutObject(&s3.PutObjectInput{
		Bucket: aws.String(terraBUCKETNAME),
		Key:    aws.String(key),
		Body:   bytes.NewReader(body),
	})

	if err != nil {
		fmt.Println(err.Error())
		return err
	}

	return nil
}

func s3KeyGen(ctx *app.CreateChefContext, filename string) string {
	okey := make([]string, 3)

	okey[0] = ctx.Request.Header.Get(design.SPACEID)
	okey[1] = ctx.Payload.Vmuid
	okey[2] = filename
	return strings.Join(okey, "/")
}

// FetchObject fetch the object
func FetchObject(ctx *app.CreateChefContext) ([]byte, error) {
	sess, err := session.NewSession()

	if err != nil {
		panic(err)
	}

	svc := s3.New(sess, &aws.Config{Region: aws.String(terraDEFAULTREGION)})

	/*  ===== DOWNLOAD/UPLOAD Tesitng
	resp, err := downloadS3object(svc, s3KeyGen(ctx, terraStatusFile))

	if err != nil {
		fmt.Println(err.Error())
		return nil, err
	}

	if err := uploadS3object(svc, s3KeyGen(ctx, "jun_tmp_file"), resp); err != nil {
		fmt.Println(err.Error())
		return nil, err
	}*/

	resp, err := CheckJobStatus(ctx, svc, s3KeyGen(ctx, terraStatusFile))

	if err != nil {
		fmt.Println(err.Error())
		return nil, err
	}

	if err := uploadS3object(svc, s3KeyGen(ctx, "jun_tmp_file"), []byte(resp)); err != nil {
		fmt.Println(err.Error())
		return nil, err
	}

	return []byte(resp), nil
}

// CheckJobStatus verifies the status of current resource
func CheckJobStatus(ctx *app.CreateChefContext, s3client *s3.S3, key string) ([]byte, error) {

	res, err := downloadS3object(s3client, key)

	if err != nil {
		fmt.Println(err.Error())
		return nil, err
	}

	jsonObj, err := gabs.ParseJSON(res)

	if err != nil {
		fmt.Println(err.Error())
		return nil, err
	}

	value, ok := jsonObj.Path("status").Data().(string)

	jsonObj.SetP("Completed", "status")

	// test upload modified file
	jsonStr := jsonObj.String()

	fmt.Println(jsonStr)

	if err := uploadS3object(s3client, s3KeyGen(ctx, "jun_tmp_file.json"), []byte(jsonStr)); err != nil {
		fmt.Println(err.Error())
		return nil, err
	}

	if ok {
		return []byte(value), nil
	}

	return []byte(""), nil
}

// UpdateTerraFile updates
func UpdateTerraFile(s3client *s3.S3, key string) ([]byte, error) {
	return nil, nil
}

// UploadObject upload the object on s3
func UploadObject() error {
	return nil
}

// JSON2File creates the file based on given json
/*
    if err := JSON2File(ctx.Payload.NodeAttributes, "atrb.json"); err != nil {
		panic(err)
	}
*/
func JSON2File(in *interface{}, filename string) error {
	jsonStr, err := json.Marshal(*in)
	if err != nil {
		fmt.Println(err.Error())
		return err
	}
	ioutil.WriteFile(filename, jsonStr, 0644)
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2012, Braille Printer Team. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

/*  Filename:    options.go
 *  Author:      Leonardo YongUkKim <dalinaum@gmail.com>,
 *               Homin Lee <homin.lee@suapapa.net>
 *  Created:     2012-10-13 12:44:45.83294 +0900 KST
 *  Description: Option parsing for braille-printer-client
 */

import (
	"flag"
	"fmt"
	"log"
	"os"
)

// TODO Customize exported (capitalized) variables, types, and functions.

var (
	CmdHelpUsage string // Custom usage string.
	CmdHelpFoot  string // Printed after help.
)

// A struct that holds braille-printer-client's parsed command line flags.
type Options struct {
	Verbose    bool
	ServerAddr string
	Lang       string
	Format     string
}

//  Create a flag.FlagSet to parse the braille-printer-client's flags.
func SetupFlags(opt *Options) *flag.FlagSet {
	fs := flag.NewFlagSet("braille-printer-client", flag.ExitOnError)
	fs.BoolVar(&opt.Verbose, "v", false, "Verbose program output.")
	fs.StringVar(&opt.ServerAddr, "a", "http://localhost:8080",
		"Address of braille-printer (server)")
	fs.StringVar(&opt.Lang, "l", "ko", "Braille language {ko|en}")
	fs.StringVar(&opt.Format, "f", "svg", "Format to print out")
	return setupUsage(fs)
}

// Check the braille-printer-client's flags and arguments for acceptable values.
// When an error is encountered, panic, exit with a non-zero status, or override
// the error.
func VerifyFlags(opt *Options, fs *flag.FlagSet) {
	switch opt.Lang {
	case "ko", "en":
	default:
		log.Fatalf("Unknown lang, %s! Use one of ko, en.", opt.Lang)
	}

	switch opt.Format {
	case "text", "svg":
	default:
		log.Fatalf("Unkonw format, %s! Use one of text or svg.",
			opt.Format)
	}
}

//  Print a help message to standard error. See CmdHelpUsage and CmdHelpFoot.
func PrintHelp() { SetupFlags(&Options{}).Usage() }

//  Hook up CmdHelpUsage and CmdHelpFoot with flag defaults to function flag.Usage.
func setupUsage(fs *flag.FlagSet) *flag.FlagSet {
	printNonEmpty := func(s string) {
		if s != "" {
			fmt.Fprintf(os.Stderr, "%s\n", s)
		}
	}
	fs.Usage = func() {
		printNonEmpty(CmdHelpUsage)
		fs.PrintDefaults()
		printNonEmpty(CmdHelpFoot)
	}
	return fs
}

//  Parse the flags, validate them, and post-process (e.g. Initialize more complex structs).
func parseFlags() (Options, []string) {
	var opt Options
	fs := SetupFlags(&opt)
	fs.Parse(os.Args[1:])
	VerifyFlags(&opt, fs)
	// Process the verified Options...
	return opt, fs.Args()
}
<commit_msg>options: Change default server address to actual braille-printer server<commit_after>// Copyright 2012, Braille Printer Team. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

/*  Filename:    options.go
 *  Author:      Leonardo YongUkKim <dalinaum@gmail.com>,
 *               Homin Lee <homin.lee@suapapa.net>
 *  Created:     2012-10-13 12:44:45.83294 +0900 KST
 *  Description: Option parsing for braille-printer-client
 */

import (
	"flag"
	"fmt"
	"log"
	"os"
)

// TODO Customize exported (capitalized) variables, types, and functions.

var (
	CmdHelpUsage string // Custom usage string.
	CmdHelpFoot  string // Printed after help.
)

// A struct that holds braille-printer-client's parsed command line flags.
type Options struct {
	Verbose    bool
	ServerAddr string
	Lang       string
	Format     string
}

//  Create a flag.FlagSet to parse the braille-printer-client's flags.
func SetupFlags(opt *Options) *flag.FlagSet {
	fs := flag.NewFlagSet("braille-printer-client", flag.ExitOnError)
	fs.BoolVar(&opt.Verbose, "v", false, "Verbose program output.")
	fs.StringVar(&opt.ServerAddr, "a", "http://braille-printer.appspot.com",
		"Address of braille-printer (server)")
	fs.StringVar(&opt.Lang, "l", "ko", "Braille language {ko|en}")
	fs.StringVar(&opt.Format, "f", "svg", "Format to print out {svg|text}")
	return setupUsage(fs)
}

// Check the braille-printer-client's flags and arguments for acceptable values.
// When an error is encountered, panic, exit with a non-zero status, or override
// the error.
func VerifyFlags(opt *Options, fs *flag.FlagSet) {
	switch opt.Lang {
	case "ko", "en":
	default:
		log.Fatalf("Unknown lang, %s! Use one of ko, en.", opt.Lang)
	}

	switch opt.Format {
	case "text", "svg":
	default:
		log.Fatalf("Unkonw format, %s! Use one of text or svg.",
			opt.Format)
	}
}

//  Print a help message to standard error. See CmdHelpUsage and CmdHelpFoot.
func PrintHelp() { SetupFlags(&Options{}).Usage() }

//  Hook up CmdHelpUsage and CmdHelpFoot with flag defaults to function flag.Usage.
func setupUsage(fs *flag.FlagSet) *flag.FlagSet {
	printNonEmpty := func(s string) {
		if s != "" {
			fmt.Fprintf(os.Stderr, "%s\n", s)
		}
	}
	fs.Usage = func() {
		printNonEmpty(CmdHelpUsage)
		fs.PrintDefaults()
		printNonEmpty(CmdHelpFoot)
	}
	return fs
}

//  Parse the flags, validate them, and post-process (e.g. Initialize more complex structs).
func parseFlags() (Options, []string) {
	var opt Options
	fs := SetupFlags(&opt)
	fs.Parse(os.Args[1:])
	VerifyFlags(&opt, fs)
	// Process the verified Options...
	return opt, fs.Args()
}
<|endoftext|>


<commit_before>package gorocks

// #include "rocksdb/c.h"
import "C"

// CompressionOpt is a value for Options.SetCompression.
type CompressionOpt int

// Known compression arguments for Options.SetCompression.
const (
	NoCompression     = CompressionOpt(0)
	SnappyCompression = CompressionOpt(1)
)

// Options represent all of the available options when opening a database with
// Open. Options should be created with NewOptions.
//
// It is usually with to call SetCache with a cache object. Otherwise, all
// data will be read off disk.
//
// To prevent memory leaks, Close must be called on an Options when the
// program no longer needs it.
type Options struct {
	Opt *C.rocksdb_options_t
}

// ReadOptions represent all of the available options when reading from a
// database.
//
// To prevent memory leaks, Close must called on a ReadOptions when the
// program no longer needs it.
type ReadOptions struct {
	Opt *C.rocksdb_readoptions_t
}

// WriteOptions represent all of the available options when writeing from a
// database.
//
// To prevent memory leaks, Close must called on a WriteOptions when the
// program no longer needs it.
type WriteOptions struct {
	Opt *C.rocksdb_writeoptions_t
}

// NewOptions allocates a new Options object.
func NewOptions() *Options {
	opt := C.rocksdb_options_create()
	return &Options{opt}
}

// NewReadOptions allocates a new ReadOptions object.
func NewReadOptions() *ReadOptions {
	opt := C.rocksdb_readoptions_create()
	return &ReadOptions{opt}
}

// NewWriteOptions allocates a new WriteOptions object.
func NewWriteOptions() *WriteOptions {
	opt := C.rocksdb_writeoptions_create()
	return &WriteOptions{opt}
}

// Close deallocates the Options, freeing its underlying C struct.
func (o *Options) Close() {
	C.rocksdb_options_destroy(o.Opt)
}

// SetComparator sets the comparator to be used for all read and write
// operations.
//
// The comparator that created a database must be the same one (technically,
// one with the same name string) that is used to perform read and write
// operations.
//
// The default comparator is usually sufficient.
func (o *Options) SetComparator(cmp *C.rocksdb_comparator_t) {
	C.rocksdb_options_set_comparator(o.Opt, cmp)
}

// SetErrorIfExists, if passed true, will cause the opening of a database that
// already exists to throw an error.
func (o *Options) SetErrorIfExists(error_if_exists bool) {
	eie := boolToUchar(error_if_exists)
	C.rocksdb_options_set_error_if_exists(o.Opt, eie)
}

// SetCache places a cache object in the database when a database is opened.
//
// This is usually wise to use. See also ReadOptions.SetFillCache.
func (o *Options) SetCache(cache *Cache) {
	C.rocksdb_options_set_cache(o.Opt, cache.Cache)
}

// SetEnv sets the Env object for the new database handle.
func (o *Options) SetEnv(env *Env) {
	C.rocksdb_options_set_env(o.Opt, env.Env)
}

// SetInfoLog sets a *C.rocksdb_logger_t object as the informational logger
// for the database.
func (o *Options) SetInfoLog(log *C.rocksdb_logger_t) {
	C.rocksdb_options_set_info_log(o.Opt, log)
}

// SetWriteBufferSize sets the number of bytes the database will build up in
// memory (backed by an unsorted log on disk) before converting to a sorted
// on-disk file.
func (o *Options) SetWriteBufferSize(s int) {
	C.rocksdb_options_set_write_buffer_size(o.Opt, C.size_t(s))
}

func (o *Options) SetMaxWriteBuffers(s int) {
	C.rocksdb_options_set_max_write_buffer_number(o.Opt, C.int(s))
}

func (o *Options) SetMinWriteBuffersToMerge(s int) {
	C.rocksdb_options_set_min_write_buffer_number_to_merge(o.Opt, C.int(s))
}

// SetParanoidChecks, when called with true, will cause the database to do
// aggressive checking of the data it is processing and will stop early if it
// detects errors.
//
// See the LevelDB documentation docs for details.
func (o *Options) SetParanoidChecks(pc bool) {
	C.rocksdb_options_set_paranoid_checks(o.Opt, boolToUchar(pc))
}

// SetMaxOpenFiles sets the number of files than can be used at once by the
// database.
//
// See the LevelDB documentation for details.
func (o *Options) SetMaxOpenFiles(n int) {
	C.rocksdb_options_set_max_open_files(o.Opt, C.int(n))
}

// SetBlockSize sets the approximate size of user data packed per block.
//
// The default is roughly 4096 uncompressed bytes. A better setting depends on
// your use case. See the LevelDB documentation for details.
func (o *Options) SetBlockSize(s int) {
	C.rocksdb_options_set_block_size(o.Opt, C.size_t(s))
}

// SetBlockRestartInterval is the number of keys between restarts points for
// delta encoding keys.
//
// Most clients should leave this parameter alone. See the LevelDB
// documentation for details.
func (o *Options) SetBlockRestartInterval(n int) {
	C.rocksdb_options_set_block_restart_interval(o.Opt, C.int(n))
}

// SetCompression sets whether to compress blocks using the specified
// compresssion algorithm.
//
// The default value is SnappyCompression and it is fast enough that it is
// unlikely you want to turn it off. The other option is NoCompression.
//
// If the LevelDB library was built without Snappy compression enabled, the
// SnappyCompression setting will be ignored.
func (o *Options) SetCompression(t CompressionOpt) {
	C.rocksdb_options_set_compression(o.Opt, C.int(t))
}

// SetCreateIfMissing causes Open to create a new database on disk if it does
// not already exist.
func (o *Options) SetCreateIfMissing(b bool) {
	C.rocksdb_options_set_create_if_missing(o.Opt, boolToUchar(b))
}

// SetFilterPolicy causes Open to create a new database that will uses filter
// created from the filter policy passed in.
func (o *Options) SetFilterPolicy(fp *FilterPolicy) {
	var policy *C.rocksdb_filterpolicy_t
	if fp != nil {
		policy = fp.Policy
	}
	C.rocksdb_options_set_filter_policy(o.Opt, policy)
}

// SetMaxBackgroundCompactions sets the maximum number of concurrent
// background jobs, submitted to the default LOW priority thread pool
func (o *Options) SetMaxBackgroundCompactions(n int) {
	C.rocksdb_options_set_max_background_compactions(o.Opt, C.int(n))
}

// SetMaxBackgroundFlushes sets the maximum number of concurrent
// background memtable flush jobs, submitted to the HIGH priority
// thread pool. By default, all background jobs (major compaction
// and memtable flush) go to the LOW priority pool. If this option
// is set to a positive number, memtable flush jobs will be submitted
// to the HIGH priority pool. It is important when the same Env is
// shared by multiple db instances. Without a separate pool, long
// running major compaction jobs could potentially block memtable
// flush jobs of other db instances, leading to unnecessary Put stalls.
func (o *Options) SetMaxBackgroundFlushes(n int) {
	C.rocksdb_options_set_max_background_compactions(o.Opt, C.int(n))
}

// SetMemtableVectorRep causes MemTableReps that are backed by a
// std::vector to be used. On iteration, the vector is sorted. This
// is useful for workloads where iteration is very rare and writes
// are generally not issued after reads begin.
func (o *Options) SetMemtableVectorRep() {
	C.rocksdb_options_set_memtable_vector_rep(o.Opt)
}

// Close deallocates the ReadOptions, freeing its underlying C struct.
func (ro *ReadOptions) Close() {
	C.rocksdb_readoptions_destroy(ro.Opt)
}

// SetVerifyChecksums controls whether all data read with this ReadOptions
// will be verified against corresponding checksums.
//
// It defaults to false. See the LevelDB documentation for details.
func (ro *ReadOptions) SetVerifyChecksums(b bool) {
	C.rocksdb_readoptions_set_verify_checksums(ro.Opt, boolToUchar(b))
}

// SetFillCache controls whether reads performed with this ReadOptions will
// fill the Cache of the server. It defaults to true.
//
// It is useful to turn this off on ReadOptions for DB.Iterator (and DB.Get)
// calls used in offline threads to prevent bulk scans from flushing out live
// user data in the cache.
//
// See also Options.SetCache
func (ro *ReadOptions) SetFillCache(b bool) {
	C.rocksdb_readoptions_set_fill_cache(ro.Opt, boolToUchar(b))
}

// SetSnapshot causes reads to provided as they were when the passed in
// Snapshot was created by DB.NewSnapshot. This is useful for getting
// consistent reads during a bulk operation.
//
// See the LevelDB documentation for details.
func (ro *ReadOptions) SetSnapshot(snap *Snapshot) {
	var s *C.rocksdb_snapshot_t
	if snap != nil {
		s = snap.snap
	}
	C.rocksdb_readoptions_set_snapshot(ro.Opt, s)
}

// Close deallocates the WriteOptions, freeing its underlying C struct.
func (wo *WriteOptions) Close() {
	C.rocksdb_writeoptions_destroy(wo.Opt)
}

// SetSync controls whether each write performed with this WriteOptions will
// be flushed from the operating system buffer cache before the write is
// considered complete.
//
// If called with true, this will signficantly slow down writes. If called
// with false, and the host machine crashes, some recent writes may be
// lost. The default is false.
//
// See the LevelDB documentation for details.
func (wo *WriteOptions) SetSync(b bool) {
	C.rocksdb_writeoptions_set_sync(wo.Opt, boolToUchar(b))
}

// DisableWAL completely disables the Write Ahead Log. Writes will not go
// to the log at all and may be lost in an event of process crash.
func (wo *WriteOptions) DisableWAL(b bool) {
	C.rocksdb_writeoptions_disable_WAL(wo.Opt, boolToInt(b))
}
<commit_msg>A few more options. TODO: comments.<commit_after>package gorocks

// #include "rocksdb/c.h"
import "C"

import (
	"time"
)

// CompressionOpt is a value for Options.SetCompression.
type CompressionOpt int

// Known compression arguments for Options.SetCompression.
const (
	NoCompression     = CompressionOpt(0)
	SnappyCompression = CompressionOpt(1)
)

// Options represent all of the available options when opening a database with
// Open. Options should be created with NewOptions.
//
// It is usually with to call SetCache with a cache object. Otherwise, all
// data will be read off disk.
//
// To prevent memory leaks, Close must be called on an Options when the
// program no longer needs it.
type Options struct {
	Opt *C.rocksdb_options_t
}

// ReadOptions represent all of the available options when reading from a
// database.
//
// To prevent memory leaks, Close must called on a ReadOptions when the
// program no longer needs it.
type ReadOptions struct {
	Opt *C.rocksdb_readoptions_t
}

// WriteOptions represent all of the available options when writeing from a
// database.
//
// To prevent memory leaks, Close must called on a WriteOptions when the
// program no longer needs it.
type WriteOptions struct {
	Opt *C.rocksdb_writeoptions_t
}

// NewOptions allocates a new Options object.
func NewOptions() *Options {
	opt := C.rocksdb_options_create()
	return &Options{opt}
}

// NewReadOptions allocates a new ReadOptions object.
func NewReadOptions() *ReadOptions {
	opt := C.rocksdb_readoptions_create()
	return &ReadOptions{opt}
}

// NewWriteOptions allocates a new WriteOptions object.
func NewWriteOptions() *WriteOptions {
	opt := C.rocksdb_writeoptions_create()
	return &WriteOptions{opt}
}

// Close deallocates the Options, freeing its underlying C struct.
func (o *Options) Close() {
	C.rocksdb_options_destroy(o.Opt)
}

// SetComparator sets the comparator to be used for all read and write
// operations.
//
// The comparator that created a database must be the same one (technically,
// one with the same name string) that is used to perform read and write
// operations.
//
// The default comparator is usually sufficient.
func (o *Options) SetComparator(cmp *C.rocksdb_comparator_t) {
	C.rocksdb_options_set_comparator(o.Opt, cmp)
}

// SetErrorIfExists, if passed true, will cause the opening of a database that
// already exists to throw an error.
func (o *Options) SetErrorIfExists(error_if_exists bool) {
	eie := boolToUchar(error_if_exists)
	C.rocksdb_options_set_error_if_exists(o.Opt, eie)
}

// SetCache places a cache object in the database when a database is opened.
//
// This is usually wise to use. See also ReadOptions.SetFillCache.
func (o *Options) SetCache(cache *Cache) {
	C.rocksdb_options_set_cache(o.Opt, cache.Cache)
}

// SetEnv sets the Env object for the new database handle.
func (o *Options) SetEnv(env *Env) {
	C.rocksdb_options_set_env(o.Opt, env.Env)
}

// SetInfoLog sets a *C.rocksdb_logger_t object as the informational logger
// for the database.
func (o *Options) SetInfoLog(log *C.rocksdb_logger_t) {
	C.rocksdb_options_set_info_log(o.Opt, log)
}

// SetWriteBufferSize sets the number of bytes the database will build up in
// memory (backed by an unsorted log on disk) before converting to a sorted
// on-disk file.
func (o *Options) SetWriteBufferSize(s int) {
	C.rocksdb_options_set_write_buffer_size(o.Opt, C.size_t(s))
}

func (o *Options) SetMaxWriteBuffers(s int) {
	C.rocksdb_options_set_max_write_buffer_number(o.Opt, C.int(s))
}

func (o *Options) SetMinWriteBuffersToMerge(s int) {
	C.rocksdb_options_set_min_write_buffer_number_to_merge(o.Opt, C.int(s))
}

// SetParanoidChecks, when called with true, will cause the database to do
// aggressive checking of the data it is processing and will stop early if it
// detects errors.
//
// See the LevelDB documentation docs for details.
func (o *Options) SetParanoidChecks(pc bool) {
	C.rocksdb_options_set_paranoid_checks(o.Opt, boolToUchar(pc))
}

// SetMaxOpenFiles sets the number of files than can be used at once by the
// database.
//
// See the LevelDB documentation for details.
func (o *Options) SetMaxOpenFiles(n int) {
	C.rocksdb_options_set_max_open_files(o.Opt, C.int(n))
}

// SetBlockSize sets the approximate size of user data packed per block.
//
// The default is roughly 4096 uncompressed bytes. A better setting depends on
// your use case. See the LevelDB documentation for details.
func (o *Options) SetBlockSize(s int) {
	C.rocksdb_options_set_block_size(o.Opt, C.size_t(s))
}

// SetBlockRestartInterval is the number of keys between restarts points for
// delta encoding keys.
//
// Most clients should leave this parameter alone. See the LevelDB
// documentation for details.
func (o *Options) SetBlockRestartInterval(n int) {
	C.rocksdb_options_set_block_restart_interval(o.Opt, C.int(n))
}

// SetCompression sets whether to compress blocks using the specified
// compresssion algorithm.
//
// The default value is SnappyCompression and it is fast enough that it is
// unlikely you want to turn it off. The other option is NoCompression.
//
// If the LevelDB library was built without Snappy compression enabled, the
// SnappyCompression setting will be ignored.
func (o *Options) SetCompression(t CompressionOpt) {
	C.rocksdb_options_set_compression(o.Opt, C.int(t))
}

// SetCreateIfMissing causes Open to create a new database on disk if it does
// not already exist.
func (o *Options) SetCreateIfMissing(b bool) {
	C.rocksdb_options_set_create_if_missing(o.Opt, boolToUchar(b))
}

// SetFilterPolicy causes Open to create a new database that will uses filter
// created from the filter policy passed in.
func (o *Options) SetFilterPolicy(fp *FilterPolicy) {
	var policy *C.rocksdb_filterpolicy_t
	if fp != nil {
		policy = fp.Policy
	}
	C.rocksdb_options_set_filter_policy(o.Opt, policy)
}

// SetMaxBackgroundCompactions sets the maximum number of concurrent
// background jobs, submitted to the default LOW priority thread pool
func (o *Options) SetMaxBackgroundCompactions(n int) {
	C.rocksdb_options_set_max_background_compactions(o.Opt, C.int(n))
}

// SetMaxBackgroundFlushes sets the maximum number of concurrent
// background memtable flush jobs, submitted to the HIGH priority
// thread pool. By default, all background jobs (major compaction
// and memtable flush) go to the LOW priority pool. If this option
// is set to a positive number, memtable flush jobs will be submitted
// to the HIGH priority pool. It is important when the same Env is
// shared by multiple db instances. Without a separate pool, long
// running major compaction jobs could potentially block memtable
// flush jobs of other db instances, leading to unnecessary Put stalls.
func (o *Options) SetMaxBackgroundFlushes(n int) {
	C.rocksdb_options_set_max_background_compactions(o.Opt, C.int(n))
}

// SetMemtableVectorRep causes MemTableReps that are backed by a
// std::vector to be used. On iteration, the vector is sorted. This
// is useful for workloads where iteration is very rare and writes
// are generally not issued after reads begin.
func (o *Options) SetMemtableVectorRep() {
	C.rocksdb_options_set_memtable_vector_rep(o.Opt)
}

func (o *Options) SetStatsDumpPeriod(period time.Duration) {
	periodSec := C.uint(period.Seconds())
	C.rocksdb_options_set_stats_dump_period_sec(o.Opt, periodSec)
}

func (o *Options) SetNumLevels(levels uint) {
	C.rocksdb_options_set_stats_dump_period_sec(o.Opt, C.uint(levels))
}

func (o *Options) SetLevel0FileNumCompactionTrigger(n int) {
	C.rocksdb_options_set_level0_file_num_compaction_trigger(o.Opt, C.int(n))
}

func (o *Options) SetLevel0SlowdownWritesTrigger(n int) {
	C.rocksdb_options_set_level0_slowdown_writes_trigger(o.Opt, C.int(n))
}

func (o *Options) SetLevel0StopWritesTrigger(n int) {
	C.rocksdb_options_set_level0_stop_writes_trigger(o.Opt, C.int(n))
}

func (o *Options) SetTargetFileSizeBase(n uint64) {
	C.rocksdb_options_set_target_file_size_base(o.Opt, C.uint64_t(n))
}

func (o *Options) SetDisableSeekCompaction(b bool) {
	C.rocksdb_options_set_disable_seek_compaction(o.Opt, boolToInt(b))
}

func (o *Options) SetMaxBytesForLevelBase(n uint64) {
	C.rocksdb_options_set_max_bytes_for_level_base(o.Opt, C.uint64_t(n))
}

// Close deallocates the ReadOptions, freeing its underlying C struct.
func (ro *ReadOptions) Close() {
	C.rocksdb_readoptions_destroy(ro.Opt)
}

// SetVerifyChecksums controls whether all data read with this ReadOptions
// will be verified against corresponding checksums.
//
// It defaults to false. See the LevelDB documentation for details.
func (ro *ReadOptions) SetVerifyChecksums(b bool) {
	C.rocksdb_readoptions_set_verify_checksums(ro.Opt, boolToUchar(b))
}

// SetFillCache controls whether reads performed with this ReadOptions will
// fill the Cache of the server. It defaults to true.
//
// It is useful to turn this off on ReadOptions for DB.Iterator (and DB.Get)
// calls used in offline threads to prevent bulk scans from flushing out live
// user data in the cache.
//
// See also Options.SetCache
func (ro *ReadOptions) SetFillCache(b bool) {
	C.rocksdb_readoptions_set_fill_cache(ro.Opt, boolToUchar(b))
}

// SetSnapshot causes reads to provided as they were when the passed in
// Snapshot was created by DB.NewSnapshot. This is useful for getting
// consistent reads during a bulk operation.
//
// See the LevelDB documentation for details.
func (ro *ReadOptions) SetSnapshot(snap *Snapshot) {
	var s *C.rocksdb_snapshot_t
	if snap != nil {
		s = snap.snap
	}
	C.rocksdb_readoptions_set_snapshot(ro.Opt, s)
}

// Close deallocates the WriteOptions, freeing its underlying C struct.
func (wo *WriteOptions) Close() {
	C.rocksdb_writeoptions_destroy(wo.Opt)
}

// SetSync controls whether each write performed with this WriteOptions will
// be flushed from the operating system buffer cache before the write is
// considered complete.
//
// If called with true, this will signficantly slow down writes. If called
// with false, and the host machine crashes, some recent writes may be
// lost. The default is false.
//
// See the LevelDB documentation for details.
func (wo *WriteOptions) SetSync(b bool) {
	C.rocksdb_writeoptions_set_sync(wo.Opt, boolToUchar(b))
}

// DisableWAL completely disables the Write Ahead Log. Writes will not go
// to the log at all and may be lost in an event of process crash.
func (wo *WriteOptions) DisableWAL(b bool) {
	C.rocksdb_writeoptions_disable_WAL(wo.Opt, boolToInt(b))
}
<|endoftext|>