<commit_before>package utils

import (
	"fmt"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"path/filepath"
	"strings"
	"time"

	engineapi "github.com/docker/engine-api/client"
	"github.com/docker/go-connections/tlsconfig"
	"gitlab.ricebook.net/platform/core/types"
)

const (
	letters       = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
	shortenLength = 7
)

func RandomString(n int) string {
	rand.Seed(time.Now().UnixNano())
	r := make([]byte, n)
	for i := 0; i < n; i++ {
		r[i] = letters[rand.Intn(len(letters))]
	}
	return string(r)
}

func TruncateID(id string) string {
	if len(id) > shortenLength {
		return id[:shortenLength]
	}
	return id
}

func Tail(path string) string {
	parts := strings.Split(path, "/")
	return parts[len(parts)-1]
}

func MakeDockerClient(endpoint string, config types.Config) (*engineapi.Client, error) {
	if !strings.HasPrefix(endpoint, "tcp://") {
		endpoint = "tcp://" + endpoint
	}

	u, err := url.Parse(endpoint)
	if err != nil {
		return nil, err
	}

	host, _, err := net.SplitHostPort(u.Host)
	if err != nil {
		return nil, err
	}

	dockerCertPath := filepath.Join(config.Docker.CertPath, host)
	options := tlsconfig.Options{
		CAFile:             filepath.Join(dockerCertPath, "ca.pem"),
		CertFile:           filepath.Join(dockerCertPath, "cert.pem"),
		KeyFile:            filepath.Join(dockerCertPath, "key.pem"),
		InsecureSkipVerify: false,
	}
	tlsc, err := tlsconfig.Client(options)
	if err != nil {
		return nil, err
	}

	cli := &http.Client{
		Transport: &http.Transport{
			TLSClientConfig: tlsc,
		},
	}

	return engineapi.NewClient(endpoint, config.Docker.APIVersion, cli, nil)
}

func GetGitRepoName(url string) (string, error) {
	if !strings.HasPrefix(url, "git@") || !strings.HasSuffix(url, ".git") {
		return "", fmt.Errorf("Bad git url format %q", url)
	}

	x := strings.SplitN(url, ":", 2)
	if len(x) != 2 {
		return "", fmt.Errorf("Bad git url format %q", url)
	}

	y := strings.SplitN(x[1], "/", 2)
	if len(y) != 2 {
		return "", fmt.Errorf("Bad git url format %q", url)
	}
	return strings.TrimSuffix(y[1], ".git"), nil
}

func GetVersion(image string) string {
	if !strings.Contains(image, ":") {
		return "unknown"
	}

	parts := strings.Split(image, ":")
	if len(parts) != 2 {
		return "unknown"
	}

	return parts[1]
}
<commit_msg>有时候并不想要tls...<commit_after>package utils

import (
	"fmt"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"path/filepath"
	"strings"
	"time"

	engineapi "github.com/docker/engine-api/client"
	"github.com/docker/go-connections/tlsconfig"
	"gitlab.ricebook.net/platform/core/types"
)

const (
	letters       = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
	shortenLength = 7
)

func RandomString(n int) string {
	rand.Seed(time.Now().UnixNano())
	r := make([]byte, n)
	for i := 0; i < n; i++ {
		r[i] = letters[rand.Intn(len(letters))]
	}
	return string(r)
}

func TruncateID(id string) string {
	if len(id) > shortenLength {
		return id[:shortenLength]
	}
	return id
}

func Tail(path string) string {
	parts := strings.Split(path, "/")
	return parts[len(parts)-1]
}

func MakeDockerClient(endpoint string, config types.Config) (*engineapi.Client, error) {
	if !strings.HasPrefix(endpoint, "tcp://") {
		endpoint = "tcp://" + endpoint
	}

	u, err := url.Parse(endpoint)
	if err != nil {
		return nil, err
	}

	host, _, err := net.SplitHostPort(u.Host)
	if err != nil {
		return nil, err
	}

	var cli *http.Client
	// if no cert path is set
	// then just use normal http client without tls
	if config.Docker.CertPath != "" {
		dockerCertPath := filepath.Join(config.Docker.CertPath, host)
		options := tlsconfig.Options{
			CAFile:             filepath.Join(dockerCertPath, "ca.pem"),
			CertFile:           filepath.Join(dockerCertPath, "cert.pem"),
			KeyFile:            filepath.Join(dockerCertPath, "key.pem"),
			InsecureSkipVerify: false,
		}
		tlsc, err := tlsconfig.Client(options)
		if err != nil {
			return nil, err
		}

		cli = &http.Client{
			Transport: &http.Transport{
				TLSClientConfig: tlsc,
			},
		}
	}

	return engineapi.NewClient(endpoint, config.Docker.APIVersion, cli, nil)
}

func GetGitRepoName(url string) (string, error) {
	if !strings.HasPrefix(url, "git@") || !strings.HasSuffix(url, ".git") {
		return "", fmt.Errorf("Bad git url format %q", url)
	}

	x := strings.SplitN(url, ":", 2)
	if len(x) != 2 {
		return "", fmt.Errorf("Bad git url format %q", url)
	}

	y := strings.SplitN(x[1], "/", 2)
	if len(y) != 2 {
		return "", fmt.Errorf("Bad git url format %q", url)
	}
	return strings.TrimSuffix(y[1], ".git"), nil
}

func GetVersion(image string) string {
	if !strings.Contains(image, ":") {
		return "unknown"
	}

	parts := strings.Split(image, ":")
	if len(parts) != 2 {
		return "unknown"
	}

	return parts[1]
}
<|endoftext|>


<commit_before>package utils

import (
	"bytes"
	"crypto/rand"
	"encoding/hex"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"syscall"

	"github.com/containers/podman/v3/pkg/lookup"
	"github.com/cri-o/cri-o/internal/dbusmgr"
	"github.com/cri-o/cri-o/server/cri/types"
	securejoin "github.com/cyphar/filepath-securejoin"
	"github.com/opencontainers/runc/libcontainer/user"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"golang.org/x/net/context"

	systemdDbus "github.com/coreos/go-systemd/v22/dbus"
	"github.com/godbus/dbus/v5"
)

// ExecCmd executes a command with args and returns its output as a string along
// with an error, if any
func ExecCmd(name string, args ...string) (string, error) {
	cmd := exec.Command(name, args...)
	var stdout bytes.Buffer
	var stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	if v, found := os.LookupEnv("XDG_RUNTIME_DIR"); found {
		cmd.Env = append(cmd.Env, fmt.Sprintf("XDG_RUNTIME_DIR=%s", v))
	}

	err := cmd.Run()
	if err != nil {
		return "", fmt.Errorf("`%v %v` failed: %v %v (%v)", name, strings.Join(args, " "), stderr.String(), stdout.String(), err)
	}

	return stdout.String(), nil
}

// StatusToExitCode converts wait status code to an exit code
func StatusToExitCode(status int) int {
	return ((status) & 0xff00) >> 8
}

// RunUnderSystemdScope adds the specified pid to a systemd scope
func RunUnderSystemdScope(mgr *dbusmgr.DbusConnManager, pid int, slice, unitName string, properties ...systemdDbus.Property) error {
	ctx := context.Background()
	var err error
	// sanity check
	if mgr == nil {
		return errors.New("dbus manager is nil")
	}
	defaultProperties := []systemdDbus.Property{
		newProp("PIDs", []uint32{uint32(pid)}),
		newProp("Delegate", true),
		newProp("DefaultDependencies", false),
	}
	properties = append(defaultProperties, properties...)
	if slice != "" {
		properties = append(properties, systemdDbus.PropSlice(slice))
	}
	ch := make(chan string)
	for {
		err := mgr.RetryOnDisconnect(func(c *systemdDbus.Conn) error {
			_, err = c.StartTransientUnitContext(ctx, unitName, "replace", properties, ch)
			return err
		})

		if err == nil {
			break
		}
		if !errors.Is(err, syscall.EAGAIN) {
			return err
		}
	}

	// Block until job is started
	<-ch

	return nil
}

func newProp(name string, units interface{}) systemdDbus.Property {
	return systemdDbus.Property{
		Name:  name,
		Value: dbus.MakeVariant(units),
	}
}

// DetachError is special error which returned in case of container detach.
type DetachError struct{}

func (DetachError) Error() string {
	return "detached from container"
}

// CopyDetachable is similar to io.Copy but support a detach key sequence to break out.
func CopyDetachable(dst io.Writer, src io.Reader, keys []byte) (int64, error) {
	var (
		written int64
		err     error
	)
	// Sanity check interfaces
	if dst == nil || src == nil {
		return 0, fmt.Errorf("src/dst reader/writer nil")
	}
	if len(keys) == 0 {
		// Default keys : ctrl-p ctrl-q
		keys = []byte{16, 17}
	}

	buf := make([]byte, 32*1024)
	for {
		nr, er := src.Read(buf)
		if nr > 0 {
			preservBuf := []byte{}
			for i, key := range keys {
				preservBuf = append(preservBuf, buf[0:nr]...)
				if nr != 1 || buf[0] != key {
					break
				}
				if i == len(keys)-1 {
					// src.Close()
					return 0, DetachError{}
				}
				nr, er = src.Read(buf)
			}
			nw, ew := dst.Write(preservBuf)
			nr = len(preservBuf)
			if nw > 0 {
				written += int64(nw)
			}
			if ew != nil {
				err = ew
				break
			}
			if nr != nw {
				err = io.ErrShortWrite
				break
			}
		}
		if er != nil {
			if er != io.EOF {
				err = er
			}
			break
		}
	}
	return written, err
}

// WriteGoroutineStacks writes out the goroutine stacks
// of the caller. Up to 32 MB is allocated to print the
// stack.
func WriteGoroutineStacks(w io.Writer) error {
	if w == nil {
		return fmt.Errorf("writer nil")
	}
	buf := make([]byte, 1<<20)
	for i := 0; ; i++ {
		n := runtime.Stack(buf, true)
		if n < len(buf) {
			buf = buf[:n]
			break
		}
		if len(buf) >= 32<<20 {
			break
		}
		buf = make([]byte, 2*len(buf))
	}
	_, err := w.Write(buf)
	return err
}

// WriteGoroutineStacksToFile write goroutine stacks
// to the specified file.
func WriteGoroutineStacksToFile(path string) error {
	f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0o666)
	if err != nil {
		return err
	}
	defer f.Close()

	err = WriteGoroutineStacks(f)
	if err != nil {
		return err
	}
	return f.Sync()
}

// GenerateID generates a random unique id.
func GenerateID() (string, error) {
	b := make([]byte, 32)
	if _, err := rand.Read(b); err != nil {
		return "", errors.Wrap(err, "generate ID")
	}
	return hex.EncodeToString(b), nil
}

// openContainerFile opens a file inside a container rootfs safely
func openContainerFile(rootfs, path string) (io.ReadCloser, error) {
	fp, err := securejoin.SecureJoin(rootfs, path)
	if err != nil {
		return nil, err
	}
	fh, err := os.Open(fp)
	if err != nil {
		// This is needed because a nil *os.File is different to a nil
		// io.ReadCloser and this causes GetExecUser to not detect that the
		// container file is missing.
		return nil, err
	}
	return fh, nil
}

// GetUserInfo returns UID, GID and additional groups for specified user
// by looking them up in /etc/passwd and /etc/group
func GetUserInfo(rootfs, userName string) (uid, gid uint32, additionalGids []uint32, _ error) {
	// We don't care if we can't open the file because
	// not all images will have these files
	passwdFile, err := openContainerFile(rootfs, "/etc/passwd")
	if err != nil {
		logrus.Warnf("Failed to open /etc/passwd: %v", err)
	} else {
		defer passwdFile.Close()
	}

	groupFile, err := openContainerFile(rootfs, "/etc/group")
	if err != nil {
		logrus.Warnf("Failed to open /etc/group: %v", err)
	} else {
		defer groupFile.Close()
	}

	execUser, err := user.GetExecUser(userName, nil, passwdFile, groupFile)
	if err != nil {
		return 0, 0, nil, errors.Wrap(err, "get exec user")
	}

	uid = uint32(execUser.Uid)
	gid = uint32(execUser.Gid)
	additionalGids = make([]uint32, 0, len(execUser.Sgids))
	for _, g := range execUser.Sgids {
		additionalGids = append(additionalGids, uint32(g))
	}

	return uid, gid, additionalGids, nil
}

// GeneratePasswd generates a container specific passwd file,
// iff uid is not defined in the containers /etc/passwd
func GeneratePasswd(username string, uid, gid uint32, homedir, rootfs, rundir string) (string, error) {
	// if UID exists inside of container rootfs /etc/passwd then
	// don't generate passwd
	if _, err := lookup.GetUser(rootfs, strconv.Itoa(int(uid))); err == nil {
		return "", nil
	}
	passwdFile := filepath.Join(rundir, "passwd")
	originPasswdFile, err := securejoin.SecureJoin(rootfs, "/etc/passwd")
	if err != nil {
		return "", errors.Wrap(err, "unable to follow symlinks to passwd file")
	}
	info, err := os.Stat(originPasswdFile)
	if err != nil {
		if os.IsNotExist(err) {
			return "", nil
		}
		return "", errors.Wrapf(err, "unable to stat passwd file %s", originPasswdFile)
	}
	// Check if passwd file is world writable
	if info.Mode().Perm()&(0o022) != 0 {
		return "", nil
	}
	passwdUID := info.Sys().(*syscall.Stat_t).Uid
	passwdGID := info.Sys().(*syscall.Stat_t).Gid

	if uid == passwdUID && info.Mode().Perm()&(0o200) != 0 {
		return "", nil
	}

	orig, err := ioutil.ReadFile(originPasswdFile)
	if err != nil {
		// If no /etc/passwd in container ignore and return
		if os.IsNotExist(err) {
			return "", nil
		}
		return "", errors.Wrapf(err, "read passwd file")
	}
	if username == "" {
		username = "default"
	}
	if homedir == "" {
		homedir = "/tmp"
	}
	pwd := fmt.Sprintf("%s%s:x:%d:%d:%s user:%s:/sbin/nologin\n", orig, username, uid, gid, username, homedir)
	if err := ioutil.WriteFile(passwdFile, []byte(pwd), info.Mode()); err != nil {
		return "", errors.Wrap(err, "failed to create temporary passwd file")
	}
	if err := os.Chown(passwdFile, int(passwdUID), int(passwdGID)); err != nil {
		return "", errors.Wrap(err, "failed to chown temporary passwd file")
	}

	return passwdFile, nil
}

// Int32Ptr is a utility function to assign to integer pointer variables
func Int32Ptr(i int32) *int32 {
	return &i
}

// EnsureSaneLogPath is a hack to fix https://issues.k8s.io/44043 which causes
// logPath to be a broken symlink to some magical Docker path. Ideally we
// wouldn't have to deal with this, but until that issue is fixed we have to
// remove the path if it's a broken symlink.
func EnsureSaneLogPath(logPath string) error {
	// If the path exists but the resolved path does not, then we have a broken
	// symlink and we need to remove it.
	fi, err := os.Lstat(logPath)
	if err != nil || fi.Mode()&os.ModeSymlink == 0 {
		// Non-existent files and non-symlinks aren't our problem.
		return nil
	}

	_, err = os.Stat(logPath)
	if os.IsNotExist(err) {
		err = os.RemoveAll(logPath)
		if err != nil {
			return fmt.Errorf("failed to remove bad log path %s: %v", logPath, err)
		}
	}
	return nil
}

func GetLabelOptions(selinuxOptions *types.SELinuxOption) []string {
	labels := []string{}
	if selinuxOptions != nil {
		if selinuxOptions.User != "" {
			labels = append(labels, "user:"+selinuxOptions.User)
		}
		if selinuxOptions.Role != "" {
			labels = append(labels, "role:"+selinuxOptions.Role)
		}
		if selinuxOptions.Type != "" {
			labels = append(labels, "type:"+selinuxOptions.Type)
		}
		if selinuxOptions.Level != "" {
			labels = append(labels, "level:"+selinuxOptions.Level)
		}
	}
	return labels
}

// SyncParent ensures a path's parent directory is synced to disk
func SyncParent(path string) error {
	return Sync(filepath.Dir(path))
}

// Sync ensures a path is synced to disk
func Sync(path string) error {
	f, err := os.OpenFile(path, os.O_RDONLY, 0o755)
	if err != nil {
		return err
	}
	defer f.Close()

	if err := syscall.Fsync(int(f.Fd())); err != nil {
		return err
	}
	return nil
}
<commit_msg>utils: close dbus conn channel<commit_after>package utils

import (
	"bytes"
	"crypto/rand"
	"encoding/hex"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"syscall"

	"github.com/containers/podman/v3/pkg/lookup"
	"github.com/cri-o/cri-o/internal/dbusmgr"
	"github.com/cri-o/cri-o/server/cri/types"
	securejoin "github.com/cyphar/filepath-securejoin"
	"github.com/opencontainers/runc/libcontainer/user"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"golang.org/x/net/context"

	systemdDbus "github.com/coreos/go-systemd/v22/dbus"
	"github.com/godbus/dbus/v5"
)

// ExecCmd executes a command with args and returns its output as a string along
// with an error, if any
func ExecCmd(name string, args ...string) (string, error) {
	cmd := exec.Command(name, args...)
	var stdout bytes.Buffer
	var stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	if v, found := os.LookupEnv("XDG_RUNTIME_DIR"); found {
		cmd.Env = append(cmd.Env, fmt.Sprintf("XDG_RUNTIME_DIR=%s", v))
	}

	err := cmd.Run()
	if err != nil {
		return "", fmt.Errorf("`%v %v` failed: %v %v (%v)", name, strings.Join(args, " "), stderr.String(), stdout.String(), err)
	}

	return stdout.String(), nil
}

// StatusToExitCode converts wait status code to an exit code
func StatusToExitCode(status int) int {
	return ((status) & 0xff00) >> 8
}

// RunUnderSystemdScope adds the specified pid to a systemd scope
func RunUnderSystemdScope(mgr *dbusmgr.DbusConnManager, pid int, slice, unitName string, properties ...systemdDbus.Property) error {
	ctx := context.Background()
	var err error
	// sanity check
	if mgr == nil {
		return errors.New("dbus manager is nil")
	}
	defaultProperties := []systemdDbus.Property{
		newProp("PIDs", []uint32{uint32(pid)}),
		newProp("Delegate", true),
		newProp("DefaultDependencies", false),
	}
	properties = append(defaultProperties, properties...)
	if slice != "" {
		properties = append(properties, systemdDbus.PropSlice(slice))
	}
	ch := make(chan string)
	for {
		err := mgr.RetryOnDisconnect(func(c *systemdDbus.Conn) error {
			_, err = c.StartTransientUnitContext(ctx, unitName, "replace", properties, ch)
			return err
		})

		if err == nil {
			break
		}
		if !errors.Is(err, syscall.EAGAIN) {
			return err
		}
	}

	// Block until job is started
	<-ch
	close(ch)

	return nil
}

func newProp(name string, units interface{}) systemdDbus.Property {
	return systemdDbus.Property{
		Name:  name,
		Value: dbus.MakeVariant(units),
	}
}

// DetachError is special error which returned in case of container detach.
type DetachError struct{}

func (DetachError) Error() string {
	return "detached from container"
}

// CopyDetachable is similar to io.Copy but support a detach key sequence to break out.
func CopyDetachable(dst io.Writer, src io.Reader, keys []byte) (int64, error) {
	var (
		written int64
		err     error
	)
	// Sanity check interfaces
	if dst == nil || src == nil {
		return 0, fmt.Errorf("src/dst reader/writer nil")
	}
	if len(keys) == 0 {
		// Default keys : ctrl-p ctrl-q
		keys = []byte{16, 17}
	}

	buf := make([]byte, 32*1024)
	for {
		nr, er := src.Read(buf)
		if nr > 0 {
			preservBuf := []byte{}
			for i, key := range keys {
				preservBuf = append(preservBuf, buf[0:nr]...)
				if nr != 1 || buf[0] != key {
					break
				}
				if i == len(keys)-1 {
					// src.Close()
					return 0, DetachError{}
				}
				nr, er = src.Read(buf)
			}
			nw, ew := dst.Write(preservBuf)
			nr = len(preservBuf)
			if nw > 0 {
				written += int64(nw)
			}
			if ew != nil {
				err = ew
				break
			}
			if nr != nw {
				err = io.ErrShortWrite
				break
			}
		}
		if er != nil {
			if er != io.EOF {
				err = er
			}
			break
		}
	}
	return written, err
}

// WriteGoroutineStacks writes out the goroutine stacks
// of the caller. Up to 32 MB is allocated to print the
// stack.
func WriteGoroutineStacks(w io.Writer) error {
	if w == nil {
		return fmt.Errorf("writer nil")
	}
	buf := make([]byte, 1<<20)
	for i := 0; ; i++ {
		n := runtime.Stack(buf, true)
		if n < len(buf) {
			buf = buf[:n]
			break
		}
		if len(buf) >= 32<<20 {
			break
		}
		buf = make([]byte, 2*len(buf))
	}
	_, err := w.Write(buf)
	return err
}

// WriteGoroutineStacksToFile write goroutine stacks
// to the specified file.
func WriteGoroutineStacksToFile(path string) error {
	f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0o666)
	if err != nil {
		return err
	}
	defer f.Close()

	err = WriteGoroutineStacks(f)
	if err != nil {
		return err
	}
	return f.Sync()
}

// GenerateID generates a random unique id.
func GenerateID() (string, error) {
	b := make([]byte, 32)
	if _, err := rand.Read(b); err != nil {
		return "", errors.Wrap(err, "generate ID")
	}
	return hex.EncodeToString(b), nil
}

// openContainerFile opens a file inside a container rootfs safely
func openContainerFile(rootfs, path string) (io.ReadCloser, error) {
	fp, err := securejoin.SecureJoin(rootfs, path)
	if err != nil {
		return nil, err
	}
	fh, err := os.Open(fp)
	if err != nil {
		// This is needed because a nil *os.File is different to a nil
		// io.ReadCloser and this causes GetExecUser to not detect that the
		// container file is missing.
		return nil, err
	}
	return fh, nil
}

// GetUserInfo returns UID, GID and additional groups for specified user
// by looking them up in /etc/passwd and /etc/group
func GetUserInfo(rootfs, userName string) (uid, gid uint32, additionalGids []uint32, _ error) {
	// We don't care if we can't open the file because
	// not all images will have these files
	passwdFile, err := openContainerFile(rootfs, "/etc/passwd")
	if err != nil {
		logrus.Warnf("Failed to open /etc/passwd: %v", err)
	} else {
		defer passwdFile.Close()
	}

	groupFile, err := openContainerFile(rootfs, "/etc/group")
	if err != nil {
		logrus.Warnf("Failed to open /etc/group: %v", err)
	} else {
		defer groupFile.Close()
	}

	execUser, err := user.GetExecUser(userName, nil, passwdFile, groupFile)
	if err != nil {
		return 0, 0, nil, errors.Wrap(err, "get exec user")
	}

	uid = uint32(execUser.Uid)
	gid = uint32(execUser.Gid)
	additionalGids = make([]uint32, 0, len(execUser.Sgids))
	for _, g := range execUser.Sgids {
		additionalGids = append(additionalGids, uint32(g))
	}

	return uid, gid, additionalGids, nil
}

// GeneratePasswd generates a container specific passwd file,
// iff uid is not defined in the containers /etc/passwd
func GeneratePasswd(username string, uid, gid uint32, homedir, rootfs, rundir string) (string, error) {
	// if UID exists inside of container rootfs /etc/passwd then
	// don't generate passwd
	if _, err := lookup.GetUser(rootfs, strconv.Itoa(int(uid))); err == nil {
		return "", nil
	}
	passwdFile := filepath.Join(rundir, "passwd")
	originPasswdFile, err := securejoin.SecureJoin(rootfs, "/etc/passwd")
	if err != nil {
		return "", errors.Wrap(err, "unable to follow symlinks to passwd file")
	}
	info, err := os.Stat(originPasswdFile)
	if err != nil {
		if os.IsNotExist(err) {
			return "", nil
		}
		return "", errors.Wrapf(err, "unable to stat passwd file %s", originPasswdFile)
	}
	// Check if passwd file is world writable
	if info.Mode().Perm()&(0o022) != 0 {
		return "", nil
	}
	passwdUID := info.Sys().(*syscall.Stat_t).Uid
	passwdGID := info.Sys().(*syscall.Stat_t).Gid

	if uid == passwdUID && info.Mode().Perm()&(0o200) != 0 {
		return "", nil
	}

	orig, err := ioutil.ReadFile(originPasswdFile)
	if err != nil {
		// If no /etc/passwd in container ignore and return
		if os.IsNotExist(err) {
			return "", nil
		}
		return "", errors.Wrapf(err, "read passwd file")
	}
	if username == "" {
		username = "default"
	}
	if homedir == "" {
		homedir = "/tmp"
	}
	pwd := fmt.Sprintf("%s%s:x:%d:%d:%s user:%s:/sbin/nologin\n", orig, username, uid, gid, username, homedir)
	if err := ioutil.WriteFile(passwdFile, []byte(pwd), info.Mode()); err != nil {
		return "", errors.Wrap(err, "failed to create temporary passwd file")
	}
	if err := os.Chown(passwdFile, int(passwdUID), int(passwdGID)); err != nil {
		return "", errors.Wrap(err, "failed to chown temporary passwd file")
	}

	return passwdFile, nil
}

// Int32Ptr is a utility function to assign to integer pointer variables
func Int32Ptr(i int32) *int32 {
	return &i
}

// EnsureSaneLogPath is a hack to fix https://issues.k8s.io/44043 which causes
// logPath to be a broken symlink to some magical Docker path. Ideally we
// wouldn't have to deal with this, but until that issue is fixed we have to
// remove the path if it's a broken symlink.
func EnsureSaneLogPath(logPath string) error {
	// If the path exists but the resolved path does not, then we have a broken
	// symlink and we need to remove it.
	fi, err := os.Lstat(logPath)
	if err != nil || fi.Mode()&os.ModeSymlink == 0 {
		// Non-existent files and non-symlinks aren't our problem.
		return nil
	}

	_, err = os.Stat(logPath)
	if os.IsNotExist(err) {
		err = os.RemoveAll(logPath)
		if err != nil {
			return fmt.Errorf("failed to remove bad log path %s: %v", logPath, err)
		}
	}
	return nil
}

func GetLabelOptions(selinuxOptions *types.SELinuxOption) []string {
	labels := []string{}
	if selinuxOptions != nil {
		if selinuxOptions.User != "" {
			labels = append(labels, "user:"+selinuxOptions.User)
		}
		if selinuxOptions.Role != "" {
			labels = append(labels, "role:"+selinuxOptions.Role)
		}
		if selinuxOptions.Type != "" {
			labels = append(labels, "type:"+selinuxOptions.Type)
		}
		if selinuxOptions.Level != "" {
			labels = append(labels, "level:"+selinuxOptions.Level)
		}
	}
	return labels
}

// SyncParent ensures a path's parent directory is synced to disk
func SyncParent(path string) error {
	return Sync(filepath.Dir(path))
}

// Sync ensures a path is synced to disk
func Sync(path string) error {
	f, err := os.OpenFile(path, os.O_RDONLY, 0o755)
	if err != nil {
		return err
	}
	defer f.Close()

	if err := syscall.Fsync(int(f.Fd())); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>// +-------------------------------------------------------------------------
// | Copyright (C) 2016 Yunify, Inc.
// +-------------------------------------------------------------------------
// | Licensed under the Apache License, Version 2.0 (the "License");
// | you may not use this work except in compliance with the License.
// | You may obtain a copy of the License in the LICENSE file, or at:
// |
// | http://www.apache.org/licenses/LICENSE-2.0
// |
// | Unless required by applicable law or agreed to in writing, software
// | distributed under the License is distributed on an "AS IS" BASIS,
// | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// | See the License for the specific language governing permissions and
// | limitations under the License.
// +-------------------------------------------------------------------------

package utils

var capitalizedToCapitalizedWordsMap = map[string]string{
	"Dns":     "DNS",
	"Dyn":     "DYN",
	"Eip":     "EIP",
	"Keypair": "KeyPair",
	"Vxnet":   "VxNet",
}

var lowerCaseToLowercaseWordsMap = map[string]string{
	"lastest": "latest", // Fix typo
}

var lowerCaseToCapitalizedWordsMap = map[string]string{
	"acl":           "ACL",
	"cors":          "CORS",
	"cpu":           "CPU",
	"datadir":       "DataDir",
	"dhcp":          "DHCP",
	"dns":           "DNS",
	"dyn":           "DYN",
	"eip":           "EIP",
	"eips":          "EIPs",
	"http":          "HTTP",
	"icp":           "ICP",
	"id":            "ID",
	"ids":           "IDs",
	"innodb":        "InnoDB",
	"io":            "IO",
	"ip":            "IP",
	"ips":           "IPs",
	"ipset":         "IPSet",
	"ipsets":        "IPSets",
	"keypair":       "KeyPair",
	"keypairs":      "KeyPairs",
	"lastest":       "Latest", // Fix typo
	"loadbalancer":  "LoadBalancer",
	"loadbalancers": "LoadBalancers",
	"md5":           "MD5",
	"newsid":        "NewSID",
	"nic":           "NIC",
	"os":            "OS",
	"opt":           "OPT",
	"qingstor":      "QingStor",
	"qingcloud":     "QingCloud",
	"qs":            "QS",
	"rdb":           "RDB",
	"rdbs":          "RDBs",
	"sha1":          "SHA1",
	"sha256":        "SHA256",
	"sql":           "SQL",
	"tmp":           "TMP",
	"tmpdir":        "TMPDir",
	"topslave":      "TopSlave",
	"trx":           "TRX",
	"ui":            "UI",
	"uri":           "URI",
	"url":           "URL",
	"usb":           "USB",
	"uuid":          "UUID",
	"vcpus":         "VCPUs",
	"vxnet":         "VxNet",
	"vxnets":        "VxNets",
}

var abbreviateWordsMap = []string{
	"ACL",
	"CORS",
	"CPU",
	"DHCP",
	"DNS",
	"DYN",
	"EIP",
	"ETag",
	"IaaS",
	"ICP",
	"ID",
	"IO",
	"IP",
	"IPSets",
	"MD5",
	"NIC",
	"OAuth",
	"OS",
	"OPT",
	"QingStor",
	"QingCloud",
	"QS",
	"RDB",
	"SQL",
	"SSO",
	"TMP",
	"TMPDir",
	"TRX",
	"UI",
	"URL",
	"UUID",
	"VCPUs",
	"VxNet",
}
<commit_msg>Add WOPI to words map<commit_after>// +-------------------------------------------------------------------------
// | Copyright (C) 2016 Yunify, Inc.
// +-------------------------------------------------------------------------
// | Licensed under the Apache License, Version 2.0 (the "License");
// | you may not use this work except in compliance with the License.
// | You may obtain a copy of the License in the LICENSE file, or at:
// |
// | http://www.apache.org/licenses/LICENSE-2.0
// |
// | Unless required by applicable law or agreed to in writing, software
// | distributed under the License is distributed on an "AS IS" BASIS,
// | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// | See the License for the specific language governing permissions and
// | limitations under the License.
// +-------------------------------------------------------------------------

package utils

var capitalizedToCapitalizedWordsMap = map[string]string{
	"Dns":     "DNS",
	"Dyn":     "DYN",
	"Eip":     "EIP",
	"Id":      "ID",
	"Ip":      "IP",
	"Keypair": "KeyPair",
	"Vxnet":   "VxNet",
}

var lowerCaseToLowercaseWordsMap = map[string]string{
	"lastest": "latest", // Fix typo
}

var lowerCaseToCapitalizedWordsMap = map[string]string{
	"acl":           "ACL",
	"cors":          "CORS",
	"cpu":           "CPU",
	"datadir":       "DataDir",
	"dhcp":          "DHCP",
	"dns":           "DNS",
	"dyn":           "DYN",
	"eip":           "EIP",
	"eips":          "EIPs",
	"http":          "HTTP",
	"icp":           "ICP",
	"id":            "ID",
	"ids":           "IDs",
	"innodb":        "InnoDB",
	"io":            "IO",
	"ip":            "IP",
	"ips":           "IPs",
	"ipset":         "IPSet",
	"ipsets":        "IPSets",
	"keypair":       "KeyPair",
	"keypairs":      "KeyPairs",
	"lastest":       "Latest", // Fix typo
	"loadbalancer":  "LoadBalancer",
	"loadbalancers": "LoadBalancers",
	"md5":           "MD5",
	"newsid":        "NewSID",
	"nic":           "NIC",
	"os":            "OS",
	"opt":           "OPT",
	"qingstor":      "QingStor",
	"qingcloud":     "QingCloud",
	"qs":            "QS",
	"rdb":           "RDB",
	"rdbs":          "RDBs",
	"sha1":          "SHA1",
	"sha256":        "SHA256",
	"sql":           "SQL",
	"tmp":           "TMP",
	"tmpdir":        "TMPDir",
	"topslave":      "TopSlave",
	"trx":           "TRX",
	"ttl":           "TTL",
	"ui":            "UI",
	"uri":           "URI",
	"url":           "URL",
	"usb":           "USB",
	"uuid":          "UUID",
	"vcpus":         "VCPUs",
	"vxnet":         "VxNet",
	"vxnets":        "VxNets",
	"wopi":          "WOPI",
}

var abbreviateWordsMap = []string{
	"ACL",
	"CORS",
	"CPU",
	"DHCP",
	"DNS",
	"DYN",
	"EIP",
	"ETag",
	"IaaS",
	"ICP",
	"ID",
	"IO",
	"IP",
	"IPSets",
	"MD5",
	"NIC",
	"OAuth",
	"OS",
	"OPT",
	"QingStor",
	"QingCloud",
	"QS",
	"RDB",
	"SQL",
	"SSO",
	"TMP",
	"TMPDir",
	"TRX",
	"UI",
	"URL",
	"UUID",
	"VCPUs",
	"VxNet",
	"WOPI",
}
<|endoftext|>


<commit_before>package neutrino

import (
	"container/heap"
	"github.com/davecgh/go-spew/spew"
	"github.com/roasbeef/btcd/chaincfg/chainhash"
	"github.com/roasbeef/btcd/wire"
	"github.com/roasbeef/btcutil"
	"github.com/roasbeef/btcutil/gcs"
	"github.com/roasbeef/btcutil/gcs/builder"
	"github.com/roasbeef/btcwallet/waddrmgr"
	"sync"
)

// GetUtxoRequest is a request to scan for OutPoint from the height StartHeight.
type GetUtxoRequest struct {
	OutPoint    *wire.OutPoint
	StartHeight uint32
	Result      func(*SpendReport, error)
}

// Interface exposes the necessary methods for interacting with the blockchain.
type Interface interface {
	GetBlockFromNetwork(chainhash.Hash, ...QueryOption) (*btcutil.Block, error)
	GetBlockHash(int64) (*chainhash.Hash, error)
	BestSnapshot() (*waddrmgr.BlockStamp, error)
	GetCFilter(blockHash chainhash.Hash, filterType wire.FilterType,
		options ...QueryOption) (*gcs.Filter, error)
}

// A PriorityQueue implements heap.Interface and holds GetUtxoRequests. The
// queue maintains that heap.Pop() will always return the GetUtxo request with
// the greatest starting height. This allows us to add new GetUtxo requests to
// an already running batch that's still in the process of catching up to the
// start height of the request.
type PriorityQueue []*GetUtxoRequest

func (pq PriorityQueue) Len() int { return len(pq) }

func (pq PriorityQueue) Less(i, j int) bool {
	// We want Pop to give us the least StartHeight.
	return pq[i].StartHeight < pq[j].StartHeight
}

func (pq PriorityQueue) Swap(i, j int) {
	pq[i], pq[j] = pq[j], pq[i]
}

// Push is called by the heap.Interface implementation to add an element to the
// end of the backing store. The heap library will then maintain the heap
// invariant.
func (pq *PriorityQueue) Push(x interface{}) {
	item := x.(*GetUtxoRequest)
	*pq = append(*pq, item)
}

// Peek returns the greatest height element in the queue without removing it.
func (pq *PriorityQueue) Peek() *GetUtxoRequest {
	return (*pq)[0]
}

// Pop is called by the heap.Interface implementation to remove an element from
// the end of the backing store. The heap library will then maintain the heap
// invariant.
func (pq *PriorityQueue) Pop() interface{} {
	old := *pq
	n := len(old)
	item := old[n-1]
	*pq = old[0 : n-1]
	return item
}

// IsEmpty returns true if the queue has no elements.
func (pq *PriorityQueue) IsEmpty() bool {
	return pq.Len() == 0
}

// UtxoScanner batches calls to GetUtxo so that a single scan can search for
// multiple outpoints. If a scan is in progress when a new element is added, we
// check whether it can safely be added to the current batch, if not it will be
// included in the next batch.
type UtxoScanner struct {
	pq      PriorityQueue
	stopped bool

	chainClient Interface

	nextBatch []*GetUtxoRequest

	cv *sync.Cond
}

// NewUtxoScanner creates a new instance of UtxoScanner using the given chain
// interface.
func NewUtxoScanner(chainClient Interface) UtxoScanner {
	return UtxoScanner{
		pq:          make(PriorityQueue, 0),
		cv:          sync.NewCond(&sync.Mutex{}),
		stopped:     false,
		chainClient: chainClient,
	}
}

// Start begins running scan batches.
func (s *UtxoScanner) Start() {
	go func() {
		for {
			s.cv.L.Lock()
			if s.stopped {
				s.cv.L.Unlock()
				return
			}

			// Re-queue previously skipped requests for next batch.
			for _, request := range s.nextBatch {
				heap.Push(&s.pq, request)
			}

			s.nextBatch = nil

			s.cv.L.Unlock()

			requests, err := s.runBatch()

			// If there was an error, then notify the currently outstanding
			// requests.
			if err != nil {
				for _, request := range requests {
					request.Result(nil, err)
				}
			}
		}
	}()
}

// getAtHeight returns all GetUtxo requests that have starting height of the
// given height.
func (s *UtxoScanner) getAtHeight(height uint32) []*GetUtxoRequest {
	s.cv.L.Lock()
	defer s.cv.L.Unlock()

	// Take any requests that are too old to go in this batch and keep them for
	// the next batch.
	for !s.pq.IsEmpty() && s.pq.Peek().StartHeight < height {
		item := heap.Pop(&s.pq).(*GetUtxoRequest)
		s.nextBatch = append(s.nextBatch, item)
	}

	var requests []*GetUtxoRequest
	for !s.pq.IsEmpty() && s.pq.Peek().StartHeight == height {
		item := heap.Pop(&s.pq).(*GetUtxoRequest)
		requests = append(requests, item)
	}

	return requests
}

// CheckTransactions finds any transactions in the block that spend the given
// outpoints.
func (s *UtxoScanner) CheckTransactions(block *wire.MsgBlock, height uint32,
	outpoints map[wire.OutPoint]struct{}) (map[wire.OutPoint]SpendReport,
	error) {
	spends := make(map[wire.OutPoint]SpendReport)

	// If we've spent the output in this block, return an
	// error stating that the output is spent.
	for _, tx := range block.Transactions {
		// Check each input to see if this transaction spends one of our
		// watched outpoints.
		for i, ti := range tx.TxIn {
			if _, ok := outpoints[ti.PreviousOutPoint]; ok {
				log.Debugf("Transaction %s spends outpoint %s", tx.TxHash(),
					ti.PreviousOutPoint)
				spends[ti.PreviousOutPoint] = SpendReport{
					SpendingTx:         tx,
					SpendingInputIndex: uint32(i),
					SpendingTxHeight:   height,
				}
			}
		}
	}

	return spends, nil
}

// filterMatches checks whether any of the filterEntries match for the given
// block.
func (s *UtxoScanner) filterMatches(hash chainhash.Hash,
	filterEntries [][]byte) (bool, error) {
	filter, err := s.chainClient.GetCFilter(hash,
		wire.GCSFilterRegular)
	if err != nil {
		return false, err
	}

	if filter != nil {
		filterKey := builder.DeriveKey(&hash)
		return filter.MatchAny(filterKey, filterEntries)
	}

	return false, nil
}

func buildFilterEntries(requests []*GetUtxoRequest) ([][]byte,
	map[wire.OutPoint]struct{}) {
	var filterEntries [][]byte
	outpoints := make(map[wire.OutPoint]struct{})

	for _, request := range requests {
		op := *request.OutPoint
		outpoints[op] = struct{}{}
		filterEntries = append(filterEntries, builder.OutPointToFilterEntry(op))
	}

	return filterEntries, outpoints
}

// runBatch runs a single batch. If there was an error, then return the
// outstanding requests.
func (s *UtxoScanner) runBatch() ([]*GetUtxoRequest, error) {
	var requests []*GetUtxoRequest

	// Take the request with the lowest block height so we can begin the scan
	// from there.
	req := s.peek()

	// Check if we were interrupted while waiting.
	if req == nil {
		return nil, nil
	}

	//log.Debugf("Running batch, looking for %d outpoints", s.Len())

	startHeight := req.StartHeight
	best, err := s.chainClient.BestSnapshot()
	if err != nil {
		return requests, err
	}

	// While scanning through the blockchain, take note of the transactions that
	// create the outpoints. If the outpoint isn't spent then return this
	// transaction.
	initialTx := make(map[wire.OutPoint]*SpendReport)

	filterEntries, outpoints := buildFilterEntries(requests)

	// Scan forward through the blockchain and look for any transactions that
	// might spend the given UTXOs.
	for height := startHeight; height <= uint32(best.Height); height++ {
		// If there are any new requests that can safely be added to this batch,
		// then try and fetch them.
		reqs := s.getAtHeight(height)
		if len(reqs) > 0 {
			for _, req := range reqs {
				log.Debugf("Adding %s (%d) to watchlist", req.OutPoint.String(),
					req.StartHeight)
				requests = append(requests, req)
				outpoints[*req.OutPoint] = struct{}{}
				filterEntries = append(filterEntries,
					builder.OutPointToFilterEntry(*req.OutPoint))
			}
		}

		hash, err := s.chainClient.GetBlockHash(int64(height))
		if err != nil {
			return requests, err
		}

		// If an outpoint is created in this block, then fetch it regardless.
		// Otherwise check to see if the filter matches any of our watched
		// outpoints.
		fetch := len(reqs) > 0
		if !fetch {
			match, err := s.filterMatches(*hash, filterEntries)
			if err != nil {
				return requests, err
			}
			fetch = match
		}

		if fetch {
			log.Debugf("Fetching block at height %d (%s)", height,
				hash.String())

			// FIXME(simon): Find out why this takes three minutes.
			// Fetch the block from the network.
			block, err := s.chainClient.GetBlockFromNetwork(*hash)
			if err != nil {
				return requests, err
			}

			log.Debugf("Got block %d (%s)", height, hash.String())

			if fetch {
				for _, request := range requests {
					if request.StartHeight == height {
						tx := s.GetOriginalTx(block.MsgBlock(),
							request.OutPoint)
						// Grab the tx that created this output.
						initialTx[*request.OutPoint] = tx

						log.Debugf("Block %d creates output %s", height,
							request.OutPoint.String())
					}
				}
			}

			spends, err := s.CheckTransactions(block.MsgBlock(), height,
				outpoints)
			if err != nil {
				return requests, err
			}

			for outPoint, spend := range spends {
				log.Debugf("Outpoint %s is spent in tx %s", outPoint.String(),
					spew.Sprint(spend))

				// Find the request this spend relates to.
				var filteredRequests []*GetUtxoRequest
				for _, request := range requests {
					if *request.OutPoint == outPoint {
						request.Result(&spend, nil)
					} else {
						filteredRequests = append(filteredRequests, request)
					}
				}
				requests = filteredRequests

				// Remove the filter from filterEntries.
				filterEntries, outpoints = buildFilterEntries(requests)
			}
		}
	}

	log.Debugf("Finished batch, %d unspent outpoints", len(requests))

	for _, request := range requests {
		tx, ok := initialTx[*request.OutPoint]
		if ok {
			request.Result(tx, nil)
		} else {
			// A nil SpendReport indicates the output was not found.
			request.Result(nil, nil)
		}
	}

	return nil, nil
}

// Stop any in-progress scan.
func (s *UtxoScanner) Stop() {
	s.cv.L.Lock()
	defer s.cv.L.Unlock()

	s.stopped = true
}

// Returns the GetUtxoRequest with the lowest block height. If no elements are
// available, then block until one is added.
func (s *UtxoScanner) peek() *GetUtxoRequest {
	s.cv.L.Lock()
	defer s.cv.L.Unlock()

	// Block until the queue is no longer empty.
	for !s.stopped && s.pq.IsEmpty() {
		s.cv.Wait()
	}

	// We return nil only in the case that we've been interrupted, so callers
	// can use this to determine that the UtxoScanner is shutting down.
	if s.stopped {
		return nil
	}

	return s.pq.Peek()
}

// Enqueue takes a GetUtxoRequest and adds it to the next applicable batch.
func (s *UtxoScanner) Enqueue(req *GetUtxoRequest) {
	log.Debugf("Enqueuing request for %s with start height %d",
		req.OutPoint.String(), req.StartHeight)
	s.cv.L.Lock()

	// Insert the request into the queue and signal any threads that might be
	// waiting for new elements.
	heap.Push(&s.pq, req)

	s.cv.L.Unlock()
	s.cv.Signal()
}

// GetOriginalTx returns a SpendReport for the UTXO, or nil if it does not exist
// in this block.
func (s *UtxoScanner) GetOriginalTx(block *wire.MsgBlock,
	point *wire.OutPoint) *SpendReport {
	for _, tx := range block.Transactions {
		if tx.TxHash() == point.Hash {
			outputs := tx.TxOut
			return &SpendReport{
				Output: outputs[point.Index],
			}
		}
	}

	log.Errorf("Failed to find tx %s", point.Hash.String())

	return nil
}
<commit_msg>utxoscanner: Fix some comments<commit_after>package neutrino

import (
	"container/heap"
	"github.com/davecgh/go-spew/spew"
	"github.com/roasbeef/btcd/chaincfg/chainhash"
	"github.com/roasbeef/btcd/wire"
	"github.com/roasbeef/btcutil"
	"github.com/roasbeef/btcutil/gcs"
	"github.com/roasbeef/btcutil/gcs/builder"
	"github.com/roasbeef/btcwallet/waddrmgr"
	"sync"
)

// GetUtxoRequest is a request to scan for OutPoint from the height StartHeight.
type GetUtxoRequest struct {
	OutPoint    *wire.OutPoint
	StartHeight uint32
	Result      func(*SpendReport, error)
}

// Interface exposes the necessary methods for interacting with the blockchain.
type Interface interface {
	GetBlockFromNetwork(chainhash.Hash, ...QueryOption) (*btcutil.Block, error)
	GetBlockHash(int64) (*chainhash.Hash, error)
	BestSnapshot() (*waddrmgr.BlockStamp, error)
	GetCFilter(blockHash chainhash.Hash, filterType wire.FilterType,
		options ...QueryOption) (*gcs.Filter, error)
}

// A PriorityQueue implements heap.Interface and holds GetUtxoRequests. The
// queue maintains that heap.Pop() will always return the GetUtxo request with
// the least starting height. This allows us to add new GetUtxo requests to
// an already running batch.
type PriorityQueue []*GetUtxoRequest

func (pq PriorityQueue) Len() int { return len(pq) }

func (pq PriorityQueue) Less(i, j int) bool {
	// We want Pop to give us the least StartHeight.
	return pq[i].StartHeight < pq[j].StartHeight
}

func (pq PriorityQueue) Swap(i, j int) {
	pq[i], pq[j] = pq[j], pq[i]
}

// Push is called by the heap.Interface implementation to add an element to the
// end of the backing store. The heap library will then maintain the heap
// invariant.
func (pq *PriorityQueue) Push(x interface{}) {
	item := x.(*GetUtxoRequest)
	*pq = append(*pq, item)
}

// Peek returns the least height element in the queue without removing it.
func (pq *PriorityQueue) Peek() *GetUtxoRequest {
	return (*pq)[0]
}

// Pop is called by the heap.Interface implementation to remove an element from
// the end of the backing store. The heap library will then maintain the heap
// invariant.
func (pq *PriorityQueue) Pop() interface{} {
	old := *pq
	n := len(old)
	item := old[n-1]
	*pq = old[0 : n-1]
	return item
}

// IsEmpty returns true if the queue has no elements.
func (pq *PriorityQueue) IsEmpty() bool {
	return pq.Len() == 0
}

// UtxoScanner batches calls to GetUtxo so that a single scan can search for
// multiple outpoints. If a scan is in progress when a new element is added, we
// check whether it can safely be added to the current batch, if not it will be
// included in the next batch.
type UtxoScanner struct {
	pq      PriorityQueue
	stopped bool

	chainClient Interface

	nextBatch []*GetUtxoRequest

	cv *sync.Cond
}

// NewUtxoScanner creates a new instance of UtxoScanner using the given chain
// interface.
func NewUtxoScanner(chainClient Interface) UtxoScanner {
	return UtxoScanner{
		pq:          make(PriorityQueue, 0),
		cv:          sync.NewCond(&sync.Mutex{}),
		stopped:     false,
		chainClient: chainClient,
	}
}

// Start begins running scan batches.
func (s *UtxoScanner) Start() {
	go func() {
		for {
			s.cv.L.Lock()
			if s.stopped {
				s.cv.L.Unlock()
				return
			}

			// Re-queue previously skipped requests for next batch.
			for _, request := range s.nextBatch {
				heap.Push(&s.pq, request)
			}

			s.nextBatch = nil

			s.cv.L.Unlock()

			requests, err := s.runBatch()

			// If there was an error, then notify the currently outstanding
			// requests.
			if err != nil {
				for _, request := range requests {
					request.Result(nil, err)
				}
			}
		}
	}()
}

// getAtHeight returns all GetUtxo requests that have starting height of the
// given height.
func (s *UtxoScanner) getAtHeight(height uint32) []*GetUtxoRequest {
	s.cv.L.Lock()
	defer s.cv.L.Unlock()

	// Take any requests that are too old to go in this batch and keep them for
	// the next batch.
	for !s.pq.IsEmpty() && s.pq.Peek().StartHeight < height {
		item := heap.Pop(&s.pq).(*GetUtxoRequest)
		s.nextBatch = append(s.nextBatch, item)
	}

	var requests []*GetUtxoRequest
	for !s.pq.IsEmpty() && s.pq.Peek().StartHeight == height {
		item := heap.Pop(&s.pq).(*GetUtxoRequest)
		requests = append(requests, item)
	}

	return requests
}

// CheckTransactions finds any transactions in the block that spend the given
// outpoints.
func (s *UtxoScanner) CheckTransactions(block *wire.MsgBlock, height uint32,
	outpoints map[wire.OutPoint]struct{}) (map[wire.OutPoint]SpendReport,
	error) {
	spends := make(map[wire.OutPoint]SpendReport)

	// If we've spent the output in this block, return an
	// error stating that the output is spent.
	for _, tx := range block.Transactions {
		// Check each input to see if this transaction spends one of our
		// watched outpoints.
		for i, ti := range tx.TxIn {
			if _, ok := outpoints[ti.PreviousOutPoint]; ok {
				log.Debugf("Transaction %s spends outpoint %s", tx.TxHash(),
					ti.PreviousOutPoint)
				spends[ti.PreviousOutPoint] = SpendReport{
					SpendingTx:         tx,
					SpendingInputIndex: uint32(i),
					SpendingTxHeight:   height,
				}
			}
		}
	}

	return spends, nil
}

// filterMatches checks whether any of the filterEntries match for the given
// block.
func (s *UtxoScanner) filterMatches(hash chainhash.Hash,
	filterEntries [][]byte) (bool, error) {
	filter, err := s.chainClient.GetCFilter(hash,
		wire.GCSFilterRegular)
	if err != nil {
		return false, err
	}

	if filter != nil {
		filterKey := builder.DeriveKey(&hash)
		return filter.MatchAny(filterKey, filterEntries)
	}

	return false, nil
}

func buildFilterEntries(requests []*GetUtxoRequest) ([][]byte,
	map[wire.OutPoint]struct{}) {
	var filterEntries [][]byte
	outpoints := make(map[wire.OutPoint]struct{})

	for _, request := range requests {
		op := *request.OutPoint
		outpoints[op] = struct{}{}
		filterEntries = append(filterEntries, builder.OutPointToFilterEntry(op))
	}

	return filterEntries, outpoints
}

// runBatch runs a single batch. If there was an error, then return the
// outstanding requests.
func (s *UtxoScanner) runBatch() ([]*GetUtxoRequest, error) {
	var requests []*GetUtxoRequest

	// Take the request with the lowest block height so we can begin the scan
	// from there.
	req := s.peek()

	// Check if we were interrupted while waiting.
	if req == nil {
		return nil, nil
	}

	//log.Debugf("Running batch, looking for %d outpoints", s.Len())

	startHeight := req.StartHeight
	best, err := s.chainClient.BestSnapshot()
	if err != nil {
		return requests, err
	}

	// While scanning through the blockchain, take note of the transactions that
	// create the outpoints. If the outpoint isn't spent then return this
	// transaction.
	initialTx := make(map[wire.OutPoint]*SpendReport)

	filterEntries, outpoints := buildFilterEntries(requests)

	// Scan forward through the blockchain and look for any transactions that
	// might spend the given UTXOs.
	for height := startHeight; height <= uint32(best.Height); height++ {
		// If there are any new requests that can safely be added to this batch,
		// then try and fetch them.
		reqs := s.getAtHeight(height)
		if len(reqs) > 0 {
			for _, req := range reqs {
				log.Debugf("Adding %s (%d) to watchlist", req.OutPoint.String(),
					req.StartHeight)
				requests = append(requests, req)
				outpoints[*req.OutPoint] = struct{}{}
				filterEntries = append(filterEntries,
					builder.OutPointToFilterEntry(*req.OutPoint))
			}
		}

		hash, err := s.chainClient.GetBlockHash(int64(height))
		if err != nil {
			return requests, err
		}

		// If an outpoint is created in this block, then fetch it regardless.
		// Otherwise check to see if the filter matches any of our watched
		// outpoints.
		fetch := len(reqs) > 0
		if !fetch {
			match, err := s.filterMatches(*hash, filterEntries)
			if err != nil {
				return requests, err
			}
			fetch = match
		}

		if fetch {
			log.Debugf("Fetching block at height %d (%s)", height,
				hash.String())

			// FIXME(simon): Find out why this takes three minutes.
			// Fetch the block from the network.
			block, err := s.chainClient.GetBlockFromNetwork(*hash)
			if err != nil {
				return requests, err
			}

			log.Debugf("Got block %d (%s)", height, hash.String())

			if fetch {
				for _, request := range requests {
					if request.StartHeight == height {
						tx := s.GetOriginalTx(block.MsgBlock(),
							request.OutPoint)
						// Grab the tx that created this output.
						initialTx[*request.OutPoint] = tx

						log.Debugf("Block %d creates output %s", height,
							request.OutPoint.String())
					}
				}
			}

			spends, err := s.CheckTransactions(block.MsgBlock(), height,
				outpoints)
			if err != nil {
				return requests, err
			}

			for outPoint, spend := range spends {
				log.Debugf("Outpoint %s is spent in tx %s", outPoint.String(),
					spew.Sprint(spend))

				// Find the request this spend relates to.
				var filteredRequests []*GetUtxoRequest
				for _, request := range requests {
					if *request.OutPoint == outPoint {
						request.Result(&spend, nil)
					} else {
						filteredRequests = append(filteredRequests, request)
					}
				}
				requests = filteredRequests

				// Remove the filter from filterEntries.
				filterEntries, outpoints = buildFilterEntries(requests)
			}
		}
	}

	log.Debugf("Finished batch, %d unspent outpoints", len(requests))

	for _, request := range requests {
		tx, ok := initialTx[*request.OutPoint]
		if ok {
			request.Result(tx, nil)
		} else {
			// A nil SpendReport indicates the output was not found.
			request.Result(nil, nil)
		}
	}

	return nil, nil
}

// Stop any in-progress scan.
func (s *UtxoScanner) Stop() {
	s.cv.L.Lock()
	defer s.cv.L.Unlock()

	s.stopped = true
}

// Returns the GetUtxoRequest with the lowest block height. If no elements are
// available, then block until one is added.
func (s *UtxoScanner) peek() *GetUtxoRequest {
	s.cv.L.Lock()
	defer s.cv.L.Unlock()

	// Block until the queue is no longer empty.
	for !s.stopped && s.pq.IsEmpty() {
		s.cv.Wait()
	}

	// We return nil only in the case that we've been interrupted, so callers
	// can use this to determine that the UtxoScanner is shutting down.
	if s.stopped {
		return nil
	}

	return s.pq.Peek()
}

// Enqueue takes a GetUtxoRequest and adds it to the next applicable batch.
func (s *UtxoScanner) Enqueue(req *GetUtxoRequest) {
	log.Debugf("Enqueuing request for %s with start height %d",
		req.OutPoint.String(), req.StartHeight)
	s.cv.L.Lock()

	// Insert the request into the queue and signal any threads that might be
	// waiting for new elements.
	heap.Push(&s.pq, req)

	s.cv.L.Unlock()
	s.cv.Signal()
}

// GetOriginalTx returns a SpendReport for the UTXO, or nil if it does not exist
// in this block.
func (s *UtxoScanner) GetOriginalTx(block *wire.MsgBlock,
	point *wire.OutPoint) *SpendReport {
	for _, tx := range block.Transactions {
		if tx.TxHash() == point.Hash {
			outputs := tx.TxOut
			return &SpendReport{
				Output: outputs[point.Index],
			}
		}
	}

	log.Errorf("Failed to find tx %s", point.Hash.String())

	return nil
}
<|endoftext|>


<commit_before>package ram

import (
	"testing"
)

var (
	ram_ak_username string
	accessKeyId     string
	status          State
)

func TestCreateAccessKey(t *testing.T) {
	var ramUser User
	client := NewTestClient()
	listParams := ListUserRequest{}
	listUserResp, err := client.ListUsers(listParams)
	if err != nil {
		t.Errorf("Failed to list user %v", err)
	}
	if len(listUserResp.Users.User) == 0 {
		//TODO create user
		userResp, err := client.CreateUser(user)
		ramUser = userResp.User
		if err != nil {
			t.Errorf("Failed to create user %v", err)
		}
	} else {
		ramUser = listUserResp.Users.User[0]
	}
	//TODO 添加到公共变量中
	ram_ak_username = ramUser.UserName

	userQuery := UserQueryRequest{UserName: ram_ak_username}
	accessKeyResponse, err := client.CreateAccessKey(userQuery)
	//加入到公共变量中
	accessKeyId = accessKeyResponse.AccessKey.AccessKeyId
	status = accessKeyResponse.AccessKey.Status

	if err != nil {
		t.Errorf("Failed to create AccessKey %v", err)
	}
	t.Logf("pass create AccessKey %v", accessKeyResponse)
}

func TestUpdateAccessKey(t *testing.T) {
	client := NewTestClient()
	updateAccessKeyRequest := UpdateAccessKeyRequest{
		UserAccessKeyId: accessKeyId,
		UserName:        ram_ak_username,
	}
	if status == Active {
		updateAccessKeyRequest.Status = Inactive
	} else {
		updateAccessKeyRequest.Status = Active
	}

	accessKeyResponse, err := client.UpdateAccessKey(updateAccessKeyRequest)
	if err != nil {
		t.Errorf("Failed to UpdateAccessKey %", err)
	}
	t.Logf("pass UpdateAccessKey %v", accessKeyResponse)
}

func TestListAccessKeys(t *testing.T) {
	client := NewTestClient()
	userQuery := UserQueryRequest{UserName: ram_ak_username}
	resp, err := client.ListAccessKeys(userQuery)
	if err != nil {
		t.Errorf("Failed to list ListAccessKeys %v", err)
	}
	t.Logf("pass ListAccessKeys %v", resp)
}

func TestDeleteAccessKey(t *testing.T) {
	client := NewTestClient()
	accessKeyRequest := UpdateAccessKeyRequest{
		UserAccessKeyId: accessKeyId,
		UserName:        ram_ak_username,
	}
	var commonResp RamCommonResponse
	commonResp, err := client.DeleteAccessKey(accessKeyRequest)
	if err != nil {
		t.Errorf("Failed to TestDeleteAccessKey %v", err)
	}
	t.Logf("pass DeleteAccessKey %v", commonResp)
}
<commit_msg>fix missing %v<commit_after>package ram

import (
	"testing"
)

var (
	ram_ak_username string
	accessKeyId     string
	status          State
)

func TestCreateAccessKey(t *testing.T) {
	var ramUser User
	client := NewTestClient()
	listParams := ListUserRequest{}
	listUserResp, err := client.ListUsers(listParams)
	if err != nil {
		t.Errorf("Failed to list user %v", err)
	}
	if len(listUserResp.Users.User) == 0 {
		//TODO create user
		userResp, err := client.CreateUser(user)
		ramUser = userResp.User
		if err != nil {
			t.Errorf("Failed to create user %v", err)
		}
	} else {
		ramUser = listUserResp.Users.User[0]
	}
	//TODO 添加到公共变量中
	ram_ak_username = ramUser.UserName

	userQuery := UserQueryRequest{UserName: ram_ak_username}
	accessKeyResponse, err := client.CreateAccessKey(userQuery)
	//加入到公共变量中
	accessKeyId = accessKeyResponse.AccessKey.AccessKeyId
	status = accessKeyResponse.AccessKey.Status

	if err != nil {
		t.Errorf("Failed to create AccessKey %v", err)
	}
	t.Logf("pass create AccessKey %v", accessKeyResponse)
}

func TestUpdateAccessKey(t *testing.T) {
	client := NewTestClient()
	updateAccessKeyRequest := UpdateAccessKeyRequest{
		UserAccessKeyId: accessKeyId,
		UserName:        ram_ak_username,
	}
	if status == Active {
		updateAccessKeyRequest.Status = Inactive
	} else {
		updateAccessKeyRequest.Status = Active
	}

	accessKeyResponse, err := client.UpdateAccessKey(updateAccessKeyRequest)
	if err != nil {
		t.Errorf("Failed to UpdateAccessKey %v", err)
	}
	t.Logf("pass UpdateAccessKey %v", accessKeyResponse)
}

func TestListAccessKeys(t *testing.T) {
	client := NewTestClient()
	userQuery := UserQueryRequest{UserName: ram_ak_username}
	resp, err := client.ListAccessKeys(userQuery)
	if err != nil {
		t.Errorf("Failed to list ListAccessKeys %v", err)
	}
	t.Logf("pass ListAccessKeys %v", resp)
}

func TestDeleteAccessKey(t *testing.T) {
	client := NewTestClient()
	accessKeyRequest := UpdateAccessKeyRequest{
		UserAccessKeyId: accessKeyId,
		UserName:        ram_ak_username,
	}
	var commonResp RamCommonResponse
	commonResp, err := client.DeleteAccessKey(accessKeyRequest)
	if err != nil {
		t.Errorf("Failed to TestDeleteAccessKey %v", err)
	}
	t.Logf("pass DeleteAccessKey %v", commonResp)
}
<|endoftext|>


<commit_before>package protocol

import (
	"reflect"
	"bufio"
	"strings"
	"testing"
)

// Create a bufio.Reader
func createReader(payload string) *bufio.Reader {
	sio := strings.NewReader(payload)
	return bufio.NewReader(sio)
}

func testMatch(t *testing.T, payload string, expected readObject) {
	var rd *bufio.Reader
	rd = createReader(payload)

	var obj readObject
	obj, _ = read(rd)

	if !reflect.DeepEqual(expected, obj) {
		t.Errorf("\nexpected: %#v\ngot: %#v\n", expected, obj)
	}
}

func testError(t *testing.T, payload string) {
	var rd *bufio.Reader
	rd = createReader(payload)

	var err error
	_, err = read(rd)

	if err == nil {
		t.Errorf("\nexpected error to be non-nil\n")
	}
}

func TestReadMessage(t *testing.T) {
	var expected = &readMessage{
		Subscription:   []byte("sub"),
		SubscriptionId: 1234,
		ReplyTo:        nil,
		Payload:        []byte("some message"),
	}

	testMatch(t, "msg sub 1234 12\r\nsome message\r\n", expected)
}

func TestReadMessageWithInvalidSubscriptionId(t *testing.T) {
	testError(t, "msg sub xxxx 12\r\nsome message\r\n")
}

func TestReadMessageWithInvalidByteCount(t *testing.T) {
	testError(t, "msg sub 1234 xx\r\nsome message\r\n")
}

func TestReadMessageWithEarlyEof(t *testing.T) {
	testError(t, "msg sub 1234 12\r\nsome message\r")
}

func TestReadOk(t *testing.T) {
	var expected = &readOk{}

	testMatch(t, "+ok\r\n", expected)
}

func TestReadErrWithoutPayload(t *testing.T) {
	var expected = &readErr{
		Payload: nil,
	}

	testMatch(t, "-err\r\n", expected)
}

func TestReadErrWithPayload(t *testing.T) {
	var expected = &readErr{
		Payload: []byte("foo"),
	}

	testMatch(t, "-err foo\r\n", expected)
}

func TestReadErrWithPayloadWithWhiteSpace(t *testing.T) {
	var expected = &readErr{
		Payload: []byte("foo bar qux"),
	}

	testMatch(t, "-err foo bar qux\r\n", expected)
}

func TestReadPing(t *testing.T) {
	var expected = &readPing{}

	testMatch(t, "ping\r\n", expected)
}

func TestReadPong(t *testing.T) {
	var expected = &readPong{}

	testMatch(t, "pong\r\n", expected)
}

func TestReadInfo(t *testing.T) {
	var expected = &readInfo{
		ServerId: "some id",
	}

	testMatch(t, "info {\"server_id\":\"some id\" }\r\n", expected)
}
<commit_msg>Prefix test helpers<commit_after>package protocol

import (
	"reflect"
	"bufio"
	"strings"
	"testing"
)

// Create a bufio.Reader
func createReader(payload string) *bufio.Reader {
	sio := strings.NewReader(payload)
	return bufio.NewReader(sio)
}

func testReadMatch(t *testing.T, payload string, expected readObject) {
	var rd *bufio.Reader
	rd = createReader(payload)

	var obj readObject
	obj, _ = read(rd)

	if !reflect.DeepEqual(expected, obj) {
		t.Errorf("\nexpected: %#v\ngot: %#v\n", expected, obj)
	}
}

func testReadError(t *testing.T, payload string) {
	var rd *bufio.Reader
	rd = createReader(payload)

	var err error
	_, err = read(rd)

	if err == nil {
		t.Errorf("\nexpected error to be non-nil\n")
	}
}

func TestReadMessage(t *testing.T) {
	var expected = &readMessage{
		Subscription:   []byte("sub"),
		SubscriptionId: 1234,
		ReplyTo:        nil,
		Payload:        []byte("some message"),
	}

	testReadMatch(t, "msg sub 1234 12\r\nsome message\r\n", expected)
}

func TestReadMessageWithInvalidSubscriptionId(t *testing.T) {
	testReadError(t, "msg sub xxxx 12\r\nsome message\r\n")
}

func TestReadMessageWithInvalidByteCount(t *testing.T) {
	testReadError(t, "msg sub 1234 xx\r\nsome message\r\n")
}

func TestReadMessageWithEarlyEof(t *testing.T) {
	testReadError(t, "msg sub 1234 12\r\nsome message\r")
}

func TestReadOk(t *testing.T) {
	var expected = &readOk{}

	testReadMatch(t, "+ok\r\n", expected)
}

func TestReadErrWithoutPayload(t *testing.T) {
	var expected = &readErr{
		Payload: nil,
	}

	testReadMatch(t, "-err\r\n", expected)
}

func TestReadErrWithPayload(t *testing.T) {
	var expected = &readErr{
		Payload: []byte("foo"),
	}

	testReadMatch(t, "-err foo\r\n", expected)
}

func TestReadErrWithPayloadWithWhiteSpace(t *testing.T) {
	var expected = &readErr{
		Payload: []byte("foo bar qux"),
	}

	testReadMatch(t, "-err foo bar qux\r\n", expected)
}

func TestReadPing(t *testing.T) {
	var expected = &readPing{}

	testReadMatch(t, "ping\r\n", expected)
}

func TestReadPong(t *testing.T) {
	var expected = &readPong{}

	testReadMatch(t, "pong\r\n", expected)
}

func TestReadInfo(t *testing.T) {
	var expected = &readInfo{
		ServerId: "some id",
	}

	testReadMatch(t, "info {\"server_id\":\"some id\" }\r\n", expected)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"log"
	"net/rpc/jsonrpc"
	"strings"

	"github.com/natefinch/pie"
)

type LuaPlugin struct{}

const namespace = "Lua"

// --- Plugin functionality ---

func add3(a, b int) int {
	// Functionality not otherwise available in Lua goes here
	return a + b + 3
}

// --- Lua wrapper code ($0 is replaced with the plugin path) ---

const luacode = `
function add3(a, b)
  return CallPlugin("$0", "Add3", a, b)
end
`

// --- Lua help text (will be syntax highlighted) ---

const luahelp = `
add3(number, number) -> number // Adds two numbers and then the number 3
`

// --- Plugin wrapper functions ---

func (LuaPlugin) Add3(jsonargs []byte, response *[]byte) (err error) {
	var args []int
	err = json.Unmarshal(jsonargs, &args)
	if err != nil || len(args) < 2 {
		// Could not unmarshal the given arguments, or too few arguments
		return errors.New("add3 requires two integer arguments")
	}
	result := add3(args[0], args[1])
	*response, err = json.Marshal(result)
	return
}

// --- Plugin functions that must be present ---

// Called once when the Plugin function is used in Algernon
func (LuaPlugin) Code(pluginPath string, response *string) error {
	*response = strings.Replace(luacode, "$0", pluginPath, -1)
	return nil
}

// Called once when the Plugin function is used in Algernon
func (LuaPlugin) Help(_ string, response *string) error {
	*response = luahelp
	return nil
}

// Called once when the Plugin or CallPlugin function is used in Algernon
func main() {
	log.SetPrefix("[plugin log] ")
	p := pie.NewProvider()
	if err := p.RegisterName(namespace, LuaPlugin{}); err != nil {
		log.Fatalf("Failed to register plugin: %s", err)
	}
	p.ServeCodec(jsonrpc.NewServerCodec)
}
<commit_msg>Fix comment<commit_after>package main

import (
	"encoding/json"
	"errors"
	"log"
	"net/rpc/jsonrpc"
	"strings"

	"github.com/natefinch/pie"
)

type LuaPlugin struct{}

const namespace = "Lua"

// --- Plugin functionality ---

func add3(a, b int) int {
	// Functionality not otherwise available in Lua goes here
	return a + b + 3
}

// --- Lua wrapper code ($0 is replaced with the plugin path) ---

const luacode = `
function add3(a, b)
  return CallPlugin("$0", "Add3", a, b)
end
`

// --- Lua help text (will be syntax highlighted) ---

const luahelp = `
add3(number, number) -> number // Adds two numbers and then the number 3
`

// --- Plugin wrapper functions ---

func (LuaPlugin) Add3(jsonargs []byte, response *[]byte) (err error) {
	var args []int
	err = json.Unmarshal(jsonargs, &args)
	if err != nil || len(args) < 2 {
		// Could not unmarshal the given arguments, or too few arguments
		return errors.New("add3 requires two integer arguments")
	}
	result := add3(args[0], args[1])
	*response, err = json.Marshal(result)
	return
}

// --- Plugin functions that must be present ---

// Called once when the Plugin function is used in Algernon
func (LuaPlugin) Code(pluginPath string, response *string) error {
	*response = strings.Replace(luacode, "$0", pluginPath, -1)
	return nil
}

// Called once when the help function is used in Algernon
func (LuaPlugin) Help(_ string, response *string) error {
	*response = luahelp
	return nil
}

// Called once when the Plugin or CallPlugin function is used in Algernon
func main() {
	log.SetPrefix("[plugin log] ")
	p := pie.NewProvider()
	if err := p.RegisterName(namespace, LuaPlugin{}); err != nil {
		log.Fatalf("Failed to register plugin: %s", err)
	}
	p.ServeCodec(jsonrpc.NewServerCodec)
}
<|endoftext|>


<commit_before>package plugin

import (
	"fmt"
	"log"

	"github.com/j6n/noye/noye"
)

// Base is a type to reduce boilerplate of plugins
// It holds the noye.Bot reference, a channel to receive
// noye.Messages on and a map of disabled things
type Base struct {
	Bot   noye.Bot
	name  string
	input chan noye.Message

	Handlers []*Handler
}

// New returns a new Base plugin
func New(name string, handlers ...*Handler) *Base {
	base := &Base{
		name:     name,
		input:    make(chan noye.Message),
		Handlers: handlers,
	}

	go base.process()
	return base
}

// Listen returns the message to receives messages upon
func (b *Base) Listen() chan noye.Message {
	return b.input
}

// Name returns the plugins name
func (b *Base) Name() string {
	return b.name
}

// Hook sets the noye.Bot reference
func (b *Base) Hook(bot noye.Bot) {
	b.Bot = bot
}

// Reply replies to the noye.Message with a formatted string
func (b *Base) Reply(msg noye.Message, f string, a ...interface{}) {
	b.Bot.Privmsg(msg.Target, fmt.Sprintf(f, a...))
}

// Error replies to the noye.Message with a readable error
func (b *Base) Error(msg noye.Message, text string, err error) {
	b.Reply(msg, "error with %s (%s)", text, err)
}

func (b *Base) process() {
	for msg := range b.input {
		for _, cmd := range b.Handlers {
			defer func() {
				if err := recover(); err != nil {
					log.Printf("recover! %s/%s from %s\n", b.Name(), cmd.Command.Command, err)
				}
			}()
			if !cmd.AcceptsFrom(msg.From) {
				b.Reply(msg, "You can't do this command.")
				continue
			}
			if cmd.Match(msg) {
				cmd.Handle(cmd.Command, msg)
			}
		}
	}
}

type Handler struct {
	*Command
	Handle func(*Command, noye.Message)
}
<commit_msg>match the command first, then see if the user was accepted<commit_after>package plugin

import (
	"fmt"
	"log"

	"github.com/j6n/noye/noye"
)

// Handler encapsulate a Command and gives it a callback
type Handler struct {
	*Command
	Handle func(*Command, noye.Message)
}

// Base is a type to reduce boilerplate of plugins
// It holds the noye.Bot reference, a channel to receive
// noye.Messages on and a map of disabled things
type Base struct {
	Bot   noye.Bot
	name  string
	input chan noye.Message

	Handlers []*Handler
}

// New returns a new Base plugin
func New(name string, handlers ...*Handler) *Base {
	base := &Base{
		name:     name,
		input:    make(chan noye.Message),
		Handlers: handlers,
	}

	go base.process()
	return base
}

// Listen returns the message to receives messages upon
func (b *Base) Listen() chan noye.Message {
	return b.input
}

// Name returns the plugins name
func (b *Base) Name() string {
	return b.name
}

// Hook sets the noye.Bot reference
func (b *Base) Hook(bot noye.Bot) {
	b.Bot = bot
}

// Reply replies to the noye.Message with a formatted string
func (b *Base) Reply(msg noye.Message, f string, a ...interface{}) {
	b.Bot.Privmsg(msg.Target, fmt.Sprintf(f, a...))
}

// Error replies to the noye.Message with a readable error
func (b *Base) Error(msg noye.Message, text string, err error) {
	b.Reply(msg, "error with %s (%s)", text, err)
}

func (b *Base) process() {
	for msg := range b.input {
		for _, cmd := range b.Handlers {
			defer func() {
				if err := recover(); err != nil {
					log.Printf("recover! %s/%s from %s\n", b.Name(), cmd.Command.Command, err)
				}
			}()
			if cmd.Match(msg) {
				if !cmd.AcceptsFrom(msg.From) {
					b.Reply(msg, "You can't do this command.")
					continue
				}
				cmd.Handle(cmd.Command, msg)
			}
		}
	}
}
<|endoftext|>


<commit_before>package data

import (
	"reflect"
	"testing"
)

func TestDataClip(t *testing.T) {
	cases := []struct {
		data []byte
		want Clip
	}{
		{
			data: []byte("this is sparta!"),
			want: Clip{data: []byte("this is sparta!"), newlines: []int{}},
		},
		{
			data: []byte("this is sparta!\nright?"),
			want: Clip{data: []byte("this is sparta!\nright?"), newlines: []int{15}},
		},
		{
			data: []byte("\n\n\n\n"),
			want: Clip{data: []byte("\n\n\n\n"), newlines: []int{0, 1, 2, 3}},
		},
	}
	for _, c := range cases {
		got := DataClip(c.data)
		if !reflect.DeepEqual(got, c.want) {
			t.Fatalf("got %q, want %q", got, c.want)
		}
	}
}

func TestCursorWrite(t *testing.T) {
	cases := []struct {
		label  string
		cs     *Cursor
		writes []rune
		at     int
		want   *Cursor
	}{
		{
			label:  "middle",
			cs:     NewCursor([]Clip{DataClip([]byte("this is sparta."))}),
			writes: []rune("n't"),
			at:     7,
			want: &Cursor{
				clips:     []Clip{DataClip([]byte("this is")), DataClip([]byte("n't")), DataClip([]byte(" sparta."))},
				i:         2,
				o:         0,
				appending: true,
			},
		},
		{
			label:  "first",
			cs:     NewCursor([]Clip{DataClip([]byte("this is sparta."))}),
			writes: []rune("hey, "),
			at:     0,
			want: &Cursor{
				clips:     []Clip{DataClip([]byte("hey, ")), DataClip([]byte("this is sparta."))},
				i:         1,
				o:         0,
				appending: true,
			},
		},
		{
			label:  "last",
			cs:     NewCursor([]Clip{DataClip([]byte("this is sparta."))}),
			writes: []rune(" isn't it?"),
			at:     15,
			want: &Cursor{
				clips:     []Clip{DataClip([]byte("this is sparta.")), DataClip([]byte(" isn't it?"))},
				i:         2,
				o:         0,
				appending: true,
			},
		},
	}
	for _, c := range cases {
		defer func() {
			if r := recover(); r != nil {
				t.Fatalf("%q panicked: %s", c.label, r)
			}
		}()
		c.cs.Shift(c.at)
		for _, r := range c.writes {
			c.cs.Write(r)
		}
		if !reflect.DeepEqual(c.cs, c.want) {
			t.Fatalf("got %v, want %v", c.cs, c.want)
		}
	}
}

func TestCursorGotoNextLine(t *testing.T) {
	type Step struct {
		i int
		o int
	}
	cases := []struct {
		label string
		clips []Clip
		wants []Step
	}{
		{
			label: "simple",
			clips: []Clip{
				DataClip([]byte("what a nice day\n")),
				DataClip([]byte("do you have breakfast?\n or shall we?")),
			},
			wants: []Step{{0, 15}, {1, 22}, {2, 0}, {2, 0}},
		},
		{
			label: "korean",
			clips: []Clip{
				DataClip([]byte("이 건\n 한글")), DataClip([]byte("테스트 입니다.\n")), // each hangul character is 3 bytes
			},
			wants: []Step{{0, 7}, {1, 20}, {2, 0}, {2, 0}},
		},
	}

	for _, c := range cases {
		cs := NewCursor(c.clips)
		for i, want := range c.wants {
			cs.GotoNextLine()
			if cs.i != want.i || cs.o != want.o {
				t.Fatalf("%s: step %d: got [%d:%d], want [%d:%d]", c.label, i, cs.i, cs.o, want.i, want.o)
			}
		}
	}
}

func TestCursorGotoPrevLine(t *testing.T) {
	type Step struct {
		i int
		o int
	}
	cases := []struct {
		label string
		clips []Clip
		wants []Step
	}{
		{
			label: "simple",
			clips: []Clip{
				DataClip([]byte("what a nice day\n")),
				DataClip([]byte("do you have breakfast?\n or shall we?")),
			},
			wants: []Step{{1, 22}, {0, 15}, {0, 0}, {0, 0}},
		},
		{
			label: "korean",
			clips: []Clip{
				DataClip([]byte("이 건\n 한글")), DataClip([]byte("테스트 입니다.\n")), // each hangul character is 3 bytes
			},
			wants: []Step{{1, 20}, {0, 7}, {0, 0}, {0, 0}},
		},
	}

	for _, c := range cases {
		cs := &Cursor{clips: c.clips, i: len(c.clips), o: 0}
		for i, want := range c.wants {
			cs.GotoPrevLine()
			if cs.i != want.i || cs.o != want.o {
				t.Fatalf("%s: step %d: got [%d:%d], want [%d:%d]", c.label, i, cs.i, cs.o, want.i, want.o)
			}
		}
	}
}

func TestCursorGotoNext(t *testing.T) {
	cases := []struct {
		label string
		clips []Clip
		nstep int
	}{
		{
			label: "numbers and newlines",
			clips: []Clip{
				DataClip([]byte("1")),
				DataClip([]byte("23\n")),
				DataClip([]byte(" 4\r\n")),
				DataClip([]byte("56 7")),
				DataClip([]byte("8\n\n\n90")),
			},
			nstep: 17,
		},
		{
			label: "korean",
			clips: []Clip{
				DataClip([]byte("한글 테스트\n")),
				DataClip([]byte("english test\n")),
			},
			nstep: 20,
		},
	}
	for _, c := range cases {
		cs := NewCursor(c.clips)
		for i := 0; i < c.nstep; i++ {
			cs.MoveNext()
		}
		if cs.i != len(c.clips) || cs.o != 0 {
			t.Fatalf("%s: got [%d:%d], want [%d:%d]", c.label, cs.i, cs.o, len(c.clips), 0)
		}
	}
}

func TestCursorGotoPrev(t *testing.T) {
	cases := []struct {
		label string
		clips []Clip
		nstep int
	}{
		{
			label: "numbers and newlines",
			clips: []Clip{
				DataClip([]byte("1")),
				DataClip([]byte("23\n")),
				DataClip([]byte(" 4\r\n")),
				DataClip([]byte("56 7")),
				DataClip([]byte("8\n\n\n90")),
			},
			nstep: 17,
		},
		{
			label: "korean",
			clips: []Clip{
				DataClip([]byte("한글 테스트\n")),
				DataClip([]byte("english test\n")),
			},
			nstep: 20,
		},
	}
	for _, c := range cases {
		cs := NewCursor(c.clips)
		cs.i = len(c.clips)
		for i := 0; i < c.nstep; i++ {
			cs.MovePrev()
		}
		if cs.i != 0 || cs.o != 0 {
			t.Fatalf("%s: got [%d:%d], want [%d:%d]", c.label, cs.i, cs.o, 0, 0)
		}
	}
}
<commit_msg>data: Shift is not existing anymore<commit_after>package data

import (
	"reflect"
	"testing"
)

func TestDataClip(t *testing.T) {
	cases := []struct {
		data []byte
		want Clip
	}{
		{
			data: []byte("this is sparta!"),
			want: Clip{data: []byte("this is sparta!"), newlines: []int{}},
		},
		{
			data: []byte("this is sparta!\nright?"),
			want: Clip{data: []byte("this is sparta!\nright?"), newlines: []int{15}},
		},
		{
			data: []byte("\n\n\n\n"),
			want: Clip{data: []byte("\n\n\n\n"), newlines: []int{0, 1, 2, 3}},
		},
	}
	for _, c := range cases {
		got := DataClip(c.data)
		if !reflect.DeepEqual(got, c.want) {
			t.Fatalf("got %q, want %q", got, c.want)
		}
	}
}

func TestCursorWrite(t *testing.T) {
	cases := []struct {
		label  string
		cs     *Cursor
		writes []rune
		at     int
		want   *Cursor
	}{
		{
			label:  "middle",
			cs:     NewCursor([]Clip{DataClip([]byte("this is sparta."))}),
			writes: []rune("n't"),
			at:     7,
			want: &Cursor{
				clips:     []Clip{DataClip([]byte("this is")), DataClip([]byte("n't")), DataClip([]byte(" sparta."))},
				i:         2,
				o:         0,
				appending: true,
			},
		},
		{
			label:  "first",
			cs:     NewCursor([]Clip{DataClip([]byte("this is sparta."))}),
			writes: []rune("hey, "),
			at:     0,
			want: &Cursor{
				clips:     []Clip{DataClip([]byte("hey, ")), DataClip([]byte("this is sparta."))},
				i:         1,
				o:         0,
				appending: true,
			},
		},
		{
			label:  "last",
			cs:     NewCursor([]Clip{DataClip([]byte("this is sparta."))}),
			writes: []rune(" isn't it?"),
			at:     15,
			want: &Cursor{
				clips:     []Clip{DataClip([]byte("this is sparta.")), DataClip([]byte(" isn't it?"))},
				i:         2,
				o:         0,
				appending: true,
			},
		},
	}
	for _, c := range cases {
		defer func() {
			if r := recover(); r != nil {
				t.Fatalf("%q panicked: %s", c.label, r)
			}
		}()
		c.cs.Move(c.at)
		for _, r := range c.writes {
			c.cs.Write(r)
		}
		if !reflect.DeepEqual(c.cs, c.want) {
			t.Fatalf("got %v, want %v", c.cs, c.want)
		}
	}
}

func TestCursorGotoNextLine(t *testing.T) {
	type Step struct {
		i int
		o int
	}
	cases := []struct {
		label string
		clips []Clip
		wants []Step
	}{
		{
			label: "simple",
			clips: []Clip{
				DataClip([]byte("what a nice day\n")),
				DataClip([]byte("do you have breakfast?\n or shall we?")),
			},
			wants: []Step{{0, 15}, {1, 22}, {2, 0}, {2, 0}},
		},
		{
			label: "korean",
			clips: []Clip{
				DataClip([]byte("이 건\n 한글")), DataClip([]byte("테스트 입니다.\n")), // each hangul character is 3 bytes
			},
			wants: []Step{{0, 7}, {1, 20}, {2, 0}, {2, 0}},
		},
	}

	for _, c := range cases {
		cs := NewCursor(c.clips)
		for i, want := range c.wants {
			cs.GotoNextLine()
			if cs.i != want.i || cs.o != want.o {
				t.Fatalf("%s: step %d: got [%d:%d], want [%d:%d]", c.label, i, cs.i, cs.o, want.i, want.o)
			}
		}
	}
}

func TestCursorGotoPrevLine(t *testing.T) {
	type Step struct {
		i int
		o int
	}
	cases := []struct {
		label string
		clips []Clip
		wants []Step
	}{
		{
			label: "simple",
			clips: []Clip{
				DataClip([]byte("what a nice day\n")),
				DataClip([]byte("do you have breakfast?\n or shall we?")),
			},
			wants: []Step{{1, 22}, {0, 15}, {0, 0}, {0, 0}},
		},
		{
			label: "korean",
			clips: []Clip{
				DataClip([]byte("이 건\n 한글")), DataClip([]byte("테스트 입니다.\n")), // each hangul character is 3 bytes
			},
			wants: []Step{{1, 20}, {0, 7}, {0, 0}, {0, 0}},
		},
	}

	for _, c := range cases {
		cs := &Cursor{clips: c.clips, i: len(c.clips), o: 0}
		for i, want := range c.wants {
			cs.GotoPrevLine()
			if cs.i != want.i || cs.o != want.o {
				t.Fatalf("%s: step %d: got [%d:%d], want [%d:%d]", c.label, i, cs.i, cs.o, want.i, want.o)
			}
		}
	}
}

func TestCursorGotoNext(t *testing.T) {
	cases := []struct {
		label string
		clips []Clip
		nstep int
	}{
		{
			label: "numbers and newlines",
			clips: []Clip{
				DataClip([]byte("1")),
				DataClip([]byte("23\n")),
				DataClip([]byte(" 4\r\n")),
				DataClip([]byte("56 7")),
				DataClip([]byte("8\n\n\n90")),
			},
			nstep: 17,
		},
		{
			label: "korean",
			clips: []Clip{
				DataClip([]byte("한글 테스트\n")),
				DataClip([]byte("english test\n")),
			},
			nstep: 20,
		},
	}
	for _, c := range cases {
		cs := NewCursor(c.clips)
		for i := 0; i < c.nstep; i++ {
			cs.MoveNext()
		}
		if cs.i != len(c.clips) || cs.o != 0 {
			t.Fatalf("%s: got [%d:%d], want [%d:%d]", c.label, cs.i, cs.o, len(c.clips), 0)
		}
	}
}

func TestCursorGotoPrev(t *testing.T) {
	cases := []struct {
		label string
		clips []Clip
		nstep int
	}{
		{
			label: "numbers and newlines",
			clips: []Clip{
				DataClip([]byte("1")),
				DataClip([]byte("23\n")),
				DataClip([]byte(" 4\r\n")),
				DataClip([]byte("56 7")),
				DataClip([]byte("8\n\n\n90")),
			},
			nstep: 17,
		},
		{
			label: "korean",
			clips: []Clip{
				DataClip([]byte("한글 테스트\n")),
				DataClip([]byte("english test\n")),
			},
			nstep: 20,
		},
	}
	for _, c := range cases {
		cs := NewCursor(c.clips)
		cs.i = len(c.clips)
		for i := 0; i < c.nstep; i++ {
			cs.MovePrev()
		}
		if cs.i != 0 || cs.o != 0 {
			t.Fatalf("%s: got [%d:%d], want [%d:%d]", c.label, cs.i, cs.o, 0, 0)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package tikv

import (
	"context"

	. "github.com/pingcap/check"
	"github.com/pingcap/errors"
	"github.com/pingcap/failpoint"
	"github.com/pingcap/parser/terror"
	"github.com/pingcap/tidb/kv"
)

// TestFailCommitPrimaryRpcErrors tests rpc errors are handled properly when
// committing primary region task.
func (s *testCommitterSuite) TestFailCommitPrimaryRpcErrors(c *C) {
	c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/mockstore/mocktikv/rpcCommitResult", `return("timeout")`), IsNil)
	defer func() {
		c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/mockstore/mocktikv/rpcCommitResult"), IsNil)
	}()
	// The rpc error will be wrapped to ErrResultUndetermined.
	t1 := s.begin(c)
	err := t1.Set([]byte("a"), []byte("a1"))
	c.Assert(err, IsNil)
	err = t1.Commit(context.Background())
	c.Assert(err, NotNil)
	c.Assert(terror.ErrorEqual(err, terror.ErrResultUndetermined), IsTrue, Commentf("%s", errors.ErrorStack(err)))

	// We don't need to call "Rollback" after "Commit" fails.
	err = t1.Rollback()
	c.Assert(err, Equals, kv.ErrInvalidTxn)
}

// TestFailCommitPrimaryRegionError tests RegionError is handled properly when
// committing primary region task.
func (s *testCommitterSuite) TestFailCommitPrimaryRegionError(c *C) {
	c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/mockstore/mocktikv/rpcCommitResult", `return("notLeader")`), IsNil)
	defer func() {
		c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/mockstore/mocktikv/rpcCommitResult"), IsNil)
	}()
	// Ensure it returns the original error without wrapped to ErrResultUndetermined
	// if it exceeds max retry timeout on RegionError.
	t2 := s.begin(c)
	err := t2.Set([]byte("b"), []byte("b1"))
	c.Assert(err, IsNil)
	err = t2.Commit(context.Background())
	c.Assert(err, NotNil)
	c.Assert(terror.ErrorNotEqual(err, terror.ErrResultUndetermined), IsTrue)
}

// TestFailCommitPrimaryRPCErrorThenRegionError tests the case when commit first
// receive a rpc timeout, then region errors afterwrards.
func (s *testCommitterSuite) TestFailCommitPrimaryRPCErrorThenRegionError(c *C) {
	c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/mockstore/mocktikv/rpcCommitResult", `1*return("timeout")->return("notLeader")`), IsNil)
	defer func() {
		c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/mockstore/mocktikv/rpcCommitResult"), IsNil)
	}()
	// The region error will be wrapped to ErrResultUndetermined.
	t1 := s.begin(c)
	err := t1.Set([]byte("a"), []byte("a1"))
	c.Assert(err, IsNil)
	err = t1.Commit(context.Background())
	c.Assert(err, NotNil)
	c.Assert(terror.ErrorEqual(err, terror.ErrResultUndetermined), IsTrue, Commentf("%s", errors.ErrorStack(err)))
}

// TestFailCommitPrimaryKeyError tests KeyError is handled properly when
// committing primary region task.
func (s *testCommitterSuite) TestFailCommitPrimaryKeyError(c *C) {
	c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/mockstore/mocktikv/rpcCommitResult", `return("keyError")`), IsNil)
	defer func() {
		c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/mockstore/mocktikv/rpcCommitResult"), IsNil)
	}()
	// Ensure it returns the original error without wrapped to ErrResultUndetermined
	// if it meets KeyError.
	t3 := s.begin(c)
	err := t3.Set([]byte("c"), []byte("c1"))
	c.Assert(err, IsNil)
	err = t3.Commit(context.Background())
	c.Assert(err, NotNil)
	c.Assert(terror.ErrorNotEqual(err, terror.ErrResultUndetermined), IsTrue)
}

func (s *testCommitterSuite) TestFailCommitTimeout(c *C) {
	c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/mockstore/mocktikv/rpcCommitTimeout", `return(true)`), IsNil)
	defer func() {
		c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/mockstore/mocktikv/rpcCommitTimeout"), IsNil)
	}()
	txn := s.begin(c)
	err := txn.Set([]byte("a"), []byte("a1"))
	c.Assert(err, IsNil)
	err = txn.Set([]byte("b"), []byte("b1"))
	c.Assert(err, IsNil)
	err = txn.Set([]byte("c"), []byte("c1"))
	c.Assert(err, IsNil)
	err = txn.Commit(context.Background())
	c.Assert(err, NotNil)

	txn2 := s.begin(c)
	value, err := txn2.Get(context.TODO(), []byte("a"))
	c.Assert(err, IsNil)
	c.Assert(len(value), Greater, 0)
	_, err = txn2.Get(context.TODO(), []byte("b"))
	c.Assert(err, IsNil)
	c.Assert(len(value), Greater, 0)
}

// TestFailPrewriteRegionError tests data race does not happen on retries
func (s *testCommitterSuite) TestFailPrewriteRegionError(c *C) {
	c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/mockstore/mocktikv/rpcPrewriteResult", `return("notLeader")`), IsNil)
	defer func() {
		c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/mockstore/mocktikv/rpcPrewriteResult"), IsNil)
	}()

	txn := s.begin(c)

	// Set the value big enough to create many batches. This increases the chance of data races.
	var bigVal [18000]byte
	for i := 0; i < 1000; i++ {
		err := txn.Set([]byte{byte(i)}, bigVal[:])
		c.Assert(err, IsNil)
	}

	committer, err := newTwoPhaseCommitterWithInit(txn, 1)
	c.Assert(err, IsNil)

	ctx := context.Background()
	err = committer.prewriteMutations(NewBackofferWithVars(ctx, 1000, nil), committer.mutations)
	c.Assert(err, NotNil)
}
<commit_msg>store/tikv: skip a test case to make race test more stable (#19256)<commit_after>// Copyright 2017 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package tikv

import (
	"context"

	. "github.com/pingcap/check"
	"github.com/pingcap/errors"
	"github.com/pingcap/failpoint"
	"github.com/pingcap/parser/terror"
	"github.com/pingcap/tidb/kv"
	"github.com/pingcap/tidb/util/israce"
)

// TestFailCommitPrimaryRpcErrors tests rpc errors are handled properly when
// committing primary region task.
func (s *testCommitterSuite) TestFailCommitPrimaryRpcErrors(c *C) {
	c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/mockstore/mocktikv/rpcCommitResult", `return("timeout")`), IsNil)
	defer func() {
		c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/mockstore/mocktikv/rpcCommitResult"), IsNil)
	}()
	// The rpc error will be wrapped to ErrResultUndetermined.
	t1 := s.begin(c)
	err := t1.Set([]byte("a"), []byte("a1"))
	c.Assert(err, IsNil)
	err = t1.Commit(context.Background())
	c.Assert(err, NotNil)
	c.Assert(terror.ErrorEqual(err, terror.ErrResultUndetermined), IsTrue, Commentf("%s", errors.ErrorStack(err)))

	// We don't need to call "Rollback" after "Commit" fails.
	err = t1.Rollback()
	c.Assert(err, Equals, kv.ErrInvalidTxn)
}

// TestFailCommitPrimaryRegionError tests RegionError is handled properly when
// committing primary region task.
func (s *testCommitterSuite) TestFailCommitPrimaryRegionError(c *C) {
	c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/mockstore/mocktikv/rpcCommitResult", `return("notLeader")`), IsNil)
	defer func() {
		c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/mockstore/mocktikv/rpcCommitResult"), IsNil)
	}()
	// Ensure it returns the original error without wrapped to ErrResultUndetermined
	// if it exceeds max retry timeout on RegionError.
	t2 := s.begin(c)
	err := t2.Set([]byte("b"), []byte("b1"))
	c.Assert(err, IsNil)
	err = t2.Commit(context.Background())
	c.Assert(err, NotNil)
	c.Assert(terror.ErrorNotEqual(err, terror.ErrResultUndetermined), IsTrue)
}

// TestFailCommitPrimaryRPCErrorThenRegionError tests the case when commit first
// receive a rpc timeout, then region errors afterwrards.
func (s *testCommitterSuite) TestFailCommitPrimaryRPCErrorThenRegionError(c *C) {
	c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/mockstore/mocktikv/rpcCommitResult", `1*return("timeout")->return("notLeader")`), IsNil)
	defer func() {
		c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/mockstore/mocktikv/rpcCommitResult"), IsNil)
	}()
	// The region error will be wrapped to ErrResultUndetermined.
	t1 := s.begin(c)
	err := t1.Set([]byte("a"), []byte("a1"))
	c.Assert(err, IsNil)
	err = t1.Commit(context.Background())
	c.Assert(err, NotNil)
	c.Assert(terror.ErrorEqual(err, terror.ErrResultUndetermined), IsTrue, Commentf("%s", errors.ErrorStack(err)))
}

// TestFailCommitPrimaryKeyError tests KeyError is handled properly when
// committing primary region task.
func (s *testCommitterSuite) TestFailCommitPrimaryKeyError(c *C) {
	c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/mockstore/mocktikv/rpcCommitResult", `return("keyError")`), IsNil)
	defer func() {
		c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/mockstore/mocktikv/rpcCommitResult"), IsNil)
	}()
	// Ensure it returns the original error without wrapped to ErrResultUndetermined
	// if it meets KeyError.
	t3 := s.begin(c)
	err := t3.Set([]byte("c"), []byte("c1"))
	c.Assert(err, IsNil)
	err = t3.Commit(context.Background())
	c.Assert(err, NotNil)
	c.Assert(terror.ErrorNotEqual(err, terror.ErrResultUndetermined), IsTrue)
}

func (s *testCommitterSuite) TestFailCommitTimeout(c *C) {
	c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/mockstore/mocktikv/rpcCommitTimeout", `return(true)`), IsNil)
	defer func() {
		c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/mockstore/mocktikv/rpcCommitTimeout"), IsNil)
	}()
	txn := s.begin(c)
	err := txn.Set([]byte("a"), []byte("a1"))
	c.Assert(err, IsNil)
	err = txn.Set([]byte("b"), []byte("b1"))
	c.Assert(err, IsNil)
	err = txn.Set([]byte("c"), []byte("c1"))
	c.Assert(err, IsNil)
	err = txn.Commit(context.Background())
	c.Assert(err, NotNil)

	txn2 := s.begin(c)
	value, err := txn2.Get(context.TODO(), []byte("a"))
	c.Assert(err, IsNil)
	c.Assert(len(value), Greater, 0)
	_, err = txn2.Get(context.TODO(), []byte("b"))
	c.Assert(err, IsNil)
	c.Assert(len(value), Greater, 0)
}

// TestFailPrewriteRegionError tests data race does not happen on retries
func (s *testCommitterSuite) TestFailPrewriteRegionError(c *C) {
	if israce.RaceEnabled {
		c.Skip("skip race test")
	}
	c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/mockstore/mocktikv/rpcPrewriteResult", `return("notLeader")`), IsNil)
	defer func() {
		c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/mockstore/mocktikv/rpcPrewriteResult"), IsNil)
	}()

	txn := s.begin(c)

	// Set the value big enough to create many batches. This increases the chance of data races.
	var bigVal [18000]byte
	for i := 0; i < 1000; i++ {
		err := txn.Set([]byte{byte(i)}, bigVal[:])
		c.Assert(err, IsNil)
	}

	committer, err := newTwoPhaseCommitterWithInit(txn, 1)
	c.Assert(err, IsNil)

	ctx := context.Background()
	err = committer.prewriteMutations(NewBackofferWithVars(ctx, 1000, nil), committer.mutations)
	c.Assert(err, NotNil)
}
<|endoftext|>


<commit_before>// Package ppclient enables low-overhead performance profiling and sends them
// to a central repository.
//
// The profiler may skip profiling cycles if it's not given any CPU cycles,
// which can only happen if the scheduler finds spare cycles or if one of the
// busy gouroutines yields control often enough. If your program is constantly
// using the threads it's allowed to (via GOMAXPROCS), the profiler may not
// have the opportunity to run. To mitigate that, try adding runtime.<yield>
// in your main loop(s).
//
// - Count the number of missed profiling collections and report them at
// least. Alternatively, let the CPU profiling be on all the time with low
// frequency and hope that profiling works despite the scheduler issues
// (assuming it relies on OS signals).
//
// - Find the cost of CPU profiling at the currently used 100hz.
package ppclient

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"runtime/pprof"
	"strings"
	"time"

	"github.com/nictuku/webpprof/ppcommon"
)

const PrrrURL = "http://localhost:8080/write"

var (
	CPUProfilingInterval = 30 * time.Second
	CPUProfilingDuration = 5 * time.Second
	Profiles             = []string{"heap", "block", "goroutine", "threadcreate"}
)

func createProfile(name string) *ppcommon.Profile {
	buf := new(bytes.Buffer)
	p := pprof.Lookup(name)
	if p == nil {
		return nil
	}
	if err := p.WriteTo(buf, 1); err != nil {
		return nil
	}
	fmt.Println("profile data", buf.String())
	return &ppcommon.Profile{"", name, buf.Bytes(), time.Now()}
}

func profileURL(name string) string {
	u := *prrrURL
	q := u.Query()
	q.Set("p", name)
	u.RawQuery = q.Encode()
	return u.String()
}

func sendProfile(p *ppcommon.Profile) error {
	// TODO: Authentication.
	// TODO: identify the program name and arguments.
	buf := new(bytes.Buffer)
	enc := json.NewEncoder(buf)
	err := enc.Encode(p)
	if err != nil {
		log.Printf("sendPr enc error %v", err)
		return err
	}
	req, err := http.NewRequest("POST", profileURL(p.Name), buf)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		log.Printf("sendProfile %v error reading response body: %v", p.Name, err)
		return err
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Printf("sendProfile %v error reading response body: %v", p.Name, err)
		return err
	}
	resp.Body.Close()
	log.Println("sendProfile got", string(body))
	return nil
}

// XXX rewrite stuff copied from net/http/pprof.
// better docs.

// Cmdline responds with the running program's
// command line, with arguments separated by NUL bytes.
func cmdline() string {
	return strings.Join(os.Args, "\x00")
}

func sendCPUProfile(buf *bytes.Buffer) {
	// Send the hpprof to the remote server.

	// TODO: Authentication.

	// TODO: identify the program name and arguments.
	fmt.Printf("%v prof result\n", string(buf.String()))
}

// cpuProfile enables CPU profiling for the provided duration and writes the
// resulting pprof-formatted report to w. As of Go 1.1, the CPU profiler
// collects counters of the stack positions at 100hz frequency. An error will
// be reported if, among other reasons, CPU profiling is already enabled when
// this function is called.
func cpuProfile(w io.Writer, duration time.Duration) error {
	if duration.Seconds() == 0 {
		duration = 30 * time.Second
	}

	if err := pprof.StartCPUProfile(w); err != nil {
		return err
	}
	time.Sleep(duration)
	pprof.StopCPUProfile() // Also finishes the write to w.
	return nil
}

func send() {
	for _, name := range Profiles {
		fmt.Println(name)
		if name == "cpu" {
			buf := new(bytes.Buffer)
			if err := cpuProfile(buf, CPUProfilingDuration); err != nil {
				log.Printf("ppclient cpuProfile error: %v", err)
			} else {
				sendCPUProfile(buf)
			}
		} else {
			fmt.Println("creating")
			p := createProfile(name)
			fmt.Println("created")
			if p == nil {
				log.Printf("createProfile %v returned an empty profile", name)
				continue
			}
			fmt.Println("sending")
			sendProfile(p)
		}
	}
}

// profiler runs in the background, waking up occasionally to collect
// performance profiles and send them to prrr.
func profiler() {
	log.Println("profiler started")
	tick := time.Tick(CPUProfilingInterval)
	for {
		send()
		<-tick

	}
}

var prrrURL *url.URL

func Start() (err error) {
	prrrURL, err = url.Parse(PrrrURL)
	if err != nil {
		return err
	}

	go profiler()
	return nil
}
<commit_msg>send -> uploadProfiles<commit_after>// Package ppclient enables low-overhead performance profiling and sends them
// to a central repository.
//
// The profiler may skip profiling cycles if it's not given any CPU cycles,
// which can only happen if the scheduler finds spare cycles or if one of the
// busy gouroutines yields control often enough. If your program is constantly
// using the threads it's allowed to (via GOMAXPROCS), the profiler may not
// have the opportunity to run. To mitigate that, try adding runtime.<yield>
// in your main loop(s).
//
// - Count the number of missed profiling collections and report them at
// least. Alternatively, let the CPU profiling be on all the time with low
// frequency and hope that profiling works despite the scheduler issues
// (assuming it relies on OS signals).
//
// - Find the cost of CPU profiling at the currently used 100hz.
package ppclient

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"runtime/pprof"
	"strings"
	"time"

	"github.com/nictuku/webpprof/ppcommon"
)

const PrrrURL = "http://localhost:8080/write"

var (
	CPUProfilingInterval = 30 * time.Second
	CPUProfilingDuration = 5 * time.Second
	Profiles             = []string{"heap", "block", "goroutine", "threadcreate"}
)

func createProfile(name string) *ppcommon.Profile {
	buf := new(bytes.Buffer)
	p := pprof.Lookup(name)
	if p == nil {
		return nil
	}
	if err := p.WriteTo(buf, 1); err != nil {
		return nil
	}
	fmt.Println("profile data", buf.String())
	return &ppcommon.Profile{"", name, buf.Bytes(), time.Now()}
}

func profileURL(name string) string {
	u := *prrrURL
	q := u.Query()
	q.Set("p", name)
	u.RawQuery = q.Encode()
	return u.String()
}

func sendProfile(p *ppcommon.Profile) error {
	// TODO: Authentication.
	// TODO: identify the program name and arguments.
	buf := new(bytes.Buffer)
	enc := json.NewEncoder(buf)
	err := enc.Encode(p)
	if err != nil {
		log.Printf("sendPr enc error %v", err)
		return err
	}
	req, err := http.NewRequest("POST", profileURL(p.Name), buf)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		log.Printf("sendProfile %v error reading response body: %v", p.Name, err)
		return err
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Printf("sendProfile %v error reading response body: %v", p.Name, err)
		return err
	}
	resp.Body.Close()
	log.Println("sendProfile got", string(body))
	return nil
}

// XXX rewrite stuff copied from net/http/pprof.
// better docs.

// Cmdline responds with the running program's
// command line, with arguments separated by NUL bytes.
func cmdline() string {
	return strings.Join(os.Args, "\x00")
}

func sendCPUProfile(buf *bytes.Buffer) {
	// Send the hpprof to the remote server.

	// TODO: Authentication.

	// TODO: identify the program name and arguments.
	fmt.Printf("%v prof result\n", string(buf.String()))
}

// cpuProfile enables CPU profiling for the provided duration and writes the
// resulting pprof-formatted report to w. As of Go 1.1, the CPU profiler
// collects counters of the stack positions at 100hz frequency. An error will
// be reported if, among other reasons, CPU profiling is already enabled when
// this function is called.
func cpuProfile(w io.Writer, duration time.Duration) error {
	if duration.Seconds() == 0 {
		duration = 30 * time.Second
	}

	if err := pprof.StartCPUProfile(w); err != nil {
		return err
	}
	time.Sleep(duration)
	pprof.StopCPUProfile() // Also finishes the write to w.
	return nil
}

func uploadProfiles() {
	for _, name := range Profiles {
		fmt.Println(name)
		if name == "cpu" {
			buf := new(bytes.Buffer)
			if err := cpuProfile(buf, CPUProfilingDuration); err != nil {
				log.Printf("ppclient cpuProfile error: %v", err)
			} else {
				// TODO: Actually send it.
				sendCPUProfile(buf)
			}
		} else {
			fmt.Println("creating")
			p := createProfile(name)
			fmt.Println("created")
			if p == nil {
				log.Printf("createProfile %v returned an empty profile", name)
				continue
			}
			fmt.Println("sending")
			sendProfile(p)
		}
	}
}

// profiler runs in the background, waking up occasionally to collect
// performance profiles and send them to prrr.
func profiler() {
	log.Println("profiler started")
	tick := time.Tick(CPUProfilingInterval)
	for {
		uploadProfiles()
		<-tick

	}
}

var prrrURL *url.URL

func Start() (err error) {
	prrrURL, err = url.Parse(PrrrURL)
	if err != nil {
		return err
	}

	go profiler()
	return nil
}
<|endoftext|>