<commit_before>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package analyzer

import (
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/signal"
	"sync"
	"sync/atomic"
	"syscall"
	"time"

	"github.com/containerd/console"
	"github.com/containerd/containerd"
	"github.com/containerd/containerd/cio"
	"github.com/containerd/containerd/cmd/ctr/commands"
	"github.com/containerd/containerd/cmd/ctr/commands/tasks"
	"github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/log"
	"github.com/containerd/containerd/mount"
	"github.com/containerd/containerd/oci"
	"github.com/containerd/containerd/platforms"
	"github.com/containerd/containerd/snapshots"
	"github.com/containerd/stargz-snapshotter/analyzer/fanotify"
	"github.com/containerd/stargz-snapshotter/analyzer/recorder"
	"github.com/opencontainers/go-digest"
	"github.com/opencontainers/image-spec/identity"
	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
	"github.com/pkg/errors"
	"github.com/rs/xid"
)

var defaultPeriod = 10 * time.Second

// Analyze analyzes the passed image then store the record of prioritized files into
// containerd's content store. This function returns the digest of that record file.
// This digest can be used to read record from the content store.
func Analyze(ctx context.Context, client *containerd.Client, ref string, opts ...Option) (digest.Digest, error) {
	var aOpts analyzerOpts
	for _, o := range opts {
		o(&aOpts)
	}
	if aOpts.terminal && aOpts.waitOnSignal {
		return "", fmt.Errorf("wait-on-signal option cannot be used with terminal option")
	}

	target, err := ioutil.TempDir("", "target")
	if err != nil {
		return "", err
	}
	defer os.RemoveAll(target)

	cs := client.ContentStore()
	is := client.ImageService()
	ss := client.SnapshotService(aOpts.snapshotter)

	img, err := is.Get(ctx, ref)
	if err != nil {
		return "", err
	}
	platformImg := containerd.NewImageWithPlatform(client, img, platforms.Default())

	// Mount the target image
	// NOTE: We cannot let containerd prepare the rootfs. We create mount namespace *before*
	//       creating the container so containerd's bundle preparation (mounting snapshots to
	//       the bundle directory, etc.) is invisible inside the pre-unshared mount namespace.
	//       This leads to runc using empty directory as the rootfs.
	if unpacked, err := platformImg.IsUnpacked(ctx, aOpts.snapshotter); err != nil {
		return "", err
	} else if !unpacked {
		if err := platformImg.Unpack(ctx, aOpts.snapshotter); err != nil {
			return "", err
		}
	}
	cleanup, err := mountImage(ctx, ss, platformImg, target)
	if err != nil {
		return "", err
	}
	defer cleanup()

	// Spawn a fanotifier process in a new mount namespace and setup recorder.
	fanotifier, err := fanotify.SpawnFanotifier("/proc/self/exe")
	if err != nil {
		return "", errors.Wrapf(err, "failed to spawn fanotifier")
	}
	defer func() {
		if err := fanotifier.Close(); err != nil {
			log.G(ctx).WithError(err).Warnf("failed to close fanotifier")
		}
	}()

	// Prepare the spec based on the specified image and runtime options.
	var sOpts []oci.SpecOpts
	if aOpts.specOpts != nil {
		gotOpts, done, err := aOpts.specOpts(platformImg, target)
		if err != nil {
			return "", err
		}
		defer func() {
			if err := done(); err != nil {
				log.G(ctx).WithError(err).Warnf("failed to cleanup container")
				return
			}
		}()
		sOpts = append(sOpts, gotOpts...)
	} else {
		sOpts = append(sOpts,
			oci.WithDefaultSpec(),
			oci.WithDefaultUnixDevices,
			oci.WithRootFSPath(target),
			oci.WithImageConfig(platformImg),
		)
	}
	sOpts = append(sOpts, oci.WithLinuxNamespace(runtimespec.LinuxNamespace{
		Type: runtimespec.MountNamespace,
		Path: fanotifier.MountNamespacePath(), // use mount namespace that the fanotifier created
	}))

	// Create the container and the task
	var container containerd.Container
	for i := 0; i < 3; i++ {
		id := xid.New().String()
		var s runtimespec.Spec
		container, err = client.NewContainer(ctx, id,
			containerd.WithImage(platformImg),
			containerd.WithSnapshotter(aOpts.snapshotter),
			containerd.WithImageStopSignal(platformImg, "SIGKILL"),

			// WithImageConfig depends on WithImage and WithSnapshotter for resolving
			// username (accesses to /etc/{passwd,group} files on the rootfs)
			containerd.WithSpec(&s, sOpts...),
		)
		if err != nil {
			if errdefs.IsAlreadyExists(err) {
				log.G(ctx).WithError(err).Warnf("failed to create container")
				continue
			}
			return "", err
		}
		break
	}
	if container == nil {
		return "", fmt.Errorf("failed to create container")
	}
	defer container.Delete(ctx, containerd.WithSnapshotCleanup)
	var ioCreator cio.Creator
	var con console.Console
	stdinC := newLazyReadCloser(os.Stdin)
	if aOpts.terminal {
		if !aOpts.stdin {
			return "", fmt.Errorf("terminal cannot be used if stdin isn't enabled")
		}
		con = console.Current()
		defer con.Reset()
		if err := con.SetRaw(); err != nil {
			return "", err
		}
		// On terminal mode, the "stderr" field is unused.
		ioCreator = cio.NewCreator(cio.WithStreams(con, con, nil), cio.WithTerminal)
	} else if aOpts.stdin {
		ioCreator = cio.NewCreator(cio.WithStreams(stdinC, os.Stdout, os.Stderr))
	} else {
		ioCreator = cio.NewCreator(cio.WithStreams(nil, os.Stdout, os.Stderr))
	}
	task, err := container.NewTask(ctx, ioCreator)
	if err != nil {
		return "", err
	}
	stdinC.registerCloser(func() { // Ensure to close IO when stdin get EOF
		task.CloseIO(ctx, containerd.WithStdinCloser)
	})

	// Start to monitor "/" and run the task.
	rc, err := recorder.NewImageRecorder(ctx, cs, img, platforms.Default())
	if err != nil {
		return "", err
	}
	defer rc.Close()
	if err := fanotifier.Start(); err != nil {
		return "", errors.Wrapf(err, "failed to start fanotifier")
	}
	var fanotifierClosed bool
	var fanotifierClosedMu sync.Mutex
	go func() {
		for {
			path, err := fanotifier.GetPath()
			if err != nil {
				if err == io.EOF {
					fanotifierClosedMu.Lock()
					isFanotifierClosed := fanotifierClosed
					fanotifierClosedMu.Unlock()
					if isFanotifierClosed {
						break
					}
				}
				log.G(ctx).WithError(err).Error("failed to get notified path")
				break
			}
			if err := rc.Record(path); err != nil {
				log.G(ctx).WithError(err).Debugf("failed to record %q", path)
			}
		}
	}()
	if aOpts.terminal {
		if err := tasks.HandleConsoleResize(ctx, task, con); err != nil {
			log.G(ctx).WithError(err).Error("failed to resize console")
		}
	} else {
		sigc := commands.ForwardAllSignals(ctx, task)
		defer commands.StopCatch(sigc)
	}
	if err := task.Start(ctx); err != nil {
		return "", err
	}

	// Wait until the task exit
	var status containerd.ExitStatus
	var killOk bool
	if aOpts.waitOnSignal { // NOTE: not functional with `terminal` option
		log.G(ctx).Infof("press Ctrl+C to terminate the container")
		status, killOk, err = waitOnSignal(ctx, container, task)
		if err != nil {
			return "", err
		}
	} else {
		if aOpts.period <= 0 {
			aOpts.period = defaultPeriod
		}
		log.G(ctx).Infof("waiting for %v ...", aOpts.period)
		status, killOk, err = waitOnTimeout(ctx, container, task, aOpts.period)
		if err != nil {
			return "", err
		}
	}
	if !killOk {
		log.G(ctx).Warnf("failed to exit task %v; manually kill it", task.ID())
	} else {
		code, _, err := status.Result()
		if err != nil {
			return "", err
		}
		log.G(ctx).Infof("container exit with code %v", code)
		if _, err := task.Delete(ctx); err != nil {
			return "", err
		}
	}

	// ensure no record comes in
	fanotifierClosedMu.Lock()
	fanotifierClosed = true
	fanotifierClosedMu.Unlock()
	if err := fanotifier.Close(); err != nil {
		log.G(ctx).WithError(err).Warnf("failed to cleanup fanotifier")
	}

	// Finish recording
	return rc.Commit(ctx)
}

func mountImage(ctx context.Context, ss snapshots.Snapshotter, image containerd.Image, mountpoint string) (func(), error) {
	diffIDs, err := image.RootFS(ctx)
	if err != nil {
		return nil, err
	}
	mounts, err := ss.Prepare(ctx, mountpoint, identity.ChainID(diffIDs).String())
	if err != nil {
		return nil, err
	}
	if err := mount.All(mounts, mountpoint); err != nil {
		if err := ss.Remove(ctx, mountpoint); err != nil && !errdefs.IsNotFound(err) {
			log.G(ctx).WithError(err).Warnf("failed to cleanup snapshot after mount error")
		}
		return nil, errors.Wrapf(err, "failed to mount rootfs at %q", mountpoint)
	}
	return func() {
		if err := mount.UnmountAll(mountpoint, 0); err != nil {
			log.G(ctx).WithError(err).Warnf("failed to unmount snapshot")
		}
		if err := ss.Remove(ctx, mountpoint); err != nil && !errdefs.IsNotFound(err) {
			log.G(ctx).WithError(err).Warnf("failed to cleanup snapshot")
		}
	}, nil
}

func waitOnSignal(ctx context.Context, container containerd.Container, task containerd.Task) (containerd.ExitStatus, bool, error) {
	statusC, err := task.Wait(ctx)
	if err != nil {
		return containerd.ExitStatus{}, false, err
	}
	sc := make(chan os.Signal, 1)
	signal.Notify(sc, syscall.SIGINT)
	defer signal.Stop(sc)
	select {
	case status := <-statusC:
		return status, true, nil
	case <-sc:
		log.G(ctx).Info("signal detected")
		status, err := killTask(ctx, container, task, statusC)
		if err != nil {
			log.G(ctx).WithError(err).Warnf("failed to kill container")
			return containerd.ExitStatus{}, false, nil
		}
		return status, true, nil
	}
}

func waitOnTimeout(ctx context.Context, container containerd.Container, task containerd.Task, period time.Duration) (containerd.ExitStatus, bool, error) {
	statusC, err := task.Wait(ctx)
	if err != nil {
		return containerd.ExitStatus{}, false, err
	}
	select {
	case status := <-statusC:
		return status, true, nil
	case <-time.After(period):
		log.G(ctx).Warnf("killing task. the time period to monitor access log (%s) has timed out", period.String())
		status, err := killTask(ctx, container, task, statusC)
		if err != nil {
			log.G(ctx).WithError(err).Warnf("failed to kill container")
			return containerd.ExitStatus{}, false, nil
		}
		return status, true, nil
	}
}

func killTask(ctx context.Context, container containerd.Container, task containerd.Task, statusC <-chan containerd.ExitStatus) (containerd.ExitStatus, error) {
	sig, err := containerd.GetStopSignal(ctx, container, syscall.SIGKILL)
	if err != nil {
		return containerd.ExitStatus{}, err
	}
	if err := task.Kill(ctx, sig, containerd.WithKillAll); err != nil && !errdefs.IsNotFound(err) {
		return containerd.ExitStatus{}, errors.Wrapf(err, "forward SIGKILL")
	}
	select {
	case status := <-statusC:
		return status, nil
	case <-time.After(5 * time.Second):
		return containerd.ExitStatus{}, fmt.Errorf("timeout")
	}
}

type lazyReadCloser struct {
	reader      io.Reader
	closer      func()
	closerMu    sync.Mutex
	initCond    *sync.Cond
	initialized int64
}

func newLazyReadCloser(r io.Reader) *lazyReadCloser {
	rc := &lazyReadCloser{reader: r, initCond: sync.NewCond(&sync.Mutex{})}
	return rc
}

func (s *lazyReadCloser) registerCloser(closer func()) {
	s.closerMu.Lock()
	s.closer = closer
	s.closerMu.Unlock()
	atomic.AddInt64(&s.initialized, 1)
	s.initCond.Broadcast()
}

func (s *lazyReadCloser) Read(p []byte) (int, error) {
	if atomic.LoadInt64(&s.initialized) <= 0 {
		// wait until initialized
		s.initCond.L.Lock()
		if atomic.LoadInt64(&s.initialized) <= 0 {
			s.initCond.Wait()
		}
		s.initCond.L.Unlock()
	}

	n, err := s.reader.Read(p)
	if err == io.EOF {
		s.closerMu.Lock()
		s.closer()
		s.closerMu.Unlock()
	}
	return n, err
}
<commit_msg>feat: log success record path count<commit_after>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package analyzer

import (
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/signal"
	"sync"
	"sync/atomic"
	"syscall"
	"time"

	"github.com/containerd/console"
	"github.com/containerd/containerd"
	"github.com/containerd/containerd/cio"
	"github.com/containerd/containerd/cmd/ctr/commands"
	"github.com/containerd/containerd/cmd/ctr/commands/tasks"
	"github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/log"
	"github.com/containerd/containerd/mount"
	"github.com/containerd/containerd/oci"
	"github.com/containerd/containerd/platforms"
	"github.com/containerd/containerd/snapshots"
	"github.com/containerd/stargz-snapshotter/analyzer/fanotify"
	"github.com/containerd/stargz-snapshotter/analyzer/recorder"
	"github.com/opencontainers/go-digest"
	"github.com/opencontainers/image-spec/identity"
	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
	"github.com/pkg/errors"
	"github.com/rs/xid"
)

var defaultPeriod = 10 * time.Second

// Analyze analyzes the passed image then store the record of prioritized files into
// containerd's content store. This function returns the digest of that record file.
// This digest can be used to read record from the content store.
func Analyze(ctx context.Context, client *containerd.Client, ref string, opts ...Option) (digest.Digest, error) {
	var aOpts analyzerOpts
	for _, o := range opts {
		o(&aOpts)
	}
	if aOpts.terminal && aOpts.waitOnSignal {
		return "", fmt.Errorf("wait-on-signal option cannot be used with terminal option")
	}

	target, err := ioutil.TempDir("", "target")
	if err != nil {
		return "", err
	}
	defer os.RemoveAll(target)

	cs := client.ContentStore()
	is := client.ImageService()
	ss := client.SnapshotService(aOpts.snapshotter)

	img, err := is.Get(ctx, ref)
	if err != nil {
		return "", err
	}
	platformImg := containerd.NewImageWithPlatform(client, img, platforms.Default())

	// Mount the target image
	// NOTE: We cannot let containerd prepare the rootfs. We create mount namespace *before*
	//       creating the container so containerd's bundle preparation (mounting snapshots to
	//       the bundle directory, etc.) is invisible inside the pre-unshared mount namespace.
	//       This leads to runc using empty directory as the rootfs.
	if unpacked, err := platformImg.IsUnpacked(ctx, aOpts.snapshotter); err != nil {
		return "", err
	} else if !unpacked {
		if err := platformImg.Unpack(ctx, aOpts.snapshotter); err != nil {
			return "", err
		}
	}
	cleanup, err := mountImage(ctx, ss, platformImg, target)
	if err != nil {
		return "", err
	}
	defer cleanup()

	// Spawn a fanotifier process in a new mount namespace and setup recorder.
	fanotifier, err := fanotify.SpawnFanotifier("/proc/self/exe")
	if err != nil {
		return "", errors.Wrapf(err, "failed to spawn fanotifier")
	}
	defer func() {
		if err := fanotifier.Close(); err != nil {
			log.G(ctx).WithError(err).Warnf("failed to close fanotifier")
		}
	}()

	// Prepare the spec based on the specified image and runtime options.
	var sOpts []oci.SpecOpts
	if aOpts.specOpts != nil {
		gotOpts, done, err := aOpts.specOpts(platformImg, target)
		if err != nil {
			return "", err
		}
		defer func() {
			if err := done(); err != nil {
				log.G(ctx).WithError(err).Warnf("failed to cleanup container")
				return
			}
		}()
		sOpts = append(sOpts, gotOpts...)
	} else {
		sOpts = append(sOpts,
			oci.WithDefaultSpec(),
			oci.WithDefaultUnixDevices,
			oci.WithRootFSPath(target),
			oci.WithImageConfig(platformImg),
		)
	}
	sOpts = append(sOpts, oci.WithLinuxNamespace(runtimespec.LinuxNamespace{
		Type: runtimespec.MountNamespace,
		Path: fanotifier.MountNamespacePath(), // use mount namespace that the fanotifier created
	}))

	// Create the container and the task
	var container containerd.Container
	for i := 0; i < 3; i++ {
		id := xid.New().String()
		var s runtimespec.Spec
		container, err = client.NewContainer(ctx, id,
			containerd.WithImage(platformImg),
			containerd.WithSnapshotter(aOpts.snapshotter),
			containerd.WithImageStopSignal(platformImg, "SIGKILL"),

			// WithImageConfig depends on WithImage and WithSnapshotter for resolving
			// username (accesses to /etc/{passwd,group} files on the rootfs)
			containerd.WithSpec(&s, sOpts...),
		)
		if err != nil {
			if errdefs.IsAlreadyExists(err) {
				log.G(ctx).WithError(err).Warnf("failed to create container")
				continue
			}
			return "", err
		}
		break
	}
	if container == nil {
		return "", fmt.Errorf("failed to create container")
	}
	defer container.Delete(ctx, containerd.WithSnapshotCleanup)
	var ioCreator cio.Creator
	var con console.Console
	stdinC := newLazyReadCloser(os.Stdin)
	if aOpts.terminal {
		if !aOpts.stdin {
			return "", fmt.Errorf("terminal cannot be used if stdin isn't enabled")
		}
		con = console.Current()
		defer con.Reset()
		if err := con.SetRaw(); err != nil {
			return "", err
		}
		// On terminal mode, the "stderr" field is unused.
		ioCreator = cio.NewCreator(cio.WithStreams(con, con, nil), cio.WithTerminal)
	} else if aOpts.stdin {
		ioCreator = cio.NewCreator(cio.WithStreams(stdinC, os.Stdout, os.Stderr))
	} else {
		ioCreator = cio.NewCreator(cio.WithStreams(nil, os.Stdout, os.Stderr))
	}
	task, err := container.NewTask(ctx, ioCreator)
	if err != nil {
		return "", err
	}
	stdinC.registerCloser(func() { // Ensure to close IO when stdin get EOF
		task.CloseIO(ctx, containerd.WithStdinCloser)
	})

	// Start to monitor "/" and run the task.
	rc, err := recorder.NewImageRecorder(ctx, cs, img, platforms.Default())
	if err != nil {
		return "", err
	}
	defer rc.Close()
	if err := fanotifier.Start(); err != nil {
		return "", errors.Wrapf(err, "failed to start fanotifier")
	}
	var fanotifierClosed bool
	var fanotifierClosedMu sync.Mutex
	go func() {
		var successCount int
		defer func() {
			log.G(ctx).Debugf("success record %d path", successCount)
		}()
		for {
			path, err := fanotifier.GetPath()
			if err != nil {
				if err == io.EOF {
					fanotifierClosedMu.Lock()
					isFanotifierClosed := fanotifierClosed
					fanotifierClosedMu.Unlock()
					if isFanotifierClosed {
						break
					}
				}
				log.G(ctx).WithError(err).Error("failed to get notified path")
				break
			}
			if err := rc.Record(path); err != nil {
				log.G(ctx).WithError(err).Debugf("failed to record %q", path)
			}
			successCount++
		}
	}()
	if aOpts.terminal {
		if err := tasks.HandleConsoleResize(ctx, task, con); err != nil {
			log.G(ctx).WithError(err).Error("failed to resize console")
		}
	} else {
		sigc := commands.ForwardAllSignals(ctx, task)
		defer commands.StopCatch(sigc)
	}
	if err := task.Start(ctx); err != nil {
		return "", err
	}

	// Wait until the task exit
	var status containerd.ExitStatus
	var killOk bool
	if aOpts.waitOnSignal { // NOTE: not functional with `terminal` option
		log.G(ctx).Infof("press Ctrl+C to terminate the container")
		status, killOk, err = waitOnSignal(ctx, container, task)
		if err != nil {
			return "", err
		}
	} else {
		if aOpts.period <= 0 {
			aOpts.period = defaultPeriod
		}
		log.G(ctx).Infof("waiting for %v ...", aOpts.period)
		status, killOk, err = waitOnTimeout(ctx, container, task, aOpts.period)
		if err != nil {
			return "", err
		}
	}
	if !killOk {
		log.G(ctx).Warnf("failed to exit task %v; manually kill it", task.ID())
	} else {
		code, _, err := status.Result()
		if err != nil {
			return "", err
		}
		log.G(ctx).Infof("container exit with code %v", code)
		if _, err := task.Delete(ctx); err != nil {
			return "", err
		}
	}

	// ensure no record comes in
	fanotifierClosedMu.Lock()
	fanotifierClosed = true
	fanotifierClosedMu.Unlock()
	if err := fanotifier.Close(); err != nil {
		log.G(ctx).WithError(err).Warnf("failed to cleanup fanotifier")
	}

	// Finish recording
	return rc.Commit(ctx)
}

func mountImage(ctx context.Context, ss snapshots.Snapshotter, image containerd.Image, mountpoint string) (func(), error) {
	diffIDs, err := image.RootFS(ctx)
	if err != nil {
		return nil, err
	}
	mounts, err := ss.Prepare(ctx, mountpoint, identity.ChainID(diffIDs).String())
	if err != nil {
		return nil, err
	}
	if err := mount.All(mounts, mountpoint); err != nil {
		if err := ss.Remove(ctx, mountpoint); err != nil && !errdefs.IsNotFound(err) {
			log.G(ctx).WithError(err).Warnf("failed to cleanup snapshot after mount error")
		}
		return nil, errors.Wrapf(err, "failed to mount rootfs at %q", mountpoint)
	}
	return func() {
		if err := mount.UnmountAll(mountpoint, 0); err != nil {
			log.G(ctx).WithError(err).Warnf("failed to unmount snapshot")
		}
		if err := ss.Remove(ctx, mountpoint); err != nil && !errdefs.IsNotFound(err) {
			log.G(ctx).WithError(err).Warnf("failed to cleanup snapshot")
		}
	}, nil
}

func waitOnSignal(ctx context.Context, container containerd.Container, task containerd.Task) (containerd.ExitStatus, bool, error) {
	statusC, err := task.Wait(ctx)
	if err != nil {
		return containerd.ExitStatus{}, false, err
	}
	sc := make(chan os.Signal, 1)
	signal.Notify(sc, syscall.SIGINT)
	defer signal.Stop(sc)
	select {
	case status := <-statusC:
		return status, true, nil
	case <-sc:
		log.G(ctx).Info("signal detected")
		status, err := killTask(ctx, container, task, statusC)
		if err != nil {
			log.G(ctx).WithError(err).Warnf("failed to kill container")
			return containerd.ExitStatus{}, false, nil
		}
		return status, true, nil
	}
}

func waitOnTimeout(ctx context.Context, container containerd.Container, task containerd.Task, period time.Duration) (containerd.ExitStatus, bool, error) {
	statusC, err := task.Wait(ctx)
	if err != nil {
		return containerd.ExitStatus{}, false, err
	}
	select {
	case status := <-statusC:
		return status, true, nil
	case <-time.After(period):
		log.G(ctx).Warnf("killing task. the time period to monitor access log (%s) has timed out", period.String())
		status, err := killTask(ctx, container, task, statusC)
		if err != nil {
			log.G(ctx).WithError(err).Warnf("failed to kill container")
			return containerd.ExitStatus{}, false, nil
		}
		return status, true, nil
	}
}

func killTask(ctx context.Context, container containerd.Container, task containerd.Task, statusC <-chan containerd.ExitStatus) (containerd.ExitStatus, error) {
	sig, err := containerd.GetStopSignal(ctx, container, syscall.SIGKILL)
	if err != nil {
		return containerd.ExitStatus{}, err
	}
	if err := task.Kill(ctx, sig, containerd.WithKillAll); err != nil && !errdefs.IsNotFound(err) {
		return containerd.ExitStatus{}, errors.Wrapf(err, "forward SIGKILL")
	}
	select {
	case status := <-statusC:
		return status, nil
	case <-time.After(5 * time.Second):
		return containerd.ExitStatus{}, fmt.Errorf("timeout")
	}
}

type lazyReadCloser struct {
	reader      io.Reader
	closer      func()
	closerMu    sync.Mutex
	initCond    *sync.Cond
	initialized int64
}

func newLazyReadCloser(r io.Reader) *lazyReadCloser {
	rc := &lazyReadCloser{reader: r, initCond: sync.NewCond(&sync.Mutex{})}
	return rc
}

func (s *lazyReadCloser) registerCloser(closer func()) {
	s.closerMu.Lock()
	s.closer = closer
	s.closerMu.Unlock()
	atomic.AddInt64(&s.initialized, 1)
	s.initCond.Broadcast()
}

func (s *lazyReadCloser) Read(p []byte) (int, error) {
	if atomic.LoadInt64(&s.initialized) <= 0 {
		// wait until initialized
		s.initCond.L.Lock()
		if atomic.LoadInt64(&s.initialized) <= 0 {
			s.initCond.Wait()
		}
		s.initCond.L.Unlock()
	}

	n, err := s.reader.Read(p)
	if err == io.EOF {
		s.closerMu.Lock()
		s.closer()
		s.closerMu.Unlock()
	}
	return n, err
}
<|endoftext|>


<commit_before>package middleware

import (
	"crypto/tls"
	"io"
	"mime/multipart"
	"net/http"
	"net/url"
	"strconv"

	"github.com/pivotal-golang/lager"
)

type Logger struct {
	logger lager.Logger
}

func NewLogger(logger lager.Logger) Middleware {
	return Logger{
		logger: logger,
	}
}

func (l Logger) Wrap(next http.Handler) http.Handler {
	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
		loggingResponseWriter := responseWriter{
			rw,
			[]byte{},
			0,
		}
		next.ServeHTTP(&loggingResponseWriter, req)

		requestCopy := *req
		requestCopy.Header["Authorization"] = nil

		response := map[string]interface{}{
			"Header":     loggingResponseWriter.Header(),
			"Body":       string(loggingResponseWriter.body),
			"StatusCode": loggingResponseWriter.statusCode,
		}

		l.logger.Debug("", lager.Data{
			"request":  fromHTTPRequest(requestCopy),
			"response": response,
		})
	})
}

type responseWriter struct {
	http.ResponseWriter
	body       []byte
	statusCode int
}

func (rw *responseWriter) Write(b []byte) (int, error) {
	rw.Header().Set("Content-Length", strconv.Itoa(len(b)))

	if rw.statusCode == 0 {
		rw.WriteHeader(http.StatusOK)
	}

	size, err := rw.ResponseWriter.Write(b)
	rw.body = b
	return size, err
}

func (rw *responseWriter) WriteHeader(s int) {
	rw.statusCode = s
	rw.ResponseWriter.WriteHeader(s)
}

// Golang 1.5 introduces a http.Request.Cancel field,
// of type <-chan struct{} which the lager library fails to deal with.
// We introduced loggableHTTPRequest as a way of handling this
// until such time as lager can deal with it.
//
// Once lager can handle the request directly, remove this struct.
// #101259402
type LoggableHTTPRequest struct {
	Method           string
	URL              *url.URL
	Proto            string
	ProtoMajor       int
	ProtoMinor       int
	Header           http.Header
	Body             io.ReadCloser
	ContentLength    int64
	TransferEncoding []string
	Close            bool
	Host             string
	Form             url.Values
	PostForm         url.Values
	MultipartForm    *multipart.Form
	Trailer          http.Header
	RemoteAddr       string
	RequestURI       string
	TLS              *tls.ConnectionState
}

func fromHTTPRequest(req http.Request) LoggableHTTPRequest {
	return LoggableHTTPRequest{
		Method:           req.Method,
		URL:              req.URL,
		Proto:            req.Proto,
		ProtoMajor:       req.ProtoMajor,
		ProtoMinor:       req.ProtoMinor,
		Header:           req.Header,
		Body:             req.Body,
		ContentLength:    req.ContentLength,
		TransferEncoding: req.TransferEncoding,
		Close:            req.Close,
		Host:             req.Host,
		Form:             req.Form,
		PostForm:         req.PostForm,
		MultipartForm:    req.MultipartForm,
		Trailer:          req.Trailer,
		RemoteAddr:       req.RemoteAddr,
		RequestURI:       req.RequestURI,
		TLS:              req.TLS,
	}
}
<commit_msg>Do not log http response body.<commit_after>package middleware

import (
	"crypto/tls"
	"io"
	"mime/multipart"
	"net/http"
	"net/url"
	"strconv"

	"github.com/pivotal-golang/lager"
)

type Logger struct {
	logger lager.Logger
}

func NewLogger(logger lager.Logger) Middleware {
	return Logger{
		logger: logger,
	}
}

func (l Logger) Wrap(next http.Handler) http.Handler {
	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
		loggingResponseWriter := responseWriter{
			rw,
			[]byte{},
			0,
		}
		next.ServeHTTP(&loggingResponseWriter, req)

		requestCopy := *req
		requestCopy.Header["Authorization"] = nil

		response := map[string]interface{}{
			"Header":     loggingResponseWriter.Header(),
			"StatusCode": loggingResponseWriter.statusCode,
		}

		l.logger.Debug("", lager.Data{
			"request":  fromHTTPRequest(requestCopy),
			"response": response,
		})
	})
}

type responseWriter struct {
	http.ResponseWriter
	body       []byte
	statusCode int
}

func (rw *responseWriter) Write(b []byte) (int, error) {
	rw.Header().Set("Content-Length", strconv.Itoa(len(b)))

	if rw.statusCode == 0 {
		rw.WriteHeader(http.StatusOK)
	}

	size, err := rw.ResponseWriter.Write(b)
	rw.body = b
	return size, err
}

func (rw *responseWriter) WriteHeader(s int) {
	rw.statusCode = s
	rw.ResponseWriter.WriteHeader(s)
}

// Golang 1.5 introduces a http.Request.Cancel field,
// of type <-chan struct{} which the lager library fails to deal with.
// We introduced loggableHTTPRequest as a way of handling this
// until such time as lager can deal with it.
//
// Once lager can handle the request directly, remove this struct.
// #101259402
type LoggableHTTPRequest struct {
	Method           string
	URL              *url.URL
	Proto            string
	ProtoMajor       int
	ProtoMinor       int
	Header           http.Header
	Body             io.ReadCloser
	ContentLength    int64
	TransferEncoding []string
	Close            bool
	Host             string
	Form             url.Values
	PostForm         url.Values
	MultipartForm    *multipart.Form
	Trailer          http.Header
	RemoteAddr       string
	RequestURI       string
	TLS              *tls.ConnectionState
}

func fromHTTPRequest(req http.Request) LoggableHTTPRequest {
	return LoggableHTTPRequest{
		Method:           req.Method,
		URL:              req.URL,
		Proto:            req.Proto,
		ProtoMajor:       req.ProtoMajor,
		ProtoMinor:       req.ProtoMinor,
		Header:           req.Header,
		Body:             req.Body,
		ContentLength:    req.ContentLength,
		TransferEncoding: req.TransferEncoding,
		Close:            req.Close,
		Host:             req.Host,
		Form:             req.Form,
		PostForm:         req.PostForm,
		MultipartForm:    req.MultipartForm,
		Trailer:          req.Trailer,
		RemoteAddr:       req.RemoteAddr,
		RequestURI:       req.RequestURI,
		TLS:              req.TLS,
	}
}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor: Aaron Meihm ameihm@mozilla.com [:alm]
package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"mig.ninja/mig"
	"os/exec"
	"path"
)

var pluginList []plugin

type plugin struct {
	name string
	path string
}

func runPlugin(r mig.RunnerResult) (err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("runPlugin() -> %v", e)
		}
	}()

	var pent *plugin
	for i := range pluginList {
		if pluginList[i].name == r.UsePlugin {
			pent = &pluginList[i]
			break
		}
	}
	if pent == nil {
		panic("unable to locate plugin")
	}

	go func() {
		buf, err := json.Marshal(r)
		if err != nil {
			mlog("%v: %v", pent.name, err)
			return
		}
		c := exec.Command(pent.path)
		stdin, err := c.StdinPipe()
		if err != nil {
			mlog("%v: %v", pent.name, err)
			return
		}
		err = c.Start()
		if err != nil {
			mlog("%v: %v", pent.name, err)
			return
		}
		wb, err := stdin.Write(buf)
		if err != nil {
			mlog("%v: %v", pent.name, err)
		}
		stdin.Close()
		err = c.Wait()
		if err != nil {
			mlog("%v: %v", pent.name, err)
		}
		mlog("%v: wrote %v bytes to plugin", pent.name, wb)
	}()

	return nil
}

func loadPlugins() (err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("loadPlugins() -> %v", e)
		}
	}()

	pluginList = make([]plugin, 0)

	// Identify any available output plugins
	dirents, err := ioutil.ReadDir(ctx.Runner.PluginDirectory)
	if err != nil {
		panic(err)
	}
	for _, x := range dirents {
		n := x.Name()
		m := x.Mode()
		if (m & 0111) == 0 {
			mlog("plugins: skipping %v (not executable)", n)
			continue
		}
		ppath := path.Join(ctx.Runner.PluginDirectory, n)
		mlog("plugins: registering %v", n)
		np := plugin{}
		np.name = n
		np.path = ppath
		pluginList = append(pluginList, np)
	}

	return nil
}
<commit_msg>[minor] remove unneccessary make() call, more explicit variable names<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor: Aaron Meihm ameihm@mozilla.com [:alm]
package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"mig.ninja/mig"
	"os/exec"
	"path"
)

var pluginList []plugin

type plugin struct {
	name string
	path string
}

func runPlugin(r mig.RunnerResult) (err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("runPlugin() -> %v", e)
		}
	}()

	var pent *plugin
	for i := range pluginList {
		if pluginList[i].name == r.UsePlugin {
			pent = &pluginList[i]
			break
		}
	}
	if pent == nil {
		panic("unable to locate plugin")
	}

	go func() {
		buf, err := json.Marshal(r)
		if err != nil {
			mlog("%v: %v", pent.name, err)
			return
		}
		c := exec.Command(pent.path)
		stdin, err := c.StdinPipe()
		if err != nil {
			mlog("%v: %v", pent.name, err)
			return
		}
		err = c.Start()
		if err != nil {
			mlog("%v: %v", pent.name, err)
			return
		}
		wb, err := stdin.Write(buf)
		if err != nil {
			mlog("%v: %v", pent.name, err)
		}
		stdin.Close()
		err = c.Wait()
		if err != nil {
			mlog("%v: %v", pent.name, err)
		}
		mlog("%v: wrote %v bytes to plugin", pent.name, wb)
	}()

	return nil
}

func loadPlugins() (err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("loadPlugins() -> %v", e)
		}
	}()

	// Identify any available output plugins
	dirents, err := ioutil.ReadDir(ctx.Runner.PluginDirectory)
	if err != nil {
		panic(err)
	}
	for _, pluginEnt := range dirents {
		pluginName := pluginEnt.Name()
		pluginMode := pluginEnt.Mode()
		if (pluginMode & 0111) == 0 {
			mlog("plugins: skipping %v (not executable)", pluginName)
			continue
		}
		ppath := path.Join(ctx.Runner.PluginDirectory, pluginName)
		mlog("plugins: registering %v", pluginName)
		np := plugin{
			name: pluginName,
			path: ppath,
		}
		pluginList = append(pluginList, np)
	}

	return nil
}
<|endoftext|>


<commit_before>package config

import (
	"encoding/hex"

	log "github.com/sirupsen/logrus"

	"github.com/bytom/consensus"
	"github.com/bytom/protocol/bc"
	"github.com/bytom/protocol/bc/types"
)

func genesisTx() *types.Tx {
	contract, err := hex.DecodeString("00148c9d063ff74ee6d9ffa88d83aeb038068366c4c4")
	if err != nil {
		log.Panicf("fail on decode genesis tx output control program")
	}

	txData := types.TxData{
		Version: 1,
		Inputs: []*types.TxInput{
			types.NewCoinbaseInput([]byte("Information is power. -- Jan/11/2013. Computing is power. -- Apr/24/2018.")),
		},
		Outputs: []*types.TxOutput{
			types.NewTxOutput(*consensus.BTMAssetID, consensus.InitialBlockSubsidy, contract),
		},
	}
	return types.NewTx(txData)
}

// GenesisBlock will return genesis block
func GenesisBlock() *types.Block {
	tx := genesisTx()
	txStatus := bc.NewTransactionStatus()
	txStatus.SetStatus(0, false)
	txStatusHash, err := bc.TxStatusMerkleRoot(txStatus.VerifyStatus)
	if err != nil {
		log.Panicf("fail on calc genesis tx status merkle root")
	}

	merkleRoot, err := bc.TxMerkleRoot([]*bc.Tx{tx.Tx})
	if err != nil {
		log.Panicf("fail on calc genesis tx merkel root")
	}

	block := &types.Block{
		BlockHeader: types.BlockHeader{
			Version:   1,
			Height:    0,
			Nonce:     9253507043297,
			Timestamp: 1524549600,
			Bits:      2161727821137910632,
			BlockCommitment: types.BlockCommitment{
				TransactionsMerkleRoot: merkleRoot,
				TransactionStatusHash:  txStatusHash,
			},
		},
		Transactions: []*types.Tx{tx},
	}
	return block
}
<commit_msg>genesis block support mutil net (#1036)<commit_after>package config

import (
	"encoding/hex"
	"time"

	log "github.com/sirupsen/logrus"

	"github.com/bytom/consensus"
	"github.com/bytom/protocol/bc"
	"github.com/bytom/protocol/bc/types"
)

func genesisTx() *types.Tx {
	contract, err := hex.DecodeString("00148c9d063ff74ee6d9ffa88d83aeb038068366c4c4")
	if err != nil {
		log.Panicf("fail on decode genesis tx output control program")
	}

	txData := types.TxData{
		Version: 1,
		Inputs: []*types.TxInput{
			types.NewCoinbaseInput([]byte("Information is power. -- Jan/11/2013. Computing is power. -- Apr/24/2018.")),
		},
		Outputs: []*types.TxOutput{
			types.NewTxOutput(*consensus.BTMAssetID, consensus.InitialBlockSubsidy, contract),
		},
	}
	return types.NewTx(txData)
}

func mainNetGenesisBlock() *types.Block {
	tx := genesisTx()
	txStatus := bc.NewTransactionStatus()
	txStatus.SetStatus(0, false)
	txStatusHash, err := bc.TxStatusMerkleRoot(txStatus.VerifyStatus)
	if err != nil {
		log.Panicf("fail on calc genesis tx status merkle root")
	}

	merkleRoot, err := bc.TxMerkleRoot([]*bc.Tx{tx.Tx})
	if err != nil {
		log.Panicf("fail on calc genesis tx merkel root")
	}

	block := &types.Block{
		BlockHeader: types.BlockHeader{
			Version:   1,
			Height:    0,
			Nonce:     9253507043297,
			Timestamp: 1524549600,
			Bits:      2161727821137910632,
			BlockCommitment: types.BlockCommitment{
				TransactionsMerkleRoot: merkleRoot,
				TransactionStatusHash:  txStatusHash,
			},
		},
		Transactions: []*types.Tx{tx},
	}
	return block
}

func testNetGenesisBlock() *types.Block {
	tx := genesisTx()
	txStatus := bc.NewTransactionStatus()
	txStatus.SetStatus(0, false)
	txStatusHash, err := bc.TxStatusMerkleRoot(txStatus.VerifyStatus)
	if err != nil {
		log.Panicf("fail on calc genesis tx status merkle root")
	}

	merkleRoot, err := bc.TxMerkleRoot([]*bc.Tx{tx.Tx})
	if err != nil {
		log.Panicf("fail on calc genesis tx merkel root")
	}

	block := &types.Block{
		BlockHeader: types.BlockHeader{
			Version:   1,
			Height:    0,
			Nonce:     9253507043297,
			Timestamp: 1528455800,
			Bits:      2233785415178221890,
			BlockCommitment: types.BlockCommitment{
				TransactionsMerkleRoot: merkleRoot,
				TransactionStatusHash:  txStatusHash,
			},
		},
		Transactions: []*types.Tx{tx},
	}
	return block
}

func soloNetGenesisBlock() *types.Block {
	tx := genesisTx()
	txStatus := bc.NewTransactionStatus()
	txStatus.SetStatus(0, false)
	txStatusHash, err := bc.TxStatusMerkleRoot(txStatus.VerifyStatus)
	if err != nil {
		log.Panicf("fail on calc genesis tx status merkle root")
	}

	merkleRoot, err := bc.TxMerkleRoot([]*bc.Tx{tx.Tx})
	if err != nil {
		log.Panicf("fail on calc genesis tx merkel root")
	}

	block := &types.Block{
		BlockHeader: types.BlockHeader{
			Version:   1,
			Height:    0,
			Nonce:     9253507043297,
			Timestamp: uint64(time.Now().Unix()),
			Bits:      2305843009214532812,
			BlockCommitment: types.BlockCommitment{
				TransactionsMerkleRoot: merkleRoot,
				TransactionStatusHash:  txStatusHash,
			},
		},
		Transactions: []*types.Tx{tx},
	}
	return block
}

// GenesisBlock will return genesis block
func GenesisBlock() *types.Block {
	return map[string]func() *types.Block{
		"main": mainNetGenesisBlock,
		"test": testNetGenesisBlock,
		"solo": soloNetGenesisBlock,
	}[consensus.ActiveNetParams.Name]()
}
<|endoftext|>


<commit_before>package mixpanel

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"github.com/nu7hatch/gouuid"
	"io"
	"net/http"
	"net/url"
	"sort"
	"strings"
	"time"
)

// The official base URL
const MixpanelBaseURL = "https://data.mixpanel.com/api/2.0/export"

// Key into the EventData map that contains the UUID of this event. Name is
// chosen to make collisions with actual keys very unlikely.
const EventIDKey = "$__$$event_id"

// Mixpanel struct represents a set of credentials used to access the Mixpanel
// API for a particular product.
type Mixpanel struct {
	Product string
	Key     string
	Secret  string
	BaseURL string
}

// EventData is a representation of each individual JSON record spit out of the
// export process.
type EventData map[string]interface{}

// New creates a Mixpanel object with the given API credentials and uses the
// official API URL.
func New(product, key, secret string) *Mixpanel {
	return NewWithURL(product, key, secret, MixpanelBaseURL)
}

// NewWithURL creates a Mixpanel object with the given API credentials and a
// custom Mixpanel API URL.
//
// I doubt this will ever be useful but there you go.
func NewWithURL(product, key, secret, baseURL string) *Mixpanel {
	m := new(Mixpanel)
	m.Product = product
	m.Key = key
	m.Secret = secret
	m.BaseURL = baseURL
	return m
}

// Add the cryptographic signature that Mixpanel API requests require.
//
// FIXME: Pretty sure sorting the pairs isn't necessary. Double check.
//
// Algorithm:
// - join key=value pairs
// - sort the pairs alphabetically
// - appending a secret
// - take MD5 hex digest.
func (m *Mixpanel) addSignature(args *url.Values) {
	hash := md5.New()

	var params []string
	for k, vs := range *args {
		for _, v := range vs {
			params = append(params, fmt.Sprintf("%s=%s", k, v))
		}
	}

	sort.StringSlice(params).Sort()

	io.WriteString(hash, strings.Join(params, "")+m.Secret)

	args.Set("sig", fmt.Sprintf("%x", hash.Sum(nil)))
}

// Generate the initial, base arguments that should be common to all Mixpanel
// API requests being created here.
func (m *Mixpanel) makeArgs(date time.Time) url.Values {
	args := url.Values{}

	args.Set("format", "json")
	args.Set("api_key", m.Key)
	args.Set("expire", fmt.Sprintf("%d", time.Now().Unix()+10000))

	day := date.Format("2006-01-02")

	args.Set("from_date", day)
	args.Set("to_date", day)

	return args
}

// ExportDate downloads event data for the given day and streams the resulting
// transformed JSON blobs as byte strings over the send-only channel passed
// to the function.
//
// The optional `moreArgs` parameter can be given to add additional URL
// parameters to the API request.
func (m *Mixpanel) ExportDate(date time.Time, output chan<- EventData, moreArgs *url.Values) error {
	args := m.makeArgs(date)

	if moreArgs != nil {
		for k, vs := range *moreArgs {
			for _, v := range vs {
				args.Add(k, v)
			}
		}
	}

	m.addSignature(&args)

	resp, err := http.Get(fmt.Sprintf("%s?%s", m.BaseURL, args.Encode()))

	if err != nil {
		return fmt.Errorf("%s: download failed: %s", m.Product, err)
	}

	defer resp.Body.Close()

	return m.TransformEventData(resp.Body, output)
}

// TransformEventData reads JSON objects line by line from `input`, performs a
// simple translation, and pipes the result back out through the `output` chan.
//
// The transformation effectively folds the properties map into the top level
// and attaches product information.
//
// Input : `{"event": "...", "properties": {"k": "v"}}`
// Output: `{"event": "...", "product: "...", "k": "v", ...}`
func (m *Mixpanel) TransformEventData(input io.Reader, output chan<- EventData) error {
	decoder := json.NewDecoder(input)

	// Don't default all numeric values to float
	decoder.UseNumber()

	for {
		var ev struct {
			Error      *string
			Event      string
			Properties map[string]interface{}
		}

		if err := decoder.Decode(&ev); err == io.EOF {
			break
		} else if err != nil {
			return fmt.Errorf("%s: Failed to parse JSON: %s", m.Product, err)
		} else if ev.Error != nil {
			return fmt.Errorf("%s: API error: %s", m.Product, *ev.Error)
		}

		if id, err := uuid.NewV4(); err == nil {
			ev.Properties[EventIDKey] = id.String()
		} else {
			return fmt.Errorf("%s: generating UUID failed: %s", m.Product, err)
		}

		ev.Properties["product"] = m.Product
		ev.Properties["event"] = ev.Event

		output <- ev.Properties
	}

	return nil
}
<commit_msg>Sorting params is definitely necessary for signature<commit_after>package mixpanel

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"github.com/nu7hatch/gouuid"
	"io"
	"net/http"
	"net/url"
	"sort"
	"strings"
	"time"
)

// The official base URL
const MixpanelBaseURL = "https://data.mixpanel.com/api/2.0/export"

// Key into the EventData map that contains the UUID of this event. Name is
// chosen to make collisions with actual keys very unlikely.
const EventIDKey = "$__$$event_id"

// Mixpanel struct represents a set of credentials used to access the Mixpanel
// API for a particular product.
type Mixpanel struct {
	Product string
	Key     string
	Secret  string
	BaseURL string
}

// EventData is a representation of each individual JSON record spit out of the
// export process.
type EventData map[string]interface{}

// New creates a Mixpanel object with the given API credentials and uses the
// official API URL.
func New(product, key, secret string) *Mixpanel {
	return NewWithURL(product, key, secret, MixpanelBaseURL)
}

// NewWithURL creates a Mixpanel object with the given API credentials and a
// custom Mixpanel API URL.
//
// I doubt this will ever be useful but there you go.
func NewWithURL(product, key, secret, baseURL string) *Mixpanel {
	m := new(Mixpanel)
	m.Product = product
	m.Key = key
	m.Secret = secret
	m.BaseURL = baseURL
	return m
}

// Add the cryptographic signature that Mixpanel API requests require.
//
// Algorithm:
// - join key=value pairs
// - sort the pairs alphabetically
// - appending a secret
// - take MD5 hex digest.
func (m *Mixpanel) addSignature(args *url.Values) {
	hash := md5.New()

	var params []string
	for k, vs := range *args {
		for _, v := range vs {
			params = append(params, fmt.Sprintf("%s=%s", k, v))
		}
	}

	sort.StringSlice(params).Sort()

	io.WriteString(hash, strings.Join(params, "")+m.Secret)
	args.Set("sig", fmt.Sprintf("%x", hash.Sum(nil)))
}

// Generate the initial, base arguments that should be common to all Mixpanel
// API requests being created here.
func (m *Mixpanel) makeArgs(date time.Time) url.Values {
	args := url.Values{}

	args.Set("format", "json")
	args.Set("api_key", m.Key)
	args.Set("expire", fmt.Sprintf("%d", time.Now().Unix()+10000))

	day := date.Format("2006-01-02")

	args.Set("from_date", day)
	args.Set("to_date", day)

	return args
}

// ExportDate downloads event data for the given day and streams the resulting
// transformed JSON blobs as byte strings over the send-only channel passed
// to the function.
//
// The optional `moreArgs` parameter can be given to add additional URL
// parameters to the API request.
func (m *Mixpanel) ExportDate(date time.Time, output chan<- EventData, moreArgs *url.Values) error {
	args := m.makeArgs(date)

	if moreArgs != nil {
		for k, vs := range *moreArgs {
			for _, v := range vs {
				args.Add(k, v)
			}
		}
	}

	m.addSignature(&args)

	resp, err := http.Get(fmt.Sprintf("%s?%s", m.BaseURL, args.Encode()))

	if err != nil {
		return fmt.Errorf("%s: download failed: %s", m.Product, err)
	}

	defer resp.Body.Close()

	return m.TransformEventData(resp.Body, output)
}

// TransformEventData reads JSON objects line by line from `input`, performs a
// simple translation, and pipes the result back out through the `output` chan.
//
// The transformation effectively folds the properties map into the top level
// and attaches product information.
//
// Input : `{"event": "...", "properties": {"k": "v"}}`
// Output: `{"event": "...", "product: "...", "k": "v", ...}`
func (m *Mixpanel) TransformEventData(input io.Reader, output chan<- EventData) error {
	decoder := json.NewDecoder(input)

	// Don't default all numeric values to float
	decoder.UseNumber()

	for {
		var ev struct {
			Error      *string
			Event      string
			Properties map[string]interface{}
		}

		if err := decoder.Decode(&ev); err == io.EOF {
			break
		} else if err != nil {
			return fmt.Errorf("%s: Failed to parse JSON: %s", m.Product, err)
		} else if ev.Error != nil {
			return fmt.Errorf("%s: API error: %s", m.Product, *ev.Error)
		}

		if id, err := uuid.NewV4(); err == nil {
			ev.Properties[EventIDKey] = id.String()
		} else {
			return fmt.Errorf("%s: generating UUID failed: %s", m.Product, err)
		}

		ev.Properties["product"] = m.Product
		ev.Properties["event"] = ev.Event

		output <- ev.Properties
	}

	return nil
}
<|endoftext|>


<commit_before>package mod_test

import (
	"io/ioutil"
	"net/http"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/prometheus/common/expfmt"
	"github.com/robustirc/bridge/robustsession"
	"github.com/robustirc/internal/health"
	"github.com/robustirc/robustirc/internal/localnet"
	"github.com/robustirc/robustirc/internal/robust"
)

func TestMessageOfDeath(t *testing.T) {
	tempdir, err := ioutil.TempDir("", "robustirc-message-of-death-")
	if err != nil {
		t.Fatalf("Could not create tempdir: %v", err)
	}

	l, err := localnet.NewLocalnet(-1, tempdir)
	if err != nil {
		t.Fatalf("Could not start local RobustIRC network: %v", err)
	}
	defer l.Kill(true)

	l.EnablePanicCommand = "1"

	// For each of the nodes, start a goroutine that verifies that the node crashes, then start it again
	var wg sync.WaitGroup

	for i := 0; i < 3; i++ {
		cmd, tempdir, addr := l.StartIRCServer(i == 0)
		wg.Add(1)
		go func(cmd *exec.Cmd, tempdir string, addr string) {
			defer wg.Done()

			terminated := make(chan error)
			skipped := make(chan bool)

			go func() {
				terminated <- cmd.Wait()
			}()

			go func() {
				// Poll messages of death counter.
				parser := &expfmt.TextParser{}
				for {
					time.Sleep(50 * time.Millisecond)
					req, err := http.NewRequest("GET", addr+"metrics", nil)
					if err != nil {
						continue
					}
					resp, err := l.Httpclient.Do(req)
					if err != nil {
						continue
					}
					if resp.StatusCode != http.StatusOK {
						continue
					}
					metrics, err := parser.TextToMetricFamilies(resp.Body)
					if err != nil {
						continue
					}
					applied, ok := metrics["applied_messages"]
					if !ok {
						continue
					}
					for _, m := range applied.GetMetric() {
						for _, labelpair := range m.GetLabel() {
							if labelpair.GetName() == "type" &&
								labelpair.GetValue() == robust.Type(robust.MessageOfDeath).String() {
								if m.GetCounter().GetValue() > 0 {
									skipped <- true
								}
							}
						}
					}
				}
			}()

			// Wait for the server to either crash or skip a message of death.
			select {
			case <-terminated:
				t.Logf("Node terminated (as expected)")
			case <-skipped:
				t.Logf("Node skipped message of death")
			}

			// Run restart.sh for that node.
			rcmd := exec.Command(filepath.Join(tempdir, "restart.sh"))
			if err := rcmd.Start(); err != nil {
				t.Errorf("Cannot restart node: %v", err)
				return
			}
			l.RecordResource("pid", strconv.Itoa(cmd.Process.Pid))

			// Ensure the node comes back up.
			started := time.Now()
			for time.Since(started) < 10*time.Second {
				if _, err := health.GetServerStatus(addr, l.NetworkPassword); err != nil {
					t.Logf("Node %s unhealthy: %v", addr, err)
					time.Sleep(1 * time.Second)
					continue
				}
				t.Logf("Node %s became healthy", addr)
				return
			}
			t.Errorf("Node did not become healthy within 10s")
		}(cmd, tempdir, addr)
	}

	// Connect and send the PANIC message.
	session, err := robustsession.Create(strings.Join(l.Servers(), ","), filepath.Join(tempdir, "cert.pem"))
	if err != nil {
		t.Fatalf("Could not create robustsession: %v", err)
	}
	foundjoin := make(chan bool)
	go func() {
		for msg := range session.Messages {
			if !strings.HasPrefix(msg, ":mod!1@") ||
				!strings.HasSuffix(msg, " JOIN #mod") {
				continue
			}
			select {
			case foundjoin <- true:
			default:
				t.Errorf("Found JOIN too early or too late (channel write did not block)")
			}
		}
	}()
	go func() {
		for err := range session.Errors {
			t.Errorf("RobustSession error: %v", err)
		}
	}()

	session.PostMessage("NICK mod")
	session.PostMessage("USER 1 2 3 4")
	session.PostMessage("PANIC")

	t.Logf("Message of death sent")

	wg.Wait()

	healthy := false
	for try := 0; try < 5; try++ {
		if l.Healthy() {
			healthy = true
			break
		}
		time.Sleep(1 * time.Second)
	}

	if !healthy {
		t.Fatalf("Expected recovery, but not all nodes are healthy")
	}

	// Verify sending a JOIN now results in an output message.
	session.PostMessage("JOIN #mod")
	select {
	case <-foundjoin:
		t.Logf("JOIN reply received, network progressing")
	case <-time.After(10 * time.Second):
		t.Errorf("Timeout waiting for JOIN message")
	}
}
<commit_msg>mod_test: record correct PID for teardown<commit_after>package mod_test

import (
	"io/ioutil"
	"net/http"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/prometheus/common/expfmt"
	"github.com/robustirc/bridge/robustsession"
	"github.com/robustirc/internal/health"
	"github.com/robustirc/robustirc/internal/localnet"
	"github.com/robustirc/robustirc/internal/robust"
)

func TestMessageOfDeath(t *testing.T) {
	tempdir, err := ioutil.TempDir("", "robustirc-message-of-death-")
	if err != nil {
		t.Fatalf("Could not create tempdir: %v", err)
	}

	l, err := localnet.NewLocalnet(-1, tempdir)
	if err != nil {
		t.Fatalf("Could not start local RobustIRC network: %v", err)
	}
	defer l.Kill(true)

	l.EnablePanicCommand = "1"

	// For each of the nodes, start a goroutine that verifies that the node crashes, then start it again
	var wg sync.WaitGroup

	for i := 0; i < 3; i++ {
		cmd, tempdir, addr := l.StartIRCServer(i == 0)
		wg.Add(1)
		go func(cmd *exec.Cmd, tempdir string, addr string) {
			defer wg.Done()

			terminated := make(chan error)
			skipped := make(chan bool)

			go func() {
				terminated <- cmd.Wait()
			}()

			go func() {
				// Poll messages of death counter.
				parser := &expfmt.TextParser{}
				for {
					time.Sleep(50 * time.Millisecond)
					req, err := http.NewRequest("GET", addr+"metrics", nil)
					if err != nil {
						continue
					}
					resp, err := l.Httpclient.Do(req)
					if err != nil {
						continue
					}
					if resp.StatusCode != http.StatusOK {
						continue
					}
					metrics, err := parser.TextToMetricFamilies(resp.Body)
					if err != nil {
						continue
					}
					applied, ok := metrics["applied_messages"]
					if !ok {
						continue
					}
					for _, m := range applied.GetMetric() {
						for _, labelpair := range m.GetLabel() {
							if labelpair.GetName() == "type" &&
								labelpair.GetValue() == robust.Type(robust.MessageOfDeath).String() {
								if m.GetCounter().GetValue() > 0 {
									skipped <- true
								}
							}
						}
					}
				}
			}()

			// Wait for the server to either crash or skip a message of death.
			select {
			case <-terminated:
				t.Logf("Node terminated (as expected)")
			case <-skipped:
				t.Logf("Node skipped message of death")
			}

			// Run restart.sh for that node.
			rcmd := exec.Command(filepath.Join(tempdir, "restart.sh"))
			if err := rcmd.Start(); err != nil {
				t.Errorf("Cannot restart node: %v", err)
				return
			}
			l.RecordResource("pid", strconv.Itoa(rcmd.Process.Pid))

			// Ensure the node comes back up.
			started := time.Now()
			for time.Since(started) < 10*time.Second {
				if _, err := health.GetServerStatus(addr, l.NetworkPassword); err != nil {
					t.Logf("Node %s unhealthy: %v", addr, err)
					time.Sleep(1 * time.Second)
					continue
				}
				t.Logf("Node %s became healthy", addr)
				return
			}
			t.Errorf("Node did not become healthy within 10s")
		}(cmd, tempdir, addr)
	}

	// Connect and send the PANIC message.
	session, err := robustsession.Create(strings.Join(l.Servers(), ","), filepath.Join(tempdir, "cert.pem"))
	if err != nil {
		t.Fatalf("Could not create robustsession: %v", err)
	}
	foundjoin := make(chan bool)
	go func() {
		for msg := range session.Messages {
			if !strings.HasPrefix(msg, ":mod!1@") ||
				!strings.HasSuffix(msg, " JOIN #mod") {
				continue
			}
			select {
			case foundjoin <- true:
			default:
				t.Errorf("Found JOIN too early or too late (channel write did not block)")
			}
		}
	}()
	go func() {
		for err := range session.Errors {
			t.Errorf("RobustSession error: %v", err)
		}
	}()

	session.PostMessage("NICK mod")
	session.PostMessage("USER 1 2 3 4")
	session.PostMessage("PANIC")

	t.Logf("Message of death sent")

	wg.Wait()

	healthy := false
	for try := 0; try < 5; try++ {
		if l.Healthy() {
			healthy = true
			break
		}
		time.Sleep(1 * time.Second)
	}

	if !healthy {
		t.Fatalf("Expected recovery, but not all nodes are healthy")
	}

	// Verify sending a JOIN now results in an output message.
	session.PostMessage("JOIN #mod")
	select {
	case <-foundjoin:
		t.Logf("JOIN reply received, network progressing")
	case <-time.After(10 * time.Second):
		t.Errorf("Timeout waiting for JOIN message")
	}
}
<|endoftext|>


<commit_before>package model

import "encoding/json"

type ReferentialId string
type ReferentialSlug string

type Referential struct {
	id   ReferentialId
	slug ReferentialSlug

	manager Referentials
	model   Model
}

type Referentials interface {
	New(slug ReferentialSlug) Referential
	Find(id ReferentialId) (Referential, bool)
	FindBySlug(slug ReferentialSlug) (Referential, bool)
	Save(stopArea *Referential) bool
	Delete(stopArea *Referential) bool
	Load() error
}

var referentials = NewMemoryReferentials()

func (referential *Referential) Id() ReferentialId {
	return referential.id
}

func (referential *Referential) Slug() ReferentialSlug {
	return referential.slug
}

func (referential *Referential) Model() Model {
	return referential.model
}

func (referential *Referential) Save() (ok bool) {
	ok = referential.manager.Save(referential)
	return
}

func (referential *Referential) MarshalJSON() ([]byte, error) {
	return json.Marshal(map[string]interface{}{
		"Id":   referential.id,
		"Slug": referential.slug,
	})
}

type MemoryReferentials struct {
	UUIDConsumer

	byId map[ReferentialId]*Referential
}

func NewMemoryReferentials() *MemoryReferentials {
	return &MemoryReferentials{
		byId: make(map[ReferentialId]*Referential),
	}
}

func CurrentReferentials() Referentials {
	return referentials
}

func (manager *MemoryReferentials) New(slug ReferentialSlug) Referential {
	model := NewMemoryModel()
	return Referential{slug: slug, manager: manager, model: model}
}

func (manager *MemoryReferentials) NewWithId(slug ReferentialSlug, id ReferentialId) Referential {
	model := NewMemoryModel()
	return Referential{id: id, slug: slug, manager: manager, model: model}
}

func (manager *MemoryReferentials) Find(id ReferentialId) (Referential, bool) {
	referential, ok := manager.byId[id]
	if ok {
		return *referential, true
	} else {
		return Referential{}, false
	}
}

func (manager *MemoryReferentials) FindBySlug(slug ReferentialSlug) (Referential, bool) {
	for _, referential := range manager.byId {
		if referential.slug == slug {
			return *referential, true
		}
	}
	return Referential{}, false
}

func (manager *MemoryReferentials) Save(referential *Referential) bool {
	if referential.Id() == "" {
		referential.id = ReferentialId(manager.NewUUID())
	}
	referential.manager = manager
	manager.byId[referential.Id()] = referential
	return true
}

func (manager *MemoryReferentials) Delete(referential *Referential) bool {
	delete(manager.byId, referential.Id())
	return true
}

func (manager *MemoryReferentials) Load() error {
	var selectReferentials []struct {
		Id   string `db:"referential_id"`
		Slug string
	}
	_, err := Database.Select(&selectReferentials, "select * from referentials")
	if err != nil {
		return err
	}

	for _, r := range selectReferentials {
		referential := manager.NewWithId(ReferentialSlug(r.Slug), ReferentialId(r.Id))
		manager.Save(&referential)
	}

	return nil
}
<commit_msg>small change on referentials Load<commit_after>package model

import "encoding/json"

type ReferentialId string
type ReferentialSlug string

type Referential struct {
	id   ReferentialId
	slug ReferentialSlug

	manager Referentials
	model   Model
}

type Referentials interface {
	New(slug ReferentialSlug) Referential
	Find(id ReferentialId) (Referential, bool)
	FindBySlug(slug ReferentialSlug) (Referential, bool)
	Save(stopArea *Referential) bool
	Delete(stopArea *Referential) bool
	Load() error
}

var referentials = NewMemoryReferentials()

func (referential *Referential) Id() ReferentialId {
	return referential.id
}

func (referential *Referential) Slug() ReferentialSlug {
	return referential.slug
}

func (referential *Referential) Model() Model {
	return referential.model
}

func (referential *Referential) Save() (ok bool) {
	ok = referential.manager.Save(referential)
	return
}

func (referential *Referential) MarshalJSON() ([]byte, error) {
	return json.Marshal(map[string]interface{}{
		"Id":   referential.id,
		"Slug": referential.slug,
	})
}

type MemoryReferentials struct {
	UUIDConsumer

	byId map[ReferentialId]*Referential
}

func NewMemoryReferentials() *MemoryReferentials {
	return &MemoryReferentials{
		byId: make(map[ReferentialId]*Referential),
	}
}

func CurrentReferentials() Referentials {
	return referentials
}

func (manager *MemoryReferentials) New(slug ReferentialSlug) Referential {
	model := NewMemoryModel()
	return Referential{slug: slug, manager: manager, model: model}
}

func (manager *MemoryReferentials) NewWithId(slug ReferentialSlug, id ReferentialId) Referential {
	model := NewMemoryModel()
	return Referential{id: id, slug: slug, manager: manager, model: model}
}

func (manager *MemoryReferentials) Find(id ReferentialId) (Referential, bool) {
	referential, ok := manager.byId[id]
	if ok {
		return *referential, true
	} else {
		return Referential{}, false
	}
}

func (manager *MemoryReferentials) FindBySlug(slug ReferentialSlug) (Referential, bool) {
	for _, referential := range manager.byId {
		if referential.slug == slug {
			return *referential, true
		}
	}
	return Referential{}, false
}

func (manager *MemoryReferentials) Save(referential *Referential) bool {
	if referential.Id() == "" {
		referential.id = ReferentialId(manager.NewUUID())
	}
	referential.manager = manager
	manager.byId[referential.Id()] = referential
	return true
}

func (manager *MemoryReferentials) Delete(referential *Referential) bool {
	delete(manager.byId, referential.Id())
	return true
}

func (manager *MemoryReferentials) Load() error {
	var selectReferentials []struct {
		Referential_id string
		Slug           string
	}
	_, err := Database.Select(&selectReferentials, "select * from referentials")
	if err != nil {
		return err
	}

	for _, r := range selectReferentials {
		referential := manager.NewWithId(ReferentialSlug(r.Slug), ReferentialId(r.Referential_id))
		manager.Save(&referential)
	}

	return nil
}
<|endoftext|>


<commit_before>package models

import (
	//"errors"
	"database/sql"
	"github.com/stretchr/testify/assert"
	"gopkg.in/DATA-DOG/go-sqlmock.v1"
	"testing"

	"github.com/eirka/eirka-libs/db"
	e "github.com/eirka/eirka-libs/errors"
)

func TestEmailIsValid(t *testing.T) {

	email := EmailModel{
		Uid:   1,
		Name:  "test",
		Email: "cool@test.com",
	}

	assert.False(t, email.IsValid(), "Should be false")

}

func TestEmailValidate(t *testing.T) {

	var err error

	mock, err := db.NewTestDb()
	assert.NoError(t, err, "An error was not expected")

	rows := sqlmock.NewRows([]string{"name", "email"}).AddRow("test", "old@test.com")
	mock.ExpectQuery(`SELECT user_name,user_email FROM users WHERE user_id`).WillReturnRows(rows)

	email := EmailModel{
		Uid:   2,
		Email: "cool@test.com",
	}

	err = email.Validate()
	if assert.NoError(t, err, "An error was not expected") {
		assert.Equal(t, email.Name, "test", "Should match")
		assert.Equal(t, email.CurrentEmail, "old@test.com", "Should match")
	}

}

func TestEmailValidateBadEmails(t *testing.T) {

	var err error

	first := EmailModel{
		Email: "notanemail",
	}

	err = first.Validate()
	if assert.Error(t, err, "An error was expected") {
		assert.Equal(t, err, e.ErrInvalidEmail, "Error should match")
	}

	second := EmailModel{
		Email: "not@anemail",
	}

	err = second.Validate()
	if assert.Error(t, err, "An error was expected") {
		assert.Equal(t, err, e.ErrInvalidEmail, "Error should match")
	}

}

func TestEmailValidateNoUser(t *testing.T) {

	var err error

	mock, err := db.NewTestDb()
	assert.NoError(t, err, "An error was not expected")

	mock.ExpectQuery(`SELECT user_name,user_email FROM users WHERE user_id`).WillReturnError(sql.ErrNoRows)

	email := EmailModel{
		Uid:   2,
		Email: "cool@test.com",
	}

	err = email.Validate()
	if assert.Error(t, err, "An error was expected") {
		assert.Equal(t, err, e.ErrUserNotExist, "Error should match")
	}

}

func TestEmailValidateSameEmail(t *testing.T) {

	var err error

	mock, err := db.NewTestDb()
	assert.NoError(t, err, "An error was not expected")

	rows := sqlmock.NewRows([]string{"name", "email"}).AddRow("test", "cool@test.com")
	mock.ExpectQuery(`SELECT user_name,user_email FROM users WHERE user_id`).WillReturnRows(rows)

	email := EmailModel{
		Uid:   2,
		Email: "cool@test.com",
	}

	err = email.Validate()
	if assert.Error(t, err, "An error was expected") {
		assert.Equal(t, err, e.ErrEmailSame, "Error should match")
	}

}

func TestEmailUpdate(t *testing.T) {

	var err error

	mock, err := db.NewTestDb()
	assert.NoError(t, err, "An error was not expected")

	mock.ExpectExec("UPDATE users SET user_email").
		WithArgs("cool@test.com", 2).
		WillReturnResult(sqlmock.NewResult(1, 1))

	email := EmailModel{
		Uid:   2,
		Email: "cool@test.com",
	}

}
<commit_msg>add model tests<commit_after>package models

import (
	//"errors"
	"database/sql"
	"github.com/stretchr/testify/assert"
	"gopkg.in/DATA-DOG/go-sqlmock.v1"
	"testing"

	"github.com/eirka/eirka-libs/db"
	e "github.com/eirka/eirka-libs/errors"
)

func TestEmailIsValid(t *testing.T) {

	email := EmailModel{
		Uid:   1,
		Name:  "test",
		Email: "cool@test.com",
	}

	assert.False(t, email.IsValid(), "Should be false")

}

func TestEmailValidate(t *testing.T) {

	var err error

	mock, err := db.NewTestDb()
	assert.NoError(t, err, "An error was not expected")

	rows := sqlmock.NewRows([]string{"name", "email"}).AddRow("test", "old@test.com")
	mock.ExpectQuery(`SELECT user_name,user_email FROM users WHERE user_id`).WillReturnRows(rows)

	email := EmailModel{
		Uid:   2,
		Email: "cool@test.com",
	}

	err = email.Validate()
	if assert.NoError(t, err, "An error was not expected") {
		assert.Equal(t, email.Name, "test", "Should match")
		assert.Equal(t, email.CurrentEmail, "old@test.com", "Should match")
	}

}

func TestEmailValidateBadEmails(t *testing.T) {

	var err error

	first := EmailModel{
		Email: "notanemail",
	}

	err = first.Validate()
	if assert.Error(t, err, "An error was expected") {
		assert.Equal(t, err, e.ErrInvalidEmail, "Error should match")
	}

	second := EmailModel{
		Email: "not@anemail",
	}

	err = second.Validate()
	if assert.Error(t, err, "An error was expected") {
		assert.Equal(t, err, e.ErrInvalidEmail, "Error should match")
	}

}

func TestEmailValidateNoUser(t *testing.T) {

	var err error

	mock, err := db.NewTestDb()
	assert.NoError(t, err, "An error was not expected")

	mock.ExpectQuery(`SELECT user_name,user_email FROM users WHERE user_id`).WillReturnError(sql.ErrNoRows)

	email := EmailModel{
		Uid:   2,
		Email: "cool@test.com",
	}

	err = email.Validate()
	if assert.Error(t, err, "An error was expected") {
		assert.Equal(t, err, e.ErrUserNotExist, "Error should match")
	}

}

func TestEmailValidateSameEmail(t *testing.T) {

	var err error

	mock, err := db.NewTestDb()
	assert.NoError(t, err, "An error was not expected")

	rows := sqlmock.NewRows([]string{"name", "email"}).AddRow("test", "cool@test.com")
	mock.ExpectQuery(`SELECT user_name,user_email FROM users WHERE user_id`).WillReturnRows(rows)

	email := EmailModel{
		Uid:   2,
		Email: "cool@test.com",
	}

	err = email.Validate()
	if assert.Error(t, err, "An error was expected") {
		assert.Equal(t, err, e.ErrEmailSame, "Error should match")
	}

}

func TestEmailUpdate(t *testing.T) {

	var err error

	mock, err := db.NewTestDb()
	assert.NoError(t, err, "An error was not expected")

	mock.ExpectExec("UPDATE users SET user_email").
		WithArgs("cool@test.com", 2).
		WillReturnResult(sqlmock.NewResult(1, 1))

	email := EmailModel{
		Uid:   2,
		Email: "cool@test.com",
	}

	err = email.Update()
	assert.NoError(t, err, "An error was not expected")

}
<|endoftext|>


<commit_before>package preparable

import (
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	log "github.com/sirupsen/logrus"
	"github.com/struCoder/pmgo/lib/process"
)

// ProcPreparable is a preparable with all the necessary informations to run
// a process. To actually run a process, call the Start() method.
type ProcPreparable interface {
	PrepareBin(bool) ([]byte, error)
	Start() (process.ProcContainer, error)
	getPath() string
	Identifier() string
	getBinPath() string
	getPidPath() string
	getOutPath() string
	getErrPath() string
}

type Preparable struct {
	Name       string
	SourcePath string
	Cmd        string
	SysFolder  string
	Language   string
	KeepAlive  bool
	Args       []string
}

// PrepareBin will compile the Golang project from SourcePath and populate Cmd with the proper
// command for the process to be executed.
// Returns the compile command output.
func (preparable *Preparable) PrepareBin(fromBinFile bool) ([]byte, error) {
	// Remove the last character '/' if present
	if preparable.SourcePath[len(preparable.SourcePath)-1] == '/' {
		preparable.SourcePath = strings.TrimSuffix(preparable.SourcePath, "/")
	}
	cmd := ""
	cmdArgs := []string{}
	binPath := preparable.getBinPath()

	if preparable.Language == "go" {
		if fromBinFile {
			os.MkdirAll(filepath.Dir(binPath), 0755)
			cmd = "cp"
			cmdArgs = []string{preparable.SourcePath, binPath}
			log.Info("copy file ", preparable.SourcePath, " to ", filepath.Dir(binPath))
		} else {
			cmd = "go"
			cmdArgs = []string{"build", "-o", binPath, preparable.SourcePath + "/."}
		}
	}

	preparable.Cmd = preparable.getBinPath()
	return exec.Command(cmd, cmdArgs...).Output()
}

// Start will execute the process based on the information presented on the preparable.
// This function should be called from inside the master to make sure
// all the watchers and process handling are done correctly.
// Returns a tuple with the process and an error in case there's any.
func (preparable *Preparable) Start() (process.ProcContainer, error) {
	proc := &process.Proc{
		Name:      preparable.Name,
		Cmd:       preparable.Cmd,
		Args:      preparable.Args,
		Path:      preparable.getPath(),
		Pidfile:   preparable.getPidPath(),
		Outfile:   preparable.getOutPath(),
		Errfile:   preparable.getErrPath(),
		KeepAlive: preparable.KeepAlive,
		Status:    &process.ProcStatus{},
	}

	err := proc.Start()
	return proc, err
}

// Identifier is a function that get proc name
func (preparable *Preparable) Identifier() string {
	return preparable.Name
}

func (preparable *Preparable) getPath() string {
	if preparable.SysFolder[len(preparable.SysFolder)-1] == '/' {
		preparable.SysFolder = strings.TrimSuffix(preparable.SysFolder, "/")
	}
	return preparable.SysFolder + "/" + preparable.Name
}

func (preparable *Preparable) getBinPath() string {
	return preparable.getPath() + "/" + preparable.Name
}

func (preparable *Preparable) getPidPath() string {
	return preparable.getBinPath() + ".pid"
}

func (preparable *Preparable) getOutPath() string {
	return preparable.getBinPath() + ".out"
}

func (preparable *Preparable) getErrPath() string {
	return preparable.getBinPath() + ".err"
}
<commit_msg>[StartBinary] (In Progress) Remove copying binary file<commit_after>package preparable

import (
	"fmt"
	"os"
	"os/exec"
	"strings"

	"github.com/struCoder/pmgo/lib/process"
)

// ProcPreparable is a preparable with all the necessary informations to run
// a process. To actually run a process, call the Start() method.
type ProcPreparable interface {
	PrepareBin(bool) ([]byte, error)
	Start() (process.ProcContainer, error)
	getPath() string
	Identifier() string
	getBinPath() string
	getPidPath() string
	getOutPath() string
	getErrPath() string
}

type Preparable struct {
	Name       string
	SourcePath string
	Cmd        string
	SysFolder  string
	Language   string
	KeepAlive  bool
	Args       []string
}

// PrepareBin will compile the Golang project from SourcePath and populate Cmd with the proper
// command for the process to be executed.
// Returns the compile command output.
func (preparable *Preparable) PrepareBin(fromBinFile bool) ([]byte, error) {
	// Remove the last character '/' if present
	if preparable.SourcePath[len(preparable.SourcePath)-1] == '/' {
		preparable.SourcePath = strings.TrimSuffix(preparable.SourcePath, "/")
	}
	binPath := preparable.getBinPath()

	var output []byte
	var err error
	if preparable.Language == "go" && !fromBinFile {
		cmd := "go"
		cmdArgs := []string{"build", "-o", binPath, preparable.SourcePath + "/."}

		output, err = exec.Command(cmd, cmdArgs...).Output()
	} else {
		info, err := os.Stat(preparable.SourcePath)
		if err != nil {
			return make([]byte, 0), err
		}
		if info.Mode()&0111 == 0 {
			return make([]byte, 0), fmt.Errorf("The given source path(%s) is not executable, neither a Go source to compile", preparable.SourcePath)
		}

		binPath = preparable.SourcePath
	}

	preparable.Cmd = binPath
	return output, err
}

// Start will execute the process based on the information presented on the preparable.
// This function should be called from inside the master to make sure
// all the watchers and process handling are done correctly.
// Returns a tuple with the process and an error in case there's any.
func (preparable *Preparable) Start() (process.ProcContainer, error) {
	proc := &process.Proc{
		Name:      preparable.Name,
		Cmd:       preparable.Cmd,
		Args:      preparable.Args,
		Path:      preparable.getPath(),
		Pidfile:   preparable.getPidPath(),
		Outfile:   preparable.getOutPath(),
		Errfile:   preparable.getErrPath(),
		KeepAlive: preparable.KeepAlive,
		Status:    &process.ProcStatus{},
	}

	err := proc.Start()
	return proc, err
}

// Identifier is a function that get proc name
func (preparable *Preparable) Identifier() string {
	return preparable.Name
}

func (preparable *Preparable) getPath() string {
	if preparable.SysFolder[len(preparable.SysFolder)-1] == '/' {
		preparable.SysFolder = strings.TrimSuffix(preparable.SysFolder, "/")
	}
	return preparable.SysFolder + "/" + preparable.Name
}

func (preparable *Preparable) getBinPath() string {
	return preparable.getPath() + "/" + preparable.Name
}

func (preparable *Preparable) getPidPath() string {
	return preparable.getBinPath() + ".pid"
}

func (preparable *Preparable) getOutPath() string {
	return preparable.getBinPath() + ".out"
}

func (preparable *Preparable) getErrPath() string {
	return preparable.getBinPath() + ".err"
}
<|endoftext|>


<commit_before>package chain

import (
	"sort"

	"github.com/btcboost/copernicus/model/blockindex"
	"github.com/btcboost/copernicus/util"
)

// Chain An in-memory blIndexed chain of blocks.
type Chain struct {
	active   		[]* blockindex.BlockIndex
	branch   		[]* blockindex.BlockIndex
	waitForTx     	map[util.Hash]* blockindex.BlockIndex
	orphan        	[]* blockindex.BlockIndex
	blockIndexMap 	map[util.Hash]* blockindex.BlockIndex
	newestBlock   	*blockindex.BlockIndex
	receiveID     	uint64
}

// Genesis Returns the blIndex entry for the genesis block of this chain,
// or nullptr if none.
func (c *Chain) Genesis() *blockindex.BlockIndex {
	if len(c.active) > 0 {
		return c.active[0]
	}

	return nil
}

// Tip Returns the blIndex entry for the tip of this chain, or nullptr if none.
func (c *Chain) Tip() *blockindex.BlockIndex {
	if len(c.active) > 0 {
		return c.active[len(c.active)-1]
	}

	return nil
}

// GetSpecIndex Returns the blIndex entry at a particular height in this chain, or nullptr
// if no such height exists.
func (c *Chain) GetIndex(height int) *blockindex.BlockIndex {
	if height < 0 || height >= len(c.active) {
		return nil
	}

	return c.active[height]
}

// Equal Compare two chains efficiently.
func (c *Chain) Equal(dst *Chain) bool {
	return len(c.active) == len(dst.active) &&
		c.active[len(c.active)-1] == dst.active[len(dst.active)-1]
}

// Contains /** Efficiently check whether a block is present in this chain
func (c *Chain) Contains(index *blockindex.BlockIndex) bool {
	return c.GetIndex(index.Height) == index
}

// Next Find the successor of a block in this chain, or nullptr if the given
// index is not found or is the tip.
func (c *Chain) Next(index *blockindex.BlockIndex) *blockindex.BlockIndex {
	if c.Contains(index) {
		return c.GetIndex(index.Height + 1)
	}
	return nil
}

// Height Return the maximal height in the chain. Is equal to chain.Tip() ?
// chain.Tip()->nHeight : -1.
func (c *Chain) Height() int {
	return len(c.active) - 1
}

// SetTip Set/initialize a chain with a given tip.
func (c *Chain) SetTip(index *BlockIndex) {
	if index == nil {
		c.active = []*BlockIndex{}
		return
	}

	tmp := make([]*BlockIndex, index.Height+1)
	copy(tmp, c.active)
	c.active = tmp
	for index != nil && c.active[index.Height] != index {
		c.active[index.Height] = index
		index = index.Prev
	}
}

// FindFork Find the last common block between this chain and a block blIndex entry.
func (chain *Chain) FindFork(blIndex *BlockIndex) *BlockIndex {
	if blIndex == nil {
		return nil
	}

	if blIndex.Height > chain.Height() {
		blIndex = blIndex.GetAncestor(chain.Height())
	}

	for blIndex != nil && !chain.Contains(blIndex) {
		blIndex = blIndex.Prev
	}
	return blIndex
}

// FindEarliestAtLeast Find the earliest block with timestamp equal or greater than the given.
func (chain *Chain) FindEarliestAtLeast(time int64) *BlockIndex {
	i := sort.Search(len(chain.Chain), func(i int) bool {
		return int64(chain.Chain[i].GetBlockTimeMax()) > time
	})
	if i == len(chain.Chain) {
		return nil
	}

	return chain.Chain[i]
}

func ActiveBestChain(bi *BlockIndex) bool {
	forkBlock := ActiveChain.FindFork(bi)
	if forkBlock == nil {
		return false
	}

	// maintain global variable NewestBlock
	NewestBlock = bi

	subHeight := bi.Height - forkBlock.Height
	tmpBi := make([]*BlockIndex, subHeight)
	tmpBi[subHeight-1] = bi
	for i := 0; i < subHeight; i++ {
		bi = bi.Prev
		tmpBi[subHeight-i-2] = bi
	}

	// maintain the global variable ActiveChain
	// todo should be locked
	ActiveChain.Chain = append(ActiveChain.Chain[:bi.Height+1], tmpBi...)

	// maintain global variable BranchChain
	removeBlockIndexFromBranchChain(tmpBi)
	addBlockIndexToBranchChain(tmpBi)

	return true
}

// should be before addBlockIndexToBranchChain()
func removeBlockIndexFromBranchChain(bis []*BlockIndex) {
	for i := 0; i < len(bis); i++ {
		for j := 0; j < len(BranchChain); {
			if BranchChain[j].BlockHash == bis[i].BlockHash {
				BranchChain = append(BranchChain[:j], BranchChain[j+1:]...)
			} else {
				j++
			}
		}
	}
}

// should be after removeBlockIndexFromBranchChain()
func addBlockIndexToBranchChain(bis []*BlockIndex) {
	BranchChain = append(BranchChain, bis...)
}

func FindMostWorkChain() *BlockIndex {
	// todo complete
	return nil
}<commit_msg>add function TipHeight(),GetMedianTimePast(),GetVersionState(),GetTipTime()<commit_after>package chain

import (
	"sort"

	"github.com/btcboost/copernicus/model/blockindex"
	"github.com/btcboost/copernicus/util"
)

// Chain An in-memory blIndexed chain of blocks.
type Chain struct {
	active   		[]* blockindex.BlockIndex
	branch   		[]* blockindex.BlockIndex
	waitForTx     	map[util.Hash]* blockindex.BlockIndex
	orphan        	[]* blockindex.BlockIndex
	blockIndexMap 	map[util.Hash]* blockindex.BlockIndex
	newestBlock   	*blockindex.BlockIndex
	receiveID     	uint64
}

// Genesis Returns the blIndex entry for the genesis block of this chain,
// or nullptr if none.
func (c *Chain) Genesis() *blockindex.BlockIndex {
	if len(c.active) > 0 {
		return c.active[0]
	}

	return nil
}

// Tip Returns the blIndex entry for the tip of this chain, or nullptr if none.
func (c *Chain) Tip() *blockindex.BlockIndex {
	if len(c.active) > 0 {
		return c.active[len(c.active)-1]
	}

	return nil
}

func (c *Chain) TipHeight() int {
	if len(c.active) > 0 {
		return c.active[len(c.active)-1].Height
	}

	return 0
}

func (c *Chain) GetMedianTimePast() int64 {

	return 0
}

func (c *Chain) GetVersionState() int {

	return 0
}

func (c *Chain) GetTipTime() int64 {

	return 0
}

// GetSpecIndex Returns the blIndex entry at a particular height in this chain, or nullptr
// if no such height exists.
func (c *Chain) GetIndex(height int) *blockindex.BlockIndex {
	if height < 0 || height >= len(c.active) {
		return nil
	}

	return c.active[height]
}

// Equal Compare two chains efficiently.
func (c *Chain) Equal(dst *Chain) bool {
	return len(c.active) == len(dst.active) &&
		c.active[len(c.active)-1] == dst.active[len(dst.active)-1]
}

// Contains /** Efficiently check whether a block is present in this chain
func (c *Chain) Contains(index *blockindex.BlockIndex) bool {
	return c.GetIndex(index.Height) == index
}

// Next Find the successor of a block in this chain, or nullptr if the given
// index is not found or is the tip.
func (c *Chain) Next(index *blockindex.BlockIndex) *blockindex.BlockIndex {
	if c.Contains(index) {
		return c.GetIndex(index.Height + 1)
	}
	return nil
}

// Height Return the maximal height in the chain. Is equal to chain.Tip() ?
// chain.Tip()->nHeight : -1.
func (c *Chain) Height() int {
	return len(c.active) - 1
}

// SetTip Set/initialize a chain with a given tip.
func (c *Chain) SetTip(index *BlockIndex) {
	if index == nil {
		c.active = []*BlockIndex{}
		return
	}

	tmp := make([]*BlockIndex, index.Height+1)
	copy(tmp, c.active)
	c.active = tmp
	for index != nil && c.active[index.Height] != index {
		c.active[index.Height] = index
		index = index.Prev
	}
}

// FindFork Find the last common block between this chain and a block blIndex entry.
func (chain *Chain) FindFork(blIndex *BlockIndex) *BlockIndex {
	if blIndex == nil {
		return nil
	}

	if blIndex.Height > chain.Height() {
		blIndex = blIndex.GetAncestor(chain.Height())
	}

	for blIndex != nil && !chain.Contains(blIndex) {
		blIndex = blIndex.Prev
	}
	return blIndex
}

// FindEarliestAtLeast Find the earliest block with timestamp equal or greater than the given.
func (chain *Chain) FindEarliestAtLeast(time int64) *BlockIndex {
	i := sort.Search(len(chain.Chain), func(i int) bool {
		return int64(chain.Chain[i].GetBlockTimeMax()) > time
	})
	if i == len(chain.Chain) {
		return nil
	}

	return chain.Chain[i]
}

func ActiveBestChain(bi *BlockIndex) bool {
	forkBlock := ActiveChain.FindFork(bi)
	if forkBlock == nil {
		return false
	}

	// maintain global variable NewestBlock
	NewestBlock = bi

	subHeight := bi.Height - forkBlock.Height
	tmpBi := make([]*BlockIndex, subHeight)
	tmpBi[subHeight-1] = bi
	for i := 0; i < subHeight; i++ {
		bi = bi.Prev
		tmpBi[subHeight-i-2] = bi
	}

	// maintain the global variable ActiveChain
	// todo should be locked
	ActiveChain.Chain = append(ActiveChain.Chain[:bi.Height+1], tmpBi...)

	// maintain global variable BranchChain
	removeBlockIndexFromBranchChain(tmpBi)
	addBlockIndexToBranchChain(tmpBi)

	return true
}

// should be before addBlockIndexToBranchChain()
func removeBlockIndexFromBranchChain(bis []*BlockIndex) {
	for i := 0; i < len(bis); i++ {
		for j := 0; j < len(BranchChain); {
			if BranchChain[j].BlockHash == bis[i].BlockHash {
				BranchChain = append(BranchChain[:j], BranchChain[j+1:]...)
			} else {
				j++
			}
		}
	}
}

// should be after removeBlockIndexFromBranchChain()
func addBlockIndexToBranchChain(bis []*BlockIndex) {
	BranchChain = append(BranchChain, bis...)
}

func FindMostWorkChain() *BlockIndex {
	// todo complete
	return nil
}<|endoftext|>


<commit_before>package msgpack

import (
	"encoding/hex"
	"fmt"
	"reflect"
	"testing"
)

func TestPack(t *testing.T) {
	t.Parallel()

	packTests := map[string]struct {
		// Expected value
		v interface{}
		// Hex encodings of typ, v
		hs string
	}{
		"Int64/0x0": {
			v:  int64(0x0),
			hs: "00",
		},
		"Int64/0x1": {
			v:  int64(0x1),
			hs: "01",
		},
		"Int64/0x7f": {
			v:  int64(0x7f),
			hs: "7f",
		},
		"Int64/0x80": {
			v:  int64(0x80),
			hs: "cc80",
		},
		"Int64/0x7fff": {
			v:  int64(0x7fff),
			hs: "cd7fff",
		},
		"Int64/0x8000": {
			v:  int64(0x8000),
			hs: "cd8000",
		},
		"Int64/0x7fffffff": {
			v:  int64(0x7fffffff),
			hs: "ce7fffffff",
		},
		"Int64/0x80000000": {
			v:  int64(0x80000000),
			hs: "ce80000000",
		},
		"Int64/0x7fffffffffffffff": {
			v:  int64(0x7fffffffffffffff),
			hs: "cf7fffffffffffffff",
		},
		"Int64/-0x1": {
			v:  int64(-0x1),
			hs: "ff",
		},
		"Int64/-0x20": {
			v:  int64(-0x20),
			hs: "e0",
		},
		"Int64/-0x21": {
			v:  int64(-0x21),
			hs: "d0df",
		},
		"Int64/-0x80": {
			v:  int64(-0x80),
			hs: "d080",
		},
		"Int64/-0x81": {
			v:  int64(-0x81),
			hs: "d1ff7f",
		},
		"Int64/-0x8000": {
			v:  int64(-0x8000),
			hs: "d18000",
		},
		"Int64/-0x8001": {
			v:  int64(-0x8001),
			hs: "d2ffff7fff",
		},
		"Int64/-0x80000000": {
			v:  int64(-0x80000000),
			hs: "d280000000",
		},
		"Int64/-0x80000001": {
			v:  int64(-0x80000001),
			hs: "d3ffffffff7fffffff",
		},
		"Int64/-0x8000000000000000": {
			v:  int64(-0x8000000000000000),
			hs: "d38000000000000000",
		},
		"Uint64/0x0": {
			v:  uint64(0x0),
			hs: "00",
		},
		"Uint64/0x1": {
			v:  uint64(0x1),
			hs: "01",
		},
		"Uint64/0x7f": {
			v:  uint64(0x7f),
			hs: "7f",
		},
		"Uint64/0xff": {
			v:  uint64(0xff),
			hs: "ccff",
		},
		"Uint64/0x100": {
			v:  uint64(0x100),
			hs: "cd0100",
		},
		"Uint64/0xffff": {
			v:  uint64(0xffff),
			hs: "cdffff",
		},
		"Uint64/0x10000": {
			v:  uint64(0x10000),
			hs: "ce00010000",
		},
		"Uint64/0xffffffff": {
			v:  uint64(0xffffffff),
			hs: "ceffffffff",
		},
		"Uint64/0x100000000": {
			v:  uint64(0x100000000),
			hs: "cf0000000100000000",
		},
		"Uint64/0xffffffffffffffff": {
			v:  uint64(0xffffffffffffffff),
			hs: "cfffffffffffffffff",
		},
		"Nil": {
			v:  nil,
			hs: "c0",
		},
		"True": {
			v:  true,
			hs: "c3",
		},
		"False": {
			v:  false,
			hs: "c2",
		},
		"Float64/1.23456": {
			v:  float64(1.23456),
			hs: "cb3ff3c0c1fc8f3238",
		},
		"MapLen/0x0": {
			v:  mapLen(0x0),
			hs: "80",
		},
		"MapLen/0x1": {
			v:  mapLen(0x1),
			hs: "81",
		},
		"MapLen/0xf": {
			v:  mapLen(0xf),
			hs: "8f",
		},
		"MapLen/0x10": {
			v:  mapLen(0x10),
			hs: "de0010",
		},
		"MapLen/0xffff": {
			v:  mapLen(0xffff),
			hs: "deffff",
		},
		"MapLen/0x10000": {
			v:  mapLen(0x10000),
			hs: "df00010000",
		},
		"MapLen/0xffffffff": {
			v:  mapLen(0xffffffff),
			hs: "dfffffffff",
		},
		"ArrayLen/0x0": {
			v:  arrayLen(0x0),
			hs: "90",
		},
		"ArrayLen/0x1": {
			v:  arrayLen(0x1),
			hs: "91",
		},
		"ArrayLen/0xf": {
			v:  arrayLen(0xf),
			hs: "9f",
		},
		"ArrayLen/0x10": {
			v:  arrayLen(0x10),
			hs: "dc0010",
		},
		"ArrayLen/0xffff": {
			v:  arrayLen(0xffff),
			hs: "dcffff",
		},
		"ArrayLen/0x10000": {
			v:  arrayLen(0x10000),
			hs: "dd00010000",
		},
		"ArrayLen/0xffffffff": {
			v:  arrayLen(0xffffffff),
			hs: "ddffffffff",
		},
		"String/Empty": {
			v:  "",
			hs: "a0",
		},
		"String/1": {
			v:  "1",
			hs: "a131",
		},
		"String/1234567890123456789012345678901": {
			v:  "1234567890123456789012345678901",
			hs: "bf31323334353637383930313233343536373839303132333435363738393031",
		},
		"String/12345678901234567890123456789012": {
			v:  "12345678901234567890123456789012",
			hs: "d9203132333435363738393031323334353637383930313233343536373839303132",
		},
		"Bytes/Empty": {
			v:  []byte(""),
			hs: "c400",
		},
		"Bytes/1": {
			v:  []byte("1"),
			hs: "c40131",
		},
		"Extension/1/Empty": {
			v:  extension{1, ""},
			hs: "c70001",
		},
		"Extension/2/1": {
			v:  extension{2, "1"},
			hs: "d40231",
		},
		"Extension/3/12": {
			v:  extension{3, "12"},
			hs: "d5033132",
		},
		"Extension/4/1234": {
			v:  extension{4, "1234"},
			hs: "d60431323334",
		},
		"Extension/5/12345678": {
			v:  extension{5, "12345678"},
			hs: "d7053132333435363738",
		},
		"Extension/6/1234567890123456": {
			v:  extension{6, "1234567890123456"},
			hs: "d80631323334353637383930313233343536",
		},
		"Extension/7/12345678901234567": {
			v:  extension{7, "12345678901234567"},
			hs: "c711073132333435363738393031323334353637",
		},
	}
	for name, tt := range packTests {
		tt := tt
		t.Run(name, func(t *testing.T) {
			t.Parallel()

			var arg string
			switch reflect.ValueOf(tt.v).Kind() {
			case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
				arg = fmt.Sprintf("%T %x", tt.v, tt.v)
			default:
				arg = fmt.Sprintf("%T %v", tt.v, tt.v)
			}

			p, err := pack(tt.v)
			if err != nil {
				t.Fatalf("pack %s returned error %v", arg, err)
			}
			h := hex.EncodeToString(p)
			if h != tt.hs {
				t.Fatalf("pack %s returned %s, want %s", arg, h, tt.hs)
			}
		})
	}
}
<commit_msg>msgpack: format and reorder packTests<commit_after>package msgpack

import (
	"encoding/hex"
	"fmt"
	"reflect"
	"testing"
)

func TestPack(t *testing.T) {
	t.Parallel()

	packTests := map[string]struct {
		// Expected value
		v interface{}
		// Hex encodings of typ, v
		hs string
	}{
		"Bool/True": {
			v:  true,
			hs: "c3",
		},
		"Bool/False": {
			v:  false,
			hs: "c2",
		},
		"Int64/0x0": {
			v:  int64(0x0),
			hs: "00",
		},
		"Int64/0x1": {
			v:  int64(0x1),
			hs: "01",
		},
		"Int64/0x7f": {
			v:  int64(0x7f),
			hs: "7f",
		},
		"Int64/0x80": {
			v:  int64(0x80),
			hs: "cc80",
		},
		"Int64/0x7fff": {
			v:  int64(0x7fff),
			hs: "cd7fff",
		},
		"Int64/0x8000": {
			v:  int64(0x8000),
			hs: "cd8000",
		},
		"Int64/0x7fffffff": {
			v:  int64(0x7fffffff),
			hs: "ce7fffffff",
		},
		"Int64/0x80000000": {
			v:  int64(0x80000000),
			hs: "ce80000000",
		},
		"Int64/0x7fffffffffffffff": {
			v:  int64(0x7fffffffffffffff),
			hs: "cf7fffffffffffffff",
		},
		"Int64/-0x1": {
			v:  int64(-0x1),
			hs: "ff",
		},
		"Int64/-0x20": {
			v:  int64(-0x20),
			hs: "e0",
		},
		"Int64/-0x21": {
			v:  int64(-0x21),
			hs: "d0df",
		},
		"Int64/-0x80": {
			v:  int64(-0x80),
			hs: "d080",
		},
		"Int64/-0x81": {
			v:  int64(-0x81),
			hs: "d1ff7f",
		},
		"Int64/-0x8000": {
			v:  int64(-0x8000),
			hs: "d18000",
		},
		"Int64/-0x8001": {
			v:  int64(-0x8001),
			hs: "d2ffff7fff",
		},
		"Int64/-0x80000000": {
			v:  int64(-0x80000000),
			hs: "d280000000",
		},
		"Int64/-0x80000001": {
			v:  int64(-0x80000001),
			hs: "d3ffffffff7fffffff",
		},
		"Int64/-0x8000000000000000": {
			v:  int64(-0x8000000000000000),
			hs: "d38000000000000000",
		},
		"Uint64/0x0": {
			v:  uint64(0x0),
			hs: "00",
		},
		"Uint64/0x1": {
			v:  uint64(0x1),
			hs: "01",
		},
		"Uint64/0x7f": {
			v:  uint64(0x7f),
			hs: "7f",
		},
		"Uint64/0xff": {
			v:  uint64(0xff),
			hs: "ccff",
		},
		"Uint64/0x100": {
			v:  uint64(0x100),
			hs: "cd0100",
		},
		"Uint64/0xffff": {
			v:  uint64(0xffff),
			hs: "cdffff",
		},
		"Uint64/0x10000": {
			v:  uint64(0x10000),
			hs: "ce00010000",
		},
		"Uint64/0xffffffff": {
			v:  uint64(0xffffffff),
			hs: "ceffffffff",
		},
		"Uint64/0x100000000": {
			v:  uint64(0x100000000),
			hs: "cf0000000100000000",
		},
		"Uint64/0xffffffffffffffff": {
			v:  uint64(0xffffffffffffffff),
			hs: "cfffffffffffffffff",
		},
		"Float64/1.23456": {
			v:  float64(1.23456),
			hs: "cb3ff3c0c1fc8f3238",
		},
		"String/Empty": {
			v:  string(""),
			hs: "a0",
		},
		"String/1": {
			v:  string("1"),
			hs: "a131",
		},
		"String/1234567890123456789012345678901": {
			v:  string("1234567890123456789012345678901"),
			hs: "bf31323334353637383930313233343536373839303132333435363738393031",
		},
		"String/12345678901234567890123456789012": {
			v:  string("12345678901234567890123456789012"),
			hs: "d9203132333435363738393031323334353637383930313233343536373839303132",
		},
		"Binary/Empty": {
			v:  []byte(""),
			hs: "c400",
		},
		"Binary/1": {
			v:  []byte("1"),
			hs: "c40131",
		},
		"MapLen/0x0": {
			v:  mapLen(0x0),
			hs: "80",
		},
		"MapLen/0x1": {
			v:  mapLen(0x1),
			hs: "81",
		},
		"MapLen/0xf": {
			v:  mapLen(0xf),
			hs: "8f",
		},
		"MapLen/0x10": {
			v:  mapLen(0x10),
			hs: "de0010",
		},
		"MapLen/0xffff": {
			v:  mapLen(0xffff),
			hs: "deffff",
		},
		"MapLen/0x10000": {
			v:  mapLen(0x10000),
			hs: "df00010000",
		},
		"MapLen/0xffffffff": {
			v:  mapLen(0xffffffff),
			hs: "dfffffffff",
		},
		"ArrayLen/0x0": {
			v:  arrayLen(0x0),
			hs: "90",
		},
		"ArrayLen/0x1": {
			v:  arrayLen(0x1),
			hs: "91",
		},
		"ArrayLen/0xf": {
			v:  arrayLen(0xf),
			hs: "9f",
		},
		"ArrayLen/0x10": {
			v:  arrayLen(0x10),
			hs: "dc0010",
		},
		"ArrayLen/0xffff": {
			v:  arrayLen(0xffff),
			hs: "dcffff",
		},
		"ArrayLen/0x10000": {
			v:  arrayLen(0x10000),
			hs: "dd00010000",
		},
		"ArrayLen/0xffffffff": {
			v:  arrayLen(0xffffffff),
			hs: "ddffffffff",
		},
		"Extension/1/Empty": {
			v:  extension{1, ""},
			hs: "c70001",
		},
		"Extension/2/1": {
			v:  extension{2, "1"},
			hs: "d40231",
		},
		"Extension/3/12": {
			v:  extension{3, "12"},
			hs: "d5033132",
		},
		"Extension/4/1234": {
			v:  extension{4, "1234"},
			hs: "d60431323334",
		},
		"Extension/5/12345678": {
			v:  extension{5, "12345678"},
			hs: "d7053132333435363738",
		},
		"Extension/6/1234567890123456": {
			v:  extension{6, "1234567890123456"},
			hs: "d80631323334353637383930313233343536",
		},
		"Extension/7/12345678901234567": {
			v:  extension{7, "12345678901234567"},
			hs: "c711073132333435363738393031323334353637",
		},
		"Nil": {
			v:  nil,
			hs: "c0",
		},
	}
	for name, tt := range packTests {
		tt := tt
		t.Run(name, func(t *testing.T) {
			t.Parallel()

			var arg string
			switch reflect.ValueOf(tt.v).Kind() {
			case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
				arg = fmt.Sprintf("%T %x", tt.v, tt.v)
			default:
				arg = fmt.Sprintf("%T %v", tt.v, tt.v)
			}

			p, err := pack(tt.v)
			if err != nil {
				t.Fatalf("pack %s returned error %v", arg, err)
			}

			h := hex.EncodeToString(p)
			if h != tt.hs {
				t.Fatalf("pack %s returned %s, want %s", arg, h, tt.hs)
			}
		})
	}
}
<|endoftext|>


<commit_before>package textrazor

const (
	EntitiesExtractor  = "entities"
	WordsExtractor     = "words"
	RelationsExtractor = "relations"

	// for now just these three
	EntitiesWordsRelations = "entities,words,relations"
)

// JSON response structs for the "entities,words" extractors;
// with the help from https://mholt.github.io/json-to-go/.

type Word struct {
	Position         int    `json:"position"`
	StartingPos      int    `json:"startingPos"`
	EndingPos        int    `json:"endingPos"`
	Stem             string `json:"stem"`
	Lemma            string `json:"lemma"`
	Token            string `json:"token"`
	PartOfSpeech     string `json:"partOfSpeech"`
	ParentPosition   int    `json:"parentPosition,omitempty"`
	RelationToParent string `json:"relationToParent,omitempty"`
}

type Sentense struct {
	Position int    `json:"position"`
	Words    []Word `json:"words"`
}

type Entity struct {
	ID              int      `json:"id"`
	Type            []string `json:"type,omitempty"`
	MatchingTokens  []int    `json:"matchingTokens"`
	EntityID        string   `json:"entityId"`
	FreebaseTypes   []string `json:"freebaseTypes,omitempty"`
	ConfidenceScore float64  `json:"confidenceScore"`
	WikiLink        string   `json:"wikiLink"`
	MatchedText     string   `json:"matchedText"`
	FreebaseID      string   `json:"freebaseId,omitempty"`
	RelevanceScore  int      `json:"relevanceScore"`
	EntityEnglishID string   `json:"entityEnglishId"`
	StartingPos     int      `json:"startingPos"`
	EndingPos       int      `json:"endingPos"`
	WikidataID      string   `json:"wikidataId,omitempty"`
}

type Param struct {
	Relation      string `json:"relation"`
	WordPositions []int  `json:"wordPositions"`
}

type Relation struct {
	ID            int     `json:"id"`
	WordPositions []int   `json:"wordPositions"`
	Params        []Param `json:"params"`
}

type Property struct {
	ID                int   `json:"id"`
	WordPositions     []int `json:"wordPositions"`
	PropertyPositions []int `json:"propertyPositions"`
}

type Response struct {
	Sentences          []Sentense `json:"sentences"`
	Language           string     `json:"language"`
	LanguageIsReliable bool       `json:"languageIsReliable"`
	Entities           []Entity   `json:"entities"`
	Relations          []Relation `json:"relations"`
	Properties         []Property `json:"properties"`
}

type Result struct {
	Resp Response `json:"response"`
	Time float64  `json:"time"`
	Ok   bool     `json:"ok"`
}

// OneLine gives a pretty one-line output of the sentence
// formatted as Stanford Parser tagging
// (http://nlp.stanford.edu:8080/parser/index.jsp).
func (s *Sentense) OneLine() string {
	str := "$"
	for _, w := range s.Words {
		str += (" " + w.Token + "/" + w.PartOfSpeech)
	}
	return str
}

// Body is the JSON request payload.
type Body struct {
	Extractors string `json:"extractors"`
	Text       string `json:"text"`
}
<commit_msg>fix (nlp): sentence typo<commit_after>package textrazor

const (
	EntitiesExtractor  = "entities"
	WordsExtractor     = "words"
	RelationsExtractor = "relations"

	// for now just these three
	EntitiesWordsRelations = "entities,words,relations"
)

// JSON response structs for the "entities,words" extractors;
// with the help from https://mholt.github.io/json-to-go/.

type Word struct {
	Position         int    `json:"position"`
	StartingPos      int    `json:"startingPos"`
	EndingPos        int    `json:"endingPos"`
	Stem             string `json:"stem"`
	Lemma            string `json:"lemma"`
	Token            string `json:"token"`
	PartOfSpeech     string `json:"partOfSpeech"`
	ParentPosition   int    `json:"parentPosition,omitempty"`
	RelationToParent string `json:"relationToParent,omitempty"`
}

type Sentence struct {
	Position int    `json:"position"`
	Words    []Word `json:"words"`
}

type Entity struct {
	ID              int      `json:"id"`
	Type            []string `json:"type,omitempty"`
	MatchingTokens  []int    `json:"matchingTokens"`
	EntityID        string   `json:"entityId"`
	FreebaseTypes   []string `json:"freebaseTypes,omitempty"`
	ConfidenceScore float64  `json:"confidenceScore"`
	WikiLink        string   `json:"wikiLink"`
	MatchedText     string   `json:"matchedText"`
	FreebaseID      string   `json:"freebaseId,omitempty"`
	RelevanceScore  int      `json:"relevanceScore"`
	EntityEnglishID string   `json:"entityEnglishId"`
	StartingPos     int      `json:"startingPos"`
	EndingPos       int      `json:"endingPos"`
	WikidataID      string   `json:"wikidataId,omitempty"`
}

type Param struct {
	Relation      string `json:"relation"`
	WordPositions []int  `json:"wordPositions"`
}

type Relation struct {
	ID            int     `json:"id"`
	WordPositions []int   `json:"wordPositions"`
	Params        []Param `json:"params"`
}

type Property struct {
	ID                int   `json:"id"`
	WordPositions     []int `json:"wordPositions"`
	PropertyPositions []int `json:"propertyPositions"`
}

type Response struct {
	Sentences          []Sentence `json:"sentences"`
	Language           string     `json:"language"`
	LanguageIsReliable bool       `json:"languageIsReliable"`
	Entities           []Entity   `json:"entities"`
	Relations          []Relation `json:"relations"`
	Properties         []Property `json:"properties"`
}

type Result struct {
	Resp Response `json:"response"`
	Time float64  `json:"time"`
	Ok   bool     `json:"ok"`
}

// OneLine gives a pretty one-line output of the sentence
// formatted as Stanford Parser tagging
// (http://nlp.stanford.edu:8080/parser/index.jsp).
func (s *Sentence) OneLine() string {
	str := "$"
	for _, w := range s.Words {
		str += (" " + w.Token + "/" + w.PartOfSpeech)
	}
	return str
}

// Body is the JSON request payload.
type Body struct {
	Extractors string `json:"extractors"`
	Text       string `json:"text"`
}
<|endoftext|>