<commit_before>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package containerd

import (
	"context"
	"encoding/json"
	"os"
	"path/filepath"
	"strings"

	"github.com/containerd/containerd/api/services/tasks/v1"
	"github.com/containerd/containerd/api/types"
	tasktypes "github.com/containerd/containerd/api/types/task"
	"github.com/containerd/containerd/cio"
	"github.com/containerd/containerd/containers"
	"github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/images"
	"github.com/containerd/containerd/oci"
	"github.com/containerd/containerd/runtime/v2/runc/options"
	"github.com/containerd/typeurl"
	prototypes "github.com/gogo/protobuf/types"
	ver "github.com/opencontainers/image-spec/specs-go"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/opencontainers/selinux/go-selinux/label"
	"github.com/pkg/errors"
)

const (
	checkpointImageNameLabel       = "org.opencontainers.image.ref.name"
	checkpointRuntimeNameLabel     = "io.containerd.checkpoint.runtime"
	checkpointSnapshotterNameLabel = "io.containerd.checkpoint.snapshotter"
)

// Container is a metadata object for container resources and task creation
type Container interface {
	// ID identifies the container
	ID() string
	// Info returns the underlying container record type
	Info(context.Context, ...InfoOpts) (containers.Container, error)
	// Delete removes the container
	Delete(context.Context, ...DeleteOpts) error
	// NewTask creates a new task based on the container metadata
	NewTask(context.Context, cio.Creator, ...NewTaskOpts) (Task, error)
	// Spec returns the OCI runtime specification
	Spec(context.Context) (*oci.Spec, error)
	// Task returns the current task for the container
	//
	// If cio.Attach options are passed the client will reattach to the IO for the running
	// task. If no task exists for the container a NotFound error is returned
	//
	// Clients must make sure that only one reader is attached to the task and consuming
	// the output from the task's fifos
	Task(context.Context, cio.Attach) (Task, error)
	// Image returns the image that the container is based on
	Image(context.Context) (Image, error)
	// Labels returns the labels set on the container
	Labels(context.Context) (map[string]string, error)
	// SetLabels sets the provided labels for the container and returns the final label set
	SetLabels(context.Context, map[string]string) (map[string]string, error)
	// Extensions returns the extensions set on the container
	Extensions(context.Context) (map[string]prototypes.Any, error)
	// Update a container
	Update(context.Context, ...UpdateContainerOpts) error
	// Checkpoint creates a checkpoint image of the current container
	Checkpoint(context.Context, string, ...CheckpointOpts) (Image, error)
}

func containerFromRecord(client *Client, c containers.Container) *container {
	return &container{
		client:   client,
		id:       c.ID,
		metadata: c,
	}
}

var _ = (Container)(&container{})

type container struct {
	client   *Client
	id       string
	metadata containers.Container
}

// ID returns the container's unique id
func (c *container) ID() string {
	return c.id
}

func (c *container) Info(ctx context.Context, opts ...InfoOpts) (containers.Container, error) {
	i := &InfoConfig{
		// default to refreshing the container's local metadata
		Refresh: true,
	}
	for _, o := range opts {
		o(i)
	}
	if i.Refresh {
		metadata, err := c.get(ctx)
		if err != nil {
			return c.metadata, err
		}
		c.metadata = metadata
	}
	return c.metadata, nil
}

func (c *container) Extensions(ctx context.Context) (map[string]prototypes.Any, error) {
	r, err := c.get(ctx)
	if err != nil {
		return nil, err
	}
	return r.Extensions, nil
}

func (c *container) Labels(ctx context.Context) (map[string]string, error) {
	r, err := c.get(ctx)
	if err != nil {
		return nil, err
	}
	return r.Labels, nil
}

func (c *container) SetLabels(ctx context.Context, labels map[string]string) (map[string]string, error) {
	container := containers.Container{
		ID:     c.id,
		Labels: labels,
	}

	var paths []string
	// mask off paths so we only muck with the labels encountered in labels.
	// Labels not in the passed in argument will be left alone.
	for k := range labels {
		paths = append(paths, strings.Join([]string{"labels", k}, "."))
	}

	r, err := c.client.ContainerService().Update(ctx, container, paths...)
	if err != nil {
		return nil, err
	}
	return r.Labels, nil
}

// Spec returns the current OCI specification for the container
func (c *container) Spec(ctx context.Context) (*oci.Spec, error) {
	r, err := c.get(ctx)
	if err != nil {
		return nil, err
	}
	var s oci.Spec
	if err := json.Unmarshal(r.Spec.Value, &s); err != nil {
		return nil, err
	}
	return &s, nil
}

// Delete deletes an existing container
// an error is returned if the container has running tasks
func (c *container) Delete(ctx context.Context, opts ...DeleteOpts) error {
	if _, err := c.loadTask(ctx, nil); err == nil {
		return errors.Wrapf(errdefs.ErrFailedPrecondition, "cannot delete running task %v", c.id)
	}
	r, err := c.get(ctx)
	if err != nil {
		return err
	}
	for _, o := range opts {
		if err := o(ctx, c.client, r); err != nil {
			return err
		}
	}
	return c.client.ContainerService().Delete(ctx, c.id)
}

func (c *container) Task(ctx context.Context, attach cio.Attach) (Task, error) {
	return c.loadTask(ctx, attach)
}

// Image returns the image that the container is based on
func (c *container) Image(ctx context.Context) (Image, error) {
	r, err := c.get(ctx)
	if err != nil {
		return nil, err
	}
	if r.Image == "" {
		return nil, errors.Wrap(errdefs.ErrNotFound, "container not created from an image")
	}
	i, err := c.client.ImageService().Get(ctx, r.Image)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to get image %s for container", r.Image)
	}
	return NewImage(c.client, i), nil
}

func (c *container) NewTask(ctx context.Context, ioCreate cio.Creator, opts ...NewTaskOpts) (_ Task, err error) {
	i, err := ioCreate(c.id)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err != nil && i != nil {
			i.Cancel()
			i.Close()
		}
	}()
	cfg := i.Config()
	request := &tasks.CreateTaskRequest{
		ContainerID: c.id,
		Terminal:    cfg.Terminal,
		Stdin:       cfg.Stdin,
		Stdout:      cfg.Stdout,
		Stderr:      cfg.Stderr,
	}
	r, err := c.get(ctx)
	if err != nil {
		return nil, err
	}
	if r.SnapshotKey != "" {
		if r.Snapshotter == "" {
			return nil, errors.Wrapf(errdefs.ErrInvalidArgument, "unable to resolve rootfs mounts without snapshotter on container")
		}

		// get the rootfs from the snapshotter and add it to the request
		s, err := c.client.getSnapshotter(ctx, r.Snapshotter)
		if err != nil {
			return nil, err
		}
		mounts, err := s.Mounts(ctx, r.SnapshotKey)
		if err != nil {
			return nil, err
		}
		spec, err := c.Spec(ctx)
		if err != nil {
			return nil, err
		}
		for _, m := range mounts {
			if spec.Linux != nil && spec.Linux.MountLabel != "" {
				context := label.FormatMountLabel("", spec.Linux.MountLabel)
				if context != "" {
					m.Options = append(m.Options, context)
				}
			}
			request.Rootfs = append(request.Rootfs, &types.Mount{
				Type:    m.Type,
				Source:  m.Source,
				Options: m.Options,
			})
		}
	}
	info := TaskInfo{
		runtime: r.Runtime.Name,
	}
	for _, o := range opts {
		if err := o(ctx, c.client, &info); err != nil {
			return nil, err
		}
	}
	if info.RootFS != nil {
		for _, m := range info.RootFS {
			request.Rootfs = append(request.Rootfs, &types.Mount{
				Type:    m.Type,
				Source:  m.Source,
				Options: m.Options,
			})
		}
	}
	if info.Options != nil {
		any, err := typeurl.MarshalAny(info.Options)
		if err != nil {
			return nil, err
		}
		request.Options = any
	}
	t := &task{
		client: c.client,
		io:     i,
		id:     c.id,
	}
	if info.Checkpoint != nil {
		request.Checkpoint = info.Checkpoint
	}
	response, err := c.client.TaskService().Create(ctx, request)
	if err != nil {
		return nil, errdefs.FromGRPC(err)
	}
	t.pid = response.Pid
	return t, nil
}

func (c *container) Update(ctx context.Context, opts ...UpdateContainerOpts) error {
	// fetch the current container config before updating it
	r, err := c.get(ctx)
	if err != nil {
		return err
	}
	for _, o := range opts {
		if err := o(ctx, c.client, &r); err != nil {
			return err
		}
	}
	if _, err := c.client.ContainerService().Update(ctx, r); err != nil {
		return errdefs.FromGRPC(err)
	}
	return nil
}

func (c *container) Checkpoint(ctx context.Context, ref string, opts ...CheckpointOpts) (Image, error) {
	index := &ocispec.Index{
		Versioned: ver.Versioned{
			SchemaVersion: 2,
		},
		Annotations: make(map[string]string),
	}
	copts := &options.CheckpointOptions{
		Exit:                false,
		OpenTcp:             false,
		ExternalUnixSockets: false,
		Terminal:            false,
		FileLocks:           true,
		EmptyNamespaces:     nil,
	}
	info, err := c.Info(ctx)
	if err != nil {
		return nil, err
	}

	img, err := c.Image(ctx)
	if err != nil {
		return nil, err
	}

	ctx, done, err := c.client.WithLease(ctx)
	if err != nil {
		return nil, err
	}
	defer done(ctx)

	// add image name to manifest
	index.Annotations[checkpointImageNameLabel] = img.Name()
	// add runtime info to index
	index.Annotations[checkpointRuntimeNameLabel] = info.Runtime.Name
	// add snapshotter info to index
	index.Annotations[checkpointSnapshotterNameLabel] = info.Snapshotter

	// process remaining opts
	for _, o := range opts {
		if err := o(ctx, c.client, &info, index, copts); err != nil {
			err = errdefs.FromGRPC(err)
			if !errdefs.IsAlreadyExists(err) {
				return nil, err
			}
		}
	}

	desc, err := writeIndex(ctx, index, c.client, c.ID()+"index")
	if err != nil {
		return nil, err
	}
	i := images.Image{
		Name:   ref,
		Target: desc,
	}
	checkpoint, err := c.client.ImageService().Create(ctx, i)
	if err != nil {
		return nil, err
	}

	return NewImage(c.client, checkpoint), nil
}

func (c *container) loadTask(ctx context.Context, ioAttach cio.Attach) (Task, error) {
	response, err := c.client.TaskService().Get(ctx, &tasks.GetRequest{
		ContainerID: c.id,
	})
	if err != nil {
		err = errdefs.FromGRPC(err)
		if errdefs.IsNotFound(err) {
			return nil, errors.Wrapf(err, "no running task found")
		}
		return nil, err
	}
	var i cio.IO
	if ioAttach != nil && response.Process.Status != tasktypes.StatusUnknown {
		// Do not attach IO for task in unknown state, because there
		// are no fifo paths anyway.
		if i, err = attachExistingIO(response, ioAttach); err != nil {
			return nil, err
		}
	}
	t := &task{
		client: c.client,
		io:     i,
		id:     response.Process.ID,
		pid:    response.Process.Pid,
	}
	return t, nil
}

func (c *container) get(ctx context.Context) (containers.Container, error) {
	return c.client.ContainerService().Get(ctx, c.id)
}

// get the existing fifo paths from the task information stored by the daemon
func attachExistingIO(response *tasks.GetResponse, ioAttach cio.Attach) (cio.IO, error) {
	fifoSet := loadFifos(response)
	return ioAttach(fifoSet)
}

// loadFifos loads the containers fifos
func loadFifos(response *tasks.GetResponse) *cio.FIFOSet {
	path := getFifoDir([]string{
		response.Process.Stdin,
		response.Process.Stdout,
		response.Process.Stderr,
	})
	closer := func() error {
		return os.RemoveAll(path)
	}
	return cio.NewFIFOSet(cio.Config{
		Stdin:    response.Process.Stdin,
		Stdout:   response.Process.Stdout,
		Stderr:   response.Process.Stderr,
		Terminal: response.Process.Terminal,
	}, closer)
}

// getFifoDir looks for any non-empty path for a stdio fifo
// and returns the dir for where it is located
func getFifoDir(paths []string) string {
	for _, p := range paths {
		if p != "" {
			return filepath.Dir(p)
		}
	}
	return ""
}
<commit_msg>Correct logic of FIFO cleanup<commit_after>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package containerd

import (
	"context"
	"encoding/json"
	"os"
	"path/filepath"
	"strings"

	"github.com/containerd/containerd/api/services/tasks/v1"
	"github.com/containerd/containerd/api/types"
	tasktypes "github.com/containerd/containerd/api/types/task"
	"github.com/containerd/containerd/cio"
	"github.com/containerd/containerd/containers"
	"github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/images"
	"github.com/containerd/containerd/oci"
	"github.com/containerd/containerd/runtime/v2/runc/options"
	"github.com/containerd/containerd/sys"
	"github.com/containerd/typeurl"
	prototypes "github.com/gogo/protobuf/types"
	ver "github.com/opencontainers/image-spec/specs-go"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/opencontainers/selinux/go-selinux/label"
	"github.com/pkg/errors"
)

const (
	checkpointImageNameLabel       = "org.opencontainers.image.ref.name"
	checkpointRuntimeNameLabel     = "io.containerd.checkpoint.runtime"
	checkpointSnapshotterNameLabel = "io.containerd.checkpoint.snapshotter"
)

// Container is a metadata object for container resources and task creation
type Container interface {
	// ID identifies the container
	ID() string
	// Info returns the underlying container record type
	Info(context.Context, ...InfoOpts) (containers.Container, error)
	// Delete removes the container
	Delete(context.Context, ...DeleteOpts) error
	// NewTask creates a new task based on the container metadata
	NewTask(context.Context, cio.Creator, ...NewTaskOpts) (Task, error)
	// Spec returns the OCI runtime specification
	Spec(context.Context) (*oci.Spec, error)
	// Task returns the current task for the container
	//
	// If cio.Attach options are passed the client will reattach to the IO for the running
	// task. If no task exists for the container a NotFound error is returned
	//
	// Clients must make sure that only one reader is attached to the task and consuming
	// the output from the task's fifos
	Task(context.Context, cio.Attach) (Task, error)
	// Image returns the image that the container is based on
	Image(context.Context) (Image, error)
	// Labels returns the labels set on the container
	Labels(context.Context) (map[string]string, error)
	// SetLabels sets the provided labels for the container and returns the final label set
	SetLabels(context.Context, map[string]string) (map[string]string, error)
	// Extensions returns the extensions set on the container
	Extensions(context.Context) (map[string]prototypes.Any, error)
	// Update a container
	Update(context.Context, ...UpdateContainerOpts) error
	// Checkpoint creates a checkpoint image of the current container
	Checkpoint(context.Context, string, ...CheckpointOpts) (Image, error)
}

func containerFromRecord(client *Client, c containers.Container) *container {
	return &container{
		client:   client,
		id:       c.ID,
		metadata: c,
	}
}

var _ = (Container)(&container{})

type container struct {
	client   *Client
	id       string
	metadata containers.Container
}

// ID returns the container's unique id
func (c *container) ID() string {
	return c.id
}

func (c *container) Info(ctx context.Context, opts ...InfoOpts) (containers.Container, error) {
	i := &InfoConfig{
		// default to refreshing the container's local metadata
		Refresh: true,
	}
	for _, o := range opts {
		o(i)
	}
	if i.Refresh {
		metadata, err := c.get(ctx)
		if err != nil {
			return c.metadata, err
		}
		c.metadata = metadata
	}
	return c.metadata, nil
}

func (c *container) Extensions(ctx context.Context) (map[string]prototypes.Any, error) {
	r, err := c.get(ctx)
	if err != nil {
		return nil, err
	}
	return r.Extensions, nil
}

func (c *container) Labels(ctx context.Context) (map[string]string, error) {
	r, err := c.get(ctx)
	if err != nil {
		return nil, err
	}
	return r.Labels, nil
}

func (c *container) SetLabels(ctx context.Context, labels map[string]string) (map[string]string, error) {
	container := containers.Container{
		ID:     c.id,
		Labels: labels,
	}

	var paths []string
	// mask off paths so we only muck with the labels encountered in labels.
	// Labels not in the passed in argument will be left alone.
	for k := range labels {
		paths = append(paths, strings.Join([]string{"labels", k}, "."))
	}

	r, err := c.client.ContainerService().Update(ctx, container, paths...)
	if err != nil {
		return nil, err
	}
	return r.Labels, nil
}

// Spec returns the current OCI specification for the container
func (c *container) Spec(ctx context.Context) (*oci.Spec, error) {
	r, err := c.get(ctx)
	if err != nil {
		return nil, err
	}
	var s oci.Spec
	if err := json.Unmarshal(r.Spec.Value, &s); err != nil {
		return nil, err
	}
	return &s, nil
}

// Delete deletes an existing container
// an error is returned if the container has running tasks
func (c *container) Delete(ctx context.Context, opts ...DeleteOpts) error {
	if _, err := c.loadTask(ctx, nil); err == nil {
		return errors.Wrapf(errdefs.ErrFailedPrecondition, "cannot delete running task %v", c.id)
	}
	r, err := c.get(ctx)
	if err != nil {
		return err
	}
	for _, o := range opts {
		if err := o(ctx, c.client, r); err != nil {
			return err
		}
	}
	return c.client.ContainerService().Delete(ctx, c.id)
}

func (c *container) Task(ctx context.Context, attach cio.Attach) (Task, error) {
	return c.loadTask(ctx, attach)
}

// Image returns the image that the container is based on
func (c *container) Image(ctx context.Context) (Image, error) {
	r, err := c.get(ctx)
	if err != nil {
		return nil, err
	}
	if r.Image == "" {
		return nil, errors.Wrap(errdefs.ErrNotFound, "container not created from an image")
	}
	i, err := c.client.ImageService().Get(ctx, r.Image)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to get image %s for container", r.Image)
	}
	return NewImage(c.client, i), nil
}

func (c *container) NewTask(ctx context.Context, ioCreate cio.Creator, opts ...NewTaskOpts) (_ Task, err error) {
	i, err := ioCreate(c.id)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err != nil && i != nil {
			i.Cancel()
			i.Close()
		}
	}()
	cfg := i.Config()
	request := &tasks.CreateTaskRequest{
		ContainerID: c.id,
		Terminal:    cfg.Terminal,
		Stdin:       cfg.Stdin,
		Stdout:      cfg.Stdout,
		Stderr:      cfg.Stderr,
	}
	r, err := c.get(ctx)
	if err != nil {
		return nil, err
	}
	if r.SnapshotKey != "" {
		if r.Snapshotter == "" {
			return nil, errors.Wrapf(errdefs.ErrInvalidArgument, "unable to resolve rootfs mounts without snapshotter on container")
		}

		// get the rootfs from the snapshotter and add it to the request
		s, err := c.client.getSnapshotter(ctx, r.Snapshotter)
		if err != nil {
			return nil, err
		}
		mounts, err := s.Mounts(ctx, r.SnapshotKey)
		if err != nil {
			return nil, err
		}
		spec, err := c.Spec(ctx)
		if err != nil {
			return nil, err
		}
		for _, m := range mounts {
			if spec.Linux != nil && spec.Linux.MountLabel != "" {
				context := label.FormatMountLabel("", spec.Linux.MountLabel)
				if context != "" {
					m.Options = append(m.Options, context)
				}
			}
			request.Rootfs = append(request.Rootfs, &types.Mount{
				Type:    m.Type,
				Source:  m.Source,
				Options: m.Options,
			})
		}
	}
	info := TaskInfo{
		runtime: r.Runtime.Name,
	}
	for _, o := range opts {
		if err := o(ctx, c.client, &info); err != nil {
			return nil, err
		}
	}
	if info.RootFS != nil {
		for _, m := range info.RootFS {
			request.Rootfs = append(request.Rootfs, &types.Mount{
				Type:    m.Type,
				Source:  m.Source,
				Options: m.Options,
			})
		}
	}
	if info.Options != nil {
		any, err := typeurl.MarshalAny(info.Options)
		if err != nil {
			return nil, err
		}
		request.Options = any
	}
	t := &task{
		client: c.client,
		io:     i,
		id:     c.id,
	}
	if info.Checkpoint != nil {
		request.Checkpoint = info.Checkpoint
	}
	response, err := c.client.TaskService().Create(ctx, request)
	if err != nil {
		return nil, errdefs.FromGRPC(err)
	}
	t.pid = response.Pid
	return t, nil
}

func (c *container) Update(ctx context.Context, opts ...UpdateContainerOpts) error {
	// fetch the current container config before updating it
	r, err := c.get(ctx)
	if err != nil {
		return err
	}
	for _, o := range opts {
		if err := o(ctx, c.client, &r); err != nil {
			return err
		}
	}
	if _, err := c.client.ContainerService().Update(ctx, r); err != nil {
		return errdefs.FromGRPC(err)
	}
	return nil
}

func (c *container) Checkpoint(ctx context.Context, ref string, opts ...CheckpointOpts) (Image, error) {
	index := &ocispec.Index{
		Versioned: ver.Versioned{
			SchemaVersion: 2,
		},
		Annotations: make(map[string]string),
	}
	copts := &options.CheckpointOptions{
		Exit:                false,
		OpenTcp:             false,
		ExternalUnixSockets: false,
		Terminal:            false,
		FileLocks:           true,
		EmptyNamespaces:     nil,
	}
	info, err := c.Info(ctx)
	if err != nil {
		return nil, err
	}

	img, err := c.Image(ctx)
	if err != nil {
		return nil, err
	}

	ctx, done, err := c.client.WithLease(ctx)
	if err != nil {
		return nil, err
	}
	defer done(ctx)

	// add image name to manifest
	index.Annotations[checkpointImageNameLabel] = img.Name()
	// add runtime info to index
	index.Annotations[checkpointRuntimeNameLabel] = info.Runtime.Name
	// add snapshotter info to index
	index.Annotations[checkpointSnapshotterNameLabel] = info.Snapshotter

	// process remaining opts
	for _, o := range opts {
		if err := o(ctx, c.client, &info, index, copts); err != nil {
			err = errdefs.FromGRPC(err)
			if !errdefs.IsAlreadyExists(err) {
				return nil, err
			}
		}
	}

	desc, err := writeIndex(ctx, index, c.client, c.ID()+"index")
	if err != nil {
		return nil, err
	}
	i := images.Image{
		Name:   ref,
		Target: desc,
	}
	checkpoint, err := c.client.ImageService().Create(ctx, i)
	if err != nil {
		return nil, err
	}

	return NewImage(c.client, checkpoint), nil
}

func (c *container) loadTask(ctx context.Context, ioAttach cio.Attach) (Task, error) {
	response, err := c.client.TaskService().Get(ctx, &tasks.GetRequest{
		ContainerID: c.id,
	})
	if err != nil {
		err = errdefs.FromGRPC(err)
		if errdefs.IsNotFound(err) {
			return nil, errors.Wrapf(err, "no running task found")
		}
		return nil, err
	}
	var i cio.IO
	if ioAttach != nil && response.Process.Status != tasktypes.StatusUnknown {
		// Do not attach IO for task in unknown state, because there
		// are no fifo paths anyway.
		if i, err = attachExistingIO(response, ioAttach); err != nil {
			return nil, err
		}
	}
	t := &task{
		client: c.client,
		io:     i,
		id:     response.Process.ID,
		pid:    response.Process.Pid,
	}
	return t, nil
}

func (c *container) get(ctx context.Context) (containers.Container, error) {
	return c.client.ContainerService().Get(ctx, c.id)
}

// get the existing fifo paths from the task information stored by the daemon
func attachExistingIO(response *tasks.GetResponse, ioAttach cio.Attach) (cio.IO, error) {
	fifoSet := loadFifos(response)
	return ioAttach(fifoSet)
}

// loadFifos loads the containers fifos
func loadFifos(response *tasks.GetResponse) *cio.FIFOSet {
	fifos := []string{
		response.Process.Stdin,
		response.Process.Stdout,
		response.Process.Stderr,
	}
	closer := func() error {
		var (
			err  error
			dirs = map[string]struct{}{}
		)
		for _, fifo := range fifos {
			if isFifo, _ := sys.IsFifo(fifo); isFifo {
				if rerr := os.Remove(fifo); err == nil {
					err = rerr
				}
				dirs[filepath.Dir(fifo)] = struct{}{}
			}
		}
		for dir := range dirs {
			// we ignore errors here because we don't
			// want to remove the directory if it isn't
			// empty
			os.Remove(dir)
		}
		return err
	}

	return cio.NewFIFOSet(cio.Config{
		Stdin:    response.Process.Stdin,
		Stdout:   response.Process.Stdout,
		Stderr:   response.Process.Stderr,
		Terminal: response.Process.Terminal,
	}, closer)
}
<|endoftext|>


<commit_before>package sentry

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"reflect"
	"strconv"
	"strings"
	"time"

	"github.com/google/go-querystring/query"
	"github.com/peterhellberg/link"
)

const (
	defaultBaseURL = "https://sentry.io/api/"
	userAgent      = "go-sentry"

	// https://docs.sentry.io/api/ratelimits/
	headerRateLimit               = "X-Sentry-Rate-Limit-Limit"
	headerRateRemaining           = "X-Sentry-Rate-Limit-Remaining"
	headerRateReset               = "X-Sentry-Rate-Limit-Reset"
	headerRateConcurrentLimit     = "X-Sentry-Rate-Limit-ConcurrentLimit"
	headerRateConcurrentRemaining = "X-Sentry-Rate-Limit-ConcurrentRemaining"
)

var errNonNilContext = errors.New("context must be non-nil")

// Client for Sentry API.
type Client struct {
	client *http.Client

	// BaseURL for API requests.
	BaseURL *url.URL

	// User agent used when communicating with Sentry.
	UserAgent string

	// Latest rate limit
	rate Rate

	// Common struct used by all services.
	common service

	// Services
	IssueAlerts         *IssueAlertsService
	MetricAlerts        *MetricAlertsService
	OrganizationMembers *OrganizationMembersService
	Organizations       *OrganizationsService
	ProjectKeys         *ProjectKeysService
	ProjectOwnerships   *ProjectOwnershipsService
	ProjectPlugins      *ProjectPluginsService
	Projects            *ProjectsService
	Teams               *TeamsService
}

type service struct {
	client *Client
}

// NewClient returns a new Sentry API client.
// If a nil httpClient is provided, the http.DefaultClient will be used.
func NewClient(httpClient *http.Client) *Client {
	if httpClient == nil {
		httpClient = http.DefaultClient
	}
	baseURL, _ := url.Parse(defaultBaseURL)

	c := &Client{
		client:    httpClient,
		BaseURL:   baseURL,
		UserAgent: userAgent,
	}
	c.common.client = c
	c.IssueAlerts = (*IssueAlertsService)(&c.common)
	c.MetricAlerts = (*MetricAlertsService)(&c.common)
	c.OrganizationMembers = (*OrganizationMembersService)(&c.common)
	c.Organizations = (*OrganizationsService)(&c.common)
	c.ProjectKeys = (*ProjectKeysService)(&c.common)
	c.ProjectOwnerships = (*ProjectOwnershipsService)(&c.common)
	c.ProjectPlugins = (*ProjectPluginsService)(&c.common)
	c.Projects = (*ProjectsService)(&c.common)
	c.Teams = (*TeamsService)(&c.common)
	return c
}

// NewOnPremiseClient returns a new Sentry API client with the provided base URL.
// Note that the base URL must be in the format "http(s)://[hostname]/api/".
// If the base URL does not have the suffix "/api/", it will be added automatically.
// If a nil httpClient is provided, the http.DefaultClient will be used.
func NewOnPremiseClient(baseURL string, httpClient *http.Client) (*Client, error) {
	baseEndpoint, err := url.Parse(baseURL)
	if err != nil {
		return nil, err
	}

	if !strings.HasSuffix(baseEndpoint.Path, "/") {
		baseEndpoint.Path += "/"
	}
	if !strings.HasSuffix(baseEndpoint.Path, "/api/") {
		baseEndpoint.Path += "api/"
	}

	c := NewClient(httpClient)
	c.BaseURL = baseEndpoint
	return c, nil
}

type ListCursorParams struct {
	// A cursor, as given in the Link header.
	// If specified, the query continues the search using this cursor.
	Cursor string `url:"cursor,omitempty"`
}

func addQuery(s string, params interface{}) (string, error) {
	v := reflect.ValueOf(params)
	if v.Kind() == reflect.Ptr && v.IsNil() {
		return s, nil
	}

	u, err := url.Parse(s)
	if err != nil {
		return s, err
	}

	qs, err := query.Values(params)
	if err != nil {
		return s, err
	}

	u.RawQuery = qs.Encode()
	return u.String(), nil
}

// NewRequest creates an API request.
func (c *Client) NewRequest(method, urlRef string, body interface{}) (*http.Request, error) {
	if !strings.HasSuffix(c.BaseURL.Path, "/") {
		return nil, fmt.Errorf("BaseURL must have a trailing slash, but %q does not", c.BaseURL)
	}

	u, err := c.BaseURL.Parse(urlRef)
	if err != nil {
		return nil, err
	}

	var buf io.ReadWriter
	if body != nil {
		buf = &bytes.Buffer{}
		enc := json.NewEncoder(buf)
		enc.SetEscapeHTML(false)
		err := enc.Encode(body)
		if err != nil {
			return nil, err
		}
	}

	req, err := http.NewRequest(method, u.String(), buf)
	if err != nil {
		return nil, err
	}

	if body != nil {
		req.Header.Set("Content-Type", "application/json")
	}
	if c.UserAgent != "" {
		req.Header.Set("User-Agent", c.UserAgent)
	}
	return req, nil
}

// Response is a Sentry API response. This wraps the standard http.Response
// and provides convenient access to things like pagination links and rate limits.
type Response struct {
	*http.Response

	// For APIs that support cursor pagination, the following field will be populated
	// to point to the next page if more results are available.
	// Set ListCursorParams.Cursor to this value when calling the endpoint again.
	Cursor string

	Rate Rate
}

func newResponse(r *http.Response) *Response {
	response := &Response{Response: r}
	response.Rate = parseRate(r)
	response.populatePaginationCursor()
	return response
}

func (r *Response) populatePaginationCursor() {
	rels := link.ParseResponse(r.Response)
	if nextRel, ok := rels["next"]; ok && nextRel.Extra["results"] == "true" {
		r.Cursor = nextRel.Extra["cursor"]
	}
}

// parseRate parses the rate limit headers.
func parseRate(r *http.Response) Rate {
	var rate Rate
	if limit := r.Header.Get(headerRateLimit); limit != "" {
		rate.Limit, _ = strconv.Atoi(limit)
	}
	if remaining := r.Header.Get(headerRateRemaining); remaining != "" {
		rate.Remaining, _ = strconv.Atoi(remaining)
	}
	if reset := r.Header.Get(headerRateReset); reset != "" {
		if v, _ := strconv.ParseInt(reset, 10, 64); v != 0 {
			rate.Reset = time.Unix(v, 0).UTC()
		}
	}
	if concurrentLimit := r.Header.Get(headerRateConcurrentLimit); concurrentLimit != "" {
		rate.ConcurrentLimit, _ = strconv.Atoi(concurrentLimit)
	}
	if concurrentRemaining := r.Header.Get(headerRateConcurrentRemaining); concurrentRemaining != "" {
		rate.ConcurrentRemaining, _ = strconv.Atoi(concurrentRemaining)
	}

	return rate
}

func (c *Client) BareDo(ctx context.Context, req *http.Request) (*Response, error) {
	if ctx == nil {
		return nil, errNonNilContext
	}

	// Check rate limit
	if err := c.checkRateLimit(req); err != nil {
		return &Response{
			Response: err.Response,
			Rate:     err.Rate,
		}, err
	}

	resp, err := c.client.Do(req)
	if err != nil {
		// If we got an error, and the context has been canceled,
		// the context's error is probably more useful.
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		default:
			return nil, err
		}
	}

	response := newResponse(resp)

	c.rate = response.Rate

	err = CheckResponse(resp)

	return response, err
}

func (c *Client) checkRateLimit(req *http.Request) *RateLimitError {
	if !c.rate.Reset.IsZero() && c.rate.Remaining == 0 && time.Now().Before(c.rate.Reset) {
		resp := &http.Response{
			Status:     http.StatusText(http.StatusTooManyRequests),
			StatusCode: http.StatusTooManyRequests,
			Request:    req,
			Header:     http.Header{},
			Body:       ioutil.NopCloser(strings.NewReader("")),
		}
		return &RateLimitError{
			Rate:     c.rate,
			Response: resp,
			Detail: fmt.Sprintf("API rate limit of %v and concurrent limit of %v still exceeded until %v, not making remote request.",
				c.rate.Limit, c.rate.ConcurrentLimit, c.rate.Reset),
		}
	}
	return nil
}

func (c *Client) Do(ctx context.Context, req *http.Request, v interface{}) (*Response, error) {
	resp, err := c.BareDo(ctx, req)
	if err != nil {
		return resp, err
	}
	defer resp.Body.Close()

	switch v := v.(type) {
	case nil:
	case io.Writer:
		_, err = io.Copy(v, resp.Body)
	default:
		dec := json.NewDecoder(resp.Body)
		dec.UseNumber()
		decErr := dec.Decode(v)
		if decErr == io.EOF {
			decErr = nil
		}
		if decErr != nil {
			err = decErr
		}
	}
	return resp, err
}

// matchHTTPResponse compares two http.Response objects. Currently, only StatusCode is checked.
func matchHTTPResponse(r1, r2 *http.Response) bool {
	if r1 == nil && r2 == nil {
		return true
	}
	if r1 != nil && r2 != nil {
		return r1.StatusCode == r2.StatusCode
	}
	return false
}

type ErrorResponse struct {
	Response *http.Response
	Detail   string `json:"detail"`
}

func (r *ErrorResponse) Error() string {
	return fmt.Sprintf(
		"%v %v: %d %v",
		r.Response.Request.Method, r.Response.Request.URL,
		r.Response.StatusCode, r.Detail)
}

func (r *ErrorResponse) Is(target error) bool {
	v, ok := target.(*ErrorResponse)
	if !ok {
		return false
	}
	if r.Detail != v.Detail ||
		!matchHTTPResponse(r.Response, v.Response) {
		return false
	}
	return true
}

type RateLimitError struct {
	Rate     Rate
	Response *http.Response
	Detail   string
}

func (r *RateLimitError) Error() string {
	return fmt.Sprintf(
		"%v %v: %d %v %v",
		r.Response.Request.Method, r.Response.Request.URL,
		r.Response.StatusCode, r.Detail, fmt.Sprintf("[rate reset in %v]", time.Until(r.Rate.Reset)))
}

func (r *RateLimitError) Is(target error) bool {
	v, ok := target.(*RateLimitError)
	if !ok {
		return false
	}
	return r.Rate == v.Rate &&
		r.Detail == v.Detail &&
		matchHTTPResponse(r.Response, v.Response)
}

func CheckResponse(r *http.Response) error {
	if c := r.StatusCode; 200 <= c && c <= 299 {
		return nil
	}

	errorResponse := &ErrorResponse{Response: r}
	data, err := ioutil.ReadAll(r.Body)
	if err == nil && data != nil {
		apiError := new(APIError)
		json.Unmarshal(data, apiError)
		if apiError.Empty() {
			errorResponse.Detail = strings.TrimSpace(string(data))
		} else {
			errorResponse.Detail = apiError.Detail()
		}
	}
	// Re-populate error response body.
	r.Body = ioutil.NopCloser(bytes.NewBuffer(data))

	switch {
	case r.StatusCode == http.StatusTooManyRequests &&
		(r.Header.Get(headerRateRemaining) == "0" || r.Header.Get(headerRateConcurrentRemaining) == "0"):
		return &RateLimitError{
			Rate:     parseRate(r),
			Response: errorResponse.Response,
			Detail:   errorResponse.Detail,
		}
	}

	return errorResponse
}

// Rate represents the rate limit for the current client.
type Rate struct {
	// The maximum number of requests allowed within the window.
	Limit int

	// The number of requests this caller has left on this endpoint within the current window
	Remaining int

	// The time when the next rate limit window begins and the count resets, measured in UTC seconds from epoch
	Reset time.Time

	// The maximum number of concurrent requests allowed within the window
	ConcurrentLimit int

	// The number of concurrent requests this caller has left on this endpoint within the current window
	ConcurrentRemaining int
}

// Bool returns a pointer to the bool value passed in.
func Bool(v bool) *bool { return &v }

// Int returns a pointer to the int value passed in.
func Int(v int) *int { return &v }

// String returns a pointer to the string value passed in.
func String(v string) *string { return &v }

// Time returns a pointer to the time.Time value passed in.
func Time(v time.Time) *time.Time { return &v }
<commit_msg>Add pointer to value converters<commit_after>package sentry

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"reflect"
	"strconv"
	"strings"
	"time"

	"github.com/google/go-querystring/query"
	"github.com/peterhellberg/link"
)

const (
	defaultBaseURL = "https://sentry.io/api/"
	userAgent      = "go-sentry"

	// https://docs.sentry.io/api/ratelimits/
	headerRateLimit               = "X-Sentry-Rate-Limit-Limit"
	headerRateRemaining           = "X-Sentry-Rate-Limit-Remaining"
	headerRateReset               = "X-Sentry-Rate-Limit-Reset"
	headerRateConcurrentLimit     = "X-Sentry-Rate-Limit-ConcurrentLimit"
	headerRateConcurrentRemaining = "X-Sentry-Rate-Limit-ConcurrentRemaining"
)

var errNonNilContext = errors.New("context must be non-nil")

// Client for Sentry API.
type Client struct {
	client *http.Client

	// BaseURL for API requests.
	BaseURL *url.URL

	// User agent used when communicating with Sentry.
	UserAgent string

	// Latest rate limit
	rate Rate

	// Common struct used by all services.
	common service

	// Services
	IssueAlerts         *IssueAlertsService
	MetricAlerts        *MetricAlertsService
	OrganizationMembers *OrganizationMembersService
	Organizations       *OrganizationsService
	ProjectKeys         *ProjectKeysService
	ProjectOwnerships   *ProjectOwnershipsService
	ProjectPlugins      *ProjectPluginsService
	Projects            *ProjectsService
	Teams               *TeamsService
}

type service struct {
	client *Client
}

// NewClient returns a new Sentry API client.
// If a nil httpClient is provided, the http.DefaultClient will be used.
func NewClient(httpClient *http.Client) *Client {
	if httpClient == nil {
		httpClient = http.DefaultClient
	}
	baseURL, _ := url.Parse(defaultBaseURL)

	c := &Client{
		client:    httpClient,
		BaseURL:   baseURL,
		UserAgent: userAgent,
	}
	c.common.client = c
	c.IssueAlerts = (*IssueAlertsService)(&c.common)
	c.MetricAlerts = (*MetricAlertsService)(&c.common)
	c.OrganizationMembers = (*OrganizationMembersService)(&c.common)
	c.Organizations = (*OrganizationsService)(&c.common)
	c.ProjectKeys = (*ProjectKeysService)(&c.common)
	c.ProjectOwnerships = (*ProjectOwnershipsService)(&c.common)
	c.ProjectPlugins = (*ProjectPluginsService)(&c.common)
	c.Projects = (*ProjectsService)(&c.common)
	c.Teams = (*TeamsService)(&c.common)
	return c
}

// NewOnPremiseClient returns a new Sentry API client with the provided base URL.
// Note that the base URL must be in the format "http(s)://[hostname]/api/".
// If the base URL does not have the suffix "/api/", it will be added automatically.
// If a nil httpClient is provided, the http.DefaultClient will be used.
func NewOnPremiseClient(baseURL string, httpClient *http.Client) (*Client, error) {
	baseEndpoint, err := url.Parse(baseURL)
	if err != nil {
		return nil, err
	}

	if !strings.HasSuffix(baseEndpoint.Path, "/") {
		baseEndpoint.Path += "/"
	}
	if !strings.HasSuffix(baseEndpoint.Path, "/api/") {
		baseEndpoint.Path += "api/"
	}

	c := NewClient(httpClient)
	c.BaseURL = baseEndpoint
	return c, nil
}

type ListCursorParams struct {
	// A cursor, as given in the Link header.
	// If specified, the query continues the search using this cursor.
	Cursor string `url:"cursor,omitempty"`
}

func addQuery(s string, params interface{}) (string, error) {
	v := reflect.ValueOf(params)
	if v.Kind() == reflect.Ptr && v.IsNil() {
		return s, nil
	}

	u, err := url.Parse(s)
	if err != nil {
		return s, err
	}

	qs, err := query.Values(params)
	if err != nil {
		return s, err
	}

	u.RawQuery = qs.Encode()
	return u.String(), nil
}

// NewRequest creates an API request.
func (c *Client) NewRequest(method, urlRef string, body interface{}) (*http.Request, error) {
	if !strings.HasSuffix(c.BaseURL.Path, "/") {
		return nil, fmt.Errorf("BaseURL must have a trailing slash, but %q does not", c.BaseURL)
	}

	u, err := c.BaseURL.Parse(urlRef)
	if err != nil {
		return nil, err
	}

	var buf io.ReadWriter
	if body != nil {
		buf = &bytes.Buffer{}
		enc := json.NewEncoder(buf)
		enc.SetEscapeHTML(false)
		err := enc.Encode(body)
		if err != nil {
			return nil, err
		}
	}

	req, err := http.NewRequest(method, u.String(), buf)
	if err != nil {
		return nil, err
	}

	if body != nil {
		req.Header.Set("Content-Type", "application/json")
	}
	if c.UserAgent != "" {
		req.Header.Set("User-Agent", c.UserAgent)
	}
	return req, nil
}

// Response is a Sentry API response. This wraps the standard http.Response
// and provides convenient access to things like pagination links and rate limits.
type Response struct {
	*http.Response

	// For APIs that support cursor pagination, the following field will be populated
	// to point to the next page if more results are available.
	// Set ListCursorParams.Cursor to this value when calling the endpoint again.
	Cursor string

	Rate Rate
}

func newResponse(r *http.Response) *Response {
	response := &Response{Response: r}
	response.Rate = parseRate(r)
	response.populatePaginationCursor()
	return response
}

func (r *Response) populatePaginationCursor() {
	rels := link.ParseResponse(r.Response)
	if nextRel, ok := rels["next"]; ok && nextRel.Extra["results"] == "true" {
		r.Cursor = nextRel.Extra["cursor"]
	}
}

// parseRate parses the rate limit headers.
func parseRate(r *http.Response) Rate {
	var rate Rate
	if limit := r.Header.Get(headerRateLimit); limit != "" {
		rate.Limit, _ = strconv.Atoi(limit)
	}
	if remaining := r.Header.Get(headerRateRemaining); remaining != "" {
		rate.Remaining, _ = strconv.Atoi(remaining)
	}
	if reset := r.Header.Get(headerRateReset); reset != "" {
		if v, _ := strconv.ParseInt(reset, 10, 64); v != 0 {
			rate.Reset = time.Unix(v, 0).UTC()
		}
	}
	if concurrentLimit := r.Header.Get(headerRateConcurrentLimit); concurrentLimit != "" {
		rate.ConcurrentLimit, _ = strconv.Atoi(concurrentLimit)
	}
	if concurrentRemaining := r.Header.Get(headerRateConcurrentRemaining); concurrentRemaining != "" {
		rate.ConcurrentRemaining, _ = strconv.Atoi(concurrentRemaining)
	}

	return rate
}

func (c *Client) BareDo(ctx context.Context, req *http.Request) (*Response, error) {
	if ctx == nil {
		return nil, errNonNilContext
	}

	// Check rate limit
	if err := c.checkRateLimit(req); err != nil {
		return &Response{
			Response: err.Response,
			Rate:     err.Rate,
		}, err
	}

	resp, err := c.client.Do(req)
	if err != nil {
		// If we got an error, and the context has been canceled,
		// the context's error is probably more useful.
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		default:
			return nil, err
		}
	}

	response := newResponse(resp)

	c.rate = response.Rate

	err = CheckResponse(resp)

	return response, err
}

func (c *Client) checkRateLimit(req *http.Request) *RateLimitError {
	if !c.rate.Reset.IsZero() && c.rate.Remaining == 0 && time.Now().Before(c.rate.Reset) {
		resp := &http.Response{
			Status:     http.StatusText(http.StatusTooManyRequests),
			StatusCode: http.StatusTooManyRequests,
			Request:    req,
			Header:     http.Header{},
			Body:       ioutil.NopCloser(strings.NewReader("")),
		}
		return &RateLimitError{
			Rate:     c.rate,
			Response: resp,
			Detail: fmt.Sprintf("API rate limit of %v and concurrent limit of %v still exceeded until %v, not making remote request.",
				c.rate.Limit, c.rate.ConcurrentLimit, c.rate.Reset),
		}
	}
	return nil
}

func (c *Client) Do(ctx context.Context, req *http.Request, v interface{}) (*Response, error) {
	resp, err := c.BareDo(ctx, req)
	if err != nil {
		return resp, err
	}
	defer resp.Body.Close()

	switch v := v.(type) {
	case nil:
	case io.Writer:
		_, err = io.Copy(v, resp.Body)
	default:
		dec := json.NewDecoder(resp.Body)
		dec.UseNumber()
		decErr := dec.Decode(v)
		if decErr == io.EOF {
			decErr = nil
		}
		if decErr != nil {
			err = decErr
		}
	}
	return resp, err
}

// matchHTTPResponse compares two http.Response objects. Currently, only StatusCode is checked.
func matchHTTPResponse(r1, r2 *http.Response) bool {
	if r1 == nil && r2 == nil {
		return true
	}
	if r1 != nil && r2 != nil {
		return r1.StatusCode == r2.StatusCode
	}
	return false
}

type ErrorResponse struct {
	Response *http.Response
	Detail   string `json:"detail"`
}

func (r *ErrorResponse) Error() string {
	return fmt.Sprintf(
		"%v %v: %d %v",
		r.Response.Request.Method, r.Response.Request.URL,
		r.Response.StatusCode, r.Detail)
}

func (r *ErrorResponse) Is(target error) bool {
	v, ok := target.(*ErrorResponse)
	if !ok {
		return false
	}
	if r.Detail != v.Detail ||
		!matchHTTPResponse(r.Response, v.Response) {
		return false
	}
	return true
}

type RateLimitError struct {
	Rate     Rate
	Response *http.Response
	Detail   string
}

func (r *RateLimitError) Error() string {
	return fmt.Sprintf(
		"%v %v: %d %v %v",
		r.Response.Request.Method, r.Response.Request.URL,
		r.Response.StatusCode, r.Detail, fmt.Sprintf("[rate reset in %v]", time.Until(r.Rate.Reset)))
}

func (r *RateLimitError) Is(target error) bool {
	v, ok := target.(*RateLimitError)
	if !ok {
		return false
	}
	return r.Rate == v.Rate &&
		r.Detail == v.Detail &&
		matchHTTPResponse(r.Response, v.Response)
}

func CheckResponse(r *http.Response) error {
	if c := r.StatusCode; 200 <= c && c <= 299 {
		return nil
	}

	errorResponse := &ErrorResponse{Response: r}
	data, err := ioutil.ReadAll(r.Body)
	if err == nil && data != nil {
		apiError := new(APIError)
		json.Unmarshal(data, apiError)
		if apiError.Empty() {
			errorResponse.Detail = strings.TrimSpace(string(data))
		} else {
			errorResponse.Detail = apiError.Detail()
		}
	}
	// Re-populate error response body.
	r.Body = ioutil.NopCloser(bytes.NewBuffer(data))

	switch {
	case r.StatusCode == http.StatusTooManyRequests &&
		(r.Header.Get(headerRateRemaining) == "0" || r.Header.Get(headerRateConcurrentRemaining) == "0"):
		return &RateLimitError{
			Rate:     parseRate(r),
			Response: errorResponse.Response,
			Detail:   errorResponse.Detail,
		}
	}

	return errorResponse
}

// Rate represents the rate limit for the current client.
type Rate struct {
	// The maximum number of requests allowed within the window.
	Limit int

	// The number of requests this caller has left on this endpoint within the current window
	Remaining int

	// The time when the next rate limit window begins and the count resets, measured in UTC seconds from epoch
	Reset time.Time

	// The maximum number of concurrent requests allowed within the window
	ConcurrentLimit int

	// The number of concurrent requests this caller has left on this endpoint within the current window
	ConcurrentRemaining int
}

// Bool returns a pointer to the bool value passed in.
func Bool(v bool) *bool { return &v }

// BoolValue returns the value of the bool pointer passed in or
// false if the pointer is nil.
func BoolValue(v *bool) bool {
	if v != nil {
		return *v
	}
	return false
}

// Int returns a pointer to the int value passed in.
func Int(v int) *int { return &v }

// IntValue returns the value of the int pointer passed in or
// 0 if the pointer is nil.
func IntValue(v *int) int {
	if v != nil {
		return *v
	}
	return 0
}

// String returns a pointer to the string value passed in.
func String(v string) *string { return &v }

// StringValue returns the value of the string pointer passed in or
// "" if the pointer is nil.
func StringValue(v *string) string {
	if v != nil {
		return *v
	}
	return ""
}

// Time returns a pointer to the time.Time value passed in.
func Time(v time.Time) *time.Time { return &v }

// TimeValue returns the value of the time.Time pointer passed in or
// time.Time{} if the pointer is nil.
func TimeValue(v *time.Time) time.Time {
	if v != nil {
		return *v
	}
	return time.Time{}
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"net/http"
	"io/ioutil"
	"encoding/json"
)

type fudocs struct {
	Location string
}

func newFudocs(Location string) *fudocs {
	return &fudocs{Location : Location}
}

func (this *fudocs) GET(w http.ResponseWriter, r *http.Request) {
	var result struct {
		File string
		Error string
	}
	file, err := ioutil.ReadFile(this.Location + r.URL.Path + ".md")
	result.File = string(file)
	result.Error = ""
	if err != nil {
		result.Error = "ioutil.Readfile: " + err.Error()
	}
	b, _ := json.Marshal(result);
	io.WriteString(w, string(b));
}

func (this *fudocs) POST(w http.ResponseWriter, r *http.Request) {

}

func (this *fudocs) PUT(w http.ResponseWriter, r *http.Request) {

}

func (this *fudocs) DELETE(w http.ResponseWriter, r *http.Request) {

}
<commit_msg>add PUT method to create markdown files<commit_after>package main

import (
	"io"
	"net/http"
	"io/ioutil"
	"encoding/json"
)

type fudocs struct {
	Location string
}

func newFudocs(Location string) *fudocs {
	return &fudocs{Location : Location}
}

func (this *fudocs) GET(w http.ResponseWriter, r *http.Request) {
	var result struct {
		File string
		Error string
	}
	file, err := ioutil.ReadFile(this.Location + r.URL.Path + ".md")
	result.File = string(file)
	result.Error = ""
	if err != nil {
		result.Error = "ioutil.Readfile: " + err.Error()
	}
	b, _ := json.Marshal(result)
	io.WriteString(w, string(b))
}

func (this *fudocs) POST(w http.ResponseWriter, r *http.Request) {

}

func (this *fudocs) PUT(w http.ResponseWriter, r *http.Request) {
	var result struct {
		Error string
	}
	err := ioutil.WriteFile(this.Location + r.URL.Path + ".md", []byte(r.FormValue("File")), 0644)
	result.Error = ""
	if err != nil {
		result.Error = "ioutil.WriteFile: " + err.Error()
	}
	b, _ := json.Marshal(result)
	io.WriteString(w, string(b))
}

func (this *fudocs) DELETE(w http.ResponseWriter, r *http.Request) {

}
<|endoftext|>


<commit_before>package main

import (
	"github.com/MerlinDMC/dsapid"
	"github.com/MerlinDMC/dsapid/server/handler"
	"github.com/MerlinDMC/dsapid/server/middleware"
	"github.com/codegangsta/martini"
)

func registerRoutes(router martini.Router) {
	// common
	router.Get("/ping", handler.CommonPing)
	router.Get("/status", handler.CommonStatus)

	// dsapi
	router.Get("/datasets", middleware.AllowCORS(), handler.DsapiList)
	router.Get("/datasets/:id", middleware.AllowCORS(), handler.DsapiDetail)
	router.Get("/datasets/:id/:path", handler.DsapiFile)

	// imgapi
	router.Get("/images", middleware.AllowCORS(), handler.ImgapiList)
	router.Get("/images/:id", middleware.AllowCORS(), handler.ImgapiDetail)
	router.Get("/images/:id/file", handler.ImgapiFile)
	router.Get("/images/:id/file:file_idx", handler.ImgapiFile)

	// public api
	router.Get("/api/datasets", middleware.AllowCORS(), handler.ApiDatasetsList)
	router.Get("/api/datasets/:id", middleware.AllowCORS(), handler.ApiDatasetsDetail)
	router.Get("/api/export/:id", handler.ApiDatasetExport)

	// private api - update
	router.Post("/api/reload/datasets", middleware.RequireRoles(dsapid.UserRoleDatasetAdmin), handler.ApiPostReloadDatasets)
	router.Post("/api/datasets/:id", middleware.RequireRoles(dsapid.UserRoleDatasetManage), handler.ApiPostDatasetUpdate)

	// private api - upload
	router.Post("/api/upload", middleware.RequireRoles(dsapid.UserRoleDatasetUpload), handler.ApiPostFileUpload)
}
<commit_msg>missed a spot that still links to the old repository<commit_after>package main

import (
	"github.com/MerlinDMC/dsapid"
	"github.com/MerlinDMC/dsapid/server/handler"
	"github.com/MerlinDMC/dsapid/server/middleware"
	"github.com/go-martini/martini"
)

func registerRoutes(router martini.Router) {
	// common
	router.Get("/ping", handler.CommonPing)
	router.Get("/status", handler.CommonStatus)

	// dsapi
	router.Get("/datasets", middleware.AllowCORS(), handler.DsapiList)
	router.Get("/datasets/:id", middleware.AllowCORS(), handler.DsapiDetail)
	router.Get("/datasets/:id/:path", handler.DsapiFile)

	// imgapi
	router.Get("/images", middleware.AllowCORS(), handler.ImgapiList)
	router.Get("/images/:id", middleware.AllowCORS(), handler.ImgapiDetail)
	router.Get("/images/:id/file", handler.ImgapiFile)
	router.Get("/images/:id/file:file_idx", handler.ImgapiFile)

	// public api
	router.Get("/api/datasets", middleware.AllowCORS(), handler.ApiDatasetsList)
	router.Get("/api/datasets/:id", middleware.AllowCORS(), handler.ApiDatasetsDetail)
	router.Get("/api/export/:id", handler.ApiDatasetExport)

	// private api - update
	router.Post("/api/reload/datasets", middleware.RequireRoles(dsapid.UserRoleDatasetAdmin), handler.ApiPostReloadDatasets)
	router.Post("/api/datasets/:id", middleware.RequireRoles(dsapid.UserRoleDatasetManage), handler.ApiPostDatasetUpdate)

	// private api - upload
	router.Post("/api/upload", middleware.RequireRoles(dsapid.UserRoleDatasetUpload), handler.ApiPostFileUpload)
}
<|endoftext|>


<commit_before>package server

import (
	"encoding/json"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"strings"
)

// Routez represents detail information on current routes
type Routez struct {
	NumRoutes int          `json:"num_routes"`
	Routes    []*RouteInfo `json:"routes"`
}

// RouteInfo has detailed information on a per connection basis.
type RouteInfo struct {
	Cid       uint64 `json:"cid"`
	URL       string `json:"url"`
	IP        string `json:"ip"`
	Port      int    `json:"port"`
	Solicited bool   `json:"solicited"`
	Subs      uint32 `json:"subscriptions"`
	Pending   int    `json:"pending_size"`
	InMsgs    int64  `json:"in_msgs"`
	OutMsgs   int64  `json:"out_msgs"`
	InBytes   int64  `json:"in_bytes"`
	OutBytes  int64  `json:"out_bytes"`
}

// HandleConnz process HTTP requests for connection information.
func (s *Server) HandleRoutez(w http.ResponseWriter, req *http.Request) {

	if req.Method == "GET" {
		r := Routez{Routes: []*RouteInfo{}}

		// Walk the list
		s.mu.Lock()
		for _, route := range s.routes {
			ri := &RouteInfo{
				Cid:       route.cid,
				Subs:      route.subs.Count(),
				Solicited: route.route.didSolicit,
				InMsgs:    route.inMsgs,
				OutMsgs:   route.outMsgs,
				InBytes:   route.inBytes,
				OutBytes:  route.outBytes,
			}

			if route.route.url != nil {
				ri.URL = route.route.url.String()
			}

			if ip, ok := route.nc.(*net.TCPConn); ok {
				addr := ip.RemoteAddr().(*net.TCPAddr)
				ri.Port = addr.Port
				ri.IP = addr.IP.String()
			}
			r.Routes = append(r.Routes, ri)
		}
		s.mu.Unlock()

		r.NumRoutes = len(r.Routes)
		b, err := json.MarshalIndent(r, "", "  ")
		if err != nil {
			Logf("Error marshalling response to /routez request: %v", err)
		}
		w.Write(b)
	} else if req.Method == "PUT" {
		body := make([]byte, 1024)
		req.Body.Read(body)
		routeURL, err := url.Parse(strings.Trim(string(body), "\x00"))
		if err != nil {
			w.WriteHeader(400)
			w.Write([]byte(fmt.Sprintf(`{"error": "could not parse URL: %v"}`, err)))
			return
		}

		s.connectToRoute(routeURL)
		w.Write([]byte(`{"status": "ok"}`))
	} else if req.Method == "DELETE" {
		body := make([]byte, 1024)
		req.Body.Read(body)
		url := strings.Trim(string(body), "\x00")

		s.mu.Lock()
		for _, route := range s.routes {
			if route.route.url != nil && route.route.url.String() == url {
				route.mu.Lock()
				route.route.didSolicit = false // don't reconnect
				route.mu.Unlock()
				route.closeConnection()
				w.WriteHeader(200)
				w.Write([]byte(`{"status": "ok"}`))
				return
			}
		}
		s.mu.Unlock()
		w.WriteHeader(404)
		w.Write([]byte(`{"error": "could not find matching route"}`))
	}
}
<commit_msg>Fix deadlocking issue<commit_after>package server

import (
	"encoding/json"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"strings"
)

// Routez represents detail information on current routes
type Routez struct {
	NumRoutes int          `json:"num_routes"`
	Routes    []*RouteInfo `json:"routes"`
}

// RouteInfo has detailed information on a per connection basis.
type RouteInfo struct {
	Cid       uint64 `json:"cid"`
	URL       string `json:"url"`
	IP        string `json:"ip"`
	Port      int    `json:"port"`
	Solicited bool   `json:"solicited"`
	Subs      uint32 `json:"subscriptions"`
	Pending   int    `json:"pending_size"`
	InMsgs    int64  `json:"in_msgs"`
	OutMsgs   int64  `json:"out_msgs"`
	InBytes   int64  `json:"in_bytes"`
	OutBytes  int64  `json:"out_bytes"`
}

// HandleConnz process HTTP requests for connection information.
func (s *Server) HandleRoutez(w http.ResponseWriter, req *http.Request) {

	if req.Method == "GET" {
		r := Routez{Routes: []*RouteInfo{}}

		// Walk the list
		s.mu.Lock()
		defer s.mu.Unlock()
		for _, route := range s.routes {
			ri := &RouteInfo{
				Cid:       route.cid,
				Subs:      route.subs.Count(),
				Solicited: route.route.didSolicit,
				InMsgs:    route.inMsgs,
				OutMsgs:   route.outMsgs,
				InBytes:   route.inBytes,
				OutBytes:  route.outBytes,
			}

			if route.route.url != nil {
				ri.URL = route.route.url.String()
			}

			if ip, ok := route.nc.(*net.TCPConn); ok {
				addr := ip.RemoteAddr().(*net.TCPAddr)
				ri.Port = addr.Port
				ri.IP = addr.IP.String()
			}
			r.Routes = append(r.Routes, ri)
		}

		r.NumRoutes = len(r.Routes)
		b, err := json.MarshalIndent(r, "", "  ")
		if err != nil {
			Logf("Error marshalling response to /routez request: %v", err)
		}
		w.Write(b)
	} else if req.Method == "PUT" {
		body := make([]byte, 1024)
		req.Body.Read(body)
		routeURL, err := url.Parse(strings.Trim(string(body), "\x00"))
		if err != nil {
			w.WriteHeader(400)
			w.Write([]byte(fmt.Sprintf(`{"error": "could not parse URL: %v"}`, err)))
			return
		}

		s.connectToRoute(routeURL)
		w.Write([]byte(`{"status": "ok"}`))
	} else if req.Method == "DELETE" {
		body := make([]byte, 1024)
		req.Body.Read(body)
		url := strings.Trim(string(body), "\x00")

		s.mu.Lock()
		defer s.mu.Unlock()
		for _, route := range s.routes {
			if route.route.url != nil && route.route.url.String() == url {
				route.mu.Lock()
				route.route.didSolicit = false // don't reconnect
				route.mu.Unlock()
				route.closeConnection()
				w.WriteHeader(200)
				w.Write([]byte(`{"status": "ok"}`))
				return
			}
		}
		w.WriteHeader(404)
		w.Write([]byte(`{"error": "could not find matching route"}`))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"os"
	"time"

	"github.com/fsouza/go-dockerclient"
	"github.com/garyburd/redigo/redis"
)

// DockerClient for running code
var DockerClient *docker.Client

// Runner runs the code
type Runner struct {
	Lang          string `json:"lang"`
	Source        string `json:"source"`
	Version       string `json:"version"`
	Timeout       int    `json:"timeout"` // How long is the code going to run
	closeNotifier <-chan bool
}

// Runnerthrottle Limit the max throttle for runner
var Runnerthrottle chan struct{}

// Run the code in the container
func (r *Runner) Run(output messages, conn redis.Conn, uuid string) {
	Runnerthrottle <- struct{}{}
	defer func() { <-Runnerthrottle }()

	container, err := r.createContainer(uuid)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: Container %s cannot be created - %v\n", uuid, err)
		return
	}

	stdoutReader, stdoutWriter := io.Pipe()
	stdinReader, stdinWriter := io.Pipe()

	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %v", err)
	}

	defer stdinWriter.Close()
	defer stdoutWriter.Close()

	go pipeStdin(conn, uuid, stdinWriter)
	go pipeStdout(stdoutReader, output)

	// Start running the container
	err = r.startContainer(container.ID)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: Container %s cannot be started - %v\n", uuid, err)
		return
	}
	defer DockerClient.RemoveContainer(docker.RemoveContainerOptions{ID: container.ID, Force: true})

	successChan := make(chan struct{})
	errorChan := make(chan error)

	go func() {
		_, err := DockerClient.WaitContainer(container.ID)
		if err != nil {
			fmt.Printf("Error: %v\n", err)
			errorChan <- err
			return
		}
		successChan <- struct{}{}
	}()

	go func() {
		err = r.attachContainer(container.ID, stdoutWriter, stdinReader)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error: Container %s cannot be attached - %v\n", uuid, err)
		}
	}()

	select {
	case <-r.closeNotifier:
		DockerClient.StopContainer(container.ID, 0)
		fmt.Fprintf(os.Stdout, "Container %s is stopped since the streamming has been halted\n", uuid)
	case <-successChan:
		fmt.Fprintf(os.Stdout, "Container %s is executed successfully\n", uuid)
	case err := <-errorChan:
		fmt.Fprintf(os.Stdout, "Container %s failed caused by - %v\n", uuid, err)
	case <-time.After(time.Duration(r.Timeout) * time.Second):
		msg := fmt.Sprintf("Container %s is terminated caused by 15 sec timeout\n", uuid)
		fmt.Fprintf(os.Stderr, msg)
		output <- msg
	}
}

func pipeStdin(conn redis.Conn, uuid string, stdin *io.PipeWriter) {
	psc := redis.PubSubConn{Conn: conn}
	psc.Subscribe(uuid + "#stdin")

	defer func() {
		psc.Unsubscribe(uuid + "#stdin")
		psc.Close()
		conn.Close()
	}()

StdinSubscriptionLoop:
	for {
		switch n := psc.Receive().(type) {
		case redis.Message:
			fmt.Printf("Message: %s %s\n", n.Channel, n.Data)
			stdin.Write(n.Data)
		case error:
			break StdinSubscriptionLoop
		}
	}
	fmt.Println("Stdin subscription closed")
}

func pipeStdout(stdout *io.PipeReader, output messages) {
	buffer := make([]byte, 512)
	for {
		n, err := stdout.Read(buffer)
		if err != nil {
			if err != io.EOF {
				stdout.Close()
				fmt.Fprintf(os.Stderr, "Error: %v\n", err)
			}

			close(output)
			break
		}

		data := buffer[0:n]
		output <- string(data)

		// Clear the buffer
		for i := 0; i < n; i++ {
			buffer[i] = 0
		}
	}
}

func getDockerClient() (*docker.Client, error) {
	dockerHost := os.Getenv("DOCKER_HOST")

	// If DOCKER_HOST exists in env, using docker-machine
	if dockerHost != "" {
		return docker.NewClientFromEnv()
	}

	// Otherwise using sock connection
	//TODO: Deal with the TLS case (even though you are not using it for now)
	endpoint := "unix:///var/run/docker.sock"
	return docker.NewClient(endpoint)
}

var imageMapper = map[string]string{
	"swift":  "koderunr-swift",
	"ruby":   "koderunr-ruby",
	"python": "koderunr-python",
	"go":     "koderunr-go",
	"c":      "koderunr-c",
	"elixir": "koderunr-erl",
}

func (r *Runner) image() string {
	return imageMapper[r.Lang]
}

func (r *Runner) createContainer(uuid string) (*docker.Container, error) {
	cmd := []string{r.Source, uuid}

	if r.Version != "" {
		cmd = append(cmd, r.Version)
	}
	return DockerClient.CreateContainer(docker.CreateContainerOptions{
		Name: uuid,
		Config: &docker.Config{
			Image:           r.image(),
			NetworkDisabled: true,
			OpenStdin:       true,
			Cmd:             cmd,
			KernelMemory:    1024 * 1024 * 4,
		},
	})
}

func (r *Runner) startContainer(containerID string) error {
	return DockerClient.StartContainer(containerID, &docker.HostConfig{
		CPUQuota: 40000,
		Memory:   50 * 1024 * 1024, // so the memory swap will be the same size
	})
}

func (r *Runner) attachContainer(containerID string, stdoutWriter *io.PipeWriter, stdinReader *io.PipeReader) error {
	return DockerClient.AttachToContainer(docker.AttachToContainerOptions{
		Container:    containerID,
		Stdin:        true,
		Stdout:       true,
		Stderr:       true,
		Stream:       true,
		OutputStream: stdoutWriter,
		ErrorStream:  stdoutWriter,
		InputStream:  stdinReader,
	})
}
<commit_msg>set pids limit.<commit_after>package main

import (
	"fmt"
	"io"
	"os"
	"time"

	"github.com/fsouza/go-dockerclient"
	"github.com/garyburd/redigo/redis"
)

// DockerClient for running code
var DockerClient *docker.Client

// Runner runs the code
type Runner struct {
	Lang          string `json:"lang"`
	Source        string `json:"source"`
	Version       string `json:"version"`
	Timeout       int    `json:"timeout"` // How long is the code going to run
	closeNotifier <-chan bool
}

// Runnerthrottle Limit the max throttle for runner
var Runnerthrottle chan struct{}

// Run the code in the container
func (r *Runner) Run(output messages, conn redis.Conn, uuid string) {
	Runnerthrottle <- struct{}{}
	defer func() { <-Runnerthrottle }()

	container, err := r.createContainer(uuid)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: Container %s cannot be created - %v\n", uuid, err)
		return
	}

	stdoutReader, stdoutWriter := io.Pipe()
	stdinReader, stdinWriter := io.Pipe()

	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %v", err)
	}

	defer stdinWriter.Close()
	defer stdoutWriter.Close()

	go pipeStdin(conn, uuid, stdinWriter)
	go pipeStdout(stdoutReader, output)

	// Start running the container
	err = r.startContainer(container.ID)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: Container %s cannot be started - %v\n", uuid, err)
		return
	}
	defer DockerClient.RemoveContainer(docker.RemoveContainerOptions{ID: container.ID, Force: true})

	successChan := make(chan struct{})
	errorChan := make(chan error)

	go func() {
		_, err := DockerClient.WaitContainer(container.ID)
		if err != nil {
			fmt.Printf("Error: %v\n", err)
			errorChan <- err
			return
		}
		successChan <- struct{}{}
	}()

	go func() {
		err = r.attachContainer(container.ID, stdoutWriter, stdinReader)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error: Container %s cannot be attached - %v\n", uuid, err)
		}
	}()

	select {
	case <-r.closeNotifier:
		DockerClient.StopContainer(container.ID, 0)
		fmt.Fprintf(os.Stdout, "Container %s is stopped since the streamming has been halted\n", uuid)
	case <-successChan:
		fmt.Fprintf(os.Stdout, "Container %s is executed successfully\n", uuid)
	case err := <-errorChan:
		fmt.Fprintf(os.Stdout, "Container %s failed caused by - %v\n", uuid, err)
	case <-time.After(time.Duration(r.Timeout) * time.Second):
		msg := fmt.Sprintf("Container %s is terminated caused by 15 sec timeout\n", uuid)
		fmt.Fprintf(os.Stderr, msg)
		output <- msg
	}
}

func pipeStdin(conn redis.Conn, uuid string, stdin *io.PipeWriter) {
	psc := redis.PubSubConn{Conn: conn}
	psc.Subscribe(uuid + "#stdin")

	defer func() {
		psc.Unsubscribe(uuid + "#stdin")
		psc.Close()
		conn.Close()
	}()

StdinSubscriptionLoop:
	for {
		switch n := psc.Receive().(type) {
		case redis.Message:
			fmt.Printf("Message: %s %s\n", n.Channel, n.Data)
			stdin.Write(n.Data)
		case error:
			break StdinSubscriptionLoop
		}
	}
	fmt.Println("Stdin subscription closed")
}

func pipeStdout(stdout *io.PipeReader, output messages) {
	buffer := make([]byte, 512)
	for {
		n, err := stdout.Read(buffer)
		if err != nil {
			if err != io.EOF {
				stdout.Close()
				fmt.Fprintf(os.Stderr, "Error: %v\n", err)
			}

			close(output)
			break
		}

		data := buffer[0:n]
		output <- string(data)

		// Clear the buffer
		for i := 0; i < n; i++ {
			buffer[i] = 0
		}
	}
}

func getDockerClient() (*docker.Client, error) {
	dockerHost := os.Getenv("DOCKER_HOST")

	// If DOCKER_HOST exists in env, using docker-machine
	if dockerHost != "" {
		return docker.NewClientFromEnv()
	}

	// Otherwise using sock connection
	//TODO: Deal with the TLS case (even though you are not using it for now)
	endpoint := "unix:///var/run/docker.sock"
	return docker.NewClient(endpoint)
}

var imageMapper = map[string]string{
	"swift":  "koderunr-swift",
	"ruby":   "koderunr-ruby",
	"python": "koderunr-python",
	"go":     "koderunr-go",
	"c":      "koderunr-c",
	"elixir": "koderunr-erl",
}

func (r *Runner) image() string {
	return imageMapper[r.Lang]
}

func (r *Runner) createContainer(uuid string) (*docker.Container, error) {
	cmd := []string{r.Source, uuid}

	if r.Version != "" {
		cmd = append(cmd, r.Version)
	}
	return DockerClient.CreateContainer(docker.CreateContainerOptions{
		Name: uuid,
		Config: &docker.Config{
			Image:           r.image(),
			NetworkDisabled: true,
			OpenStdin:       true,
			Cmd:             cmd,
			KernelMemory:    1024 * 1024 * 4,
			PidsLimit:       5,
		},
	})
}

func (r *Runner) startContainer(containerID string) error {
	return DockerClient.StartContainer(containerID, &docker.HostConfig{
		CPUQuota: 40000,
		Memory:   50 * 1024 * 1024, // so the memory swap will be the same size
	})
}

func (r *Runner) attachContainer(containerID string, stdoutWriter *io.PipeWriter, stdinReader *io.PipeReader) error {
	return DockerClient.AttachToContainer(docker.AttachToContainerOptions{
		Container:    containerID,
		Stdin:        true,
		Stdout:       true,
		Stderr:       true,
		Stream:       true,
		OutputStream: stdoutWriter,
		ErrorStream:  stdoutWriter,
		InputStream:  stdinReader,
	})
}
<|endoftext|>


<commit_before>package core

import (
	"bufio"
	"os"
	"strings"
	"time"

	"github.com/fsnotify/fsnotify"
	"github.com/tinycedar/lily/common"
	"github.com/tinycedar/lily/conf"
)

const (
	systemHosts = "C:/Windows/System32/drivers/etc/hosts"
)

// var batcher *Batcher

func FireHostsSwitch() {
	common.Info("============================== Fire hosts switch ==============================")
	// if batcher != nil {
	// 	batcher.Close()
	// }
	doProcess()
	// batcher = initSystemHostsWatcher()
	// go startSystemHostsWatcher()
}

// 1. Find collection of same domain names between system hosts and currentHostIndex
// 2. Disconnect the TCP connections(http:80 & https:443) of collection found above
func doProcess() {
	overlapHostConfigMap := getOverlapHostConfigMap()
	common.Info("overlapHostConfigMap: %v", overlapHostConfigMap)
	if len(overlapHostConfigMap) == 0 {
		return
	}
	table := getTCPTable()
	for i := uint32(0); i < uint32(table.dwNumEntries); i++ {
		row := table.table[i]
		if row.dwOwningPid <= 0 {
			continue
		}
		ip := row.displayIP(row.dwRemoteAddr)
		port := row.displayPort(row.dwRemotePort)
		if _, ok := overlapHostConfigMap[ip]; !ok {
			continue
		}
		if port == 80 || port == 443 {
			if err := CloseTCPEntry(row); err != nil {
				common.Error("Fail to close TCP connections: Pid = %v, Addr = %v:%v\n", row.dwOwningPid, ip, port)
			} else {
				common.Info("Succeed to close TCP connections: Pid = %v, Addr = %v:%v", row.dwOwningPid, ip, port)
			}
		}
	}
}

func getOverlapHostConfigMap() map[string]bool {
	result := make(map[string]bool)
	var currentHostConfigMap map[string]string
	if current := conf.Config.HostConfigModel.RootAt(conf.Config.CurrentHostIndex); current != nil {
		currentHostConfigMap = readHostConfigMap("conf/hosts/" + current.Text() + ".hosts")
	}
	common.Info("currentHostConfigMap: %v", currentHostConfigMap)
	if len(currentHostConfigMap) == 0 {
		return result
	}
	common.Info("systemConfigMap: %v", readHostConfigMap(systemHosts))
	for k, v := range readHostConfigMap(systemHosts) {
		v2, ok := currentHostConfigMap[k]
		common.Info("k: %s, %s - %s", k, v, v2)
		if ok && v != v2 {
			result[v] = true
		}
	}
	return result
}

func process() {
	// hostConfigMap := readFile()
	browserProcessMap := getBrowserProcessMap()
	table := getTCPTable()
	// group by process
	tcpRowByProcessNameMap := make(map[string][]*MIB_TCPROW2)
	for i := uint32(0); i < uint32(table.dwNumEntries); i++ {
		row := table.table[i]
		if row.dwOwningPid <= 0 {
			continue
		}
		// remoteAddr := row.displayIP(row.dwRemoteAddr)
		// if _, ok := hostConfigMap[remoteAddr]; ok {
		// common.Info("====== remoteAddr= %v\tbrowserProcessMap = %v\tpid = %v", remoteAddr, browserProcessMap, row.dwOwningPid)
		if processName, ok := browserProcessMap[uint32(row.dwOwningPid)]; ok {
			pidSlice, ok := tcpRowByProcessNameMap[processName]
			if !ok {
				pidSlice = []*MIB_TCPROW2{}
			}
			pidSlice = append(pidSlice, row)
			tcpRowByProcessNameMap[processName] = pidSlice
		}
		// }
		// common.Info("\t%-6d\t%s:%-16d\t%s:%-16d\t%d\t%d\n", row.dwState, row.displayIP(row.dwLocalAddr), row.displayPort(row.dwLocalPort), row.displayIP(row.dwRemoteAddr), row.displayPort(row.dwRemotePort), row.dwOwningPid, row.dwOffloadState)
	}
	browsers := []string{}
	for k := range tcpRowByProcessNameMap {
		browsers = append(browsers, k)
	}
	common.Info("Browsers: %v", browsers)
	for processName, rowSlice := range tcpRowByProcessNameMap {
		success := true
		for _, row := range rowSlice {
			if err := CloseTCPEntry(row); err != nil {
				success = false
				common.Error("Fail to close TCP connections: %s, %v\n", processName, row.dwOwningPid)
			}
		}
		if success {
			common.Info("Succeed to close TCP connections: %s", processName)
		}
	}
}

func readHostConfigMap(path string) map[string]string {
	hostConfigMap := make(map[string]string)
	file, err := os.Open(path)
	if err != nil {
		common.Error("Fail to open system_hosts: %s", err)
	}
	defer file.Close()
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		if line != "" && !strings.HasPrefix(line, "#") {
			config := trimDuplicateSpaces(line)
			if len(config) == 2 {
				hostConfigMap[config[1]] = config[0]
			}
		}
	}
	if err := scanner.Err(); err != nil {
		common.Error("Fail to read system_hosts: %s", err)
	}
	return hostConfigMap
}

func trimDuplicateSpaces(line string) []string {
	temp := []string{}
	line = strings.TrimSpace(line)
	for _, v := range strings.SplitN(line, " ", 2) {
		if trimed := strings.TrimSpace(v); trimed != "" {
			temp = append(temp, trimed)
		}
	}
	return temp
}

func initSystemHostsWatcher() *Batcher {
	batcher, err := New(time.Millisecond * 300)
	if err != nil {
		common.Error("Fail to initialize batcher")
	}
	if err = batcher.Add(systemHosts); err != nil {
		common.Error("Fail to add system hosts: %s", systemHosts)
	}
	return batcher
}

func startSystemHostsWatcher() {
	if batcher == nil {
		common.Error("Fail to start system hosts watcher, watcher is nil")
		return
	}
	for events := range batcher.Events {
		for _, event := range events {
			if event.Op&fsnotify.Write == fsnotify.Write {
				common.Info("modified file: %v", event)
				doProcess()
				break
			}
		}
	}
	// never return
}
<commit_msg>misc: Fix build error<commit_after>package core

import (
	"bufio"
	"os"
	"strings"

	"github.com/tinycedar/lily/common"
	"github.com/tinycedar/lily/conf"
)

const (
	systemHosts = "C:/Windows/System32/drivers/etc/hosts"
)

// var batcher *Batcher

func FireHostsSwitch() {
	common.Info("============================== Fire hosts switch ==============================")
	// if batcher != nil {
	// 	batcher.Close()
	// }
	doProcess()
	// batcher = initSystemHostsWatcher()
	// go startSystemHostsWatcher()
}

// 1. Find collection of same domain names between system hosts and currentHostIndex
// 2. Disconnect the TCP connections(http:80 & https:443) of collection found above
func doProcess() {
	overlapHostConfigMap := getOverlapHostConfigMap()
	common.Info("overlapHostConfigMap: %v", overlapHostConfigMap)
	if len(overlapHostConfigMap) == 0 {
		return
	}
	table := getTCPTable()
	for i := uint32(0); i < uint32(table.dwNumEntries); i++ {
		row := table.table[i]
		if row.dwOwningPid <= 0 {
			continue
		}
		ip := row.displayIP(row.dwRemoteAddr)
		port := row.displayPort(row.dwRemotePort)
		if _, ok := overlapHostConfigMap[ip]; !ok {
			continue
		}
		if port == 80 || port == 443 {
			if err := CloseTCPEntry(row); err != nil {
				common.Error("Fail to close TCP connections: Pid = %v, Addr = %v:%v\n", row.dwOwningPid, ip, port)
			} else {
				common.Info("Succeed to close TCP connections: Pid = %v, Addr = %v:%v", row.dwOwningPid, ip, port)
			}
		}
	}
}

func getOverlapHostConfigMap() map[string]bool {
	result := make(map[string]bool)
	var currentHostConfigMap map[string]string
	if current := conf.Config.HostConfigModel.RootAt(conf.Config.CurrentHostIndex); current != nil {
		currentHostConfigMap = readHostConfigMap("conf/hosts/" + current.Text() + ".hosts")
	}
	common.Info("currentHostConfigMap: %v", currentHostConfigMap)
	if len(currentHostConfigMap) == 0 {
		return result
	}
	common.Info("systemConfigMap: %v", readHostConfigMap(systemHosts))
	for k, v := range readHostConfigMap(systemHosts) {
		v2, ok := currentHostConfigMap[k]
		common.Info("k: %s, %s - %s", k, v, v2)
		if ok && v != v2 {
			result[v] = true
		}
	}
	return result
}

func process() {
	// hostConfigMap := readFile()
	browserProcessMap := getBrowserProcessMap()
	table := getTCPTable()
	// group by process
	tcpRowByProcessNameMap := make(map[string][]*MIB_TCPROW2)
	for i := uint32(0); i < uint32(table.dwNumEntries); i++ {
		row := table.table[i]
		if row.dwOwningPid <= 0 {
			continue
		}
		// remoteAddr := row.displayIP(row.dwRemoteAddr)
		// if _, ok := hostConfigMap[remoteAddr]; ok {
		// common.Info("====== remoteAddr= %v\tbrowserProcessMap = %v\tpid = %v", remoteAddr, browserProcessMap, row.dwOwningPid)
		if processName, ok := browserProcessMap[uint32(row.dwOwningPid)]; ok {
			pidSlice, ok := tcpRowByProcessNameMap[processName]
			if !ok {
				pidSlice = []*MIB_TCPROW2{}
			}
			pidSlice = append(pidSlice, row)
			tcpRowByProcessNameMap[processName] = pidSlice
		}
		// }
		// common.Info("\t%-6d\t%s:%-16d\t%s:%-16d\t%d\t%d\n", row.dwState, row.displayIP(row.dwLocalAddr), row.displayPort(row.dwLocalPort), row.displayIP(row.dwRemoteAddr), row.displayPort(row.dwRemotePort), row.dwOwningPid, row.dwOffloadState)
	}
	browsers := []string{}
	for k := range tcpRowByProcessNameMap {
		browsers = append(browsers, k)
	}
	common.Info("Browsers: %v", browsers)
	for processName, rowSlice := range tcpRowByProcessNameMap {
		success := true
		for _, row := range rowSlice {
			if err := CloseTCPEntry(row); err != nil {
				success = false
				common.Error("Fail to close TCP connections: %s, %v\n", processName, row.dwOwningPid)
			}
		}
		if success {
			common.Info("Succeed to close TCP connections: %s", processName)
		}
	}
}

func readHostConfigMap(path string) map[string]string {
	hostConfigMap := make(map[string]string)
	file, err := os.Open(path)
	if err != nil {
		common.Error("Fail to open system_hosts: %s", err)
	}
	defer file.Close()
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		if line != "" && !strings.HasPrefix(line, "#") {
			config := trimDuplicateSpaces(line)
			if len(config) == 2 {
				hostConfigMap[config[1]] = config[0]
			}
		}
	}
	if err := scanner.Err(); err != nil {
		common.Error("Fail to read system_hosts: %s", err)
	}
	return hostConfigMap
}

func trimDuplicateSpaces(line string) []string {
	temp := []string{}
	line = strings.TrimSpace(line)
	for _, v := range strings.SplitN(line, " ", 2) {
		if trimed := strings.TrimSpace(v); trimed != "" {
			temp = append(temp, trimed)
		}
	}
	return temp
}

// func initSystemHostsWatcher() *Batcher {
// 	batcher, err := New(time.Millisecond * 300)
// 	if err != nil {
// 		common.Error("Fail to initialize batcher")
// 	}
// 	if err = batcher.Add(systemHosts); err != nil {
// 		common.Error("Fail to add system hosts: %s", systemHosts)
// 	}
// 	return batcher
// }

// func startSystemHostsWatcher() {
// 	if batcher == nil {
// 		common.Error("Fail to start system hosts watcher, watcher is nil")
// 		return
// 	}
// 	for events := range batcher.Events {
// 		for _, event := range events {
// 			if event.Op&fsnotify.Write == fsnotify.Write {
// 				common.Info("modified file: %v", event)
// 				doProcess()
// 				break
// 			}
// 		}
// 	}
// 	// never return
// }
<|endoftext|>


<commit_before>// gorewind is an event store server written in Python that talks ZeroMQ.
// Copyright (C) 2013  Jens Rantil
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

// Contains the server loop. Deals with incoming requests and delegates
// them to the event store.
package server

import (
	"bytes"
	"errors"
	"log"
	"container/list"
	"time"
	"sync"
	zmq "github.com/alecthomas/gozmq"
)

type InitParams struct {
	Store *EventStore
	CommandSocketZPath *string
	EvPubSocketZPath *string
}

// Check all required initialization parameters are set.
func checkAllInitParamsSet(p *InitParams) error {
	if p.Store == nil {
		return errors.New("Missing param: Store")
	}
	if p.CommandSocketZPath == nil {
		return errors.New("Missing param: CommandSocketZPath")
	}
	if p.EvPubSocketZPath == nil {
		return errors.New("Missing param: EvPubSocketZPath")
	}
	return nil
}

// A server instance. Can be run.
type Server struct {
	params InitParams

	evpubsock *zmq.Socket
	commandsock *zmq.Socket
	context *zmq.Context

	runningMutex sync.Mutex
	running bool
	stopChan chan bool
}

// IsRunning returns true if the server is running, false otherwise.
func (v *Server) IsRunning() bool {
	v.runningMutex.Lock()
	defer v.runningMutex.Unlock()
	return v.running
}

// Stop stops the server.
func (v* Server) Stop() error {
	if v.IsRunning() {
		return errors.New("Not running.")
	}

	select {
	case v.stopChan <- true:
	default:
		return errors.New("Stop already signalled.")
	}
	<-v.stopChan
	// v.running is modified by Server.Run(...)

	if v.IsRunning() {
		return errors.New("Signalled stopped, but never stopped.")
	}

	return nil
}

// Initialize a new event store server and return a handle to it. The
// event store is not started. It's up to the caller to execute Run()
// on the server handle.
func New(params *InitParams) (*Server, error) {
	if params == nil {
		return nil, errors.New("Missing init params")
	}
	if err := checkAllInitParamsSet(params); err != nil {
		return nil, err
	}

	server := Server{
		params: *params,
		running: false,
	}

	var allOkay *bool = new(bool)
	*allOkay = false
	defer func() {
		if (!*allOkay) {
			server.closeZmq()
		}
	}()

	context, err := zmq.NewContext()
	if err != nil {
		return nil, err
	}
	server.context = &context

	commandsock, err := context.NewSocket(zmq.ROUTER)
	if err != nil {
		return nil, err
	}
	server.commandsock = &commandsock
	err = commandsock.Bind(*params.CommandSocketZPath)
	if err != nil {
		return nil, err
	}

	evpubsock, err := context.NewSocket(zmq.PUB)
	if err != nil {
		return nil, err
	}
	server.evpubsock = &evpubsock
	if binderr := evpubsock.Bind(*params.EvPubSocketZPath); binderr != nil {
		return nil, err
	}

	*allOkay = true

	return &server, nil
}

func (v *Server) closeZmq() {
	(*v.evpubsock).Close()
	v.evpubsock = nil
	(*v.commandsock).Close()
	v.commandsock = nil
	(*v.context).Close()
	v.context = nil
}

func (v *Server) setRunningState(newState bool) {
	v.runningMutex.Lock()
	defer v.runningMutex.Unlock()
	v.running = newState
}

// Runs the server that distributes requests to workers.
// Panics on error since it is an essential piece of code required to
// run the application correctly.
func (v *Server) Run() {
	v.setRunningState(true)
	defer v.setRunningState(false)
	loopServer((*v).params.Store, *(*v).evpubsock, *(*v).commandsock, v.stopChan)
}

// The result of an asynchronous zmq.Poll call.
type zmqPollResult struct {
	err error
}

// Polls a bunch of ZeroMQ sockets and notifies the result through a
// channel. This makes it possible to combine ZeroMQ polling with Go's
// own built-in channels.
func asyncPoll(notifier chan zmqPollResult, items zmq.PollItems, stop chan bool) {
	for {
		timeout := time.Duration(1)*time.Second
		count, err := zmq.Poll(items, int64(timeout))
		if count > 0 || err != nil {
			notifier <- zmqPollResult{err}
		}

		select {
		case <-stop:
			stop <- true
			return
		default:
		}
	}
}

func stopPoller(cancelChan chan bool) {
	cancelChan <- true
	<-cancelChan
}

// The core ZeroMQ messaging loop. Handles requests and responses
// asynchronously using the router socket. Every request is delegated to
// a goroutine for maximum concurrency.
//
// `gozmq` does currently not support copy-free messages/frames. This
// means that every message passing through this function needs to be
// copied in-memory. If this becomes a bottleneck in the future,
// multiple router sockets can be hooked to this final router to scale
// message copying.
//
// TODO: Make this a type function of `Server` to remove a lot of
// parameters.
func loopServer(estore *EventStore, evpubsock, frontend zmq.Socket,
stop chan bool) {
	toPoll := zmq.PollItems{
		zmq.PollItem{Socket: frontend, zmq.Events: zmq.POLLIN},
	}

	pubchan := make(chan StoredEvent)
	estore.RegisterPublishedEventsChannel(pubchan)
	go publishAllSavedEvents(pubchan, evpubsock)

	pollchan := make(chan zmqPollResult)
	respchan := make(chan zMsg)

	pollCancel := make(chan bool)
	defer stopPoller(pollCancel)

	go asyncPoll(pollchan, toPoll, pollCancel)
	for {
		select {
		case res := <-pollchan:
			if res.err != nil {
				log.Print("Could not poll:", res.err)
			}
			if res.err == nil && toPoll[0].REvents&zmq.POLLIN != 0 {
				msg, _ := toPoll[0].Socket.RecvMultipart(0)
				zmsg := zMsg(msg)
				go handleRequest(respchan, estore, zmsg)
			}
			go asyncPoll(pollchan, toPoll, pollCancel)
		case frames := <-respchan:
			if err := frontend.SendMultipart(frames, 0); err != nil {
				log.Println(err)
			}
		case <- stop:
			stop <- true
			return
		}
	}
}

// Publishes stored events to event listeners.
//
// Pops previously stored messages off a channel and published them to a
// ZeroMQ socket.
func publishAllSavedEvents(toPublish chan StoredEvent, evpub zmq.Socket) {
	msg := make(zMsg, 3)
	for {
		event := <-toPublish

		msg[0] = event.Stream
		msg[1] = event.Id
		msg[2] = event.Data

		if err := evpub.SendMultipart(msg, 0); err != nil {
			log.Println(err)
		}
	}
}

// A single frame in a ZeroMQ message.
type zFrame []byte

// A ZeroMQ message.
//
// I wish it could have been `[]zFrame`, but that would make conversion
// from `[][]byte` pretty messy[1].
//
// [1] http://stackoverflow.com/a/15650327/260805
type zMsg [][]byte

// Handles a single ZeroMQ RES/REQ loop synchronously.
//
// The full request message stored in `msg` and the full ZeroMQ response
// is pushed to `respchan`. The function does not return any error
// because it is expected to be called asynchronously as a goroutine.
func handleRequest(respchan chan zMsg, estore *EventStore, msg zMsg) {

	// TODO: Rename to 'framelist'
	parts := list.New()
	for _, msgpart := range msg {
		parts.PushBack(msgpart)
	}

	resptemplate := list.New()
	emptyFrame := zFrame("")
	for true {
		resptemplate.PushBack(parts.Remove(parts.Front()))

		if bytes.Equal(parts.Front().Value.(zFrame), emptyFrame) {
			break
		}
	}

	if parts.Len() == 0 {
		errstr := "Incoming command was empty. Ignoring it."
		log.Println(errstr)
		response := copyList(resptemplate)
		response.PushBack(zFrame("ERROR " + errstr))
		respchan <- listToFrames(response)
		return
	}

	command := string(parts.Front().Value.(zFrame))
	switch command {
	case "PUBLISH":
		parts.Remove(parts.Front())
		if parts.Len() != 2 {
			// TODO: Constantify this error message
			errstr := "Wrong number of frames for PUBLISH."
			log.Println(errstr)
			response := copyList(resptemplate)
			response.PushBack(zFrame("ERROR " + errstr))
			respchan <- listToFrames(response)
		} else {
			estream := parts.Remove(parts.Front())
			data := parts.Remove(parts.Front())
			newevent := UnstoredEvent{
				Stream: estream.(StreamName),
				Data: data.(zFrame),
			}
			newId, err := estore.Add(newevent)
			if err != nil {
				sErr := err.Error()
				log.Println(sErr)

				response := copyList(resptemplate)
				response.PushBack(zFrame("ERROR " + sErr))
				respchan <- listToFrames(response)
			} else {
				// the event was added
				response := copyList(resptemplate)
				response.PushBack(zFrame("PUBLISHED"))
				response.PushBack(zFrame(newId))
				respchan <- listToFrames(response)
			}
		}
	case "QUERY":
		parts.Remove(parts.Front())
		if parts.Len() != 3 {
			// TODO: Constantify this error message
			errstr := "Wrong number of frames for QUERY."
			log.Println(errstr)
			response := copyList(resptemplate)
			response.PushBack(zFrame("ERROR " + errstr))
			respchan <- listToFrames(response)
		} else {
			estream := parts.Remove(parts.Front())
			fromid := parts.Remove(parts.Front())
			toid := parts.Remove(parts.Front())

			req := QueryRequest{
				Stream: estream.(zFrame),
				FromId: fromid.(zFrame),
				ToId: toid.(zFrame),
			}
			events := make(chan StoredEvent)
			// TODO: Handle errors returned below
			go estore.Query(req, events)
			for eventdata := range(events) {
				response := copyList(resptemplate)
				response.PushBack(eventdata.Stream)
				response.PushBack(eventdata.Id)
				response.PushBack(eventdata.Data)

				respchan <- listToFrames(response)
			}
			response := copyList(resptemplate)
			response.PushBack(zFrame("END"))
			respchan <- listToFrames(response)
		}
	default:
		// TODO: Move these error strings out as constants of
		//       this package.

		// TODO: Move the chunk of code below into a separate
		// function and reuse for similar piece of code above.
		// TODO: Constantify this error message
		errstr := "Unknown request type."
		log.Println(errstr)
		response := copyList(resptemplate)
		response.PushBack(zFrame("ERROR " + errstr))
		respchan <- listToFrames(response)
	}
}

// Convert a doubly linked list of message frames to a slice of message
// fram
func listToFrames(l *list.List) zMsg {
	frames := make(zMsg, l.Len())
	i := 0
	for e := l.Front(); e != nil; e = e.Next() {
		frames[i] = e.Value.(zFrame)
	}
	return frames
}

// Helper function for copying a doubly linked list.
func copyList(l *list.List) *list.List {
	replica := list.New()
	replica.PushBackList(l)
	return replica
}

<commit_msg>Query result events are prepended with "EVENT"<commit_after>// gorewind is an event store server written in Python that talks ZeroMQ.
// Copyright (C) 2013  Jens Rantil
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

// Contains the server loop. Deals with incoming requests and delegates
// them to the event store.
package server

import (
	"bytes"
	"errors"
	"log"
	"container/list"
	"time"
	"sync"
	zmq "github.com/alecthomas/gozmq"
)

type InitParams struct {
	Store *EventStore
	CommandSocketZPath *string
	EvPubSocketZPath *string
}

// Check all required initialization parameters are set.
func checkAllInitParamsSet(p *InitParams) error {
	if p.Store == nil {
		return errors.New("Missing param: Store")
	}
	if p.CommandSocketZPath == nil {
		return errors.New("Missing param: CommandSocketZPath")
	}
	if p.EvPubSocketZPath == nil {
		return errors.New("Missing param: EvPubSocketZPath")
	}
	return nil
}

// A server instance. Can be run.
type Server struct {
	params InitParams

	evpubsock *zmq.Socket
	commandsock *zmq.Socket
	context *zmq.Context

	runningMutex sync.Mutex
	running bool
	stopChan chan bool
}

// IsRunning returns true if the server is running, false otherwise.
func (v *Server) IsRunning() bool {
	v.runningMutex.Lock()
	defer v.runningMutex.Unlock()
	return v.running
}

// Stop stops the server.
func (v* Server) Stop() error {
	if v.IsRunning() {
		return errors.New("Not running.")
	}

	select {
	case v.stopChan <- true:
	default:
		return errors.New("Stop already signalled.")
	}
	<-v.stopChan
	// v.running is modified by Server.Run(...)

	if v.IsRunning() {
		return errors.New("Signalled stopped, but never stopped.")
	}

	return nil
}

// Initialize a new event store server and return a handle to it. The
// event store is not started. It's up to the caller to execute Run()
// on the server handle.
func New(params *InitParams) (*Server, error) {
	if params == nil {
		return nil, errors.New("Missing init params")
	}
	if err := checkAllInitParamsSet(params); err != nil {
		return nil, err
	}

	server := Server{
		params: *params,
		running: false,
	}

	var allOkay *bool = new(bool)
	*allOkay = false
	defer func() {
		if (!*allOkay) {
			server.closeZmq()
		}
	}()

	context, err := zmq.NewContext()
	if err != nil {
		return nil, err
	}
	server.context = &context

	commandsock, err := context.NewSocket(zmq.ROUTER)
	if err != nil {
		return nil, err
	}
	server.commandsock = &commandsock
	err = commandsock.Bind(*params.CommandSocketZPath)
	if err != nil {
		return nil, err
	}

	evpubsock, err := context.NewSocket(zmq.PUB)
	if err != nil {
		return nil, err
	}
	server.evpubsock = &evpubsock
	if binderr := evpubsock.Bind(*params.EvPubSocketZPath); binderr != nil {
		return nil, err
	}

	*allOkay = true

	return &server, nil
}

func (v *Server) closeZmq() {
	(*v.evpubsock).Close()
	v.evpubsock = nil
	(*v.commandsock).Close()
	v.commandsock = nil
	(*v.context).Close()
	v.context = nil
}

func (v *Server) setRunningState(newState bool) {
	v.runningMutex.Lock()
	defer v.runningMutex.Unlock()
	v.running = newState
}

// Runs the server that distributes requests to workers.
// Panics on error since it is an essential piece of code required to
// run the application correctly.
func (v *Server) Run() {
	v.setRunningState(true)
	defer v.setRunningState(false)
	loopServer((*v).params.Store, *(*v).evpubsock, *(*v).commandsock, v.stopChan)
}

// The result of an asynchronous zmq.Poll call.
type zmqPollResult struct {
	err error
}

// Polls a bunch of ZeroMQ sockets and notifies the result through a
// channel. This makes it possible to combine ZeroMQ polling with Go's
// own built-in channels.
func asyncPoll(notifier chan zmqPollResult, items zmq.PollItems, stop chan bool) {
	for {
		timeout := time.Duration(1)*time.Second
		count, err := zmq.Poll(items, int64(timeout))
		if count > 0 || err != nil {
			notifier <- zmqPollResult{err}
		}

		select {
		case <-stop:
			stop <- true
			return
		default:
		}
	}
}

func stopPoller(cancelChan chan bool) {
	cancelChan <- true
	<-cancelChan
}

// The core ZeroMQ messaging loop. Handles requests and responses
// asynchronously using the router socket. Every request is delegated to
// a goroutine for maximum concurrency.
//
// `gozmq` does currently not support copy-free messages/frames. This
// means that every message passing through this function needs to be
// copied in-memory. If this becomes a bottleneck in the future,
// multiple router sockets can be hooked to this final router to scale
// message copying.
//
// TODO: Make this a type function of `Server` to remove a lot of
// parameters.
func loopServer(estore *EventStore, evpubsock, frontend zmq.Socket,
stop chan bool) {
	toPoll := zmq.PollItems{
		zmq.PollItem{Socket: frontend, zmq.Events: zmq.POLLIN},
	}

	pubchan := make(chan StoredEvent)
	estore.RegisterPublishedEventsChannel(pubchan)
	go publishAllSavedEvents(pubchan, evpubsock)

	pollchan := make(chan zmqPollResult)
	respchan := make(chan zMsg)

	pollCancel := make(chan bool)
	defer stopPoller(pollCancel)

	go asyncPoll(pollchan, toPoll, pollCancel)
	for {
		select {
		case res := <-pollchan:
			if res.err != nil {
				log.Print("Could not poll:", res.err)
			}
			if res.err == nil && toPoll[0].REvents&zmq.POLLIN != 0 {
				msg, _ := toPoll[0].Socket.RecvMultipart(0)
				zmsg := zMsg(msg)
				go handleRequest(respchan, estore, zmsg)
			}
			go asyncPoll(pollchan, toPoll, pollCancel)
		case frames := <-respchan:
			if err := frontend.SendMultipart(frames, 0); err != nil {
				log.Println(err)
			}
		case <- stop:
			stop <- true
			return
		}
	}
}

// Publishes stored events to event listeners.
//
// Pops previously stored messages off a channel and published them to a
// ZeroMQ socket.
func publishAllSavedEvents(toPublish chan StoredEvent, evpub zmq.Socket) {
	msg := make(zMsg, 3)
	for {
		event := <-toPublish

		msg[0] = event.Stream
		msg[1] = event.Id
		msg[2] = event.Data

		if err := evpub.SendMultipart(msg, 0); err != nil {
			log.Println(err)
		}
	}
}

// A single frame in a ZeroMQ message.
type zFrame []byte

// A ZeroMQ message.
//
// I wish it could have been `[]zFrame`, but that would make conversion
// from `[][]byte` pretty messy[1].
//
// [1] http://stackoverflow.com/a/15650327/260805
type zMsg [][]byte

// Handles a single ZeroMQ RES/REQ loop synchronously.
//
// The full request message stored in `msg` and the full ZeroMQ response
// is pushed to `respchan`. The function does not return any error
// because it is expected to be called asynchronously as a goroutine.
func handleRequest(respchan chan zMsg, estore *EventStore, msg zMsg) {

	// TODO: Rename to 'framelist'
	parts := list.New()
	for _, msgpart := range msg {
		parts.PushBack(msgpart)
	}

	resptemplate := list.New()
	emptyFrame := zFrame("")
	for true {
		resptemplate.PushBack(parts.Remove(parts.Front()))

		if bytes.Equal(parts.Front().Value.(zFrame), emptyFrame) {
			break
		}
	}

	if parts.Len() == 0 {
		errstr := "Incoming command was empty. Ignoring it."
		log.Println(errstr)
		response := copyList(resptemplate)
		response.PushBack(zFrame("ERROR " + errstr))
		respchan <- listToFrames(response)
		return
	}

	command := string(parts.Front().Value.(zFrame))
	switch command {
	case "PUBLISH":
		parts.Remove(parts.Front())
		if parts.Len() != 2 {
			// TODO: Constantify this error message
			errstr := "Wrong number of frames for PUBLISH."
			log.Println(errstr)
			response := copyList(resptemplate)
			response.PushBack(zFrame("ERROR " + errstr))
			respchan <- listToFrames(response)
		} else {
			estream := parts.Remove(parts.Front())
			data := parts.Remove(parts.Front())
			newevent := UnstoredEvent{
				Stream: estream.(StreamName),
				Data: data.(zFrame),
			}
			newId, err := estore.Add(newevent)
			if err != nil {
				sErr := err.Error()
				log.Println(sErr)

				response := copyList(resptemplate)
				response.PushBack(zFrame("ERROR " + sErr))
				respchan <- listToFrames(response)
			} else {
				// the event was added
				response := copyList(resptemplate)
				response.PushBack(zFrame("PUBLISHED"))
				response.PushBack(zFrame(newId))
				respchan <- listToFrames(response)
			}
		}
	case "QUERY":
		parts.Remove(parts.Front())
		if parts.Len() != 3 {
			// TODO: Constantify this error message
			errstr := "Wrong number of frames for QUERY."
			log.Println(errstr)
			response := copyList(resptemplate)
			response.PushBack(zFrame("ERROR " + errstr))
			respchan <- listToFrames(response)
		} else {
			estream := parts.Remove(parts.Front())
			fromid := parts.Remove(parts.Front())
			toid := parts.Remove(parts.Front())

			req := QueryRequest{
				Stream: estream.(zFrame),
				FromId: fromid.(zFrame),
				ToId: toid.(zFrame),
			}
			events := make(chan StoredEvent)
			// TODO: Handle errors returned below
			go estore.Query(req, events)
			for eventdata := range(events) {
				response := copyList(resptemplate)
				response.PushBack([]byte("EVENT"))
				response.PushBack(eventdata.Id)
				response.PushBack(eventdata.Data)

				respchan <- listToFrames(response)
			}
			response := copyList(resptemplate)
			response.PushBack(zFrame("END"))
			respchan <- listToFrames(response)
		}
	default:
		// TODO: Move these error strings out as constants of
		//       this package.

		// TODO: Move the chunk of code below into a separate
		// function and reuse for similar piece of code above.
		// TODO: Constantify this error message
		errstr := "Unknown request type."
		log.Println(errstr)
		response := copyList(resptemplate)
		response.PushBack(zFrame("ERROR " + errstr))
		respchan <- listToFrames(response)
	}
}

// Convert a doubly linked list of message frames to a slice of message
// fram
func listToFrames(l *list.List) zMsg {
	frames := make(zMsg, l.Len())
	i := 0
	for e := l.Front(); e != nil; e = e.Next() {
		frames[i] = e.Value.(zFrame)
	}
	return frames
}

// Helper function for copying a doubly linked list.
func copyList(l *list.List) *list.List {
	replica := list.New()
	replica.PushBackList(l)
	return replica
}

<|endoftext|>