<commit_before>package podtask

import (
	"fmt"
	"time"

	"code.google.com/p/go-uuid/uuid"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	log "github.com/golang/glog"
	mesos "github.com/mesos/mesos-go/mesosproto"
	mutil "github.com/mesos/mesos-go/mesosutil"
	"github.com/mesosphere/kubernetes-mesos/pkg/offers"
	annotation "github.com/mesosphere/kubernetes-mesos/pkg/scheduler/meta"
	"github.com/mesosphere/kubernetes-mesos/pkg/scheduler/metrics"
)

const (
	containerCpus = 0.25 // initial CPU allocated for executor
	containerMem  = 64   // initial MB of memory allocated for executor
)

type StateType int

const (
	StatePending StateType = iota
	StateRunning
	StateFinished
	StateUnknown
)

type FlagType string

const (
	Launched = FlagType("launched")
	Bound    = FlagType("bound")
	Deleted  = FlagType("deleted")
)

// A struct that describes a pod task.
type T struct {
	ID          string
	Pod         *api.Pod
	TaskInfo    *mesos.TaskInfo
	Offer       offers.Perishable
	State       StateType
	Ports       []HostPortMapping
	Flags       map[FlagType]struct{}
	podKey      string
	CreateTime  time.Time
	UpdatedTime time.Time // time of the most recent StatusUpdate we've seen from the mesos master
	launchTime  time.Time
	bindTime    time.Time
	mapper      HostPortMappingFunc
}

func (t *T) HasAcceptedOffer() bool {
	return t.TaskInfo != nil && t.TaskInfo.TaskId != nil
}

func (t *T) GetOfferId() string {
	if t.Offer == nil {
		return ""
	}
	return t.Offer.Details().Id.GetValue()
}

// Fill the TaskInfo in the T, should be called during k8s scheduling,
// before binding.
func (t *T) FillFromDetails(details *mesos.Offer) error {
	if details == nil {
		//programming error
		panic("offer details are nil")
	}

	log.V(3).Infof("Recording offer(s) %v against pod %v", details.Id, t.Pod.Name)

	t.TaskInfo.TaskId = mutil.NewTaskID(t.ID)
	t.TaskInfo.SlaveId = details.GetSlaveId()
	t.TaskInfo.Resources = []*mesos.Resource{
		mutil.NewScalarResource("cpus", containerCpus),
		mutil.NewScalarResource("mem", containerMem),
	}
	if mapping, err := t.mapper(t, details); err != nil {
		t.ClearTaskInfo()
		return err
	} else {
		ports := []uint64{}
		for _, entry := range mapping {
			ports = append(ports, entry.OfferPort)
		}
		t.Ports = mapping
		if portsResource := rangeResource("ports", ports); portsResource != nil {
			t.TaskInfo.Resources = append(t.TaskInfo.Resources, portsResource)
		}
	}
	return nil
}

// Clear offer-related details from the task, should be called if/when an offer
// has already been assigned to a task but for some reason is no longer valid.
func (t *T) ClearTaskInfo() {
	log.V(3).Infof("Clearing offer(s) from pod %v", t.Pod.Name)
	t.Offer = nil
	t.TaskInfo.TaskId = nil
	t.TaskInfo.SlaveId = nil
	t.TaskInfo.Resources = nil
	t.TaskInfo.Data = nil
	t.Ports = nil
}

func (t *T) AcceptOffer(offer *mesos.Offer) bool {
	if offer == nil {
		return false
	}
	var (
		cpus float64 = 0
		mem  float64 = 0
	)
	for _, resource := range offer.Resources {
		if resource.GetName() == "cpus" {
			cpus = *resource.GetScalar().Value
		}

		if resource.GetName() == "mem" {
			mem = *resource.GetScalar().Value
		}
	}
	if _, err := t.mapper(t, offer); err != nil {
		log.V(3).Info(err)
		return false
	}
	if (cpus < containerCpus) || (mem < containerMem) {
		log.V(3).Infof("not enough resources: cpus: %f mem: %f", cpus, mem)
		return false
	}
	return true
}

func (t *T) Set(f FlagType) {
	t.Flags[f] = struct{}{}
	if Launched == f {
		t.launchTime = time.Now()
		queueWaitTime := t.launchTime.Sub(t.CreateTime)
		metrics.QueueWaitTime.Observe(metrics.InMicroseconds(queueWaitTime))
	}
}

func (t *T) Has(f FlagType) (exists bool) {
	_, exists = t.Flags[f]
	return
}

// create a duplicate task, one that refers to the same pod specification and
// executor as the current task. all other state is reset to "factory settings"
// (as if returned from New())
func (t *T) dup() (*T, error) {
	ctx := api.WithNamespace(api.NewDefaultContext(), t.Pod.Namespace)
	return New(ctx, t.Pod, t.TaskInfo.Executor)
}

func New(ctx api.Context, pod *api.Pod, executor *mesos.ExecutorInfo) (*T, error) {
	if pod == nil {
		return nil, fmt.Errorf("illegal argument: pod was nil")
	}
	if executor == nil {
		return nil, fmt.Errorf("illegal argument: executor was nil")
	}
	key, err := MakePodKey(ctx, pod.Name)
	if err != nil {
		return nil, err
	}
	taskId := uuid.NewUUID().String()
	task := &T{
		ID:       taskId,
		Pod:      pod,
		TaskInfo: newTaskInfo("Pod"),
		State:    StatePending,
		podKey:   key,
		mapper:   defaultHostPortMapping,
		Flags:    make(map[FlagType]struct{}),
	}
	task.TaskInfo.Executor = executor
	task.CreateTime = time.Now()
	return task, nil
}

func (t *T) SaveRecoveryInfo(dict map[string]string) {
	dict[annotation.TaskIdKey] = t.ID
	dict[annotation.SlaveIdKey] = t.TaskInfo.SlaveId.GetValue()
	dict[annotation.OfferIdKey] = t.Offer.Details().Id.GetValue()
	dict[annotation.ExecutorIdKey] = t.TaskInfo.Executor.ExecutorId.GetValue()
}

// reconstruct a task from metadata stashed in a pod entry. there are limited pod states that
// support reconstruction. if we expect to be able to reconstruct state but encounter errors
// in the process then those errors are returned. if the pod is in a seemingly valid state but
// otherwise does not support task reconstruction return false. if we're able to reconstruct
// state then return a reconstructed task and true.
//
// at this time task reconstruction is only supported for pods that have been annotated with
// binding metadata, which implies that they've previously been associated with a task and
// that mesos knows about it.
//
func RecoverFrom(pod *api.Pod) (*T, bool, error) {
	if pod == nil {
		return nil, false, fmt.Errorf("illegal argument: pod was nil")
	}
	if pod.Status.Host == "" || len(pod.Annotations) == 0 {
		//TODO(jdef) if Status.Host != "" then it's likely that the task has launched
		//but is not yet bound -- so annotations may be on the way. The binding may also
		//have failed but we haven't been processed the TASK_FAILED yet.
		log.V(1).Infof("skipping recovery for unbound pod %v/%v", pod.Namespace, pod.Name)
		return nil, false, nil
	}

	// only process pods that are not in a terminal state
	switch pod.Status.Phase {
	case api.PodPending, api.PodRunning, api.PodUnknown: // continue
	default:
		log.V(1).Infof("skipping recovery for terminal pod %v/%v", pod.Namespace, pod.Name)
		return nil, false, nil
	}

	ctx := api.WithNamespace(api.NewDefaultContext(), pod.Namespace)
	key, err := MakePodKey(ctx, pod.Name)
	if err != nil {
		return nil, false, err
	}

	//TODO(jdef) recover ports (and other resource requirements?) from the pod spec as well

	now := time.Now()
	t := &T{
		Pod:        pod,
		TaskInfo:   newTaskInfo("Pod"),
		CreateTime: now,
		podKey:     key,
		State:      StatePending, // possibly running? mesos will tell us during reconciliation
		Flags:      make(map[FlagType]struct{}),
		mapper:     defaultHostPortMapping,
		launchTime: now,
		bindTime:   now,
	}
	var (
		offerId  string
		hostname string
	)
	for _, k := range []string{
		annotation.BindingHostKey,
		annotation.TaskIdKey,
		annotation.SlaveIdKey,
		annotation.OfferIdKey,
		annotation.ExecutorIdKey,
	} {
		v, found := pod.Annotations[k]
		if !found {
			return nil, false, fmt.Errorf("incomplete metadata: missing value for pod annotation: %v", k)
		}
		switch k {
		case annotation.BindingHostKey:
			hostname = v
		case annotation.SlaveIdKey:
			t.TaskInfo.SlaveId = mutil.NewSlaveID(v)
		case annotation.OfferIdKey:
			offerId = v
		case annotation.TaskIdKey:
			t.ID = v
			t.TaskInfo.TaskId = mutil.NewTaskID(v)
		case annotation.ExecutorIdKey:
			t.TaskInfo.Executor = &mesos.ExecutorInfo{ExecutorId: mutil.NewExecutorID(v)}
		}
	}
	t.Offer = offers.Expired(offerId, hostname, 0)
	t.Flags[Launched] = struct{}{}
	t.Flags[Bound] = struct{}{}
	return t, true, nil
}

type HostPortMapping struct {
	ContainerIdx int // index of the container in the pod spec
	PortIdx      int // index of the port in a container's port spec
	OfferPort    uint64
}

// abstracts the way that host ports are mapped to pod container ports
type HostPortMappingFunc func(t *T, offer *mesos.Offer) ([]HostPortMapping, error)

type PortAllocationError struct {
	PodId string
	Ports []uint64
}

func (err *PortAllocationError) Error() string {
	return fmt.Sprintf("Could not schedule pod %s: %d port(s) could not be allocated", err.PodId, len(err.Ports))
}

type DuplicateHostPortError struct {
	m1, m2 HostPortMapping
}

func (err *DuplicateHostPortError) Error() string {
	return fmt.Sprintf(
		"Host port %d is specified for container %d, pod %d and container %d, pod %d",
		err.m1.OfferPort, err.m1.ContainerIdx, err.m1.PortIdx, err.m2.ContainerIdx, err.m2.PortIdx)
}

// default k8s host port mapping implementation: hostPort == 0 means containerPort remains pod-private
func defaultHostPortMapping(t *T, offer *mesos.Offer) ([]HostPortMapping, error) {
	requiredPorts := make(map[uint64]HostPortMapping)
	mapping := []HostPortMapping{}
	if t.Pod == nil {
		// programming error
		panic("task.Pod is nil")
	}
	for i, container := range t.Pod.Spec.Containers {
		// strip all port==0 from this array; k8s already knows what to do with zero-
		// ports (it does not create 'port bindings' on the minion-host); we need to
		// remove the wildcards from this array since they don't consume host resources
		for pi, port := range container.Ports {
			if port.HostPort == 0 {
				continue // ignore
			}
			m := HostPortMapping{
				ContainerIdx: i,
				PortIdx:      pi,
				OfferPort:    uint64(port.HostPort),
			}
			if entry, inuse := requiredPorts[uint64(port.HostPort)]; inuse {
				return nil, &DuplicateHostPortError{entry, m}
			}
			requiredPorts[uint64(port.HostPort)] = m
		}
	}
	for _, resource := range offer.Resources {
		if resource.GetName() == "ports" {
			for _, r := range (*resource).GetRanges().Range {
				bp := r.GetBegin()
				ep := r.GetEnd()
				for port, _ := range requiredPorts {
					log.V(3).Infof("evaluating port range {%d:%d} %d", bp, ep, port)
					if (bp <= port) && (port <= ep) {
						mapping = append(mapping, requiredPorts[port])
						delete(requiredPorts, port)
					}
				}
			}
		}
	}
	unsatisfiedPorts := len(requiredPorts)
	if unsatisfiedPorts > 0 {
		err := &PortAllocationError{
			PodId: t.Pod.Name,
		}
		for p, _ := range requiredPorts {
			err.Ports = append(err.Ports, p)
		}
		return nil, err
	}
	return mapping, nil
}
<commit_msg>removed TODO<commit_after>package podtask

import (
	"fmt"
	"time"

	"code.google.com/p/go-uuid/uuid"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	log "github.com/golang/glog"
	mesos "github.com/mesos/mesos-go/mesosproto"
	mutil "github.com/mesos/mesos-go/mesosutil"
	"github.com/mesosphere/kubernetes-mesos/pkg/offers"
	annotation "github.com/mesosphere/kubernetes-mesos/pkg/scheduler/meta"
	"github.com/mesosphere/kubernetes-mesos/pkg/scheduler/metrics"
)

const (
	containerCpus = 0.25 // initial CPU allocated for executor
	containerMem  = 64   // initial MB of memory allocated for executor
)

type StateType int

const (
	StatePending StateType = iota
	StateRunning
	StateFinished
	StateUnknown
)

type FlagType string

const (
	Launched = FlagType("launched")
	Bound    = FlagType("bound")
	Deleted  = FlagType("deleted")
)

// A struct that describes a pod task.
type T struct {
	ID          string
	Pod         *api.Pod
	TaskInfo    *mesos.TaskInfo
	Offer       offers.Perishable
	State       StateType
	Ports       []HostPortMapping
	Flags       map[FlagType]struct{}
	podKey      string
	CreateTime  time.Time
	UpdatedTime time.Time // time of the most recent StatusUpdate we've seen from the mesos master
	launchTime  time.Time
	bindTime    time.Time
	mapper      HostPortMappingFunc
}

func (t *T) HasAcceptedOffer() bool {
	return t.TaskInfo != nil && t.TaskInfo.TaskId != nil
}

func (t *T) GetOfferId() string {
	if t.Offer == nil {
		return ""
	}
	return t.Offer.Details().Id.GetValue()
}

// Fill the TaskInfo in the T, should be called during k8s scheduling,
// before binding.
func (t *T) FillFromDetails(details *mesos.Offer) error {
	if details == nil {
		//programming error
		panic("offer details are nil")
	}

	log.V(3).Infof("Recording offer(s) %v against pod %v", details.Id, t.Pod.Name)

	t.TaskInfo.TaskId = mutil.NewTaskID(t.ID)
	t.TaskInfo.SlaveId = details.GetSlaveId()
	t.TaskInfo.Resources = []*mesos.Resource{
		mutil.NewScalarResource("cpus", containerCpus),
		mutil.NewScalarResource("mem", containerMem),
	}
	if mapping, err := t.mapper(t, details); err != nil {
		t.ClearTaskInfo()
		return err
	} else {
		ports := []uint64{}
		for _, entry := range mapping {
			ports = append(ports, entry.OfferPort)
		}
		t.Ports = mapping
		if portsResource := rangeResource("ports", ports); portsResource != nil {
			t.TaskInfo.Resources = append(t.TaskInfo.Resources, portsResource)
		}
	}
	return nil
}

// Clear offer-related details from the task, should be called if/when an offer
// has already been assigned to a task but for some reason is no longer valid.
func (t *T) ClearTaskInfo() {
	log.V(3).Infof("Clearing offer(s) from pod %v", t.Pod.Name)
	t.Offer = nil
	t.TaskInfo.TaskId = nil
	t.TaskInfo.SlaveId = nil
	t.TaskInfo.Resources = nil
	t.TaskInfo.Data = nil
	t.Ports = nil
}

func (t *T) AcceptOffer(offer *mesos.Offer) bool {
	if offer == nil {
		return false
	}
	var (
		cpus float64 = 0
		mem  float64 = 0
	)
	for _, resource := range offer.Resources {
		if resource.GetName() == "cpus" {
			cpus = *resource.GetScalar().Value
		}

		if resource.GetName() == "mem" {
			mem = *resource.GetScalar().Value
		}
	}
	if _, err := t.mapper(t, offer); err != nil {
		log.V(3).Info(err)
		return false
	}
	if (cpus < containerCpus) || (mem < containerMem) {
		log.V(3).Infof("not enough resources: cpus: %f mem: %f", cpus, mem)
		return false
	}
	return true
}

func (t *T) Set(f FlagType) {
	t.Flags[f] = struct{}{}
	if Launched == f {
		t.launchTime = time.Now()
		queueWaitTime := t.launchTime.Sub(t.CreateTime)
		metrics.QueueWaitTime.Observe(metrics.InMicroseconds(queueWaitTime))
	}
}

func (t *T) Has(f FlagType) (exists bool) {
	_, exists = t.Flags[f]
	return
}

// create a duplicate task, one that refers to the same pod specification and
// executor as the current task. all other state is reset to "factory settings"
// (as if returned from New())
func (t *T) dup() (*T, error) {
	ctx := api.WithNamespace(api.NewDefaultContext(), t.Pod.Namespace)
	return New(ctx, t.Pod, t.TaskInfo.Executor)
}

func New(ctx api.Context, pod *api.Pod, executor *mesos.ExecutorInfo) (*T, error) {
	if pod == nil {
		return nil, fmt.Errorf("illegal argument: pod was nil")
	}
	if executor == nil {
		return nil, fmt.Errorf("illegal argument: executor was nil")
	}
	key, err := MakePodKey(ctx, pod.Name)
	if err != nil {
		return nil, err
	}
	taskId := uuid.NewUUID().String()
	task := &T{
		ID:       taskId,
		Pod:      pod,
		TaskInfo: newTaskInfo("Pod"),
		State:    StatePending,
		podKey:   key,
		mapper:   defaultHostPortMapping,
		Flags:    make(map[FlagType]struct{}),
	}
	task.TaskInfo.Executor = executor
	task.CreateTime = time.Now()
	return task, nil
}

func (t *T) SaveRecoveryInfo(dict map[string]string) {
	dict[annotation.TaskIdKey] = t.ID
	dict[annotation.SlaveIdKey] = t.TaskInfo.SlaveId.GetValue()
	dict[annotation.OfferIdKey] = t.Offer.Details().Id.GetValue()
	dict[annotation.ExecutorIdKey] = t.TaskInfo.Executor.ExecutorId.GetValue()
}

// reconstruct a task from metadata stashed in a pod entry. there are limited pod states that
// support reconstruction. if we expect to be able to reconstruct state but encounter errors
// in the process then those errors are returned. if the pod is in a seemingly valid state but
// otherwise does not support task reconstruction return false. if we're able to reconstruct
// state then return a reconstructed task and true.
//
// at this time task reconstruction is only supported for pods that have been annotated with
// binding metadata, which implies that they've previously been associated with a task and
// that mesos knows about it.
//
func RecoverFrom(pod *api.Pod) (*T, bool, error) {
	if pod == nil {
		return nil, false, fmt.Errorf("illegal argument: pod was nil")
	}
	if pod.Status.Host == "" || len(pod.Annotations) == 0 {
		// if Status.Host != "" then it's likely that the task has launched
		// but is not yet bound -- so annotations may be on the way. The binding
		// may also have failed but we haven't been processed the TASK_FAILED yet.
		log.V(1).Infof("skipping recovery for unbound pod %v/%v", pod.Namespace, pod.Name)
		return nil, false, nil
	}

	// only process pods that are not in a terminal state
	switch pod.Status.Phase {
	case api.PodPending, api.PodRunning, api.PodUnknown: // continue
	default:
		log.V(1).Infof("skipping recovery for terminal pod %v/%v", pod.Namespace, pod.Name)
		return nil, false, nil
	}

	ctx := api.WithNamespace(api.NewDefaultContext(), pod.Namespace)
	key, err := MakePodKey(ctx, pod.Name)
	if err != nil {
		return nil, false, err
	}

	//TODO(jdef) recover ports (and other resource requirements?) from the pod spec as well

	now := time.Now()
	t := &T{
		Pod:        pod,
		TaskInfo:   newTaskInfo("Pod"),
		CreateTime: now,
		podKey:     key,
		State:      StatePending, // possibly running? mesos will tell us during reconciliation
		Flags:      make(map[FlagType]struct{}),
		mapper:     defaultHostPortMapping,
		launchTime: now,
		bindTime:   now,
	}
	var (
		offerId  string
		hostname string
	)
	for _, k := range []string{
		annotation.BindingHostKey,
		annotation.TaskIdKey,
		annotation.SlaveIdKey,
		annotation.OfferIdKey,
		annotation.ExecutorIdKey,
	} {
		v, found := pod.Annotations[k]
		if !found {
			return nil, false, fmt.Errorf("incomplete metadata: missing value for pod annotation: %v", k)
		}
		switch k {
		case annotation.BindingHostKey:
			hostname = v
		case annotation.SlaveIdKey:
			t.TaskInfo.SlaveId = mutil.NewSlaveID(v)
		case annotation.OfferIdKey:
			offerId = v
		case annotation.TaskIdKey:
			t.ID = v
			t.TaskInfo.TaskId = mutil.NewTaskID(v)
		case annotation.ExecutorIdKey:
			t.TaskInfo.Executor = &mesos.ExecutorInfo{ExecutorId: mutil.NewExecutorID(v)}
		}
	}
	t.Offer = offers.Expired(offerId, hostname, 0)
	t.Flags[Launched] = struct{}{}
	t.Flags[Bound] = struct{}{}
	return t, true, nil
}

type HostPortMapping struct {
	ContainerIdx int // index of the container in the pod spec
	PortIdx      int // index of the port in a container's port spec
	OfferPort    uint64
}

// abstracts the way that host ports are mapped to pod container ports
type HostPortMappingFunc func(t *T, offer *mesos.Offer) ([]HostPortMapping, error)

type PortAllocationError struct {
	PodId string
	Ports []uint64
}

func (err *PortAllocationError) Error() string {
	return fmt.Sprintf("Could not schedule pod %s: %d port(s) could not be allocated", err.PodId, len(err.Ports))
}

type DuplicateHostPortError struct {
	m1, m2 HostPortMapping
}

func (err *DuplicateHostPortError) Error() string {
	return fmt.Sprintf(
		"Host port %d is specified for container %d, pod %d and container %d, pod %d",
		err.m1.OfferPort, err.m1.ContainerIdx, err.m1.PortIdx, err.m2.ContainerIdx, err.m2.PortIdx)
}

// default k8s host port mapping implementation: hostPort == 0 means containerPort remains pod-private
func defaultHostPortMapping(t *T, offer *mesos.Offer) ([]HostPortMapping, error) {
	requiredPorts := make(map[uint64]HostPortMapping)
	mapping := []HostPortMapping{}
	if t.Pod == nil {
		// programming error
		panic("task.Pod is nil")
	}
	for i, container := range t.Pod.Spec.Containers {
		// strip all port==0 from this array; k8s already knows what to do with zero-
		// ports (it does not create 'port bindings' on the minion-host); we need to
		// remove the wildcards from this array since they don't consume host resources
		for pi, port := range container.Ports {
			if port.HostPort == 0 {
				continue // ignore
			}
			m := HostPortMapping{
				ContainerIdx: i,
				PortIdx:      pi,
				OfferPort:    uint64(port.HostPort),
			}
			if entry, inuse := requiredPorts[uint64(port.HostPort)]; inuse {
				return nil, &DuplicateHostPortError{entry, m}
			}
			requiredPorts[uint64(port.HostPort)] = m
		}
	}
	for _, resource := range offer.Resources {
		if resource.GetName() == "ports" {
			for _, r := range (*resource).GetRanges().Range {
				bp := r.GetBegin()
				ep := r.GetEnd()
				for port, _ := range requiredPorts {
					log.V(3).Infof("evaluating port range {%d:%d} %d", bp, ep, port)
					if (bp <= port) && (port <= ep) {
						mapping = append(mapping, requiredPorts[port])
						delete(requiredPorts, port)
					}
				}
			}
		}
	}
	unsatisfiedPorts := len(requiredPorts)
	if unsatisfiedPorts > 0 {
		err := &PortAllocationError{
			PodId: t.Pod.Name,
		}
		for p, _ := range requiredPorts {
			err.Ports = append(err.Ports, p)
		}
		return nil, err
	}
	return mapping, nil
}
<|endoftext|>


<commit_before>package shim

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sync"
	"syscall"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/containerd/monitor"
	"github.com/docker/containerd/oci"
	"github.com/docker/containerkit"
	specs "github.com/opencontainers/runtime-spec/specs-go"
)

/*
├── libcontainerd
│   ├── containerd
│   │   └── ff2e86955c2be43f0e3c300fbd3786599301bd8efcaa5a386587f132e73af242
│   │       ├── init
│   │       │   ├── control
│   │       │   ├── exit
│   │       │   ├── log.json
│   │       │   ├── pid
│   │       │   ├── process.json
│   │       │   ├── shim-log.json
│   │       │   └── starttime
│   │       └── state.json
*/

var (
	ErrNotFifo             = errors.New("shim: IO is not a valid fifo on disk")
	errInitProcessNotExist = errors.New("shim: init process does not exist")
)

type Opts struct {
	Name        string
	RuntimeName string
	RuntimeArgs []string
	RuntimeRoot string
	NoPivotRoot bool
	Root        string
	Timeout     time.Duration
}

func New(opts Opts) (*Shim, error) {
	if err := os.MkdirAll(filepath.Dir(opts.Root), 0711); err != nil {
		return nil, err
	}
	if err := os.Mkdir(opts.Root, 0711); err != nil {
		return nil, err
	}
	r, err := oci.New(oci.Opts{
		Root: opts.RuntimeRoot,
		Name: opts.RuntimeName,
		Args: opts.RuntimeArgs,
	})
	if err != nil {
		return nil, err
	}
	m, err := monitor.New()
	if err != nil {
		return nil, err
	}
	s := &Shim{
		root:      opts.Root,
		name:      opts.Name,
		timeout:   opts.Timeout,
		runtime:   r,
		processes: make(map[string]*process),
		m:         m,
	}
	go s.startMonitor()
	f, err := os.Create(filepath.Join(opts.Root, "state.json"))
	if err != nil {
		return nil, err
	}
	err = json.NewEncoder(f).Encode(s)
	f.Close()
	return s, err
}

// Load will load an existing shim with all its information restored from the
// provided path
func Load(root string) (*Shim, error) {
	f, err := os.Open(filepath.Join(root, "state.json"))
	if err != nil {
		return nil, err
	}
	var s Shim
	err = json.NewDecoder(f).Decode(&s)
	f.Close()
	if err != nil {
		return nil, err
	}
	dirs, err := ioutil.ReadDir(root)
	if err != nil {
		return nil, err
	}
	for _, d := range dirs {
		if !d.IsDir() {
			continue
		}
		name := d.Name()
		if f, err = os.Open(filepath.Join(root, name, "process.json")); err != nil {
			return nil, err
		}
		var p process
		err = json.NewDecoder(f).Decode(&p)
		f.Close()
		if err != nil {
			return nil, err
		}
		s.processes[name] = &p
	}
	return &s, nil
}

// Shim is a container runtime that adds a shim process as the container's parent
// to hold open stdio and other resources so that higher level daemons can exit and
// load running containers for handling upgrades and/or crashes
//
// The shim uses an OCI compliant runtime as its executor
type Shim struct {
	// root holds runtime state information for the containers
	// launched by the runtime
	root        string
	name        string
	timeout     time.Duration
	noPivotRoot bool
	runtime     *oci.OCIRuntime
	pmu         sync.Mutex
	processes   map[string]*process
	bundle      string
	checkpoint  string
	m           *monitor.Monitor
}

type state struct {
	// Bundle is the path to the container's bundle
	Bundle string `json:"bundle"`
	// OCI runtime binary name
	Runtime string `json:"runtime"`
	// OCI runtime args
	RuntimeArgs []string `json:"runtimeArgs"`
	// Shim binary name
	Name string `json:"shim"`
	/// NoPivotRoot option
	NoPivotRoot bool `json:"noPivotRoot"`
	// Timeout for container start
	Timeout time.Duration `json:"timeout"`
}

func (s *Shim) MarshalJSON() ([]byte, error) {
	st := state{
		Name:        s.name,
		Bundle:      s.bundle,
		Runtime:     s.runtime.Name(),
		RuntimeArgs: s.runtime.Args(),
		NoPivotRoot: s.noPivotRoot,
		Timeout:     s.timeout,
	}
	return json.Marshal(st)
}

func (s *Shim) UnmarshalJSON(b []byte) error {
	var st state
	if err := json.Unmarshal(b, &st); err != nil {
		return err
	}
	s.name = st.Name
	s.bundle = st.Bundle
	s.timeout = st.Timeout
	s.noPivotRoot = st.NoPivotRoot
	r, err := oci.New(oci.Opts{
		Name: st.Runtime,
		Args: st.RuntimeArgs,
	})
	if err != nil {
		return err
	}
	s.runtime = r
	s.processes = make(map[string]*process)
	return nil
}

func (s *Shim) Create(c *containerkit.Container) (containerkit.ProcessDelegate, error) {
	s.bundle = c.Path()
	var (
		root = filepath.Join(s.root, "init")
		cmd  = s.command(c.ID(), c.Path(), s.runtime.Name())
	)
	// exec the shim inside the state directory setup with the process
	// information for what is being run
	cmd.Dir = root
	// make sure the shim is in a new process group
	cmd.SysProcAttr = &syscall.SysProcAttr{
		Setpgid: true,
	}
	p, err := s.startCommand(c, cmd)
	if err != nil {
		return nil, err
	}
	if err := s.m.Add(p); err != nil {
		return nil, err
	}
	s.pmu.Lock()
	s.processes["init"] = p
	s.pmu.Unlock()

	f, err := os.Create(filepath.Join(s.root, "state.json"))
	if err != nil {
		return nil, err
	}
	err = json.NewEncoder(f).Encode(s)
	f.Close()
	// ~TODO: oom and stats stuff here
	return p, err
}

func (s *Shim) Start(c *containerkit.Container) error {
	p, err := s.getContainerInit(c)
	if err != nil {
		return err
	}
	var (
		errC = make(chan error, 1)
		cmd  = s.runtime.Command("start", c.ID())
	)
	go func() {
		out, err := cmd.CombinedOutput()
		if err != nil {
			errC <- fmt.Errorf("%s: %q", err, out)
		}
		errC <- nil
	}()
	select {
	case err := <-errC:
		if err != nil {
			return err
		}
	case <-p.done:
		if !p.success {
			if cmd.Process != nil {
				cmd.Process.Kill()
			}
			cmd.Wait()
			return ErrShimExited
		}
		err := <-errC
		if err != nil {
			return err
		}
	}
	return nil
}

func (s *Shim) Delete(c *containerkit.Container) error {
	if err := s.runtime.Delete(c); err != nil {
		return err
	}
	return os.RemoveAll(s.root)
}

var errnotimpl = errors.New("NOT IMPL RIGHT NOW, CHILL")

func (s *Shim) Exec(c *containerkit.Container, p *containerkit.Process) (containerkit.ProcessDelegate, error) {
	return nil, errnotimpl
}

func (s *Shim) Load(id string) (containerkit.ProcessDelegate, error) {
	return nil, errnotimpl
}

func (s *Shim) getContainerInit(c *containerkit.Container) (*process, error) {
	s.pmu.Lock()
	p, ok := s.processes["init"]
	s.pmu.Unlock()
	if !ok {
		return nil, errInitProcessNotExist
	}
	return p, nil
}

func (s *Shim) startCommand(c *containerkit.Container, cmd *exec.Cmd) (*process, error) {
	p, err := newProcess(filepath.Join(s.root, "init"), s.noPivotRoot, s.checkpoint, c, cmd)
	if err != nil {
		return nil, err
	}
	if err := cmd.Start(); err != nil {
		close(p.done)
		if checkShimNotFound(err) {
			return nil, fmt.Errorf("%s not install on system", s.name)
		}
		return nil, err
	}
	// make sure it does not die before we get the container's pid
	defer func() {
		go p.checkExited()
	}()
	if err := p.waitForCreate(s.timeout); err != nil {
		return nil, err
	}
	return p, nil
}

func (s *Shim) command(args ...string) *exec.Cmd {
	return exec.Command(s.name, args...)
}

func (s *Shim) startMonitor() {
	for m := range s.m.Events() {
		p := m.(*process)
		close(p.done)
		if err := s.m.Remove(p); err != nil {
			logrus.Error(err)
		}
	}
}

// checkShimNotFound checks the error returned from a exec call to see if the binary
// that was called exists on the system and returns true if the shim binary does not exist
func checkShimNotFound(err error) bool {
	if exitError, ok := err.(*exec.Error); ok {
		e := exitError.Err
		return e == exec.ErrNotFound || e == os.ErrNotExist
	}
	return false
}

// getFifoPath returns the path to the fifo on disk as long as the provided
// interface is an *os.File and has a valid path on the Name() method call
func getFifoPath(v interface{}) (string, error) {
	f, ok := v.(*os.File)
	if !ok {
		return "", ErrNotFifo
	}
	p := f.Name()
	if p == "" {
		return "", ErrNotFifo
	}
	return p, nil
}

func getRootIDs(s *specs.Spec) (int, int, error) {
	if s == nil {
		return 0, 0, nil
	}
	var hasUserns bool
	for _, ns := range s.Linux.Namespaces {
		if ns.Type == specs.UserNamespace {
			hasUserns = true
			break
		}
	}
	if !hasUserns {
		return 0, 0, nil
	}
	uid := hostIDFromMap(0, s.Linux.UIDMappings)
	gid := hostIDFromMap(0, s.Linux.GIDMappings)
	return uid, gid, nil
}

func hostIDFromMap(id uint32, mp []specs.IDMapping) int {
	for _, m := range mp {
		if (id >= m.ContainerID) && (id <= (m.ContainerID + m.Size - 1)) {
			return int(m.HostID + (id - m.ContainerID))
		}
	}
	return 0
}
<commit_msg>Implement Load for shim<commit_after>package shim

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sync"
	"syscall"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/containerd/monitor"
	"github.com/docker/containerd/oci"
	"github.com/docker/containerkit"
	specs "github.com/opencontainers/runtime-spec/specs-go"
)

/*
├── libcontainerd
│   ├── containerd
│   │   └── ff2e86955c2be43f0e3c300fbd3786599301bd8efcaa5a386587f132e73af242
│   │       ├── init
│   │       │   ├── control
│   │       │   ├── exit
│   │       │   ├── log.json
│   │       │   ├── pid
│   │       │   ├── process.json
│   │       │   ├── shim-log.json
│   │       │   └── starttime
│   │       └── state.json
*/

var (
	ErrNotFifo             = errors.New("shim: IO is not a valid fifo on disk")
	errInitProcessNotExist = errors.New("shim: init process does not exist")
)

type Opts struct {
	Name        string
	RuntimeName string
	RuntimeArgs []string
	RuntimeRoot string
	NoPivotRoot bool
	Root        string
	Timeout     time.Duration
}

func New(opts Opts) (*Shim, error) {
	if err := os.MkdirAll(filepath.Dir(opts.Root), 0711); err != nil {
		return nil, err
	}
	if err := os.Mkdir(opts.Root, 0711); err != nil {
		return nil, err
	}
	r, err := oci.New(oci.Opts{
		Root: opts.RuntimeRoot,
		Name: opts.RuntimeName,
		Args: opts.RuntimeArgs,
	})
	if err != nil {
		return nil, err
	}
	m, err := monitor.New()
	if err != nil {
		return nil, err
	}
	s := &Shim{
		root:      opts.Root,
		name:      opts.Name,
		timeout:   opts.Timeout,
		runtime:   r,
		processes: make(map[string]*process),
		m:         m,
	}
	go s.startMonitor()
	f, err := os.Create(filepath.Join(opts.Root, "state.json"))
	if err != nil {
		return nil, err
	}
	err = json.NewEncoder(f).Encode(s)
	f.Close()
	return s, err
}

// Load will load an existing shim with all its information restored from the
// provided path
func Load(root string) (*Shim, error) {
	f, err := os.Open(filepath.Join(root, "state.json"))
	if err != nil {
		return nil, err
	}
	var s Shim
	err = json.NewDecoder(f).Decode(&s)
	f.Close()
	if err != nil {
		return nil, err
	}
	m, err := monitor.New()
	if err != nil {
		return nil, err
	}
	s.m = m
	dirs, err := ioutil.ReadDir(root)
	if err != nil {
		return nil, err
	}
	for _, d := range dirs {
		if !d.IsDir() {
			continue
		}
		name := d.Name()
		if f, err = os.Open(filepath.Join(root, name, "process.json")); err != nil {
			return nil, err
		}
		var p process
		err = json.NewDecoder(f).Decode(&p)
		f.Close()
		if err != nil {
			return nil, err
		}
		s.processes[name] = &p
		if err := s.m.Add(&p); err != nil {
			return nil, err
		}
	}
	return &s, nil
}

// Shim is a container runtime that adds a shim process as the container's parent
// to hold open stdio and other resources so that higher level daemons can exit and
// load running containers for handling upgrades and/or crashes
//
// The shim uses an OCI compliant runtime as its executor
type Shim struct {
	// root holds runtime state information for the containers
	// launched by the runtime
	root        string
	name        string
	timeout     time.Duration
	noPivotRoot bool
	runtime     *oci.OCIRuntime
	pmu         sync.Mutex
	processes   map[string]*process
	bundle      string
	checkpoint  string
	m           *monitor.Monitor
}

type state struct {
	// Bundle is the path to the container's bundle
	Bundle string `json:"bundle"`
	// OCI runtime binary name
	Runtime string `json:"runtime"`
	// OCI runtime args
	RuntimeArgs []string `json:"runtimeArgs"`
	// Shim binary name
	Name string `json:"shim"`
	/// NoPivotRoot option
	NoPivotRoot bool `json:"noPivotRoot"`
	// Timeout for container start
	Timeout time.Duration `json:"timeout"`
}

func (s *Shim) MarshalJSON() ([]byte, error) {
	st := state{
		Name:        s.name,
		Bundle:      s.bundle,
		Runtime:     s.runtime.Name(),
		RuntimeArgs: s.runtime.Args(),
		NoPivotRoot: s.noPivotRoot,
		Timeout:     s.timeout,
	}
	return json.Marshal(st)
}

func (s *Shim) UnmarshalJSON(b []byte) error {
	var st state
	if err := json.Unmarshal(b, &st); err != nil {
		return err
	}
	s.name = st.Name
	s.bundle = st.Bundle
	s.timeout = st.Timeout
	s.noPivotRoot = st.NoPivotRoot
	r, err := oci.New(oci.Opts{
		Name: st.Runtime,
		Args: st.RuntimeArgs,
	})
	if err != nil {
		return err
	}
	s.runtime = r
	s.processes = make(map[string]*process)
	return nil
}

func (s *Shim) Create(c *containerkit.Container) (containerkit.ProcessDelegate, error) {
	s.bundle = c.Path()
	var (
		root = filepath.Join(s.root, "init")
		cmd  = s.command(c.ID(), c.Path(), s.runtime.Name())
	)
	// exec the shim inside the state directory setup with the process
	// information for what is being run
	cmd.Dir = root
	// make sure the shim is in a new process group
	cmd.SysProcAttr = &syscall.SysProcAttr{
		Setpgid: true,
	}
	p, err := s.startCommand(c, cmd)
	if err != nil {
		return nil, err
	}
	if err := s.m.Add(p); err != nil {
		return nil, err
	}
	s.pmu.Lock()
	s.processes["init"] = p
	s.pmu.Unlock()

	f, err := os.Create(filepath.Join(s.root, "state.json"))
	if err != nil {
		return nil, err
	}
	err = json.NewEncoder(f).Encode(s)
	f.Close()
	// ~TODO: oom and stats stuff here
	return p, err
}

func (s *Shim) Start(c *containerkit.Container) error {
	p, err := s.getContainerInit()
	if err != nil {
		return err
	}
	var (
		errC = make(chan error, 1)
		cmd  = s.runtime.Command("start", c.ID())
	)
	go func() {
		out, err := cmd.CombinedOutput()
		if err != nil {
			errC <- fmt.Errorf("%s: %q", err, out)
		}
		errC <- nil
	}()
	select {
	case err := <-errC:
		if err != nil {
			return err
		}
	case <-p.done:
		if !p.success {
			if cmd.Process != nil {
				cmd.Process.Kill()
			}
			cmd.Wait()
			return ErrShimExited
		}
		err := <-errC
		if err != nil {
			return err
		}
	}
	return nil
}

func (s *Shim) Delete(c *containerkit.Container) error {
	if err := s.runtime.Delete(c); err != nil {
		return err
	}
	return os.RemoveAll(s.root)
}

var errnotimpl = errors.New("NOT IMPL RIGHT NOW, CHILL")

func (s *Shim) Exec(c *containerkit.Container, p *containerkit.Process) (containerkit.ProcessDelegate, error) {
	return nil, errnotimpl
}

func (s *Shim) Load(id string) (containerkit.ProcessDelegate, error) {
	return s.getContainerInit()
}

func (s *Shim) getContainerInit() (*process, error) {
	s.pmu.Lock()
	p, ok := s.processes["init"]
	s.pmu.Unlock()
	if !ok {
		return nil, errInitProcessNotExist
	}
	return p, nil
}

func (s *Shim) startCommand(c *containerkit.Container, cmd *exec.Cmd) (*process, error) {
	p, err := newProcess(filepath.Join(s.root, "init"), s.noPivotRoot, s.checkpoint, c, cmd)
	if err != nil {
		return nil, err
	}
	if err := cmd.Start(); err != nil {
		close(p.done)
		if checkShimNotFound(err) {
			return nil, fmt.Errorf("%s not install on system", s.name)
		}
		return nil, err
	}
	// make sure it does not die before we get the container's pid
	defer func() {
		go p.checkExited()
	}()
	if err := p.waitForCreate(s.timeout); err != nil {
		return nil, err
	}
	return p, nil
}

func (s *Shim) command(args ...string) *exec.Cmd {
	return exec.Command(s.name, args...)
}

func (s *Shim) startMonitor() {
	for m := range s.m.Events() {
		p := m.(*process)
		close(p.done)
		if err := s.m.Remove(p); err != nil {
			logrus.Error(err)
		}
	}
}

// checkShimNotFound checks the error returned from a exec call to see if the binary
// that was called exists on the system and returns true if the shim binary does not exist
func checkShimNotFound(err error) bool {
	if exitError, ok := err.(*exec.Error); ok {
		e := exitError.Err
		return e == exec.ErrNotFound || e == os.ErrNotExist
	}
	return false
}

// getFifoPath returns the path to the fifo on disk as long as the provided
// interface is an *os.File and has a valid path on the Name() method call
func getFifoPath(v interface{}) (string, error) {
	f, ok := v.(*os.File)
	if !ok {
		return "", ErrNotFifo
	}
	p := f.Name()
	if p == "" {
		return "", ErrNotFifo
	}
	return p, nil
}

func getRootIDs(s *specs.Spec) (int, int, error) {
	if s == nil {
		return 0, 0, nil
	}
	var hasUserns bool
	for _, ns := range s.Linux.Namespaces {
		if ns.Type == specs.UserNamespace {
			hasUserns = true
			break
		}
	}
	if !hasUserns {
		return 0, 0, nil
	}
	uid := hostIDFromMap(0, s.Linux.UIDMappings)
	gid := hostIDFromMap(0, s.Linux.GIDMappings)
	return uid, gid, nil
}

func hostIDFromMap(id uint32, mp []specs.IDMapping) int {
	for _, m := range mp {
		if (id >= m.ContainerID) && (id <= (m.ContainerID + m.Size - 1)) {
			return int(m.HostID + (id - m.ContainerID))
		}
	}
	return 0
}
<|endoftext|>


<commit_before>package sink

import (
	"strconv"
	"time"
	"bytes"

	"net/http"

	"os"

	"fmt"

	log "github.com/sirupsen/logrus"
)

// HttpSink ...
type HttpSink struct {
	address     string
	workerCount int
	stopCh      chan interface{}
	putCh       chan []byte
}

// NewHttp ...
func NewHttp() (*HttpSink, error) {
	address := os.Getenv("SINK_HTTP_ADDRESS")
	if address == "" {
		return nil, fmt.Errorf("[sink/http] Missing SINK_HTTP_ADDRESS (example: http://miau.com:8080/biau)")
	}

	workerCountStr := os.Getenv("SINK_WORKER_COUNT")
	if workerCountStr == "" {
		workerCountStr = "1"
	}
	workerCount, err := strconv.Atoi(workerCountStr)
	if err != nil {
		return nil, fmt.Errorf("Invalid SINK_WORKER_COUNT, must be an integer")
	}

	return &HttpSink{
		address:     address,
		workerCount: workerCount,
		stopCh:      make(chan interface{}),
		putCh:       make(chan []byte, 1000),
	}, nil
}

// Start ...
func (s *HttpSink) Start() error {
	// Stop chan for all tasks to depend on
	s.stopCh = make(chan interface{})

	for i := 0; i < s.workerCount; i++ {
		go s.send(i)
	}

	// wait forever for a stop signal to happen
	for {
		select {
		case <-s.stopCh:
			break
		}
		break
	}

	return nil
}

// Stop ...
func (s *HttpSink) Stop() {
	log.Infof("[sink/http] ensure writer queue is empty (%d messages left)", len(s.putCh))

	for len(s.putCh) > 0 {
		log.Info("[sink/http] Waiting for queue to drain - (%d messages left)", len(s.putCh))
		time.Sleep(1 * time.Second)
	}

	close(s.stopCh)
}

// Put ..
func (s *HttpSink) Put(data []byte) error {
	s.putCh <- data

	return nil
}

func (s *HttpSink) send(id int) {
	log.Infof("[sink/http/%d] Starting writer", id)

	for {
		select {
		case data := <-s.putCh:
			_, err := http.Post(s.address, "text/json", bytes.NewReader(data))
			if err != nil {
				log.Errorf("[sink/http/%d] %s", id, err)
			} else {
				log.Debugf("[sink/http/%d] publish ok", id)
			}
		}
	}
}
<commit_msg>http sink: post using application/json content-type<commit_after>package sink

import (
	"strconv"
	"time"
	"bytes"

	"net/http"

	"os"

	"fmt"

	log "github.com/sirupsen/logrus"
)

// HttpSink ...
type HttpSink struct {
	address     string
	workerCount int
	stopCh      chan interface{}
	putCh       chan []byte
}

// NewHttp ...
func NewHttp() (*HttpSink, error) {
	address := os.Getenv("SINK_HTTP_ADDRESS")
	if address == "" {
		return nil, fmt.Errorf("[sink/http] Missing SINK_HTTP_ADDRESS (example: http://miau.com:8080/biau)")
	}

	workerCountStr := os.Getenv("SINK_WORKER_COUNT")
	if workerCountStr == "" {
		workerCountStr = "1"
	}
	workerCount, err := strconv.Atoi(workerCountStr)
	if err != nil {
		return nil, fmt.Errorf("Invalid SINK_WORKER_COUNT, must be an integer")
	}

	return &HttpSink{
		address:     address,
		workerCount: workerCount,
		stopCh:      make(chan interface{}),
		putCh:       make(chan []byte, 1000),
	}, nil
}

// Start ...
func (s *HttpSink) Start() error {
	// Stop chan for all tasks to depend on
	s.stopCh = make(chan interface{})

	for i := 0; i < s.workerCount; i++ {
		go s.send(i)
	}

	// wait forever for a stop signal to happen
	for {
		select {
		case <-s.stopCh:
			break
		}
		break
	}

	return nil
}

// Stop ...
func (s *HttpSink) Stop() {
	log.Infof("[sink/http] ensure writer queue is empty (%d messages left)", len(s.putCh))

	for len(s.putCh) > 0 {
		log.Info("[sink/http] Waiting for queue to drain - (%d messages left)", len(s.putCh))
		time.Sleep(1 * time.Second)
	}

	close(s.stopCh)
}

// Put ..
func (s *HttpSink) Put(data []byte) error {
	s.putCh <- data

	return nil
}

func (s *HttpSink) send(id int) {
	log.Infof("[sink/http/%d] Starting writer", id)

	for {
		select {
		case data := <-s.putCh:
			_, err := http.Post(s.address, "application/json; charset=utf-8", bytes.NewBuffer(data[:]))
			if err != nil {
				log.Errorf("[sink/http/%d] %s", id, err)
			} else {
				log.Debugf("[sink/http/%d] publish ok", id)
			}
		}
	}
}
<|endoftext|>


<commit_before>/*
Package embed allows for storing data resources in a virtual filesystem
that gets compiled directly into the output program, eliminating the need
to distribute data files with the application. This is especially useful for small
web servers that need to deliver content files.

The data is gzipped to save space.

An external tool for generating output files can be found at http://github.com/cratonica/embedder

Author: Clint Caywood

http://github.com/cratonica/embed
*/
package embed

import (
	"bytes"
	"compress/gzip"
	"encoding/binary"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

// A mapping of resource identifiers to associated data
type ResourceMap map[string][]byte

// A compressed resource map that can be used for serialization
type PackedResourceMap []byte

var byteOrder binary.ByteOrder = binary.LittleEndian

// Packs the map of resource identifiers => []byte into
// a buffer. This process is reversed by calling Unpack.
func Pack(data ResourceMap) (PackedResourceMap, error) {
	var buf bytes.Buffer
	writer, err := gzip.NewWriterLevel(&buf, gzip.BestCompression)
	if err != nil {
		return nil, err
	}
	keys := make([]string, len(data))
	idx := 0
	for key := range data {
		keys[idx] = key
		idx++
	}
	sort.Strings(keys)
	for _, i := range keys {
		v := data[i]
		if err = binary.Write(writer, byteOrder, int32(len(i))); err != nil {
			return nil, err
		}
		if _, err = writer.Write([]byte(i)); err != nil {
			return nil, err
		}
		if err = binary.Write(writer, byteOrder, int32(len(v))); err != nil {
			return nil, err
		}
		if _, err = writer.Write(v); err != nil {
		}
	}
	if err = writer.Close(); err != nil {
		return nil, err
	}
	result := buf.Bytes()
	return result, nil
}

type DeserializationError struct {
	what     string
	expected int
	actual   int
}

func (this *DeserializationError) Error() string {
	return fmt.Sprintf("Failure deserializing resource: expected %v to be %v bytes, but only read %v bytes", this.what, this.expected, this.actual)
}

func NewDeserializationError(what string, expected int, actual int) *DeserializationError {
	return &DeserializationError{what, expected, actual}
}

// Reads a buffer generated by a call to Pack, returning the original map
func Unpack(data PackedResourceMap) (ResourceMap, error) {
	result := make(map[string][]byte)
	buf := bytes.NewBuffer(data)
	reader, err := gzip.NewReader(buf)
	defer reader.Close()
	if err != nil {
		return nil, err
	}
	for {
		var size int32
		err := binary.Read(reader, byteOrder, &size)
		if err != nil {
			if err == io.EOF {
				break
			}
			return nil, err
		}
		keyBuf, err := readAll(reader, int(size))
		if err != nil {
			return nil, err
		}
		err = binary.Read(reader, byteOrder, &size)
		if err != nil {
			return nil, err
		}
		dataBuf, err := readAll(reader, int(size))
		if err != nil {
			return nil, err
		}
		result[string(keyBuf)] = dataBuf
	}
	return result, nil
}

// Recursively packs all files in the given directory into
// a resource map. Directory delimiters are converted to Unix-style /
// regardless of the host operating system.
func CreateFromFiles(path string) (ResourceMap, error) {
	result := make(map[string][]byte)
	err := crawl(filepath.Clean(path), filepath.Clean(path), result)
	if err != nil {
		return nil, err
	}
	return result, nil
}

// Generates Go code containing the given data that can be included in the target project
func GenerateGoCode(packageName string, varName string, data PackedResourceMap) string {
	template := `
package %v

import "github.com/cratonica/embed"

var %v embed.PackedResourceMap = %#v
`
	return fmt.Sprintf(template, packageName, varName, data)
}

func readAll(reader io.Reader, size int) ([]byte, error) {
	result := make([]byte, size)
	totalBytes := 0
	for totalBytes < size {
		readSize, err := reader.Read(result[totalBytes:])
		if err != nil {
			return nil, err
		}
		totalBytes += readSize
	}
	return result, nil
}

func crawl(rootPath string, currentPath string, dest map[string][]byte) error {
	files, err := ioutil.ReadDir(currentPath)
	if err != nil {
		return err
	}
	for _, file := range files {
		fullPath := filepath.Join(currentPath, file.Name())
		if file.IsDir() {
			err := crawl(rootPath, fullPath, dest)
			if err != nil {
				return err
			}
		} else {
			buf, err := ioutil.ReadFile(fullPath)
			if err != nil {
				log.Printf("Failure reading file %v: %v", fullPath, err)
			} else {
				key := strings.TrimLeft(fullPath, rootPath)
				if strings.HasPrefix(key, string(os.PathSeparator)) {
					key = key[1:]
				}
				// Convert to unix style for internal use
				key = strings.Replace(key, string(os.PathSeparator), "/", -1)
				dest[key] = buf
			}
		}
	}
	return nil
}
<commit_msg>Fix bug where reads land on exact file boundaries<commit_after>/*
Package embed allows for storing data resources in a virtual filesystem
that gets compiled directly into the output program, eliminating the need
to distribute data files with the application. This is especially useful for small
web servers that need to deliver content files.

The data is gzipped to save space.

An external tool for generating output files can be found at http://github.com/cratonica/embedder

Author: Clint Caywood

http://github.com/cratonica/embed
*/
package embed

import (
	"bytes"
	"compress/gzip"
	"encoding/binary"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

// A mapping of resource identifiers to associated data
type ResourceMap map[string][]byte

// A compressed resource map that can be used for serialization
type PackedResourceMap []byte

var byteOrder binary.ByteOrder = binary.LittleEndian

// Packs the map of resource identifiers => []byte into
// a buffer. This process is reversed by calling Unpack.
func Pack(data ResourceMap) (PackedResourceMap, error) {
	var buf bytes.Buffer
	writer, err := gzip.NewWriterLevel(&buf, gzip.BestCompression)
	if err != nil {
		return nil, err
	}
	keys := make([]string, len(data))
	idx := 0
	for key := range data {
		keys[idx] = key
		idx++
	}
	sort.Strings(keys)
	for _, i := range keys {
		v := data[i]
		if err = binary.Write(writer, byteOrder, int32(len(i))); err != nil {
			return nil, err
		}
		if _, err = writer.Write([]byte(i)); err != nil {
			return nil, err
		}
		if err = binary.Write(writer, byteOrder, int32(len(v))); err != nil {
			return nil, err
		}
		if _, err = writer.Write(v); err != nil {
		}
	}
	if err = writer.Close(); err != nil {
		return nil, err
	}
	result := buf.Bytes()
	return result, nil
}

type DeserializationError struct {
	what     string
	expected int
	actual   int
}

func (this *DeserializationError) Error() string {
	return fmt.Sprintf("Failure deserializing resource: expected %v to be %v bytes, but only read %v bytes", this.what, this.expected, this.actual)
}

func NewDeserializationError(what string, expected int, actual int) *DeserializationError {
	return &DeserializationError{what, expected, actual}
}

// Reads a buffer generated by a call to Pack, returning the original map
func Unpack(data PackedResourceMap) (ResourceMap, error) {
	result := make(map[string][]byte)
	buf := bytes.NewBuffer(data)
	reader, err := gzip.NewReader(buf)
	defer reader.Close()
	if err != nil {
		return nil, err
	}
	for {
		var size int32
		err := binary.Read(reader, byteOrder, &size)
		if err != nil {
			if err == io.EOF {
				break
			}
			return nil, err
		}
		keyBuf, err := readAll(reader, int(size))
		if err != nil {
			return nil, err
		}
		err = binary.Read(reader, byteOrder, &size)
		if err != nil {
			return nil, err
		}
		dataBuf, err := readAll(reader, int(size))
		if err != nil {
			return nil, err
		}
		result[string(keyBuf)] = dataBuf
	}
	return result, nil
}

// Recursively packs all files in the given directory into
// a resource map. Directory delimiters are converted to Unix-style /
// regardless of the host operating system.
func CreateFromFiles(path string) (ResourceMap, error) {
	result := make(map[string][]byte)
	err := crawl(filepath.Clean(path), filepath.Clean(path), result)
	if err != nil {
		return nil, err
	}
	return result, nil
}

// Generates Go code containing the given data that can be included in the target project
func GenerateGoCode(packageName string, varName string, data PackedResourceMap) string {
	template := `
package %v

import "github.com/cratonica/embed"

var %v embed.PackedResourceMap = %#v
`
	return fmt.Sprintf(template, packageName, varName, data)
}

func readAll(reader io.Reader, size int) ([]byte, error) {
	result := make([]byte, size)
	totalBytes := 0
	for totalBytes < size {
		readSize, err := reader.Read(result[totalBytes:])
		if err != nil {
			if err == io.EOF && totalBytes+readSize == size {
				return result, nil
			} else {
				return nil, err
			}
		}
		totalBytes += readSize
	}
	return result, nil
}

func crawl(rootPath string, currentPath string, dest map[string][]byte) error {
	files, err := ioutil.ReadDir(currentPath)
	if err != nil {
		return err
	}
	for _, file := range files {
		fullPath := filepath.Join(currentPath, file.Name())
		if file.IsDir() {
			err := crawl(rootPath, fullPath, dest)
			if err != nil {
				return err
			}
		} else {
			buf, err := ioutil.ReadFile(fullPath)
			if err != nil {
				log.Printf("Failure reading file %v: %v", fullPath, err)
			} else {
				key := strings.TrimLeft(fullPath, rootPath)
				if strings.HasPrefix(key, string(os.PathSeparator)) {
					key = key[1:]
				}
				// Convert to unix style for internal use
				key = strings.Replace(key, string(os.PathSeparator), "/", -1)
				dest[key] = buf
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package aws

import (
	"bytes"
	"io/ioutil"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
)

const (
	// New versions of github.com/aws/aws-sdk-go/aws have these consts
	// but the version currently pinned by bosh-cli v2 does not

	// ErrCodeNoSuchBucket for service response error code
	// "NoSuchBucket".
	//
	// The specified bucket does not exist.
	ErrCodeNoSuchBucket = "NoSuchBucket"

	// ErrCodeNoSuchKey for service response error code
	// "NoSuchKey".
	//
	// The specified key does not exist.
	ErrCodeNoSuchKey = "NoSuchKey"
)

// EnsureBucketExists checks if the named bucket exists and creates it if it doesn't
func EnsureBucketExists(name, region string) error {
	sess, err := session.NewSession(aws.NewConfig().WithCredentialsChainVerboseErrors(true))
	if err != nil {
		return err
	}

	client := s3.New(sess, &aws.Config{Region: &region})

	_, err = client.HeadBucket(&s3.HeadBucketInput{Bucket: &name})
	if err == nil {
		return nil
	}

	if err.(awserr.Error).Code() != ErrCodeNoSuchBucket && err.(awserr.Error).Code() != "NotFound" {
		return err
	}

	_, err = client.CreateBucket(&s3.CreateBucketInput{
		Bucket: &name,
		CreateBucketConfiguration: &s3.CreateBucketConfiguration{
			LocationConstraint: &region,
		},
	})
	if err != nil {
		return err
	}

	versioningStatus := "Enabled"
	_, err = client.PutBucketVersioning(&s3.PutBucketVersioningInput{
		Bucket: &name,
		VersioningConfiguration: &s3.VersioningConfiguration{
			Status: &versioningStatus,
		},
	})

	return err
}

// WriteFile writes the specified S3 object
func WriteFile(bucket, path, region string, contents []byte) error {
	sess, err := session.NewSession(aws.NewConfig().WithCredentialsChainVerboseErrors(true))
	if err != nil {
		return err
	}
	client := s3.New(sess, &aws.Config{Region: &region})

	_, err = client.PutObject(&s3.PutObjectInput{
		Bucket: &bucket,
		Key:    &path,
		Body:   bytes.NewReader(contents),
	})
	return err
}

// HasFile returns true if the specified S3 object exists
func HasFile(bucket, path, region string) (bool, error) {
	sess, err := session.NewSession(aws.NewConfig().WithCredentialsChainVerboseErrors(true))
	if err != nil {
		return false, err
	}
	client := s3.New(sess, &aws.Config{Region: &region})

	_, err = client.HeadObject(&s3.HeadObjectInput{Bucket: &bucket, Key: &path})
	if err != nil {
		errCode := err.(awserr.Error).Code()
		if errCode == ErrCodeNoSuchKey || errCode == "NotFound" {
			return false, nil
		}
		return false, err
	}

	return true, nil
}

// EnsureFileExists checks for the named file in S3 and creates it if it doesn't
// Second argument is true if new file was created
func EnsureFileExists(bucket, path, region string, defaultContents []byte) ([]byte, bool, error) {
	sess, err := session.NewSession(aws.NewConfig().WithCredentialsChainVerboseErrors(true))
	if err != nil {
		return nil, false, err
	}

	client := s3.New(sess, &aws.Config{Region: &region})

	output, err := client.GetObject(&s3.GetObjectInput{Bucket: &bucket, Key: &path})
	if err == nil {
		var contents []byte
		contents, err = ioutil.ReadAll(output.Body)
		if err != nil {
			return nil, false, err
		}

		// Successfully loaded file
		return contents, true, nil
	}

	if err.(awserr.Error).Code() != ErrCodeNoSuchKey {
		return nil, false, err
	}

	_, err = client.PutObject(&s3.PutObjectInput{
		Bucket: &bucket,
		Key:    &path,
		Body:   bytes.NewReader(defaultContents),
	})
	if err != nil {
		return nil, false, err
	}

	// Created file from given contents
	return defaultContents, true, nil
}

// LoadFile loads a file from S3
func LoadFile(bucket, path, region string) ([]byte, error) {
	sess, err := session.NewSession(aws.NewConfig().WithCredentialsChainVerboseErrors(true))
	if err != nil {
		return nil, err
	}

	client := s3.New(sess, &aws.Config{Region: &region})

	output, err := client.GetObject(&s3.GetObjectInput{Bucket: &bucket, Key: &path})
	if err != nil {
		return nil, err
	}

	return ioutil.ReadAll(output.Body)
}
<commit_msg>make local constants private<commit_after>package aws

import (
	"bytes"
	"io/ioutil"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
)

const (
	// New versions of github.com/aws/aws-sdk-go/aws have these consts
	// but the version currently pinned by bosh-cli v2 does not

	// ErrCodeNoSuchBucket for service response error code
	// "NoSuchBucket".
	//
	// The specified bucket does not exist.
	awsErrCodeNoSuchBucket = "NoSuchBucket"

	// ErrCodeNoSuchKey for service response error code
	// "NoSuchKey".
	//
	// The specified key does not exist.
	awsErrCodeNoSuchKey = "NoSuchKey"

	// Returned when calling HEAD on non-existant bucket or object
	awsErrCodeNotFound = "NotFound"
)

// EnsureBucketExists checks if the named bucket exists and creates it if it doesn't
func EnsureBucketExists(name, region string) error {
	sess, err := session.NewSession(aws.NewConfig().WithCredentialsChainVerboseErrors(true))
	if err != nil {
		return err
	}

	client := s3.New(sess, &aws.Config{Region: &region})

	_, err = client.HeadBucket(&s3.HeadBucketInput{Bucket: &name})
	if err == nil {
		return nil
	}

	awsErrCode := err.(awserr.Error).Code()
	if awsErrCode != awsErrCodeNotFound && awsErrCode != awsErrCodeNoSuchBucket {
		return err
	}

	_, err = client.CreateBucket(&s3.CreateBucketInput{
		Bucket: &name,
		CreateBucketConfiguration: &s3.CreateBucketConfiguration{
			LocationConstraint: &region,
		},
	})
	if err != nil {
		return err
	}

	versioningStatus := "Enabled"
	_, err = client.PutBucketVersioning(&s3.PutBucketVersioningInput{
		Bucket: &name,
		VersioningConfiguration: &s3.VersioningConfiguration{
			Status: &versioningStatus,
		},
	})

	return err
}

// WriteFile writes the specified S3 object
func WriteFile(bucket, path, region string, contents []byte) error {
	sess, err := session.NewSession(aws.NewConfig().WithCredentialsChainVerboseErrors(true))
	if err != nil {
		return err
	}
	client := s3.New(sess, &aws.Config{Region: &region})

	_, err = client.PutObject(&s3.PutObjectInput{
		Bucket: &bucket,
		Key:    &path,
		Body:   bytes.NewReader(contents),
	})
	return err
}

// HasFile returns true if the specified S3 object exists
func HasFile(bucket, path, region string) (bool, error) {
	sess, err := session.NewSession(aws.NewConfig().WithCredentialsChainVerboseErrors(true))
	if err != nil {
		return false, err
	}
	client := s3.New(sess, &aws.Config{Region: &region})

	_, err = client.HeadObject(&s3.HeadObjectInput{Bucket: &bucket, Key: &path})
	if err != nil {
		awsErrCode := err.(awserr.Error).Code()
		if awsErrCode == awsErrCodeNotFound || awsErrCode == awsErrCodeNoSuchKey {
			return false, nil
		}
		return false, err
	}

	return true, nil
}

// EnsureFileExists checks for the named file in S3 and creates it if it doesn't
// Second argument is true if new file was created
func EnsureFileExists(bucket, path, region string, defaultContents []byte) ([]byte, bool, error) {
	sess, err := session.NewSession(aws.NewConfig().WithCredentialsChainVerboseErrors(true))
	if err != nil {
		return nil, false, err
	}

	client := s3.New(sess, &aws.Config{Region: &region})

	output, err := client.GetObject(&s3.GetObjectInput{Bucket: &bucket, Key: &path})
	if err == nil {
		var contents []byte
		contents, err = ioutil.ReadAll(output.Body)
		if err != nil {
			return nil, false, err
		}

		// Successfully loaded file
		return contents, true, nil
	}

	awsErrCode := err.(awserr.Error).Code()
	if awsErrCode != awsErrCodeNoSuchKey && awsErrCode != awsErrCodeNotFound {
		return nil, false, err
	}

	_, err = client.PutObject(&s3.PutObjectInput{
		Bucket: &bucket,
		Key:    &path,
		Body:   bytes.NewReader(defaultContents),
	})
	if err != nil {
		return nil, false, err
	}

	// Created file from given contents
	return defaultContents, true, nil
}

// LoadFile loads a file from S3
func LoadFile(bucket, path, region string) ([]byte, error) {
	sess, err := session.NewSession(aws.NewConfig().WithCredentialsChainVerboseErrors(true))
	if err != nil {
		return nil, err
	}

	client := s3.New(sess, &aws.Config{Region: &region})

	output, err := client.GetObject(&s3.GetObjectInput{Bucket: &bucket, Key: &path})
	if err != nil {
		return nil, err
	}

	return ioutil.ReadAll(output.Body)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/lxc/lxd/lxd/state"
	storagePools "github.com/lxc/lxd/lxd/storage"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
)

func storagePoolUpdate(state *state.State, name, newDescription string, newConfig map[string]string, withDB bool) error {
	pool, err := storagePools.GetPoolByName(state, name)
	if err != nil {
		return err
	}

	return pool.Update(!withDB, newDescription, newConfig, nil)
}

// storagePoolDBCreate creates a storage pool DB entry and returns the created Pool ID.
func storagePoolDBCreate(s *state.State, poolName, poolDescription string, driver string, config map[string]string) (int64, error) {
	// Check that the storage pool does not already exist.
	_, err := s.Cluster.GetStoragePoolID(poolName)
	if err == nil {
		return -1, fmt.Errorf("The storage pool already exists")
	}

	// Make sure that we don't pass a nil to the next function.
	if config == nil {
		config = map[string]string{}
	}
	err = storagePoolValidate(poolName, driver, config)
	if err != nil {
		return -1, err
	}

	// Fill in the defaults
	err = storagePoolFillDefault(poolName, driver, config)
	if err != nil {
		return -1, err
	}

	// Create the database entry for the storage pool.
	id, err := dbStoragePoolCreateAndUpdateCache(s, poolName, poolDescription, driver, config)
	if err != nil {
		return -1, fmt.Errorf("Error inserting %s into database: %s", poolName, err)
	}

	return id, nil
}

func storagePoolValidate(poolName string, driverName string, config map[string]string) error {
	// Check if the storage pool name is valid.
	err := storagePools.ValidName(poolName)
	if err != nil {
		return err
	}

	// Validate the requested storage pool configuration.
	err = storagePoolValidateConfig(poolName, driverName, config, nil)
	if err != nil {
		return err
	}

	return nil
}

func storagePoolCreateGlobal(state *state.State, req api.StoragePoolsPost) error {
	// Create the database entry.
	id, err := storagePoolDBCreate(state, req.Name, req.Description, req.Driver, req.Config)
	if err != nil {
		return err
	}

	// Define a function which reverts everything.  Defer this function
	// so that it doesn't need to be explicitly called in every failing
	// return path. Track whether or not we want to undo the changes
	// using a closure.
	tryUndo := true
	defer func() {
		if !tryUndo {
			return
		}

		dbStoragePoolDeleteAndUpdateCache(state, req.Name)
	}()

	_, err = storagePoolCreateLocal(state, id, req, false)
	if err != nil {
		return err
	}

	tryUndo = false
	return nil
}

// This performs all non-db related work needed to create the pool.
func storagePoolCreateLocal(state *state.State, id int64, req api.StoragePoolsPost, isNotification bool) (map[string]string, error) {
	tryUndo := true

	// Make a copy of the req for later diff.
	var updatedConfig map[string]string
	var updatedReq api.StoragePoolsPost
	shared.DeepCopy(&req, &updatedReq)

	// Fill in the defaults.
	err := storagePoolFillDefault(updatedReq.Name, updatedReq.Driver, updatedReq.Config)
	if err != nil {
		return nil, err
	}

	// Create the pool.
	pool, err := storagePools.CreatePool(state, id, &updatedReq, isNotification, nil)
	if err != nil {
		return nil, err
	}

	// Mount the pool.
	_, err = pool.Mount()
	if err != nil {
		return nil, err
	}

	// Record the updated config.
	updatedConfig = updatedReq.Config

	// Setup revert function.
	defer func() {
		if !tryUndo {
			return
		}

		pool.Delete(isNotification, nil)
	}()

	// In case the storage pool config was changed during the pool creation,
	// we need to update the database to reflect this change. This can e.g.
	// happen, when we create a loop file image. This means we append ".img"
	// to the path the user gave us and update the config in the storage
	// callback. So diff the config here to see if something like this has
	// happened.
	configDiff, _ := storagePools.ConfigDiff(req.Config, updatedConfig)
	if len(configDiff) > 0 {
		// Create the database entry for the storage pool.
		err = state.Cluster.UpdateStoragePool(req.Name, req.Description, updatedConfig)
		if err != nil {
			return nil, fmt.Errorf("Error inserting %s into database: %s", req.Name, err)
		}
	}

	// Success, update the closure to mark that the changes should be kept.
	tryUndo = false

	return updatedConfig, nil
}

// Helper around the low-level DB API, which also updates the driver names cache.
func dbStoragePoolCreateAndUpdateCache(s *state.State, poolName string, poolDescription string, poolDriver string, poolConfig map[string]string) (int64, error) {
	id, err := s.Cluster.CreateStoragePool(poolName, poolDescription, poolDriver, poolConfig)
	if err != nil {
		return id, err
	}

	// Update the storage drivers cache in api_1.0.go.
	storagePoolDriversCacheUpdate(s)

	return id, nil
}

// Helper around the low-level DB API, which also updates the driver names
// cache.
func dbStoragePoolDeleteAndUpdateCache(s *state.State, poolName string) error {
	_, err := s.Cluster.RemoveStoragePool(poolName)
	if err != nil {
		return err
	}

	// Update the storage drivers cache in api_1.0.go.
	storagePoolDriversCacheUpdate(s)

	return err
}
<commit_msg>lxd/storage/pools/utils: Updates comment and error for storagePoolCreateLocal<commit_after>package main

import (
	"fmt"

	"github.com/pkg/errors"

	"github.com/lxc/lxd/lxd/state"
	storagePools "github.com/lxc/lxd/lxd/storage"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
)

func storagePoolUpdate(state *state.State, name, newDescription string, newConfig map[string]string, withDB bool) error {
	pool, err := storagePools.GetPoolByName(state, name)
	if err != nil {
		return err
	}

	return pool.Update(!withDB, newDescription, newConfig, nil)
}

// storagePoolDBCreate creates a storage pool DB entry and returns the created Pool ID.
func storagePoolDBCreate(s *state.State, poolName, poolDescription string, driver string, config map[string]string) (int64, error) {
	// Check that the storage pool does not already exist.
	_, err := s.Cluster.GetStoragePoolID(poolName)
	if err == nil {
		return -1, fmt.Errorf("The storage pool already exists")
	}

	// Make sure that we don't pass a nil to the next function.
	if config == nil {
		config = map[string]string{}
	}
	err = storagePoolValidate(poolName, driver, config)
	if err != nil {
		return -1, err
	}

	// Fill in the defaults
	err = storagePoolFillDefault(poolName, driver, config)
	if err != nil {
		return -1, err
	}

	// Create the database entry for the storage pool.
	id, err := dbStoragePoolCreateAndUpdateCache(s, poolName, poolDescription, driver, config)
	if err != nil {
		return -1, fmt.Errorf("Error inserting %s into database: %s", poolName, err)
	}

	return id, nil
}

func storagePoolValidate(poolName string, driverName string, config map[string]string) error {
	// Check if the storage pool name is valid.
	err := storagePools.ValidName(poolName)
	if err != nil {
		return err
	}

	// Validate the requested storage pool configuration.
	err = storagePoolValidateConfig(poolName, driverName, config, nil)
	if err != nil {
		return err
	}

	return nil
}

func storagePoolCreateGlobal(state *state.State, req api.StoragePoolsPost) error {
	// Create the database entry.
	id, err := storagePoolDBCreate(state, req.Name, req.Description, req.Driver, req.Config)
	if err != nil {
		return err
	}

	// Define a function which reverts everything.  Defer this function
	// so that it doesn't need to be explicitly called in every failing
	// return path. Track whether or not we want to undo the changes
	// using a closure.
	tryUndo := true
	defer func() {
		if !tryUndo {
			return
		}

		dbStoragePoolDeleteAndUpdateCache(state, req.Name)
	}()

	_, err = storagePoolCreateLocal(state, id, req, false)
	if err != nil {
		return err
	}

	tryUndo = false
	return nil
}

// This performs local pool setup and updates DB record if config was changed during pool setup.
func storagePoolCreateLocal(state *state.State, id int64, req api.StoragePoolsPost, isNotification bool) (map[string]string, error) {
	tryUndo := true

	// Make a copy of the req for later diff.
	var updatedConfig map[string]string
	var updatedReq api.StoragePoolsPost
	shared.DeepCopy(&req, &updatedReq)

	// Fill in the defaults.
	err := storagePoolFillDefault(updatedReq.Name, updatedReq.Driver, updatedReq.Config)
	if err != nil {
		return nil, err
	}

	// Create the pool.
	pool, err := storagePools.CreatePool(state, id, &updatedReq, isNotification, nil)
	if err != nil {
		return nil, err
	}

	// Mount the pool.
	_, err = pool.Mount()
	if err != nil {
		return nil, err
	}

	// Record the updated config.
	updatedConfig = updatedReq.Config

	// Setup revert function.
	defer func() {
		if !tryUndo {
			return
		}

		pool.Delete(isNotification, nil)
	}()

	// In case the storage pool config was changed during the pool creation,
	// we need to update the database to reflect this change. This can e.g.
	// happen, when we create a loop file image. This means we append ".img"
	// to the path the user gave us and update the config in the storage
	// callback. So diff the config here to see if something like this has
	// happened.
	configDiff, _ := storagePools.ConfigDiff(req.Config, updatedConfig)
	if len(configDiff) > 0 {
		// Create the database entry for the storage pool.
		err = state.Cluster.UpdateStoragePool(req.Name, req.Description, updatedConfig)
		if err != nil {
			return nil, errors.Wrapf(err, "Error updating storage pool config after local create for %q", req.Name)
		}
	}

	// Success, update the closure to mark that the changes should be kept.
	tryUndo = false

	return updatedConfig, nil
}

// Helper around the low-level DB API, which also updates the driver names cache.
func dbStoragePoolCreateAndUpdateCache(s *state.State, poolName string, poolDescription string, poolDriver string, poolConfig map[string]string) (int64, error) {
	id, err := s.Cluster.CreateStoragePool(poolName, poolDescription, poolDriver, poolConfig)
	if err != nil {
		return id, err
	}

	// Update the storage drivers cache in api_1.0.go.
	storagePoolDriversCacheUpdate(s)

	return id, nil
}

// Helper around the low-level DB API, which also updates the driver names
// cache.
func dbStoragePoolDeleteAndUpdateCache(s *state.State, poolName string) error {
	_, err := s.Cluster.RemoveStoragePool(poolName)
	if err != nil {
		return err
	}

	// Update the storage drivers cache in api_1.0.go.
	storagePoolDriversCacheUpdate(s)

	return err
}
<|endoftext|>


<commit_before>// Copyright 2014-2016 CNI authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package skel provides skeleton code for a CNI plugin.
// In particular, it implements argument parsing and validation.
package skel

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"

	"github.com/containernetworking/cni/pkg/types"
	"github.com/containernetworking/cni/pkg/version"
)

// CmdArgs captures all the arguments passed in to the plugin
// via both env vars and stdin
type CmdArgs struct {
	ContainerID string
	Netns       string
	IfName      string
	Args        string
	Path        string
	StdinData   []byte
}

type dispatcher struct {
	Getenv func(string) string
	Stdin  io.Reader
	Stdout io.Writer
	Stderr io.Writer

	ConfVersionDecoder version.ConfigDecoder
	VersionReconciler  version.Reconciler
}

type reqForCmdEntry map[string]bool

func (t *dispatcher) getCmdArgsFromEnv() (string, *CmdArgs, error) {
	var cmd, contID, netns, ifName, args, path string

	vars := []struct {
		name      string
		val       *string
		reqForCmd reqForCmdEntry
	}{
		{
			"CNI_COMMAND",
			&cmd,
			reqForCmdEntry{
				"ADD": true,
				"DEL": true,
			},
		},
		{
			"CNI_CONTAINERID",
			&contID,
			reqForCmdEntry{
				"ADD": false,
				"DEL": false,
			},
		},
		{
			"CNI_NETNS",
			&netns,
			reqForCmdEntry{
				"ADD": true,
				"DEL": false,
			},
		},
		{
			"CNI_IFNAME",
			&ifName,
			reqForCmdEntry{
				"ADD": true,
				"DEL": true,
			},
		},
		{
			"CNI_ARGS",
			&args,
			reqForCmdEntry{
				"ADD": false,
				"DEL": false,
			},
		},
		{
			"CNI_PATH",
			&path,
			reqForCmdEntry{
				"ADD": true,
				"DEL": true,
			},
		},
	}

	argsMissing := false
	for _, v := range vars {
		*v.val = t.Getenv(v.name)
		if *v.val == "" {
			if v.reqForCmd[cmd] || v.name == "CNI_COMMAND" {
				fmt.Fprintf(t.Stderr, "%v env variable missing\n", v.name)
				argsMissing = true
			}
		}
	}

	if argsMissing {
		return "", nil, fmt.Errorf("required env variables missing")
	}

	stdinData, err := ioutil.ReadAll(t.Stdin)
	if err != nil {
		return "", nil, fmt.Errorf("error reading from stdin: %v", err)
	}

	cmdArgs := &CmdArgs{
		ContainerID: contID,
		Netns:       netns,
		IfName:      ifName,
		Args:        args,
		Path:        path,
		StdinData:   stdinData,
	}
	return cmd, cmdArgs, nil
}

func createTypedError(f string, args ...interface{}) *types.Error {
	return &types.Error{
		Code: 100,
		Msg:  fmt.Sprintf(f, args...),
	}
}

func (t *dispatcher) checkVersionAndCall(cmdArgs *CmdArgs, pluginVersionInfo version.PluginInfo, toCall func(*CmdArgs) error) error {
	configVersion, err := t.ConfVersionDecoder.Decode(cmdArgs.StdinData)
	if err != nil {
		return err
	}
	verErr := t.VersionReconciler.Check(configVersion, pluginVersionInfo)
	if verErr != nil {
		return &types.Error{
			Code:    types.ErrIncompatibleCNIVersion,
			Msg:     "incompatible CNI versions",
			Details: verErr.Details(),
		}
	}
	return toCall(cmdArgs)
}

func (t *dispatcher) pluginMain(cmdAdd, cmdDel func(_ *CmdArgs) error, versionInfo version.PluginInfo) *types.Error {
	cmd, cmdArgs, err := t.getCmdArgsFromEnv()
	if err != nil {
		return createTypedError(err.Error())
	}

	switch cmd {
	case "ADD":
		err = t.checkVersionAndCall(cmdArgs, versionInfo, cmdAdd)
	case "DEL":
		err = t.checkVersionAndCall(cmdArgs, versionInfo, cmdDel)
	case "VERSION":
		err = versionInfo.Encode(t.Stdout)
	default:
		return createTypedError("unknown CNI_COMMAND: %v", cmd)
	}

	if err != nil {
		if e, ok := err.(*types.Error); ok {
			// don't wrap Error in Error
			return e
		}
		return createTypedError(err.Error())
	}
	return nil
}

// PluginMain is the "main" for a plugin. It accepts
// two callback functions for add and del commands.
func PluginMain(cmdAdd, cmdDel func(_ *CmdArgs) error, versionInfo version.PluginInfo) {
	caller := dispatcher{
		Getenv: os.Getenv,
		Stdin:  os.Stdin,
		Stdout: os.Stdout,
		Stderr: os.Stderr,
	}

	err := caller.pluginMain(cmdAdd, cmdDel, versionInfo)
	if err != nil {
		dieErr(err)
	}
}

func dieErr(e *types.Error) {
	if err := e.Print(); err != nil {
		log.Print("Error writing error JSON to stdout: ", err)
	}
	os.Exit(1)
}
<commit_msg>skel: adds PluginMainWithError which returns a *types.Error<commit_after>// Copyright 2014-2016 CNI authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package skel provides skeleton code for a CNI plugin.
// In particular, it implements argument parsing and validation.
package skel

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"

	"github.com/containernetworking/cni/pkg/types"
	"github.com/containernetworking/cni/pkg/version"
)

// CmdArgs captures all the arguments passed in to the plugin
// via both env vars and stdin
type CmdArgs struct {
	ContainerID string
	Netns       string
	IfName      string
	Args        string
	Path        string
	StdinData   []byte
}

type dispatcher struct {
	Getenv func(string) string
	Stdin  io.Reader
	Stdout io.Writer
	Stderr io.Writer

	ConfVersionDecoder version.ConfigDecoder
	VersionReconciler  version.Reconciler
}

type reqForCmdEntry map[string]bool

func (t *dispatcher) getCmdArgsFromEnv() (string, *CmdArgs, error) {
	var cmd, contID, netns, ifName, args, path string

	vars := []struct {
		name      string
		val       *string
		reqForCmd reqForCmdEntry
	}{
		{
			"CNI_COMMAND",
			&cmd,
			reqForCmdEntry{
				"ADD": true,
				"DEL": true,
			},
		},
		{
			"CNI_CONTAINERID",
			&contID,
			reqForCmdEntry{
				"ADD": false,
				"DEL": false,
			},
		},
		{
			"CNI_NETNS",
			&netns,
			reqForCmdEntry{
				"ADD": true,
				"DEL": false,
			},
		},
		{
			"CNI_IFNAME",
			&ifName,
			reqForCmdEntry{
				"ADD": true,
				"DEL": true,
			},
		},
		{
			"CNI_ARGS",
			&args,
			reqForCmdEntry{
				"ADD": false,
				"DEL": false,
			},
		},
		{
			"CNI_PATH",
			&path,
			reqForCmdEntry{
				"ADD": true,
				"DEL": true,
			},
		},
	}

	argsMissing := false
	for _, v := range vars {
		*v.val = t.Getenv(v.name)
		if *v.val == "" {
			if v.reqForCmd[cmd] || v.name == "CNI_COMMAND" {
				fmt.Fprintf(t.Stderr, "%v env variable missing\n", v.name)
				argsMissing = true
			}
		}
	}

	if argsMissing {
		return "", nil, fmt.Errorf("required env variables missing")
	}

	stdinData, err := ioutil.ReadAll(t.Stdin)
	if err != nil {
		return "", nil, fmt.Errorf("error reading from stdin: %v", err)
	}

	cmdArgs := &CmdArgs{
		ContainerID: contID,
		Netns:       netns,
		IfName:      ifName,
		Args:        args,
		Path:        path,
		StdinData:   stdinData,
	}
	return cmd, cmdArgs, nil
}

func createTypedError(f string, args ...interface{}) *types.Error {
	return &types.Error{
		Code: 100,
		Msg:  fmt.Sprintf(f, args...),
	}
}

func (t *dispatcher) checkVersionAndCall(cmdArgs *CmdArgs, pluginVersionInfo version.PluginInfo, toCall func(*CmdArgs) error) error {
	configVersion, err := t.ConfVersionDecoder.Decode(cmdArgs.StdinData)
	if err != nil {
		return err
	}
	verErr := t.VersionReconciler.Check(configVersion, pluginVersionInfo)
	if verErr != nil {
		return &types.Error{
			Code:    types.ErrIncompatibleCNIVersion,
			Msg:     "incompatible CNI versions",
			Details: verErr.Details(),
		}
	}
	return toCall(cmdArgs)
}

func (t *dispatcher) pluginMain(cmdAdd, cmdDel func(_ *CmdArgs) error, versionInfo version.PluginInfo) *types.Error {
	cmd, cmdArgs, err := t.getCmdArgsFromEnv()
	if err != nil {
		return createTypedError(err.Error())
	}

	switch cmd {
	case "ADD":
		err = t.checkVersionAndCall(cmdArgs, versionInfo, cmdAdd)
	case "DEL":
		err = t.checkVersionAndCall(cmdArgs, versionInfo, cmdDel)
	case "VERSION":
		err = versionInfo.Encode(t.Stdout)
	default:
		return createTypedError("unknown CNI_COMMAND: %v", cmd)
	}

	if err != nil {
		if e, ok := err.(*types.Error); ok {
			// don't wrap Error in Error
			return e
		}
		return createTypedError(err.Error())
	}
	return nil
}

// PluginMainWithError is the core "main" for a plugin. It accepts
// callback functions for add and del CNI commands and returns an error.
//
// The caller must also specify what CNI spec versions the plugin supports.
//
// It is the responsibility of the caller to check for non-nil error return.
//
// For a plugin to comply with the CNI spec, it must print any error to stdout
// as JSON and then exit with nonzero status code.
//
// To let this package automatically handle errors and call os.Exit(1) for you,
// use PluginMain() instead.
func PluginMainWithError(cmdAdd, cmdDel func(_ *CmdArgs) error, versionInfo version.PluginInfo) *types.Error {
	return (&dispatcher{
		Getenv: os.Getenv,
		Stdin:  os.Stdin,
		Stdout: os.Stdout,
		Stderr: os.Stderr,
	}).pluginMain(cmdAdd, cmdDel, versionInfo)
}

// PluginMain is the core "main" for a plugin which includes automatic error handling.
//
// The caller must also specify what CNI spec versions the plugin supports.
//
// When an error occurs in either cmdAdd or cmdDel, PluginMain will print the error
// as JSON to stdout and call os.Exit(1).
//
// To have more control over error handling, use PluginMainWithError() instead.
func PluginMain(cmdAdd, cmdDel func(_ *CmdArgs) error, versionInfo version.PluginInfo) {
	if e := PluginMainWithError(cmdAdd, cmdDel, versionInfo); e != nil {
		if err := e.Print(); err != nil {
			log.Print("Error writing error JSON to stdout: ", err)
		}
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package views

const list = `
<!DOCTYPE html>

    <meta charset="utf-8" />
    <title>alexandria</title>
    <link rel="stylesheet" href="/assets/styles.css" />

alexandria

    {{if .}}

        Sign-in

    {{end}}

    <input id="filter" name="filter" type="text" placeholder="Search" />

          Title
          Author
          Added
          Editions

`
<commit_msg>Remove protocol on jquery include<commit_after>package views

const list = `
<!DOCTYPE html>

    <meta charset="utf-8" />
    <title>alexandria</title>
    <link rel="stylesheet" href="/assets/styles.css" />

alexandria

    {{if .}}

        Sign-in

    {{end}}

    <input id="filter" name="filter" type="text" placeholder="Search" />

          Title
          Author
          Added
          Editions

`
<|endoftext|>