<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package runtime

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"io"
	"net/url"
	"reflect"

	"k8s.io/apimachinery/pkg/conversion/queryparams"
	"k8s.io/apimachinery/pkg/runtime/schema"
)

// codec binds an encoder and decoder.
type codec struct {
	Encoder
	Decoder
}

// NewCodec creates a Codec from an Encoder and Decoder.
func NewCodec(e Encoder, d Decoder) Codec {
	return codec{e, d}
}

// Encode is a convenience wrapper for encoding to a []byte from an Encoder
func Encode(e Encoder, obj Object) ([]byte, error) {
	// TODO: reuse buffer
	buf := &bytes.Buffer{}
	if err := e.Encode(obj, buf); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

// Decode is a convenience wrapper for decoding data into an Object.
func Decode(d Decoder, data []byte) (Object, error) {
	obj, _, err := d.Decode(data, nil, nil)
	return obj, err
}

// DecodeInto performs a Decode into the provided object.
func DecodeInto(d Decoder, data []byte, into Object) error {
	out, gvk, err := d.Decode(data, nil, into)
	if err != nil {
		return err
	}
	if out != into {
		return fmt.Errorf("unable to decode %s into %v", gvk, reflect.TypeOf(into))
	}
	return nil
}

// EncodeOrDie is a version of Encode which will panic instead of returning an error. For tests.
func EncodeOrDie(e Encoder, obj Object) string {
	bytes, err := Encode(e, obj)
	if err != nil {
		panic(err)
	}
	return string(bytes)
}

// DefaultingSerializer invokes defaulting after decoding.
type DefaultingSerializer struct {
	Defaulter ObjectDefaulter
	Decoder   Decoder
	// Encoder is optional to allow this type to be used as both a Decoder and an Encoder
	Encoder
}

// Decode performs a decode and then allows the defaulter to act on the provided object.
func (d DefaultingSerializer) Decode(data []byte, defaultGVK *schema.GroupVersionKind, into Object) (Object, *schema.GroupVersionKind, error) {
	obj, gvk, err := d.Decoder.Decode(data, defaultGVK, into)
	if err != nil {
		return obj, gvk, err
	}
	d.Defaulter.Default(obj)
	return obj, gvk, nil
}

// UseOrCreateObject returns obj if the canonical ObjectKind returned by the provided typer matches gvk, or
// invokes the ObjectCreator to instantiate a new gvk. Returns an error if the typer cannot find the object.
func UseOrCreateObject(t ObjectTyper, c ObjectCreater, gvk schema.GroupVersionKind, obj Object) (Object, error) {
	if obj != nil {
		kinds, _, err := t.ObjectKinds(obj)
		if err != nil {
			return nil, err
		}
		for _, kind := range kinds {
			if gvk == kind {
				return obj, nil
			}
		}
	}
	return c.New(gvk)
}

// NoopEncoder converts an Decoder to a Serializer or Codec for code that expects them but only uses decoding.
type NoopEncoder struct {
	Decoder
}

var _ Serializer = NoopEncoder{}

func (n NoopEncoder) Encode(obj Object, w io.Writer) error {
	return fmt.Errorf("encoding is not allowed for this codec: %v", reflect.TypeOf(n.Decoder))
}

// NoopDecoder converts an Encoder to a Serializer or Codec for code that expects them but only uses encoding.
type NoopDecoder struct {
	Encoder
}

var _ Serializer = NoopDecoder{}

func (n NoopDecoder) Decode(data []byte, gvk *schema.GroupVersionKind, into Object) (Object, *schema.GroupVersionKind, error) {
	return nil, nil, fmt.Errorf("decoding is not allowed for this codec: %v", reflect.TypeOf(n.Encoder))
}

// NewParameterCodec creates a ParameterCodec capable of transforming url values into versioned objects and back.
func NewParameterCodec(scheme *Scheme) ParameterCodec {
	return &parameterCodec{
		typer:     scheme,
		convertor: scheme,
		creator:   scheme,
	}
}

// parameterCodec implements conversion to and from query parameters and objects.
type parameterCodec struct {
	typer     ObjectTyper
	convertor ObjectConvertor
	creator   ObjectCreater
}

var _ ParameterCodec = &parameterCodec{}

// DecodeParameters converts the provided url.Values into an object of type From with the kind of into, and then
// converts that object to into (if necessary). Returns an error if the operation cannot be completed.
func (c *parameterCodec) DecodeParameters(parameters url.Values, from schema.GroupVersion, into Object) error {
	if len(parameters) == 0 {
		return nil
	}
	targetGVKs, _, err := c.typer.ObjectKinds(into)
	if err != nil {
		return err
	}
	for i := range targetGVKs {
		if targetGVKs[i].GroupVersion() == from {
			return c.convertor.Convert(&parameters, into, nil)
		}
	}
	input, err := c.creator.New(from.WithKind(targetGVKs[0].Kind))
	if err != nil {
		return err
	}
	if err := c.convertor.Convert(&parameters, input, nil); err != nil {
		return err
	}
	return c.convertor.Convert(input, into, nil)
}

// EncodeParameters converts the provided object into the to version, then converts that object to url.Values.
// Returns an error if conversion is not possible.
func (c *parameterCodec) EncodeParameters(obj Object, to schema.GroupVersion) (url.Values, error) {
	gvks, _, err := c.typer.ObjectKinds(obj)
	if err != nil {
		return nil, err
	}
	gvk := gvks[0]
	if to != gvk.GroupVersion() {
		out, err := c.convertor.ConvertToVersion(obj, to)
		if err != nil {
			return nil, err
		}
		obj = out
	}
	return queryparams.Convert(obj)
}

type base64Serializer struct {
	Encoder
	Decoder
}

func NewBase64Serializer(e Encoder, d Decoder) Serializer {
	return &base64Serializer{e, d}
}

func (s base64Serializer) Encode(obj Object, stream io.Writer) error {
	e := base64.NewEncoder(base64.StdEncoding, stream)
	err := s.Encoder.Encode(obj, e)
	e.Close()
	return err
}

func (s base64Serializer) Decode(data []byte, defaults *schema.GroupVersionKind, into Object) (Object, *schema.GroupVersionKind, error) {
	out := make([]byte, base64.StdEncoding.DecodedLen(len(data)))
	n, err := base64.StdEncoding.Decode(out, data)
	if err != nil {
		return nil, nil, err
	}
	return s.Decoder.Decode(out[:n], defaults, into)
}

// SerializerInfoForMediaType returns the first info in types that has a matching media type (which cannot
// include media-type parameters), or the first info with an empty media type, or false if no type matches.
func SerializerInfoForMediaType(types []SerializerInfo, mediaType string) (SerializerInfo, bool) {
	for _, info := range types {
		if info.MediaType == mediaType {
			return info, true
		}
	}
	for _, info := range types {
		if len(info.MediaType) == 0 {
			return info, true
		}
	}
	return SerializerInfo{}, false
}

var (
	// InternalGroupVersioner will always prefer the internal version for a given group version kind.
	InternalGroupVersioner GroupVersioner = internalGroupVersioner{}
	// DisabledGroupVersioner will reject all kinds passed to it.
	DisabledGroupVersioner GroupVersioner = disabledGroupVersioner{}
)

type internalGroupVersioner struct{}

// KindForGroupVersionKinds returns an internal Kind if one is found, or converts the first provided kind to the internal version.
func (internalGroupVersioner) KindForGroupVersionKinds(kinds []schema.GroupVersionKind) (schema.GroupVersionKind, bool) {
	for _, kind := range kinds {
		if kind.Version == APIVersionInternal {
			return kind, true
		}
	}
	for _, kind := range kinds {
		return schema.GroupVersionKind{Group: kind.Group, Version: APIVersionInternal, Kind: kind.Kind}, true
	}
	return schema.GroupVersionKind{}, false
}

type disabledGroupVersioner struct{}

// KindForGroupVersionKinds returns false for any input.
func (disabledGroupVersioner) KindForGroupVersionKinds(kinds []schema.GroupVersionKind) (schema.GroupVersionKind, bool) {
	return schema.GroupVersionKind{}, false
}

// GroupVersioners implements GroupVersioner and resolves to the first exact match for any kind.
type GroupVersioners []GroupVersioner

// KindForGroupVersionKinds returns the first match of any of the group versioners, or false if no match occured.
func (gvs GroupVersioners) KindForGroupVersionKinds(kinds []schema.GroupVersionKind) (schema.GroupVersionKind, bool) {
	for _, gv := range gvs {
		target, ok := gv.KindForGroupVersionKinds(kinds)
		if !ok {
			continue
		}
		return target, true
	}
	return schema.GroupVersionKind{}, false
}

// Assert that schema.GroupVersion and GroupVersions implement GroupVersioner
var _ GroupVersioner = schema.GroupVersion{}
var _ GroupVersioner = schema.GroupVersions{}
var _ GroupVersioner = multiGroupVersioner{}

type multiGroupVersioner struct {
	target             schema.GroupVersion
	acceptedGroupKinds []schema.GroupKind
}

// NewMultiGroupVersioner returns the provided group version for any kind that matches one of the provided group kinds.
// Kind may be empty in the provided group kind, in which case any kind will match.
func NewMultiGroupVersioner(gv schema.GroupVersion, groupKinds ...schema.GroupKind) GroupVersioner {
	if len(groupKinds) == 0 || (len(groupKinds) == 1 && groupKinds[0].Group == gv.Group) {
		return gv
	}
	return multiGroupVersioner{target: gv, acceptedGroupKinds: groupKinds}
}

// KindForGroupVersionKinds returns the target group version if any kind matches any of the original group kinds. It will
// use the originating kind where possible.
func (v multiGroupVersioner) KindForGroupVersionKinds(kinds []schema.GroupVersionKind) (schema.GroupVersionKind, bool) {
	for _, src := range kinds {
		for _, kind := range v.acceptedGroupKinds {
			if kind.Group != src.Group {
				continue
			}
			if len(kind.Kind) > 0 && kind.Kind != src.Kind {
				continue
			}
			return v.target.WithKind(src.Kind), true
		}
	}
	return schema.GroupVersionKind{}, false
}
<commit_msg>make the parameter codec include defaulters<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package runtime

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"io"
	"net/url"
	"reflect"

	"k8s.io/apimachinery/pkg/conversion/queryparams"
	"k8s.io/apimachinery/pkg/runtime/schema"
)

// codec binds an encoder and decoder.
type codec struct {
	Encoder
	Decoder
}

// NewCodec creates a Codec from an Encoder and Decoder.
func NewCodec(e Encoder, d Decoder) Codec {
	return codec{e, d}
}

// Encode is a convenience wrapper for encoding to a []byte from an Encoder
func Encode(e Encoder, obj Object) ([]byte, error) {
	// TODO: reuse buffer
	buf := &bytes.Buffer{}
	if err := e.Encode(obj, buf); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

// Decode is a convenience wrapper for decoding data into an Object.
func Decode(d Decoder, data []byte) (Object, error) {
	obj, _, err := d.Decode(data, nil, nil)
	return obj, err
}

// DecodeInto performs a Decode into the provided object.
func DecodeInto(d Decoder, data []byte, into Object) error {
	out, gvk, err := d.Decode(data, nil, into)
	if err != nil {
		return err
	}
	if out != into {
		return fmt.Errorf("unable to decode %s into %v", gvk, reflect.TypeOf(into))
	}
	return nil
}

// EncodeOrDie is a version of Encode which will panic instead of returning an error. For tests.
func EncodeOrDie(e Encoder, obj Object) string {
	bytes, err := Encode(e, obj)
	if err != nil {
		panic(err)
	}
	return string(bytes)
}

// DefaultingSerializer invokes defaulting after decoding.
type DefaultingSerializer struct {
	Defaulter ObjectDefaulter
	Decoder   Decoder
	// Encoder is optional to allow this type to be used as both a Decoder and an Encoder
	Encoder
}

// Decode performs a decode and then allows the defaulter to act on the provided object.
func (d DefaultingSerializer) Decode(data []byte, defaultGVK *schema.GroupVersionKind, into Object) (Object, *schema.GroupVersionKind, error) {
	obj, gvk, err := d.Decoder.Decode(data, defaultGVK, into)
	if err != nil {
		return obj, gvk, err
	}
	d.Defaulter.Default(obj)
	return obj, gvk, nil
}

// UseOrCreateObject returns obj if the canonical ObjectKind returned by the provided typer matches gvk, or
// invokes the ObjectCreator to instantiate a new gvk. Returns an error if the typer cannot find the object.
func UseOrCreateObject(t ObjectTyper, c ObjectCreater, gvk schema.GroupVersionKind, obj Object) (Object, error) {
	if obj != nil {
		kinds, _, err := t.ObjectKinds(obj)
		if err != nil {
			return nil, err
		}
		for _, kind := range kinds {
			if gvk == kind {
				return obj, nil
			}
		}
	}
	return c.New(gvk)
}

// NoopEncoder converts an Decoder to a Serializer or Codec for code that expects them but only uses decoding.
type NoopEncoder struct {
	Decoder
}

var _ Serializer = NoopEncoder{}

func (n NoopEncoder) Encode(obj Object, w io.Writer) error {
	return fmt.Errorf("encoding is not allowed for this codec: %v", reflect.TypeOf(n.Decoder))
}

// NoopDecoder converts an Encoder to a Serializer or Codec for code that expects them but only uses encoding.
type NoopDecoder struct {
	Encoder
}

var _ Serializer = NoopDecoder{}

func (n NoopDecoder) Decode(data []byte, gvk *schema.GroupVersionKind, into Object) (Object, *schema.GroupVersionKind, error) {
	return nil, nil, fmt.Errorf("decoding is not allowed for this codec: %v", reflect.TypeOf(n.Encoder))
}

// NewParameterCodec creates a ParameterCodec capable of transforming url values into versioned objects and back.
func NewParameterCodec(scheme *Scheme) ParameterCodec {
	return &parameterCodec{
		typer:     scheme,
		convertor: scheme,
		creator:   scheme,
		defaulter: scheme,
	}
}

// parameterCodec implements conversion to and from query parameters and objects.
type parameterCodec struct {
	typer     ObjectTyper
	convertor ObjectConvertor
	creator   ObjectCreater
	defaulter ObjectDefaulter
}

var _ ParameterCodec = &parameterCodec{}

// DecodeParameters converts the provided url.Values into an object of type From with the kind of into, and then
// converts that object to into (if necessary). Returns an error if the operation cannot be completed.
func (c *parameterCodec) DecodeParameters(parameters url.Values, from schema.GroupVersion, into Object) error {
	if len(parameters) == 0 {
		return nil
	}
	targetGVKs, _, err := c.typer.ObjectKinds(into)
	if err != nil {
		return err
	}
	for i := range targetGVKs {
		if targetGVKs[i].GroupVersion() == from {
			if err := c.convertor.Convert(&parameters, into, nil); err != nil {
				return err
			}
			// in the case where we going into the same object we're receiving, default on the outbound object
			if c.defaulter != nil {
				c.defaulter.Default(into)
			}
			return nil
		}
	}

	input, err := c.creator.New(from.WithKind(targetGVKs[0].Kind))
	if err != nil {
		return err
	}
	if err := c.convertor.Convert(&parameters, input, nil); err != nil {
		return err
	}
	// if we have defaulter, default the input before converting to output
	if c.defaulter != nil {
		c.defaulter.Default(input)
	}
	return c.convertor.Convert(input, into, nil)
}

// EncodeParameters converts the provided object into the to version, then converts that object to url.Values.
// Returns an error if conversion is not possible.
func (c *parameterCodec) EncodeParameters(obj Object, to schema.GroupVersion) (url.Values, error) {
	gvks, _, err := c.typer.ObjectKinds(obj)
	if err != nil {
		return nil, err
	}
	gvk := gvks[0]
	if to != gvk.GroupVersion() {
		out, err := c.convertor.ConvertToVersion(obj, to)
		if err != nil {
			return nil, err
		}
		obj = out
	}
	return queryparams.Convert(obj)
}

type base64Serializer struct {
	Encoder
	Decoder
}

func NewBase64Serializer(e Encoder, d Decoder) Serializer {
	return &base64Serializer{e, d}
}

func (s base64Serializer) Encode(obj Object, stream io.Writer) error {
	e := base64.NewEncoder(base64.StdEncoding, stream)
	err := s.Encoder.Encode(obj, e)
	e.Close()
	return err
}

func (s base64Serializer) Decode(data []byte, defaults *schema.GroupVersionKind, into Object) (Object, *schema.GroupVersionKind, error) {
	out := make([]byte, base64.StdEncoding.DecodedLen(len(data)))
	n, err := base64.StdEncoding.Decode(out, data)
	if err != nil {
		return nil, nil, err
	}
	return s.Decoder.Decode(out[:n], defaults, into)
}

// SerializerInfoForMediaType returns the first info in types that has a matching media type (which cannot
// include media-type parameters), or the first info with an empty media type, or false if no type matches.
func SerializerInfoForMediaType(types []SerializerInfo, mediaType string) (SerializerInfo, bool) {
	for _, info := range types {
		if info.MediaType == mediaType {
			return info, true
		}
	}
	for _, info := range types {
		if len(info.MediaType) == 0 {
			return info, true
		}
	}
	return SerializerInfo{}, false
}

var (
	// InternalGroupVersioner will always prefer the internal version for a given group version kind.
	InternalGroupVersioner GroupVersioner = internalGroupVersioner{}
	// DisabledGroupVersioner will reject all kinds passed to it.
	DisabledGroupVersioner GroupVersioner = disabledGroupVersioner{}
)

type internalGroupVersioner struct{}

// KindForGroupVersionKinds returns an internal Kind if one is found, or converts the first provided kind to the internal version.
func (internalGroupVersioner) KindForGroupVersionKinds(kinds []schema.GroupVersionKind) (schema.GroupVersionKind, bool) {
	for _, kind := range kinds {
		if kind.Version == APIVersionInternal {
			return kind, true
		}
	}
	for _, kind := range kinds {
		return schema.GroupVersionKind{Group: kind.Group, Version: APIVersionInternal, Kind: kind.Kind}, true
	}
	return schema.GroupVersionKind{}, false
}

type disabledGroupVersioner struct{}

// KindForGroupVersionKinds returns false for any input.
func (disabledGroupVersioner) KindForGroupVersionKinds(kinds []schema.GroupVersionKind) (schema.GroupVersionKind, bool) {
	return schema.GroupVersionKind{}, false
}

// GroupVersioners implements GroupVersioner and resolves to the first exact match for any kind.
type GroupVersioners []GroupVersioner

// KindForGroupVersionKinds returns the first match of any of the group versioners, or false if no match occured.
func (gvs GroupVersioners) KindForGroupVersionKinds(kinds []schema.GroupVersionKind) (schema.GroupVersionKind, bool) {
	for _, gv := range gvs {
		target, ok := gv.KindForGroupVersionKinds(kinds)
		if !ok {
			continue
		}
		return target, true
	}
	return schema.GroupVersionKind{}, false
}

// Assert that schema.GroupVersion and GroupVersions implement GroupVersioner
var _ GroupVersioner = schema.GroupVersion{}
var _ GroupVersioner = schema.GroupVersions{}
var _ GroupVersioner = multiGroupVersioner{}

type multiGroupVersioner struct {
	target             schema.GroupVersion
	acceptedGroupKinds []schema.GroupKind
}

// NewMultiGroupVersioner returns the provided group version for any kind that matches one of the provided group kinds.
// Kind may be empty in the provided group kind, in which case any kind will match.
func NewMultiGroupVersioner(gv schema.GroupVersion, groupKinds ...schema.GroupKind) GroupVersioner {
	if len(groupKinds) == 0 || (len(groupKinds) == 1 && groupKinds[0].Group == gv.Group) {
		return gv
	}
	return multiGroupVersioner{target: gv, acceptedGroupKinds: groupKinds}
}

// KindForGroupVersionKinds returns the target group version if any kind matches any of the original group kinds. It will
// use the originating kind where possible.
func (v multiGroupVersioner) KindForGroupVersionKinds(kinds []schema.GroupVersionKind) (schema.GroupVersionKind, bool) {
	for _, src := range kinds {
		for _, kind := range v.acceptedGroupKinds {
			if kind.Group != src.Group {
				continue
			}
			if len(kind.Kind) > 0 && kind.Kind != src.Kind {
				continue
			}
			return v.target.WithKind(src.Kind), true
		}
	}
	return schema.GroupVersionKind{}, false
}
<|endoftext|>


<commit_before>// vm: tabstop=2 shiftwidth=2

package main

import (
	"bytes"
	"strings"
	"errors"
	"encoding/binary"
	"encoding/hex"
	"time"
	"crypto/rand"
	"github.com/crooks/yamn/keymgr"
	"code.google.com/p/go.crypto/nacl/box"
)

// timestamp creates a Mixmaster formatted timestamp, consisting of an intro
// string concatented to the number of days since Epoch (little Endian).
func timestamp() (stamp []byte) {
	d := uint16(time.Now().UTC().Unix() / 86400)
	days := make([]byte, 2)
	binary.LittleEndian.PutUint16(days, d)
	stamp = append([]byte("0000\x00"), days...)
	return
}

// Generate a public/private ECC key pair
func eccGenerate() (public, private []byte) {
	puba, priva, err := box.GenerateKey(rand.Reader)
	if err != nil {
		panic(err)
	}
	public = make([]byte, 32)
	private = make([]byte, 32)
	copy(public[:], puba[:])
	copy(private[:], priva[:])
	if len(public) != 32 {
		panic("Invalid pubkey length generated")
	}
	if len(private) != 32 {
		panic("Invalid seckey length generated")
	}
	return
}

/*
Slot Header Format
[ Recipient key ID		 16 Bytes ]
[ Sender Public key		 32 Bytes ]
[ Xsalsa20 Nonce		 24 Bytes ]
[ Encrypted data		408 Bytes ] (392 + Overhead)
[ Random padding		 32 Bytes ]
*/

type slotHead struct {
	recipientKeyid []byte
	recipientPK []byte
	//sender_pubkey []byte
	//nonce []byte
	data []byte
}

func (h *slotHead) encodeHead() []byte {
	// Generate an ECC key pair
	sendpub, sendpriv, err := box.GenerateKey(rand.Reader)
	if err != nil {
		panic(err)
	}
	var nonce [24]byte
	copy(nonce[:], randbytes(24))
	buf := new(bytes.Buffer)
	buf.Write(h.recipientKeyid)
	buf.Write(sendpub[:])
	buf.Write(nonce[:])
	var pk [32]byte
	copy(pk[:], h.recipientPK)
	buf.Write(box.Seal(nil, h.data, &nonce, &pk, sendpriv))
	err = bufLenCheck(buf.Len(), 480)
	if err != nil {
		panic(err)
	}
	buf.Write(randbytes(headerBytes - buf.Len()))
	return buf.Bytes()
}

// decodeHead decodes a slot header
func decodeHead(b []byte, secret *keymgr.Secring) (data []byte, err error) {
	/*
	Decode functions should return their associated structs but, in this
	instance, the only field of value is the decrypted data.
	*/
	err = lenCheck(len(b), headerBytes)
	if err != nil {
		return
	}
	var keyid string
	keyid = hex.EncodeToString(b[0:16])
	var sk []byte
	sk, err = secret.GetSK(keyid)
	if err != nil {
		return
	}
	var sender_pk [32]byte
	copy(sender_pk[:], b[16:48])
	var recipient_sk [32]byte
	copy(recipient_sk[:], sk)
	var nonce [24]byte
	copy(nonce[:], b[48:72])
	var auth bool
	data, auth = box.Open(
		nil,
		b[72:72+408],
		&nonce,
		&sender_pk,
		&recipient_sk)
	if ! auth {
		err = errors.New("Authentication failed decrypting slot data")
		return
	}
	err = lenCheck(len(data), 392)
	return
}

/*
Encrypted Data
[ Packet ID		 16 Bytes ]
[ AES-CTR key		 32 Bytes ]
[ Packet type ID	  1 Byte  ]
[ Padded packet info	256 Bytes ]
[ Timestamp		  7 Bytes ]
[ Anti-tag digest	 64 Bytes ]
[ Padding		 16 Bytes ]
Total	392 Bytes
*/

type slotData struct {
	packetID []byte
	aesKey []byte
	packetType uint8
	packetInfo []byte
	timestamp []byte
	tagHash []byte
}

func (d slotData) encodeData() (b []byte, err error) {
	err = lenCheck(len(d.packetID), 16)
	if err != nil {
		return
	}
	err = lenCheck(len(d.aesKey), 32)
	if err != nil {
		return
	}
	err = lenCheck(len(d.packetInfo), 256)
	if err != nil {
		return
	}
	err = lenCheck(len(d.tagHash), 64)
	if err != nil {
		return
	}
	buf := new(bytes.Buffer)
	buf.Write(d.packetID)
	buf.Write(d.aesKey)
	buf.WriteByte(d.packetType)
	buf.Write(d.packetInfo)
	buf.Write(d.timestamp)
	buf.Write(d.tagHash)
	err = bufLenCheck(buf.Len(), 376)
	if err != nil {
		return
	}
	buf.WriteString(strings.Repeat("\x00", 392 - buf.Len()))
	b = buf.Bytes()
	return
}

func (data *slotData) decodeData(b []byte) (err error) {
	err = lenCheck(len(b), 392)
	if err != nil {
		return
	}
	data.packetID = b[0:16]
	data.aesKey = b[16:48]
	data.packetType = uint8(b[48])
	data.packetInfo = b[49:305]
	data.timestamp = b[305:312]
	//TODO Test timestamp
	data.tagHash = b[312:376]
	//Padding[376:392]
	return
}

/*
Final Hop
[ Chunk num		  1 Byte  ]
[ Num chunks		  1 Byte  ]
[ Message ID		 16 Bytes ]
[ AES-CTR IV		 16 Bytes ]
[ Body length		  4 Bytes ]
[ Delivery method	  1 Byte  ]

Currently supported delivery methods are:-
[ SMTP				  0 ]
[ Dummy - discard 		255 ]
*/

type slotFinal struct {
	chunkNum uint8
	numChunks uint8
	messageID []byte
	aesIV []byte
	bodyBytes int
	deliveryMethod uint8
}

func (f *slotFinal) encodeFinal() []byte {
	buf := new(bytes.Buffer)
	buf.WriteByte(f.chunkNum)
	buf.WriteByte(f.numChunks)
	buf.Write(f.messageID)
	buf.Write(f.aesIV)
	tmp := make([]byte, 4)
	binary.LittleEndian.PutUint32(tmp, uint32(f.bodyBytes))
	buf.Write(tmp)
	buf.WriteByte(f.deliveryMethod)
	err := bufLenCheck(buf.Len(), 39)
	if err != nil {
		panic(err)
	}
	buf.WriteString(strings.Repeat("\x00", 256 - buf.Len()))
	return buf.Bytes()
}

func (f *slotFinal) decodeFinal(b []byte) (err error) {
	err = lenCheck(len(b), 256)
	if err != nil {
		return
	}
	f.chunkNum = b[0]
	f.numChunks = b[1]
	f.messageID = b[2:18]
	f.aesIV = b[18:34]
	f.bodyBytes = int(binary.LittleEndian.Uint32(b[34:38]))
	f.deliveryMethod = b[38]
	return
}

/*
Intermediate Hop
[ 11 * AES-CTR IVs		176 Bytes ]
[ Next hop address		 80 Bytes ]

IVs are:
[ 9 * Header slots		144 Bytes ]
[ 1 * Deterministic header	 16 Bytes ]
[ 1 * Payload header		 16 Bytes ]
*/

type slotIntermediate struct {
	aesIVs []byte
	nextHop []byte
}

func (i *slotIntermediate) setNextHop(nh string) {
	i.nextHop = []byte(nh + strings.Repeat("\x00", 80 - len(nh)))
}

func (i *slotIntermediate) getNextHop() string {
	return strings.TrimRight(string(i.nextHop), "\x00")
}

func (i *slotIntermediate) encodeIntermediate() []byte {
	var err error
	err = lenCheck(len(i.aesIVs), (maxChainLength + 1) * 16)
	if err != nil {
		panic(err)
	}
	buf := new(bytes.Buffer)
	buf.Write(i.aesIVs)
	buf.Write(i.nextHop)
	err = bufLenCheck(buf.Len(), 256)
	if err != nil {
		panic(err)
	}
	return buf.Bytes()
}

func (i *slotIntermediate) decodeIntermediate(b []byte) (err error) {
	err = lenCheck(len(b), 256)
	if err != nil {
		return
	}
	i.aesIVs = b[:176]
	i.nextHop = b[176:]
	return
}
<commit_msg>Tweaked to compile with go-1.4<commit_after>// vm: tabstop=2 shiftwidth=2

package main

import (
	"bytes"
	"strings"
	"errors"
	"encoding/binary"
	"encoding/hex"
	"time"
	"crypto/rand"
	"github.com/crooks/yamn/keymgr"
	"golang.org/x/crypto/nacl/box"
	//"code.google.com/p/go.crypto/nacl/box"
)

// timestamp creates a Mixmaster formatted timestamp, consisting of an intro
// string concatented to the number of days since Epoch (little Endian).
func timestamp() (stamp []byte) {
	d := uint16(time.Now().UTC().Unix() / 86400)
	days := make([]byte, 2)
	binary.LittleEndian.PutUint16(days, d)
	stamp = append([]byte("0000\x00"), days...)
	return
}

// Generate a public/private ECC key pair
func eccGenerate() (public, private []byte) {
	puba, priva, err := box.GenerateKey(rand.Reader)
	if err != nil {
		panic(err)
	}
	public = make([]byte, 32)
	private = make([]byte, 32)
	copy(public[:], puba[:])
	copy(private[:], priva[:])
	if len(public) != 32 {
		panic("Invalid pubkey length generated")
	}
	if len(private) != 32 {
		panic("Invalid seckey length generated")
	}
	return
}

/*
Slot Header Format
[ Recipient key ID		 16 Bytes ]
[ Sender Public key		 32 Bytes ]
[ Xsalsa20 Nonce		 24 Bytes ]
[ Encrypted data		408 Bytes ] (392 + Overhead)
[ Random padding		 32 Bytes ]
*/

type slotHead struct {
	recipientKeyid []byte
	recipientPK []byte
	//sender_pubkey []byte
	//nonce []byte
	data []byte
}

func (h *slotHead) encodeHead() []byte {
	// Generate an ECC key pair
	sendpub, sendpriv, err := box.GenerateKey(rand.Reader)
	if err != nil {
		panic(err)
	}
	var nonce [24]byte
	copy(nonce[:], randbytes(24))
	buf := new(bytes.Buffer)
	buf.Write(h.recipientKeyid)
	buf.Write(sendpub[:])
	buf.Write(nonce[:])
	var pk [32]byte
	copy(pk[:], h.recipientPK)
	buf.Write(box.Seal(nil, h.data, &nonce, &pk, sendpriv))
	err = bufLenCheck(buf.Len(), 480)
	if err != nil {
		panic(err)
	}
	buf.Write(randbytes(headerBytes - buf.Len()))
	return buf.Bytes()
}

// decodeHead decodes a slot header
func decodeHead(b []byte, secret *keymgr.Secring) (data []byte, err error) {
	/*
	Decode functions should return their associated structs but, in this
	instance, the only field of value is the decrypted data.
	*/
	err = lenCheck(len(b), headerBytes)
	if err != nil {
		return
	}
	var keyid string
	keyid = hex.EncodeToString(b[0:16])
	var sk []byte
	sk, err = secret.GetSK(keyid)
	if err != nil {
		return
	}
	var sender_pk [32]byte
	copy(sender_pk[:], b[16:48])
	var recipient_sk [32]byte
	copy(recipient_sk[:], sk)
	var nonce [24]byte
	copy(nonce[:], b[48:72])
	var auth bool
	data, auth = box.Open(
		nil,
		b[72:72+408],
		&nonce,
		&sender_pk,
		&recipient_sk)
	if ! auth {
		err = errors.New("Authentication failed decrypting slot data")
		return
	}
	err = lenCheck(len(data), 392)
	return
}

/*
Encrypted Data
[ Packet ID		 16 Bytes ]
[ AES-CTR key		 32 Bytes ]
[ Packet type ID	  1 Byte  ]
[ Padded packet info	256 Bytes ]
[ Timestamp		  7 Bytes ]
[ Anti-tag digest	 64 Bytes ]
[ Padding		 16 Bytes ]
Total	392 Bytes
*/

type slotData struct {
	packetID []byte
	aesKey []byte
	packetType uint8
	packetInfo []byte
	timestamp []byte
	tagHash []byte
}

func (d slotData) encodeData() (b []byte, err error) {
	err = lenCheck(len(d.packetID), 16)
	if err != nil {
		return
	}
	err = lenCheck(len(d.aesKey), 32)
	if err != nil {
		return
	}
	err = lenCheck(len(d.packetInfo), 256)
	if err != nil {
		return
	}
	err = lenCheck(len(d.tagHash), 64)
	if err != nil {
		return
	}
	buf := new(bytes.Buffer)
	buf.Write(d.packetID)
	buf.Write(d.aesKey)
	buf.WriteByte(d.packetType)
	buf.Write(d.packetInfo)
	buf.Write(d.timestamp)
	buf.Write(d.tagHash)
	err = bufLenCheck(buf.Len(), 376)
	if err != nil {
		return
	}
	buf.WriteString(strings.Repeat("\x00", 392 - buf.Len()))
	b = buf.Bytes()
	return
}

func (data *slotData) decodeData(b []byte) (err error) {
	err = lenCheck(len(b), 392)
	if err != nil {
		return
	}
	data.packetID = b[0:16]
	data.aesKey = b[16:48]
	data.packetType = uint8(b[48])
	data.packetInfo = b[49:305]
	data.timestamp = b[305:312]
	//TODO Test timestamp
	data.tagHash = b[312:376]
	//Padding[376:392]
	return
}

/*
Final Hop
[ Chunk num		  1 Byte  ]
[ Num chunks		  1 Byte  ]
[ Message ID		 16 Bytes ]
[ AES-CTR IV		 16 Bytes ]
[ Body length		  4 Bytes ]
[ Delivery method	  1 Byte  ]

Currently supported delivery methods are:-
[ SMTP				  0 ]
[ Dummy - discard 		255 ]
*/

type slotFinal struct {
	chunkNum uint8
	numChunks uint8
	messageID []byte
	aesIV []byte
	bodyBytes int
	deliveryMethod uint8
}

func (f *slotFinal) encodeFinal() []byte {
	buf := new(bytes.Buffer)
	buf.WriteByte(f.chunkNum)
	buf.WriteByte(f.numChunks)
	buf.Write(f.messageID)
	buf.Write(f.aesIV)
	tmp := make([]byte, 4)
	binary.LittleEndian.PutUint32(tmp, uint32(f.bodyBytes))
	buf.Write(tmp)
	buf.WriteByte(f.deliveryMethod)
	err := bufLenCheck(buf.Len(), 39)
	if err != nil {
		panic(err)
	}
	buf.WriteString(strings.Repeat("\x00", 256 - buf.Len()))
	return buf.Bytes()
}

func (f *slotFinal) decodeFinal(b []byte) (err error) {
	err = lenCheck(len(b), 256)
	if err != nil {
		return
	}
	f.chunkNum = b[0]
	f.numChunks = b[1]
	f.messageID = b[2:18]
	f.aesIV = b[18:34]
	f.bodyBytes = int(binary.LittleEndian.Uint32(b[34:38]))
	f.deliveryMethod = b[38]
	return
}

/*
Intermediate Hop
[ 11 * AES-CTR IVs		176 Bytes ]
[ Next hop address		 80 Bytes ]

IVs are:
[ 9 * Header slots		144 Bytes ]
[ 1 * Deterministic header	 16 Bytes ]
[ 1 * Payload header		 16 Bytes ]
*/

type slotIntermediate struct {
	aesIVs []byte
	nextHop []byte
}

func (i *slotIntermediate) setNextHop(nh string) {
	i.nextHop = []byte(nh + strings.Repeat("\x00", 80 - len(nh)))
}

func (i *slotIntermediate) getNextHop() string {
	return strings.TrimRight(string(i.nextHop), "\x00")
}

func (i *slotIntermediate) encodeIntermediate() []byte {
	var err error
	err = lenCheck(len(i.aesIVs), (maxChainLength + 1) * 16)
	if err != nil {
		panic(err)
	}
	buf := new(bytes.Buffer)
	buf.Write(i.aesIVs)
	buf.Write(i.nextHop)
	err = bufLenCheck(buf.Len(), 256)
	if err != nil {
		panic(err)
	}
	return buf.Bytes()
}

func (i *slotIntermediate) decodeIntermediate(b []byte) (err error) {
	err = lenCheck(len(b), 256)
	if err != nil {
		return
	}
	i.aesIVs = b[:176]
	i.nextHop = b[176:]
	return
}
<|endoftext|>


<commit_before>package scipipe

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	str "strings"
)

// ================== SciTask ==================

type SciTask struct {
	Name          string
	Command       string
	CustomExecute func(*SciTask)
	InTargets     map[string]*FileTarget
	OutTargets    map[string]*FileTarget
	Params        map[string]string
	Done          chan int
}

func NewSciTask(name string, cmdPat string, inTargets map[string]*FileTarget, outPathFuncs map[string]func(*SciTask) string, outPortsDoStream map[string]bool, params map[string]string, prepend string) *SciTask {
	t := &SciTask{
		Name:       name,
		InTargets:  inTargets,
		OutTargets: make(map[string]*FileTarget),
		Params:     params,
		Command:    "",
		Done:       make(chan int),
	}
	// Create out targets
	Debug.Printf("Task:%s: Creating outTargets now ... [%s]", name, cmdPat)
	outTargets := make(map[string]*FileTarget)
	for oname, ofun := range outPathFuncs {
		opath := ofun(t)
		otgt := NewFileTarget(opath)
		if outPortsDoStream[oname] {
			otgt.doStream = true
		}
		Debug.Printf("Task:%s: Creating outTarget with path %s ... [%s]", name, cmdPat)
		outTargets[oname] = otgt
	}
	t.OutTargets = outTargets
	t.Command = formatCommand(cmdPat, inTargets, outTargets, params, prepend)
	Debug.Printf("Task:%s: Created formatted command: %s [%s]", name, t.Command, cmdPat)
	return t
}

// --------------- SciTask API methods ----------------

func (t *SciTask) GetInPath(inPort string) string {
	return t.InTargets[inPort].GetPath()
}

func (t *SciTask) Execute() {
	defer close(t.Done)
	if !t.anyOutputExists() && !t.fifosInOutTargetsMissing() {
		if t.CustomExecute != nil {
			Info.Printf("Task:%-16s Executing task.\n", t.Command)
			t.CustomExecute(t)
		} else {
			t.executeCommand(t.Command)
		}
		t.atomizeTargets()
	}
	Debug.Printf("Task:%s: Starting to send Done in t.Execute() ...) [%s]\n", t.Name, t.Command)
	t.Done <- 1
	Debug.Printf("Task:%s: Done sending Done, in t.Execute() [%s]\n", t.Name, t.Command)
}

// --------------- SciTask Helper methods ----------------

// Check if any output file target, or temporary file targets, exist
func (t *SciTask) anyOutputExists() (anyFileExists bool) {
	anyFileExists = false
	for _, tgt := range t.OutTargets {
		opath := tgt.GetPath()
		otmpPath := tgt.GetTempPath()
		if !tgt.doStream {
			if _, err := os.Stat(opath); err == nil {
				Warning.Printf("Task:%-16s Output file already exists: %s, so skipping... [%s]\n", t.Name, opath, t.Command)
				anyFileExists = true
			}
			if _, err := os.Stat(otmpPath); err == nil {
				Warning.Printf("Task:%-16s Temporary Output file already exists: %s, so skipping... [%s]\n", t.Name, otmpPath, t.Command)
				anyFileExists = true
			}
		}
	}
	return
}

// Check if any FIFO files for this tasks exist, for out-ports specified to support streaming
func (t *SciTask) anyFifosExist() (anyFifosExist bool) {
	anyFifosExist = false
	for _, tgt := range t.OutTargets {
		ofifoPath := tgt.GetFifoPath()
		if tgt.doStream {
			if _, err := os.Stat(ofifoPath); err == nil {
				Warning.Printf("Task:%-16s Output FIFO already exists: %s. Check your workflow for correctness! [%s]\n", t.Name, ofifoPath, t.Command)
				anyFifosExist = true
			}
		}
	}
	return
}

// Make sure that FIFOs that are supposed to exist, really exists
func (t *SciTask) fifosInOutTargetsMissing() (fifosInOutTargetsMissing bool) {
	fifosInOutTargetsMissing = false
	for _, tgt := range t.OutTargets {
		if tgt.doStream {
			ofifoPath := tgt.GetFifoPath()
			if _, err := os.Stat(ofifoPath); err != nil {
				Warning.Printf("Task:%-16s FIFO Output file missing, for streaming output: %s. Check your workflow for correctness! [%s]\n", t.Name, t.Command, ofifoPath)
				fifosInOutTargetsMissing = true
			}
		}
	}
	return
}

func (t *SciTask) executeCommand(cmd string) {
	Info.Printf("Task:%-16s Executing command: %s\n", t.Name, cmd)
	_, err := exec.Command("bash", "-c", cmd).Output()
	Check(err)
}

// Create FIFO files for all out-ports that are specified to support streaming
func (t *SciTask) createFifos() {
	Debug.Printf("Task:%s: Now creating fifos for task [%s]\n", t.Name, t.Command)
	for _, otgt := range t.OutTargets {
		if otgt.doStream {
			otgt.CreateFifo()
		}
	}
}

// Rename temporary output files to their proper file names
func (t *SciTask) atomizeTargets() {
	for _, tgt := range t.OutTargets {
		if !tgt.doStream {
			Debug.Printf("Atomizing file: %s -> %s", tgt.GetTempPath(), tgt.GetPath())
			tgt.Atomize()
			Debug.Printf("Done atomizing file: %s -> %s", tgt.GetTempPath(), tgt.GetPath())
		} else {
			Debug.Printf("Target is streaming, so not atomizing: %s", tgt.GetPath())
		}
	}
}

// Clean up any remaining FIFOs
// TODO: this is actually not really used anymore ...
func (t *SciTask) cleanUpFifos() {
	for _, tgt := range t.OutTargets {
		if tgt.doStream {
			Debug.Printf("Task:%s: Cleaning up FIFO for output target: %s [%s]\n", t.Name, tgt.GetFifoPath(), t.Command)
			tgt.RemoveFifo()
		} else {
			Debug.Printf("Task:%s: output target is not FIFO, so not removing any FIFO: %s [%s]\n", t.Name, tgt.GetPath(), t.Command)
		}
	}
}

// ================== Helper functions==================

func formatCommand(cmd string, inTargets map[string]*FileTarget, outTargets map[string]*FileTarget, params map[string]string, prepend string) string {

	// Debug.Println("Formatting command with the following data:")
	// Debug.Println("prepend:", prepend)
	// Debug.Println("cmd:", cmd)
	// Debug.Println("inTargets:", inTargets)
	// Debug.Println("outTargets:", outTargets)
	// Debug.Println("params:", params)

	r := getShellCommandPlaceHolderRegex()
	ms := r.FindAllStringSubmatch(cmd, -1)
	for _, m := range ms {
		placeHolderStr := m[0]
		typ := m[1]
		name := m[2]
		var filePath string
		if typ == "o" || typ == "os" {
			// Out-ports
			if outTargets[name] == nil {
				msg := fmt.Sprint("Missing outpath for outport '", name, "' for command '", cmd, "'")
				Check(errors.New(msg))
			} else {
				if typ == "o" {
					filePath = outTargets[name].GetTempPath() // Means important to Atomize afterwards!
				} else if typ == "os" {
					filePath = outTargets[name].GetFifoPath()
				}
			}
		} else if typ == "i" {
			// In-ports
			if inTargets[name] == nil {
				msg := fmt.Sprint("Missing intarget for inport '", name, "' for command '", cmd, "'")
				Check(errors.New(msg))
			} else if inTargets[name].GetPath() == "" {
				msg := fmt.Sprint("Missing inpath for inport '", name, "' for command '", cmd, "'")
				Check(errors.New(msg))
			} else {
				if inTargets[name].doStream {
					filePath = inTargets[name].GetFifoPath()
				} else {
					filePath = inTargets[name].GetPath()
				}
			}
		} else if typ == "p" {
			if params[name] == "" {
				msg := fmt.Sprint("Missing param value param '", name, "' for command '", cmd, "'")
				Check(errors.New(msg))
			} else {
				filePath = params[name]
			}
		}
		if filePath == "" {
			msg := fmt.Sprint("Replace failed for port ", name, " for command '", cmd, "'")
			Check(errors.New(msg))
		}
		cmd = str.Replace(cmd, placeHolderStr, filePath, -1)
	}
	// Add prepend string to the command
	if prepend != "" {
		cmd = fmt.Sprintf("%s %s", prepend, cmd)
	}
	return cmd
}
<commit_msg>Log level fix: Executed commands, and existing outputs as AUDIT<commit_after>package scipipe

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	str "strings"
)

// ================== SciTask ==================

type SciTask struct {
	Name          string
	Command       string
	CustomExecute func(*SciTask)
	InTargets     map[string]*FileTarget
	OutTargets    map[string]*FileTarget
	Params        map[string]string
	Done          chan int
}

func NewSciTask(name string, cmdPat string, inTargets map[string]*FileTarget, outPathFuncs map[string]func(*SciTask) string, outPortsDoStream map[string]bool, params map[string]string, prepend string) *SciTask {
	t := &SciTask{
		Name:       name,
		InTargets:  inTargets,
		OutTargets: make(map[string]*FileTarget),
		Params:     params,
		Command:    "",
		Done:       make(chan int),
	}
	// Create out targets
	Debug.Printf("Task:%s: Creating outTargets now ... [%s]", name, cmdPat)
	outTargets := make(map[string]*FileTarget)
	for oname, ofun := range outPathFuncs {
		opath := ofun(t)
		otgt := NewFileTarget(opath)
		if outPortsDoStream[oname] {
			otgt.doStream = true
		}
		Debug.Printf("Task:%s: Creating outTarget with path %s ... [%s]", name, cmdPat)
		outTargets[oname] = otgt
	}
	t.OutTargets = outTargets
	t.Command = formatCommand(cmdPat, inTargets, outTargets, params, prepend)
	Debug.Printf("Task:%s: Created formatted command: %s [%s]", name, t.Command, cmdPat)
	return t
}

// --------------- SciTask API methods ----------------

func (t *SciTask) GetInPath(inPort string) string {
	return t.InTargets[inPort].GetPath()
}

func (t *SciTask) Execute() {
	defer close(t.Done)
	if !t.anyOutputExists() && !t.fifosInOutTargetsMissing() {
		if t.CustomExecute != nil {
			Info.Printf("Task:%-16s Executing task.\n", t.Command)
			t.CustomExecute(t)
		} else {
			t.executeCommand(t.Command)
		}
		t.atomizeTargets()
	}
	Debug.Printf("Task:%s: Starting to send Done in t.Execute() ...) [%s]\n", t.Name, t.Command)
	t.Done <- 1
	Debug.Printf("Task:%s: Done sending Done, in t.Execute() [%s]\n", t.Name, t.Command)
}

// --------------- SciTask Helper methods ----------------

// Check if any output file target, or temporary file targets, exist
func (t *SciTask) anyOutputExists() (anyFileExists bool) {
	anyFileExists = false
	for _, tgt := range t.OutTargets {
		opath := tgt.GetPath()
		otmpPath := tgt.GetTempPath()
		if !tgt.doStream {
			if _, err := os.Stat(opath); err == nil {
				Audit.Printf("Task:%-12s Output file already exists: %s, so skipping... [%s]\n", t.Name, opath, t.Command)
				anyFileExists = true
			}
			if _, err := os.Stat(otmpPath); err == nil {
				Warning.Printf("Task:%-16s Temporary Output file already exists: %s, so skipping... [%s]\n", t.Name, otmpPath, t.Command)
				anyFileExists = true
			}
		}
	}
	return
}

// Check if any FIFO files for this tasks exist, for out-ports specified to support streaming
func (t *SciTask) anyFifosExist() (anyFifosExist bool) {
	anyFifosExist = false
	for _, tgt := range t.OutTargets {
		ofifoPath := tgt.GetFifoPath()
		if tgt.doStream {
			if _, err := os.Stat(ofifoPath); err == nil {
				Warning.Printf("Task:%-16s Output FIFO already exists: %s. Check your workflow for correctness! [%s]\n", t.Name, ofifoPath, t.Command)
				anyFifosExist = true
			}
		}
	}
	return
}

// Make sure that FIFOs that are supposed to exist, really exists
func (t *SciTask) fifosInOutTargetsMissing() (fifosInOutTargetsMissing bool) {
	fifosInOutTargetsMissing = false
	for _, tgt := range t.OutTargets {
		if tgt.doStream {
			ofifoPath := tgt.GetFifoPath()
			if _, err := os.Stat(ofifoPath); err != nil {
				Warning.Printf("Task:%-16s FIFO Output file missing, for streaming output: %s. Check your workflow for correctness! [%s]\n", t.Name, t.Command, ofifoPath)
				fifosInOutTargetsMissing = true
			}
		}
	}
	return
}

func (t *SciTask) executeCommand(cmd string) {
	Audit.Printf("Task:%-12s Executing command: %s\n", t.Name, cmd)
	_, err := exec.Command("bash", "-c", cmd).Output()
	Check(err)
}

// Create FIFO files for all out-ports that are specified to support streaming
func (t *SciTask) createFifos() {
	Debug.Printf("Task:%s: Now creating fifos for task [%s]\n", t.Name, t.Command)
	for _, otgt := range t.OutTargets {
		if otgt.doStream {
			otgt.CreateFifo()
		}
	}
}

// Rename temporary output files to their proper file names
func (t *SciTask) atomizeTargets() {
	for _, tgt := range t.OutTargets {
		if !tgt.doStream {
			Debug.Printf("Atomizing file: %s -> %s", tgt.GetTempPath(), tgt.GetPath())
			tgt.Atomize()
			Debug.Printf("Done atomizing file: %s -> %s", tgt.GetTempPath(), tgt.GetPath())
		} else {
			Debug.Printf("Target is streaming, so not atomizing: %s", tgt.GetPath())
		}
	}
}

// Clean up any remaining FIFOs
// TODO: this is actually not really used anymore ...
func (t *SciTask) cleanUpFifos() {
	for _, tgt := range t.OutTargets {
		if tgt.doStream {
			Debug.Printf("Task:%s: Cleaning up FIFO for output target: %s [%s]\n", t.Name, tgt.GetFifoPath(), t.Command)
			tgt.RemoveFifo()
		} else {
			Debug.Printf("Task:%s: output target is not FIFO, so not removing any FIFO: %s [%s]\n", t.Name, tgt.GetPath(), t.Command)
		}
	}
}

// ================== Helper functions==================

func formatCommand(cmd string, inTargets map[string]*FileTarget, outTargets map[string]*FileTarget, params map[string]string, prepend string) string {

	// Debug.Println("Formatting command with the following data:")
	// Debug.Println("prepend:", prepend)
	// Debug.Println("cmd:", cmd)
	// Debug.Println("inTargets:", inTargets)
	// Debug.Println("outTargets:", outTargets)
	// Debug.Println("params:", params)

	r := getShellCommandPlaceHolderRegex()
	ms := r.FindAllStringSubmatch(cmd, -1)
	for _, m := range ms {
		placeHolderStr := m[0]
		typ := m[1]
		name := m[2]
		var filePath string
		if typ == "o" || typ == "os" {
			// Out-ports
			if outTargets[name] == nil {
				msg := fmt.Sprint("Missing outpath for outport '", name, "' for command '", cmd, "'")
				Check(errors.New(msg))
			} else {
				if typ == "o" {
					filePath = outTargets[name].GetTempPath() // Means important to Atomize afterwards!
				} else if typ == "os" {
					filePath = outTargets[name].GetFifoPath()
				}
			}
		} else if typ == "i" {
			// In-ports
			if inTargets[name] == nil {
				msg := fmt.Sprint("Missing intarget for inport '", name, "' for command '", cmd, "'")
				Check(errors.New(msg))
			} else if inTargets[name].GetPath() == "" {
				msg := fmt.Sprint("Missing inpath for inport '", name, "' for command '", cmd, "'")
				Check(errors.New(msg))
			} else {
				if inTargets[name].doStream {
					filePath = inTargets[name].GetFifoPath()
				} else {
					filePath = inTargets[name].GetPath()
				}
			}
		} else if typ == "p" {
			if params[name] == "" {
				msg := fmt.Sprint("Missing param value param '", name, "' for command '", cmd, "'")
				Check(errors.New(msg))
			} else {
				filePath = params[name]
			}
		}
		if filePath == "" {
			msg := fmt.Sprint("Replace failed for port ", name, " for command '", cmd, "'")
			Check(errors.New(msg))
		}
		cmd = str.Replace(cmd, placeHolderStr, filePath, -1)
	}
	// Add prepend string to the command
	if prepend != "" {
		cmd = fmt.Sprintf("%s %s", prepend, cmd)
	}
	return cmd
}
<|endoftext|>


<commit_before>package task

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"

	"gopkg.in/yaml.v2"
)

var (
	TaskFilePath = "Taskfile.yml"
	ShExists     bool
	ShPath       string

	Tasks = make(map[string]*Task)
)

func init() {
	var err error
	ShPath, err = exec.LookPath("sh")
	if err != nil {
		return
	}
	ShExists = true
}

type Task struct {
	Cmds      []string
	Deps      []string
	Sources   []string
	Generates []string
}

type TaskNotFoundError struct {
	taskName string
}

func (err *TaskNotFoundError) Error() string {
	return fmt.Sprintf(`Task "%s" not found`, err.taskName)
}

type TaskRunError struct {
	taskName string
	err      error
}

func (err *TaskRunError) Error() string {
	return fmt.Sprintf(`Failed to run task "%s": %v`, err.taskName, err.err)
}

func Run() {
	log.SetFlags(0)

	args := os.Args[1:]
	if len(args) == 0 {
		log.Fatal("No argument given")
	}

	file, err := ioutil.ReadFile(TaskFilePath)
	if err != nil {
		if os.IsNotExist(err) {
			log.Fatal("Taskfile.yml not found")
		}
		log.Fatal(err)
	}

	if err = yaml.Unmarshal(file, &Tasks); err != nil {
		log.Fatal(err)
	}

	for _, a := range args {
		if err = RunTask(a); err != nil {
			log.Fatal(err)
		}
	}
}

func RunTask(name string) error {
	t, ok := Tasks[name]
	if !ok {
		return &TaskNotFoundError{name}
	}

	if isTaskUpToDate(t) {
		log.Printf(`Task "%s" is up to date`, name)
		return nil
	}

	for _, d := range t.Deps {
		if err := RunTask(d); err != nil {
			return err
		}
	}

	for _, c := range t.Cmds {
		if err := runCommand(c); err != nil {
			return &TaskRunError{name, err}
		}
	}
	return nil
}

func isTaskUpToDate(t *Task) bool {
	if len(t.Sources) == 0 || len(t.Generates) == 0 {
		return false
	}

	sourcesMaxTime, err := getPatternsMaxTime(t.Sources)
	if err != nil || sourcesMaxTime.IsZero() {
		return false
	}

	generatesMinTime, err := getPatternsMinTime(t.Generates)
	if err != nil || generatesMinTime.IsZero() {
		return false
	}

	return generatesMinTime.After(sourcesMaxTime)
}

func runCommand(c string) error {
	var cmd *exec.Cmd
	if ShExists {
		cmd = exec.Command(ShPath, "-c", c)
	} else {
		cmd = exec.Command("cmd", "/C", c)
	}
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return err
	}
	return nil
}
<commit_msg>Make golint happy<commit_after>package task

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"

	"gopkg.in/yaml.v2"
)

var (
	// TaskFilePath is the default Taskfile
	TaskFilePath = "Taskfile.yml"
	// ShExists is true if Bash was found
	ShExists bool
	// ShPath constains the Bash path if found
	ShPath string

	// Tasks constains the tasks parsed from Taskfile
	Tasks = make(map[string]*Task)
)

func init() {
	var err error
	ShPath, err = exec.LookPath("sh")
	if err != nil {
		return
	}
	ShExists = true
}

// Task represents a task
type Task struct {
	Cmds      []string
	Deps      []string
	Sources   []string
	Generates []string
}

type taskNotFoundError struct {
	taskName string
}

func (err *taskNotFoundError) Error() string {
	return fmt.Sprintf(`Task "%s" not found`, err.taskName)
}

type taskRunError struct {
	taskName string
	err      error
}

func (err *taskRunError) Error() string {
	return fmt.Sprintf(`Failed to run task "%s": %v`, err.taskName, err.err)
}

// Run runs Task
func Run() {
	log.SetFlags(0)

	args := os.Args[1:]
	if len(args) == 0 {
		log.Fatal("No argument given")
	}

	file, err := ioutil.ReadFile(TaskFilePath)
	if err != nil {
		if os.IsNotExist(err) {
			log.Fatal("Taskfile.yml not found")
		}
		log.Fatal(err)
	}

	if err = yaml.Unmarshal(file, &Tasks); err != nil {
		log.Fatal(err)
	}

	for _, a := range args {
		if err = RunTask(a); err != nil {
			log.Fatal(err)
		}
	}
}

// RunTask runs a task by its name
func RunTask(name string) error {
	t, ok := Tasks[name]
	if !ok {
		return &taskNotFoundError{name}
	}

	if isTaskUpToDate(t) {
		log.Printf(`Task "%s" is up to date`, name)
		return nil
	}

	for _, d := range t.Deps {
		if err := RunTask(d); err != nil {
			return err
		}
	}

	for _, c := range t.Cmds {
		if err := runCommand(c); err != nil {
			return &taskRunError{name, err}
		}
	}
	return nil
}

func isTaskUpToDate(t *Task) bool {
	if len(t.Sources) == 0 || len(t.Generates) == 0 {
		return false
	}

	sourcesMaxTime, err := getPatternsMaxTime(t.Sources)
	if err != nil || sourcesMaxTime.IsZero() {
		return false
	}

	generatesMinTime, err := getPatternsMinTime(t.Generates)
	if err != nil || generatesMinTime.IsZero() {
		return false
	}

	return generatesMinTime.After(sourcesMaxTime)
}

func runCommand(c string) error {
	var cmd *exec.Cmd
	if ShExists {
		cmd = exec.Command(ShPath, "-c", c)
	} else {
		cmd = exec.Command("cmd", "/C", c)
	}
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package scipipe

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	str "strings"
)

// ================== SciTask ==================

type SciTask struct {
	Name          string
	Command       string
	CustomExecute func(*SciTask)
	InTargets     map[string]*FileTarget
	OutTargets    map[string]*FileTarget
	Params        map[string]string
	Done          chan int
}

func NewSciTask(name string, cmdPat string, inTargets map[string]*FileTarget, outPathFuncs map[string]func(*SciTask) string, outPortsDoStream map[string]bool, params map[string]string, prepend string) *SciTask {
	t := &SciTask{
		Name:       name,
		InTargets:  inTargets,
		OutTargets: make(map[string]*FileTarget),
		Params:     params,
		Command:    "",
		Done:       make(chan int),
	}
	// Create out targets
	Debug.Printf("Task:%s: Creating outTargets now ... [%s]", name, cmdPat)
	outTargets := make(map[string]*FileTarget)
	for oname, ofun := range outPathFuncs {
		opath := ofun(t)
		otgt := NewFileTarget(opath)
		if outPortsDoStream[oname] {
			otgt.doStream = true
		}
		Debug.Printf("Task:%s: Creating outTarget with path %s ... [%s]", name, cmdPat)
		outTargets[oname] = otgt
	}
	t.OutTargets = outTargets
	t.Command = formatCommand(cmdPat, inTargets, outTargets, params, prepend)
	Debug.Printf("Task:%s: Created formatted command: %s [%s]", name, t.Command, cmdPat)
	return t
}

// --------------- SciTask API methods ----------------

func (t *SciTask) GetInPath(inPort string) string {
	return t.InTargets[inPort].GetPath()
}

func (t *SciTask) Execute() {
	defer close(t.Done)
	if !t.anyOutputExists() && !t.fifosInOutTargetsMissing() {
		Debug.Printf("Task:%-12s Executing task. [%s]\n", t.Name, t.Command)
		if t.CustomExecute != nil {
			t.CustomExecute(t)
		} else {
			t.executeCommand(t.Command)
		}
		Debug.Printf("Task:%-12s Atomizing targets. [%s]\n", t.Name, t.Command)
		t.atomizeTargets()
	}
	Debug.Printf("Task:%s: Starting to send Done in t.Execute() ...) [%s]\n", t.Name, t.Command)
	t.Done <- 1
	Debug.Printf("Task:%s: Done sending Done, in t.Execute() [%s]\n", t.Name, t.Command)
}

// --------------- SciTask Helper methods ----------------

// Check if any output file target, or temporary file targets, exist
func (t *SciTask) anyOutputExists() (anyFileExists bool) {
	anyFileExists = false
	for _, tgt := range t.OutTargets {
		opath := tgt.GetPath()
		otmpPath := tgt.GetTempPath()
		if !tgt.doStream {
			if _, err := os.Stat(opath); err == nil {
				Audit.Printf("Task:%-12s Output file already exists: %s, so skipping... [%s]\n", t.Name, opath, t.Command)
				anyFileExists = true
			}
			if _, err := os.Stat(otmpPath); err == nil {
				Warning.Printf("Task:%-12s Temporary Output file already exists: %s. Check your workflow for correctness! [%s]\n", t.Name, otmpPath, t.Command)
				anyFileExists = true
			}
		}
	}
	return
}

// Check if any FIFO files for this tasks exist, for out-ports specified to support streaming
func (t *SciTask) anyFifosExist() (anyFifosExist bool) {
	anyFifosExist = false
	for _, tgt := range t.OutTargets {
		ofifoPath := tgt.GetFifoPath()
		if tgt.doStream {
			if _, err := os.Stat(ofifoPath); err == nil {
				Warning.Printf("Task:%-12s Output FIFO already exists: %s. Check your workflow for correctness! [%s]\n", t.Name, ofifoPath, t.Command)
				anyFifosExist = true
			}
		}
	}
	return
}

// Make sure that FIFOs that are supposed to exist, really exists
func (t *SciTask) fifosInOutTargetsMissing() (fifosInOutTargetsMissing bool) {
	fifosInOutTargetsMissing = false
	for _, tgt := range t.OutTargets {
		if tgt.doStream {
			ofifoPath := tgt.GetFifoPath()
			if _, err := os.Stat(ofifoPath); err != nil {
				Warning.Printf("Task:%-12s FIFO Output file missing, for streaming output: %s. Check your workflow for correctness! [%s]\n", t.Name, t.Command, ofifoPath)
				fifosInOutTargetsMissing = true
			}
		}
	}
	return
}

func (t *SciTask) executeCommand(cmd string) {
	Audit.Printf("Task:%-12s Executing command: %s\n", t.Name, cmd)
	out, err := exec.Command("bash", "-c", cmd).CombinedOutput()
	if err != nil {
		Error.Println("Command failed, with output:\n", string(out))
		os.Exit(126)
	}
}

// Create FIFO files for all out-ports that are specified to support streaming
func (t *SciTask) createFifos() {
	Debug.Printf("Task:%s: Now creating fifos for task [%s]\n", t.Name, t.Command)
	for _, otgt := range t.OutTargets {
		if otgt.doStream {
			otgt.CreateFifo()
		}
	}
}

// Rename temporary output files to their proper file names
func (t *SciTask) atomizeTargets() {
	for _, tgt := range t.OutTargets {
		if !tgt.doStream {
			Debug.Printf("Atomizing file: %s -> %s", tgt.GetTempPath(), tgt.GetPath())
			tgt.Atomize()
			Debug.Printf("Done atomizing file: %s -> %s", tgt.GetTempPath(), tgt.GetPath())
		} else {
			Debug.Printf("Target is streaming, so not atomizing: %s", tgt.GetPath())
		}
	}
}

// Clean up any remaining FIFOs
// TODO: this is actually not really used anymore ...
func (t *SciTask) cleanUpFifos() {
	for _, tgt := range t.OutTargets {
		if tgt.doStream {
			Debug.Printf("Task:%s: Cleaning up FIFO for output target: %s [%s]\n", t.Name, tgt.GetFifoPath(), t.Command)
			tgt.RemoveFifo()
		} else {
			Debug.Printf("Task:%s: output target is not FIFO, so not removing any FIFO: %s [%s]\n", t.Name, tgt.GetPath(), t.Command)
		}
	}
}

// ================== Helper functions==================

func formatCommand(cmd string, inTargets map[string]*FileTarget, outTargets map[string]*FileTarget, params map[string]string, prepend string) string {

	// Debug.Println("Formatting command with the following data:")
	// Debug.Println("prepend:", prepend)
	// Debug.Println("cmd:", cmd)
	// Debug.Println("inTargets:", inTargets)
	// Debug.Println("outTargets:", outTargets)
	// Debug.Println("params:", params)

	r := getShellCommandPlaceHolderRegex()
	ms := r.FindAllStringSubmatch(cmd, -1)
	for _, m := range ms {
		placeHolderStr := m[0]
		typ := m[1]
		name := m[2]
		var filePath string
		if typ == "o" || typ == "os" {
			// Out-ports
			if outTargets[name] == nil {
				msg := fmt.Sprint("Missing outpath for outport '", name, "' for command '", cmd, "'")
				Check(errors.New(msg))
			} else {
				if typ == "o" {
					filePath = outTargets[name].GetTempPath() // Means important to Atomize afterwards!
				} else if typ == "os" {
					filePath = outTargets[name].GetFifoPath()
				}
			}
		} else if typ == "i" {
			// In-ports
			if inTargets[name] == nil {
				msg := fmt.Sprint("Missing intarget for inport '", name, "' for command '", cmd, "'")
				Check(errors.New(msg))
			} else if inTargets[name].GetPath() == "" {
				msg := fmt.Sprint("Missing inpath for inport '", name, "' for command '", cmd, "'")
				Check(errors.New(msg))
			} else {
				if inTargets[name].doStream {
					filePath = inTargets[name].GetFifoPath()
				} else {
					filePath = inTargets[name].GetPath()
				}
			}
		} else if typ == "p" {
			if params[name] == "" {
				msg := fmt.Sprint("Missing param value param '", name, "' for command '", cmd, "'")
				Check(errors.New(msg))
			} else {
				filePath = params[name]
			}
		}
		if filePath == "" {
			msg := fmt.Sprint("Replace failed for port ", name, " for command '", cmd, "'")
			Check(errors.New(msg))
		}
		cmd = str.Replace(cmd, placeHolderStr, filePath, -1)
	}
	// Add prepend string to the command
	if prepend != "" {
		cmd = fmt.Sprintf("%s %s", prepend, cmd)
	}
	return cmd
}
<commit_msg>Make log messages about existing files more readable<commit_after>package scipipe

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	str "strings"
)

// ================== SciTask ==================

type SciTask struct {
	Name          string
	Command       string
	CustomExecute func(*SciTask)
	InTargets     map[string]*FileTarget
	OutTargets    map[string]*FileTarget
	Params        map[string]string
	Done          chan int
}

func NewSciTask(name string, cmdPat string, inTargets map[string]*FileTarget, outPathFuncs map[string]func(*SciTask) string, outPortsDoStream map[string]bool, params map[string]string, prepend string) *SciTask {
	t := &SciTask{
		Name:       name,
		InTargets:  inTargets,
		OutTargets: make(map[string]*FileTarget),
		Params:     params,
		Command:    "",
		Done:       make(chan int),
	}
	// Create out targets
	Debug.Printf("Task:%s: Creating outTargets now ... [%s]", name, cmdPat)
	outTargets := make(map[string]*FileTarget)
	for oname, ofun := range outPathFuncs {
		opath := ofun(t)
		otgt := NewFileTarget(opath)
		if outPortsDoStream[oname] {
			otgt.doStream = true
		}
		Debug.Printf("Task:%s: Creating outTarget with path %s ... [%s]", name, cmdPat)
		outTargets[oname] = otgt
	}
	t.OutTargets = outTargets
	t.Command = formatCommand(cmdPat, inTargets, outTargets, params, prepend)
	Debug.Printf("Task:%s: Created formatted command: %s [%s]", name, t.Command, cmdPat)
	return t
}

// --------------- SciTask API methods ----------------

func (t *SciTask) GetInPath(inPort string) string {
	return t.InTargets[inPort].GetPath()
}

func (t *SciTask) Execute() {
	defer close(t.Done)
	if !t.anyOutputExists() && !t.fifosInOutTargetsMissing() {
		Debug.Printf("Task:%-12s Executing task. [%s]\n", t.Name, t.Command)
		if t.CustomExecute != nil {
			t.CustomExecute(t)
		} else {
			t.executeCommand(t.Command)
		}
		Debug.Printf("Task:%-12s Atomizing targets. [%s]\n", t.Name, t.Command)
		t.atomizeTargets()
	}
	Debug.Printf("Task:%s: Starting to send Done in t.Execute() ...) [%s]\n", t.Name, t.Command)
	t.Done <- 1
	Debug.Printf("Task:%s: Done sending Done, in t.Execute() [%s]\n", t.Name, t.Command)
}

// --------------- SciTask Helper methods ----------------

// Check if any output file target, or temporary file targets, exist
func (t *SciTask) anyOutputExists() (anyFileExists bool) {
	anyFileExists = false
	for _, tgt := range t.OutTargets {
		opath := tgt.GetPath()
		otmpPath := tgt.GetTempPath()
		if !tgt.doStream {
			if _, err := os.Stat(opath); err == nil {
				Info.Printf("Task:%-12s Output file already exists, so skipping: %s\n", t.Name, opath)
				anyFileExists = true
			}
			if _, err := os.Stat(otmpPath); err == nil {
				Warning.Printf("Task:%-12s Temp   file already exists, so skipping: %s (Note: If resuming form a failed run, clean up .tmp files first).\n", t.Name, otmpPath)
				anyFileExists = true
			}
		}
	}
	return
}

// Check if any FIFO files for this tasks exist, for out-ports specified to support streaming
func (t *SciTask) anyFifosExist() (anyFifosExist bool) {
	anyFifosExist = false
	for _, tgt := range t.OutTargets {
		ofifoPath := tgt.GetFifoPath()
		if tgt.doStream {
			if _, err := os.Stat(ofifoPath); err == nil {
				Warning.Printf("Task:%-12s Output FIFO already exists, so skipping: %s (Note: If resuming form a failed run, clean up .fifo files first).\n", t.Name, ofifoPath)
				anyFifosExist = true
			}
		}
	}
	return
}

// Make sure that FIFOs that are supposed to exist, really exists
func (t *SciTask) fifosInOutTargetsMissing() (fifosInOutTargetsMissing bool) {
	fifosInOutTargetsMissing = false
	for _, tgt := range t.OutTargets {
		if tgt.doStream {
			ofifoPath := tgt.GetFifoPath()
			if _, err := os.Stat(ofifoPath); err != nil {
				Warning.Printf("Task:%-12s FIFO Output file missing, for streaming output: %s. Check your workflow for correctness! [%s]\n", t.Name, t.Command, ofifoPath)
				fifosInOutTargetsMissing = true
			}
		}
	}
	return
}

func (t *SciTask) executeCommand(cmd string) {
	Audit.Printf("Task:%-12s Executing command: %s\n", t.Name, cmd)
	out, err := exec.Command("bash", "-c", cmd).CombinedOutput()
	if err != nil {
		Error.Println("Command failed, with output:\n", string(out))
		os.Exit(126)
	}
}

// Create FIFO files for all out-ports that are specified to support streaming
func (t *SciTask) createFifos() {
	Debug.Printf("Task:%s: Now creating fifos for task [%s]\n", t.Name, t.Command)
	for _, otgt := range t.OutTargets {
		if otgt.doStream {
			otgt.CreateFifo()
		}
	}
}

// Rename temporary output files to their proper file names
func (t *SciTask) atomizeTargets() {
	for _, tgt := range t.OutTargets {
		if !tgt.doStream {
			Debug.Printf("Atomizing file: %s -> %s", tgt.GetTempPath(), tgt.GetPath())
			tgt.Atomize()
			Debug.Printf("Done atomizing file: %s -> %s", tgt.GetTempPath(), tgt.GetPath())
		} else {
			Debug.Printf("Target is streaming, so not atomizing: %s", tgt.GetPath())
		}
	}
}

// Clean up any remaining FIFOs
// TODO: this is actually not really used anymore ...
func (t *SciTask) cleanUpFifos() {
	for _, tgt := range t.OutTargets {
		if tgt.doStream {
			Debug.Printf("Task:%s: Cleaning up FIFO for output target: %s [%s]\n", t.Name, tgt.GetFifoPath(), t.Command)
			tgt.RemoveFifo()
		} else {
			Debug.Printf("Task:%s: output target is not FIFO, so not removing any FIFO: %s [%s]\n", t.Name, tgt.GetPath(), t.Command)
		}
	}
}

// ================== Helper functions==================

func formatCommand(cmd string, inTargets map[string]*FileTarget, outTargets map[string]*FileTarget, params map[string]string, prepend string) string {

	// Debug.Println("Formatting command with the following data:")
	// Debug.Println("prepend:", prepend)
	// Debug.Println("cmd:", cmd)
	// Debug.Println("inTargets:", inTargets)
	// Debug.Println("outTargets:", outTargets)
	// Debug.Println("params:", params)

	r := getShellCommandPlaceHolderRegex()
	ms := r.FindAllStringSubmatch(cmd, -1)
	for _, m := range ms {
		placeHolderStr := m[0]
		typ := m[1]
		name := m[2]
		var filePath string
		if typ == "o" || typ == "os" {
			// Out-ports
			if outTargets[name] == nil {
				msg := fmt.Sprint("Missing outpath for outport '", name, "' for command '", cmd, "'")
				Check(errors.New(msg))
			} else {
				if typ == "o" {
					filePath = outTargets[name].GetTempPath() // Means important to Atomize afterwards!
				} else if typ == "os" {
					filePath = outTargets[name].GetFifoPath()
				}
			}
		} else if typ == "i" {
			// In-ports
			if inTargets[name] == nil {
				msg := fmt.Sprint("Missing intarget for inport '", name, "' for command '", cmd, "'")
				Check(errors.New(msg))
			} else if inTargets[name].GetPath() == "" {
				msg := fmt.Sprint("Missing inpath for inport '", name, "' for command '", cmd, "'")
				Check(errors.New(msg))
			} else {
				if inTargets[name].doStream {
					filePath = inTargets[name].GetFifoPath()
				} else {
					filePath = inTargets[name].GetPath()
				}
			}
		} else if typ == "p" {
			if params[name] == "" {
				msg := fmt.Sprint("Missing param value param '", name, "' for command '", cmd, "'")
				Check(errors.New(msg))
			} else {
				filePath = params[name]
			}
		}
		if filePath == "" {
			msg := fmt.Sprint("Replace failed for port ", name, " for command '", cmd, "'")
			Check(errors.New(msg))
		}
		cmd = str.Replace(cmd, placeHolderStr, filePath, -1)
	}
	// Add prepend string to the command
	if prepend != "" {
		cmd = fmt.Sprintf("%s %s", prepend, cmd)
	}
	return cmd
}
<|endoftext|>


<commit_before>package orchestrate

import (
	"fmt"
	"html/template"
	"net/http"

	"github.com/apex/log"
	"github.com/facebookgo/grace/gracehttp"
	"github.com/gin-contrib/multitemplate"
	"github.com/gin-gonic/gin"
	"github.com/jmoiron/sqlx"
	"github.com/spf13/viper"
	"gopkg.in/gin-contrib/cors.v1"

	"github.com/thetorproject/proteus/proteus-common/middleware"

	"github.com/thetorproject/proteus/proteus-orchestrate/orchestrate/api/v1"
	"github.com/thetorproject/proteus/proteus-orchestrate/orchestrate/sched"
)

var ctx = log.WithFields(log.Fields{
	"cmd": "events",
})

func initDatabase() (*sqlx.DB, error) {
	db, err := sqlx.Open("postgres", viper.GetString("database.url"))
	if err != nil {
		ctx.Error("failed to open database")
		return nil, err
	}
	return db, err
}

func loadTemplates(list ...string) multitemplate.Render {
	r := multitemplate.New()
	for _, x := range list {
		templateString, err := Asset("data/templates/" + x)
		if err != nil {
			ctx.WithError(err).Error("failed to load template")
		}

		tmplMessage, err := template.New(x).Parse(string(templateString))
		if err != nil {
			ctx.WithError(err).Error("failed to parse template")
		}
		r.Add(x, tmplMessage)
	}
	return r
}

// Start starts the events backend including the web handlers
func Start() {
	var (
		err error
	)

	dbMiddleware, err := middleware.InitDatabaseMiddleware("postgres", viper.GetString("database.url"))
	if err != nil {
		ctx.WithError(err).Error("failed to init database middleware")
		return
	}
	defer dbMiddleware.DB.Close()

	authMiddleware, err := middleware.InitAuthMiddleware(dbMiddleware.DB)
	if err != nil {
		ctx.WithError(err).Error("failed to initialise authMiddlewareDevice")
		return
	}
	schedMiddleware, err := sched.InitSchedMiddleware(dbMiddleware.DB)
	if err != nil {
		ctx.WithError(err).Error("failed to initialise schedMiddleware")
		return
	}

	router := gin.Default()
	router.Use(schedMiddleware.MiddlewareFunc())
	router.Use(dbMiddleware.MiddlewareFunc())
	router.Use(cors.New(middleware.CorsConfig()))
	router.HTMLRender = loadTemplates("home.tmpl")
	router.GET("/", func(c *gin.Context) {
		c.HTML(http.StatusOK, "home.tmpl", gin.H{
			"title":                "proteus-events",
			"componentName":        "proteus-events",
			"componentDescription": LongDescription,
		})
	})
	err = apiv1.BindAPI(router, authMiddleware)
	if err != nil {
		ctx.WithError(err).Error("failed to BinAPI")
		return
	}

	Addr := fmt.Sprintf("%s:%d", viper.GetString("api.address"),
		viper.GetInt("api.port"))
	ctx.Infof("starting on %s", Addr)

	s := &http.Server{
		Addr:    Addr,
		Handler: router,
	}
	gracehttp.Serve(s)
}
<commit_msg>Rename ctx pkg from event -> orchestrate<commit_after>package orchestrate

import (
	"fmt"
	"html/template"
	"net/http"

	"github.com/apex/log"
	"github.com/facebookgo/grace/gracehttp"
	"github.com/gin-contrib/multitemplate"
	"github.com/gin-gonic/gin"
	"github.com/jmoiron/sqlx"
	"github.com/spf13/viper"
	"gopkg.in/gin-contrib/cors.v1"

	"github.com/thetorproject/proteus/proteus-common/middleware"

	"github.com/thetorproject/proteus/proteus-orchestrate/orchestrate/api/v1"
	"github.com/thetorproject/proteus/proteus-orchestrate/orchestrate/sched"
)

var ctx = log.WithFields(log.Fields{
	"pkg": "orchestrate",
})

func initDatabase() (*sqlx.DB, error) {
	db, err := sqlx.Open("postgres", viper.GetString("database.url"))
	if err != nil {
		ctx.Error("failed to open database")
		return nil, err
	}
	return db, err
}

func loadTemplates(list ...string) multitemplate.Render {
	r := multitemplate.New()
	for _, x := range list {
		templateString, err := Asset("data/templates/" + x)
		if err != nil {
			ctx.WithError(err).Error("failed to load template")
		}

		tmplMessage, err := template.New(x).Parse(string(templateString))
		if err != nil {
			ctx.WithError(err).Error("failed to parse template")
		}
		r.Add(x, tmplMessage)
	}
	return r
}

// Start starts the events backend including the web handlers
func Start() {
	var (
		err error
	)

	dbMiddleware, err := middleware.InitDatabaseMiddleware("postgres", viper.GetString("database.url"))
	if err != nil {
		ctx.WithError(err).Error("failed to init database middleware")
		return
	}
	defer dbMiddleware.DB.Close()

	authMiddleware, err := middleware.InitAuthMiddleware(dbMiddleware.DB)
	if err != nil {
		ctx.WithError(err).Error("failed to initialise authMiddlewareDevice")
		return
	}
	schedMiddleware, err := sched.InitSchedMiddleware(dbMiddleware.DB)
	if err != nil {
		ctx.WithError(err).Error("failed to initialise schedMiddleware")
		return
	}

	router := gin.Default()
	router.Use(schedMiddleware.MiddlewareFunc())
	router.Use(dbMiddleware.MiddlewareFunc())
	router.Use(cors.New(middleware.CorsConfig()))
	router.HTMLRender = loadTemplates("home.tmpl")
	router.GET("/", func(c *gin.Context) {
		c.HTML(http.StatusOK, "home.tmpl", gin.H{
			"title":                "proteus-events",
			"componentName":        "proteus-events",
			"componentDescription": LongDescription,
		})
	})
	err = apiv1.BindAPI(router, authMiddleware)
	if err != nil {
		ctx.WithError(err).Error("failed to BinAPI")
		return
	}

	Addr := fmt.Sprintf("%s:%d", viper.GetString("api.address"),
		viper.GetInt("api.port"))
	ctx.Infof("starting on %s", Addr)

	s := &http.Server{
		Addr:    Addr,
		Handler: router,
	}
	gracehttp.Serve(s)
}
<|endoftext|>


<commit_before>package consul

import (
	"fmt"
	"log"
	"time"

	"github.com/AsynkronIT/protoactor-go/cluster"
	"github.com/AsynkronIT/protoactor-go/eventstream"
	"github.com/hashicorp/consul/api"
)

type ConsulProvider struct {
	deregistered       bool
	shutdown           bool
	kvKey              string
	id                 string
	clusterName        string
	index              uint64 //consul blocking index
	client             *api.Client
	ttl                time.Duration
	refreshTTL         time.Duration
	deregisterCritical time.Duration
	blockingWaitTime   time.Duration
}

func New() (*ConsulProvider, error) {
	client, err := api.NewClient(&api.Config{})
	if err != nil {
		return nil, err
	}
	p := &ConsulProvider{
		client:             client,
		ttl:                3 * time.Second,
		refreshTTL:         1 * time.Second,
		deregisterCritical: 10 * time.Second,
		blockingWaitTime:   20 * time.Second,
		weight:             5,
	}
	return p, nil
}

func (p *ConsulProvider) RegisterMember(clusterName string, address string, port int, knownKinds []string) error {
	p.id = fmt.Sprintf("%v@%v:%v", clusterName, address, port)
	p.clusterName = clusterName
	s := &api.AgentServiceRegistration{
		ID:      p.id,
		Name:    clusterName,
		Tags:    knownKinds,
		Address: address,
		Port:    port,
		Check: &api.AgentServiceCheck{
			DeregisterCriticalServiceAfter: p.deregisterCritical.String(),
			TTL: p.ttl.String(),
		},
	}
	err := p.client.Agent().ServiceRegister(s)

	if err != nil {
		return err
	}

	//register a unique ID for the current process
	//similar to UID for Akka ActorSystem
	//TODO: Orleans just use an int32 for the unique id called Generation.
	p.kvKey = fmt.Sprintf("%v/%v:%v", clusterName, address, port)
	_, err = p.client.KV().Put(&api.KVPair{
		Key:   p.kvKey,
		Value: []byte(time.Now().UTC().Format(time.RFC3339) + "\n" + fmt.Sprint(5)), //currently, just a semi unique id for this member
	}, &api.WriteOptions{})

	if err != nil {
		return err
	}

	//IMPORTANT: do these ops sync directly after registering.
	//this will ensure that the local node sees its own information upon startup.

	//force our own TTL to be OK
	p.blockingUpdateTTL()
	//force our own existence to be part of the first status update
	p.blockingStatusChange()

	p.UpdateTTL()
	return nil
}

func (p *ConsulProvider) blockingUpdateTTL() {
	refresh := func() error {
		err := p.client.Agent().PassTTL("service:"+p.id, "")
		if err != nil {
			return err
		}
		return nil
	}
	//	log.Println("[CLUSTER] [CONSUL] Refreshing service TTL")
	err := refresh()
	if err != nil {
		log.Println("[CLUSTER] [CONSUL] Failure refreshing service TTL")
	}
}

func (p *ConsulProvider) UpdateTTL() {
	go func() {
		for !p.shutdown {
			p.blockingUpdateTTL()
			time.Sleep(p.refreshTTL)
		}
	}()
}

func (p *ConsulProvider) DeregisterMember() error {
	err := p.client.Agent().ServiceDeregister(p.id)
	if err != nil {
		fmt.Println(err)
		return err
	}
	_, err = p.client.KV().Delete(p.kvKey, &api.WriteOptions{})
	if err != nil {
		fmt.Println(err)
		return err
	}
	p.deregistered = true
	return nil
}

func (p *ConsulProvider) Shutdown() error {
	p.shutdown = true
	if !p.deregistered {
		err := p.DeregisterMember()
		if err != nil {
			return err
		}
	}
	return nil
}

//call this directly after registering the service
func (p *ConsulProvider) blockingStatusChange() {
	p.notifyStatuses()
}

func (p *ConsulProvider) notifyStatuses() {

	statuses, meta, err := p.client.Health().Service(p.clusterName, "", false, &api.QueryOptions{
		WaitIndex: p.index,
		WaitTime:  p.blockingWaitTime,
	})
	if err != nil {
		log.Printf("Error %v", err)
		return
	}
	p.index = meta.LastIndex

	//fetch additional info per member from the consul KV store
	kvKey := p.clusterName + "/"
	kv, _, err := p.client.KV().List(kvKey, &api.QueryOptions{})
	if err != nil {
		log.Printf("Error %v", err)
		return
	}
	kvMap := make(map[string]string)
	for _, v := range kv {
		kvMap[v.Key] = string(v.Value)
	}

	res := make(cluster.ClusterTopologyEvent, len(statuses))
	for i, v := range statuses {
		key := fmt.Sprintf("%v/%v:%v", p.clusterName, v.Service.Address, v.Service.Port)
		memberID := kvMap[key]
		ms := &cluster.MemberStatus{
			MemberID: memberID,
			Host:     v.Service.Address,
			Port:     v.Service.Port,
			Kinds:    v.Service.Tags,
			Alive:    len(v.Checks) > 0 && v.Checks.AggregatedStatus() == api.HealthPassing,
		}
		res[i] = ms
	}
	//the reason why we want this in a batch and not as individual messages is that
	//if we have an atomic batch, we can calculate what nodes have left the cluster
	//passing events one by one, we can't know if someone left or just havent changed status for a long time

	//publish the current cluster topology onto the event stream
	eventstream.Publish(res)
}

func (p *ConsulProvider) MonitorMemberStatusChanges() {

	go func() {
		for !p.shutdown {
			p.notifyStatuses()
		}
	}()
}
<commit_msg>Add weight to consul provider.<commit_after>package consul

import (
	"fmt"
	"log"
	"time"

	"github.com/AsynkronIT/protoactor-go/cluster"
	"github.com/AsynkronIT/protoactor-go/eventstream"
	"github.com/hashicorp/consul/api"
)

type ConsulProvider struct {
	deregistered       bool
	shutdown           bool
	kvKey              string
	id                 string
	clusterName        string
	index              uint64 //consul blocking index
	weight             int
	client             *api.Client
	ttl                time.Duration
	refreshTTL         time.Duration
	deregisterCritical time.Duration
	blockingWaitTime   time.Duration
}

func New() (*ConsulProvider, error) {
	client, err := api.NewClient(&api.Config{})
	if err != nil {
		return nil, err
	}
	p := &ConsulProvider{
		client:             client,
		ttl:                3 * time.Second,
		refreshTTL:         1 * time.Second,
		deregisterCritical: 10 * time.Second,
		blockingWaitTime:   20 * time.Second,
		weight:             5,
	}
	return p, nil
}

func (p *ConsulProvider) RegisterMember(clusterName string, address string, port int, knownKinds []string) error {
	p.id = fmt.Sprintf("%v@%v:%v", clusterName, address, port)
	p.clusterName = clusterName
	s := &api.AgentServiceRegistration{
		ID:      p.id,
		Name:    clusterName,
		Tags:    knownKinds,
		Address: address,
		Port:    port,
		Check: &api.AgentServiceCheck{
			DeregisterCriticalServiceAfter: p.deregisterCritical.String(),
			TTL: p.ttl.String(),
		},
	}
	err := p.client.Agent().ServiceRegister(s)

	if err != nil {
		return err
	}

	//register a unique ID for the current process
	//similar to UID for Akka ActorSystem
	//TODO: Orleans just use an int32 for the unique id called Generation.
	p.kvKey = fmt.Sprintf("%v/%v:%v", clusterName, address, port)
	_, err = p.client.KV().Put(&api.KVPair{
		Key:   p.kvKey,
		Value: []byte(time.Now().UTC().Format(time.RFC3339) + "\n" + fmt.Sprint(5)), //currently, just a semi unique id for this member
	}, &api.WriteOptions{})

	if err != nil {
		return err
	}

	//IMPORTANT: do these ops sync directly after registering.
	//this will ensure that the local node sees its own information upon startup.

	//force our own TTL to be OK
	p.blockingUpdateTTL()
	//force our own existence to be part of the first status update
	p.blockingStatusChange()

	p.UpdateTTL()
	return nil
}

func (p *ConsulProvider) blockingUpdateTTL() {
	refresh := func() error {
		err := p.client.Agent().PassTTL("service:"+p.id, "")
		if err != nil {
			return err
		}
		return nil
	}
	//	log.Println("[CLUSTER] [CONSUL] Refreshing service TTL")
	err := refresh()
	if err != nil {
		log.Println("[CLUSTER] [CONSUL] Failure refreshing service TTL")
	}
}

func (p *ConsulProvider) UpdateTTL() {
	go func() {
		for !p.shutdown {
			p.blockingUpdateTTL()
			time.Sleep(p.refreshTTL)
		}
	}()
}

func (p *ConsulProvider) DeregisterMember() error {
	err := p.client.Agent().ServiceDeregister(p.id)
	if err != nil {
		fmt.Println(err)
		return err
	}
	_, err = p.client.KV().Delete(p.kvKey, &api.WriteOptions{})
	if err != nil {
		fmt.Println(err)
		return err
	}
	p.deregistered = true
	return nil
}

func (p *ConsulProvider) Shutdown() error {
	p.shutdown = true
	if !p.deregistered {
		err := p.DeregisterMember()
		if err != nil {
			return err
		}
	}
	return nil
}

//call this directly after registering the service
func (p *ConsulProvider) blockingStatusChange() {
	p.notifyStatuses()
}

func (p *ConsulProvider) notifyStatuses() {

	statuses, meta, err := p.client.Health().Service(p.clusterName, "", false, &api.QueryOptions{
		WaitIndex: p.index,
		WaitTime:  p.blockingWaitTime,
	})
	if err != nil {
		log.Printf("Error %v", err)
		return
	}
	p.index = meta.LastIndex

	//fetch additional info per member from the consul KV store
	kvKey := p.clusterName + "/"
	kv, _, err := p.client.KV().List(kvKey, &api.QueryOptions{})
	if err != nil {
		log.Printf("Error %v", err)
		return
	}
	kvMap := make(map[string]string)
	for _, v := range kv {
		kvMap[v.Key] = string(v.Value)
	}

	res := make(cluster.ClusterTopologyEvent, len(statuses))
	for i, v := range statuses {
		key := fmt.Sprintf("%v/%v:%v", p.clusterName, v.Service.Address, v.Service.Port)
		memberID := kvMap[key]
		ms := &cluster.MemberStatus{
			MemberID: memberID,
			Host:     v.Service.Address,
			Port:     v.Service.Port,
			Kinds:    v.Service.Tags,
			Alive:    len(v.Checks) > 0 && v.Checks.AggregatedStatus() == api.HealthPassing,
		}
		res[i] = ms
	}
	//the reason why we want this in a batch and not as individual messages is that
	//if we have an atomic batch, we can calculate what nodes have left the cluster
	//passing events one by one, we can't know if someone left or just havent changed status for a long time

	//publish the current cluster topology onto the event stream
	eventstream.Publish(res)
}

func (p *ConsulProvider) MonitorMemberStatusChanges() {

	go func() {
		for !p.shutdown {
			p.notifyStatuses()
		}
	}()
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"sort"
	"time"

	"github.com/spf13/cobra"

	"github.com/weaveworks/flux"
	"github.com/weaveworks/flux/api/v10"
	"github.com/weaveworks/flux/api/v6"
	"github.com/weaveworks/flux/registry"
	"github.com/weaveworks/flux/update"
)

type imageListOpts struct {
	*rootOpts
	namespace string
	workload  string
	limit     int

	// Deprecated
	controller string
}

func newImageList(parent *rootOpts) *imageListOpts {
	return &imageListOpts{rootOpts: parent}
}

func (opts *imageListOpts) Command() *cobra.Command {
	cmd := &cobra.Command{
		Use:     "list-images",
		Short:   "Show deployed and available images.",
		Example: makeExample("fluxctl list-images --namespace default --workload=deployment/foo"),
		RunE:    opts.RunE,
	}
	cmd.Flags().StringVarP(&opts.namespace, "namespace", "n", "", "Namespace")
	cmd.Flags().StringVarP(&opts.workload, "workload", "w", "", "Show images for this workload")
	cmd.Flags().IntVarP(&opts.limit, "limit", "l", 10, "Number of images to show (0 for all)")

	// Deprecated
	cmd.Flags().StringVarP(&opts.controller, "controller", "c", "", "Show images for this controller")
	cmd.Flags().MarkDeprecated("controller", "changed to --workload, use that instead")

	return cmd
}

func (opts *imageListOpts) RunE(cmd *cobra.Command, args []string) error {
	if len(args) != 0 {
		return errorWantedNoArgs
	}

	imageOpts := v10.ListImagesOptions{
		Spec:      update.ResourceSpecAll,
		Namespace: opts.namespace,
	}
	// Backwards compatibility with --controller until we remove it
	switch {
	case opts.workload != "" && opts.controller != "":
		return newUsageError("can't specify both the controller and workload")
	case opts.controller != "":
		opts.workload = opts.controller
	}
	if len(opts.workload) > 0 {
		id, err := flux.ParseResourceIDOptionalNamespace(opts.namespace, opts.workload)
		if err != nil {
			return err
		}
		imageOpts.Spec = update.MakeResourceSpec(id)
		imageOpts.Namespace = ""
	}

	ctx := context.Background()

	workloads, err := opts.API.ListImagesWithOptions(ctx, imageOpts)
	if err != nil {
		return err
	}

	sort.Sort(imageStatusByName(workloads))

	out := newTabwriter()

	fmt.Fprintln(out, "WORKLOAD\tCONTAINER\tIMAGE\tCREATED")
	for _, workload := range workloads {
		if len(workload.Containers) == 0 {
			fmt.Fprintf(out, "%s\t\t\t\n", workload.ID)
			continue
		}

		workloadName := workload.ID.String()
		for _, container := range workload.Containers {
			var lineCount int
			containerName := container.Name
			reg, repo, currentTag := container.Current.ID.Components()
			if reg != "" {
				reg += "/"
			}
			if len(container.Available) == 0 {
				availableErr := container.AvailableError
				if availableErr == "" {
					availableErr = registry.ErrNoImageData.Error()
				}
				fmt.Fprintf(out, "%s\t%s\t%s%s\t%s\n", workloadName, containerName, reg, repo, availableErr)
			} else {
				fmt.Fprintf(out, "%s\t%s\t%s%s\t\n", workloadName, containerName, reg, repo)
			}
			foundRunning := false
			for _, available := range container.Available {
				running := "|  "
				_, _, tag := available.ID.Components()
				if currentTag == tag {
					running = "'->"
					foundRunning = true
				} else if foundRunning {
					running = "   "
				}

				lineCount++
				var printEllipsis, printLine bool
				if opts.limit <= 0 || lineCount <= opts.limit {
					printEllipsis, printLine = false, true
				} else if container.Current.ID == available.ID {
					printEllipsis, printLine = lineCount > (opts.limit+1), true
				}
				if printEllipsis {
					fmt.Fprintf(out, "\t\t%s (%d image(s) omitted)\t\n", ":", lineCount-opts.limit-1)
				}
				if printLine {
					createdAt := ""
					if !available.CreatedAt.IsZero() {
						createdAt = available.CreatedAt.Format(time.RFC822)
					}
					fmt.Fprintf(out, "\t\t%s %s\t%s\n", running, tag, createdAt)
				}
			}
			workloadName = ""
		}
	}
	out.Flush()
	return nil
}

type imageStatusByName []v6.ImageStatus

func (s imageStatusByName) Len() int {
	return len(s)
}

func (s imageStatusByName) Less(a, b int) bool {
	return s[a].ID.String() < s[b].ID.String()
}

func (s imageStatusByName) Swap(a, b int) {
	s[a], s[b] = s[b], s[a]
}
<commit_msg>Show unknown tag for workload in list-images<commit_after>package main

import (
	"context"
	"fmt"
	"sort"
	"time"

	"github.com/spf13/cobra"

	"github.com/weaveworks/flux"
	"github.com/weaveworks/flux/api/v10"
	"github.com/weaveworks/flux/api/v6"
	"github.com/weaveworks/flux/registry"
	"github.com/weaveworks/flux/update"
)

type imageListOpts struct {
	*rootOpts
	namespace string
	workload  string
	limit     int

	// Deprecated
	controller string
}

func newImageList(parent *rootOpts) *imageListOpts {
	return &imageListOpts{rootOpts: parent}
}

func (opts *imageListOpts) Command() *cobra.Command {
	cmd := &cobra.Command{
		Use:     "list-images",
		Short:   "Show deployed and available images.",
		Example: makeExample("fluxctl list-images --namespace default --workload=deployment/foo"),
		RunE:    opts.RunE,
	}
	cmd.Flags().StringVarP(&opts.namespace, "namespace", "n", "", "Namespace")
	cmd.Flags().StringVarP(&opts.workload, "workload", "w", "", "Show images for this workload")
	cmd.Flags().IntVarP(&opts.limit, "limit", "l", 10, "Number of images to show (0 for all)")

	// Deprecated
	cmd.Flags().StringVarP(&opts.controller, "controller", "c", "", "Show images for this controller")
	cmd.Flags().MarkDeprecated("controller", "changed to --workload, use that instead")

	return cmd
}

func (opts *imageListOpts) RunE(cmd *cobra.Command, args []string) error {
	if len(args) != 0 {
		return errorWantedNoArgs
	}

	imageOpts := v10.ListImagesOptions{
		Spec:      update.ResourceSpecAll,
		Namespace: opts.namespace,
	}
	// Backwards compatibility with --controller until we remove it
	switch {
	case opts.workload != "" && opts.controller != "":
		return newUsageError("can't specify both the controller and workload")
	case opts.controller != "":
		opts.workload = opts.controller
	}
	if len(opts.workload) > 0 {
		id, err := flux.ParseResourceIDOptionalNamespace(opts.namespace, opts.workload)
		if err != nil {
			return err
		}
		imageOpts.Spec = update.MakeResourceSpec(id)
		imageOpts.Namespace = ""
	}

	ctx := context.Background()

	workloads, err := opts.API.ListImagesWithOptions(ctx, imageOpts)
	if err != nil {
		return err
	}

	sort.Sort(imageStatusByName(workloads))

	out := newTabwriter()

	fmt.Fprintln(out, "WORKLOAD\tCONTAINER\tIMAGE\tCREATED")
	for _, workload := range workloads {
		if len(workload.Containers) == 0 {
			fmt.Fprintf(out, "%s\t\t\t\n", workload.ID)
			continue
		}

		workloadName := workload.ID.String()
		for _, container := range workload.Containers {
			var lineCount int
			containerName := container.Name
			reg, repo, currentTag := container.Current.ID.Components()
			if reg != "" {
				reg += "/"
			}
			if len(container.Available) == 0 {
				availableErr := container.AvailableError
				if availableErr == "" {
					availableErr = registry.ErrNoImageData.Error()
				}
				fmt.Fprintf(out, "%s\t%s\t%s%s\t%s\n", workloadName, containerName, reg, repo, availableErr)
			} else {
				fmt.Fprintf(out, "%s\t%s\t%s%s\t\n", workloadName, containerName, reg, repo)
			}
			foundRunning := false
			for _, available := range container.Available {
				running := "|  "
				_, _, tag := available.ID.Components()
				if currentTag == tag {
					running = "'->"
					foundRunning = true
				} else if foundRunning {
					running = "   "
				}

				lineCount++
				var printEllipsis, printLine bool
				if opts.limit <= 0 || lineCount <= opts.limit {
					printEllipsis, printLine = false, true
				} else if container.Current.ID == available.ID {
					printEllipsis, printLine = lineCount > (opts.limit+1), true
				}
				if printEllipsis {
					fmt.Fprintf(out, "\t\t%s (%d image(s) omitted)\t\n", ":", lineCount-opts.limit-1)
				}
				if printLine {
					createdAt := ""
					if !available.CreatedAt.IsZero() {
						createdAt = available.CreatedAt.Format(time.RFC822)
					}
					fmt.Fprintf(out, "\t\t%s %s\t%s\n", running, tag, createdAt)
				}
			}
			if !foundRunning {
				running := "'->"
				if currentTag == "" {
					currentTag = "(untagged)"
				}
				fmt.Fprintf(out, "\t\t%s %s\t%s\n", running, currentTag, "?")

			}
			workloadName = ""
		}
	}
	out.Flush()
	return nil
}

type imageStatusByName []v6.ImageStatus

func (s imageStatusByName) Len() int {
	return len(s)
}

func (s imageStatusByName) Less(a, b int) bool {
	return s[a].ID.String() < s[b].ID.String()
}

func (s imageStatusByName) Swap(a, b int) {
	s[a], s[b] = s[b], s[a]
}
<|endoftext|>


<commit_before>package rw

import (
	"fmt"
	"io"
	"sync/atomic"
	"time"
)

// WaitingWriter TODO
type WaitingWriter struct {
	W io.Writer // underlying Writer
	P []byte    // byte sequence to wait for

	spin uint32
	offs int
}

// WaitWriter TODO
func WaitWriter(w io.Writer, p []byte) *WaitingWriter {
	return &WaitingWriter{
		W: w,
		P: p,
	}
}

// Write TODO
func (ww *WaitingWriter) Write(p []byte) (int, error) {
	for _, b := range p {
		if ww.P[ww.offs] == b {
			ww.offs++
			if ww.offs == len(ww.P) {
				ww.offs = 0
				atomic.CompareAndSwapUint32(&ww.spin, 0, 1)
			}
		} else {
			ww.offs = 0
		}
	}
	return ww.W.Write(p)
}

// Wait TODO
func (ww *WaitingWriter) Wait(d time.Duration) error {
	t := time.After(d)
	for {
		select {
		case <-t:
			return fmt.Errorf("timeout waiting for p=%q", ww.P)
		default:
			if atomic.CompareAndSwapUint32(&ww.spin, 1, 0) {
				return nil
			}
		}
	}
}
<commit_msg>Don't starve while spin-waiting<commit_after>package rw

import (
	"fmt"
	"io"
	"runtime"
	"sync/atomic"
	"time"
)

// WaitingWriter TODO
type WaitingWriter struct {
	W io.Writer // underlying Writer
	P []byte    // byte sequence to wait for

	spin uint32
	offs int
}

// WaitWriter TODO
func WaitWriter(w io.Writer, p []byte) *WaitingWriter {
	return &WaitingWriter{
		W: w,
		P: p,
	}
}

// Write TODO
func (ww *WaitingWriter) Write(p []byte) (int, error) {
	for _, b := range p {
		if ww.P[ww.offs] == b {
			ww.offs++
			if ww.offs == len(ww.P) {
				ww.offs = 0
				atomic.CompareAndSwapUint32(&ww.spin, 0, 1)
			}
		} else {
			ww.offs = 0
		}
	}
	return ww.W.Write(p)
}

// Wait TODO
func (ww *WaitingWriter) Wait(d time.Duration) error {
	t := time.After(d)
	for {
		select {
		case <-t:
			return fmt.Errorf("timeout waiting for p=%q", ww.P)
		default:
			if atomic.CompareAndSwapUint32(&ww.spin, 1, 0) {
				return nil
			}
			runtime.Gosched()
		}
	}
}
<|endoftext|>