<commit_before>// Go Riemann client
//
// TODO(cloudhead): pbEventsToEvents should parse Attributes field
package raidman

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"net"
	"os"
	"sync"

	pb "code.google.com/p/goprotobuf/proto"
	"github.com/swdunlop/raidman/proto"
)

type network interface {
	Send(message *proto.Msg, conn net.Conn) (*proto.Msg, error)
}

type tcp struct{}

type udp struct{}

// Client represents a connection to a Riemann server
type Client struct {
	sync.Mutex
	net        network
	connection net.Conn
}

// An Event represents a single Riemann event
type Event struct {
	Ttl         float32
	Time        int64
	Tags        []string
	Host        string // Defaults to os.Hostname()
	State       string
	Service     string
	Metric      interface{} // Could be Int, Float32, Float64
	Attributes  map[string]interface{}
	Description string
}

// Dial establishes a connection to a Riemann server at addr, on the network
// netwrk.
//
// Known networks are "tcp", "tcp4", "tcp6", "udp", "udp4", and "udp6".
func Dial(netwrk, addr string) (c *Client, err error) {
	c = new(Client)

	var cnet network
	switch netwrk {
	case "tcp", "tcp4", "tcp6":
		cnet = new(tcp)
	case "udp", "udp4", "udp6":
		cnet = new(udp)
	default:
		return nil, fmt.Errorf("dial %q: unsupported network %q", netwrk, netwrk)
	}

	c.net = cnet
	c.connection, err = net.Dial(netwrk, addr)
	if err != nil {
		return nil, err
	}

	return c, nil
}

func (network *tcp) Send(message *proto.Msg, conn net.Conn) (*proto.Msg, error) {
	msg := &proto.Msg{}
	data, err := pb.Marshal(message)
	if err != nil {
		return msg, err
	}
	b := new(bytes.Buffer)
	if err = binary.Write(b, binary.BigEndian, uint32(len(data))); err != nil {
		return msg, err
	}
	if _, err = conn.Write(b.Bytes()); err != nil {
		return msg, err
	}
	if _, err = conn.Write(data); err != nil {
		return msg, err
	}
	var header uint32
	if err = binary.Read(conn, binary.BigEndian, &header); err != nil {
		return msg, err
	}
	response := make([]byte, header)
	if err = readFully(conn, response); err != nil {
		return msg, err
	}
	if err = pb.Unmarshal(response, msg); err != nil {
		return msg, err
	}
	if msg.GetOk() != true {
		return msg, errors.New(msg.GetError())
	}
	return msg, nil
}

func readFully(r io.Reader, p []byte) error {
	for len(p) > 0 {
		n, err := r.Read(p)
		p = p[n:]
		if err != nil {
			return err
		}
	}
	return nil
}

func (network *udp) Send(message *proto.Msg, conn net.Conn) (*proto.Msg, error) {
	data, err := pb.Marshal(message)
	if err != nil {
		return nil, err
	}
	if _, err = conn.Write(data); err != nil {
		return nil, err
	}

	return nil, nil
}

func eventToPbEvent(event *Event) (*proto.Event, error) {
	var e proto.Event

	h := event.Host
	if event.Host == "" {
		h, _ = os.Hostname()
	}
	e.Host = &h

	if event.State != "" {
		e.State = &event.State
	}
	if event.Service != "" {
		e.Service = &event.Service
	}
	if event.Description != "" {
		e.Description = &event.Description
	}
	if event.Ttl != 0 {
		e.Ttl = &event.Ttl
	}
	if event.Time != 0 {
		e.Time = &event.Time
	}
	if len(event.Tags) > 0 {
		e.Tags = event.Tags
	}

	err := assignEventMetric(&e, event.Metric)
	if err != nil {
		return nil, err
	}

	attrs := make([]proto.Attribute, len(event.Attributes))
	i := 0
	for k, v := range event.Attributes {
		switch x := v.(type) {
		case string:
			attrs[i].Key = &k
			attrs[i].Value = &x
		case bool:
			if x {
				attrs[i].Key = &k
			}
		default:
			return nil, fmt.Errorf("Attribute %v has invalid type (type %T)", k, v)
		}
		i++
	}

	return &e, nil
}

func assignEventMetric(e *proto.Event, v interface{}) error {
	switch x := v.(type) {
	case nil:
		// do nothing; an event without a metric is legitimate
	case int:
		i := int64(x)
		e.MetricSint64 = &i
	case int32:
		i := int64(x)
		e.MetricSint64 = &i
	case int64:
		e.MetricSint64 = &x
	case float32:
		e.MetricF = &x
	case float64:
		e.MetricD = &x
	default:
		return fmt.Errorf("Metric of invalid type (type %T)", v)
	}
	return nil
}

func pbEventsToEvents(pbEvents []*proto.Event) []Event {
	var events []Event

	for _, event := range pbEvents {
		e := Event{
			State:       event.GetState(),
			Service:     event.GetService(),
			Host:        event.GetHost(),
			Description: event.GetDescription(),
			Ttl:         event.GetTtl(),
			Time:        event.GetTime(),
			Tags:        event.GetTags(),
		}
		if event.MetricF != nil {
			e.Metric = event.GetMetricF()
		} else if event.MetricD != nil {
			e.Metric = event.GetMetricD()
		} else {
			e.Metric = event.GetMetricSint64()
		}

		events = append(events, e)
	}

	return events
}

// Send sends an event to Riemann
func (c *Client) Send(event *Event) error {
	e, err := eventToPbEvent(event)
	if err != nil {
		return err
	}
	message := &proto.Msg{}
	message.Events = append(message.Events, e)
	c.Lock()
	defer c.Unlock()
	_, err = c.net.Send(message, c.connection)
	if err != nil {
		return err
	}

	return nil
}

// Query returns a list of events matched by query
func (c *Client) Query(q string) ([]Event, error) {
	switch c.net.(type) {
	case *udp:
		return nil, errors.New("Querying over UDP is not supported")
	}
	query := &proto.Query{}
	query.String_ = pb.String(q)
	message := &proto.Msg{}
	message.Query = query
	c.Lock()
	defer c.Unlock()
	response, err := c.net.Send(message, c.connection)
	if err != nil {
		return nil, err
	}
	return pbEventsToEvents(response.GetEvents()), nil
}

// Close closes the connection to Riemann
func (c *Client) Close() {
	c.Lock()
	c.connection.Close()
	c.Unlock()
}
<commit_msg>refactored private net class into a more general Transport interface<commit_after>// Go Riemann client
//
// TODO(cloudhead): pbEventsToEvents should parse Attributes field
package raidman

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"net"
	"os"
	"sync"

	pb "code.google.com/p/goprotobuf/proto"
	"github.com/swdunlop/raidman/proto"
)

// Client represents a connection to a Riemann server
type Client struct {
	sync.Mutex
	transport Transport
}

// An Event represents a single Riemann event
type Event struct {
	Ttl         float32
	Time        int64
	Tags        []string
	Host        string // Defaults to os.Hostname()
	State       string
	Service     string
	Metric      interface{} // Could be Int, Float32, Float64
	Attributes  map[string]interface{}
	Description string
}

// Dial establishes a connection to a Riemann server at addr, on the network
// netwrk.
//
// Known networks are "tcp", "tcp4", "tcp6", "udp", "udp4", and "udp6".
func Dial(network, addr string) (*Client, error) {
	client := &Client{}
	rwc, err := net.Dial(network, addr)
	if err != nil {
		return nil, err
	}

	switch network {
	case "udp", "udp4", "udp6":
		client.transport = &UdpTransport{rwc}
	default:
		client.transport = &TcpTransport{rwc}
	}

	return client, nil
}

func eventToPbEvent(event *Event) (*proto.Event, error) {
	var e proto.Event

	h := event.Host
	if event.Host == "" {
		h, _ = os.Hostname()
	}
	e.Host = &h

	if event.State != "" {
		e.State = &event.State
	}
	if event.Service != "" {
		e.Service = &event.Service
	}
	if event.Description != "" {
		e.Description = &event.Description
	}
	if event.Ttl != 0 {
		e.Ttl = &event.Ttl
	}
	if event.Time != 0 {
		e.Time = &event.Time
	}
	if len(event.Tags) > 0 {
		e.Tags = event.Tags
	}

	err := assignEventMetric(&e, event.Metric)
	if err != nil {
		return nil, err
	}

	attrs := make([]proto.Attribute, len(event.Attributes))
	i := 0
	for k, v := range event.Attributes {
		switch x := v.(type) {
		case string:
			attrs[i].Key = &k
			attrs[i].Value = &x
		case bool:
			if x {
				attrs[i].Key = &k
			}
		default:
			return nil, fmt.Errorf("Attribute %v has invalid type (type %T)", k, v)
		}
		i++
	}

	return &e, nil
}

func assignEventMetric(e *proto.Event, v interface{}) error {
	switch x := v.(type) {
	case nil:
		// do nothing; an event without a metric is legitimate
	case int:
		i := int64(x)
		e.MetricSint64 = &i
	case int32:
		i := int64(x)
		e.MetricSint64 = &i
	case int64:
		e.MetricSint64 = &x
	case float32:
		e.MetricF = &x
	case float64:
		e.MetricD = &x
	default:
		return fmt.Errorf("Metric of invalid type (type %T)", v)
	}
	return nil
}

// Send sends an event to Riemann and waits for an acknowledgement, if a TCP client was used.
func (c *Client) Send(event *Event) error {
	c.Lock()
	defer c.Unlock()

	evt, err := eventToPbEvent(event)
	if err != nil {
		return err
	}

	req := &proto.Msg{Events: []*proto.Event{evt}}
	err = c.transport.WriteMsg(req)
	switch {
	case err != nil:
		return err
	case c.isUsingUdp():
		return nil
	}

	_, err = c.transport.ReadMsg()
	return err
}

// Query returns a list of events matched by query; Query cannot be used with UDP clients.
func (c *Client) Query(q string) ([]Event, error) {
	c.Lock()
	defer c.Unlock()

	if c.isUsingUdp() {
		return nil, errors.New("Querying over UDP is not supported")
	}

	req := &proto.Msg{Query: &proto.Query{String_: pb.String(q)}}
	err := c.transport.WriteMsg(req)
	if err != nil {
		return nil, err
	}

	rsp, err := c.transport.ReadMsg()
	if err != nil {
		return nil, err
	}
	return pbEventsToEvents(rsp.GetEvents()), nil
}

func (c *Client) isUsingUdp() bool {
	_, yes := c.transport.(*UdpTransport)
	return yes
}

// Close closes the connection to Riemann
func (c *Client) Close() error {
	c.Lock()
	defer c.Unlock()
	return c.transport.Close()
}

func pbEventsToEvents(pbEvents []*proto.Event) []Event {
	var events []Event

	for _, event := range pbEvents {
		e := Event{
			State:       event.GetState(),
			Service:     event.GetService(),
			Host:        event.GetHost(),
			Description: event.GetDescription(),
			Ttl:         event.GetTtl(),
			Time:        event.GetTime(),
			Tags:        event.GetTags(),
		}
		if event.MetricF != nil {
			e.Metric = event.GetMetricF()
		} else if event.MetricD != nil {
			e.Metric = event.GetMetricD()
		} else {
			e.Metric = event.GetMetricSint64()
		}

		events = append(events, e)
	}

	return events
}

type Transport interface {
	WriteMsg(msg *proto.Msg) error
	ReadMsg() (*proto.Msg, error)
	Close() error
}

type TcpTransport struct {
	rwc io.ReadWriteCloser
}

var _ Transport = &TcpTransport{}

func (tp *TcpTransport) WriteMsg(msg *proto.Msg) error {
	data, err := pb.Marshal(msg)
	if err != nil {
		return err
	}

	sz := len(data)
	buf := bytes.NewBuffer(make([]byte, 0, sz+4))
	binary.Write(buf, binary.BigEndian, uint32(sz))
	buf.Write(data)
	_, err = tp.rwc.Write(buf.Bytes())
	return err
}

func (tp *TcpTransport) ReadMsg() (*proto.Msg, error) {
	var sz uint32
	err := binary.Read(tp.rwc, binary.BigEndian, &sz)
	if err != nil {
		return nil, err
	}
	p := make([]byte, sz)
	err = readFully(tp.rwc, p)
	if err != nil {
		return nil, err
	}

	msg := new(proto.Msg)
	err = pb.Unmarshal(p, msg)
	if err != nil {
		return nil, err
	}
	return msg, nil
}

func (tp *TcpTransport) Close() error {
	return tp.rwc.Close()
}

type UdpTransport struct {
	rwc io.ReadWriteCloser
}

var _ Transport = &UdpTransport{}

func (tp *UdpTransport) WriteMsg(msg *proto.Msg) error {
	data, err := pb.Marshal(msg)
	if err != nil {
		return err
	}
	_, err = tp.rwc.Write(data)
	return err
}

func (tp *UdpTransport) ReadMsg() (*proto.Msg, error) {
	p := make([]byte, 1<<16) // max udp data size is less than 64k
	n, err := tp.rwc.Read(p)
	if err != nil {
		return nil, err
	}
	p = p[:n]

	msg := new(proto.Msg)
	err = pb.Unmarshal(p, msg)
	if err != nil {
		return nil, err
	}
	return msg, nil
}

func (tp *UdpTransport) Close() error {
	return tp.rwc.Close()
}

func readFully(r io.Reader, p []byte) error {
	for len(p) > 0 {
		n, err := r.Read(p)
		p = p[n:]
		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package gae

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"runtime"
	"strings"
	"sync"
	"time"

	"appengine"
	"appengine/taskqueue"
	"code.google.com/p/go-uuid/uuid"
	"github.com/go-martini/martini"
	"github.com/knightso/base/gae/bq"
	"github.com/martini-contrib/render"
)

type ExMartini struct {
	*martini.Martini
	martini.Router
}

type MartiniOption struct {
	AdditionalHandlers []martini.Handler
	Log2bq             bool
}

func NewMartini(option MartiniOption) *ExMartini {
	r := martini.NewRouter()
	m := martini.New()

	m.Use(func(c martini.Context, r *http.Request, l *log.Logger) {
		ac := appengine.NewContext(r)
		gaelog := log.New(logWriter{ac}, l.Prefix(), l.Flags())
		c.Map(gaelog)
	})
	m.Use(func(c martini.Context, r *http.Request) {
		ac := appengine.NewContext(r)
		c.Next()
		for l := popLog(); l != ""; l = popLog() {
			if option.Log2bq == false {
				ac.Debugf(l)
				continue
			}

			id := uuid.NewUUID()
			uuidString := id.String()
			now := time.Now()
			record := make(map[string]interface{})
			record["id"] = uuidString
			record["date"] = now
			record["log"] = l

			err := bq.SendLog(ac, "debuglog", uuidString, record)
			if err != nil {
				ac.Warningf("%s", err.Error())
				continue
			}

			ac.Debugf(l)
		}
	})
	m.Use(martini.Logger())
	m.Use(martini.Recovery())
	m.Use(render.Renderer())
	for _, h := range option.AdditionalHandlers {
		m.Use(h)
	}
	m.MapTo(r, (*martini.Route)(nil))
	m.Action(r.Handle)

	return &ExMartini{m, r}
}

type logWriter struct {
	ac appengine.Context
}

func (w logWriter) Write(p []byte) (n int, err error) {
	w.ac.Debugf(string(p))
	return len(p), nil
}

var logs []string
var logMutex sync.Mutex

var LOG_ENABLED bool

func init() {
	logs = make([]string, 0)
}

func Logf(s string, a ...interface{}) {
	if !LOG_ENABLED {
		return
	}

	logMutex.Lock()
	defer logMutex.Unlock()

	logs = append(logs, location()+": "+fmt.Sprintf(s, a...))
}

func popLog() string {
	if !LOG_ENABLED {
		return ""
	}

	logMutex.Lock()
	defer logMutex.Unlock()

	if len(logs) == 0 {
		return ""
	}

	s := logs[0]
	logs = logs[1:]

	return s
}

func location() string {
	_, file, line, ok := runtime.Caller(2)
	if ok {
		if index := strings.LastIndex(file, "/"); index >= 0 {
			file = file[index+1:]
		} else if index = strings.LastIndex(file, "\\"); index >= 0 {
			file = file[index+1:]
		}
	} else {
		file = "???"
		line = -1
	}
	return fmt.Sprintf("%s:%d", file, line)
}
<commit_msg>remove the package that has not been used<commit_after>package gae

import (
	"fmt"
	"log"
	"net/http"
	"runtime"
	"strings"
	"sync"
	"time"

	"appengine"

	"code.google.com/p/go-uuid/uuid"
	"github.com/go-martini/martini"
	"github.com/knightso/base/gae/bq"
	"github.com/martini-contrib/render"
)

type ExMartini struct {
	*martini.Martini
	martini.Router
}

type MartiniOption struct {
	AdditionalHandlers []martini.Handler
	Log2bq             bool
}

func NewMartini(option MartiniOption) *ExMartini {
	r := martini.NewRouter()
	m := martini.New()

	m.Use(func(c martini.Context, r *http.Request, l *log.Logger) {
		ac := appengine.NewContext(r)
		gaelog := log.New(logWriter{ac}, l.Prefix(), l.Flags())
		c.Map(gaelog)
	})
	m.Use(func(c martini.Context, r *http.Request) {
		ac := appengine.NewContext(r)
		c.Next()
		for l := popLog(); l != ""; l = popLog() {
			if option.Log2bq == false {
				ac.Debugf(l)
				continue
			}

			id := uuid.NewUUID()
			uuidString := id.String()
			now := time.Now()
			record := make(map[string]interface{})
			record["id"] = uuidString
			record["date"] = now
			record["log"] = l

			err := bq.SendLog(ac, "debuglog", uuidString, record)
			if err != nil {
				ac.Warningf("%s", err.Error())
				continue
			}

			ac.Debugf(l)
		}
	})
	m.Use(martini.Logger())
	m.Use(martini.Recovery())
	m.Use(render.Renderer())
	for _, h := range option.AdditionalHandlers {
		m.Use(h)
	}
	m.MapTo(r, (*martini.Route)(nil))
	m.Action(r.Handle)

	return &ExMartini{m, r}
}

type logWriter struct {
	ac appengine.Context
}

func (w logWriter) Write(p []byte) (n int, err error) {
	w.ac.Debugf(string(p))
	return len(p), nil
}

var logs []string
var logMutex sync.Mutex

var LOG_ENABLED bool

func init() {
	logs = make([]string, 0)
}

func Logf(s string, a ...interface{}) {
	if !LOG_ENABLED {
		return
	}

	logMutex.Lock()
	defer logMutex.Unlock()

	logs = append(logs, location()+": "+fmt.Sprintf(s, a...))
}

func popLog() string {
	if !LOG_ENABLED {
		return ""
	}

	logMutex.Lock()
	defer logMutex.Unlock()

	if len(logs) == 0 {
		return ""
	}

	s := logs[0]
	logs = logs[1:]

	return s
}

func location() string {
	_, file, line, ok := runtime.Caller(2)
	if ok {
		if index := strings.LastIndex(file, "/"); index >= 0 {
			file = file[index+1:]
		} else if index = strings.LastIndex(file, "\\"); index >= 0 {
			file = file[index+1:]
		}
	} else {
		file = "???"
		line = -1
	}
	return fmt.Sprintf("%s:%d", file, line)
}
<|endoftext|>


<commit_before>package dynamodb

import (
	"fmt"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute"
	"github.com/ohsu-comp-bio/funnel/proto/tes"
	"golang.org/x/net/context"
)

// GetTask gets a task, which describes a running task
func (db *DynamoDB) GetTask(ctx context.Context, req *tes.GetTaskRequest) (*tes.Task, error) {
	var task *tes.Task
	var response *dynamodb.GetItemOutput
	var err error

	switch req.View {
	case tes.TaskView_MINIMAL:
		response, err = db.getMinimalView(ctx, req.Id)
	case tes.TaskView_BASIC:
		response, err = db.getBasicView(ctx, req.Id)
	case tes.TaskView_FULL:
		response, err = db.getFullView(ctx, req.Id)
	}
	if err != nil {
		return nil, err
	}

	if response.Item == nil {
		return nil, tes.ErrNotFound
	}

	err = dynamodbattribute.UnmarshalMap(response.Item, &task)
	if err != nil {
		return nil, fmt.Errorf("failed to DynamoDB unmarshal Task, %v", err)
	}

	return task, nil
}

// ListTasks returns a list of taskIDs
func (db *DynamoDB) ListTasks(ctx context.Context, req *tes.ListTasksRequest) (*tes.ListTasksResponse, error) {

	var tasks []*tes.Task
	var query *dynamodb.QueryInput
	pageSize := int64(tes.GetPageSize(req.GetPageSize()))

	query = &dynamodb.QueryInput{
		TableName:              aws.String(db.taskTable),
		Limit:                  aws.Int64(pageSize),
		ScanIndexForward:       aws.Bool(false),
		ConsistentRead:         aws.Bool(true),
		KeyConditionExpression: aws.String(fmt.Sprintf("%s = :v1", db.partitionKey)),
		ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{
			":v1": {
				S: aws.String(db.partitionValue),
			},
		},
	}

	if req.View == tes.TaskView_MINIMAL {
		query.ExpressionAttributeNames = map[string]*string{
			"#state": aws.String("state"),
		}
		query.ProjectionExpression = aws.String("id, #state")
	}

	if req.PageToken != "" {
		query.ExclusiveStartKey = map[string]*dynamodb.AttributeValue{
			db.partitionKey: {
				S: aws.String(db.partitionValue),
			},
			"id": {
				S: aws.String(req.PageToken),
			},
		}
	}

	response, err := db.client.QueryWithContext(ctx, query)

	if err != nil {
		return nil, err
	}

	if req.View == tes.TaskView_FULL {
		for _, item := range response.Items {
			// TODO handle errors
			_ = db.getContent(ctx, item)
			_ = db.getExecutorOutput(ctx, item, "stdout", db.stdoutTable)
			_ = db.getExecutorOutput(ctx, item, "stderr", db.stderrTable)
		}
	}

	err = dynamodbattribute.UnmarshalListOfMaps(response.Items, &tasks)
	if err != nil {
		return nil, fmt.Errorf("failed to DynamoDB unmarshal Tasks, %v", err)
	}

	out := tes.ListTasksResponse{
		Tasks: tasks,
	}

	if response.LastEvaluatedKey != nil {
		out.NextPageToken = *response.LastEvaluatedKey["id"].S
	}

	return &out, nil
}
<commit_msg>api/tes: ListTasks state filter for DynamoDB<commit_after>package dynamodb

import (
	"fmt"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute"
	"github.com/ohsu-comp-bio/funnel/proto/tes"
	"golang.org/x/net/context"
	"strconv"
)

// GetTask gets a task, which describes a running task
func (db *DynamoDB) GetTask(ctx context.Context, req *tes.GetTaskRequest) (*tes.Task, error) {
	var task *tes.Task
	var response *dynamodb.GetItemOutput
	var err error

	switch req.View {
	case tes.TaskView_MINIMAL:
		response, err = db.getMinimalView(ctx, req.Id)
	case tes.TaskView_BASIC:
		response, err = db.getBasicView(ctx, req.Id)
	case tes.TaskView_FULL:
		response, err = db.getFullView(ctx, req.Id)
	}
	if err != nil {
		return nil, err
	}

	if response.Item == nil {
		return nil, tes.ErrNotFound
	}

	err = dynamodbattribute.UnmarshalMap(response.Item, &task)
	if err != nil {
		return nil, fmt.Errorf("failed to DynamoDB unmarshal Task, %v", err)
	}

	return task, nil
}

// ListTasks returns a list of taskIDs
func (db *DynamoDB) ListTasks(ctx context.Context, req *tes.ListTasksRequest) (*tes.ListTasksResponse, error) {

	var tasks []*tes.Task
	var query *dynamodb.QueryInput
	pageSize := int64(tes.GetPageSize(req.GetPageSize()))

	query = &dynamodb.QueryInput{
		TableName:              aws.String(db.taskTable),
		Limit:                  aws.Int64(pageSize),
		ScanIndexForward:       aws.Bool(false),
		ConsistentRead:         aws.Bool(true),
		KeyConditionExpression: aws.String(fmt.Sprintf("%s = :v1", db.partitionKey)),
		ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{
			":v1": {
				S: aws.String(db.partitionValue),
			},
		},
	}

	if req.StateFilter != tes.Unknown {
		query.ExpressionAttributeNames = map[string]*string{
			"#state": aws.String("state"),
		}
		query.ExpressionAttributeValues[":stateFilter"] = &dynamodb.AttributeValue{
			N: aws.String(strconv.Itoa(int(req.StateFilter))),
		}
		query.FilterExpression = aws.String("#state = :stateFilter")
	}

	if req.View == tes.TaskView_MINIMAL {
		query.ExpressionAttributeNames = map[string]*string{
			"#state": aws.String("state"),
		}
		query.ProjectionExpression = aws.String("id, #state")
	}

	if req.PageToken != "" {
		query.ExclusiveStartKey = map[string]*dynamodb.AttributeValue{
			db.partitionKey: {
				S: aws.String(db.partitionValue),
			},
			"id": {
				S: aws.String(req.PageToken),
			},
		}
	}

	response, err := db.client.QueryWithContext(ctx, query)

	if err != nil {
		return nil, err
	}

	if req.View == tes.TaskView_FULL {
		for _, item := range response.Items {
			// TODO handle errors
			_ = db.getContent(ctx, item)
			_ = db.getExecutorOutput(ctx, item, "stdout", db.stdoutTable)
			_ = db.getExecutorOutput(ctx, item, "stderr", db.stderrTable)
		}
	}

	err = dynamodbattribute.UnmarshalListOfMaps(response.Items, &tasks)
	if err != nil {
		return nil, fmt.Errorf("failed to DynamoDB unmarshal Tasks, %v", err)
	}

	out := tes.ListTasksResponse{
		Tasks: tasks,
	}

	if response.LastEvaluatedKey != nil {
		out.NextPageToken = *response.LastEvaluatedKey["id"].S
	}

	return &out, nil
}
<|endoftext|>


<commit_before>package smux

import (
	"io"
	"sync"
	"sync/atomic"
	"time"

	"github.com/pkg/errors"
)

const (
	defaultAcceptBacklog = 1024
	defaultCloseWait     = 1024
)

const (
	errBrokenPipe      = "broken pipe"
	errInvalidProtocol = "invalid protocol version"
)

// Session defines a multiplexed connection for streams
type Session struct {
	conn      io.ReadWriteCloser
	writeLock sync.Mutex

	config       *Config
	nextStreamID uint32 // next stream identifier

	bucket     int32
	bucketCond *sync.Cond

	streamBuffers map[uint32][]byte  // stream input buffer
	streams       map[uint32]*Stream // all streams in this session
	streamLock    sync.Mutex         // locks streams && frameQueues

	die            chan struct{} // flag session has died
	dieLock        sync.Mutex
	chAccepts      chan *Stream
	chClosedStream chan uint32

	dataReady int32 // flag data has arrived
}

func newSession(config *Config, conn io.ReadWriteCloser, client bool) *Session {
	s := new(Session)
	s.die = make(chan struct{})
	s.conn = conn
	s.config = config
	s.streams = make(map[uint32]*Stream)
	s.streamBuffers = make(map[uint32][]byte)
	s.chAccepts = make(chan *Stream, defaultAcceptBacklog)
	s.chClosedStream = make(chan uint32, defaultCloseWait)
	s.bucket = int32(config.MaxReceiveBuffer)
	s.bucketCond = sync.NewCond(&sync.Mutex{})
	if client {
		s.nextStreamID = 1
	} else {
		s.nextStreamID = 2
	}
	go s.recvLoop()
	go s.monitor()
	go s.keepalive()
	return s
}

// OpenStream is used to create a new stream
func (s *Session) OpenStream() (*Stream, error) {
	if s.IsClosed() {
		return nil, errors.New(errBrokenPipe)
	}

	sid := atomic.AddUint32(&s.nextStreamID, 2)
	stream := newStream(sid, s.config.MaxFrameSize, s)

	s.streamLock.Lock()
	s.streams[sid] = stream
	s.streamLock.Unlock()

	s.writeFrame(newFrame(cmdSYN, sid))
	return stream, nil
}

// AcceptStream is used to block until the next available stream
// is ready to be accepted.
func (s *Session) AcceptStream() (*Stream, error) {
	select {
	case stream := <-s.chAccepts:
		return stream, nil
	case <-s.die:
		return nil, errors.New(errBrokenPipe)
	}
}

// Close is used to close the session and all streams.
func (s *Session) Close() error {
	s.dieLock.Lock()
	defer s.dieLock.Unlock()

	select {
	case <-s.die:
		return errors.New(errBrokenPipe)
	default:
		s.streamLock.Lock()
		for k := range s.streams {
			s.streams[k].Close()
		}
		s.streamLock.Unlock()
		s.writeFrame(newFrame(cmdTerminate, 0))
		s.conn.Close()
		close(s.die)
		s.bucketCond.Signal()
	}
	return nil
}

// IsClosed does a safe check to see if we have shutdown
func (s *Session) IsClosed() bool {
	select {
	case <-s.die:
		return true
	default:
		return false
	}
}

// NumStreams returns the number of currently open streams
func (s *Session) NumStreams() int {
	if s.IsClosed() {
		return 0
	}
	s.streamLock.Lock()
	defer s.streamLock.Unlock()
	return len(s.streams)
}

// notify the session that a stream has closed
func (s *Session) streamClosed(sid uint32) {
	go func() {
		select {
		case s.chClosedStream <- sid:
		case <-s.die:
		}
	}()
}

// nonblocking read from session pool, for streams
func (s *Session) nioread(sid uint32, p []byte) (n int) {
	s.streamLock.Lock()
	n = copy(p, s.streamBuffers[sid])
	if n > 0 {
		s.streamBuffers[sid] = s.streamBuffers[sid][n:]
		atomic.AddInt32(&s.bucket, int32(n))
		s.bucketCond.Signal()
	}
	s.streamLock.Unlock()
	return
}

// session read a frame from underlying connection
func (s *Session) readFrame(buffer []byte) (f Frame, err error) {
	if _, err := io.ReadFull(s.conn, buffer[:headerSize]); err != nil {
		return f, errors.Wrap(err, "readFrame")
	}

	dec := rawHeader(buffer)
	if dec.Version() != version {
		return f, errors.New(errInvalidProtocol)
	}

	if length := dec.Length(); length > 0 {
		if _, err := io.ReadFull(s.conn, buffer[headerSize:headerSize+length]); err != nil {
			return f, errors.Wrap(err, "readFrame")
		}
		f.UnmarshalBinary(buffer[:headerSize+length])
		return f, nil
	}
	f.UnmarshalBinary(buffer[:headerSize])
	return f, nil
}

// monitors streams
func (s *Session) monitor() {
	for {
		select {
		case sid := <-s.chClosedStream:
			s.streamLock.Lock()
			delete(s.streams, sid)
			if n := len(s.streamBuffers[sid]); n > 0 { // return remaining tokens to the bucket
				atomic.AddInt32(&s.bucket, int32(n))
				s.bucketCond.Signal()
			}
			delete(s.streamBuffers, sid)
			s.streamLock.Unlock()
		case <-s.die:
			return
		}
	}
}

// recvLoop keeps on reading from underlying connection if tokens are available
func (s *Session) recvLoop() {
	buffer := make([]byte, (1<<16)+headerSize)
	for {
		s.bucketCond.L.Lock()
		for atomic.LoadInt32(&s.bucket) <= 0 && !s.IsClosed() {
			s.bucketCond.Wait()
		}
		s.bucketCond.L.Unlock()

		if s.IsClosed() {
			return
		}

		if f, err := s.readFrame(buffer); err == nil {
			atomic.StoreInt32(&s.dataReady, 1)

			switch f.cmd {
			case cmdNOP:
			case cmdTerminate:
				s.Close()
				return
			case cmdSYN:
				s.streamLock.Lock()
				if _, ok := s.streams[f.sid]; !ok {
					stream := newStream(f.sid, s.config.MaxFrameSize, s)
					s.streams[f.sid] = stream
					go func() { s.chAccepts <- stream }()
				} else { // stream exists, RST the peer
					go s.writeFrame(newFrame(cmdRST, f.sid))
				}
				s.streamLock.Unlock()
			case cmdRST:
				s.streamLock.Lock()
				if stream, ok := s.streams[f.sid]; ok {
					stream.markRST()
					stream.notifyReadEvent()
				}
				s.streamLock.Unlock()
			case cmdPSH:
				s.streamLock.Lock()
				if stream, ok := s.streams[f.sid]; ok {
					atomic.AddInt32(&s.bucket, -int32(len(f.data)))
					s.streamBuffers[f.sid] = append(s.streamBuffers[f.sid], f.data...)
					stream.notifyReadEvent()
				} else { // stream is absent
					go s.writeFrame(newFrame(cmdRST, f.sid))
				}
				s.streamLock.Unlock()
			default:
				s.Close()
				return
			}
		} else {
			s.Close()
			return
		}
	}
}

func (s *Session) keepalive() {
	tickerPing := time.NewTicker(s.config.KeepAliveInterval)
	tickerTimeout := time.NewTicker(s.config.KeepAliveTimeout)
	defer tickerPing.Stop()
	defer tickerTimeout.Stop()
	for {
		select {
		case <-tickerPing.C:
			s.writeFrame(newFrame(cmdNOP, 0))
		case <-tickerTimeout.C:
			if !atomic.CompareAndSwapInt32(&s.dataReady, 1, 0) &&
				int(atomic.LoadInt32(&s.bucket)) == s.config.MaxReceiveBuffer {
				s.Close()
				return
			}
		case <-s.die:
			return
		}
	}
}

// writeFrame writes the frame to the underlying connection, and returns len(f.data) if successful
func (s *Session) writeFrame(f Frame) (n int, err error) {
	bts, _ := f.MarshalBinary()
	s.writeLock.Lock()
	_, err = s.conn.Write(bts)
	s.writeLock.Unlock()
	return len(f.data), err
}

// writeBinary writes the byte slice to the underlying connection
func (s *Session) writeBinary(bts []byte) (n int, err error) {
	s.writeLock.Lock()
	n, err = s.conn.Write(bts)
	s.writeLock.Unlock()
	return n, err
}
<commit_msg>add comments<commit_after>package smux

import (
	"io"
	"sync"
	"sync/atomic"
	"time"

	"github.com/pkg/errors"
)

const (
	defaultAcceptBacklog = 1024
	defaultCloseWait     = 1024
)

const (
	errBrokenPipe      = "broken pipe"
	errInvalidProtocol = "invalid protocol version"
)

// Session defines a multiplexed connection for streams
type Session struct {
	conn      io.ReadWriteCloser
	writeLock sync.Mutex

	config       *Config
	nextStreamID uint32 // next stream identifier

	bucket     int32      // token bucket
	bucketCond *sync.Cond // used for waiting for tokens

	streamBuffers map[uint32][]byte  // stream input buffer
	streams       map[uint32]*Stream // all streams in this session
	streamLock    sync.Mutex         // locks streams && frameQueues

	die            chan struct{} // flag session has died
	dieLock        sync.Mutex
	chAccepts      chan *Stream
	chClosedStream chan uint32

	dataReady int32 // flag data has arrived
}

func newSession(config *Config, conn io.ReadWriteCloser, client bool) *Session {
	s := new(Session)
	s.die = make(chan struct{})
	s.conn = conn
	s.config = config
	s.streams = make(map[uint32]*Stream)
	s.streamBuffers = make(map[uint32][]byte)
	s.chAccepts = make(chan *Stream, defaultAcceptBacklog)
	s.chClosedStream = make(chan uint32, defaultCloseWait)
	s.bucket = int32(config.MaxReceiveBuffer)
	s.bucketCond = sync.NewCond(&sync.Mutex{})
	if client {
		s.nextStreamID = 1
	} else {
		s.nextStreamID = 2
	}
	go s.recvLoop()
	go s.monitor()
	go s.keepalive()
	return s
}

// OpenStream is used to create a new stream
func (s *Session) OpenStream() (*Stream, error) {
	if s.IsClosed() {
		return nil, errors.New(errBrokenPipe)
	}

	sid := atomic.AddUint32(&s.nextStreamID, 2)
	stream := newStream(sid, s.config.MaxFrameSize, s)

	s.streamLock.Lock()
	s.streams[sid] = stream
	s.streamLock.Unlock()

	s.writeFrame(newFrame(cmdSYN, sid))
	return stream, nil
}

// AcceptStream is used to block until the next available stream
// is ready to be accepted.
func (s *Session) AcceptStream() (*Stream, error) {
	select {
	case stream := <-s.chAccepts:
		return stream, nil
	case <-s.die:
		return nil, errors.New(errBrokenPipe)
	}
}

// Close is used to close the session and all streams.
func (s *Session) Close() error {
	s.dieLock.Lock()
	defer s.dieLock.Unlock()

	select {
	case <-s.die:
		return errors.New(errBrokenPipe)
	default:
		s.streamLock.Lock()
		for k := range s.streams {
			s.streams[k].Close()
		}
		s.streamLock.Unlock()
		s.writeFrame(newFrame(cmdTerminate, 0))
		s.conn.Close()
		close(s.die)
		s.bucketCond.Signal()
	}
	return nil
}

// IsClosed does a safe check to see if we have shutdown
func (s *Session) IsClosed() bool {
	select {
	case <-s.die:
		return true
	default:
		return false
	}
}

// NumStreams returns the number of currently open streams
func (s *Session) NumStreams() int {
	if s.IsClosed() {
		return 0
	}
	s.streamLock.Lock()
	defer s.streamLock.Unlock()
	return len(s.streams)
}

// notify the session that a stream has closed
func (s *Session) streamClosed(sid uint32) {
	go func() {
		select {
		case s.chClosedStream <- sid:
		case <-s.die:
		}
	}()
}

// nonblocking read from session pool, for streams
func (s *Session) nioread(sid uint32, p []byte) (n int) {
	s.streamLock.Lock()
	n = copy(p, s.streamBuffers[sid])
	if n > 0 {
		s.streamBuffers[sid] = s.streamBuffers[sid][n:]
		atomic.AddInt32(&s.bucket, int32(n))
		s.bucketCond.Signal()
	}
	s.streamLock.Unlock()
	return
}

// session read a frame from underlying connection
func (s *Session) readFrame(buffer []byte) (f Frame, err error) {
	if _, err := io.ReadFull(s.conn, buffer[:headerSize]); err != nil {
		return f, errors.Wrap(err, "readFrame")
	}

	dec := rawHeader(buffer)
	if dec.Version() != version {
		return f, errors.New(errInvalidProtocol)
	}

	if length := dec.Length(); length > 0 {
		if _, err := io.ReadFull(s.conn, buffer[headerSize:headerSize+length]); err != nil {
			return f, errors.Wrap(err, "readFrame")
		}
		f.UnmarshalBinary(buffer[:headerSize+length])
		return f, nil
	}
	f.UnmarshalBinary(buffer[:headerSize])
	return f, nil
}

// monitors streams
func (s *Session) monitor() {
	for {
		select {
		case sid := <-s.chClosedStream:
			s.streamLock.Lock()
			delete(s.streams, sid)
			if n := len(s.streamBuffers[sid]); n > 0 { // return remaining tokens to the bucket
				atomic.AddInt32(&s.bucket, int32(n))
				s.bucketCond.Signal()
			}
			delete(s.streamBuffers, sid)
			s.streamLock.Unlock()
		case <-s.die:
			return
		}
	}
}

// recvLoop keeps on reading from underlying connection if tokens are available
func (s *Session) recvLoop() {
	buffer := make([]byte, (1<<16)+headerSize)
	for {
		s.bucketCond.L.Lock()
		for atomic.LoadInt32(&s.bucket) <= 0 && !s.IsClosed() {
			s.bucketCond.Wait()
		}
		s.bucketCond.L.Unlock()

		if s.IsClosed() {
			return
		}

		if f, err := s.readFrame(buffer); err == nil {
			atomic.StoreInt32(&s.dataReady, 1)

			switch f.cmd {
			case cmdNOP:
			case cmdTerminate:
				s.Close()
				return
			case cmdSYN:
				s.streamLock.Lock()
				if _, ok := s.streams[f.sid]; !ok {
					stream := newStream(f.sid, s.config.MaxFrameSize, s)
					s.streams[f.sid] = stream
					go func() { s.chAccepts <- stream }()
				} else { // stream exists, RST the peer
					go s.writeFrame(newFrame(cmdRST, f.sid))
				}
				s.streamLock.Unlock()
			case cmdRST:
				s.streamLock.Lock()
				if stream, ok := s.streams[f.sid]; ok {
					stream.markRST()
					stream.notifyReadEvent()
				}
				s.streamLock.Unlock()
			case cmdPSH:
				s.streamLock.Lock()
				if stream, ok := s.streams[f.sid]; ok {
					atomic.AddInt32(&s.bucket, -int32(len(f.data)))
					s.streamBuffers[f.sid] = append(s.streamBuffers[f.sid], f.data...)
					stream.notifyReadEvent()
				} else { // stream is absent
					go s.writeFrame(newFrame(cmdRST, f.sid))
				}
				s.streamLock.Unlock()
			default:
				s.Close()
				return
			}
		} else {
			s.Close()
			return
		}
	}
}

func (s *Session) keepalive() {
	tickerPing := time.NewTicker(s.config.KeepAliveInterval)
	tickerTimeout := time.NewTicker(s.config.KeepAliveTimeout)
	defer tickerPing.Stop()
	defer tickerTimeout.Stop()
	for {
		select {
		case <-tickerPing.C:
			s.writeFrame(newFrame(cmdNOP, 0))
		case <-tickerTimeout.C:
			if !atomic.CompareAndSwapInt32(&s.dataReady, 1, 0) &&
				int(atomic.LoadInt32(&s.bucket)) == s.config.MaxReceiveBuffer {
				s.Close()
				return
			}
		case <-s.die:
			return
		}
	}
}

// writeFrame writes the frame to the underlying connection, and returns len(f.data) if successful
func (s *Session) writeFrame(f Frame) (n int, err error) {
	bts, _ := f.MarshalBinary()
	s.writeLock.Lock()
	_, err = s.conn.Write(bts)
	s.writeLock.Unlock()
	return len(f.data), err
}

// writeBinary writes the byte slice to the underlying connection
func (s *Session) writeBinary(bts []byte) (n int, err error) {
	s.writeLock.Lock()
	n, err = s.conn.Write(bts)
	s.writeLock.Unlock()
	return n, err
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backup

import (
	"fmt"
	"github.com/jacobsa/comeback/blob"
	"io"
	"io/ioutil"
	"runtime"
)

// An object that knows how to save files to some underlying storage.
type FileSaver interface {
	// Save the contents of the supplied reader to underlying storage, returning
	// a list of scores of blobs that should be concatenated in order to recover
	// its contents.
	Save(r io.Reader) (scores []blob.Score, err error)
}

// Create a file saver that uses the supplied blob store, splitting files into
// chunks of the specified size.
func NewFileSaver(store blob.Store, chunkSize uint32) (FileSaver, error) {
	if chunkSize == 0 {
		return nil, fmt.Errorf("Chunk size must be positive.")
	}

	saver := &fileSaver{blobStore: store, chunkSize: chunkSize}
	startWorkers(saver)
	runtime.SetFinalizer(saver, stopWorkers)

	return saver, nil
}

type fileSaverWork struct {
	index int
	data []byte
	score blob.Score
	err error
	result chan<- fileSaverWork
}

type fileSaver struct {
	blobStore blob.Store
	chunkSize uint32
	work chan<- fileSaverWork
}

func startWorkers(f *fileSaver) {
	work := make(chan fileSaverWork)
	f.work = work

	processWork := func() {
		for item := range work {
			item.score, item.err = f.blobStore.Store(item.data)
			item.result <- item
		}
	}

	const numWorkers = 1
	for i := 0; i < numWorkers; i++ {
		go processWork()
	}
}

func stopWorkers(f *fileSaver) {
	close(f.work)
}

// Read 16 MiB from the supplied reader, returning less iff the reader returns
// an error (including EOF). Do not treat EOF as an error condition.
func getChunk(r io.Reader, chunkSize uint32) ([]byte, error) {
	r = io.LimitReader(r, int64(chunkSize))
	return ioutil.ReadAll(r)
}

func (s *fileSaver) Save(r io.Reader) (scores []blob.Score, err error) {
	// Turn the file into chunks, saving each to the blob store.
	scores = []blob.Score{}
	for {
		chunk, err := getChunk(r, s.chunkSize)
		if err != nil {
			return nil, fmt.Errorf("Reading chunk: %v", err)
		}

		// Are we done?
		if len(chunk) == 0 {
			break
		}

		// Store the chunk.
		score, err := s.blobStore.Store(chunk)
		if err != nil {
			return nil, fmt.Errorf("Storing chunk: %v", err)
		}

		scores = append(scores, score)
	}

	return scores, nil
}
<commit_msg>Run in parallel.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backup

import (
	"fmt"
	"github.com/jacobsa/comeback/blob"
	"io"
	"io/ioutil"
	"runtime"
)

// An object that knows how to save files to some underlying storage.
type FileSaver interface {
	// Save the contents of the supplied reader to underlying storage, returning
	// a list of scores of blobs that should be concatenated in order to recover
	// its contents.
	Save(r io.Reader) (scores []blob.Score, err error)
}

// Create a file saver that uses the supplied blob store, splitting files into
// chunks of the specified size.
func NewFileSaver(store blob.Store, chunkSize uint32) (FileSaver, error) {
	if chunkSize == 0 {
		return nil, fmt.Errorf("Chunk size must be positive.")
	}

	saver := &fileSaver{blobStore: store, chunkSize: chunkSize}
	startWorkers(saver)
	runtime.SetFinalizer(saver, stopWorkers)

	return saver, nil
}

type fileSaverResult struct {
	index int
	score blob.Score
	err error
}

type fileSaverWork struct {
	index int
	data []byte
	resultChan chan<- fileSaverResult
}

type fileSaver struct {
	blobStore blob.Store
	chunkSize uint32
	work chan<- fileSaverWork
}

func startWorkers(s *fileSaver) {
	work := make(chan fileSaverWork)
	s.work = work

	processWork := func() {
		for item := range work {
			score, err := s.blobStore.Store(item.data)
			item.resultChan <- fileSaverResult{item.index, score, err}
		}
	}

	const numWorkers = 1
	for i := 0; i < numWorkers; i++ {
		go processWork()
	}
}

func stopWorkers(s *fileSaver) {
	close(s.work)
}

// Read 16 MiB from the supplied reader, returning less iff the reader returns
// an error (including EOF). Do not treat EOF as an error condition.
func getChunk(r io.Reader, chunkSize uint32) ([]byte, error) {
	r = io.LimitReader(r, int64(chunkSize))
	return ioutil.ReadAll(r)
}

func (s *fileSaver) Save(r io.Reader) (scores []blob.Score, err error) {
	// Turn the file into chunks, giving each to the saver's workers to store in
	// the blob store.
	var numBlobs int
	results := make(chan fileSaverResult)

	for {
		var chunk []byte
		chunk, err = getChunk(r, s.chunkSize)
		if err != nil {
			err = fmt.Errorf("Reading chunk: %v", err)
			return
		}

		// Are we done?
		if len(chunk) == 0 {
			break
		}

		// Add the chunk to the queue of work.
		workItem := fileSaverWork{numBlobs, chunk, results}
		s.work <- workItem
		numBlobs++
	}

	// Read back scores.
	scores = make([]blob.Score, numBlobs)
	for result := range results {
		if result.err != nil {
			err = fmt.Errorf("Storing chunk %d: %v", result.index, result.err)
			return
		}

		scores[result.index] = result.score
	}

	return scores, nil
}
<|endoftext|>


<commit_before>// monitor.go contains methods of s/s/pps/server/api_server.go:APIServer that
// pertain to these fields of APIServer:
//   - monitorCancels,
//   - crashingMonitorCancels,
//   - pollCancel, and
//   - monitorCancelsMu (which protects all of the other fields).
//
// Other functions of APIServer.go should not access any of
// these fields directly (particularly APIServer.monitorCancelsMu, to avoid
// deadlocks) and should instead interact with monitorPipeline and such via
// methods in this file.
//
// Likewise, to avoid reentrancy deadlocks (A -> B -> A), methods in this file
// should avoid calling other methods of APIServer defined outside this file and
// shouldn't call each other.
package server

import (
	"context"
	"time"

	opentracing "github.com/opentracing/opentracing-go"
	log "github.com/sirupsen/logrus"
	"golang.org/x/sync/errgroup"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/errors"
	"github.com/pachyderm/pachyderm/src/client/pkg/tracing"
	"github.com/pachyderm/pachyderm/src/client/pkg/tracing/extended"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/backoff"
	"github.com/pachyderm/pachyderm/src/server/pkg/ppsconsts"
	"github.com/pachyderm/pachyderm/src/server/pkg/ppsutil"
	workerserver "github.com/pachyderm/pachyderm/src/server/worker/server"
)

// startMonitorThread is a helper used by startMonitor, startCrashingMonitor,
// and startPipelinePoller. It doesn't manipulate any of APIServer's fields,
// just wrapps the passed function in a goroutine, and returns a cancel() fn to
// cancel it and block until it returns.
func startMonitorThread(parent context.Context, name string, f func(context.Context)) func() {
	ctx, cancel := context.WithCancel(parent)
	done := make(chan struct{})
	go func() {
		f(ctx)
		close(done)
	}()
	return func() {
		cancel()
		select {
		case <-done:
			return
		case <-time.After(time.Minute):
			// restart pod rather than permanently locking up the PPS master (which
			// would break the PPS API)
			panic(name + " blocked for over a minute after cancellation; restarting pod")
		}
	}
}

// startMonitor starts a new goroutine running monitorPipeline for
// 'pipelineInfo.Pipeline'.
//
// Every running pipeline with standby == true or a cron input has a
// corresponding goroutine running monitorPipeline() that puts the pipeline in
// and out of standby in response to new output commits appearing in that
// pipeline's output repo.
func (a *apiServer) startMonitor(ppsMasterCtx context.Context, pipelineInfo *pps.PipelineInfo) {
	pipeline := pipelineInfo.Pipeline.Name
	a.monitorCancelsMu.Lock()
	defer a.monitorCancelsMu.Unlock()
	if _, ok := a.monitorCancels[pipeline]; !ok {
		a.monitorCancels[pipeline] = startMonitorThread(
			ppsMasterCtx, "monitorPipeline for "+pipeline, func(ctx context.Context) {
				// monitorPipeline needs auth privileges to call subscribeCommit and
				// blockCommit
				// TODO(msteffen): run the pipeline monitor as the pipeline user, rather
				// than as the PPS superuser
				if err := a.sudo(a.env.GetPachClient(ctx),
					func(superUserClient *client.APIClient) error {
						a.monitorPipeline(superUserClient, pipelineInfo)
						return nil
					}); err != nil {
					log.Errorf("error monitoring %q: %v", pipeline, err)
				}
			})
	}
}

// cancelMonitor cancels the monitorPipeline goroutine for 'pipeline'. See
// a.startMonitor().
func (a *apiServer) cancelMonitor(pipeline string) {
	a.monitorCancelsMu.Lock()
	defer a.monitorCancelsMu.Unlock()
	if cancel, ok := a.monitorCancels[pipeline]; ok {
		cancel()
		delete(a.monitorCancels, pipeline)
	}
}

// startCrashingMonitor starts a new goroutine running monitorCrashingPipeline
// for 'pipelineInfo.Pipeline'
//
// Every crashing pipeline has a corresponding goro running
// monitorCrashingPipeline that checks to see if the issues have resolved
// themselves and moves the pipeline out of crashing if they have.
func (a *apiServer) startCrashingMonitor(ppsMasterCtx context.Context, parallelism uint64, pipelineInfo *pps.PipelineInfo) {
	pipeline := pipelineInfo.Pipeline.Name
	a.monitorCancelsMu.Lock()
	defer a.monitorCancelsMu.Unlock()
	if _, ok := a.crashingMonitorCancels[pipeline]; !ok {
		a.crashingMonitorCancels[pipeline] = startMonitorThread(ppsMasterCtx,
			"monitorCrashingPipeline for "+pipeline,
			func(ctx context.Context) {
				a.monitorCrashingPipeline(ctx, parallelism, pipelineInfo)
			})
	}
}

// cancelCrashingMonitor cancels the monitorCrashingPipeline goroutine for
// 'pipeline'. See a.startCrashingMonitor().
func (a *apiServer) cancelCrashingMonitor(pipeline string) {
	a.monitorCancelsMu.Lock()
	defer a.monitorCancelsMu.Unlock()
	if cancel, ok := a.crashingMonitorCancels[pipeline]; ok {
		cancel()
		delete(a.crashingMonitorCancels, pipeline)
	}
}

// cancelAllMonitorsAndCrashingMonitors overlaps with cancelMonitor and
// cancelCrashingMonitor, but also iterates over the existing members of
// a.{crashingM,m}onitorCancels in the critical section, so that all monitors
// can be cancelled without the risk that a new monitor is added between cancels
func (a *apiServer) cancelAllMonitorsAndCrashingMonitors() {
	// cancel all monitorPipeline goroutines
	a.monitorCancelsMu.Lock()
	defer a.monitorCancelsMu.Unlock()
	for _, c := range a.monitorCancels {
		c()
	}
	for _, c := range a.crashingMonitorCancels {
		c()
	}
	a.monitorCancels = make(map[string]func())
	a.crashingMonitorCancels = make(map[string]func())
}

//////////////////////////////////////////////////////////////////////////////
//                     Monitor Functions                                    //
// - These do not lock monitorCancelsMu, but they are called by the         //
//   functions above, which do. They can in turn call each other but cannot //
//   call any of the functions above or any functions outside this file (or //
//   else they will trigger a reentrancy deadlock:                          //
//                 A (lock succeeds) -> B -> A (lock fails)                 //
//////////////////////////////////////////////////////////////////////////////

func (a *apiServer) monitorPipeline(pachClient *client.APIClient, pipelineInfo *pps.PipelineInfo) {
	log.Printf("PPS master: monitoring pipeline %q", pipelineInfo.Pipeline.Name)
	var eg errgroup.Group
	pps.VisitInput(pipelineInfo.Input, func(in *pps.Input) {
		if in.Cron != nil {
			eg.Go(func() error {
				return backoff.RetryNotify(func() error {
					return a.makeCronCommits(pachClient, in)
				}, backoff.NewInfiniteBackOff(), notifyCtx(pachClient.Ctx(), "cron for "+in.Cron.Name))
			})
		}
	})
	if pipelineInfo.Standby {
		// Capacity 1 gives us a bit of buffer so we don't needlessly go into
		// standby when SubscribeCommit takes too long to return.
		ciChan := make(chan *pfs.CommitInfo, 1)
		eg.Go(func() error {
			defer close(ciChan)
			return backoff.RetryNotify(func() error {
				return pachClient.SubscribeCommitF(pipelineInfo.Pipeline.Name, "",
					client.NewCommitProvenance(ppsconsts.SpecRepo, pipelineInfo.Pipeline.Name, pipelineInfo.SpecCommit.ID),
					"", pfs.CommitState_READY, func(ci *pfs.CommitInfo) error {
						ciChan <- ci
						return nil
					})
			}, backoff.NewInfiniteBackOff(), notifyCtx(pachClient.Ctx(), "SubscribeCommit"))
		})
		eg.Go(func() error {
			return backoff.RetryNotify(func() error {
				span, ctx := extended.AddPipelineSpanToAnyTrace(pachClient.Ctx(),
					a.env.GetEtcdClient(), pipelineInfo.Pipeline.Name, "/pps.Master/MonitorPipeline",
					"standby", pipelineInfo.Standby)
				if span != nil {
					pachClient = pachClient.WithCtx(ctx)
				}
				defer tracing.FinishAnySpan(span)

				if err := a.transitionPipelineState(pachClient.Ctx(),
					pipelineInfo.Pipeline.Name,
					pps.PipelineState_PIPELINE_RUNNING,
					pps.PipelineState_PIPELINE_STANDBY, ""); err != nil {

					pte := &ppsutil.PipelineTransitionError{}
					if errors.As(err, &pte) && pte.Current == pps.PipelineState_PIPELINE_PAUSED {
						// pipeline is stopped, exit monitorPipeline (which pausing the
						// pipeline should also do). monitorPipeline will be called when
						// it transitions back to running
						// TODO(msteffen): this should happen in the pipeline
						// controller
						return nil
					}
					return err
				}
				var (
					childSpan     opentracing.Span
					oldCtx        = ctx
					oldPachClient = pachClient
				)
				defer func() {
					tracing.FinishAnySpan(childSpan) // Finish any dangling children of 'span'
				}()
				for {
					// finish span from previous loops
					tracing.FinishAnySpan(childSpan)
					childSpan = nil

					var ci *pfs.CommitInfo
					select {
					case ci = <-ciChan:
						if ci.Finished != nil {
							continue
						}
						childSpan, ctx = tracing.AddSpanToAnyExisting(
							oldCtx, "/pps.Master/MonitorPipeline_SpinUp",
							"pipeline", pipelineInfo.Pipeline.Name, "commit", ci.Commit.ID)
						if childSpan != nil {
							pachClient = oldPachClient.WithCtx(ctx)
						}

						if err := a.transitionPipelineState(pachClient.Ctx(),
							pipelineInfo.Pipeline.Name,
							pps.PipelineState_PIPELINE_STANDBY,
							pps.PipelineState_PIPELINE_RUNNING, ""); err != nil {

							pte := &ppsutil.PipelineTransitionError{}
							if errors.As(err, &pte) && pte.Current == pps.PipelineState_PIPELINE_PAUSED {
								// pipeline is stopped, exit monitorPipeline (see above)
								return nil
							}
							return err
						}

						// Stay running while commits are available
					running:
						for {
							// Wait for the commit to be finished before blocking on the
							// job because the job may not exist yet.
							if _, err := pachClient.BlockCommit(ci.Commit.Repo.Name, ci.Commit.ID); err != nil {
								return err
							}
							if _, err := pachClient.InspectJobOutputCommit(ci.Commit.Repo.Name, ci.Commit.ID, true); err != nil {
								return err
							}

							select {
							case ci = <-ciChan:
							default:
								break running
							}
						}

						if err := a.transitionPipelineState(pachClient.Ctx(),
							pipelineInfo.Pipeline.Name,
							pps.PipelineState_PIPELINE_RUNNING,
							pps.PipelineState_PIPELINE_STANDBY, ""); err != nil {

							pte := &ppsutil.PipelineTransitionError{}
							if errors.As(err, &pte) && pte.Current == pps.PipelineState_PIPELINE_PAUSED {
								// pipeline is stopped; monitorPipeline will be called when it
								// transitions back to running
								// TODO(msteffen): this should happen in the pipeline
								// controller
								return nil
							}
							return err
						}
					case <-pachClient.Ctx().Done():
						return pachClient.Ctx().Err()
					}
				}
			}, backoff.NewInfiniteBackOff(), func(err error, d time.Duration) error {
				select {
				case <-pachClient.Ctx().Done():
					return context.DeadlineExceeded
				default:
					log.Printf("error in monitorPipeline: %v: retrying in: %v", err, d)
				}
				return nil
			})
		})
	}
	if err := eg.Wait(); err != nil {
		log.Printf("error in monitorPipeline: %v", err)
	}
}

// allWorkersUp is a helper used by monitorCrashingPipelinejkjk
func (a *apiServer) allWorkersUp(ctx context.Context, parallelism64 uint64, pipelineInfo *pps.PipelineInfo) (bool, error) {
	parallelism := int(parallelism64)
	if parallelism == 0 {
		parallelism = 1
	}
	workerPoolID := ppsutil.PipelineRcName(pipelineInfo.Pipeline.Name,
		pipelineInfo.Version)
	workerStatus, err := workerserver.Status(ctx, workerPoolID,
		a.env.GetEtcdClient(), a.etcdPrefix, a.workerGrpcPort)
	if err != nil {
		return false, err
	}
	return parallelism == len(workerStatus), nil
}

func (a *apiServer) monitorCrashingPipeline(ctx context.Context, parallelism uint64, pipelineInfo *pps.PipelineInfo) {
For:
	for {
		select {
		case <-time.After(crashingBackoff):
		case <-ctx.Done():
			break For
		}
		time.Sleep(crashingBackoff)
		workersUp, err := a.allWorkersUp(ctx, parallelism, pipelineInfo)
		if err != nil {
			log.Printf("error in monitorCrashingPipeline: %v", err)
			continue
		}
		if workersUp {
			if err := a.transitionPipelineState(ctx, pipelineInfo.Pipeline.Name,
				pps.PipelineState_PIPELINE_CRASHING,
				pps.PipelineState_PIPELINE_RUNNING, ""); err != nil {

				pte := &ppsutil.PipelineTransitionError{}
				if errors.As(err, &pte) && pte.Current == pps.PipelineState_PIPELINE_CRASHING {
					log.Print(err) // Pipeline has moved to STOPPED or been updated--give up
					return
				}
				log.Printf("error in monitorCrashingPipeline: %v", err)
				continue
			}
			break
		}
	}
}
<commit_msg>Refactor monitor fns to use new backoff code<commit_after>// monitor.go contains methods of s/s/pps/server/api_server.go:APIServer that
// pertain to these fields of APIServer:
//   - monitorCancels,
//   - crashingMonitorCancels,
//   - pollCancel, and
//   - monitorCancelsMu (which protects all of the other fields).
//
// Other functions of APIServer.go should not access any of
// these fields directly (particularly APIServer.monitorCancelsMu, to avoid
// deadlocks) and should instead interact with monitorPipeline and such via
// methods in this file.
//
// Likewise, to avoid reentrancy deadlocks (A -> B -> A), methods in this file
// should avoid calling other methods of APIServer defined outside this file and
// shouldn't call each other.
package server

import (
	"context"
	"time"

	opentracing "github.com/opentracing/opentracing-go"
	log "github.com/sirupsen/logrus"
	"golang.org/x/sync/errgroup"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/errors"
	"github.com/pachyderm/pachyderm/src/client/pkg/tracing"
	"github.com/pachyderm/pachyderm/src/client/pkg/tracing/extended"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/backoff"
	"github.com/pachyderm/pachyderm/src/server/pkg/ppsconsts"
	"github.com/pachyderm/pachyderm/src/server/pkg/ppsutil"
	workerserver "github.com/pachyderm/pachyderm/src/server/worker/server"
)

// startMonitorThread is a helper used by startMonitor, startCrashingMonitor,
// and startPipelinePoller. It doesn't manipulate any of APIServer's fields,
// just wrapps the passed function in a goroutine, and returns a cancel() fn to
// cancel it and block until it returns.
func startMonitorThread(parent context.Context, name string, f func(context.Context)) func() {
	ctx, cancel := context.WithCancel(parent)
	done := make(chan struct{})
	go func() {
		f(ctx)
		close(done)
	}()
	return func() {
		cancel()
		select {
		case <-done:
			return
		case <-time.After(time.Minute):
			// restart pod rather than permanently locking up the PPS master (which
			// would break the PPS API)
			panic(name + " blocked for over a minute after cancellation; restarting pod")
		}
	}
}

// startMonitor starts a new goroutine running monitorPipeline for
// 'pipelineInfo.Pipeline'.
//
// Every running pipeline with standby == true or a cron input has a
// corresponding goroutine running monitorPipeline() that puts the pipeline in
// and out of standby in response to new output commits appearing in that
// pipeline's output repo.
func (a *apiServer) startMonitor(ppsMasterCtx context.Context, pipelineInfo *pps.PipelineInfo) {
	pipeline := pipelineInfo.Pipeline.Name
	a.monitorCancelsMu.Lock()
	defer a.monitorCancelsMu.Unlock()
	if _, ok := a.monitorCancels[pipeline]; !ok {
		a.monitorCancels[pipeline] = startMonitorThread(
			ppsMasterCtx, "monitorPipeline for "+pipeline, func(ctx context.Context) {
				// monitorPipeline needs auth privileges to call subscribeCommit and
				// blockCommit
				// TODO(msteffen): run the pipeline monitor as the pipeline user, rather
				// than as the PPS superuser
				if err := a.sudo(a.env.GetPachClient(ctx),
					func(superUserClient *client.APIClient) error {
						a.monitorPipeline(superUserClient, pipelineInfo)
						return nil
					}); err != nil {
					log.Errorf("error monitoring %q: %v", pipeline, err)
				}
			})
	}
}

// cancelMonitor cancels the monitorPipeline goroutine for 'pipeline'. See
// a.startMonitor().
func (a *apiServer) cancelMonitor(pipeline string) {
	a.monitorCancelsMu.Lock()
	defer a.monitorCancelsMu.Unlock()
	if cancel, ok := a.monitorCancels[pipeline]; ok {
		cancel()
		delete(a.monitorCancels, pipeline)
	}
}

// startCrashingMonitor starts a new goroutine running monitorCrashingPipeline
// for 'pipelineInfo.Pipeline'
//
// Every crashing pipeline has a corresponding goro running
// monitorCrashingPipeline that checks to see if the issues have resolved
// themselves and moves the pipeline out of crashing if they have.
func (a *apiServer) startCrashingMonitor(ppsMasterCtx context.Context, parallelism uint64, pipelineInfo *pps.PipelineInfo) {
	pipeline := pipelineInfo.Pipeline.Name
	a.monitorCancelsMu.Lock()
	defer a.monitorCancelsMu.Unlock()
	if _, ok := a.crashingMonitorCancels[pipeline]; !ok {
		a.crashingMonitorCancels[pipeline] = startMonitorThread(ppsMasterCtx,
			"monitorCrashingPipeline for "+pipeline,
			func(ctx context.Context) {
				a.monitorCrashingPipeline(ctx, parallelism, pipelineInfo)
			})
	}
}

// cancelCrashingMonitor cancels the monitorCrashingPipeline goroutine for
// 'pipeline'. See a.startCrashingMonitor().
func (a *apiServer) cancelCrashingMonitor(pipeline string) {
	a.monitorCancelsMu.Lock()
	defer a.monitorCancelsMu.Unlock()
	if cancel, ok := a.crashingMonitorCancels[pipeline]; ok {
		cancel()
		delete(a.crashingMonitorCancels, pipeline)
	}
}

// cancelAllMonitorsAndCrashingMonitors overlaps with cancelMonitor and
// cancelCrashingMonitor, but also iterates over the existing members of
// a.{crashingM,m}onitorCancels in the critical section, so that all monitors
// can be cancelled without the risk that a new monitor is added between cancels
func (a *apiServer) cancelAllMonitorsAndCrashingMonitors() {
	// cancel all monitorPipeline goroutines
	a.monitorCancelsMu.Lock()
	defer a.monitorCancelsMu.Unlock()
	for _, c := range a.monitorCancels {
		c()
	}
	for _, c := range a.crashingMonitorCancels {
		c()
	}
	a.monitorCancels = make(map[string]func())
	a.crashingMonitorCancels = make(map[string]func())
}

//////////////////////////////////////////////////////////////////////////////
//                     Monitor Functions                                    //
// - These do not lock monitorCancelsMu, but they are called by the         //
//   functions above, which do. They can in turn call each other but cannot //
//   call any of the functions above or any functions outside this file (or //
//   else they will trigger a reentrancy deadlock:                          //
//                 A (lock succeeds) -> B -> A (lock fails)                 //
//////////////////////////////////////////////////////////////////////////////

func (a *apiServer) monitorPipeline(pachClient *client.APIClient, pipelineInfo *pps.PipelineInfo) {
	log.Printf("PPS master: monitoring pipeline %q", pipelineInfo.Pipeline.Name)
	var eg errgroup.Group
	pps.VisitInput(pipelineInfo.Input, func(in *pps.Input) {
		if in.Cron != nil {
			eg.Go(func() error {
				return backoff.RetryNotify(func() error {
					return a.makeCronCommits(pachClient, in)
				}, backoff.NewInfiniteBackOff(), notifyCtx(pachClient.Ctx(), "cron for "+in.Cron.Name))
			})
		}
	})
	if pipelineInfo.Standby {
		// Capacity 1 gives us a bit of buffer so we don't needlessly go into
		// standby when SubscribeCommit takes too long to return.
		ciChan := make(chan *pfs.CommitInfo, 1)
		eg.Go(func() error {
			defer close(ciChan)
			return backoff.RetryNotify(func() error {
				return pachClient.SubscribeCommitF(pipelineInfo.Pipeline.Name, "",
					client.NewCommitProvenance(ppsconsts.SpecRepo, pipelineInfo.Pipeline.Name, pipelineInfo.SpecCommit.ID),
					"", pfs.CommitState_READY, func(ci *pfs.CommitInfo) error {
						ciChan <- ci
						return nil
					})
			}, backoff.NewInfiniteBackOff(), notifyCtx(pachClient.Ctx(), "SubscribeCommit"))
		})
		eg.Go(func() error {
			return backoff.RetryNotify(func() error {
				span, ctx := extended.AddPipelineSpanToAnyTrace(pachClient.Ctx(),
					a.env.GetEtcdClient(), pipelineInfo.Pipeline.Name, "/pps.Master/MonitorPipeline",
					"standby", pipelineInfo.Standby)
				if span != nil {
					pachClient = pachClient.WithCtx(ctx)
				}
				defer tracing.FinishAnySpan(span)

				if err := a.transitionPipelineState(pachClient.Ctx(),
					pipelineInfo.Pipeline.Name,
					pps.PipelineState_PIPELINE_RUNNING,
					pps.PipelineState_PIPELINE_STANDBY, ""); err != nil {

					pte := &ppsutil.PipelineTransitionError{}
					if errors.As(err, &pte) && pte.Current == pps.PipelineState_PIPELINE_PAUSED {
						// pipeline is stopped, exit monitorPipeline (which pausing the
						// pipeline should also do). monitorPipeline will be called when
						// it transitions back to running
						// TODO(msteffen): this should happen in the pipeline
						// controller
						return nil
					}
					return err
				}
				var (
					childSpan     opentracing.Span
					oldCtx        = ctx
					oldPachClient = pachClient
				)
				defer func() {
					tracing.FinishAnySpan(childSpan) // Finish any dangling children of 'span'
				}()
				for {
					// finish span from previous loops
					tracing.FinishAnySpan(childSpan)
					childSpan = nil

					var ci *pfs.CommitInfo
					select {
					case ci = <-ciChan:
						if ci.Finished != nil {
							continue
						}
						childSpan, ctx = tracing.AddSpanToAnyExisting(
							oldCtx, "/pps.Master/MonitorPipeline_SpinUp",
							"pipeline", pipelineInfo.Pipeline.Name, "commit", ci.Commit.ID)
						if childSpan != nil {
							pachClient = oldPachClient.WithCtx(ctx)
						}

						if err := a.transitionPipelineState(pachClient.Ctx(),
							pipelineInfo.Pipeline.Name,
							pps.PipelineState_PIPELINE_STANDBY,
							pps.PipelineState_PIPELINE_RUNNING, ""); err != nil {

							pte := &ppsutil.PipelineTransitionError{}
							if errors.As(err, &pte) && pte.Current == pps.PipelineState_PIPELINE_PAUSED {
								// pipeline is stopped, exit monitorPipeline (see above)
								return nil
							}
							return err
						}

						// Stay running while commits are available
					running:
						for {
							// Wait for the commit to be finished before blocking on the
							// job because the job may not exist yet.
							if _, err := pachClient.BlockCommit(ci.Commit.Repo.Name, ci.Commit.ID); err != nil {
								return err
							}
							if _, err := pachClient.InspectJobOutputCommit(ci.Commit.Repo.Name, ci.Commit.ID, true); err != nil {
								return err
							}

							select {
							case ci = <-ciChan:
							default:
								break running
							}
						}

						if err := a.transitionPipelineState(pachClient.Ctx(),
							pipelineInfo.Pipeline.Name,
							pps.PipelineState_PIPELINE_RUNNING,
							pps.PipelineState_PIPELINE_STANDBY, ""); err != nil {

							pte := &ppsutil.PipelineTransitionError{}
							if errors.As(err, &pte) && pte.Current == pps.PipelineState_PIPELINE_PAUSED {
								// pipeline is stopped; monitorPipeline will be called when it
								// transitions back to running
								// TODO(msteffen): this should happen in the pipeline
								// controller
								return nil
							}
							return err
						}
					case <-pachClient.Ctx().Done():
						return pachClient.Ctx().Err()
					}
				}
			}, backoff.NewInfiniteBackOff(),
				backoff.NotifyCtx(pachClient.Ctx(), "monitorPipeline for "+pipelineInfo.Pipeline.Name))
		})
	}
	if err := eg.Wait(); err != nil {
		log.Printf("error in monitorPipeline: %v", err)
	}
}

// allWorkersUp is a helper used by monitorCrashingPipelinejkjk
func (a *apiServer) allWorkersUp(ctx context.Context, parallelism64 uint64, pipelineInfo *pps.PipelineInfo) (bool, error) {
	parallelism := int(parallelism64)
	if parallelism == 0 {
		parallelism = 1
	}
	workerPoolID := ppsutil.PipelineRcName(pipelineInfo.Pipeline.Name,
		pipelineInfo.Version)
	workerStatus, err := workerserver.Status(ctx, workerPoolID,
		a.env.GetEtcdClient(), a.etcdPrefix, a.workerGrpcPort)
	if err != nil {
		return false, err
	}
	return parallelism == len(workerStatus), nil
}

func (a *apiServer) monitorCrashingPipeline(ctx context.Context, parallelism uint64, pipelineInfo *pps.PipelineInfo) {
	pipeline := pipelineInfo.Pipeline.Name
	if err := backoff.RetryUntilCancel(ctx,
		func() error {
			workersUp, err := a.allWorkersUp(ctx, parallelism, pipelineInfo)
			if err != nil {
				return errors.Wrap(err, "could not check if all workers are up")
			}
			if workersUp {
				if err := a.transitionPipelineState(ctx, pipeline,
					pps.PipelineState_PIPELINE_CRASHING,
					pps.PipelineState_PIPELINE_RUNNING, ""); err != nil {

					pte := &ppsutil.PipelineTransitionError{}
					if errors.As(err, &pte) && pte.Current == pps.PipelineState_PIPELINE_CRASHING {
						log.Error(err)
						return nil // Pipeline has moved to STOPPED or been updated--give up
					}
					return errors.Wrap(err, "could not transition pipeline to RUNNING")
				}
			}
			return nil
		},
		&backoff.ZeroBackOff{},
		backoff.NotifyContinue("monitorCrashingPipeline for "+pipeline),
	); err != nil && ctx.Err() == nil {
		// retryUntilCancel should exit iff 'ctx' is cancelled, so this should be
		// unreachable (restart master if not)
		panic("monitorCrashingPipeline is exiting early, this should never happen")
	}

}
<|endoftext|>


<commit_before>package reactor

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"path/filepath"
	"strings"

	"github.com/kirillrdy/nadeshiko/html"
	"github.com/sparkymat/reactor/property"
	"github.com/sparkymat/webdsl/css"
)

type fileMapping struct {
	filePath string
	webPath  string
}

type reactor struct {
	name              string
	javascriptFolders []fileMapping
	cssFolders        []fileMapping
	props             map[string]Property
}

func New(name string) reactor {
	r := reactor{name: name}
	r.props = make(map[string]Property)
	return r
}

func (r *reactor) SetStringProperty(name string, value string) {
	property := Property{propertyType: property.String, value: value}
	r.props[name] = property
}

func (r *reactor) SetIntegerProperty(name string, value int64) {
	property := Property{propertyType: property.Integer, value: fmt.Sprintf("%v", value)}
	r.props[name] = property
}

func (r *reactor) SetFloatProperty(name string, value float64) {
	property := Property{propertyType: property.Float, value: fmt.Sprintf("%v", value)}
	r.props[name] = property
}

func (r *reactor) SetObjectProperty(name string, value interface{}) {
	jsonValue, _ := json.Marshal(value)

	// FIXME: Ignoring error for now
	property := Property{propertyType: property.Object, value: strings.Replace(string(jsonValue), "\"", "\\\"\\", -1)}
	r.props[name] = property
}

func (r *reactor) MapJavascriptFolder(file string, web string) {
	r.javascriptFolders = append(r.javascriptFolders, fileMapping{filePath: file, webPath: web})
}

func (r *reactor) MapCssFolder(file string, web string) {
	r.cssFolders = append(r.cssFolders, fileMapping{filePath: file, webPath: web})
}

func (r reactor) Html() html.Node {
	headNodes := []html.Node{}

	headNodes = append(headNodes, html.Title().Text(r.name))

	// List javascript files
	for _, jsMap := range r.javascriptFolders {
		files, err := ioutil.ReadDir(jsMap.filePath)
		if err == nil {
			for _, jsFile := range files {
				if filepath.Ext(jsFile.Name()) == ".js" {
					finalPath := fmt.Sprintf("/%v/%v", jsMap.webPath, jsFile.Name())
					headNodes = append(headNodes, html.Script().Type("text/javascript").Src(finalPath))
				}
			}
		}
	}

	// List css files
	for _, cssMap := range r.cssFolders {
		files, err := ioutil.ReadDir(cssMap.filePath)

		if err == nil {
			for _, cssFile := range files {
				if filepath.Ext(cssFile.Name()) == ".css" {
					finalPath := fmt.Sprintf("/%v/%v", cssMap.webPath, cssFile.Name())
					headNodes = append(headNodes, html.Link().Rel("stylesheet").Href(finalPath))
				}
			}
		}
	}

	propsList := []string{}

	if r.props != nil {
		for key, value := range r.props {
			propsList = append(propsList, fmt.Sprintf("%v: %v", key, value.String()))
		}
	}

	propsString := fmt.Sprintf("{%v}", strings.Join(propsList, ","))

	return html.Html().Children(
		html.Head().Children(headNodes...),
		html.Body().Children(
			html.Div().Id(css.Id("app-container")),
			html.Script().TextUnsafe(fmt.Sprintf("React.render(React.createElement(%v, %v), document.getElementById('app-container'));", r.name, propsString)),
		),
	)
}
<commit_msg>Look at me. I render now.<commit_after>package reactor

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"path/filepath"
	"strings"

	"github.com/kirillrdy/nadeshiko/html"
	"github.com/sparkymat/reactor/property"
	"github.com/sparkymat/webdsl/css"
)

type fileMapping struct {
	filePath string
	webPath  string
}

type reactor struct {
	name              string
	javascriptFolders []fileMapping
	cssFolders        []fileMapping
	props             map[string]Property
}

func New(name string) reactor {
	r := reactor{name: name}
	r.props = make(map[string]Property)
	return r
}

func (r *reactor) SetStringProperty(name string, value string) {
	property := Property{propertyType: property.String, value: value}
	r.props[name] = property
}

func (r *reactor) SetIntegerProperty(name string, value int64) {
	property := Property{propertyType: property.Integer, value: fmt.Sprintf("%v", value)}
	r.props[name] = property
}

func (r *reactor) SetFloatProperty(name string, value float64) {
	property := Property{propertyType: property.Float, value: fmt.Sprintf("%v", value)}
	r.props[name] = property
}

func (r *reactor) SetObjectProperty(name string, value interface{}) {
	jsonValue, _ := json.Marshal(value)

	// FIXME: Ignoring error for now
	property := Property{propertyType: property.Object, value: strings.Replace(string(jsonValue), "\"", "\\\"\\", -1)}
	r.props[name] = property
}

func (r *reactor) MapJavascriptFolder(file string, web string) {
	r.javascriptFolders = append(r.javascriptFolders, fileMapping{filePath: file, webPath: web})
}

func (r *reactor) MapCssFolder(file string, web string) {
	r.cssFolders = append(r.cssFolders, fileMapping{filePath: file, webPath: web})
}

func (r reactor) Html() html.Node {
	headNodes := []html.Node{}

	headNodes = append(headNodes, html.Title().Text(r.name))

	// List javascript files
	for _, jsMap := range r.javascriptFolders {
		files, err := ioutil.ReadDir(jsMap.filePath)
		if err == nil {
			for _, jsFile := range files {
				if filepath.Ext(jsFile.Name()) == ".js" {
					finalPath := fmt.Sprintf("/%v/%v", jsMap.webPath, jsFile.Name())
					headNodes = append(headNodes, html.Script().Type("text/javascript").Src(finalPath))
				}
			}
		}
	}

	// List css files
	for _, cssMap := range r.cssFolders {
		files, err := ioutil.ReadDir(cssMap.filePath)

		if err == nil {
			for _, cssFile := range files {
				if filepath.Ext(cssFile.Name()) == ".css" {
					finalPath := fmt.Sprintf("/%v/%v", cssMap.webPath, cssFile.Name())
					headNodes = append(headNodes, html.Link().Rel("stylesheet").Href(finalPath))
				}
			}
		}
	}

	propsList := []string{}

	if r.props != nil {
		for key, value := range r.props {
			propsList = append(propsList, fmt.Sprintf("%v: %v", key, value.String()))
		}
	}

	propsString := fmt.Sprintf("{%v}", strings.Join(propsList, ","))

	return html.Html().Children(
		html.Head().Children(headNodes...),
		html.Body().Children(
			html.Div().Id(css.Id("app-container")),
			html.Script().TextUnsafe(fmt.Sprintf("ReactDOM.render(React.createElement(%v, %v), document.getElementById('app-container'));", r.name, propsString)),
		),
	)
}
<|endoftext|>


<commit_before>package models

import "errors"

type ResultEnum int

const (
	SUCCESS ResultEnum = iota
	FAIL
)

func (r ResultEnum) MarshalJSON() ([]byte, error) {
	switch r {
	case SUCCESS:
		return []byte("success"), nil
	case FAIL:
		return []byte("fail"), nil
	}

	return nil, errors.New("Uknown value of ResultEnum: " + r)
}

func (r ResultEnum) UnmarshalJSON(data []byte) error {
	switch data {
	case []byte("success"):
		r = SUCCESS
		return nil
	case []byte("fail"):
		r = FAIL
		return nil
	}

	return errors.New("Uknown value for ResultEnum: " + data)
}
<commit_msg>Simplified models/result.go marshal method<commit_after>package models

import "errors"

type ResultEnum int

const (
	SUCCESS ResultEnum = iota
	FAIL
)

func (r ResultEnum) MarshalJSON() ([]byte, error) {
	if r == SUCCESS || r == FAIL {
		return []byte(r.String()), nil
	}

	return nil, errors.New("Uknown value of ResultEnum: " + r)
}

func (r ResultEnum) UnmarshalJSON(data []byte) error {
	switch data {
	case []byte("success"):
		r = SUCCESS
		return nil
	case []byte("fail"):
		r = FAIL
		return nil
	}

	return errors.New("Uknown value for ResultEnum: " + data)
}

func (r ResultEnum) String() string {
	switch r {
	case SUCCESS:
		return "success"
	case FAIL:
		return "fail"
	default:
		return "!INVALID VALUE (value: " + r + ")!"
	}
}<|endoftext|>


<commit_before>package main

import (
	"bsearch/index"
	"bsearch/parser"
	"flag"
	"fmt"
	"os"
	"log"
 	"runtime/pprof"
	"net"
)

func usage() {
	fmt.Fprintf(os.Stderr, "usage: bsearch <path to index blob>\n")
	flag.PrintDefaults()
	os.Exit(1)
}

type headers map[string]string

func (h headers) Add(k, v string) {
	h[k] = v
}

var cpuprofile = flag.String("cpuprofile", "", "Write cpu profile to file")
var listenport = flag.String("listen", "", "TCP port to listen to")

func main() {
	flag.Parse()
	if flag.NArg() != 1 || listenport == nil {
		usage()
	}
	dbname := flag.Arg(0)
	in, err := index.Open(dbname)
	if err != nil {
		log.Fatal(os.Stderr, "bindex.Open: %v\n", err)
	}
	defer in.Close()

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	ln, err := net.Listen("tcp", *listenport)
	if err != nil {
		log.Fatal("listen: %v\n", err)
	}

	for {
		conn, err := ln.Accept()
		if err != nil {
			log.Fatal("accept %v\n", err)
		}
		go handle(conn, in)
	}
}

func handle(conn net.Conn, in *index.Index) {
	b := [2048]byte{}
	n, err := conn.Read(b[:])
	if err != nil {
		log.Fatal("conn.Read %v\n", err)
	}
	bq := b[:n]
	log.Printf("query: %v\n", string(bq))
	p := parser.Parse(in, string(bq))
	q := p.Stack[0]

	docarr := make([]*index.IbDoc, 0)

	s := index.NullDoc()
	for true {
		d := q.NextDoc(s)
		if d == nil {
			break
		}
		docarr = append(docarr, d)
		*s = *d
		s.Inc()
	}
	h := make(headers)
	q.ProcessHeaders(h)
	for k, v := range h {
		conn.Write([]byte(fmt.Sprintf("info:%v:%v\n", k, v)))
	}
	conn.Write([]byte(in.Header()))
	conn.Write([]byte("\n"))
	log.Printf("len: %v", len(docarr))
	for o, d := range docarr {
		if d == nil {
			log.Fatalf("nil in docarr at %v", o)
		}
		doc, exists := in.Docs[d.Id]
		if !exists {
			log.Printf("Doc %v does not exist", d.Id)
		}
		conn.Write(doc)
		conn.Write([]byte("\n"))
	}
}
<commit_msg>Log query timing data.<commit_after>package main

import (
	"bsearch/index"
	"bsearch/parser"
	"flag"
	"fmt"
	"os"
	"log"
 	"runtime/pprof"
	"net"
	"time"
)

func usage() {
	fmt.Fprintf(os.Stderr, "usage: bsearch <path to index blob>\n")
	flag.PrintDefaults()
	os.Exit(1)
}

type headers map[string]string

func (h headers) Add(k, v string) {
	h[k] = v
}

var cpuprofile = flag.String("cpuprofile", "", "Write cpu profile to file")
var listenport = flag.String("listen", "", "TCP port to listen to")

func main() {
	flag.Parse()
	if flag.NArg() != 1 || listenport == nil {
		usage()
	}
	dbname := flag.Arg(0)
	in, err := index.Open(dbname)
	if err != nil {
		log.Fatal(os.Stderr, "bindex.Open: %v\n", err)
	}
	defer in.Close()

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	ln, err := net.Listen("tcp", *listenport)
	if err != nil {
		log.Fatal("listen: %v\n", err)
	}

	for {
		conn, err := ln.Accept()
		if err != nil {
			log.Fatal("accept %v\n", err)
		}
		go handle(conn, in)
	}
}

func handle(conn net.Conn, in *index.Index) {
	b := [2048]byte{}
	n, err := conn.Read(b[:])
	if err != nil {
		log.Fatal("conn.Read %v\n", err)
	}
	bq := b[:n]
	log.Printf("query: %v\n", string(bq))
	t1 := time.Now()
	p := parser.Parse(in, string(bq))
	q := p.Stack[0]

	docarr := make([]*index.IbDoc, 0)

	s := index.NullDoc()
	for true {
		d := q.NextDoc(s)
		if d == nil {
			break
		}
		docarr = append(docarr, d)
		*s = *d
		s.Inc()
	}
	h := make(headers)
	q.ProcessHeaders(h)
	t2 := time.Now()

	log.Printf("Query time: %v\n", t2.Sub(t1))

	for k, v := range h {
		conn.Write([]byte(fmt.Sprintf("info:%v:%v\n", k, v)))
	}
	conn.Write([]byte(in.Header()))
	conn.Write([]byte("\n"))
	for o, d := range docarr {
		if d == nil {
			log.Fatalf("nil in docarr at %v", o)
		}
		doc, exists := in.Docs[d.Id]
		if !exists {
			log.Printf("Doc %v does not exist", d.Id)
		}
		conn.Write(doc)
		conn.Write([]byte("\n"))
	}
}
<|endoftext|>


<commit_before>package runner

import (
	"fmt"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"syscall"
	"time"

	"github.com/cloudfoundry-incubator/garden/client"
	"github.com/cloudfoundry-incubator/garden/client/connection"
	"github.com/cloudfoundry-incubator/garden/warden"
	"github.com/onsi/ginkgo"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"
)

type Runner struct {
	Command *exec.Cmd

	network string
	addr    string

	bin  string
	argv []string

	binPath    string
	rootFSPath string

	tmpdir string
}

func New(network, addr string, bin, binPath, rootFSPath string, argv ...string) *Runner {
	return &Runner{
		network: network,
		addr:    addr,

		bin:  bin,
		argv: argv,

		binPath:    binPath,
		rootFSPath: rootFSPath,

		tmpdir: filepath.Join(
			os.TempDir(),
			fmt.Sprintf("test-warden-%d", ginkgo.GinkgoParallelNode()),
		),
	}
}

func (r *Runner) Run(signals <-chan os.Signal, ready chan<- struct{}) error {
	logger := lager.NewLogger("warden-runner")
	logger.RegisterSink(lager.NewWriterSink(ginkgo.GinkgoWriter, lager.DEBUG))

	err := os.MkdirAll(r.tmpdir, 0755)
	if err != nil {
		return err
	}

	depotPath := filepath.Join(r.tmpdir, "containers")
	overlaysPath := filepath.Join(r.tmpdir, "overlays")
	snapshotsPath := filepath.Join(r.tmpdir, "snapshots")
	graphPath := filepath.Join(r.tmpdir, "graph")

	if err := os.MkdirAll(depotPath, 0755); err != nil {
		return err
	}

	if err := os.MkdirAll(snapshotsPath, 0755); err != nil {
		return err
	}

	wardenArgs := append(
		r.argv,
		"--listenNetwork", r.network,
		"--listenAddr", r.addr,
		"--bin", r.binPath,
		"--rootfs", r.rootFSPath,
		"--depot", depotPath,
		"--overlays", overlaysPath,
		"--snapshots", snapshotsPath,
		"--graph", graphPath,
		"--logLevel", "debug",
		"--disableQuotas",
		"--networkPool", fmt.Sprintf("10.250.%d.0/24", ginkgo.GinkgoParallelNode()),
		"--portPoolStart", strconv.Itoa(51000+(1000*ginkgo.GinkgoParallelNode())),
		"--portPoolSize", "1000",
		"--uidPoolStart", strconv.Itoa(10000*ginkgo.GinkgoParallelNode()),
		"--tag", strconv.Itoa(ginkgo.GinkgoParallelNode()),
	)

	var signal os.Signal

	r.Command = exec.Command(r.bin, wardenArgs...)

	process := ifrit.Envoke(&ginkgomon.Runner{
		Name:              "warden-linux",
		Command:           r.Command,
		AnsiColorCode:     "31m",
		StartCheck:        "warden-linux.started",
		StartCheckTimeout: 10 * time.Second,
		Cleanup: func() {
			if signal == syscall.SIGKILL {
				logger.Info("removing-tmp-dirs")
				if err := os.RemoveAll(r.tmpdir); err != nil {
					logger.Error("cleanup-tempdirs-failed", err, lager.Data{"tmpdir": r.tmpdir})
				} else {
					logger.Info("tmp-dirs-removed")
				}
			}
		},
	})

	close(ready)

	var waitErr error

dance:
	for {
		select {
		case signal = <-signals:
			if signal == syscall.SIGKILL {
				logger.Info("received-sigkill")
				if err := r.destroyContainers(); err != nil {
					logger.Error("destroy-containers-failed", err)
					return err
				}
				logger.Info("destroyed-containers")
			}

			process.Signal(syscall.SIGTERM)
		case waitErr = <-process.Wait():
			break dance
		}
	}

	logger.Info("process-exited")

	return waitErr
}

func (r *Runner) TryDial() error {
	conn, dialErr := net.DialTimeout(r.network, r.addr, 100*time.Millisecond)

	if dialErr == nil {
		conn.Close()
		return nil
	}

	return dialErr
}

func (r *Runner) NewClient() warden.Client {
	return client.New(connection.New(r.network, r.addr))
}

func (r *Runner) destroyContainers() error {
	client := r.NewClient()

	containers, err := client.Containers(nil)
	if err != nil {
		return err
	}

	for _, container := range containers {
		err := client.Destroy(container.Handle())
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>runner takes graphPath as argument<commit_after>package runner

import (
	"fmt"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"syscall"
	"time"

	"github.com/cloudfoundry-incubator/garden/client"
	"github.com/cloudfoundry-incubator/garden/client/connection"
	"github.com/cloudfoundry-incubator/garden/warden"
	"github.com/onsi/ginkgo"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"
)

type Runner struct {
	Command *exec.Cmd

	network string
	addr    string

	bin  string
	argv []string

	binPath    string
	rootFSPath string

	tmpdir    string
	graphPath string
}

func New(network, addr string, bin, binPath, rootFSPath, graphPath string, argv ...string) *Runner {
	return &Runner{
		network: network,
		addr:    addr,

		bin:  bin,
		argv: argv,

		binPath:    binPath,
		rootFSPath: rootFSPath,
		graphPath:  filepath.Join(graphPath, fmt.Sprintf("test-warden-%d", ginkgo.GinkgoParallelNode())),
		tmpdir: filepath.Join(
			os.TempDir(),
			fmt.Sprintf("test-warden-%d", ginkgo.GinkgoParallelNode()),
		),
	}
}

func (r *Runner) Run(signals <-chan os.Signal, ready chan<- struct{}) error {
	logger := lager.NewLogger("warden-runner")
	logger.RegisterSink(lager.NewWriterSink(ginkgo.GinkgoWriter, lager.DEBUG))

	err := os.MkdirAll(r.tmpdir, 0755)
	if err != nil {
		return err
	}

	depotPath := filepath.Join(r.tmpdir, "containers")
	overlaysPath := filepath.Join(r.tmpdir, "overlays")
	snapshotsPath := filepath.Join(r.tmpdir, "snapshots")

	if err := os.MkdirAll(depotPath, 0755); err != nil {
		return err
	}

	if err := os.MkdirAll(snapshotsPath, 0755); err != nil {
		return err
	}

	wardenArgs := append(
		r.argv,
		"--listenNetwork", r.network,
		"--listenAddr", r.addr,
		"--bin", r.binPath,
		"--rootfs", r.rootFSPath,
		"--depot", depotPath,
		"--overlays", overlaysPath,
		"--snapshots", snapshotsPath,
		"--graph", r.graphPath,
		"--logLevel", "debug",
		"--disableQuotas",
		"--networkPool", fmt.Sprintf("10.250.%d.0/24", ginkgo.GinkgoParallelNode()),
		"--portPoolStart", strconv.Itoa(51000+(1000*ginkgo.GinkgoParallelNode())),
		"--portPoolSize", "1000",
		"--uidPoolStart", strconv.Itoa(10000*ginkgo.GinkgoParallelNode()),
		"--tag", strconv.Itoa(ginkgo.GinkgoParallelNode()),
	)

	var signal os.Signal

	r.Command = exec.Command(r.bin, wardenArgs...)

	process := ifrit.Envoke(&ginkgomon.Runner{
		Name:              "warden-linux",
		Command:           r.Command,
		AnsiColorCode:     "31m",
		StartCheck:        "warden-linux.started",
		StartCheckTimeout: 10 * time.Second,
		Cleanup: func() {
			if signal == syscall.SIGKILL {
				logger.Info("removing-tmp-dirs")
				if err := os.RemoveAll(r.tmpdir); err != nil {
					logger.Error("cleanup-tempdirs-failed", err, lager.Data{"tmpdir": r.tmpdir})
				} else {
					logger.Info("tmp-dirs-removed")
				}
			}
		},
	})

	close(ready)

	var waitErr error

dance:
	for {
		select {
		case signal = <-signals:
			if signal == syscall.SIGKILL {
				logger.Info("received-sigkill")
				if err := r.destroyContainers(); err != nil {
					logger.Error("destroy-containers-failed", err)
					return err
				}
				logger.Info("destroyed-containers")
			}

			process.Signal(syscall.SIGTERM)
		case waitErr = <-process.Wait():
			break dance
		}
	}

	logger.Info("process-exited")

	return waitErr
}

func (r *Runner) TryDial() error {
	conn, dialErr := net.DialTimeout(r.network, r.addr, 100*time.Millisecond)

	if dialErr == nil {
		conn.Close()
		return nil
	}

	return dialErr
}

func (r *Runner) NewClient() warden.Client {
	return client.New(connection.New(r.network, r.addr))
}

func (r *Runner) destroyContainers() error {
	client := r.NewClient()

	containers, err := client.Containers(nil)
	if err != nil {
		return err
	}

	for _, container := range containers {
		err := client.Destroy(container.Handle())
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before><commit_msg>remove limit<commit_after><|endoftext|>


<commit_before>package hpcloud

/*
 BadRequest describes the response from a JSON resource when the
 data which was sent in the original request was malformed or not
 compliant with the layout specified in the HPCloud documentation
*/
type BadRequest struct {
	B struct {
		Message string `json:"message"`
		Details string `json:"details"`
		Code    int64  `json:"code"`
	} `json:"BadRequest"`
}

/*
 NotFound describes the response from a JSON resource when the
 resource which was interacted with in the original request was
 not able to be found.
*/
type NotFound struct {
	NF struct {
		Message string `json:"message"`
		Code    int    `json:"code"`
	} `json:"itemNotFound"`
}

/*
 Unauthorized describes the response from a JSON resource when the
 request could not be completed due to none or incorrect authentication
 was used to make the request.
*/
type Unauthorized struct {
	U struct {
		Code            int64  `json:"code"`
		Details         string `json:"details"`
		Message         string `json:"message"`
		OtherAttributes struct {
		} `json:"otherAttributes"`
	} `json:"unauthorized"`
}

/*
 Forbidden describes the response from a JSON resource when the
 request could not be completed due to the user making the request
 being disabled or suspended.
*/
type Forbidden struct {
	F struct {
		Code            int64  `json:"code"`
		Details         string `json:"details"`
		Message         string `json:"message"`
		OtherAttributes struct {
		} `json:"otherAttributes"`
	} `json:"forbidden"`
}

/*
 InternalServerError describes the response from a JSON resource when the
 request could not be completed due to the request causing the service
 to return a 500 status code.
*/
type InternalServerError struct {
	ISE struct {
		Code            int64  `json:"code"`
		Details         string `json:"details"`
		Message         string `json:"message"`
		OtherAttributes struct {
		} `json:"otherAttributes"`
	} `json:"internalServerError"`
}

type SubToken struct {
	ID string `json:"id"`
}

type Flavor_ struct {
	Name  string `json:"name"`
	ID    int64  `json:"id"`
	Links []Link `json:"links"`
}

type Flavors struct {
	F []Flavor_ `json:"flavors"`
}

type Tenants struct {
	T []Tenant `json:"tenants"`
}

type FailureResponse interface {
	Code() int64
	Details() string
	Message() string
}

func (u Unauthorized) Code() int64 {
	return u.U.Code
}

func (b BadRequest) Code() int64 {
	return b.B.Code
}

func (f Forbidden) Code() int64 {
	return f.F.Code
}

func (nf NotFound) Code() int64 {
	return nf.NF.Code
}

func (ise InternalServerError) Code() int64 {
	return ise.ISE.Code
}

func (u Unauthorized) Details() string {
	return u.U.Details
}

func (b BadRequest) Details() string {
	return b.B.Details
}

func (f Forbidden) Details() string {
	return f.F.Details
}

func (ise InternalServerError) Details() string {
	return ise.ISE.Details
}

func (nf NotFound) Details() string {
	return nf.NF.Details
}

func (u Unauthorized) Message() string {
	return u.U.Message
}

func (b BadRequest) Message() string {
	return b.B.Message
}

func (f Forbidden) Message() string {
	return f.F.Message
}

func (ise InternalServerError) Message() string {
	return ise.ISE.Message
}

func (nf NotFound) Message() string {
	return nf.NF.Message
}
<commit_msg>Added missing field on NotFound struct.<commit_after>package hpcloud

/*
 BadRequest describes the response from a JSON resource when the
 data which was sent in the original request was malformed or not
 compliant with the layout specified in the HPCloud documentation
*/
type BadRequest struct {
	B struct {
		Message string `json:"message"`
		Details string `json:"details"`
		Code    int64  `json:"code"`
	} `json:"BadRequest"`
}

/*
 NotFound describes the response from a JSON resource when the
 resource which was interacted with in the original request was
 not able to be found.
*/
type NotFound struct {
	NF struct {
		Message string `json:"message"`
		Details string `json:"details"`
		Code    int    `json:"code"`
	} `json:"itemNotFound"`
}

/*
 Unauthorized describes the response from a JSON resource when the
 request could not be completed due to none or incorrect authentication
 was used to make the request.
*/
type Unauthorized struct {
	U struct {
		Code            int64  `json:"code"`
		Details         string `json:"details"`
		Message         string `json:"message"`
		OtherAttributes struct {
		} `json:"otherAttributes"`
	} `json:"unauthorized"`
}

/*
 Forbidden describes the response from a JSON resource when the
 request could not be completed due to the user making the request
 being disabled or suspended.
*/
type Forbidden struct {
	F struct {
		Code            int64  `json:"code"`
		Details         string `json:"details"`
		Message         string `json:"message"`
		OtherAttributes struct {
		} `json:"otherAttributes"`
	} `json:"forbidden"`
}

/*
 InternalServerError describes the response from a JSON resource when the
 request could not be completed due to the request causing the service
 to return a 500 status code.
*/
type InternalServerError struct {
	ISE struct {
		Code            int64  `json:"code"`
		Details         string `json:"details"`
		Message         string `json:"message"`
		OtherAttributes struct {
		} `json:"otherAttributes"`
	} `json:"internalServerError"`
}

type SubToken struct {
	ID string `json:"id"`
}

type Flavor_ struct {
	Name  string `json:"name"`
	ID    int64  `json:"id"`
	Links []Link `json:"links"`
}

type Flavors struct {
	F []Flavor_ `json:"flavors"`
}

type Tenants struct {
	T []Tenant `json:"tenants"`
}

type FailureResponse interface {
	Code() int64
	Details() string
	Message() string
}

func (u Unauthorized) Code() int64 {
	return u.U.Code
}

func (b BadRequest) Code() int64 {
	return b.B.Code
}

func (f Forbidden) Code() int64 {
	return f.F.Code
}

func (nf NotFound) Code() int64 {
	return nf.NF.Code
}

func (ise InternalServerError) Code() int64 {
	return ise.ISE.Code
}

func (u Unauthorized) Details() string {
	return u.U.Details
}

func (b BadRequest) Details() string {
	return b.B.Details
}

func (f Forbidden) Details() string {
	return f.F.Details
}

func (ise InternalServerError) Details() string {
	return ise.ISE.Details
}

func (nf NotFound) Details() string {
	return nf.NF.Details
}

func (u Unauthorized) Message() string {
	return u.U.Message
}

func (b BadRequest) Message() string {
	return b.B.Message
}

func (f Forbidden) Message() string {
	return f.F.Message
}

func (ise InternalServerError) Message() string {
	return ise.ISE.Message
}

func (nf NotFound) Message() string {
	return nf.NF.Message
}
<|endoftext|>


<commit_before>package analytics

import (
	"sync"

	ga "github.com/jpillora/go-ogle-analytics"
)

const (
	id       = "UA-42684979-8"
	category = "coreos-linux-controller"
)

var (
	mu     sync.Mutex
	client *ga.Client
)

func Enable() {
	mu.Lock()
	defer mu.Unlock()
	client = mustNewClient()
}

func Disable() {
	mu.Lock()
	defer mu.Unlock()
	client = nil
}

func send(e *ga.Event) {
	mu.Lock()
	c := client
	mu.Unlock()

	if c == nil {
		return
	}
	// error is ignored intentionally. we try to send event to GA in a best effort approach.
	c.Send(e)
}

func mustNewClient() *ga.Client {
	client, err := ga.NewClient(id)
	if err != nil {
		panic(err)
	}
	return client
}

func ControllerStarted() {
	send(ga.NewEvent(category, "controller_started"))
}
<commit_msg>internal/analytics: fix analytics strings<commit_after>package analytics

import (
	"sync"

	ga "github.com/jpillora/go-ogle-analytics"
)

const (
	id       = "UA-42684979-8"
	category = "container-linux-update-operator"
)

var (
	mu     sync.Mutex
	client *ga.Client
)

func Enable() {
	mu.Lock()
	defer mu.Unlock()
	client = mustNewClient()
}

func Disable() {
	mu.Lock()
	defer mu.Unlock()
	client = nil
}

func send(e *ga.Event) {
	mu.Lock()
	c := client
	mu.Unlock()

	if c == nil {
		return
	}
	// error is ignored intentionally. we try to send event to GA in a best effort approach.
	c.Send(e)
}

func mustNewClient() *ga.Client {
	client, err := ga.NewClient(id)
	if err != nil {
		panic(err)
	}
	return client
}

func ControllerStarted() {
	send(ga.NewEvent(category, "operator_started"))
}
<|endoftext|>


<commit_before>// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.

package beacon

import "net"

type Broadcast struct {
	conn   *net.UDPConn
	port   int
	inbox  chan []byte
	outbox chan recv
}

func NewBroadcast(port int) (*Broadcast, error) {
	conn, err := net.ListenUDP("udp4", &net.UDPAddr{Port: port})
	if err != nil {
		return nil, err
	}
	b := &Broadcast{
		conn:   conn,
		port:   port,
		inbox:  make(chan []byte),
		outbox: make(chan recv, 16),
	}

	go genericReader(b.conn, b.outbox)
	go b.writer()

	return b, nil
}

func (b *Broadcast) Send(data []byte) {
	b.inbox <- data
}

func (b *Broadcast) Recv() ([]byte, net.Addr) {
	recv := <-b.outbox
	return recv.data, recv.src
}

func (b *Broadcast) writer() {
	for bs := range b.inbox {

		addrs, err := net.InterfaceAddrs()
		if err != nil {
			l.Warnln("Broadcast: interface addresses:", err)
			continue
		}

		var dsts []net.IP
		for _, addr := range addrs {
			if iaddr, ok := addr.(*net.IPNet); ok && len(iaddr.IP) >= 4 && iaddr.IP.IsGlobalUnicast() && iaddr.IP.To4() != nil {
				baddr := bcast(iaddr)
				dsts = append(dsts, baddr.IP)
			}
		}

		if len(dsts) == 0 {
			// Fall back to the general IPv4 broadcast address
			dsts = append(dsts, net.IP{0xff, 0xff, 0xff, 0xff})
		}

		if debug {
			l.Debugln("addresses:", dsts)
		}

		for _, ip := range dsts {
			dst := &net.UDPAddr{IP: ip, Port: b.port}

			_, err := b.conn.WriteTo(bs, dst)
			if err != nil {
				if debug {
					l.Debugln(err)
				}
			} else if debug {
				l.Debugf("sent %d bytes to %s", len(bs), dst)
			}
		}
	}
}

func bcast(ip *net.IPNet) *net.IPNet {
	var bc = &net.IPNet{}
	bc.IP = make([]byte, len(ip.IP))
	copy(bc.IP, ip.IP)
	bc.Mask = ip.Mask

	offset := len(bc.IP) - len(bc.Mask)
	for i := range bc.IP {
		if i-offset >= 0 {
			bc.IP[i] = ip.IP[i] | ^ip.Mask[i-offset]
		}
	}
	return bc
}
<commit_msg>Silence discovery warnings (fixes #1388)<commit_after>// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.

package beacon

import "net"

type Broadcast struct {
	conn   *net.UDPConn
	port   int
	inbox  chan []byte
	outbox chan recv
}

func NewBroadcast(port int) (*Broadcast, error) {
	conn, err := net.ListenUDP("udp4", &net.UDPAddr{Port: port})
	if err != nil {
		return nil, err
	}
	b := &Broadcast{
		conn:   conn,
		port:   port,
		inbox:  make(chan []byte),
		outbox: make(chan recv, 16),
	}

	go genericReader(b.conn, b.outbox)
	go b.writer()

	return b, nil
}

func (b *Broadcast) Send(data []byte) {
	b.inbox <- data
}

func (b *Broadcast) Recv() ([]byte, net.Addr) {
	recv := <-b.outbox
	return recv.data, recv.src
}

func (b *Broadcast) writer() {
	for bs := range b.inbox {

		addrs, err := net.InterfaceAddrs()
		if err != nil {
			if debug {
				l.Debugln("Broadcast: interface addresses:", err)
			}
			continue
		}

		var dsts []net.IP
		for _, addr := range addrs {
			if iaddr, ok := addr.(*net.IPNet); ok && len(iaddr.IP) >= 4 && iaddr.IP.IsGlobalUnicast() && iaddr.IP.To4() != nil {
				baddr := bcast(iaddr)
				dsts = append(dsts, baddr.IP)
			}
		}

		if len(dsts) == 0 {
			// Fall back to the general IPv4 broadcast address
			dsts = append(dsts, net.IP{0xff, 0xff, 0xff, 0xff})
		}

		if debug {
			l.Debugln("addresses:", dsts)
		}

		for _, ip := range dsts {
			dst := &net.UDPAddr{IP: ip, Port: b.port}

			_, err := b.conn.WriteTo(bs, dst)
			if err != nil {
				if debug {
					l.Debugln(err)
				}
			} else if debug {
				l.Debugf("sent %d bytes to %s", len(bs), dst)
			}
		}
	}
}

func bcast(ip *net.IPNet) *net.IPNet {
	var bc = &net.IPNet{}
	bc.IP = make([]byte, len(ip.IP))
	copy(bc.IP, ip.IP)
	bc.Mask = ip.Mask

	offset := len(bc.IP) - len(bc.Mask)
	for i := range bc.IP {
		if i-offset >= 0 {
			bc.IP[i] = ip.IP[i] | ^ip.Mask[i-offset]
		}
	}
	return bc
}
<|endoftext|>