<commit_before>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package mqtt

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/TheThingsNetwork/ttn/utils/random"
	"github.com/apex/log"
	MQTT "github.com/eclipse/paho.mqtt.golang"
)

const QoS = 0x00

// Client connects to the MQTT server and can publish/subscribe on uplink, downlink and activations from devices
type Client interface {
	Connect() error
	Disconnect()

	IsConnected() bool

	// Uplink pub/sub
	PublishUplink(payload UplinkMessage) Token
	SubscribeDeviceUplink(appID string, devID string, handler UplinkHandler) Token
	SubscribeAppUplink(appID string, handler UplinkHandler) Token
	SubscribeUplink(handler UplinkHandler) Token
	UnsubscribeDeviceUplink(appID string, devID string) Token
	UnsubscribeAppUplink(appID string) Token
	UnsubscribeUplink() Token

	// Downlink pub/sub
	PublishDownlink(payload DownlinkMessage) Token
	SubscribeDeviceDownlink(appID string, devID string, handler DownlinkHandler) Token
	SubscribeAppDownlink(appID string, handler DownlinkHandler) Token
	SubscribeDownlink(handler DownlinkHandler) Token
	UnsubscribeDeviceDownlink(appID string, devID string) Token
	UnsubscribeAppDownlink(appID string) Token
	UnsubscribeDownlink() Token

	// Activation pub/sub
	PublishActivation(payload Activation) Token
	SubscribeDeviceActivations(appID string, devID string, handler ActivationHandler) Token
	SubscribeAppActivations(appID string, handler ActivationHandler) Token
	SubscribeActivations(handler ActivationHandler) Token
	UnsubscribeDeviceActivations(appID string, devID string) Token
	UnsubscribeAppActivations(appID string) Token
	UnsubscribeActivations() Token
}

// Token is returned on asyncronous functions
type Token interface {
	// Wait for the function to finish
	Wait() bool
	// Wait for the function to finish or return false after a certain time
	WaitTimeout(time.Duration) bool
	// The error associated with the result of the function (nil if everything okay)
	Error() error
}

type simpleToken struct {
	err error
}

// Wait always returns true
func (t *simpleToken) Wait() bool {
	return true
}

// WaitTimeout always returns true
func (t *simpleToken) WaitTimeout(_ time.Duration) bool {
	return true
}

// Error contains the error if present
func (t *simpleToken) Error() error {
	return t.err
}

// UplinkHandler is called for uplink messages
type UplinkHandler func(client Client, appID string, devID string, req UplinkMessage)

// DownlinkHandler is called for downlink messages
type DownlinkHandler func(client Client, appID string, devID string, req DownlinkMessage)

// ActivationHandler is called for activations
type ActivationHandler func(client Client, appID string, devID string, req Activation)

// DefaultClient is the default MQTT client for The Things Network
type DefaultClient struct {
	mqtt          MQTT.Client
	ctx           log.Interface
	subscriptions map[string]MQTT.MessageHandler
}

// NewClient creates a new DefaultClient
func NewClient(ctx log.Interface, id, username, password string, brokers ...string) Client {
	mqttOpts := MQTT.NewClientOptions()

	for _, broker := range brokers {
		mqttOpts.AddBroker(broker)
	}

	mqttOpts.SetClientID(fmt.Sprintf("%s-%s", id, random.String(16)))
	mqttOpts.SetUsername(username)
	mqttOpts.SetPassword(password)

	// TODO: Some tuning of these values probably won't hurt:
	mqttOpts.SetKeepAlive(30 * time.Second)
	mqttOpts.SetPingTimeout(10 * time.Second)

	mqttOpts.SetCleanSession(true)

	mqttOpts.SetDefaultPublishHandler(func(client MQTT.Client, msg MQTT.Message) {
		ctx.WithField("message", msg).Warn("Received unhandled message")
	})

	var reconnecting bool

	mqttOpts.SetConnectionLostHandler(func(client MQTT.Client, err error) {
		ctx.WithError(err).Warn("Disconnected, reconnecting...")
		reconnecting = true
	})

	ttnClient := &DefaultClient{
		ctx:           ctx,
		subscriptions: make(map[string]MQTT.MessageHandler),
	}

	mqttOpts.SetOnConnectHandler(func(client MQTT.Client) {
		ctx.Info("Connected to MQTT")
		if reconnecting {
			for topic, handler := range ttnClient.subscriptions {
				ctx.Infof("Re-subscribing to %s", topic)
				ttnClient.subscribe(topic, handler)
			}
			reconnecting = false
		}
	})

	ttnClient.mqtt = MQTT.NewClient(mqttOpts)

	return ttnClient
}

var (
	// ConnectRetries says how many times the client should retry a failed connection
	ConnectRetries = 10
	// ConnectRetryDelay says how long the client should wait between retries
	ConnectRetryDelay = time.Second
)

// Connect to the MQTT broker. It will retry for ConnectRetries times with a delay of ConnectRetryDelay between retries
func (c *DefaultClient) Connect() error {
	if c.mqtt.IsConnected() {
		return nil
	}
	var err error
	for retries := 0; retries < ConnectRetries; retries++ {
		c.ctx.Debug("Connecting to MQTT...")
		token := c.mqtt.Connect()
		token.Wait()
		err = token.Error()
		if err == nil {
			break
		}
		<-time.After(ConnectRetryDelay)
	}
	if err != nil {
		return fmt.Errorf("Could not connect: %s", err)
	}
	return nil
}

func (c *DefaultClient) subscribe(topic string, handler MQTT.MessageHandler) Token {
	c.subscriptions[topic] = handler
	return c.mqtt.Subscribe(topic, QoS, handler)
}

func (c *DefaultClient) unsubscribe(topic string) Token {
	delete(c.subscriptions, topic)
	return c.mqtt.Unsubscribe(topic)
}

// Disconnect from the MQTT broker
func (c *DefaultClient) Disconnect() {
	if !c.mqtt.IsConnected() {
		return
	}
	c.ctx.Debug("Disconnecting from MQTT")
	c.mqtt.Disconnect(25)
}

// IsConnected returns true if there is a connection to the MQTT broker
func (c *DefaultClient) IsConnected() bool {
	return c.mqtt.IsConnected()
}

// PublishUplink publishes an uplink message to the MQTT broker
func (c *DefaultClient) PublishUplink(dataUp UplinkMessage) Token {
	topic := DeviceTopic{dataUp.AppID, dataUp.DevID, Uplink}
	dataUp.AppID = ""
	dataUp.DevID = ""
	msg, err := json.Marshal(dataUp)
	if err != nil {
		return &simpleToken{fmt.Errorf("Unable to marshal the message payload")}
	}
	return c.mqtt.Publish(topic.String(), QoS, false, msg)
}

// SubscribeDeviceUplink subscribes to all uplink messages for the given application and device
func (c *DefaultClient) SubscribeDeviceUplink(appID string, devID string, handler UplinkHandler) Token {
	topic := DeviceTopic{appID, devID, Uplink}
	return c.subscribe(topic.String(), func(mqtt MQTT.Client, msg MQTT.Message) {
		// Determine the actual topic
		topic, err := ParseDeviceTopic(msg.Topic())
		if err != nil {
			c.ctx.WithField("topic", msg.Topic()).WithError(err).Warn("Received message on invalid uplink topic")
			return
		}

		// Unmarshal the payload
		dataUp := &UplinkMessage{}
		err = json.Unmarshal(msg.Payload(), dataUp)
		dataUp.AppID = topic.AppID
		dataUp.DevID = topic.DevID

		if err != nil {
			c.ctx.WithError(err).Warn("Could not unmarshal uplink")
			return
		}

		// Call the uplink handler
		handler(c, topic.AppID, topic.DevID, *dataUp)
	})
}

// SubscribeAppUplink subscribes to all uplink messages for the given application
func (c *DefaultClient) SubscribeAppUplink(appID string, handler UplinkHandler) Token {
	return c.SubscribeDeviceUplink(appID, "", handler)
}

// SubscribeUplink subscribes to all uplink messages that the current user has access to
func (c *DefaultClient) SubscribeUplink(handler UplinkHandler) Token {
	return c.SubscribeDeviceUplink("", "", handler)
}

// UnsubscribeDeviceUplink unsubscribes from the uplink messages for the given application and device
func (c *DefaultClient) UnsubscribeDeviceUplink(appID string, devID string) Token {
	topic := DeviceTopic{appID, devID, Uplink}
	return c.unsubscribe(topic.String())
}

// UnsubscribeAppUplink unsubscribes from the uplink messages for the given application
func (c *DefaultClient) UnsubscribeAppUplink(appID string) Token {
	return c.UnsubscribeDeviceUplink(appID, "")
}

// UnsubscribeUplink unsubscribes from the uplink messages that the current user has access to
func (c *DefaultClient) UnsubscribeUplink() Token {
	return c.UnsubscribeDeviceUplink("", "")
}

// PublishDownlink publishes a downlink message
func (c *DefaultClient) PublishDownlink(dataDown DownlinkMessage) Token {
	topic := DeviceTopic{dataDown.AppID, dataDown.DevID, Downlink}
	dataDown.AppID = ""
	dataDown.DevID = ""
	msg, err := json.Marshal(dataDown)
	if err != nil {
		return &simpleToken{fmt.Errorf("Unable to marshal the message payload")}
	}
	return c.mqtt.Publish(topic.String(), QoS, false, msg)
}

// SubscribeDeviceDownlink subscribes to all downlink messages for the given application and device
func (c *DefaultClient) SubscribeDeviceDownlink(appID string, devID string, handler DownlinkHandler) Token {
	topic := DeviceTopic{appID, devID, Downlink}
	return c.subscribe(topic.String(), func(mqtt MQTT.Client, msg MQTT.Message) {
		// Determine the actual topic
		topic, err := ParseDeviceTopic(msg.Topic())
		if err != nil {
			c.ctx.WithField("topic", msg.Topic()).WithError(err).Warn("Received message on invalid Downlink topic")
			return
		}

		// Unmarshal the payload
		dataDown := &DownlinkMessage{}
		err = json.Unmarshal(msg.Payload(), dataDown)
		if err != nil {
			c.ctx.WithError(err).Warn("Could not unmarshal Downlink")
			return
		}
		dataDown.AppID = topic.AppID
		dataDown.DevID = topic.DevID

		// Call the Downlink handler
		handler(c, topic.AppID, topic.DevID, *dataDown)
	})
}

// SubscribeAppDownlink subscribes to all downlink messages for the given application
func (c *DefaultClient) SubscribeAppDownlink(appID string, handler DownlinkHandler) Token {
	return c.SubscribeDeviceDownlink(appID, "", handler)
}

// SubscribeDownlink subscribes to all downlink messages that the current user has access to
func (c *DefaultClient) SubscribeDownlink(handler DownlinkHandler) Token {
	return c.SubscribeDeviceDownlink("", "", handler)
}

// UnsubscribeDeviceDownlink unsubscribes from the downlink messages for the given application and device
func (c *DefaultClient) UnsubscribeDeviceDownlink(appID string, devID string) Token {
	topic := DeviceTopic{appID, devID, Downlink}
	return c.unsubscribe(topic.String())
}

// UnsubscribeAppDownlink unsubscribes from the downlink messages for the given application
func (c *DefaultClient) UnsubscribeAppDownlink(appID string) Token {
	return c.UnsubscribeDeviceDownlink(appID, "")
}

// UnsubscribeDownlink unsubscribes from the downlink messages that the current user has access to
func (c *DefaultClient) UnsubscribeDownlink() Token {
	return c.UnsubscribeDeviceDownlink("", "")
}

// PublishActivation publishes an activation
func (c *DefaultClient) PublishActivation(activation Activation) Token {
	topic := DeviceTopic{activation.AppID, activation.DevID, Activations}
	activation.AppID = ""
	activation.DevID = ""
	msg, err := json.Marshal(activation)
	if err != nil {
		return &simpleToken{fmt.Errorf("Unable to marshal the message payload")}
	}
	return c.mqtt.Publish(topic.String(), QoS, false, msg)
}

// SubscribeDeviceActivations subscribes to all activations for the given application and device
func (c *DefaultClient) SubscribeDeviceActivations(appID string, devID string, handler ActivationHandler) Token {
	topic := DeviceTopic{appID, devID, Activations}
	return c.subscribe(topic.String(), func(mqtt MQTT.Client, msg MQTT.Message) {
		// Determine the actual topic
		topic, err := ParseDeviceTopic(msg.Topic())
		if err != nil {
			c.ctx.WithField("topic", msg.Topic()).WithError(err).Warn("Received message on invalid Activations topic")
			return
		}

		// Unmarshal the payload
		activation := &Activation{}
		err = json.Unmarshal(msg.Payload(), activation)
		if err != nil {
			c.ctx.WithError(err).Warn("Could not unmarshal Activation")
			return
		}
		activation.AppID = topic.AppID
		activation.DevID = topic.DevID

		// Call the Activation handler
		handler(c, topic.AppID, topic.DevID, *activation)
	})
}

// SubscribeAppActivations subscribes to all activations for the given application
func (c *DefaultClient) SubscribeAppActivations(appID string, handler ActivationHandler) Token {
	return c.SubscribeDeviceActivations(appID, "", handler)
}

// SubscribeActivations subscribes to all activations that the current user has access to
func (c *DefaultClient) SubscribeActivations(handler ActivationHandler) Token {
	return c.SubscribeDeviceActivations("", "", handler)
}

// UnsubscribeDeviceActivations unsubscribes from the activations for the given application and device
func (c *DefaultClient) UnsubscribeDeviceActivations(appID string, devID string) Token {
	topic := DeviceTopic{appID, devID, Activations}
	return c.unsubscribe(topic.String())
}

// UnsubscribeAppActivations unsubscribes from the activations for the given application
func (c *DefaultClient) UnsubscribeAppActivations(appID string) Token {
	return c.UnsubscribeDeviceActivations(appID, "")
}

// UnsubscribeActivations unsubscribes from the activations that the current user has access to
func (c *DefaultClient) UnsubscribeActivations() Token {
	return c.UnsubscribeDeviceActivations("", "")
}
<commit_msg>Don't wait more than 1 second in MQTT connection attempt<commit_after>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package mqtt

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/TheThingsNetwork/ttn/utils/random"
	"github.com/apex/log"
	MQTT "github.com/eclipse/paho.mqtt.golang"
)

const QoS = 0x00

// Client connects to the MQTT server and can publish/subscribe on uplink, downlink and activations from devices
type Client interface {
	Connect() error
	Disconnect()

	IsConnected() bool

	// Uplink pub/sub
	PublishUplink(payload UplinkMessage) Token
	SubscribeDeviceUplink(appID string, devID string, handler UplinkHandler) Token
	SubscribeAppUplink(appID string, handler UplinkHandler) Token
	SubscribeUplink(handler UplinkHandler) Token
	UnsubscribeDeviceUplink(appID string, devID string) Token
	UnsubscribeAppUplink(appID string) Token
	UnsubscribeUplink() Token

	// Downlink pub/sub
	PublishDownlink(payload DownlinkMessage) Token
	SubscribeDeviceDownlink(appID string, devID string, handler DownlinkHandler) Token
	SubscribeAppDownlink(appID string, handler DownlinkHandler) Token
	SubscribeDownlink(handler DownlinkHandler) Token
	UnsubscribeDeviceDownlink(appID string, devID string) Token
	UnsubscribeAppDownlink(appID string) Token
	UnsubscribeDownlink() Token

	// Activation pub/sub
	PublishActivation(payload Activation) Token
	SubscribeDeviceActivations(appID string, devID string, handler ActivationHandler) Token
	SubscribeAppActivations(appID string, handler ActivationHandler) Token
	SubscribeActivations(handler ActivationHandler) Token
	UnsubscribeDeviceActivations(appID string, devID string) Token
	UnsubscribeAppActivations(appID string) Token
	UnsubscribeActivations() Token
}

// Token is returned on asyncronous functions
type Token interface {
	// Wait for the function to finish
	Wait() bool
	// Wait for the function to finish or return false after a certain time
	WaitTimeout(time.Duration) bool
	// The error associated with the result of the function (nil if everything okay)
	Error() error
}

type simpleToken struct {
	err error
}

// Wait always returns true
func (t *simpleToken) Wait() bool {
	return true
}

// WaitTimeout always returns true
func (t *simpleToken) WaitTimeout(_ time.Duration) bool {
	return true
}

// Error contains the error if present
func (t *simpleToken) Error() error {
	return t.err
}

// UplinkHandler is called for uplink messages
type UplinkHandler func(client Client, appID string, devID string, req UplinkMessage)

// DownlinkHandler is called for downlink messages
type DownlinkHandler func(client Client, appID string, devID string, req DownlinkMessage)

// ActivationHandler is called for activations
type ActivationHandler func(client Client, appID string, devID string, req Activation)

// DefaultClient is the default MQTT client for The Things Network
type DefaultClient struct {
	mqtt          MQTT.Client
	ctx           log.Interface
	subscriptions map[string]MQTT.MessageHandler
}

// NewClient creates a new DefaultClient
func NewClient(ctx log.Interface, id, username, password string, brokers ...string) Client {
	mqttOpts := MQTT.NewClientOptions()

	for _, broker := range brokers {
		mqttOpts.AddBroker(broker)
	}

	mqttOpts.SetClientID(fmt.Sprintf("%s-%s", id, random.String(16)))
	mqttOpts.SetUsername(username)
	mqttOpts.SetPassword(password)

	// TODO: Some tuning of these values probably won't hurt:
	mqttOpts.SetKeepAlive(30 * time.Second)
	mqttOpts.SetPingTimeout(10 * time.Second)

	mqttOpts.SetCleanSession(true)

	mqttOpts.SetDefaultPublishHandler(func(client MQTT.Client, msg MQTT.Message) {
		ctx.WithField("message", msg).Warn("Received unhandled message")
	})

	var reconnecting bool

	mqttOpts.SetConnectionLostHandler(func(client MQTT.Client, err error) {
		ctx.WithError(err).Warn("Disconnected, reconnecting...")
		reconnecting = true
	})

	ttnClient := &DefaultClient{
		ctx:           ctx,
		subscriptions: make(map[string]MQTT.MessageHandler),
	}

	mqttOpts.SetOnConnectHandler(func(client MQTT.Client) {
		ctx.Info("Connected to MQTT")
		if reconnecting {
			for topic, handler := range ttnClient.subscriptions {
				ctx.Infof("Re-subscribing to %s", topic)
				ttnClient.subscribe(topic, handler)
			}
			reconnecting = false
		}
	})

	ttnClient.mqtt = MQTT.NewClient(mqttOpts)

	return ttnClient
}

var (
	// ConnectRetries says how many times the client should retry a failed connection
	ConnectRetries = 10
	// ConnectRetryDelay says how long the client should wait between retries
	ConnectRetryDelay = time.Second
)

// Connect to the MQTT broker. It will retry for ConnectRetries times with a delay of ConnectRetryDelay between retries
func (c *DefaultClient) Connect() error {
	if c.mqtt.IsConnected() {
		return nil
	}
	var err error
	for retries := 0; retries < ConnectRetries; retries++ {
		token := c.mqtt.Connect()
		finished := token.WaitTimeout(1 * time.Second)
		err = token.Error()
		if finished && err == nil {
			break
		}
		c.ctx.WithError(err).Warn("Could not connect to MQTT Broker. Retrying...")
		<-time.After(ConnectRetryDelay)
	}
	if err != nil {
		return fmt.Errorf("Could not connect to MQTT Broker: %s", err)
	}
	return nil
}

func (c *DefaultClient) subscribe(topic string, handler MQTT.MessageHandler) Token {
	c.subscriptions[topic] = handler
	return c.mqtt.Subscribe(topic, QoS, handler)
}

func (c *DefaultClient) unsubscribe(topic string) Token {
	delete(c.subscriptions, topic)
	return c.mqtt.Unsubscribe(topic)
}

// Disconnect from the MQTT broker
func (c *DefaultClient) Disconnect() {
	if !c.mqtt.IsConnected() {
		return
	}
	c.ctx.Debug("Disconnecting from MQTT")
	c.mqtt.Disconnect(25)
}

// IsConnected returns true if there is a connection to the MQTT broker
func (c *DefaultClient) IsConnected() bool {
	return c.mqtt.IsConnected()
}

// PublishUplink publishes an uplink message to the MQTT broker
func (c *DefaultClient) PublishUplink(dataUp UplinkMessage) Token {
	topic := DeviceTopic{dataUp.AppID, dataUp.DevID, Uplink}
	dataUp.AppID = ""
	dataUp.DevID = ""
	msg, err := json.Marshal(dataUp)
	if err != nil {
		return &simpleToken{fmt.Errorf("Unable to marshal the message payload")}
	}
	return c.mqtt.Publish(topic.String(), QoS, false, msg)
}

// SubscribeDeviceUplink subscribes to all uplink messages for the given application and device
func (c *DefaultClient) SubscribeDeviceUplink(appID string, devID string, handler UplinkHandler) Token {
	topic := DeviceTopic{appID, devID, Uplink}
	return c.subscribe(topic.String(), func(mqtt MQTT.Client, msg MQTT.Message) {
		// Determine the actual topic
		topic, err := ParseDeviceTopic(msg.Topic())
		if err != nil {
			c.ctx.WithField("topic", msg.Topic()).WithError(err).Warn("Received message on invalid uplink topic")
			return
		}

		// Unmarshal the payload
		dataUp := &UplinkMessage{}
		err = json.Unmarshal(msg.Payload(), dataUp)
		dataUp.AppID = topic.AppID
		dataUp.DevID = topic.DevID

		if err != nil {
			c.ctx.WithError(err).Warn("Could not unmarshal uplink")
			return
		}

		// Call the uplink handler
		handler(c, topic.AppID, topic.DevID, *dataUp)
	})
}

// SubscribeAppUplink subscribes to all uplink messages for the given application
func (c *DefaultClient) SubscribeAppUplink(appID string, handler UplinkHandler) Token {
	return c.SubscribeDeviceUplink(appID, "", handler)
}

// SubscribeUplink subscribes to all uplink messages that the current user has access to
func (c *DefaultClient) SubscribeUplink(handler UplinkHandler) Token {
	return c.SubscribeDeviceUplink("", "", handler)
}

// UnsubscribeDeviceUplink unsubscribes from the uplink messages for the given application and device
func (c *DefaultClient) UnsubscribeDeviceUplink(appID string, devID string) Token {
	topic := DeviceTopic{appID, devID, Uplink}
	return c.unsubscribe(topic.String())
}

// UnsubscribeAppUplink unsubscribes from the uplink messages for the given application
func (c *DefaultClient) UnsubscribeAppUplink(appID string) Token {
	return c.UnsubscribeDeviceUplink(appID, "")
}

// UnsubscribeUplink unsubscribes from the uplink messages that the current user has access to
func (c *DefaultClient) UnsubscribeUplink() Token {
	return c.UnsubscribeDeviceUplink("", "")
}

// PublishDownlink publishes a downlink message
func (c *DefaultClient) PublishDownlink(dataDown DownlinkMessage) Token {
	topic := DeviceTopic{dataDown.AppID, dataDown.DevID, Downlink}
	dataDown.AppID = ""
	dataDown.DevID = ""
	msg, err := json.Marshal(dataDown)
	if err != nil {
		return &simpleToken{fmt.Errorf("Unable to marshal the message payload")}
	}
	return c.mqtt.Publish(topic.String(), QoS, false, msg)
}

// SubscribeDeviceDownlink subscribes to all downlink messages for the given application and device
func (c *DefaultClient) SubscribeDeviceDownlink(appID string, devID string, handler DownlinkHandler) Token {
	topic := DeviceTopic{appID, devID, Downlink}
	return c.subscribe(topic.String(), func(mqtt MQTT.Client, msg MQTT.Message) {
		// Determine the actual topic
		topic, err := ParseDeviceTopic(msg.Topic())
		if err != nil {
			c.ctx.WithField("topic", msg.Topic()).WithError(err).Warn("Received message on invalid Downlink topic")
			return
		}

		// Unmarshal the payload
		dataDown := &DownlinkMessage{}
		err = json.Unmarshal(msg.Payload(), dataDown)
		if err != nil {
			c.ctx.WithError(err).Warn("Could not unmarshal Downlink")
			return
		}
		dataDown.AppID = topic.AppID
		dataDown.DevID = topic.DevID

		// Call the Downlink handler
		handler(c, topic.AppID, topic.DevID, *dataDown)
	})
}

// SubscribeAppDownlink subscribes to all downlink messages for the given application
func (c *DefaultClient) SubscribeAppDownlink(appID string, handler DownlinkHandler) Token {
	return c.SubscribeDeviceDownlink(appID, "", handler)
}

// SubscribeDownlink subscribes to all downlink messages that the current user has access to
func (c *DefaultClient) SubscribeDownlink(handler DownlinkHandler) Token {
	return c.SubscribeDeviceDownlink("", "", handler)
}

// UnsubscribeDeviceDownlink unsubscribes from the downlink messages for the given application and device
func (c *DefaultClient) UnsubscribeDeviceDownlink(appID string, devID string) Token {
	topic := DeviceTopic{appID, devID, Downlink}
	return c.unsubscribe(topic.String())
}

// UnsubscribeAppDownlink unsubscribes from the downlink messages for the given application
func (c *DefaultClient) UnsubscribeAppDownlink(appID string) Token {
	return c.UnsubscribeDeviceDownlink(appID, "")
}

// UnsubscribeDownlink unsubscribes from the downlink messages that the current user has access to
func (c *DefaultClient) UnsubscribeDownlink() Token {
	return c.UnsubscribeDeviceDownlink("", "")
}

// PublishActivation publishes an activation
func (c *DefaultClient) PublishActivation(activation Activation) Token {
	topic := DeviceTopic{activation.AppID, activation.DevID, Activations}
	activation.AppID = ""
	activation.DevID = ""
	msg, err := json.Marshal(activation)
	if err != nil {
		return &simpleToken{fmt.Errorf("Unable to marshal the message payload")}
	}
	return c.mqtt.Publish(topic.String(), QoS, false, msg)
}

// SubscribeDeviceActivations subscribes to all activations for the given application and device
func (c *DefaultClient) SubscribeDeviceActivations(appID string, devID string, handler ActivationHandler) Token {
	topic := DeviceTopic{appID, devID, Activations}
	return c.subscribe(topic.String(), func(mqtt MQTT.Client, msg MQTT.Message) {
		// Determine the actual topic
		topic, err := ParseDeviceTopic(msg.Topic())
		if err != nil {
			c.ctx.WithField("topic", msg.Topic()).WithError(err).Warn("Received message on invalid Activations topic")
			return
		}

		// Unmarshal the payload
		activation := &Activation{}
		err = json.Unmarshal(msg.Payload(), activation)
		if err != nil {
			c.ctx.WithError(err).Warn("Could not unmarshal Activation")
			return
		}
		activation.AppID = topic.AppID
		activation.DevID = topic.DevID

		// Call the Activation handler
		handler(c, topic.AppID, topic.DevID, *activation)
	})
}

// SubscribeAppActivations subscribes to all activations for the given application
func (c *DefaultClient) SubscribeAppActivations(appID string, handler ActivationHandler) Token {
	return c.SubscribeDeviceActivations(appID, "", handler)
}

// SubscribeActivations subscribes to all activations that the current user has access to
func (c *DefaultClient) SubscribeActivations(handler ActivationHandler) Token {
	return c.SubscribeDeviceActivations("", "", handler)
}

// UnsubscribeDeviceActivations unsubscribes from the activations for the given application and device
func (c *DefaultClient) UnsubscribeDeviceActivations(appID string, devID string) Token {
	topic := DeviceTopic{appID, devID, Activations}
	return c.unsubscribe(topic.String())
}

// UnsubscribeAppActivations unsubscribes from the activations for the given application
func (c *DefaultClient) UnsubscribeAppActivations(appID string) Token {
	return c.UnsubscribeDeviceActivations(appID, "")
}

// UnsubscribeActivations unsubscribes from the activations that the current user has access to
func (c *DefaultClient) UnsubscribeActivations() Token {
	return c.UnsubscribeDeviceActivations("", "")
}
<|endoftext|>


<commit_before>// config.go
//
// This file implements the configuration part for when you need the API
// key to modify things in the Atlas configuration and manage measurements.

package atlas

import (

)

const (
	apiEndpoint = "https://atlas.ripe.net/api/v2/"
)
<commit_msg>Fix file name in comment.<commit_after>// common.go
//
// This file implements the configuration part for when you need the API
// key to modify things in the Atlas configuration and manage measurements.

package atlas

import (

)

const (
	apiEndpoint = "https://atlas.ripe.net/api/v2/"
)
<|endoftext|>


<commit_before>package selenium

import (
	"log"
)

var debugFlag = false

func setDebug(debug bool) {
	debugFlag = debug
}

func debugLog(format string, args ...interface{}) {
	if !debugFlag {
		return
	}
	log.Printf(format+"\n", args...)
}
<commit_msg>Fix debug accessibility<commit_after>package selenium

import (
	"log"
)

var debugFlag = false

func SetDebug(debug bool) {
	debugFlag = debug
}

func debugLog(format string, args ...interface{}) {
	if !debugFlag {
		return
	}
	log.Printf(format+"\n", args...)
}
<|endoftext|>


<commit_before>// common.go
//
// This file implements the configuration part for when you need the API
// key to modify things in the Atlas configuration and manage measurements.

package atlas

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"

	"github.com/pkg/errors"
)

const (
	apiEndpoint = "https://atlas.ripe.net/api/v2"
)

// getPageNum returns the value of the page= parameter
func getPageNum(url string) (page string) {
	re := regexp.MustCompile(`page=(\d+)`)
	if m := re.FindStringSubmatch(url); len(m) >= 1 {
		return m[1]
	}
	return ""
}

// AddQueryParameters adds query parameters to the URL.
func AddQueryParameters(baseURL string, queryParams map[string]string) string {
	if len(queryParams) == 0 {
		return baseURL
	}
	baseURL += "?"
	params := url.Values{}
	for key, value := range queryParams {
		params.Add(key, value)
	}
	return baseURL + params.Encode()
}

// addAPIKey insert the key into options if needed
func (c *Client) addAPIKey(opts map[string]string) map[string]string {
	key, ok := c.HasAPIKey()
	// Insert key
	if ok {
		opts["key"] = key
	}
	return opts
}

// prepareRequest insert all pre-defined stuff
func (c *Client) prepareRequest(method, what string, opts map[string]string) (req *http.Request) {
	var endPoint string

	// This is a hack to fetch direct urls for results
	if method == "FETCH" {
		endPoint = what
		method = "GET"
	} else {
		if c.config.endpoint != "" {
			endPoint = fmt.Sprintf("%s/%s", c.config.endpoint, what)
		}
	}

	c.mergeGlobalOptions(opts)
	c.debug("Options:\n%v", opts)
	baseURL := AddQueryParameters(endPoint, opts)

	req, err := http.NewRequest(method, baseURL, nil)
	if err != nil {
		c.log.Printf("error parsing %s: %v", baseURL, err)
		return &http.Request{}
	}

	c.debug("req.url=%s", baseURL)
	// We need these when we POST
	if method == "POST" {
		req.Header.Set("Content-Type", "application/json")
		req.Header.Set("Accept", "application/json")
	}

	return
}

// client.handleAPIResponse check status code & return undecoded APIError
func (c *Client) handleAPIResponse(r *http.Response) ([]byte, error) {
	if r == nil {
		return []byte{}, fmt.Errorf("error: r is nil")
	}

	// Everything is fine
	if r.StatusCode == http.StatusOK || r.StatusCode == 0 {
		return []byte{}, nil
	}

	// Everything is fine too (200-2xx)
	if r.StatusCode >= http.StatusOK && r.StatusCode < http.StatusMultipleChoices {
		return []byte{}, nil
	}

	// Check this condition (3xx are handled directly)
	if r.StatusCode >= http.StatusMultipleChoices && r.StatusCode < http.StatusBadRequest {
		return []byte{}, nil
	}

	// Everything else is an error
	body, err := ioutil.ReadAll(r.Body)
	defer r.Body.Close()

	if err != nil {
		return body, errors.Wrap(err, "read body")
	}

	var e APIError

	err = json.Unmarshal(body, &e)
	if err != nil {
		return body, errors.Wrapf(err, "decoding error raw=%v", body)
	}

	return body, e
}

func (c *Client) mergeGlobalOptions(opts map[string]string) {
	for k, v := range c.opts {
		opts[k] = v
	}
}
<commit_msg>Create decodeAPIError to extract interesting stuff from API.<commit_after>// common.go
//
// This file implements the configuration part for when you need the API
// key to modify things in the Atlas configuration and manage measurements.

package atlas

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"

	"github.com/pkg/errors"
)

const (
	apiEndpoint = "https://atlas.ripe.net/api/v2"
)

// getPageNum returns the value of the page= parameter
func getPageNum(url string) (page string) {
	re := regexp.MustCompile(`page=(\d+)`)
	if m := re.FindStringSubmatch(url); len(m) >= 1 {
		return m[1]
	}
	return ""
}

// AddQueryParameters adds query parameters to the URL.
func AddQueryParameters(baseURL string, queryParams map[string]string) string {
	if len(queryParams) == 0 {
		return baseURL
	}
	baseURL += "?"
	params := url.Values{}
	for key, value := range queryParams {
		params.Add(key, value)
	}
	return baseURL + params.Encode()
}

// addAPIKey insert the key into options if needed
func (c *Client) addAPIKey(opts map[string]string) map[string]string {
	key, ok := c.HasAPIKey()
	// Insert key
	if ok {
		opts["key"] = key
	}
	return opts
}

// prepareRequest insert all pre-defined stuff
func (c *Client) prepareRequest(method, what string, opts map[string]string) (req *http.Request) {
	var endPoint string

	// This is a hack to fetch direct urls for results
	if method == "FETCH" {
		endPoint = what
		method = "GET"
	} else {
		if c.config.endpoint != "" {
			endPoint = fmt.Sprintf("%s/%s", c.config.endpoint, what)
		}
	}

	c.mergeGlobalOptions(opts)
	c.debug("Options:\n%v", opts)
	baseURL := AddQueryParameters(endPoint, opts)

	req, err := http.NewRequest(method, baseURL, nil)
	if err != nil {
		c.log.Printf("error parsing %s: %v", baseURL, err)
		return &http.Request{}
	}

	c.debug("req.url=%s", baseURL)
	// We need these when we POST
	if method == "POST" {
		req.Header.Set("Content-Type", "application/json")
		req.Header.Set("Accept", "application/json")
	}

	return
}

// decodeAPIError does the deserialisation
func decodeAPIError(body []byte) (*APIError, error) {
	var e APIError

	err := json.Unmarshal(body, &e)
	return &e, err
}

// client.handleAPIResponse check status code & return undecoded APIError
func (c *Client) handleAPIResponse(r *http.Response) ([]byte, error) {
	if r == nil {
		return []byte{}, fmt.Errorf("error: r is nil")
	}

	// Everything is fine
	if r.StatusCode == http.StatusOK || r.StatusCode == 0 {
		return []byte{}, nil
	}

	// Everything is fine too (200-2xx)
	if r.StatusCode >= http.StatusOK && r.StatusCode < http.StatusMultipleChoices {
		return []byte{}, nil
	}

	// Check this condition (3xx are handled directly)
	if r.StatusCode >= http.StatusMultipleChoices && r.StatusCode < http.StatusBadRequest {
		return []byte{}, nil
	}

	// Everything else is an error
	body, err := ioutil.ReadAll(r.Body)
	defer r.Body.Close()

	if err != nil {
		return body, errors.Wrap(err, "read body")
	}

	var e APIError

	err = json.Unmarshal(body, &e)
	if err != nil {
		return body, errors.Wrapf(err, "decoding error raw=%v", body)
	}

	return body, e
}

func (c *Client) mergeGlobalOptions(opts map[string]string) {
	for k, v := range c.opts {
		opts[k] = v
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package addresser

import (
	"github.com/juju/errors"
	"github.com/juju/names"

	"github.com/juju/juju/api/base"
	"github.com/juju/juju/api/common"
	"github.com/juju/juju/api/watcher"
	"github.com/juju/juju/apiserver/params"
)

const addresserFacade = "Addresser"

// API provides access to the InstancePoller API facade.
type API struct {
	*common.EnvironWatcher

	facade base.FacadeCaller
}

// NewAPI creates a new client-side Addresser facade.
func NewAPI(caller base.APICaller) *API {
	if caller == nil {
		panic("caller is nil")
	}
	facadeCaller := base.NewFacadeCaller(caller, addresserFacade)
	return &API{
		EnvironWatcher: common.NewEnvironWatcher(facadeCaller),
		facade:         facadeCaller,
	}
}

// IPAddress provides access to methods of a state.IPAddress through the
// facade.
func (api *API) IPAddress(tag names.IPAddressTag) (*IPAddress, error) {
	life, err := common.Life(api.facade, tag)
	if err != nil {
		return nil, errors.Trace(err)
	}
	return &IPAddress{api.facade, tag, life}, nil
}

var newEntityWatcher = watcher.NewEntityWatcher

// WatchIPAddresses returns a EntityWatcher for observing the
// tags of IP addresses with changes in life cycle.
// The initial event will contain the tags of any IP addresses
// which are no longer Alive.
func (api *API) WatchIPAddresses() (watcher.EntityWatcher, error) {
	var result params.EntityWatchResult
	err := api.facade.FacadeCall("WatchIPAddresses", nil, &result)
	if err != nil {
		return nil, err
	}
	if err != nil {
		return nil, err
	}
	if result.Error != nil {
		return nil, result.Error
	}
	w := newEntityWatcher(api.facade.RawAPICaller(), result)
	return w, nil
}
<commit_msg>Added TODO for later change to bulk requests.<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package addresser

import (
	"github.com/juju/errors"
	"github.com/juju/names"

	"github.com/juju/juju/api/base"
	"github.com/juju/juju/api/common"
	"github.com/juju/juju/api/watcher"
	"github.com/juju/juju/apiserver/params"
)

const addresserFacade = "Addresser"

// API provides access to the InstancePoller API facade.
type API struct {
	*common.EnvironWatcher

	facade base.FacadeCaller
}

// NewAPI creates a new client-side Addresser facade.
func NewAPI(caller base.APICaller) *API {
	if caller == nil {
		panic("caller is nil")
	}
	facadeCaller := base.NewFacadeCaller(caller, addresserFacade)
	return &API{
		EnvironWatcher: common.NewEnvironWatcher(facadeCaller),
		facade:         facadeCaller,
	}
}

// IPAddress provides access to methods of a state.IPAddress through the
// facade.
func (api *API) IPAddress(tag names.IPAddressTag) (*IPAddress, error) {
	// TODO(mue) Change approach to use bulk requests for retrieval
	// and later removal.
	life, err := common.Life(api.facade, tag)
	if err != nil {
		return nil, errors.Trace(err)
	}
	return &IPAddress{api.facade, tag, life}, nil
}

var newEntityWatcher = watcher.NewEntityWatcher

// WatchIPAddresses returns a EntityWatcher for observing the
// tags of IP addresses with changes in life cycle.
// The initial event will contain the tags of any IP addresses
// which are no longer Alive.
func (api *API) WatchIPAddresses() (watcher.EntityWatcher, error) {
	var result params.EntityWatchResult
	err := api.facade.FacadeCall("WatchIPAddresses", nil, &result)
	if err != nil {
		return nil, err
	}
	if err != nil {
		return nil, err
	}
	if result.Error != nil {
		return nil, result.Error
	}
	w := newEntityWatcher(api.facade.RawAPICaller(), result)
	return w, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Circonus, Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package api

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"reflect"
	"testing"
)

var (
	testOutlierReport = OutlierReport{
		CID:              "/outlier_report/1234",
		Created:          1483033102,
		CreatedBy:        "/user/1234",
		LastModified:     1483033102,
		LastModifiedBy:   "/user/1234",
		Config:           "",
		MetricClusterCID: "/metric_cluster/1234",
		Tags:             []string{"cat:tag"},
		Title:            "foo bar",
	}
)

func testOutlierReportServer() *httptest.Server {
	f := func(w http.ResponseWriter, r *http.Request) {
		path := r.URL.Path
		if path == "/outlier_report/1234" {
			switch r.Method {
			case "GET":
				ret, err := json.Marshal(testOutlierReport)
				if err != nil {
					panic(err)
				}
				w.WriteHeader(200)
				w.Header().Set("Content-Type", "application/json")
				fmt.Fprintln(w, string(ret))
			case "PUT":
				defer r.Body.Close()
				b, err := ioutil.ReadAll(r.Body)
				if err != nil {
					panic(err)
				}
				w.WriteHeader(200)
				w.Header().Set("Content-Type", "application/json")
				fmt.Fprintln(w, string(b))
			case "DELETE":
				w.WriteHeader(200)
				w.Header().Set("Content-Type", "application/json")
			default:
				w.WriteHeader(404)
				fmt.Fprintln(w, fmt.Sprintf("not found: %s %s", r.Method, path))
			}
		} else if path == "/outlier_report" {
			switch r.Method {
			case "GET":
				c := []OutlierReport{testOutlierReport}
				ret, err := json.Marshal(c)
				if err != nil {
					panic(err)
				}
				w.WriteHeader(200)
				w.Header().Set("Content-Type", "application/json")
				fmt.Fprintln(w, string(ret))
			case "POST":
				defer r.Body.Close()
				_, err := ioutil.ReadAll(r.Body)
				if err != nil {
					panic(err)
				}
				ret, err := json.Marshal(testOutlierReport)
				if err != nil {
					panic(err)
				}
				w.WriteHeader(200)
				w.Header().Set("Content-Type", "application/json")
				fmt.Fprintln(w, string(ret))
			default:
				w.WriteHeader(404)
				fmt.Fprintln(w, fmt.Sprintf("not found: %s %s", r.Method, path))
			}
		} else {
			w.WriteHeader(404)
			fmt.Fprintln(w, fmt.Sprintf("not found: %s %s", r.Method, path))
		}
	}

	return httptest.NewServer(http.HandlerFunc(f))
}

func TestFetchOutlierReport(t *testing.T) {
	server := testOutlierReportServer()
	defer server.Close()

	ac := &Config{
		TokenKey: "abc123",
		TokenApp: "test",
		URL:      server.URL,
	}
	apih, err := NewAPI(ac)
	if err != nil {
		t.Errorf("Expected no error, got '%v'", err)
	}

	t.Log("without CID")
	{
		cid := ""
		expectedError := errors.New("Invalid outlier report CID [none]")
		_, err := apih.FetchOutlierReport(CIDType(&cid))
		if err == nil {
			t.Fatalf("Expected error")
		}
		if err.Error() != expectedError.Error() {
			t.Fatalf("Expected %+v got '%+v'", expectedError, err)
		}
	}

	t.Log("with valid CID")
	{
		cid := "/outlier_report/1234"
		report, err := apih.FetchOutlierReport(CIDType(&cid))
		if err != nil {
			t.Fatalf("Expected no error, got '%v'", err)
		}

		actualType := reflect.TypeOf(report)
		expectedType := "*api.OutlierReport"
		if actualType.String() != expectedType {
			t.Fatalf("Expected %s, got %s", expectedType, actualType.String())
		}

		if report.CID != testOutlierReport.CID {
			t.Fatalf("CIDs do not match: %+v != %+v\n", report, testOutlierReport)
		}
	}

	t.Log("with invalid CID")
	{
		cid := "/invalid"
		expectedError := errors.New("Invalid outlier report CID [/invalid]")
		_, err := apih.FetchOutlierReport(CIDType(&cid))
		if err == nil {
			t.Fatalf("Expected error")
		}
		if err.Error() != expectedError.Error() {
			t.Fatalf("Expected %+v got '%+v'", expectedError, err)
		}
	}
}

func TestFetchOutlierReports(t *testing.T) {
	server := testOutlierReportServer()
	defer server.Close()

	ac := &Config{
		TokenKey: "abc123",
		TokenApp: "test",
		URL:      server.URL,
	}
	apih, err := NewAPI(ac)
	if err != nil {
		t.Errorf("Expected no error, got '%v'", err)
	}

	reports, err := apih.FetchOutlierReports()
	if err != nil {
		t.Fatalf("Expected no error, got '%v'", err)
	}

	actualType := reflect.TypeOf(reports)
	expectedType := "*[]api.OutlierReport"
	if actualType.String() != expectedType {
		t.Fatalf("Expected %s, got %s", expectedType, actualType.String())
	}

}

func TestCreateOutlierReport(t *testing.T) {
	server := testOutlierReportServer()
	defer server.Close()

	var apih *API

	ac := &Config{
		TokenKey: "abc123",
		TokenApp: "test",
		URL:      server.URL,
	}
	apih, err := NewAPI(ac)
	if err != nil {
		t.Errorf("Expected no error, got '%v'", err)
	}

	report, err := apih.CreateOutlierReport(&testOutlierReport)
	if err != nil {
		t.Fatalf("Expected no error, got '%v'", err)
	}

	actualType := reflect.TypeOf(report)
	expectedType := "*api.OutlierReport"
	if actualType.String() != expectedType {
		t.Fatalf("Expected %s, got %s", expectedType, actualType.String())
	}
}

func TestUpdateOutlierReport(t *testing.T) {
	server := testOutlierReportServer()
	defer server.Close()

	var apih *API

	ac := &Config{
		TokenKey: "abc123",
		TokenApp: "test",
		URL:      server.URL,
	}
	apih, err := NewAPI(ac)
	if err != nil {
		t.Errorf("Expected no error, got '%v'", err)
	}

	t.Log("valid OutlierReport")
	{
		report, err := apih.UpdateOutlierReport(&testOutlierReport)
		if err != nil {
			t.Fatalf("Expected no error, got '%v'", err)
		}

		actualType := reflect.TypeOf(report)
		expectedType := "*api.OutlierReport"
		if actualType.String() != expectedType {
			t.Fatalf("Expected %s, got %s", expectedType, actualType.String())
		}
	}

	t.Log("Test with invalid CID")
	{
		expectedError := errors.New("Invalid outlier report CID [/invalid]")
		x := &OutlierReport{CID: "/invalid"}
		_, err := apih.UpdateOutlierReport(x)
		if err == nil {
			t.Fatal("Expected an error")
		}
		if err.Error() != expectedError.Error() {
			t.Fatalf("Expected %+v got '%+v'", expectedError, err)
		}
	}
}

func TestDeleteOutlierReport(t *testing.T) {
	server := testOutlierReportServer()
	defer server.Close()

	var apih *API

	ac := &Config{
		TokenKey: "abc123",
		TokenApp: "test",
		URL:      server.URL,
	}
	apih, err := NewAPI(ac)
	if err != nil {
		t.Errorf("Expected no error, got '%v'", err)
	}

	t.Log("valid OutlierReport")
	{
		_, err := apih.DeleteOutlierReport(&testOutlierReport)
		if err != nil {
			t.Fatalf("Expected no error, got '%v'", err)
		}
	}

	t.Log("Test with invalid CID")
	{
		expectedError := errors.New("Invalid outlier report CID [/invalid]")
		x := &OutlierReport{CID: "/invalid"}
		_, err := apih.UpdateOutlierReport(x)
		if err == nil {
			t.Fatal("Expected an error")
		}
		if err.Error() != expectedError.Error() {
			t.Fatalf("Expected %+v got '%+v'", expectedError, err)
		}
	}
}
<commit_msg>add: search test coverage<commit_after>// Copyright 2016 Circonus, Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package api

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"reflect"
	"testing"
)

var (
	testOutlierReport = OutlierReport{
		CID:              "/outlier_report/1234",
		Created:          1483033102,
		CreatedBy:        "/user/1234",
		LastModified:     1483033102,
		LastModifiedBy:   "/user/1234",
		Config:           "",
		MetricClusterCID: "/metric_cluster/1234",
		Tags:             []string{"cat:tag"},
		Title:            "foo bar",
	}
)

func testOutlierReportServer() *httptest.Server {
	f := func(w http.ResponseWriter, r *http.Request) {
		path := r.URL.Path
		if path == "/outlier_report/1234" {
			switch r.Method {
			case "GET":
				ret, err := json.Marshal(testOutlierReport)
				if err != nil {
					panic(err)
				}
				w.WriteHeader(200)
				w.Header().Set("Content-Type", "application/json")
				fmt.Fprintln(w, string(ret))
			case "PUT":
				defer r.Body.Close()
				b, err := ioutil.ReadAll(r.Body)
				if err != nil {
					panic(err)
				}
				w.WriteHeader(200)
				w.Header().Set("Content-Type", "application/json")
				fmt.Fprintln(w, string(b))
			case "DELETE":
				w.WriteHeader(200)
				w.Header().Set("Content-Type", "application/json")
			default:
				w.WriteHeader(404)
				fmt.Fprintln(w, fmt.Sprintf("not found: %s %s", r.Method, path))
			}
		} else if path == "/outlier_report" {
			switch r.Method {
			case "GET":
				reqURL := r.URL.String()
				var c []OutlierReport
				if reqURL == "/outlier_report?search=requests+per+second" {
					c = []OutlierReport{testOutlierReport}
				} else if reqURL == "/outlier_report?f_tags_has=service%3Aweb" {
					c = []OutlierReport{testOutlierReport}
				} else if reqURL == "/outlier_report?f_tags_has=service%3Aweb&search=requests+per+second" {
					c = []OutlierReport{testOutlierReport}
				} else if reqURL == "/outlier_report" {
					c = []OutlierReport{testOutlierReport}
				} else {
					c = []OutlierReport{}
				}
				if len(c) > 0 {
					ret, err := json.Marshal(c)
					if err != nil {
						panic(err)
					}
					w.WriteHeader(200)
					w.Header().Set("Content-Type", "application/json")
					fmt.Fprintln(w, string(ret))
				} else {
					w.WriteHeader(404)
					fmt.Fprintln(w, fmt.Sprintf("not found: %s %s", r.Method, reqURL))
				}
			case "POST":
				defer r.Body.Close()
				_, err := ioutil.ReadAll(r.Body)
				if err != nil {
					panic(err)
				}
				ret, err := json.Marshal(testOutlierReport)
				if err != nil {
					panic(err)
				}
				w.WriteHeader(200)
				w.Header().Set("Content-Type", "application/json")
				fmt.Fprintln(w, string(ret))
			default:
				w.WriteHeader(404)
				fmt.Fprintln(w, fmt.Sprintf("not found: %s %s", r.Method, path))
			}
		} else {
			w.WriteHeader(404)
			fmt.Fprintln(w, fmt.Sprintf("not found: %s %s", r.Method, path))
		}
	}

	return httptest.NewServer(http.HandlerFunc(f))
}

func TestFetchOutlierReport(t *testing.T) {
	server := testOutlierReportServer()
	defer server.Close()

	ac := &Config{
		TokenKey: "abc123",
		TokenApp: "test",
		URL:      server.URL,
	}
	apih, err := NewAPI(ac)
	if err != nil {
		t.Errorf("Expected no error, got '%v'", err)
	}

	t.Log("without CID")
	{
		cid := ""
		expectedError := errors.New("Invalid outlier report CID [none]")
		_, err := apih.FetchOutlierReport(CIDType(&cid))
		if err == nil {
			t.Fatalf("Expected error")
		}
		if err.Error() != expectedError.Error() {
			t.Fatalf("Expected %+v got '%+v'", expectedError, err)
		}
	}

	t.Log("with valid CID")
	{
		cid := "/outlier_report/1234"
		report, err := apih.FetchOutlierReport(CIDType(&cid))
		if err != nil {
			t.Fatalf("Expected no error, got '%v'", err)
		}

		actualType := reflect.TypeOf(report)
		expectedType := "*api.OutlierReport"
		if actualType.String() != expectedType {
			t.Fatalf("Expected %s, got %s", expectedType, actualType.String())
		}

		if report.CID != testOutlierReport.CID {
			t.Fatalf("CIDs do not match: %+v != %+v\n", report, testOutlierReport)
		}
	}

	t.Log("with invalid CID")
	{
		cid := "/invalid"
		expectedError := errors.New("Invalid outlier report CID [/invalid]")
		_, err := apih.FetchOutlierReport(CIDType(&cid))
		if err == nil {
			t.Fatalf("Expected error")
		}
		if err.Error() != expectedError.Error() {
			t.Fatalf("Expected %+v got '%+v'", expectedError, err)
		}
	}
}

func TestFetchOutlierReports(t *testing.T) {
	server := testOutlierReportServer()
	defer server.Close()

	ac := &Config{
		TokenKey: "abc123",
		TokenApp: "test",
		URL:      server.URL,
	}
	apih, err := NewAPI(ac)
	if err != nil {
		t.Errorf("Expected no error, got '%v'", err)
	}

	reports, err := apih.FetchOutlierReports()
	if err != nil {
		t.Fatalf("Expected no error, got '%v'", err)
	}

	actualType := reflect.TypeOf(reports)
	expectedType := "*[]api.OutlierReport"
	if actualType.String() != expectedType {
		t.Fatalf("Expected %s, got %s", expectedType, actualType.String())
	}

}

func TestCreateOutlierReport(t *testing.T) {
	server := testOutlierReportServer()
	defer server.Close()

	var apih *API

	ac := &Config{
		TokenKey: "abc123",
		TokenApp: "test",
		URL:      server.URL,
	}
	apih, err := NewAPI(ac)
	if err != nil {
		t.Errorf("Expected no error, got '%v'", err)
	}

	report, err := apih.CreateOutlierReport(&testOutlierReport)
	if err != nil {
		t.Fatalf("Expected no error, got '%v'", err)
	}

	actualType := reflect.TypeOf(report)
	expectedType := "*api.OutlierReport"
	if actualType.String() != expectedType {
		t.Fatalf("Expected %s, got %s", expectedType, actualType.String())
	}
}

func TestUpdateOutlierReport(t *testing.T) {
	server := testOutlierReportServer()
	defer server.Close()

	var apih *API

	ac := &Config{
		TokenKey: "abc123",
		TokenApp: "test",
		URL:      server.URL,
	}
	apih, err := NewAPI(ac)
	if err != nil {
		t.Errorf("Expected no error, got '%v'", err)
	}

	t.Log("valid OutlierReport")
	{
		report, err := apih.UpdateOutlierReport(&testOutlierReport)
		if err != nil {
			t.Fatalf("Expected no error, got '%v'", err)
		}

		actualType := reflect.TypeOf(report)
		expectedType := "*api.OutlierReport"
		if actualType.String() != expectedType {
			t.Fatalf("Expected %s, got %s", expectedType, actualType.String())
		}
	}

	t.Log("Test with invalid CID")
	{
		expectedError := errors.New("Invalid outlier report CID [/invalid]")
		x := &OutlierReport{CID: "/invalid"}
		_, err := apih.UpdateOutlierReport(x)
		if err == nil {
			t.Fatal("Expected an error")
		}
		if err.Error() != expectedError.Error() {
			t.Fatalf("Expected %+v got '%+v'", expectedError, err)
		}
	}
}

func TestDeleteOutlierReport(t *testing.T) {
	server := testOutlierReportServer()
	defer server.Close()

	var apih *API

	ac := &Config{
		TokenKey: "abc123",
		TokenApp: "test",
		URL:      server.URL,
	}
	apih, err := NewAPI(ac)
	if err != nil {
		t.Errorf("Expected no error, got '%v'", err)
	}

	t.Log("valid OutlierReport")
	{
		_, err := apih.DeleteOutlierReport(&testOutlierReport)
		if err != nil {
			t.Fatalf("Expected no error, got '%v'", err)
		}
	}

	t.Log("Test with invalid CID")
	{
		expectedError := errors.New("Invalid outlier report CID [/invalid]")
		x := &OutlierReport{CID: "/invalid"}
		_, err := apih.UpdateOutlierReport(x)
		if err == nil {
			t.Fatal("Expected an error")
		}
		if err.Error() != expectedError.Error() {
			t.Fatalf("Expected %+v got '%+v'", expectedError, err)
		}
	}
}

func TestSearchOutlierReports(t *testing.T) {
	server := testOutlierReportServer()
	defer server.Close()

	var apih *API

	ac := &Config{
		TokenKey: "abc123",
		TokenApp: "test",
		URL:      server.URL,
	}
	apih, err := NewAPI(ac)
	if err != nil {
		t.Errorf("Expected no error, got '%v'", err)
	}

	search := SearchQueryType("requests per second")
	filter := SearchFilterType(map[string][]string{"f_tags_has": []string{"service:web"}})

	t.Log("no search, no filter")
	{
		reports, err := apih.SearchOutlierReports(nil, nil)
		if err != nil {
			t.Fatalf("Expected no error, got '%v'", err)
		}

		actualType := reflect.TypeOf(reports)
		expectedType := "*[]api.OutlierReport"
		if actualType.String() != expectedType {
			t.Fatalf("Expected %s, got %s", expectedType, actualType.String())
		}
	}

	t.Log("search, no filter")
	{
		reports, err := apih.SearchOutlierReports(&search, nil)
		if err != nil {
			t.Fatalf("Expected no error, got '%v'", err)
		}

		actualType := reflect.TypeOf(reports)
		expectedType := "*[]api.OutlierReport"
		if actualType.String() != expectedType {
			t.Fatalf("Expected %s, got %s", expectedType, actualType.String())
		}
	}

	t.Log("no search, filter")
	{
		reports, err := apih.SearchOutlierReports(nil, &filter)
		if err != nil {
			t.Fatalf("Expected no error, got '%v'", err)
		}

		actualType := reflect.TypeOf(reports)
		expectedType := "*[]api.OutlierReport"
		if actualType.String() != expectedType {
			t.Fatalf("Expected %s, got %s", expectedType, actualType.String())
		}
	}

	t.Log("search, filter")
	{
		reports, err := apih.SearchOutlierReports(&search, &filter)
		if err != nil {
			t.Fatalf("Expected no error, got '%v'", err)
		}

		actualType := reflect.TypeOf(reports)
		expectedType := "*[]api.OutlierReport"
		if actualType.String() != expectedType {
			t.Fatalf("Expected %s, got %s", expectedType, actualType.String())
		}
	}
}
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.164"
<commit_msg>functions: 0.3.165 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.165"
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.460"
<commit_msg>fnserver: 0.3.461 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.461"
<|endoftext|>


<commit_before>package godingtalk

import (
	"testing"
)

func TestEncryption(t *testing.T) {
	str, err := c.Encrypt("Hello")
    if err!=nil {
        t.Error(err)
    } else {
        t.Log(str)
    }
}
<commit_msg>fixed test case<commit_after>package godingtalk

import (
	"testing"
)

func TestEncryption(t *testing.T) {
	str, err := c.Encrypt("Hello")
    if err!=nil {
        t.Log(err)
    } else {
        t.Log(str)
    }
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"strings"

	"github.com/drone/drone/shared/build/log"
)

type SshConfigFileSection struct {
	Host         string
	ForwardAgent string
	User         string
	HostName     string
	Port         string
}

// parseSshConfigFileSection parses a section from the ~/.ssh/config file
func parseSshConfigFileSection(content string) *SshConfigFileSection {
	section := &SshConfigFileSection{}

	for n, line := range strings.Split(content, "\n") {
		line = strings.TrimSpace(line)
		if line == "" {
			continue
		}

		if n == 0 {
			section.Host = line
		} else if strings.HasPrefix(line, "ForwardAgent") {
			section.ForwardAgent = strings.TrimSpace(strings.TrimPrefix(line, "ForwardAgent"))
		} else if strings.HasPrefix(line, "User") {
			section.User = strings.TrimSpace(strings.TrimPrefix(line, "User"))
		} else if strings.HasPrefix(line, "HostName") {
			section.HostName = strings.TrimSpace(strings.TrimPrefix(line, "HostName"))
		} else if strings.HasPrefix(line, "Port") {
			section.Port = strings.TrimSpace(strings.TrimPrefix(line, "Port"))
		}
	}
	log.Debugf("parsed ssh config file section: %s", section.Host)
	return section
}

// parseSshConfigFile parses the ~/.ssh/config file and build a list of section
func parseSshConfigFile(path string) (map[string]*SshConfigFileSection, error) {
	log.Debugf("parsing ssh config file: %s", path)
	content, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}
	sections := make(map[string]*SshConfigFileSection)
	for _, split := range strings.Split(string(content), "Host ") {
		split = strings.TrimSpace(split)
		if split == "" {
			continue
		}

		section := parseSshConfigFileSection(split)
		sections[section.Host] = section
	}

	return sections, nil
}
<commit_msg>use logrus<commit_after>package main

import (
	"io/ioutil"
	"strings"

	log "github.com/Sirupsen/logrus"
)

type SshConfigFileSection struct {
	Host         string
	ForwardAgent string
	User         string
	HostName     string
	Port         string
}

// parseSshConfigFileSection parses a section from the ~/.ssh/config file
func parseSshConfigFileSection(content string) *SshConfigFileSection {
	section := &SshConfigFileSection{}

	for n, line := range strings.Split(content, "\n") {
		line = strings.TrimSpace(line)
		if line == "" {
			continue
		}

		if n == 0 {
			section.Host = line
		} else if strings.HasPrefix(line, "ForwardAgent") {
			section.ForwardAgent = strings.TrimSpace(strings.TrimPrefix(line, "ForwardAgent"))
		} else if strings.HasPrefix(line, "User") {
			section.User = strings.TrimSpace(strings.TrimPrefix(line, "User"))
		} else if strings.HasPrefix(line, "HostName") {
			section.HostName = strings.TrimSpace(strings.TrimPrefix(line, "HostName"))
		} else if strings.HasPrefix(line, "Port") {
			section.Port = strings.TrimSpace(strings.TrimPrefix(line, "Port"))
		}
	}
	log.Debugf("parsed ssh config file section: %s", section.Host)
	return section
}

// parseSshConfigFile parses the ~/.ssh/config file and build a list of section
func parseSshConfigFile(path string) (map[string]*SshConfigFileSection, error) {
	log.Debugf("parsing ssh config file: %s", path)
	content, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}
	sections := make(map[string]*SshConfigFileSection)
	for _, split := range strings.Split(string(content), "Host ") {
		split = strings.TrimSpace(split)
		if split == "" {
			continue
		}

		section := parseSshConfigFileSection(split)
		sections[section.Host] = section
	}

	return sections, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
)

type Config struct {
	TrelloUser       string   `json:"trello_user"`
	TrelloKey        string   `json:"trello_key"`
	TrelloToken      string   `json:"trello_token"`
	BoardName        string   `json:"board_name"`
	StartListName    string   `json:"start_list_name"`
	FinishedListName string   `json:"finished_list_name"`
	NotifyChannel    string   `json:"notify_channel"`
	InfoChannel      string   `json:"info_channel"`
	ReportDays       int      `json:"report_days"`
	ReportLists      []string `json:"report_lists"`
	SlackToken       string   `json:"slack_token"`
	ListenURL        string   `json:"listen_url"`
	Port             string   `json:"port"`
}

func LoadConfig(filename string) Config {

	var config Config

	file, err := ioutil.ReadFile(filename)
	if err != nil {
		fmt.Printf("Error reading config file: %v\n", err)
	}

	json.Unmarshal(file, &config)

	return config
}
<commit_msg>Update config system<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
)

type Config struct {
	TrelloUser  string `json:"trello_user"`
	TrelloKey   string `json:"trello_key"`
	TrelloToken string `json:"trello_token"`
	SlackToken  string `json:"slack_token"`
	ListenURL   string `json:"listen_url"`
	Port        string `json:"port"`
}

type BoardConfig struct {
	BoardName         string       `json:"board_name"`
	NotifyChannelName string       `json:"notify_channel_name"`
	ListConfigs       []ListConfig `json:"list_configs"`
}

type ListConfig struct {
	ListName        string `json:"list_name"`
	OnAction        string `json:"on_action"`
	MessageTemplate string `json:"message_template"`
}

func LoadConfig(filename string) Config {

	var config Config

	file, err := ioutil.ReadFile(filename)
	if err != nil {
		fmt.Printf("Error reading config file: %v\n", err)
	}

	json.Unmarshal(file, &config)

	return config
}
<|endoftext|>


<commit_before>package cfg

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"reflect"
	"strings"
	"text/template"

	"launchpad.net/goyaml"
)

func LoadConfig(configPath string, configStruct interface{}) error {
	bytes, err := ioutil.ReadFile(configPath)
	if err != nil {
		return err
	}

	bytes, err = substitute(bytes)
	if err != nil {
		return err
	}

	if err = goyaml.Unmarshal(bytes, configStruct); err != nil {
		return err
	}

	if err = validate(configStruct); err != nil {
		return err
	}

	return nil
}

type templateData struct {
	Env map[string]string
}

func substitute(in []byte) ([]byte, error) {
	t, err := template.New("config").Parse(string(in))
	if err != nil {
		return nil, err
	}

	data := &templateData{
		Env: make(map[string]string),
	}

	values := os.Environ()
	for _, val := range values {
		keyval := strings.SplitN(val, "=", 2)
		if len(keyval) != 2 {
			continue
		}
		data.Env[keyval[0]] = keyval[1]
	}

	buffer := &bytes.Buffer{}
	if err = t.Execute(buffer, data); err != nil {
		return nil, err
	}

	return buffer.Bytes(), nil
}

func validate(configStruct interface{}) error {
	return validateStruct(
		reflect.TypeOf(configStruct).Elem(),
		reflect.ValueOf(configStruct).Elem())
}

func validateStruct(typ reflect.Type, val reflect.Value) error {
	for idx := 0; idx < val.NumField(); idx++ {
		field := typ.Field(idx)
		if field.Type.Kind() == reflect.Struct {
			if err := validateStruct(val.Field(idx).Type(), val.Field(idx)); err != nil {
				return err
			}
		} else if field.Type.Kind() == reflect.Bool || TypeIsNumeric(field.Type.Kind()) { // no way to tell if boolean field was provided or not
			continue
		} else {
			if field.Tag.Get("config") != "optional" {
				if val.Field(idx).Len() == 0 {
					return errors.New(
						fmt.Sprintf("Missing required config field: %v", field.Name))
				}
			}
		}
	}
	return nil
}

var NumericTypes = []reflect.Kind{reflect.Int,
	reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
	reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64,
	reflect.Float32, reflect.Float64}

func TypeIsNumeric(typ reflect.Kind) bool {
	for _, numericType := range NumericTypes {
		if typ == numericType {
			return true
		}
	}
	return false
}
<commit_msg>Update goyaml dependency: it moved<commit_after>package cfg

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"reflect"
	"strings"
	"text/template"

	"github.com/go-yaml/yaml"
)

func LoadConfig(configPath string, configStruct interface{}) error {
	bytes, err := ioutil.ReadFile(configPath)
	if err != nil {
		return err
	}

	bytes, err = substitute(bytes)
	if err != nil {
		return err
	}

	if err = yaml.Unmarshal(bytes, configStruct); err != nil {
		return err
	}

	if err = validate(configStruct); err != nil {
		return err
	}

	return nil
}

type templateData struct {
	Env map[string]string
}

func substitute(in []byte) ([]byte, error) {
	t, err := template.New("config").Parse(string(in))
	if err != nil {
		return nil, err
	}

	data := &templateData{
		Env: make(map[string]string),
	}

	values := os.Environ()
	for _, val := range values {
		keyval := strings.SplitN(val, "=", 2)
		if len(keyval) != 2 {
			continue
		}
		data.Env[keyval[0]] = keyval[1]
	}

	buffer := &bytes.Buffer{}
	if err = t.Execute(buffer, data); err != nil {
		return nil, err
	}

	return buffer.Bytes(), nil
}

func validate(configStruct interface{}) error {
	return validateStruct(
		reflect.TypeOf(configStruct).Elem(),
		reflect.ValueOf(configStruct).Elem())
}

func validateStruct(typ reflect.Type, val reflect.Value) error {
	for idx := 0; idx < val.NumField(); idx++ {
		field := typ.Field(idx)
		if field.Type.Kind() == reflect.Struct {
			if err := validateStruct(val.Field(idx).Type(), val.Field(idx)); err != nil {
				return err
			}
		} else if field.Type.Kind() == reflect.Bool || TypeIsNumeric(field.Type.Kind()) { // no way to tell if boolean field was provided or not
			continue
		} else {
			if field.Tag.Get("config") != "optional" {
				if val.Field(idx).Len() == 0 {
					return errors.New(
						fmt.Sprintf("Missing required config field: %v", field.Name))
				}
			}
		}
	}
	return nil
}

var NumericTypes = []reflect.Kind{reflect.Int,
	reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
	reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64,
	reflect.Float32, reflect.Float64}

func TypeIsNumeric(typ reflect.Kind) bool {
	for _, numericType := range NumericTypes {
		if typ == numericType {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package runc

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/inconshreveable/log15"
	"github.com/polydawn/gosh"
	"go.polydawn.net/meep"

	"go.polydawn.net/repeatr/api/def"
	"go.polydawn.net/repeatr/core/assets"
	"go.polydawn.net/repeatr/core/executor"
	"go.polydawn.net/repeatr/core/executor/basicjob"
	"go.polydawn.net/repeatr/core/executor/cradle"
	"go.polydawn.net/repeatr/core/executor/util"
	"go.polydawn.net/repeatr/lib/flak"
	"go.polydawn.net/repeatr/lib/streamer"
)

// interface assertion
var _ executor.Executor = &Executor{}

type Executor struct {
	workspacePath string
}

func (e *Executor) Configure(workspacePath string) {
	e.workspacePath = workspacePath
}

func (e *Executor) Start(f def.Formula, id executor.JobID, stdin io.Reader, log log15.Logger) executor.Job {
	// TODO this function sig and its interface are long overdue for an aggressive refactor.
	// - `journal` is Rong.  The streams mux should be accessible after this function's scope!
	//   - either that or it's time to get cracking on saving the stream mux as an output
	// - `journal` should still be a thing, but it should be a logger.
	// - All these other values should move along in a `Job` struct
	//   - `BasicJob` sorta started, but is drunk:
	//      - if we're gonna have that, it's incomplete on the inputs
	//      - for some reason it mixes in responsibility for waiting for some of the ouputs
	//      - that use of channels and public fields is stupidly indefensive
	//   - The current `Job` interface is in the wrong package
	// - almost all of the scopes in these functions is wrong
	//   - they should be realigned until they actually assist the defers and cleanups
	//     - e.g. withErrorCapture, withJobWorkPath, withFilesystems, etc

	// Fill in default config for anything still blank.
	f = *cradle.ApplyDefaults(&f)

	job := basicjob.New(id)
	jobReady := make(chan struct{})

	go func() {
		// Run the formula in a temporary directory
		flak.WithDir(func(dir string) {

			// spool our output to a muxed stream
			var strm streamer.Mux
			strm = streamer.CborFileMux(filepath.Join(dir, "log"))
			outS := strm.Appender(1)
			errS := strm.Appender(2)
			job.Streams = strm
			defer func() {
				// Regardless of how the job ends (or even if it fails the remaining setup), output streams must be terminated.
				outS.Close()
				errS.Close()
			}()

			// Job is ready to stream process output
			close(jobReady)

			job.Result = e.Run(f, job, dir, stdin, outS, errS, log)
		}, e.workspacePath, "job", string(job.Id()))

		// Directory is clean; job complete
		close(job.WaitChan)
	}()

	<-jobReady
	return job
}

// Executes a job, catching any panics.
func (e *Executor) Run(f def.Formula, j executor.Job, d string, stdin io.Reader, outS, errS io.WriteCloser, journal log15.Logger) executor.JobResult {
	r := executor.JobResult{
		ID:       j.Id(),
		ExitCode: -1,
	}

	r.Error = meep.RecoverPanics(func() {
		e.Execute(f, j, d, &r, stdin, outS, errS, journal)
	})
	return r
}

// Execute a formula in a specified directory. MAY PANIC.
func (e *Executor) Execute(formula def.Formula, job executor.Job, jobPath string, result *executor.JobResult, stdin io.Reader, stdout, stderr io.WriteCloser, journal log15.Logger) {
	rootfsPath := filepath.Join(jobPath, "rootfs")

	// Prepare inputs
	transmat := util.DefaultTransmat()
	inputArenas := util.ProvisionInputs(transmat, formula.Inputs, journal)
	util.ProvisionOutputs(formula.Outputs, rootfsPath, journal)

	// Assemble filesystem
	assembly := util.AssembleFilesystem(
		util.BestAssembler(),
		rootfsPath,
		formula.Inputs,
		inputArenas,
		formula.Action.Escapes.Mounts,
		journal,
	)
	defer assembly.Teardown()
	if formula.Action.Cradle == nil || *(formula.Action.Cradle) == true {
		cradle.MakeCradle(rootfsPath, formula)
	}

	// Emit config for runc.
	runcConfigJsonPath := filepath.Join(jobPath, "config.json")
	cfg := EmitRuncConfigStruct(formula, job, rootfsPath, stdin != nil)
	buf, err := json.Marshal(cfg)
	if err != nil {
		panic(executor.UnknownError.Wrap(err))
	}
	ioutil.WriteFile(runcConfigJsonPath, buf, 0600)

	// Routing logs through a fifo appears to work, but we're going to use a file as a buffer anyway:
	//  in the event of nasty breakdowns, it's preferable that the runc log remain readable even if repeatr was the process to end first.
	logPath := filepath.Join(jobPath, "runc-debug.log")

	// Get handle to invokable runc plugin.
	runcPath := filepath.Join(assets.Get("runc"), "runc")

	// Prepare command to exec
	args := []string{
		"--root", filepath.Join(e.workspacePath, "shared"), // a tmpfs would be appropriate
		"--log", logPath,
		"--log-format", "json",
		"run",
		"--bundle", jobPath,
		string(job.Id()),
	}
	cmd := exec.Command(runcPath, args...)
	cmd.Stdin = stdin
	cmd.Stdout = stdout
	cmd.Stderr = stderr

	// launch execution.
	// transform gosh's typed errors to repeatr's hierarchical errors.
	// this is... not untroubled code: since we're invoking a helper that's then
	//  proxying the exec even further, most errors are fatal (the mapping here is
	//   very different than in e.g. chroot executor, and provides much less meaning).
	startedExec := time.Now()
	journal.Info("Beginning execution!")
	var proc gosh.Proc
	meep.Try(func() {
		proc = gosh.ExecProcCmd(cmd)
	}, meep.TryPlan{
		{ByType: gosh.NoSuchCommandError{}, Handler: func(err error) {
			panic(executor.ConfigError.New("runc binary is missing"))
		}},
		{ByType: gosh.NoArgumentsError{}, Handler: func(err error) {
			panic(executor.UnknownError.Wrap(err))
		}},
		{ByType: gosh.NoSuchCwdError{}, Handler: func(err error) {
			panic(executor.UnknownError.Wrap(err))
		}},
		{ByType: gosh.ProcMonitorError{}, Handler: func(err error) {
			panic(executor.TaskExecError.Wrap(err))
		}},
		{CatchAny: true, Handler: func(err error) {
			panic(executor.UnknownError.Wrap(err))
		}},
	})

	var runcLog io.ReadCloser
	runcLog, err = os.OpenFile(logPath, os.O_CREATE|os.O_RDONLY, 0644)
	// note this open races child; doesn't matter.
	if err != nil {
		panic(executor.TaskExecError.New("failed to tail runc log: %s", err))
	}
	// swaddle the file in userland-interruptable reader;
	//  obviously we don't want to stop watching the logs when we hit the end of the still-growing file.
	runcLog = streamer.NewTailReader(runcLog)

	// Proxy runc's logs out in realtime; also, detect errors and exit statuses from the stream.
	var realError error
	var someError bool // see the "NOTE WELL" section below -.-
	var tailerDone sync.WaitGroup
	tailerDone.Add(1)
	go func() {
		defer tailerDone.Done()
		dec := json.NewDecoder(runcLog)
		for {
			// Parse log lines.
			var logMsg map[string]string
			err := dec.Decode(&logMsg)
			if err != nil {
				if err == io.EOF {
					return
				}
				panic(executor.TaskExecError.New("unparsable log from runc: %s", err))
			}
			// remap
			if _, ok := logMsg["msg"]; !ok {
				logMsg["msg"] = ""
			}
			ctx := log15.Ctx{}
			for k, v := range logMsg {
				if k == "msg" {
					continue
				}
				ctx["runc-"+k] = v
			}

			//fmt.Printf("\n\n---\n%s\n---\n\n", logMsg["msg"])

			// Attempt to filter and normalize errors.
			// We want to be clear in representing which category of errors are coming up:
			//
			//  - Type 1.a: Exit codes of the contained user process.
			//    - These aren't errors that we raise as such: they're just an int code to report.
			//  - Type 1.b: Errors from invalid user configuration (e.g. no such executable, which prevents the process from ever starting) (we expect these to be reproducible!).
			//    - These kinds of errors should be mapped onto clear types themselves: we want a "NoSuchCwdError", not just a string vomit.
			//  - Type 2: Errors from runc being unable to function (e.g. maybe your kernel doesn't support cgroups, or other bizarre and serious issue?), where hopefully we can advise the user of this in a clear fashion.
			//  - Type 3: Runc crashing in an unrecognized way (which should result in either patches to our recognizers, or bugs filed upstream to runc).
			//
			// This is HARD.
			//
			// NOTE WELL: we cannot guarantee to capture all semantic runc failure modes.
			//  Errors may slip through with exit status 1: there are still many fail states
			//  which runc does not log with sufficient consistency or a sufficiently separate
			//  control channel for us to be able to reliably disambiguate them from stderr
			//  output of a successfully executing job!
			//
			// We have whitelisted recognizers for what we can, but oddities may remain.
			for _, tr := range []struct {
				prefix, suffix string
				err            error
			}{
				{"container_linux.go:262: starting container process caused \"exec: \\\"", ": executable file not found in $PATH\"\n",
					executor.NoSuchCommandError.New("command %q not found", formula.Action.Entrypoint[0])},
				{"container_linux.go:262: starting container process caused \"exec: \\\"", ": no such file or directory\"\n",
					executor.NoSuchCommandError.New("command %q not found", formula.Action.Entrypoint[0])},
				{"container_linux.go:262: starting container process caused \"chdir to cwd (\\\"", "\\\") set in config.json failed: not a directory\"\n",
					executor.NoSuchCwdError.New("cannot set cwd to %q: no such file or directory", formula.Action.Cwd)},
				{"container_linux.go:262: starting container process caused \"chdir to cwd (\\\"", "\\\") set in config.json failed: no such file or directory\"\n",
					executor.NoSuchCwdError.New("cannot set cwd to %q: no such file or directory", formula.Action.Cwd)},
				// Note: Some other errors were previously raised in the pattern of `executor.TaskExecError.New("runc cannot operate in this environment!")`,
				// but none of these are currently here because we cachebusted our known error strings when upgrading runc.
			} {
				if !strings.HasPrefix(logMsg["msg"], tr.prefix) {
					continue
				}
				if !strings.HasSuffix(logMsg["msg"], tr.suffix) {
					continue
				}
				realError = tr.err
				break
			}

			// Log again.
			// The level of alarm we raise depends:
			//  - With runc, everything we hear is at least a warning;
			//  - If we recognized it above, it's no more than a warning;
			//  - If we *didn't* recognize and handle it explicitly, and
			//    we can see a clear indication it's fatal, then log big and red.
			if realError == nil && ctx["runc-level"] == "error" {
				journal.Error(logMsg["msg"], ctx)
				someError = true
			} else {
				journal.Warn(logMsg["msg"], ctx)
			}
		}
	}()

	// Wait for the job to complete.
	result.ExitCode = proc.GetExitCode()
	journal.Info("Execution done!",
		"elapsed", time.Now().Sub(startedExec).Seconds(),
	)
	// Tell the log tailer to drain as soon as the proc exits.
	runcLog.Close()
	// Wait for the tailer routine to drain & exit (this sync guards the err vars).
	tailerDone.Wait()

	// If we had a CnC error (rather than the real subprocess exit code):
	//  - reset code to -1 because the runc exit code wasn't really from the job command
	//  - finally, raise the error
	// FIXME we WISH we could zero the output buffers because runc pushes duplicate error messages
	//  down a channel that's indistinguishable from the application stderr... but that's tricky for several reasons:
	//  - we support streaming them out, right?
	//  - that means we'd have to have been blocking them already; we can't zero retroactively.
	//  - there's no "all clear" signal available from runc that would let us know we're clear to start flushing the stream if we blocked it.
	//  - So, we're unable to pass the executor compat tests until patches to runc clean up this behavior.
	if someError && realError == nil {
		realError = executor.UnknownError.New("runc errored in an unrecognized fashion")
	}
	if realError != nil {
		result.ExitCode = -1
		panic(realError)
	}

	// Save outputs
	result.Outputs = util.PreserveOutputs(transmat, formula.Outputs, rootfsPath, journal)
}
<commit_msg>Enable runc debug logging.<commit_after>package runc

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/inconshreveable/log15"
	"github.com/polydawn/gosh"
	"go.polydawn.net/meep"

	"go.polydawn.net/repeatr/api/def"
	"go.polydawn.net/repeatr/core/assets"
	"go.polydawn.net/repeatr/core/executor"
	"go.polydawn.net/repeatr/core/executor/basicjob"
	"go.polydawn.net/repeatr/core/executor/cradle"
	"go.polydawn.net/repeatr/core/executor/util"
	"go.polydawn.net/repeatr/lib/flak"
	"go.polydawn.net/repeatr/lib/streamer"
)

// interface assertion
var _ executor.Executor = &Executor{}

type Executor struct {
	workspacePath string
}

func (e *Executor) Configure(workspacePath string) {
	e.workspacePath = workspacePath
}

func (e *Executor) Start(f def.Formula, id executor.JobID, stdin io.Reader, log log15.Logger) executor.Job {
	// TODO this function sig and its interface are long overdue for an aggressive refactor.
	// - `journal` is Rong.  The streams mux should be accessible after this function's scope!
	//   - either that or it's time to get cracking on saving the stream mux as an output
	// - `journal` should still be a thing, but it should be a logger.
	// - All these other values should move along in a `Job` struct
	//   - `BasicJob` sorta started, but is drunk:
	//      - if we're gonna have that, it's incomplete on the inputs
	//      - for some reason it mixes in responsibility for waiting for some of the ouputs
	//      - that use of channels and public fields is stupidly indefensive
	//   - The current `Job` interface is in the wrong package
	// - almost all of the scopes in these functions is wrong
	//   - they should be realigned until they actually assist the defers and cleanups
	//     - e.g. withErrorCapture, withJobWorkPath, withFilesystems, etc

	// Fill in default config for anything still blank.
	f = *cradle.ApplyDefaults(&f)

	job := basicjob.New(id)
	jobReady := make(chan struct{})

	go func() {
		// Run the formula in a temporary directory
		flak.WithDir(func(dir string) {

			// spool our output to a muxed stream
			var strm streamer.Mux
			strm = streamer.CborFileMux(filepath.Join(dir, "log"))
			outS := strm.Appender(1)
			errS := strm.Appender(2)
			job.Streams = strm
			defer func() {
				// Regardless of how the job ends (or even if it fails the remaining setup), output streams must be terminated.
				outS.Close()
				errS.Close()
			}()

			// Job is ready to stream process output
			close(jobReady)

			job.Result = e.Run(f, job, dir, stdin, outS, errS, log)
		}, e.workspacePath, "job", string(job.Id()))

		// Directory is clean; job complete
		close(job.WaitChan)
	}()

	<-jobReady
	return job
}

// Executes a job, catching any panics.
func (e *Executor) Run(f def.Formula, j executor.Job, d string, stdin io.Reader, outS, errS io.WriteCloser, journal log15.Logger) executor.JobResult {
	r := executor.JobResult{
		ID:       j.Id(),
		ExitCode: -1,
	}

	r.Error = meep.RecoverPanics(func() {
		e.Execute(f, j, d, &r, stdin, outS, errS, journal)
	})
	return r
}

// Execute a formula in a specified directory. MAY PANIC.
func (e *Executor) Execute(formula def.Formula, job executor.Job, jobPath string, result *executor.JobResult, stdin io.Reader, stdout, stderr io.WriteCloser, journal log15.Logger) {
	rootfsPath := filepath.Join(jobPath, "rootfs")

	// Prepare inputs
	transmat := util.DefaultTransmat()
	inputArenas := util.ProvisionInputs(transmat, formula.Inputs, journal)
	util.ProvisionOutputs(formula.Outputs, rootfsPath, journal)

	// Assemble filesystem
	assembly := util.AssembleFilesystem(
		util.BestAssembler(),
		rootfsPath,
		formula.Inputs,
		inputArenas,
		formula.Action.Escapes.Mounts,
		journal,
	)
	defer assembly.Teardown()
	if formula.Action.Cradle == nil || *(formula.Action.Cradle) == true {
		cradle.MakeCradle(rootfsPath, formula)
	}

	// Emit config for runc.
	runcConfigJsonPath := filepath.Join(jobPath, "config.json")
	cfg := EmitRuncConfigStruct(formula, job, rootfsPath, stdin != nil)
	buf, err := json.Marshal(cfg)
	if err != nil {
		panic(executor.UnknownError.Wrap(err))
	}
	ioutil.WriteFile(runcConfigJsonPath, buf, 0600)

	// Routing logs through a fifo appears to work, but we're going to use a file as a buffer anyway:
	//  in the event of nasty breakdowns, it's preferable that the runc log remain readable even if repeatr was the process to end first.
	logPath := filepath.Join(jobPath, "runc-debug.log")

	// Get handle to invokable runc plugin.
	runcPath := filepath.Join(assets.Get("runc"), "runc")

	// Prepare command to exec
	args := []string{
		"--root", filepath.Join(e.workspacePath, "shared"), // a tmpfs would be appropriate
		"--debug",
		"--log", logPath,
		"--log-format", "json",
		"run",
		"--bundle", jobPath,
		string(job.Id()),
	}
	cmd := exec.Command(runcPath, args...)
	cmd.Stdin = stdin
	cmd.Stdout = stdout
	cmd.Stderr = stderr

	// launch execution.
	// transform gosh's typed errors to repeatr's hierarchical errors.
	// this is... not untroubled code: since we're invoking a helper that's then
	//  proxying the exec even further, most errors are fatal (the mapping here is
	//   very different than in e.g. chroot executor, and provides much less meaning).
	startedExec := time.Now()
	journal.Info("Beginning execution!")
	var proc gosh.Proc
	meep.Try(func() {
		proc = gosh.ExecProcCmd(cmd)
	}, meep.TryPlan{
		{ByType: gosh.NoSuchCommandError{}, Handler: func(err error) {
			panic(executor.ConfigError.New("runc binary is missing"))
		}},
		{ByType: gosh.NoArgumentsError{}, Handler: func(err error) {
			panic(executor.UnknownError.Wrap(err))
		}},
		{ByType: gosh.NoSuchCwdError{}, Handler: func(err error) {
			panic(executor.UnknownError.Wrap(err))
		}},
		{ByType: gosh.ProcMonitorError{}, Handler: func(err error) {
			panic(executor.TaskExecError.Wrap(err))
		}},
		{CatchAny: true, Handler: func(err error) {
			panic(executor.UnknownError.Wrap(err))
		}},
	})

	var runcLog io.ReadCloser
	runcLog, err = os.OpenFile(logPath, os.O_CREATE|os.O_RDONLY, 0644)
	// note this open races child; doesn't matter.
	if err != nil {
		panic(executor.TaskExecError.New("failed to tail runc log: %s", err))
	}
	// swaddle the file in userland-interruptable reader;
	//  obviously we don't want to stop watching the logs when we hit the end of the still-growing file.
	runcLog = streamer.NewTailReader(runcLog)

	// Proxy runc's logs out in realtime; also, detect errors and exit statuses from the stream.
	var realError error
	var someError bool // see the "NOTE WELL" section below -.-
	var tailerDone sync.WaitGroup
	tailerDone.Add(1)
	go func() {
		defer tailerDone.Done()
		dec := json.NewDecoder(runcLog)
		for {
			// Parse log lines.
			var logMsg map[string]interface{}
			err := dec.Decode(&logMsg)
			if err != nil {
				if err == io.EOF {
					return
				}
				panic(executor.TaskExecError.New("unparsable log from runc: %s", err))
			}
			// remap
			if _, ok := logMsg["msg"]; !ok {
				logMsg["msg"] = ""
			}
			ctx := log15.Ctx{}
			for k, v := range logMsg {
				if k == "msg" {
					continue
				}
				ctx["runc-"+k] = v
			}

			//fmt.Printf("\n\n---\n%s\n---\n\n", logMsg["msg"])

			// Attempt to filter and normalize errors.
			// We want to be clear in representing which category of errors are coming up:
			//
			//  - Type 1.a: Exit codes of the contained user process.
			//    - These aren't errors that we raise as such: they're just an int code to report.
			//  - Type 1.b: Errors from invalid user configuration (e.g. no such executable, which prevents the process from ever starting) (we expect these to be reproducible!).
			//    - These kinds of errors should be mapped onto clear types themselves: we want a "NoSuchCwdError", not just a string vomit.
			//  - Type 2: Errors from runc being unable to function (e.g. maybe your kernel doesn't support cgroups, or other bizarre and serious issue?), where hopefully we can advise the user of this in a clear fashion.
			//  - Type 3: Runc crashing in an unrecognized way (which should result in either patches to our recognizers, or bugs filed upstream to runc).
			//
			// This is HARD.
			//
			// NOTE WELL: we cannot guarantee to capture all semantic runc failure modes.
			//  Errors may slip through with exit status 1: there are still many fail states
			//  which runc does not log with sufficient consistency or a sufficiently separate
			//  control channel for us to be able to reliably disambiguate them from stderr
			//  output of a successfully executing job!
			//
			// We have whitelisted recognizers for what we can, but oddities may remain.
			for _, tr := range []struct {
				prefix, suffix string
				err            error
			}{
				{"container_linux.go:262: starting container process caused \"exec: \\\"", ": executable file not found in $PATH\"\n",
					executor.NoSuchCommandError.New("command %q not found", formula.Action.Entrypoint[0])},
				{"container_linux.go:262: starting container process caused \"exec: \\\"", ": no such file or directory\"\n",
					executor.NoSuchCommandError.New("command %q not found", formula.Action.Entrypoint[0])},
				{"container_linux.go:262: starting container process caused \"chdir to cwd (\\\"", "\\\") set in config.json failed: not a directory\"\n",
					executor.NoSuchCwdError.New("cannot set cwd to %q: no such file or directory", formula.Action.Cwd)},
				{"container_linux.go:262: starting container process caused \"chdir to cwd (\\\"", "\\\") set in config.json failed: no such file or directory\"\n",
					executor.NoSuchCwdError.New("cannot set cwd to %q: no such file or directory", formula.Action.Cwd)},
				// Note: Some other errors were previously raised in the pattern of `executor.TaskExecError.New("runc cannot operate in this environment!")`,
				// but none of these are currently here because we cachebusted our known error strings when upgrading runc.
			} {
				if !strings.HasPrefix(logMsg["msg"].(string), tr.prefix) {
					continue
				}
				if !strings.HasSuffix(logMsg["msg"].(string), tr.suffix) {
					continue
				}
				realError = tr.err
				break
			}

			// Log again.
			// The level of alarm we raise depends:
			//  - If it's clearly flagged debug level, accept that;
			//  - If we recognized it above, it's no more than a warning;
			//  - If we *didn't* recognize and handle it explicitly, and
			//    we can see a clear indication it's fatal, then log big and red.
			switch ctx["runc-level"] {
			case "debug":
				journal.Debug(logMsg["msg"].(string), ctx)
			default:
				fallthrough
			case "error":
				if realError == nil {
					journal.Error(logMsg["msg"].(string), ctx)
					someError = true
					break
				}
				fallthrough
			case "warning":
				journal.Warn(logMsg["msg"].(string), ctx)
			}
		}
	}()

	// Wait for the job to complete.
	result.ExitCode = proc.GetExitCode()
	journal.Info("Execution done!",
		"elapsed", time.Now().Sub(startedExec).Seconds(),
	)
	// Tell the log tailer to drain as soon as the proc exits.
	runcLog.Close()
	// Wait for the tailer routine to drain & exit (this sync guards the err vars).
	tailerDone.Wait()

	// If we had a CnC error (rather than the real subprocess exit code):
	//  - reset code to -1 because the runc exit code wasn't really from the job command
	//  - finally, raise the error
	// FIXME we WISH we could zero the output buffers because runc pushes duplicate error messages
	//  down a channel that's indistinguishable from the application stderr... but that's tricky for several reasons:
	//  - we support streaming them out, right?
	//  - that means we'd have to have been blocking them already; we can't zero retroactively.
	//  - there's no "all clear" signal available from runc that would let us know we're clear to start flushing the stream if we blocked it.
	//  - So, we're unable to pass the executor compat tests until patches to runc clean up this behavior.
	if someError && realError == nil {
		realError = executor.UnknownError.New("runc errored in an unrecognized fashion")
	}
	if realError != nil {
		result.ExitCode = -1
		panic(realError)
	}

	// Save outputs
	result.Outputs = util.PreserveOutputs(transmat, formula.Outputs, rootfsPath, journal)
}
<|endoftext|>


<commit_before>package makex

import (
	"flag"
	"os"
	"runtime"
	"time"

	"sourcegraph.com/sourcegraph/rwvfs"
)

type Config struct {
	FS           FileSystem
	ParallelJobs int
	Verbose      bool
	DryRun       bool
}

var Default = Config{
	ParallelJobs: 1,
}

func (c *Config) fs() FileSystem {
	if c.FS != nil {
		return c.FS
	}
	dir, err := os.Getwd()
	if err != nil {
		dir = "."
	}
	return NewFileSystem(rwvfs.OS(dir))
}

func (c *Config) pathExists(path string) (bool, error) {
	_, err := c.fs().Stat(path)
	if os.IsNotExist(err) {
		return false, nil
	} else if err != nil {
		return false, err
	}
	return true, nil
}

func (c *Config) modTime(path string) (time.Time, error) {
	s, err := c.fs().Stat(path)
	return s.ModTime(), err
}

// Flags adds makex command-line flags to an existing flag.FlagSet (or the
// global FlagSet if fs is nil).
func Flags(fs *flag.FlagSet, conf *Config, prefix string) {
	if fs == nil {
		fs = flag.CommandLine
	}
	fs.BoolVar(&conf.DryRun, prefix+"n", false, "dry run (don't actually run any commands)")
	fs.IntVar(&conf.ParallelJobs, prefix+"j", runtime.GOMAXPROCS(0), "number of jobs to run in parallel")
	fs.BoolVar(&conf.Verbose, prefix+"v", false, "verbose")
}
<commit_msg>nil check<commit_after>package makex

import (
	"flag"
	"os"
	"runtime"
	"time"

	"sourcegraph.com/sourcegraph/rwvfs"
)

type Config struct {
	FS           FileSystem
	ParallelJobs int
	Verbose      bool
	DryRun       bool
}

var Default = Config{
	ParallelJobs: 1,
}

func (c *Config) fs() FileSystem {
	if c.FS != nil {
		return c.FS
	}
	dir, err := os.Getwd()
	if err != nil {
		dir = "."
	}
	return NewFileSystem(rwvfs.OS(dir))
}

func (c *Config) pathExists(path string) (bool, error) {
	_, err := c.fs().Stat(path)
	if os.IsNotExist(err) {
		return false, nil
	} else if err != nil {
		return false, err
	}
	return true, nil
}

func (c *Config) modTime(path string) (time.Time, error) {
	s, err := c.fs().Stat(path)
	if err != nil {
		return time.Time{}, err
	}
	return s.ModTime(), nil
}

// Flags adds makex command-line flags to an existing flag.FlagSet (or the
// global FlagSet if fs is nil).
func Flags(fs *flag.FlagSet, conf *Config, prefix string) {
	if fs == nil {
		fs = flag.CommandLine
	}
	fs.BoolVar(&conf.DryRun, prefix+"n", false, "dry run (don't actually run any commands)")
	fs.IntVar(&conf.ParallelJobs, prefix+"j", runtime.GOMAXPROCS(0), "number of jobs to run in parallel")
	fs.BoolVar(&conf.Verbose, prefix+"v", false, "verbose")
}
<|endoftext|>