<commit_before>// Copyright 2017 The go-darwin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package hdiutil

import (
	"bytes"
	"os/exec"
	"strconv"
)

// AttachFlag implements a hdiutil attach command flag interface.
type AttachFlag interface {
	attachFlag() string
}

type rwType int

const (
	// Readonly force the resulting device to be read-only.
	readonly rwType = 1 << iota
	// Readwrite attempt to override the DiskImages framework's decision to attach a particular image read-only.
	// For example, -readwrite can be used to modify the HFS+ filesystem on a HFS+/ISO hybrid CD image.
	readwrite
)

func (rw rwType) attachFlag() string {
	switch rw {
	case readonly:
		return "-readonly"
	case readwrite:
		return "-readwrite"
	default:
		return ""
	}
}

// kernel attempt to attach this image without a helper process; fail if unsupported.
// Only UDRW, UDRO, UDZO, ULFO, and UDSP images are supported in-kernel.
// Encryption and HTTP are supported by the kernel driver.
// -nokernel attach with a helper process. This is (again) the default as of Mac OS X 10.5.
type kernel bool

func (k kernel) attachFlag() string { return boolNoFlag(bool(k), "kernel") }

// notremovable prevent this image from being detached. Only root can use this option.
// A reboot is necessary to cleanly detach an image attached with -notremovable.
type notRemovable bool

func (n notRemovable) attachFlag() string { return boolFlag(bool(n), "notremovable") }

// mount indicate whether filesystems in the image should be mounted or not.
// The default is required (attach will fail if no filesystems mount).
type mount string

func (m mount) attachFlag() string { return stringFlag(string(m), "mount") }

type noMount bool

func (n noMount) attachFlag() string { return boolFlag(bool(n), "nomount") }

// MountRoot mount volumes on subdirectories of path instead of under /Volumes. path must exist.
// Full mount point paths must be less than MNAMELEN characters (increased from 90 to 1024 in Mac OS X 10.6).
type MountRoot string

func (m MountRoot) attachFlag() string { return stringFlag(string(m), "mountroot") }

// MountRandom like -mountroot, but mount point directory names are randomized with mkdtemp(3).
type MountRandom string

func (m MountRandom) attachFlag() string { return stringFlag(string(m), "mountrandom") }

// MountPoint assuming only one volume, mount it at path instead of in /Volumes.
// See fstab(5) for ways a system administrator can make particular volumes automatically mount in particular filesystem locations by editing the file /etc/fstab.
type MountPoint string

func (m MountPoint) attachFlag() string { return stringFlag(string(m), "mountpoint") }

// noBrowse render any volumes invisible in applications such as the macOS Finder.
type noBrowse bool

func (n noBrowse) attachFlag() string { return boolFlag(bool(n), "nobrowse") }

// owners specify that owners on any filesystems be honored or not.
type owners string

const (
	// OwnersOn is -owners on
	ownersOn owners = "on"
	// OwnersOff is -owners off
	ownersOff owners = "off"
)

func (o owners) attachFlag() string { return stringFlag(string(o), "owners") }

// drivekey specify a key/value pair to be set on the device in the IOKit registry.
type drivekey struct {
	Key   string
	Value string
}

// section attach a subsection of a disk image.
// subspec is any of <offset>, <first-last>, or <start,count> in 0-based sectors.
// Ranges are inclusive.
type section [2]int

func (s section) attachFlag() string {
	var arg string
	for v := range s {
		arg = arg + strconv.Itoa(v)
	}
	return stringFlag(arg, "section")
}

// verify do [not] verify the image.
// By default, hdiutil attach attempts to intelligently verify images that contain checksums before attaching them.
// If hdiutil can write to an image it has verified, attach will store an attribute with
// the image so that it will not be verified again unless its timestamp changes.
// To maintain backwards compatibility, hdid(8) does not attempt to verify images before attaching them.
// Preferences keys: skip-verify, skip-verify-remote, skip-verify-locked, skip-previously-verified
type verify bool

func (v verify) attachFlag() string { return boolNoFlag(bool(v), "verify") }

// ignoreBadChecksums specify whether bad checksums should be ignored.
// The default is to abort when a bad checksum is detected.
// Preferences key: ignore-bad-checksums
type ignoreBadChecksums bool

func (i ignoreBadChecksums) attachFlag() string { return boolNoFlag(bool(i), "ignoreBadChecksums") }

// idme do perform IDME actions on IDME images.
// IDME actions are not performed by default.
// Preferences key: skip-idme
type idme bool

func (i idme) attachFlag() string { return boolNoFlag(bool(i), "idme") }

// idmeReveal do [not] reveal (in the Finder) the results of IDME processing.
// Preferences key: skip-idme-reveal
type idmeReveal bool

func (i idmeReveal) attachFlag() string { return boolNoFlag(bool(i), "idmereveal") }

// idmeTrash do [not] put IDME images in the trash after processing.
// Preferences key: skip-idme-trash
type idmeTrash bool

func (i idmeTrash) attachFlag() string { return boolNoFlag(bool(i), "idmetrash") }

// autoOpen do [not] auto-open volumes (in the Finder) after attaching an image.
// By default, double-clicking a read-only disk image causes the resulting volume to be opened in the Finder.
// hdiutil defaults to -noautoopen.
type autoOpen bool

func (a autoOpen) attachFlag() string { return boolNoFlag(bool(a), "autoopen") }

// autoOpenRO do [not] auto-open read-only volumes.
// Preferences key: auto-open-ro-root
type autoOpenRO bool

func (a autoOpenRO) attachFlag() string { return boolNoFlag(bool(a), "autoopenro") }

// autoOpenRW do [not] auto-open read/write volumes.
// Preferences key: auto-open-rw-root
type autoOpenRW bool

func (a autoOpenRW) attachFlag() string { return boolNoFlag(bool(a), "autoopenrw") }

// autoFsck do [not] force automatic file system checking before mounting a disk image.
// By default, only quarantined images (e.g. downloaded from the Internet) that have not previously passed fsck are checked.
// Preferences key: auto-fsck
type autoFsck bool

func (a autoFsck) attachFlag() string { return boolNoFlag(bool(a), "autofsck") }

const (
	Readonly             rwType             = readonly
	ReadWrite            rwType             = readwrite
	Kernel               kernel             = true
	NoKernel             kernel             = false
	NotRemovable         notRemovable       = true
	MountRequired        mount              = "required"
	MountOptional        mount              = "optional"
	MountSuppressed      mount              = "suppressed"
	NoMount              noMount            = true
	NoBrowse             noBrowse           = true
	OwnersOn             owners             = ownersOn
	OwnersOff            owners             = ownersOff
	Verify               verify             = true
	NoVerify             verify             = false
	IgnoreBadChecksums   ignoreBadChecksums = true
	NoIgnoreBadChecksums ignoreBadChecksums = false
	Idme                 idme               = true
	NoIdme               idme               = false
	IdmeReveal           idmeReveal         = true
	NoIdmeReveal         idmeReveal         = false
	IdmeTrash            idmeTrash          = true
	NoIdmeTrash          idmeTrash          = false
	AutoOpen             autoOpen           = true
	NoAutoOpen           autoOpen           = false
	AutoOpenRO           autoOpenRO         = true
	NoAutoOpenRO         autoOpenRO         = false
	AutoOpenRW           autoOpenRW         = true
	NoAutoOpenRW         autoOpenRW         = false
	AutoFsck             autoFsck           = true
	NoAutoFsck           autoFsck           = false
)

// Attach attach the image file. The returns device node path and error.
func Attach(image string, flags ...AttachFlag) (string, error) {
	cmd := exec.Command(hdiutilPath, "attach", image)

	if len(flags) != 0 {
		args := []string{}
		for _, f := range flags {
			args = append(args, f.attachFlag())
		}
		cmd.Args = append(cmd.Args, args...)
	}

	out, err := cmd.Output()
	if err != nil {
		return "", err
	}

	return string(bytes.TrimSpace(out)), nil
}
<commit_msg>attach: rename attach flags to add 'Attach' prefix<commit_after>// Copyright 2017 The go-darwin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package hdiutil

import (
	"bytes"
	"os/exec"
	"strconv"
)

// AttachFlag implements a hdiutil attach command flag interface.
type AttachFlag interface {
	attachFlag() string
}

type rwType int

const (
	// Readonly force the resulting device to be read-only.
	readonly rwType = 1 << iota
	// Readwrite attempt to override the DiskImages framework's decision to attach a particular image read-only.
	// For example, -readwrite can be used to modify the HFS+ filesystem on a HFS+/ISO hybrid CD image.
	readwrite
)

func (rw rwType) attachFlag() string {
	switch rw {
	case readonly:
		return "-readonly"
	case readwrite:
		return "-readwrite"
	default:
		return ""
	}
}

// kernel attempt to attach this image without a helper process; fail if unsupported.
// Only UDRW, UDRO, UDZO, ULFO, and UDSP images are supported in-kernel.
// Encryption and HTTP are supported by the kernel driver.
// -nokernel attach with a helper process. This is (again) the default as of Mac OS X 10.5.
type kernel bool

func (k kernel) attachFlag() string { return boolNoFlag(bool(k), "kernel") }

// notremovable prevent this image from being detached. Only root can use this option.
// A reboot is necessary to cleanly detach an image attached with -notremovable.
type notRemovable bool

func (n notRemovable) attachFlag() string { return boolFlag(bool(n), "notremovable") }

// mount indicate whether filesystems in the image should be mounted or not.
// The default is required (attach will fail if no filesystems mount).
type mount string

func (m mount) attachFlag() string { return stringFlag(string(m), "mount") }

type noMount bool

func (n noMount) attachFlag() string { return boolFlag(bool(n), "nomount") }

// MountRoot mount volumes on subdirectories of path instead of under /Volumes. path must exist.
// Full mount point paths must be less than MNAMELEN characters (increased from 90 to 1024 in Mac OS X 10.6).
type MountRoot string

func (m MountRoot) attachFlag() string { return stringFlag(string(m), "mountroot") }

// MountRandom like -mountroot, but mount point directory names are randomized with mkdtemp(3).
type MountRandom string

func (m MountRandom) attachFlag() string { return stringFlag(string(m), "mountrandom") }

// MountPoint assuming only one volume, mount it at path instead of in /Volumes.
// See fstab(5) for ways a system administrator can make particular volumes automatically mount in particular filesystem locations by editing the file /etc/fstab.
type MountPoint string

func (m MountPoint) attachFlag() string { return stringFlag(string(m), "mountpoint") }

// noBrowse render any volumes invisible in applications such as the macOS Finder.
type noBrowse bool

func (n noBrowse) attachFlag() string { return boolFlag(bool(n), "nobrowse") }

// owners specify that owners on any filesystems be honored or not.
type owners string

const (
	// OwnersOn is -owners on
	ownersOn owners = "on"
	// OwnersOff is -owners off
	ownersOff owners = "off"
)

func (o owners) attachFlag() string { return stringFlag(string(o), "owners") }

// drivekey specify a key/value pair to be set on the device in the IOKit registry.
type drivekey struct {
	Key   string
	Value string
}

// section attach a subsection of a disk image.
// subspec is any of <offset>, <first-last>, or <start,count> in 0-based sectors.
// Ranges are inclusive.
type section [2]int

func (s section) attachFlag() string {
	var arg string
	for v := range s {
		arg = arg + strconv.Itoa(v)
	}
	return stringFlag(arg, "section")
}

// verify do [not] verify the image.
// By default, hdiutil attach attempts to intelligently verify images that contain checksums before attaching them.
// If hdiutil can write to an image it has verified, attach will store an attribute with
// the image so that it will not be verified again unless its timestamp changes.
// To maintain backwards compatibility, hdid(8) does not attempt to verify images before attaching them.
// Preferences keys: skip-verify, skip-verify-remote, skip-verify-locked, skip-previously-verified
type verify bool

func (v verify) attachFlag() string { return boolNoFlag(bool(v), "verify") }

// ignoreBadChecksums specify whether bad checksums should be ignored.
// The default is to abort when a bad checksum is detected.
// Preferences key: ignore-bad-checksums
type ignoreBadChecksums bool

func (i ignoreBadChecksums) attachFlag() string { return boolNoFlag(bool(i), "ignoreBadChecksums") }

// idme do perform IDME actions on IDME images.
// IDME actions are not performed by default.
// Preferences key: skip-idme
type idme bool

func (i idme) attachFlag() string { return boolNoFlag(bool(i), "idme") }

// idmeReveal do [not] reveal (in the Finder) the results of IDME processing.
// Preferences key: skip-idme-reveal
type idmeReveal bool

func (i idmeReveal) attachFlag() string { return boolNoFlag(bool(i), "idmereveal") }

// idmeTrash do [not] put IDME images in the trash after processing.
// Preferences key: skip-idme-trash
type idmeTrash bool

func (i idmeTrash) attachFlag() string { return boolNoFlag(bool(i), "idmetrash") }

// autoOpen do [not] auto-open volumes (in the Finder) after attaching an image.
// By default, double-clicking a read-only disk image causes the resulting volume to be opened in the Finder.
// hdiutil defaults to -noautoopen.
type autoOpen bool

func (a autoOpen) attachFlag() string { return boolNoFlag(bool(a), "autoopen") }

// autoOpenRO do [not] auto-open read-only volumes.
// Preferences key: auto-open-ro-root
type autoOpenRO bool

func (a autoOpenRO) attachFlag() string { return boolNoFlag(bool(a), "autoopenro") }

// autoOpenRW do [not] auto-open read/write volumes.
// Preferences key: auto-open-rw-root
type autoOpenRW bool

func (a autoOpenRW) attachFlag() string { return boolNoFlag(bool(a), "autoopenrw") }

// autoFsck do [not] force automatic file system checking before mounting a disk image.
// By default, only quarantined images (e.g. downloaded from the Internet) that have not previously passed fsck are checked.
// Preferences key: auto-fsck
type autoFsck bool

func (a autoFsck) attachFlag() string { return boolNoFlag(bool(a), "autofsck") }

const (
	AttachReadonly             rwType             = readonly
	AttachReadWrite            rwType             = readwrite
	AttachKernel               kernel             = true
	AttachNoKernel             kernel             = false
	AttachNotRemovable         notRemovable       = true
	AttachMountRequired        mount              = "required"
	AttachMountOptional        mount              = "optional"
	AttachMountSuppressed      mount              = "suppressed"
	AttachNoMount              noMount            = true
	AttachNoBrowse             noBrowse           = true
	AttachOwnersOn             owners             = ownersOn
	AttachOwnersOff            owners             = ownersOff
	AttachVerify               verify             = true
	AttachNoVerify             verify             = false
	AttachIgnoreBadChecksums   ignoreBadChecksums = true
	AttachNoIgnoreBadChecksums ignoreBadChecksums = false
	AttachIdme                 idme               = true
	AttachNoIdme               idme               = false
	AttachIdmeReveal           idmeReveal         = true
	AttachNoIdmeReveal         idmeReveal         = false
	AttachIdmeTrash            idmeTrash          = true
	AttachNoIdmeTrash          idmeTrash          = false
	AttachAutoOpen             autoOpen           = true
	AttachNoAutoOpen           autoOpen           = false
	AttachAutoOpenRO           autoOpenRO         = true
	AttachNoAutoOpenRO         autoOpenRO         = false
	AttachAutoOpenRW           autoOpenRW         = true
	AttachNoAutoOpenRW         autoOpenRW         = false
	AttachAutoFsck             autoFsck           = true
	AttachNoAutoFsck           autoFsck           = false
)

// Attach attach the image file. The returns device node path and error.
func Attach(image string, flags ...AttachFlag) (string, error) {
	cmd := exec.Command(hdiutilPath, "attach", image)

	if len(flags) != 0 {
		args := []string{}
		for _, f := range flags {
			args = append(args, f.attachFlag())
		}
		cmd.Args = append(cmd.Args, args...)
	}

	out, err := cmd.Output()
	if err != nil {
		return "", err
	}

	return string(bytes.TrimSpace(out)), nil
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/spf13/cobra"

	"github.com/ovh/cds/cli"
	"github.com/ovh/cds/sdk"
)

var applicationVariableCmd = cli.Command{
	Name:  "variable",
	Short: "Manage CDS application variables",
}

func applicationVariable() *cobra.Command {
	return cli.NewCommand(applicationVariableCmd, nil, []*cobra.Command{
		cli.NewCommand(applicationVariableCreateCmd, applicationCreateVariableRun, nil, withAllCommandModifiers()...),
		cli.NewGetCommand(applicationVariableShowCmd, applicationVariableShowRun, nil, withAllCommandModifiers()...),
		cli.NewListCommand(applicationVariableListCmd, applicationListVariableRun, nil, withAllCommandModifiers()...),
		cli.NewCommand(applicationVariableDeleteCmd, applicationDeleteVariableRun, nil, withAllCommandModifiers()...),
		cli.NewCommand(applicationVariableUpdateCmd, applicationUpdateVariableRun, nil, withAllCommandModifiers()...),
	})
}

var applicationVariableCreateCmd = cli.Command{
	Name:  "add",
	Short: "Add a new variable on application. variable type can be one of password, text, string, key, boolean, number, repository",
	Ctx: []cli.Arg{
		{Name: _ProjectKey},
	},
	Args: []cli.Arg{
		{Name: _ApplicationName},
		{Name: "variable-name"},
		{Name: "variable-type"},
		{Name: "variable-value"},
	},
}

func applicationCreateVariableRun(v cli.Values) error {
	variable := &sdk.Variable{
		Name:  v.GetString("variable-name"),
		Type:  v.GetString("variable-type"),
		Value: v.GetString("variable-value"),
	}
	return client.ApplicationVariableCreate(v.GetString(_ProjectKey), v.GetString(_ApplicationName), variable)
}

var applicationVariableListCmd = cli.Command{
	Name:  "list",
	Short: "List CDS application variables",
	Ctx: []cli.Arg{
		{Name: _ProjectKey},
		{Name: _ApplicationName},
	},
}

func applicationListVariableRun(v cli.Values) (cli.ListResult, error) {
	variables, err := client.ApplicationVariablesList(v.GetString(_ProjectKey), v.GetString(_ApplicationName))
	if err != nil {
		return nil, err
	}
	return cli.AsListResult(variables), nil
}

var applicationVariableDeleteCmd = cli.Command{
	Name:  "delete",
	Short: "Delete CDS application variable",
	Ctx: []cli.Arg{
		{Name: _ProjectKey},
		{Name: _ApplicationName},
	},
	Args: []cli.Arg{
		{Name: "variable-name"},
	},
}

func applicationDeleteVariableRun(v cli.Values) error {
	return client.ApplicationVariableDelete(v.GetString(_ProjectKey), v.GetString(_ApplicationName), v.GetString("variable-name"))
}

var applicationVariableShowCmd = cli.Command{
	Name:  "show",
	Short: "Show a CDS application variable",
	Ctx: []cli.Arg{
		{Name: _ProjectKey},
		{Name: _ApplicationName},
	},
	Args: []cli.Arg{
		{Name: "variable-name"},
	},
}

func applicationVariableShowRun(v cli.Values) (interface{}, error) {
	return client.ApplicationVariableGet(v.GetString(_ProjectKey), v.GetString(_ApplicationName), v.GetString("variable-name"))
}

var applicationVariableUpdateCmd = cli.Command{
	Name:  "update",
	Short: "Update CDS application variable value",
	Ctx: []cli.Arg{
		{Name: _ProjectKey},
		{Name: _ApplicationName},
	},
	Args: []cli.Arg{
		{Name: "variable-oldname"},
		{Name: "variable-name"},
		{Name: "variable-type"},
		{Name: "variable-value"},
	},
}

func applicationUpdateVariableRun(v cli.Values) error {
	variable, err := client.ApplicationVariableGet(v.GetString(_ProjectKey), v.GetString(_ApplicationName), v.GetString("variable-oldname"))
	if err != nil {
		return err
	}
	variable.Name = v.GetString("variable-name")
	variable.Value = v.GetString("variable-value")
	variable.Type = v.GetString("variable-type")
	return client.ApplicationVariableUpdate(v.GetString(_ProjectKey), v.GetString(_ApplicationName), variable)
}
<commit_msg>fix(cli): application variable argument order (#4070)<commit_after>package main

import (
	"github.com/spf13/cobra"

	"github.com/ovh/cds/cli"
	"github.com/ovh/cds/sdk"
)

var applicationVariableCmd = cli.Command{
	Name:  "variable",
	Short: "Manage CDS application variables",
}

func applicationVariable() *cobra.Command {
	return cli.NewCommand(applicationVariableCmd, nil, []*cobra.Command{
		cli.NewCommand(applicationVariableCreateCmd, applicationCreateVariableRun, nil, withAllCommandModifiers()...),
		cli.NewGetCommand(applicationVariableShowCmd, applicationVariableShowRun, nil, withAllCommandModifiers()...),
		cli.NewListCommand(applicationVariableListCmd, applicationListVariableRun, nil, withAllCommandModifiers()...),
		cli.NewCommand(applicationVariableDeleteCmd, applicationDeleteVariableRun, nil, withAllCommandModifiers()...),
		cli.NewCommand(applicationVariableUpdateCmd, applicationUpdateVariableRun, nil, withAllCommandModifiers()...),
	})
}

var applicationVariableCreateCmd = cli.Command{
	Name:  "add",
	Short: "Add a new variable on application. variable type can be one of password, text, string, key, boolean, number, repository",
	Ctx: []cli.Arg{
		{Name: _ProjectKey},
		{Name: _ApplicationName},
	},
	Args: []cli.Arg{
		{Name: "variable-name", Weight: 1},
		{Name: "variable-type", Weight: 2},
		{Name: "variable-value", Weight: 3},
	},
}

func applicationCreateVariableRun(v cli.Values) error {
	variable := &sdk.Variable{
		Name:  v.GetString("variable-name"),
		Type:  v.GetString("variable-type"),
		Value: v.GetString("variable-value"),
	}
	return client.ApplicationVariableCreate(v.GetString(_ProjectKey), v.GetString(_ApplicationName), variable)
}

var applicationVariableListCmd = cli.Command{
	Name:  "list",
	Short: "List CDS application variables",
	Ctx: []cli.Arg{
		{Name: _ProjectKey},
		{Name: _ApplicationName},
	},
}

func applicationListVariableRun(v cli.Values) (cli.ListResult, error) {
	variables, err := client.ApplicationVariablesList(v.GetString(_ProjectKey), v.GetString(_ApplicationName))
	if err != nil {
		return nil, err
	}
	return cli.AsListResult(variables), nil
}

var applicationVariableDeleteCmd = cli.Command{
	Name:  "delete",
	Short: "Delete CDS application variable",
	Ctx: []cli.Arg{
		{Name: _ProjectKey},
		{Name: _ApplicationName},
	},
	Args: []cli.Arg{
		{Name: "variable-name"},
	},
}

func applicationDeleteVariableRun(v cli.Values) error {
	return client.ApplicationVariableDelete(v.GetString(_ProjectKey), v.GetString(_ApplicationName), v.GetString("variable-name"))
}

var applicationVariableShowCmd = cli.Command{
	Name:  "show",
	Short: "Show a CDS application variable",
	Ctx: []cli.Arg{
		{Name: _ProjectKey},
		{Name: _ApplicationName},
	},
	Args: []cli.Arg{
		{Name: "variable-name"},
	},
}

func applicationVariableShowRun(v cli.Values) (interface{}, error) {
	return client.ApplicationVariableGet(v.GetString(_ProjectKey), v.GetString(_ApplicationName), v.GetString("variable-name"))
}

var applicationVariableUpdateCmd = cli.Command{
	Name:  "update",
	Short: "Update CDS application variable value",
	Ctx: []cli.Arg{
		{Name: _ProjectKey},
		{Name: _ApplicationName},
	},
	Args: []cli.Arg{
		{Name: "variable-oldname", Weight: 1},
		{Name: "variable-name", Weight: 2},
		{Name: "variable-type", Weight: 3},
		{Name: "variable-value", Weight: 4},
	},
}

func applicationUpdateVariableRun(v cli.Values) error {
	variable, err := client.ApplicationVariableGet(v.GetString(_ProjectKey), v.GetString(_ApplicationName), v.GetString("variable-oldname"))
	if err != nil {
		return err
	}
	variable.Name = v.GetString("variable-name")
	variable.Value = v.GetString("variable-value")
	variable.Type = v.GetString("variable-type")
	return client.ApplicationVariableUpdate(v.GetString(_ProjectKey), v.GetString(_ApplicationName), variable)
}
<|endoftext|>


<commit_before>package logstash

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net"
	"regexp"
	"strings"
	"time"

	"github.com/gliderlabs/logspout/router"
)

func init() {
	router.AdapterFactories.Register(NewLogstashAdapter, "logstash")
}

// LogstashAdapter is an adapter that streams UDP JSON to Logstash.
type LogstashAdapter struct {
	conn  net.Conn
	route *router.Route
}

var javaExceptionPattern = regexp.MustCompile("(^.+Exception: .+)|(^\\s+at .+)|(^\\s+... \\d+ more)|(^\\s*Caused by:.+)")

var lastMessage = make(map[string]LogstashMessage)
var collapseMessage = make(map[string]chan LogstashMessage)

// NewLogstashAdapter creates a LogstashAdapter with UDP as the default transport.
func NewLogstashAdapter(route *router.Route) (router.LogAdapter, error) {
	transport, found := router.AdapterTransports.Lookup(route.AdapterTransport("udp"))
	if !found {
		return nil, errors.New("unable to find adapter: " + route.Adapter)
	}

	conn, err := transport.Dial(route.Address, route.Options)
	if err != nil {
		return nil, err
	}

	return &LogstashAdapter{
		route: route,
		conn:  conn,
	}, nil
}

func (a *LogstashAdapter) collapseIfNeeded(message LogstashMessage, sendMessage func(message LogstashMessage)) {
	if javaExceptionPattern.MatchString(message.Message) {
		if _, present := collapseMessage[message.Docker.ID]; !present {
			channel := make(chan LogstashMessage)
			collapseMessage[message.Docker.ID] = channel
			go func(channel chan LogstashMessage, message LogstashMessage) {
				receive := true
				data := message
				for receive == true {
					select {
					case followingMessage := <-channel:
						data.Message = strings.Join([]string{data.Message, followingMessage.Message}, "\n")
					case <-time.After(time.Millisecond * 200):
						sendMessage(data)
						delete(collapseMessage, message.Docker.ID)
						receive = false
					}
				}
			}(channel, message)
		} else {
			collapseMessage[message.Docker.ID] <- message
		}
	} else {
		sendMessage(message)
	}
}

// Stream implements the router.LogAdapter interface.
func (a *LogstashAdapter) Stream(logstream chan *router.Message) {
	for m := range logstream {
		dockerInfo := DockerInfo{
			Name:     m.Container.Name,
			ID:       m.Container.ID,
			Image:    m.Container.Config.Image,
			Hostname: m.Container.Config.Hostname,
		}
		msg := LogstashMessage{
			Message: m.Data,
			Docker:  dockerInfo,
		}

		var jsonMsg map[string]interface{}
		err := json.Unmarshal([]byte(m.Data), &jsonMsg)
		if err == nil {
			msg.MessageInfo = jsonMsg
		}

		fmt.Println("Send ", msg.Message)

		a.collapseIfNeeded(msg, func(message LogstashMessage) {
			js, err := json.Marshal(message)
			if err != nil {
				log.Println("logstash:", err)
			}
			_, err = a.conn.Write(js)
			if err != nil {
				log.Println("logstash:", err)
			}
		})
	}
}

// DockerInfo : informations embbeded in message to logstash
type DockerInfo struct {
	Name     string `json:"name"`
	ID       string `json:"id"`
	Image    string `json:"image"`
	Hostname string `json:"hostname"`
}

// LogstashMessage is a simple JSON input to Logstash.
type LogstashMessage struct {
	Message     string                 `json:"message"`
	MessageInfo map[string]interface{} `json:"messageInfo"`
	Docker      DockerInfo             `json:"docker"`
}
<commit_msg>Add time to Logstahs message.<commit_after>package logstash

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net"
	"regexp"
	"strings"
	"time"

	"github.com/gliderlabs/logspout/router"
)

func init() {
	router.AdapterFactories.Register(NewLogstashAdapter, "logstash")
}

// LogstashAdapter is an adapter that streams UDP JSON to Logstash.
type LogstashAdapter struct {
	conn  net.Conn
	route *router.Route
}

var javaExceptionPattern = regexp.MustCompile("(^.+Exception: .+)|(^\\s+at .+)|(^\\s+... \\d+ more)|(^\\s*Caused by:.+)")

var lastMessage = make(map[string]LogstashMessage)
var collapseMessage = make(map[string]chan LogstashMessage)

// NewLogstashAdapter creates a LogstashAdapter with UDP as the default transport.
func NewLogstashAdapter(route *router.Route) (router.LogAdapter, error) {
	transport, found := router.AdapterTransports.Lookup(route.AdapterTransport("udp"))
	if !found {
		return nil, errors.New("unable to find adapter: " + route.Adapter)
	}

	conn, err := transport.Dial(route.Address, route.Options)
	if err != nil {
		return nil, err
	}

	return &LogstashAdapter{
		route: route,
		conn:  conn,
	}, nil
}

func (a *LogstashAdapter) collapseIfNeeded(message LogstashMessage, sendMessage func(message LogstashMessage)) {
	if javaExceptionPattern.MatchString(message.Message) {
		if _, present := collapseMessage[message.Docker.ID]; !present {
			channel := make(chan LogstashMessage)
			collapseMessage[message.Docker.ID] = channel
			go func(channel chan LogstashMessage, message LogstashMessage) {
				receive := true
				data := message
				for receive == true {
					select {
					case followingMessage := <-channel:
						data.Message = strings.Join([]string{data.Message, followingMessage.Message}, "\n")
					case <-time.After(time.Millisecond * 200):
						sendMessage(data)
						delete(collapseMessage, message.Docker.ID)
						receive = false
					}
				}
			}(channel, message)
		} else {
			collapseMessage[message.Docker.ID] <- message
		}
	} else {
		sendMessage(message)
	}
}

// Stream implements the router.LogAdapter interface.
func (a *LogstashAdapter) Stream(logstream chan *router.Message) {
	for m := range logstream {
		dockerInfo := DockerInfo{
			Name:     m.Container.Name,
			ID:       m.Container.ID,
			Image:    m.Container.Config.Image,
			Hostname: m.Container.Config.Hostname,
		}
		msg := LogstashMessage{
			Message: m.Data,
			Time:    m.Time,
			Docker:  dockerInfo,
		}

		var jsonMsg map[string]interface{}
		err := json.Unmarshal([]byte(m.Data), &jsonMsg)
		if err == nil {
			msg.MessageInfo = jsonMsg
		}

		fmt.Println("Send ", msg.Message)

		a.collapseIfNeeded(msg, func(message LogstashMessage) {
			js, err := json.Marshal(message)
			if err != nil {
				log.Println("logstash:", err)
			}
			_, err = a.conn.Write(js)
			if err != nil {
				log.Println("logstash:", err)
			}
		})
	}
}

// DockerInfo : informations embbeded in message to logstash
type DockerInfo struct {
	Name     string `json:"name"`
	ID       string `json:"id"`
	Image    string `json:"image"`
	Hostname string `json:"hostname"`
}

// LogstashMessage is a simple JSON input to Logstash
type LogstashMessage struct {
	Message     string                 `json:"message"`
	Time        time.Time              `json:"time"`
	MessageInfo map[string]interface{} `json:"messageInfo"`
	Docker      DockerInfo             `json:"docker"`
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"github.com/spf13/cobra"
	"k8s.io/kops/channels/pkg/channels"
	"k8s.io/kops/util/pkg/tables"
	"net/url"
	"os"
	"strings"
)

type ApplyChannelCmd struct {
	Yes   bool
	Files []string
}

var applyChannel ApplyChannelCmd

func init() {
	cmd := &cobra.Command{
		Use:   "channel",
		Short: "Apply channel",
		Run: func(cmd *cobra.Command, args []string) {
			err := applyChannel.Run(args)
			if err != nil {
				exitWithError(err)
			}
		},
	}

	cmd.Flags().BoolVar(&applyChannel.Yes, "yes", false, "Apply update")
	cmd.Flags().StringSliceVar(&applyChannel.Files, "f", []string{}, "Apply from a local file")

	applyCmd.AddCommand(cmd)
}

func (c *ApplyChannelCmd) Run(args []string) error {
	k8sClient, err := rootCommand.KubernetesClient()
	if err != nil {
		return err
	}

	var addons []*channels.Addon
	for _, name := range args {
		location, err := url.Parse(name)
		if err != nil {
			return fmt.Errorf("unable to parse argument %q as url", name)
		}
		if !location.IsAbs() {
			// We recognize the following "well-known" format:
			// <name> with no slashes ->
			if strings.Contains(name, "/") {
				return fmt.Errorf("Channel format not recognized (did you mean to use `-f` to specify a local file?): %q", name)
			}
			expanded := "https://raw.githubusercontent.com/kubernetes/kops/master/addons/" + name + "/addon.yaml"
			location, err = url.Parse(expanded)
			if err != nil {
				return fmt.Errorf("unable to parse expanded argument %q as url", expanded)
			}
		}
		o, err := channels.LoadAddons(name, location)
		if err != nil {
			return fmt.Errorf("error loading channel %q: %v", location, err)
		}

		current, err := o.GetCurrent()
		if err != nil {
			return fmt.Errorf("error processing latest versions in %q: %v", location, err)
		}
		addons = append(addons, current...)
	}

	for _, f := range c.Files {
		location, err := url.Parse(f)
		if err != nil {
			return fmt.Errorf("unable to parse argument %q as url", f)
		}
		if !location.IsAbs() {
			cwd, err := os.Getwd()
			if err != nil {
				return fmt.Errorf("error getting current directory: %v", err)
			}
			baseURL, err := url.Parse(cwd + string(os.PathSeparator))
			if err != nil {
				return fmt.Errorf("error building url for current directory %q: %v", cwd, err)
			}
			location = baseURL.ResolveReference(location)
		}
		o, err := channels.LoadAddons(f, location)
		if err != nil {
			return fmt.Errorf("error loading file %q: %v", f, err)
		}

		current, err := o.GetCurrent()
		if err != nil {
			return fmt.Errorf("error processing latest versions in %q: %v", f, err)
		}
		addons = append(addons, current...)
	}

	var updates []*channels.AddonUpdate
	var needUpdates []*channels.Addon
	for _, addon := range addons {
		// TODO: Cache lookups to prevent repeated lookups?
		update, err := addon.GetRequiredUpdates(k8sClient)
		if err != nil {
			return fmt.Errorf("error checking for required update: %v", err)
		}
		if update != nil {
			updates = append(updates, update)
			needUpdates = append(needUpdates, addon)
		}
	}

	if len(updates) == 0 {
		fmt.Printf("No update required\n")
		return nil
	}

	{
		t := &tables.Table{}
		t.AddColumn("NAME", func(r *channels.AddonUpdate) string {
			return r.Name
		})
		t.AddColumn("CURRENT", func(r *channels.AddonUpdate) string {
			if r.ExistingVersion == nil {
				return "-"
			}
			if r.ExistingVersion.Version != nil {
				return *r.ExistingVersion.Version
			}
			return "?"
		})
		t.AddColumn("UPDATE", func(r *channels.AddonUpdate) string {
			if r.NewVersion == nil {
				return "-"
			}
			if r.NewVersion.Version != nil {
				return *r.NewVersion.Version
			}
			return "?"
		})

		columns := []string{"NAME", "CURRENT", "UPDATE"}
		err := t.Render(updates, os.Stdout, columns...)
		if err != nil {
			return err
		}
	}

	if !c.Yes {
		fmt.Printf("\nMust specify --yes to update\n")
		return nil
	}

	for _, needUpdate := range needUpdates {
		update, err := needUpdate.EnsureUpdated(k8sClient)
		if err != nil {
			return fmt.Errorf("error updating %q: %v", needUpdate.Name, err)
		}
		if update.NewVersion.Version != nil {
			fmt.Printf("Updated %q to %d\n", update.Name, *update.NewVersion)
		} else {
			fmt.Printf("Updated %q\n", update.Name)
		}
	}

	fmt.Printf("\n")

	return nil
}
<commit_msg>Fix segfault in apply_channel<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"github.com/spf13/cobra"
	"k8s.io/kops/channels/pkg/channels"
	"k8s.io/kops/util/pkg/tables"
	"net/url"
	"os"
	"strings"
)

type ApplyChannelCmd struct {
	Yes   bool
	Files []string
}

var applyChannel ApplyChannelCmd

func init() {
	cmd := &cobra.Command{
		Use:   "channel",
		Short: "Apply channel",
		Run: func(cmd *cobra.Command, args []string) {
			err := applyChannel.Run(args)
			if err != nil {
				exitWithError(err)
			}
		},
	}

	cmd.Flags().BoolVar(&applyChannel.Yes, "yes", false, "Apply update")
	cmd.Flags().StringSliceVar(&applyChannel.Files, "f", []string{}, "Apply from a local file")

	applyCmd.AddCommand(cmd)
}

func (c *ApplyChannelCmd) Run(args []string) error {
	k8sClient, err := rootCommand.KubernetesClient()
	if err != nil {
		return err
	}

	var addons []*channels.Addon
	for _, name := range args {
		location, err := url.Parse(name)
		if err != nil {
			return fmt.Errorf("unable to parse argument %q as url", name)
		}
		if !location.IsAbs() {
			// We recognize the following "well-known" format:
			// <name> with no slashes ->
			if strings.Contains(name, "/") {
				return fmt.Errorf("Channel format not recognized (did you mean to use `-f` to specify a local file?): %q", name)
			}
			expanded := "https://raw.githubusercontent.com/kubernetes/kops/master/addons/" + name + "/addon.yaml"
			location, err = url.Parse(expanded)
			if err != nil {
				return fmt.Errorf("unable to parse expanded argument %q as url", expanded)
			}
		}
		o, err := channels.LoadAddons(name, location)
		if err != nil {
			return fmt.Errorf("error loading channel %q: %v", location, err)
		}

		current, err := o.GetCurrent()
		if err != nil {
			return fmt.Errorf("error processing latest versions in %q: %v", location, err)
		}
		addons = append(addons, current...)
	}

	for _, f := range c.Files {
		location, err := url.Parse(f)
		if err != nil {
			return fmt.Errorf("unable to parse argument %q as url", f)
		}
		if !location.IsAbs() {
			cwd, err := os.Getwd()
			if err != nil {
				return fmt.Errorf("error getting current directory: %v", err)
			}
			baseURL, err := url.Parse(cwd + string(os.PathSeparator))
			if err != nil {
				return fmt.Errorf("error building url for current directory %q: %v", cwd, err)
			}
			location = baseURL.ResolveReference(location)
		}
		o, err := channels.LoadAddons(f, location)
		if err != nil {
			return fmt.Errorf("error loading file %q: %v", f, err)
		}

		current, err := o.GetCurrent()
		if err != nil {
			return fmt.Errorf("error processing latest versions in %q: %v", f, err)
		}
		addons = append(addons, current...)
	}

	var updates []*channels.AddonUpdate
	var needUpdates []*channels.Addon
	for _, addon := range addons {
		// TODO: Cache lookups to prevent repeated lookups?
		update, err := addon.GetRequiredUpdates(k8sClient)
		if err != nil {
			return fmt.Errorf("error checking for required update: %v", err)
		}
		if update != nil {
			updates = append(updates, update)
			needUpdates = append(needUpdates, addon)
		}
	}

	if len(updates) == 0 {
		fmt.Printf("No update required\n")
		return nil
	}

	{
		t := &tables.Table{}
		t.AddColumn("NAME", func(r *channels.AddonUpdate) string {
			return r.Name
		})
		t.AddColumn("CURRENT", func(r *channels.AddonUpdate) string {
			if r.ExistingVersion == nil {
				return "-"
			}
			if r.ExistingVersion.Version != nil {
				return *r.ExistingVersion.Version
			}
			return "?"
		})
		t.AddColumn("UPDATE", func(r *channels.AddonUpdate) string {
			if r.NewVersion == nil {
				return "-"
			}
			if r.NewVersion.Version != nil {
				return *r.NewVersion.Version
			}
			return "?"
		})

		columns := []string{"NAME", "CURRENT", "UPDATE"}
		err := t.Render(updates, os.Stdout, columns...)
		if err != nil {
			return err
		}
	}

	if !c.Yes {
		fmt.Printf("\nMust specify --yes to update\n")
		return nil
	}

	for _, needUpdate := range needUpdates {
		update, err := needUpdate.EnsureUpdated(k8sClient)
		if err != nil {
			return fmt.Errorf("error updating %q: %v", needUpdate.Name, err)
		}
		// Could have been a concurrent request
		if update != nil {
			if update.NewVersion.Version != nil {
				fmt.Printf("Updated %q to %d\n", update.Name, *update.NewVersion)
			} else {
				fmt.Printf("Updated %q\n", update.Name)
			}
		}
	}

	fmt.Printf("\n")

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Davis Webb
// Copyright 2015 Luke Shumaker

package store

import (
	"database/sql"
	he "httpentity"
)

var _ he.Entity = &Group{}
var _ he.NetEntity = &Group{}
var dirGroups he.Entity = newDirGroups()

// Model /////////////////////////////////////////////////////////////

type Group struct {
	Name string
}

func GetGroupByName(con DB, name string) *Group {
	var group Group
	err := con.QueryRow("select * from groups where name=?", name).Scan(&group)
	switch {
	case err == sql.ErrNoRows:
		// group does not exist
		return nil
	case err != nil:
		panic(err)
	default:
		return &group
	}
}

func NewGroup(con DB, name string) *Group {
	g := &Group{Name: name}
	_, err := con.Exec("INSERT INTO groups VALUES (?)", g.Name)
	if err != nil {
		panic(err)
	}

	return g
}

func (o *Group) Subentity(name string, req he.Request) he.Entity {
	panic("TODO: API: (*Group).Subentity()")
}

func (o *Group) Methods() map[string]he.Handler {
	return map[string]he.Handler{
		"GET": func(req he.Request) he.Response {
			panic("TODO: API: (*Group).Methods()[\"GET\"]")
		},
		"PUT": func(req he.Request) he.Response {
			panic("TODO: API: (*Group).Methods()[\"PUT\"]")
		},
		"PATCH": func(req he.Request) he.Response {
			panic("TODO: API: (*Group).Methods()[\"PATCH\"]")
		},
		"DELETE": func(req he.Request) he.Response {
			panic("TODO: API: (*Group).Methods()[\"DELETE\"]")
		},
	}
}

// View //////////////////////////////////////////////////////////////

func (o *Group) Encoders() map[string]he.Encoder {
	return defaultEncoders(o)
}

// Directory ("Controller") //////////////////////////////////////////

type t_dirGroups struct {
	methods map[string]he.Handler
}

func newDirGroups() t_dirGroups {
	r := t_dirGroups{}
	r.methods = map[string]he.Handler{
		"POST": func(req he.Request) he.Response {
			con := getConnection()
			defer con.Close()
			return req.StatusCreated(r, NewGroup(con /*TODO NAME*/, "").Name)
		},
	}
	return r
}

func (d t_dirGroups) Methods() map[string]he.Handler {
	return d.methods
}

func (d t_dirGroups) Subentity(name string, request he.Request) he.Entity {
	con := getConnection()
	defer con.Close()
	return GetGroupByName(con, name)
}
<commit_msg>implement group POST<commit_after>// Copyright 2015 Davis Webb
// Copyright 2015 Luke Shumaker

package store

import (
	"database/sql"
	he "httpentity"
	"strings"
)

var _ he.Entity = &Group{}
var _ he.NetEntity = &Group{}
var dirGroups he.Entity = newDirGroups()

// Model /////////////////////////////////////////////////////////////

type Group struct {
	Name string
}

func GetGroupByName(con DB, name string) *Group {
	var group Group
	err := con.QueryRow("select * from groups where name=?", name).Scan(&group)
	switch {
	case err == sql.ErrNoRows:
		// group does not exist
		return nil
	case err != nil:
		panic(err)
	default:
		return &group
	}
}

func NewGroup(con DB, name string) *Group {
	g := &Group{Name: name}
	_, err := con.Exec("INSERT INTO groups VALUES (?)", g.Name)
	if err != nil {
		panic(err)
	}

	return g
}

func (o *Group) Subentity(name string, req he.Request) he.Entity {
	panic("TODO: API: (*Group).Subentity()")
}

func (o *Group) Methods() map[string]he.Handler {
	return map[string]he.Handler{
		"GET": func(req he.Request) he.Response {
			panic("TODO: API: (*Group).Methods()[\"GET\"]")
		},
		"PUT": func(req he.Request) he.Response {
			panic("TODO: API: (*Group).Methods()[\"PUT\"]")
		},
		"PATCH": func(req he.Request) he.Response {
			panic("TODO: API: (*Group).Methods()[\"PATCH\"]")
		},
		"DELETE": func(req he.Request) he.Response {
			panic("TODO: API: (*Group).Methods()[\"DELETE\"]")
		},
	}
}

// View //////////////////////////////////////////////////////////////

func (o *Group) Encoders() map[string]he.Encoder {
	return defaultEncoders(o)
}

// Directory ("Controller") //////////////////////////////////////////

type t_dirGroups struct {
	methods map[string]he.Handler
}

func newDirGroups() t_dirGroups {
	r := t_dirGroups{}
	r.methods = map[string]he.Handler{
		"POST": func(req he.Request) he.Response {
			db := req.Things["db"].(DB)
			badbody := req.StatusBadRequest("submitted body not what expected")
			hash, ok := req.Entity.(map[string]interface{}); if !ok { return badbody }
			groupname, ok := hash["groupname"].(string)    ; if !ok { return badbody }

			groupname = strings.ToLower(groupname)

			group := NewGroup(db, groupname)
			if group == nil {
				return req.StatusConflict(he.NetString("a group with that name already exists"))
			} else {
				return req.StatusCreated(r, groupname)
			}
		},
	}
	return r
}

func (d t_dirGroups) Methods() map[string]he.Handler {
	return d.methods
}

func (d t_dirGroups) Subentity(name string, request he.Request) he.Entity {
	con := getConnection()
	defer con.Close()
	return GetGroupByName(con, name)
}
<|endoftext|>


<commit_before>package server

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
	"testing"

	"google.golang.org/grpc"

	"github.com/pachyderm/pachyderm/src/common"
	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pfs/discovery"
	"github.com/pachyderm/pachyderm/src/pfs/drive"
	"github.com/pachyderm/pachyderm/src/pfs/fuse"
	"github.com/pachyderm/pachyderm/src/pfs/pfsutil"
	"github.com/pachyderm/pachyderm/src/pfs/route"
	"github.com/pachyderm/pachyderm/src/pkg/btrfs"
	"github.com/pachyderm/pachyderm/src/pkg/executil"
	"github.com/pachyderm/pachyderm/src/pkg/grpctest"
	"github.com/stretchr/testify/require"
)

const (
	// TODO(pedge): large numbers of shards takes forever because
	// we are doing tons of btrfs operations on init, is there anything
	// we can do about that?
	testShardsPerServer = 8
	testNumServers      = 8
	testSize            = 1000
)

var (
	counter int32
)

func init() {
	// TODO(pedge): needed in tests? will not be needed for golang 1.5 for sure
	runtime.GOMAXPROCS(runtime.NumCPU())
	executil.SetDebug(true)
	common.ForceLogColors()
}

func TestBtrfsFFI(t *testing.T) {
	t.Parallel()
	driver := drive.NewBtrfsDriver(getBtrfsRootDir(t), btrfs.NewFFIAPI())
	runTest(t, driver, testSimple)
}

func TestBtrfsExec(t *testing.T) {
	t.Parallel()
	driver := drive.NewBtrfsDriver(getBtrfsRootDir(t), btrfs.NewExecAPI())
	runTest(t, driver, testSimple)
}

func TestFuseMount(t *testing.T) {
	t.Parallel()
	driver := drive.NewBtrfsDriver(getBtrfsRootDir(t), btrfs.NewExecAPI())
	runTest(t, driver, testMount)
}

func getBtrfsRootDir(t *testing.T) string {
	// TODO(pedge)
	rootDir := os.Getenv("PFS_BTRFS_ROOT")
	if rootDir == "" {
		t.Fatal("PFS_BTRFS_ROOT not set")
	}
	return rootDir
}

func testGetVersion(t *testing.T, apiClient pfs.ApiClient) {
	getVersionResponse, err := pfsutil.GetVersion(apiClient)
	require.NoError(t, err)
	require.Equal(t, common.VersionString(), pfs.VersionString(getVersionResponse.Version))
}

func testSimple(t *testing.T, apiClient pfs.ApiClient) {
	repositoryName := testRepositoryName()

	err := pfsutil.InitRepository(apiClient, repositoryName)
	require.NoError(t, err)

	getCommitInfoResponse, err := pfsutil.GetCommitInfo(apiClient, repositoryName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, getCommitInfoResponse)
	require.Equal(t, "scratch", getCommitInfoResponse.CommitInfo.Commit.Id)
	require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, getCommitInfoResponse.CommitInfo.CommitType)
	require.Nil(t, getCommitInfoResponse.CommitInfo.ParentCommit)

	branchResponse, err := pfsutil.Branch(apiClient, repositoryName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, branchResponse)
	newCommitID := branchResponse.Commit.Id

	getCommitInfoResponse, err = pfsutil.GetCommitInfo(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)
	require.NotNil(t, getCommitInfoResponse)
	require.Equal(t, newCommitID, getCommitInfoResponse.CommitInfo.Commit.Id)
	require.Equal(t, pfs.CommitType_COMMIT_TYPE_WRITE, getCommitInfoResponse.CommitInfo.CommitType)
	require.Equal(t, "scratch", getCommitInfoResponse.CommitInfo.ParentCommit.Id)

	err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/b")
	require.NoError(t, err)
	err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/c")
	require.NoError(t, err)

	var wg sync.WaitGroup
	for i := 0; i < testSize; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			_, iErr := pfsutil.PutFile(apiClient, repositoryName, newCommitID, fmt.Sprintf("a/b/file%d", i), strings.NewReader(fmt.Sprintf("hello%d", i)))
			require.NoError(t, iErr)
			_, iErr = pfsutil.PutFile(apiClient, repositoryName, newCommitID, fmt.Sprintf("a/c/file%d", i), strings.NewReader(fmt.Sprintf("hello%d", i)))
			require.NoError(t, iErr)
		}()
	}
	wg.Wait()

	err = pfsutil.Commit(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)

	getCommitInfoResponse, err = pfsutil.GetCommitInfo(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)
	require.NotNil(t, getCommitInfoResponse)
	require.Equal(t, newCommitID, getCommitInfoResponse.CommitInfo.Commit.Id)
	require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, getCommitInfoResponse.CommitInfo.CommitType)
	require.Equal(t, "scratch", getCommitInfoResponse.CommitInfo.ParentCommit.Id)

	wg = sync.WaitGroup{}
	for i := 0; i < testSize; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			buffer := bytes.NewBuffer(nil)
			iErr := pfsutil.GetFile(apiClient, repositoryName, newCommitID, fmt.Sprintf("a/b/file%d", i), buffer)
			require.NoError(t, iErr)
			require.Equal(t, fmt.Sprintf("hello%d", i), buffer.String())
			buffer = bytes.NewBuffer(nil)
			iErr = pfsutil.GetFile(apiClient, repositoryName, newCommitID, fmt.Sprintf("a/c/file%d", i), buffer)
			require.NoError(t, iErr)
			require.Equal(t, fmt.Sprintf("hello%d", i), buffer.String())
		}()
	}
	wg.Wait()

	listFilesResponse, err := pfsutil.ListFiles(apiClient, repositoryName, newCommitID, "a/b", 0, 1)
	require.NoError(t, err)
	require.Equal(t, testSize, len(listFilesResponse.FileInfo))
	listFilesResponse, err = pfsutil.ListFiles(apiClient, repositoryName, newCommitID, "a/c", 0, 1)
	require.NoError(t, err)
	require.Equal(t, testSize, len(listFilesResponse.FileInfo))

	var fileInfos [7][]*pfs.FileInfo
	wg = sync.WaitGroup{}
	for i := 0; i < 7; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			listFilesResponse, iErr := pfsutil.ListFiles(apiClient, repositoryName, newCommitID, "a/b", uint64(i), 7)
			require.NoError(t, iErr)
			fileInfos[i] = listFilesResponse.FileInfo
		}()
	}
	wg.Wait()
	count := 0
	for i := 0; i < 7; i++ {
		count += len(fileInfos[i])
	}
	require.Equal(t, testSize, count)
}

func testMount(t *testing.T, apiClient pfs.ApiClient) {
	repositoryName := testRepositoryName()

	err := pfsutil.InitRepository(apiClient, repositoryName)
	require.NoError(t, err)

	directory, err := ioutil.TempDir("", "testMount")
	require.NoError(t, err)
	mounter := fuse.NewMounter()
	go func() {
		err = mounter.Mount(apiClient, repositoryName, directory, 0, 1)
		require.NoError(t, err)
	}()
	mounter.Ready()
}

func testRepositoryName() string {
	// TODO could be nice to add callee to this string to make it easy to
	// recover results for debugging
	return fmt.Sprintf("test-%d", atomic.AddInt32(&counter, 1))
}

func runTest(
	t *testing.T,
	driver drive.Driver,
	f func(t *testing.T, apiClient pfs.ApiClient),
) {
	grpctest.Run(
		t,
		testNumServers,
		func(servers map[string]*grpc.Server) {
			discoveryClient := discovery.NewMockClient()
			i := 0
			addresses := make([]string, testNumServers)
			for address := range servers {
				shards := make([]string, testShardsPerServer)
				for j := 0; j < testShardsPerServer; j++ {
					shards[j] = fmt.Sprintf("%d", (i*testShardsPerServer)+j)
				}
				_ = discoveryClient.Set(address+"-master", strings.Join(shards, ","))
				addresses[i] = address
				i++
			}
			_ = discoveryClient.Set("all-addresses", strings.Join(addresses, ","))
			for address, server := range servers {
				combinedAPIServer := NewCombinedAPIServer(
					route.NewSharder(
						testShardsPerServer*testNumServers,
					),
					route.NewRouter(
						route.NewDiscoveryAddresser(
							discoveryClient,
						),
						route.NewDialer(),
						address,
					),
					driver,
				)
				pfs.RegisterApiServer(server, combinedAPIServer)
				pfs.RegisterInternalApiServer(server, combinedAPIServer)
			}
		},
		func(t *testing.T, clientConns map[string]*grpc.ClientConn) {
			var clientConn *grpc.ClientConn
			for _, c := range clientConns {
				clientConn = c
				break
			}
			for _, c := range clientConns {
				if c != clientConn {
					_ = c.Close()
				}
			}
			f(
				t,
				pfs.NewApiClient(
					clientConn,
				),
			)
		},
	)
}
<commit_msg>Adds a failing fuse test.<commit_after>package server

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"google.golang.org/grpc"

	"github.com/pachyderm/pachyderm/src/common"
	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pfs/discovery"
	"github.com/pachyderm/pachyderm/src/pfs/drive"
	"github.com/pachyderm/pachyderm/src/pfs/fuse"
	"github.com/pachyderm/pachyderm/src/pfs/pfsutil"
	"github.com/pachyderm/pachyderm/src/pfs/route"
	"github.com/pachyderm/pachyderm/src/pkg/btrfs"
	"github.com/pachyderm/pachyderm/src/pkg/executil"
	"github.com/pachyderm/pachyderm/src/pkg/grpctest"
	"github.com/stretchr/testify/require"
)

const (
	// TODO(pedge): large numbers of shards takes forever because
	// we are doing tons of btrfs operations on init, is there anything
	// we can do about that?
	testShardsPerServer = 8
	testNumServers      = 8
	testSize            = 1000
)

var (
	counter int32
)

func init() {
	// TODO(pedge): needed in tests? will not be needed for golang 1.5 for sure
	runtime.GOMAXPROCS(runtime.NumCPU())
	executil.SetDebug(true)
	common.ForceLogColors()
}

func TestBtrfsFFI(t *testing.T) {
	t.Parallel()
	driver := drive.NewBtrfsDriver(getBtrfsRootDir(t), btrfs.NewFFIAPI())
	runTest(t, driver, testSimple)
}

func TestBtrfsExec(t *testing.T) {
	t.Parallel()
	driver := drive.NewBtrfsDriver(getBtrfsRootDir(t), btrfs.NewExecAPI())
	runTest(t, driver, testSimple)
}

func TestFuseMount(t *testing.T) {
	t.Parallel()
	driver := drive.NewBtrfsDriver(getBtrfsRootDir(t), btrfs.NewExecAPI())
	runTest(t, driver, testMount)
}

func getBtrfsRootDir(t *testing.T) string {
	// TODO(pedge)
	rootDir := os.Getenv("PFS_BTRFS_ROOT")
	if rootDir == "" {
		t.Fatal("PFS_BTRFS_ROOT not set")
	}
	return rootDir
}

func testGetVersion(t *testing.T, apiClient pfs.ApiClient) {
	getVersionResponse, err := pfsutil.GetVersion(apiClient)
	require.NoError(t, err)
	require.Equal(t, common.VersionString(), pfs.VersionString(getVersionResponse.Version))
}

func testSimple(t *testing.T, apiClient pfs.ApiClient) {
	repositoryName := testRepositoryName()

	err := pfsutil.InitRepository(apiClient, repositoryName)
	require.NoError(t, err)

	getCommitInfoResponse, err := pfsutil.GetCommitInfo(apiClient, repositoryName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, getCommitInfoResponse)
	require.Equal(t, "scratch", getCommitInfoResponse.CommitInfo.Commit.Id)
	require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, getCommitInfoResponse.CommitInfo.CommitType)
	require.Nil(t, getCommitInfoResponse.CommitInfo.ParentCommit)

	branchResponse, err := pfsutil.Branch(apiClient, repositoryName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, branchResponse)
	newCommitID := branchResponse.Commit.Id

	getCommitInfoResponse, err = pfsutil.GetCommitInfo(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)
	require.NotNil(t, getCommitInfoResponse)
	require.Equal(t, newCommitID, getCommitInfoResponse.CommitInfo.Commit.Id)
	require.Equal(t, pfs.CommitType_COMMIT_TYPE_WRITE, getCommitInfoResponse.CommitInfo.CommitType)
	require.Equal(t, "scratch", getCommitInfoResponse.CommitInfo.ParentCommit.Id)

	err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/b")
	require.NoError(t, err)
	err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/c")
	require.NoError(t, err)

	var wg sync.WaitGroup
	for i := 0; i < testSize; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			_, iErr := pfsutil.PutFile(apiClient, repositoryName, newCommitID, fmt.Sprintf("a/b/file%d", i), strings.NewReader(fmt.Sprintf("hello%d", i)))
			require.NoError(t, iErr)
			_, iErr = pfsutil.PutFile(apiClient, repositoryName, newCommitID, fmt.Sprintf("a/c/file%d", i), strings.NewReader(fmt.Sprintf("hello%d", i)))
			require.NoError(t, iErr)
		}()
	}
	wg.Wait()

	err = pfsutil.Commit(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)

	getCommitInfoResponse, err = pfsutil.GetCommitInfo(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)
	require.NotNil(t, getCommitInfoResponse)
	require.Equal(t, newCommitID, getCommitInfoResponse.CommitInfo.Commit.Id)
	require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, getCommitInfoResponse.CommitInfo.CommitType)
	require.Equal(t, "scratch", getCommitInfoResponse.CommitInfo.ParentCommit.Id)

	wg = sync.WaitGroup{}
	for i := 0; i < testSize; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			buffer := bytes.NewBuffer(nil)
			iErr := pfsutil.GetFile(apiClient, repositoryName, newCommitID, fmt.Sprintf("a/b/file%d", i), buffer)
			require.NoError(t, iErr)
			require.Equal(t, fmt.Sprintf("hello%d", i), buffer.String())
			buffer = bytes.NewBuffer(nil)
			iErr = pfsutil.GetFile(apiClient, repositoryName, newCommitID, fmt.Sprintf("a/c/file%d", i), buffer)
			require.NoError(t, iErr)
			require.Equal(t, fmt.Sprintf("hello%d", i), buffer.String())
		}()
	}
	wg.Wait()

	listFilesResponse, err := pfsutil.ListFiles(apiClient, repositoryName, newCommitID, "a/b", 0, 1)
	require.NoError(t, err)
	require.Equal(t, testSize, len(listFilesResponse.FileInfo))
	listFilesResponse, err = pfsutil.ListFiles(apiClient, repositoryName, newCommitID, "a/c", 0, 1)
	require.NoError(t, err)
	require.Equal(t, testSize, len(listFilesResponse.FileInfo))

	var fileInfos [7][]*pfs.FileInfo
	wg = sync.WaitGroup{}
	for i := 0; i < 7; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			listFilesResponse, iErr := pfsutil.ListFiles(apiClient, repositoryName, newCommitID, "a/b", uint64(i), 7)
			require.NoError(t, iErr)
			fileInfos[i] = listFilesResponse.FileInfo
		}()
	}
	wg.Wait()
	count := 0
	for i := 0; i < 7; i++ {
		count += len(fileInfos[i])
	}
	require.Equal(t, testSize, count)
}

func testMount(t *testing.T, apiClient pfs.ApiClient) {
	repositoryName := testRepositoryName()

	err := pfsutil.InitRepository(apiClient, repositoryName)
	require.NoError(t, err)

	directory, err := ioutil.TempDir("", "testMount")
	require.NoError(t, err)
	mounter := fuse.NewMounter()
	go func() {
		err = mounter.Mount(apiClient, repositoryName, directory, 0, 1)
		require.NoError(t, err)
	}()
	mounter.Ready()

	_, err = os.Stat(filepath.Join(directory, "scratch"))
	require.NoError(t, err)

	branchResponse, err := pfsutil.Branch(apiClient, repositoryName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, branchResponse)
	newCommitID := branchResponse.Commit.Id

	_, err = os.Stat(filepath.Join(directory, newCommitID))
	require.NoError(t, err)

	err = ioutil.WriteFile(filepath.Join(directory, newCommitID, "foo"), []byte("foo"), 0666)
	require.NoError(t, err)

	_, err = pfsutil.PutFile(apiClient, repositoryName, newCommitID, "bar", strings.NewReader("bar"))
	require.NoError(t, err)

	err = pfsutil.Commit(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)

	data, err := ioutil.ReadFile(filepath.Join(directory, newCommitID, "foo"))
	require.NoError(t, err)
	require.Equal(t, "foo", string(data))

	data, err = ioutil.ReadFile(filepath.Join(directory, newCommitID, "bar"))
	require.NoError(t, err)
	require.Equal(t, "bar", string(data))
}

func testRepositoryName() string {
	// TODO could be nice to add callee to this string to make it easy to
	// recover results for debugging
	return fmt.Sprintf("test-%d", atomic.AddInt32(&counter, 1))
}

func runTest(
	t *testing.T,
	driver drive.Driver,
	f func(t *testing.T, apiClient pfs.ApiClient),
) {
	grpctest.Run(
		t,
		testNumServers,
		func(servers map[string]*grpc.Server) {
			discoveryClient := discovery.NewMockClient()
			i := 0
			addresses := make([]string, testNumServers)
			for address := range servers {
				shards := make([]string, testShardsPerServer)
				for j := 0; j < testShardsPerServer; j++ {
					shards[j] = fmt.Sprintf("%d", (i*testShardsPerServer)+j)
				}
				_ = discoveryClient.Set(address+"-master", strings.Join(shards, ","))
				addresses[i] = address
				i++
			}
			_ = discoveryClient.Set("all-addresses", strings.Join(addresses, ","))
			for address, server := range servers {
				combinedAPIServer := NewCombinedAPIServer(
					route.NewSharder(
						testShardsPerServer*testNumServers,
					),
					route.NewRouter(
						route.NewDiscoveryAddresser(
							discoveryClient,
						),
						route.NewDialer(),
						address,
					),
					driver,
				)
				pfs.RegisterApiServer(server, combinedAPIServer)
				pfs.RegisterInternalApiServer(server, combinedAPIServer)
			}
		},
		func(t *testing.T, clientConns map[string]*grpc.ClientConn) {
			var clientConn *grpc.ClientConn
			for _, c := range clientConns {
				clientConn = c
				break
			}
			for _, c := range clientConns {
				if c != clientConn {
					_ = c.Close()
				}
			}
			f(
				t,
				pfs.NewApiClient(
					clientConn,
				),
			)
		},
	)
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import (
	"runtime"
	"testing"
)

func TestMulticastJoinAndLeave(t *testing.T) {
	if runtime.GOOS == "windows" {
		return
	}
	addr := &UDPAddr{
		IP:   IPv4zero,
		Port: 0,
	}
	// open a UDPConn
	conn, err := ListenUDP("udp4", addr)
	if err != nil {
		t.Fatal(err)
	}
	defer conn.Close()

	// try to join group
	mcast := IPv4(224, 0, 0, 251)
	err = conn.JoinGroup(mcast)
	if err != nil {
		t.Fatal(err)
	}

	// try to leave group
	err = conn.LeaveGroup(mcast)
	if err != nil {
		t.Fatal(err)
	}
}

func TestJoinFailureWithIPv6Address(t *testing.T) {
	addr := &UDPAddr{
		IP:   IPv4zero,
		Port: 0,
	}

	// open a UDPConn
	conn, err := ListenUDP("udp4", addr)
	if err != nil {
		t.Fatal(err)
	}
	defer conn.Close()
	// try to join group
	mcast := ParseIP("ff02::1")
	err = conn.JoinGroup(mcast)
	if err == nil {
		t.Fatal("JoinGroup succeeded, should fail")
	}
	t.Logf("%s", err)
}
<commit_msg>net: fix multicast tests<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import (
	"runtime"
	"testing"
)

func TestMulticastJoinAndLeave(t *testing.T) {
	if runtime.GOOS == "windows" {
		return
	}

	addr := &UDPAddr{
		IP:   IPv4zero,
		Port: 0,
	}
	// open a UDPConn
	conn, err := ListenUDP("udp4", addr)
	if err != nil {
		t.Fatal(err)
	}
	defer conn.Close()

	// try to join group
	mcast := IPv4(224, 0, 0, 254)
	err = conn.JoinGroup(mcast)
	if err != nil {
		t.Fatal(err)
	}

	// try to leave group
	err = conn.LeaveGroup(mcast)
	if err != nil {
		t.Fatal(err)
	}
}

func TestJoinFailureWithIPv6Address(t *testing.T) {
	addr := &UDPAddr{
		IP:   IPv4zero,
		Port: 0,
	}

	// open a UDPConn
	conn, err := ListenUDP("udp4", addr)
	if err != nil {
		t.Fatal(err)
	}
	defer conn.Close()

	// try to join group
	mcast := ParseIP("ff02::1")
	err = conn.JoinGroup(mcast)
	if err == nil {
		t.Fatal("JoinGroup succeeded, should fail")
	}
	t.Logf("%s", err)
}
<|endoftext|>


<commit_before>// Package fire implements a small and opinionated framework for Go providing
// Ember compatible JSON APIs.
package fire

import (
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"sync"

	"github.com/labstack/echo"
	"github.com/labstack/echo/engine"
	"github.com/labstack/echo/engine/standard"
	"gopkg.in/tomb.v2"
)

// Map is a general purpose map used for configuration.
type Map map[string]interface{}

// A Component that can be mounted in an application.
type Component interface {
	// Describe must return a ComponentInfo struct that describes the component.
	Describe() ComponentInfo
}

// A RoutableComponent is a component that accepts requests from a router for
// routes that haven been registered using Register().
type RoutableComponent interface {
	Component

	// Register will be called by the application with a new echo router.
	Register(router *echo.Echo)
}

// A BootableComponent is an extended component with additional methods for
// setup and teardown.
type BootableComponent interface {
	Component

	// Setup will be called before the applications starts and allows further
	// initialization.
	Setup() error

	// Teardown will be called after applications has stopped and allows proper
	// cleanup.
	Teardown() error
}

// A Phase is used in conjunction with a InspectorComponent and denotes a phase
// the application will undergo.
type Phase int

const (
	// Registration is the phase in which components get registered.
	Registration Phase = iota

	// Setup is the phase in which components get set up.
	Setup

	// Run is the phase in which the application handles requests.
	Run

	// Teardown is the phase in which components get teared down.
	Teardown

	// Termination is the phase in which the applications is terminated.
	Termination
)

// String returns the string representation of the phase.
func (p Phase) String() string {
	switch p {
	case Registration:
		return "Registration"
	case Setup:
		return "Setup"
	case Run:
		return "Run"
	case Teardown:
		return "Teardown"
	case Termination:
		return "Termination"
	}

	return ""
}

// An InspectorComponent is an extended component that is able to inspect the
// boot process of an application and inspect all used components and the router
// instance.
type InspectorComponent interface {
	Component

	// Before is called by the application before the specified phase is
	// initiated by the passed application.
	Before(phase Phase, app *Application)
}

// A ReporterComponent is an extended component that is responsible for
// reporting errors.
type ReporterComponent interface {
	Component

	// Report is called by the application on every occurring error.
	Report(err error) error
}

// An Application provides a simple way to combine multiple components.
type Application struct {
	components []Component
	routables  []RoutableComponent
	bootables  []BootableComponent
	inspectors []InspectorComponent
	reporters  []ReporterComponent

	router *echo.Echo
	mutex  sync.Mutex
	server engine.Server
	tomb   tomb.Tomb
}

// New creates and returns a new Application.
func New() *Application {
	return &Application{
		router: echo.New(),
	}
}

// Mount will mount the passed Component in the application.
//
// Note: Each component should only be mounted once before calling Run or Start.
func (a *Application) Mount(component Component) {
	// synchronize access
	a.mutex.Lock()
	defer a.mutex.Unlock()

	// check status
	if a.server != nil {
		panic("Application has already been started")
	}

	// check component
	if component == nil {
		panic("Mount must be called with a component")
	}

	// add routable component
	if c, ok := component.(RoutableComponent); ok {
		a.routables = append(a.routables, c)
	}

	// add bootable component
	if c, ok := component.(BootableComponent); ok {
		a.bootables = append(a.bootables, c)
	}

	// add inspector
	if c, ok := component.(InspectorComponent); ok {
		a.inspectors = append(a.inspectors, c)
	}

	// add reporter
	if c, ok := component.(ReporterComponent); ok {
		a.reporters = append(a.reporters, c)
	}

	a.components = append(a.components, component)
}

// Components will return all so far registered components.
func (a *Application) Components() []Component {
	return a.components
}

// Router returns the used echo router for this application.
func (a *Application) Router() *echo.Echo {
	return a.router
}

// Start will start the application using a new server listening on the
// specified address.
//
// See StartWith.
func (a *Application) Start(addr string) {
	a.StartWith(standard.New(addr))
}

// StartSecure will start the application with a new server listening on the
// specified address using the provided TLS certificate.
//
// See StartWith.
func (a *Application) StartSecure(addr, certFile, keyFile string) {
	a.StartWith(standard.WithTLS(addr, certFile, keyFile))
}

// StartWith will start the application using the specified server.
//
// Note: Any errors that occur during the boot process of the application and
// later during request processing are reported using the registered reporters.
// If there are no reporters or one of the reporter fails to report the error,
// the calling goroutine will panic and print the error (see Exec).
func (a *Application) StartWith(server engine.Server) {
	// synchronize access
	a.mutex.Lock()
	defer a.mutex.Unlock()

	// check status
	if a.server != nil {
		panic("Application has already been started")
	}

	// check server
	if server == nil {
		panic("StartWith must be called with a server")
	}

	// set server
	a.server = server

	// run app
	a.tomb.Go(a.runner)
}

// Exec will execute the passed function in the context of the application
// and call all reporters if an error occurs.
//
// Note: If a reporter fails to report an occurring error, the current goroutine
// will panic and print the original error and the reporter's error.
func (a *Application) Exec(fn func() error) {
	err := fn()
	if err != nil {
		a.report(err)
	}
}

// Stop will stop a running application and wait until it has been properly stopped.
func (a *Application) Stop() {
	// synchronize access
	a.mutex.Lock()
	defer a.mutex.Unlock()

	// kill controlling tomb
	a.tomb.Kill(nil)

	// stop app by stopping the server
	a.server.Stop()

	// wait until goroutine finishes
	a.tomb.Wait()
}

// Yield will block the calling goroutine until the the application has been
// stopped. It will automatically stop the application if the process receives
// the SIGINT signal.
func (a *Application) Yield() {
	// prepare signal pipeline
	interrupt := make(chan os.Signal, 1)
	signal.Notify(interrupt, os.Interrupt)

	select {
	// wait for interrupt and stop app
	case <-interrupt:
		a.Stop()
	// wait for app to close and return
	case <-a.tomb.Dead():
		return
	}
}

func (a *Application) runner() error {
	a.Exec(a.boot)
	return nil
}

func (a *Application) boot() error {
	// set error handler
	a.router.SetHTTPErrorHandler(a.errorHandler)

	// signal before register event
	for _, i := range a.inspectors {
		i.Before(Registration, a)
	}

	// TODO: Create group and pass group?

	// register routable components
	for _, c := range a.routables {
		c.Register(a.router)
	}

	// signal before setup event
	for _, i := range a.inspectors {
		i.Before(Setup, a)
	}

	// setup bootable components
	for _, c := range a.bootables {
		err := c.Setup()
		if err != nil {
			return err
		}
	}

	// signal before run event
	for _, i := range a.inspectors {
		i.Before(Run, a)
	}

	// run router
	err := a.router.Run(a.server)
	if err != nil {
		select {
		case <-a.tomb.Dying():
			// Stop() has been called and therefore the error returned by run
			// can be ignored as it is always the underlying listener failing
		default:
			return err
		}
	}

	// signal after run event
	for _, i := range a.inspectors {
		i.Before(Teardown, a)
	}

	// teardown bootable components
	for _, c := range a.bootables {
		err := c.Teardown()
		if err != nil {
			return err
		}
	}

	// signal after teardown event
	for _, i := range a.inspectors {
		i.Before(Termination, a)
	}

	return nil
}

func (a *Application) errorHandler(err error, ctx echo.Context) {
	// treat echo.HTTPError instances as already treated errors
	if he, ok := err.(*echo.HTTPError); ok && http.StatusText(he.Code) != "" {
		// write response if not yet committed
		if !ctx.Response().Committed() {
			ctx.NoContent(he.Code)
		}

		return
	}

	// report error
	a.report(err)

	// write response if not yet committed
	if !ctx.Response().Committed() {
		ctx.NoContent(http.StatusInternalServerError)
	}
}

func (a *Application) report(err error) {
	// prepare variable that tracks if the error has at least been reported once
	var reportedOnce bool

	// iterate over all reporters
	for _, r := range a.reporters {
		// attempt to report error
		rErr := r.Report(err)
		if rErr != nil {
			name := r.Describe().Name
			panic(fmt.Sprintf("%s returned '%s' while reporting '%s'", name, rErr, err))
		}

		// mark report
		reportedOnce = true
	}

	// check tracker
	if !reportedOnce {
		panic(fmt.Sprintf("No reporter found to report '%s'", err))
	}
}
<commit_msg>cleanup<commit_after>// Package fire implements a small and opinionated framework for Go providing
// Ember compatible JSON APIs.
package fire

import (
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"sync"

	"github.com/labstack/echo"
	"github.com/labstack/echo/engine"
	"github.com/labstack/echo/engine/standard"
	"gopkg.in/tomb.v2"
)

// Map is a general purpose map used for configuration.
type Map map[string]interface{}

// A Component that can be mounted in an application.
type Component interface {
	// Describe must return a ComponentInfo struct that describes the component.
	Describe() ComponentInfo
}

// A RoutableComponent is a component that accepts requests from a router for
// routes that haven been registered using Register().
type RoutableComponent interface {
	Component

	// Register will be called by the application with a new echo router.
	Register(router *echo.Echo)
}

// A BootableComponent is an extended component with additional methods for
// setup and teardown.
type BootableComponent interface {
	Component

	// Setup will be called before the applications starts and allows further
	// initialization.
	Setup() error

	// Teardown will be called after applications has stopped and allows proper
	// cleanup.
	Teardown() error
}

// A Phase is used in conjunction with a InspectorComponent and denotes a phase
// the application will undergo.
type Phase int

const (
	// Registration is the phase in which components get registered.
	Registration Phase = iota

	// Setup is the phase in which components get set up.
	Setup

	// Run is the phase in which the application handles requests.
	Run

	// Teardown is the phase in which components get teared down.
	Teardown

	// Termination is the phase in which the applications is terminated.
	Termination
)

// String returns the string representation of the phase.
func (p Phase) String() string {
	switch p {
	case Registration:
		return "Registration"
	case Setup:
		return "Setup"
	case Run:
		return "Run"
	case Teardown:
		return "Teardown"
	case Termination:
		return "Termination"
	}

	return ""
}

// An InspectorComponent is an extended component that is able to inspect the
// boot process of an application and inspect all used components and the router
// instance.
type InspectorComponent interface {
	Component

	// Before is called by the application before the specified phase is
	// initiated by the passed application.
	Before(phase Phase, app *Application)
}

// A ReporterComponent is an extended component that is responsible for
// reporting errors.
type ReporterComponent interface {
	Component

	// Report is called by the application on every occurring error.
	Report(err error) error
}

// An Application provides a simple way to combine multiple components.
type Application struct {
	components []Component
	routables  []RoutableComponent
	bootables  []BootableComponent
	inspectors []InspectorComponent
	reporters  []ReporterComponent

	router *echo.Echo
	mutex  sync.Mutex
	server engine.Server
	tomb   tomb.Tomb
}

// New creates and returns a new Application.
func New() *Application {
	return &Application{
		router: echo.New(),
	}
}

// Mount will mount the passed Component in the application.
//
// Note: Each component should only be mounted once before calling Run or Start.
func (a *Application) Mount(component Component) {
	// synchronize access
	a.mutex.Lock()
	defer a.mutex.Unlock()

	// check status
	if a.server != nil {
		panic("Application has already been started")
	}

	// check component
	if component == nil {
		panic("Mount must be called with a component")
	}

	// add routable component
	if c, ok := component.(RoutableComponent); ok {
		a.routables = append(a.routables, c)
	}

	// add bootable component
	if c, ok := component.(BootableComponent); ok {
		a.bootables = append(a.bootables, c)
	}

	// add inspector
	if c, ok := component.(InspectorComponent); ok {
		a.inspectors = append(a.inspectors, c)
	}

	// add reporter
	if c, ok := component.(ReporterComponent); ok {
		a.reporters = append(a.reporters, c)
	}

	a.components = append(a.components, component)
}

// Components will return all so far registered components.
func (a *Application) Components() []Component {
	return a.components
}

// Router returns the used echo router for this application.
func (a *Application) Router() *echo.Echo {
	return a.router
}

// Start will start the application using a new server listening on the
// specified address.
//
// See StartWith.
func (a *Application) Start(addr string) {
	a.StartWith(standard.New(addr))
}

// StartSecure will start the application with a new server listening on the
// specified address using the provided TLS certificate.
//
// See StartWith.
func (a *Application) StartSecure(addr, certFile, keyFile string) {
	a.StartWith(standard.WithTLS(addr, certFile, keyFile))
}

// StartWith will start the application using the specified server.
//
// Note: Any errors that occur during the boot process of the application and
// later during request processing are reported using the registered reporters.
// If there are no reporters or one of the reporter fails to report the error,
// the calling goroutine will panic and print the error (see Exec).
func (a *Application) StartWith(server engine.Server) {
	// synchronize access
	a.mutex.Lock()
	defer a.mutex.Unlock()

	// check status
	if a.server != nil {
		panic("Application has already been started")
	}

	// check server
	if server == nil {
		panic("StartWith must be called with a server")
	}

	// set server
	a.server = server

	// run app
	a.tomb.Go(a.runner)
}

// Exec will execute the passed function in the context of the application
// and call all reporters if an error occurs.
//
// Note: If a reporter fails to report an occurring error, the current goroutine
// will panic and print the original error and the reporter's error.
func (a *Application) Exec(fn func() error) {
	err := fn()
	if err != nil {
		a.report(err)
	}
}

// Stop will stop a running application and wait until it has been properly stopped.
func (a *Application) Stop() {
	// synchronize access
	a.mutex.Lock()
	defer a.mutex.Unlock()

	// kill controlling tomb
	a.tomb.Kill(nil)

	// stop app by stopping the server
	a.server.Stop()

	// wait until goroutine finishes
	a.tomb.Wait()
}

// Yield will block the calling goroutine until the the application has been
// stopped. It will automatically stop the application if the process receives
// the SIGINT signal.
func (a *Application) Yield() {
	// prepare signal pipeline
	interrupt := make(chan os.Signal, 1)
	signal.Notify(interrupt, os.Interrupt)

	select {
	// wait for interrupt and stop app
	case <-interrupt:
		a.Stop()
	// wait for app to close and return
	case <-a.tomb.Dead():
		return
	}
}

func (a *Application) runner() error {
	a.Exec(a.boot)
	return nil
}

func (a *Application) boot() error {
	// set error handler
	a.router.SetHTTPErrorHandler(a.errorHandler)

	// signal before registration event
	for _, i := range a.inspectors {
		i.Before(Registration, a)
	}

	// register routable components
	for _, c := range a.routables {
		c.Register(a.router)
	}

	// signal before setup event
	for _, i := range a.inspectors {
		i.Before(Setup, a)
	}

	// setup bootable components
	for _, c := range a.bootables {
		err := c.Setup()
		if err != nil {
			return err
		}
	}

	// signal before run event
	for _, i := range a.inspectors {
		i.Before(Run, a)
	}

	// run router
	err := a.router.Run(a.server)
	if err != nil {
		select {
		case <-a.tomb.Dying():
			// Stop() has been called and therefore the error returned by run
			// can be ignored as it is always the underlying listener failing
		default:
			return err
		}
	}

	// signal after run event
	for _, i := range a.inspectors {
		i.Before(Teardown, a)
	}

	// teardown bootable components
	for _, c := range a.bootables {
		err := c.Teardown()
		if err != nil {
			return err
		}
	}

	// signal after teardown event
	for _, i := range a.inspectors {
		i.Before(Termination, a)
	}

	return nil
}

func (a *Application) errorHandler(err error, ctx echo.Context) {
	// treat echo.HTTPError instances as already treated errors
	if he, ok := err.(*echo.HTTPError); ok && http.StatusText(he.Code) != "" {
		// write response if not yet committed
		if !ctx.Response().Committed() {
			ctx.NoContent(he.Code)
		}

		return
	}

	// report error
	a.report(err)

	// write response if not yet committed
	if !ctx.Response().Committed() {
		ctx.NoContent(http.StatusInternalServerError)
	}
}

func (a *Application) report(err error) {
	// prepare variable that tracks if the error has at least been reported once
	var reportedOnce bool

	// iterate over all reporters
	for _, r := range a.reporters {
		// attempt to report error
		rErr := r.Report(err)
		if rErr != nil {
			name := r.Describe().Name
			panic(fmt.Sprintf("%s returned '%s' while reporting '%s'", name, rErr, err))
		}

		// mark report
		reportedOnce = true
	}

	// check tracker
	if !reportedOnce {
		panic(fmt.Sprintf("No reporter found to report '%s'", err))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"strings"
)

type Application interface {
	BuildCmd() string

	BuildOutputDir() string

	PostDeployCmd() string

	RunCmd(port int) string

	HealthEndpoint() string

	// e.g. prod -> Target{...}
	Target(server string) *Target
}

type AppImpl struct {
	def ApplicationDef
}

type Target struct {
	Ssh string // e.g. user@host

	SshPort int // optional

	Base int // camus base port, e.g. 8000
}

type ApplicationDef struct {
	Name string

	BuildCmd string

	BuildOutputDir string

	PostDeployCmd string

	// needs a %PORT% part for port subsitution
	RunCmd string

	HealthEndpoint string

	// e.g. user@host  (no path)
	Targets map[string]*Target
}

func ApplicationFromConfig(isClient bool, file string) (Application, error) {
	var def ApplicationDef

	data, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, err
	}

	json.Unmarshal(data, &def)

	errMsg := func(str string, args ...interface{}) (Application, error) {
		return nil, fmt.Errorf("deploy.json: "+str, args...)
	}

	if isClient {
		if len(def.Name) == 0 {
			return errMsg("Missing Name")
		}
		if len(def.BuildCmd) == 0 {
			return errMsg("Missing BuildCmd")
		}
		if len(def.BuildOutputDir) == 0 {
			return errMsg("Missing BuildOutputDir")
		}

		foundTarget := false
		for name, target := range def.Targets {
			foundTarget = true
			if len(target.Ssh) == 0 {
				return errMsg("Missing %s.Ssh", name)
			}
			if target.Base == 0 {
				return errMsg("Missing %s.Base", name)
			}
			if target.SshPort == 0 {
				target.SshPort = 22
			}
		}

		if !foundTarget {
			return errMsg("No targets specified")
		}
	}

	if len(def.RunCmd) == 0 {
		return errMsg("Missing RunCmd")
	}

	if len(def.HealthEndpoint) == 0 {
		if isClient {
			return errMsg("Missing HealthEndpoint")
		} else {
			def.HealthEndpoint = "/"
		}
	}

	return &AppImpl{def}, nil
}

func (a *AppImpl) RunCmd(port int) string {
	return strings.Replace(a.def.RunCmd, "%PORT%", fmt.Sprintf("%d", port), -1)
}
func (a *AppImpl) Target(server string) *Target {
	return a.def.Targets[server]
}

func (a *AppImpl) BuildCmd() string {
	return a.def.BuildCmd
}
func (a *AppImpl) PostDeployCmd() string {
	return a.def.PostDeployCmd
}
func (a *AppImpl) BuildOutputDir() string {
	return a.def.BuildOutputDir
}
func (a *AppImpl) HealthEndpoint() string {
	return a.def.HealthEndpoint
}
<commit_msg>Better error message if deploy.json fails to parse<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"strings"
)

type Application interface {
	BuildCmd() string

	BuildOutputDir() string

	PostDeployCmd() string

	RunCmd(port int) string

	HealthEndpoint() string

	// e.g. prod -> Target{...}
	Target(server string) *Target
}

type AppImpl struct {
	def ApplicationDef
}

type Target struct {
	Ssh string // e.g. user@host

	SshPort int // optional

	Base int // camus base port, e.g. 8000
}

type ApplicationDef struct {
	Name string

	BuildCmd string

	BuildOutputDir string

	PostDeployCmd string

	// needs a %PORT% part for port subsitution
	RunCmd string

	HealthEndpoint string

	// e.g. user@host  (no path)
	Targets map[string]*Target
}

func ApplicationFromConfig(isClient bool, file string) (Application, error) {
	var def ApplicationDef

	data, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, err
	}

	errMsg := func(str string, args ...interface{}) (Application, error) {
		return nil, fmt.Errorf("deploy.json: "+str, args...)
	}

	if err := json.Unmarshal(data, &def); err != nil {
		return errMsg(fmt.Sprintf("Invalid json %s", err))
	}

	if isClient {
		if len(def.Name) == 0 {
			return errMsg("Missing Name")
		}
		if len(def.BuildCmd) == 0 {
			return errMsg("Missing BuildCmd")
		}
		if len(def.BuildOutputDir) == 0 {
			return errMsg("Missing BuildOutputDir")
		}

		foundTarget := false
		for name, target := range def.Targets {
			foundTarget = true
			if len(target.Ssh) == 0 {
				return errMsg("Missing %s.Ssh", name)
			}
			if target.Base == 0 {
				return errMsg("Missing %s.Base", name)
			}
			if target.SshPort == 0 {
				target.SshPort = 22
			}
		}

		if !foundTarget {
			return errMsg("No targets specified")
		}
	}

	if len(def.RunCmd) == 0 {
		return errMsg("Missing RunCmd")
	}

	if len(def.HealthEndpoint) == 0 {
		if isClient {
			return errMsg("Missing HealthEndpoint")
		} else {
			def.HealthEndpoint = "/"
		}
	}

	return &AppImpl{def}, nil
}

func (a *AppImpl) RunCmd(port int) string {
	return strings.Replace(a.def.RunCmd, "%PORT%", fmt.Sprintf("%d", port), -1)
}
func (a *AppImpl) Target(server string) *Target {
	return a.def.Targets[server]
}

func (a *AppImpl) BuildCmd() string {
	return a.def.BuildCmd
}
func (a *AppImpl) PostDeployCmd() string {
	return a.def.PostDeployCmd
}
func (a *AppImpl) BuildOutputDir() string {
	return a.def.BuildOutputDir
}
func (a *AppImpl) HealthEndpoint() string {
	return a.def.HealthEndpoint
}
<|endoftext|>