<commit_before>package http

import (
	http "net/http"
	"strings"

	influxdb "github.com/influxdata/influxdb"
	"github.com/influxdata/influxdb/authorizer"
	"github.com/influxdata/influxdb/chronograf/server"
	"github.com/influxdata/influxdb/query"
	"github.com/influxdata/influxdb/storage"
	"go.uber.org/zap"
)

// APIHandler is a collection of all the service handlers.
type APIHandler struct {
	BucketHandler        *BucketHandler
	UserHandler          *UserHandler
	OrgHandler           *OrgHandler
	AuthorizationHandler *AuthorizationHandler
	DashboardHandler     *DashboardHandler
	AssetHandler         *AssetHandler
	ChronografHandler    *ChronografHandler
	ScraperHandler       *ScraperHandler
	SourceHandler        *SourceHandler
	MacroHandler         *MacroHandler
	TaskHandler          *TaskHandler
	TelegrafHandler      *TelegrafHandler
	QueryHandler         *FluxHandler
	ProtoHandler         *ProtoHandler
	WriteHandler         *WriteHandler
	SetupHandler         *SetupHandler
	SessionHandler       *SessionHandler
}

// APIBackend is all services and associated parameters required to construct
// an APIHandler.
type APIBackend struct {
	DeveloperMode bool
	Logger        *zap.Logger

	NewBucketService func(*influxdb.Source) (influxdb.BucketService, error)
	NewQueryService  func(*influxdb.Source) (query.ProxyQueryService, error)

	PointsWriter                    storage.PointsWriter
	AuthorizationService            influxdb.AuthorizationService
	BucketService                   influxdb.BucketService
	SessionService                  influxdb.SessionService
	UserService                     influxdb.UserService
	OrganizationService             influxdb.OrganizationService
	UserResourceMappingService      influxdb.UserResourceMappingService
	LabelService                    influxdb.LabelService
	DashboardService                influxdb.DashboardService
	DashboardOperationLogService    influxdb.DashboardOperationLogService
	BucketOperationLogService       influxdb.BucketOperationLogService
	UserOperationLogService         influxdb.UserOperationLogService
	OrganizationOperationLogService influxdb.OrganizationOperationLogService
	SourceService                   influxdb.SourceService
	MacroService                    influxdb.MacroService
	BasicAuthService                influxdb.BasicAuthService
	OnboardingService               influxdb.OnboardingService
	ProxyQueryService               query.ProxyQueryService
	TaskService                     influxdb.TaskService
	TelegrafService                 influxdb.TelegrafConfigStore
	ScraperTargetStoreService       influxdb.ScraperTargetStoreService
	SecretService                   influxdb.SecretService
	LookupService                   influxdb.LookupService
	ChronografService               *server.Service
	ProtoService                    influxdb.ProtoService
}

// NewAPIHandler constructs all api handlers beneath it and returns an APIHandler
func NewAPIHandler(b *APIBackend) *APIHandler {
	h := &APIHandler{}

	sessionBackend := NewSessionBackend(b)
	h.SessionHandler = NewSessionHandler(sessionBackend)

	h.BucketHandler = NewBucketHandler(b.UserResourceMappingService, b.LabelService, b.UserService)
	h.BucketHandler.BucketService = authorizer.NewBucketService(b.BucketService)
	h.BucketHandler.OrganizationService = b.OrganizationService
	h.BucketHandler.BucketOperationLogService = b.BucketOperationLogService

	h.OrgHandler = NewOrgHandler(b.UserResourceMappingService, b.LabelService, b.UserService)
	h.OrgHandler.OrganizationService = authorizer.NewOrgService(b.OrganizationService)
	h.OrgHandler.OrganizationOperationLogService = b.OrganizationOperationLogService
	h.OrgHandler.SecretService = b.SecretService

	h.UserHandler = NewUserHandler()
	h.UserHandler.UserService = b.UserService
	h.UserHandler.BasicAuthService = b.BasicAuthService
	h.UserHandler.UserOperationLogService = b.UserOperationLogService

	h.DashboardHandler = NewDashboardHandler(b.UserResourceMappingService, b.LabelService, b.UserService)
	h.DashboardHandler.DashboardService = b.DashboardService
	h.DashboardHandler.DashboardOperationLogService = b.DashboardOperationLogService

	h.MacroHandler = NewMacroHandler()
	h.MacroHandler.MacroService = authorizer.NewMacroService(b.MacroService)

	h.AuthorizationHandler = NewAuthorizationHandler(b.UserService)
	h.AuthorizationHandler.OrganizationService = b.OrganizationService
	h.AuthorizationHandler.AuthorizationService = b.AuthorizationService
	h.AuthorizationHandler.LookupService = b.LookupService
	h.AuthorizationHandler.Logger = b.Logger.With(zap.String("handler", "auth"))

	h.ScraperHandler = NewScraperHandler()
	h.ScraperHandler.ScraperStorageService = b.ScraperTargetStoreService
	h.ScraperHandler.BucketService = b.BucketService
	h.ScraperHandler.OrganizationService = b.OrganizationService
	h.ScraperHandler.Logger = b.Logger.With(zap.String("handler", "scraper"))

	h.SourceHandler = NewSourceHandler()
	h.SourceHandler.SourceService = b.SourceService
	h.SourceHandler.NewBucketService = b.NewBucketService
	h.SourceHandler.NewQueryService = b.NewQueryService

	h.SetupHandler = NewSetupHandler()
	h.SetupHandler.OnboardingService = b.OnboardingService

	h.TaskHandler = NewTaskHandler(b.UserResourceMappingService, b.LabelService, b.Logger, b.UserService)
	h.TaskHandler.TaskService = b.TaskService
	h.TaskHandler.AuthorizationService = b.AuthorizationService
	h.TaskHandler.UserResourceMappingService = b.UserResourceMappingService

	h.TelegrafHandler = NewTelegrafHandler(
		b.Logger.With(zap.String("handler", "telegraf")),
		b.UserResourceMappingService,
		b.LabelService,
		authorizer.NewTelegrafConfigService(b.TelegrafService, b.UserResourceMappingService),
		b.UserService,
		b.OrganizationService,
	)

	h.WriteHandler = NewWriteHandler(b.PointsWriter)
	h.WriteHandler.OrganizationService = b.OrganizationService
	h.WriteHandler.BucketService = b.BucketService
	h.WriteHandler.Logger = b.Logger.With(zap.String("handler", "write"))

	h.QueryHandler = NewFluxHandler()
	h.QueryHandler.OrganizationService = b.OrganizationService
	h.QueryHandler.Logger = b.Logger.With(zap.String("handler", "query"))
	h.QueryHandler.ProxyQueryService = b.ProxyQueryService

	h.ProtoHandler = NewProtoHandler(NewProtoBackend(b))

	h.ChronografHandler = NewChronografHandler(b.ChronografService)

	return h
}

var apiLinks = map[string]interface{}{
	// when adding new links, please take care to keep this list alphabetical
	// as this makes it easier to verify values against the swagger document.
	"authorizations": "/api/v2/authorizations",
	"buckets":        "/api/v2/buckets",
	"dashboards":     "/api/v2/dashboards",
	"external": map[string]string{
		"statusFeed": "https://www.influxdata.com/feed/json",
	},
	"macros": "/api/v2/macros",
	"me":     "/api/v2/me",
	"orgs":   "/api/v2/orgs",
	"protos": "/api/v2/protos",
	"query": map[string]string{
		"self":        "/api/v2/query",
		"ast":         "/api/v2/query/ast",
		"analyze":     "/api/v2/query/analyze",
		"spec":        "/api/v2/query/spec",
		"suggestions": "/api/v2/query/suggestions",
	},
	"setup":          "/api/v2/setup",
	"signin":         "/api/v2/signin",
	"signout":        "/api/v2/signout",
	"sources":        "/api/v2/sources",
	"scrapertargets": "/api/v2/scrapertargets",
	"system": map[string]string{
		"metrics": "/metrics",
		"debug":   "/debug/pprof",
		"health":  "/health",
	},
	"tasks":     "/api/v2/tasks",
	"telegrafs": "/api/v2/telegrafs",
	"users":     "/api/v2/users",
	"write":     "/api/v2/write",
}

func (h *APIHandler) serveLinks(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()
	if err := encodeResponse(ctx, w, http.StatusOK, apiLinks); err != nil {
		EncodeError(ctx, err, w)
		return
	}
}

// ServeHTTP delegates a request to the appropriate subhandler.
func (h *APIHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	setCORSResponseHeaders(w, r)
	if r.Method == "OPTIONS" {
		return
	}

	// Serve the links base links for the API.
	if r.URL.Path == "/api/v2/" || r.URL.Path == "/api/v2" {
		h.serveLinks(w, r)
		return
	}

	if r.URL.Path == "/api/v2/signin" || r.URL.Path == "/api/v2/signout" {
		h.SessionHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/setup") {
		h.SetupHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/write") {
		h.WriteHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/query") {
		h.QueryHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/buckets") {
		h.BucketHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/users") {
		h.UserHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/me") {
		h.UserHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/orgs") {
		h.OrgHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/authorizations") {
		h.AuthorizationHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/dashboards") {
		h.DashboardHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/sources") {
		h.SourceHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/scrapertargets") {
		h.ScraperHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/tasks") {
		h.TaskHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/telegrafs") {
		h.TelegrafHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/macros") {
		h.MacroHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/protos") {
		h.ProtoHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/chronograf/") {
		h.ChronografHandler.ServeHTTP(w, r)
		return
	}

	notFoundHandler(w, r)
}
<commit_msg>feat(http): wire up authorized authorization service<commit_after>package http

import (
	http "net/http"
	"strings"

	influxdb "github.com/influxdata/influxdb"
	"github.com/influxdata/influxdb/authorizer"
	"github.com/influxdata/influxdb/chronograf/server"
	"github.com/influxdata/influxdb/query"
	"github.com/influxdata/influxdb/storage"
	"go.uber.org/zap"
)

// APIHandler is a collection of all the service handlers.
type APIHandler struct {
	BucketHandler        *BucketHandler
	UserHandler          *UserHandler
	OrgHandler           *OrgHandler
	AuthorizationHandler *AuthorizationHandler
	DashboardHandler     *DashboardHandler
	AssetHandler         *AssetHandler
	ChronografHandler    *ChronografHandler
	ScraperHandler       *ScraperHandler
	SourceHandler        *SourceHandler
	MacroHandler         *MacroHandler
	TaskHandler          *TaskHandler
	TelegrafHandler      *TelegrafHandler
	QueryHandler         *FluxHandler
	ProtoHandler         *ProtoHandler
	WriteHandler         *WriteHandler
	SetupHandler         *SetupHandler
	SessionHandler       *SessionHandler
}

// APIBackend is all services and associated parameters required to construct
// an APIHandler.
type APIBackend struct {
	DeveloperMode bool
	Logger        *zap.Logger

	NewBucketService func(*influxdb.Source) (influxdb.BucketService, error)
	NewQueryService  func(*influxdb.Source) (query.ProxyQueryService, error)

	PointsWriter                    storage.PointsWriter
	AuthorizationService            influxdb.AuthorizationService
	BucketService                   influxdb.BucketService
	SessionService                  influxdb.SessionService
	UserService                     influxdb.UserService
	OrganizationService             influxdb.OrganizationService
	UserResourceMappingService      influxdb.UserResourceMappingService
	LabelService                    influxdb.LabelService
	DashboardService                influxdb.DashboardService
	DashboardOperationLogService    influxdb.DashboardOperationLogService
	BucketOperationLogService       influxdb.BucketOperationLogService
	UserOperationLogService         influxdb.UserOperationLogService
	OrganizationOperationLogService influxdb.OrganizationOperationLogService
	SourceService                   influxdb.SourceService
	MacroService                    influxdb.MacroService
	BasicAuthService                influxdb.BasicAuthService
	OnboardingService               influxdb.OnboardingService
	ProxyQueryService               query.ProxyQueryService
	TaskService                     influxdb.TaskService
	TelegrafService                 influxdb.TelegrafConfigStore
	ScraperTargetStoreService       influxdb.ScraperTargetStoreService
	SecretService                   influxdb.SecretService
	LookupService                   influxdb.LookupService
	ChronografService               *server.Service
	ProtoService                    influxdb.ProtoService
}

// NewAPIHandler constructs all api handlers beneath it and returns an APIHandler
func NewAPIHandler(b *APIBackend) *APIHandler {
	h := &APIHandler{}

	sessionBackend := NewSessionBackend(b)
	h.SessionHandler = NewSessionHandler(sessionBackend)

	h.BucketHandler = NewBucketHandler(b.UserResourceMappingService, b.LabelService, b.UserService)
	h.BucketHandler.BucketService = authorizer.NewBucketService(b.BucketService)
	h.BucketHandler.OrganizationService = b.OrganizationService
	h.BucketHandler.BucketOperationLogService = b.BucketOperationLogService

	h.OrgHandler = NewOrgHandler(b.UserResourceMappingService, b.LabelService, b.UserService)
	h.OrgHandler.OrganizationService = authorizer.NewOrgService(b.OrganizationService)
	h.OrgHandler.OrganizationOperationLogService = b.OrganizationOperationLogService
	h.OrgHandler.SecretService = b.SecretService

	h.UserHandler = NewUserHandler()
	h.UserHandler.UserService = b.UserService
	h.UserHandler.BasicAuthService = b.BasicAuthService
	h.UserHandler.UserOperationLogService = b.UserOperationLogService

	h.DashboardHandler = NewDashboardHandler(b.UserResourceMappingService, b.LabelService, b.UserService)
	h.DashboardHandler.DashboardService = b.DashboardService
	h.DashboardHandler.DashboardOperationLogService = b.DashboardOperationLogService

	h.MacroHandler = NewMacroHandler()
	h.MacroHandler.MacroService = authorizer.NewMacroService(b.MacroService)

	h.AuthorizationHandler = NewAuthorizationHandler(b.UserService)
	h.AuthorizationHandler.OrganizationService = b.OrganizationService
	h.AuthorizationHandler.AuthorizationService = authorizer.NewAuthorizationService(b.AuthorizationService)
	h.AuthorizationHandler.LookupService = b.LookupService
	h.AuthorizationHandler.Logger = b.Logger.With(zap.String("handler", "auth"))

	h.ScraperHandler = NewScraperHandler()
	h.ScraperHandler.ScraperStorageService = b.ScraperTargetStoreService
	h.ScraperHandler.BucketService = b.BucketService
	h.ScraperHandler.OrganizationService = b.OrganizationService
	h.ScraperHandler.Logger = b.Logger.With(zap.String("handler", "scraper"))

	h.SourceHandler = NewSourceHandler()
	h.SourceHandler.SourceService = b.SourceService
	h.SourceHandler.NewBucketService = b.NewBucketService
	h.SourceHandler.NewQueryService = b.NewQueryService

	h.SetupHandler = NewSetupHandler()
	h.SetupHandler.OnboardingService = b.OnboardingService

	h.TaskHandler = NewTaskHandler(b.UserResourceMappingService, b.LabelService, b.Logger, b.UserService)
	h.TaskHandler.TaskService = b.TaskService
	h.TaskHandler.AuthorizationService = b.AuthorizationService
	h.TaskHandler.UserResourceMappingService = b.UserResourceMappingService

	h.TelegrafHandler = NewTelegrafHandler(
		b.Logger.With(zap.String("handler", "telegraf")),
		b.UserResourceMappingService,
		b.LabelService,
		authorizer.NewTelegrafConfigService(b.TelegrafService, b.UserResourceMappingService),
		b.UserService,
		b.OrganizationService,
	)

	h.WriteHandler = NewWriteHandler(b.PointsWriter)
	h.WriteHandler.OrganizationService = b.OrganizationService
	h.WriteHandler.BucketService = b.BucketService
	h.WriteHandler.Logger = b.Logger.With(zap.String("handler", "write"))

	h.QueryHandler = NewFluxHandler()
	h.QueryHandler.OrganizationService = b.OrganizationService
	h.QueryHandler.Logger = b.Logger.With(zap.String("handler", "query"))
	h.QueryHandler.ProxyQueryService = b.ProxyQueryService

	h.ProtoHandler = NewProtoHandler(NewProtoBackend(b))

	h.ChronografHandler = NewChronografHandler(b.ChronografService)

	return h
}

var apiLinks = map[string]interface{}{
	// when adding new links, please take care to keep this list alphabetical
	// as this makes it easier to verify values against the swagger document.
	"authorizations": "/api/v2/authorizations",
	"buckets":        "/api/v2/buckets",
	"dashboards":     "/api/v2/dashboards",
	"external": map[string]string{
		"statusFeed": "https://www.influxdata.com/feed/json",
	},
	"macros": "/api/v2/macros",
	"me":     "/api/v2/me",
	"orgs":   "/api/v2/orgs",
	"protos": "/api/v2/protos",
	"query": map[string]string{
		"self":        "/api/v2/query",
		"ast":         "/api/v2/query/ast",
		"analyze":     "/api/v2/query/analyze",
		"spec":        "/api/v2/query/spec",
		"suggestions": "/api/v2/query/suggestions",
	},
	"setup":          "/api/v2/setup",
	"signin":         "/api/v2/signin",
	"signout":        "/api/v2/signout",
	"sources":        "/api/v2/sources",
	"scrapertargets": "/api/v2/scrapertargets",
	"system": map[string]string{
		"metrics": "/metrics",
		"debug":   "/debug/pprof",
		"health":  "/health",
	},
	"tasks":     "/api/v2/tasks",
	"telegrafs": "/api/v2/telegrafs",
	"users":     "/api/v2/users",
	"write":     "/api/v2/write",
}

func (h *APIHandler) serveLinks(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()
	if err := encodeResponse(ctx, w, http.StatusOK, apiLinks); err != nil {
		EncodeError(ctx, err, w)
		return
	}
}

// ServeHTTP delegates a request to the appropriate subhandler.
func (h *APIHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	setCORSResponseHeaders(w, r)
	if r.Method == "OPTIONS" {
		return
	}

	// Serve the links base links for the API.
	if r.URL.Path == "/api/v2/" || r.URL.Path == "/api/v2" {
		h.serveLinks(w, r)
		return
	}

	if r.URL.Path == "/api/v2/signin" || r.URL.Path == "/api/v2/signout" {
		h.SessionHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/setup") {
		h.SetupHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/write") {
		h.WriteHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/query") {
		h.QueryHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/buckets") {
		h.BucketHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/users") {
		h.UserHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/me") {
		h.UserHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/orgs") {
		h.OrgHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/authorizations") {
		h.AuthorizationHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/dashboards") {
		h.DashboardHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/sources") {
		h.SourceHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/scrapertargets") {
		h.ScraperHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/tasks") {
		h.TaskHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/telegrafs") {
		h.TelegrafHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/macros") {
		h.MacroHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/api/v2/protos") {
		h.ProtoHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/chronograf/") {
		h.ChronografHandler.ServeHTTP(w, r)
		return
	}

	notFoundHandler(w, r)
}
<|endoftext|>


<commit_before>package taskrunner

import (
	"context"
	"fmt"

	log "github.com/hashicorp/go-hclog"
	"github.com/hashicorp/nomad/client/allocrunner/interfaces"
	"github.com/hashicorp/nomad/client/devicemanager"
	"github.com/hashicorp/nomad/plugins/device"
	"github.com/hashicorp/nomad/plugins/drivers"
)

// deviceHook is used to retrieve device mounting information.
type deviceHook struct {
	logger log.Logger
	dm     devicemanager.Manager
}

func newDeviceHook(dm devicemanager.Manager, logger log.Logger) *deviceHook {
	h := &deviceHook{
		dm: dm,
	}
	h.logger = logger.Named(h.Name())
	return h
}

func (*deviceHook) Name() string {
	return "devices"
}

func (h *deviceHook) Prestart(ctx context.Context, req *interfaces.TaskPrestartRequest, resp *interfaces.TaskPrestartResponse) error {
	if len(req.TaskResources.Devices) == 0 {
		resp.Done = true
		return nil
	}

	// Capture the responses
	var reservations []*device.ContainerReservation
	for _, req := range req.TaskResources.Devices {
		// Ask the device manager for the reservation information
		res, err := h.dm.Reserve(req)
		if err != nil {
			return fmt.Errorf("failed to reserve device %s: %v", req.ID(), err)
		}

		reservations = append(reservations, res)
	}

	// Build the response
	for _, res := range reservations {
		for k, v := range res.Envs {
			if resp.Env == nil {
				resp.Env = make(map[string]string)
			}

			resp.Env[k] = v
		}

		for _, m := range res.Mounts {
			resp.Mounts = append(resp.Mounts, convertMount(m))
		}

		for _, d := range res.Devices {
			resp.Devices = append(resp.Devices, convertDevice(d))
		}
	}

	resp.Done = true
	return nil
}

func convertMount(in *device.Mount) *drivers.MountConfig {
	if in == nil {
		return nil
	}

	return &drivers.MountConfig{
		TaskPath: in.TaskPath,
		HostPath: in.HostPath,
		Readonly: in.ReadOnly,
	}
}

func convertDevice(in *device.DeviceSpec) *drivers.DeviceConfig {
	if in == nil {
		return nil
	}

	return &drivers.DeviceConfig{
		TaskPath:    in.TaskPath,
		HostPath:    in.HostPath,
		Permissions: in.CgroupPerms,
	}
}
<commit_msg>assume that slices contain only non-nil items<commit_after>package taskrunner

import (
	"context"
	"fmt"

	log "github.com/hashicorp/go-hclog"
	"github.com/hashicorp/nomad/client/allocrunner/interfaces"
	"github.com/hashicorp/nomad/client/devicemanager"
	"github.com/hashicorp/nomad/plugins/device"
	"github.com/hashicorp/nomad/plugins/drivers"
)

// deviceHook is used to retrieve device mounting information.
type deviceHook struct {
	logger log.Logger
	dm     devicemanager.Manager
}

func newDeviceHook(dm devicemanager.Manager, logger log.Logger) *deviceHook {
	h := &deviceHook{
		dm: dm,
	}
	h.logger = logger.Named(h.Name())
	return h
}

func (*deviceHook) Name() string {
	return "devices"
}

func (h *deviceHook) Prestart(ctx context.Context, req *interfaces.TaskPrestartRequest, resp *interfaces.TaskPrestartResponse) error {
	if len(req.TaskResources.Devices) == 0 {
		resp.Done = true
		return nil
	}

	// Capture the responses
	var reservations []*device.ContainerReservation
	for _, req := range req.TaskResources.Devices {
		// Ask the device manager for the reservation information
		res, err := h.dm.Reserve(req)
		if err != nil {
			return fmt.Errorf("failed to reserve device %s: %v", req.ID(), err)
		}

		reservations = append(reservations, res)
	}

	// Build the response
	for _, res := range reservations {
		for k, v := range res.Envs {
			if resp.Env == nil {
				resp.Env = make(map[string]string)
			}

			resp.Env[k] = v
		}

		for _, m := range res.Mounts {
			resp.Mounts = append(resp.Mounts, convertMount(m))
		}

		for _, d := range res.Devices {
			resp.Devices = append(resp.Devices, convertDevice(d))
		}
	}

	resp.Done = true
	return nil
}

func convertMount(in *device.Mount) *drivers.MountConfig {
	return &drivers.MountConfig{
		TaskPath: in.TaskPath,
		HostPath: in.HostPath,
		Readonly: in.ReadOnly,
	}
}

func convertDevice(in *device.DeviceSpec) *drivers.DeviceConfig {
	return &drivers.DeviceConfig{
		TaskPath:    in.TaskPath,
		HostPath:    in.HostPath,
		Permissions: in.CgroupPerms,
	}
}
<|endoftext|>


<commit_before>package etcdstorerunner

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"sync"
	"time"

	"github.com/cloudfoundry/storeadapter"
	"github.com/cloudfoundry/storeadapter/etcdstoreadapter"
	"github.com/cloudfoundry/storeadapter/workerpool"
	etcdclient "github.com/coreos/go-etcd/etcd"
	"github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

type ETCDClusterRunner struct {
	startingPort int
	numNodes     int
	etcdSessions []*gexec.Session
	running      bool
	client       *etcdclient.Client

	mutex *sync.RWMutex
}

func NewETCDClusterRunner(startingPort int, numNodes int) *ETCDClusterRunner {
	return &ETCDClusterRunner{
		startingPort: startingPort,
		numNodes:     numNodes,

		mutex: &sync.RWMutex{},
	}
}

func (etcd *ETCDClusterRunner) Start() {
	etcd.start(true)
}

func (etcd *ETCDClusterRunner) Stop() {
	etcd.stop(true)
}

func (etcd *ETCDClusterRunner) KillWithFire() {
	etcd.kill()
}

func (etcd *ETCDClusterRunner) GoAway() {
	etcd.stop(false)
}

func (etcd *ETCDClusterRunner) ComeBack() {
	etcd.start(false)
}

func (etcd *ETCDClusterRunner) NodeURLS() []string {
	urls := make([]string, etcd.numNodes)
	for i := 0; i < etcd.numNodes; i++ {
		urls[i] = "http://" + etcd.clientUrl(i)
	}
	return urls
}

func (etcd *ETCDClusterRunner) DiskUsage() (bytes int64, err error) {
	fi, err := os.Stat(etcd.tmpPathTo("log", 0))
	if err != nil {
		return 0, err
	}
	return fi.Size(), nil
}

func (etcd *ETCDClusterRunner) Reset() {
	etcd.mutex.RLock()
	running := etcd.running
	etcd.mutex.RUnlock()

	if running {
		response, err := etcd.client.Get("/", false, false)
		Ω(err).ShouldNot(HaveOccurred())
		for _, doomed := range response.Node.Nodes {
			etcd.client.Delete(doomed.Key, true)
		}
	}
}

func (etcd *ETCDClusterRunner) FastForwardTime(seconds int) {
	etcd.mutex.RLock()
	running := etcd.running
	etcd.mutex.RUnlock()

	if running {
		response, err := etcd.client.Get("/", false, true)
		Ω(err).ShouldNot(HaveOccurred())
		etcd.fastForwardTime(response.Node, seconds)
	}
}

func (etcd *ETCDClusterRunner) Adapter() storeadapter.StoreAdapter {
	pool := workerpool.NewWorkerPool(10)
	adapter := etcdstoreadapter.NewETCDStoreAdapter(etcd.NodeURLS(), pool)
	adapter.Connect()
	return adapter
}

func (etcd *ETCDClusterRunner) start(nuke bool) {
	etcd.mutex.RLock()
	running := etcd.running
	etcd.mutex.RUnlock()

	if running {
		return
	}

	etcd.mutex.Lock()

	etcd.etcdSessions = make([]*gexec.Session, etcd.numNodes)

	for i := 0; i < etcd.numNodes; i++ {
		if nuke {
			etcd.nukeArtifacts(i)
		}

		if etcd.detectRunningEtcd(i) {
			log.Fatalf("Detected an ETCD already running on %s", etcd.clientUrl(i))
		}

		os.MkdirAll(etcd.tmpPath(i), 0700)
		args := []string{"-data-dir", etcd.tmpPath(i), "-addr", etcd.clientUrl(i), "-peer-addr", etcd.serverUrl(i), "-name", etcd.nodeName(i)}
		if i != 0 {
			args = append(args, "-peers", etcd.serverUrl(0))
		}

		session, err := gexec.Start(
			exec.Command("etcd", args...),
			gexec.NewPrefixedWriter("\x1b[32m[o]\x1b[33m[etcd_cluster]\x1b[0m ", ginkgo.GinkgoWriter),
			gexec.NewPrefixedWriter("\x1b[91m[e]\x1b[33m[etcd_cluster]\x1b[0m ", ginkgo.GinkgoWriter),
		)
		Ω(err).ShouldNot(HaveOccurred(), "Make sure etcd is compiled and on your $PATH.")

		etcd.etcdSessions[i] = session

		Eventually(func() bool {
			return etcd.detectRunningEtcd(i)
		}, 3, 0.05).Should(BeTrue(), "Expected ETCD to be up and running")
	}

	etcd.client = etcdclient.NewClient(etcd.NodeURLS())
	etcd.running = true

	etcd.mutex.Unlock()
}

func (etcd *ETCDClusterRunner) stop(nuke bool) {
	etcd.mutex.Lock()
	defer etcd.mutex.Unlock()

	if etcd.running {
		for i := 0; i < etcd.numNodes; i++ {
			etcd.etcdSessions[i].Interrupt().Wait(5 * time.Second)
			if nuke {
				etcd.nukeArtifacts(i)
			}
		}
		etcd.markAsStopped()
	}
}

func (etcd *ETCDClusterRunner) kill() {
	etcd.mutex.Lock()
	defer etcd.mutex.Unlock()

	if etcd.running {
		for i := 0; i < etcd.numNodes; i++ {
			etcd.etcdSessions[i].Kill().Wait(5 * time.Second)
			etcd.nukeArtifacts(i)
		}
		etcd.markAsStopped()
	}
}

func (etcd *ETCDClusterRunner) markAsStopped() {
	etcd.etcdSessions = nil
	etcd.running = false
	etcd.client = nil
}

func (etcd *ETCDClusterRunner) detectRunningEtcd(index int) bool {
	client := etcdclient.NewClient([]string{})
	return client.SetCluster([]string{"http://" + etcd.clientUrl(index)})
}

func (etcd *ETCDClusterRunner) fastForwardTime(etcdNode *etcdclient.Node, seconds int) {
	if etcdNode.Dir == true {
		for _, child := range etcdNode.Nodes {
			etcd.fastForwardTime(child, seconds)
		}
	} else {
		if etcdNode.TTL == 0 {
			return
		}
		if etcdNode.TTL <= int64(seconds) {
			_, err := etcd.client.Delete(etcdNode.Key, true)
			Ω(err).ShouldNot(HaveOccurred())
		} else {
			_, err := etcd.client.Set(etcdNode.Key, etcdNode.Value, uint64(etcdNode.TTL-int64(seconds)))
			Ω(err).ShouldNot(HaveOccurred())
		}
	}
}

func (etcd *ETCDClusterRunner) clientUrl(index int) string {
	return fmt.Sprintf("127.0.0.1:%d", etcd.port(index))
}

func (etcd *ETCDClusterRunner) serverUrl(index int) string {
	return fmt.Sprintf("127.0.0.1:%d", etcd.port(index)+3000)
}

func (etcd *ETCDClusterRunner) nodeName(index int) string {
	return fmt.Sprintf("node%d", index)
}

func (etcd *ETCDClusterRunner) port(index int) int {
	return etcd.startingPort + index
}

func (etcd *ETCDClusterRunner) tmpPath(index int) string {
	return fmt.Sprintf("/tmp/ETCD_%d", etcd.port(index))
}

func (etcd *ETCDClusterRunner) tmpPathTo(subdir string, index int) string {
	return fmt.Sprintf("/%s/%s", etcd.tmpPath(index), subdir)
}

func (etcd *ETCDClusterRunner) nukeArtifacts(index int) {
	os.RemoveAll(etcd.tmpPath(index))
}
<commit_msg>storerunner: fix deadlock when etcd not in $PATH<commit_after>package etcdstorerunner

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"sync"
	"time"

	"github.com/cloudfoundry/storeadapter"
	"github.com/cloudfoundry/storeadapter/etcdstoreadapter"
	"github.com/cloudfoundry/storeadapter/workerpool"
	etcdclient "github.com/coreos/go-etcd/etcd"
	"github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

type ETCDClusterRunner struct {
	startingPort int
	numNodes     int
	etcdSessions []*gexec.Session
	running      bool
	client       *etcdclient.Client

	mutex *sync.RWMutex
}

func NewETCDClusterRunner(startingPort int, numNodes int) *ETCDClusterRunner {
	return &ETCDClusterRunner{
		startingPort: startingPort,
		numNodes:     numNodes,

		mutex: &sync.RWMutex{},
	}
}

func (etcd *ETCDClusterRunner) Start() {
	etcd.start(true)
}

func (etcd *ETCDClusterRunner) Stop() {
	etcd.stop(true)
}

func (etcd *ETCDClusterRunner) KillWithFire() {
	etcd.kill()
}

func (etcd *ETCDClusterRunner) GoAway() {
	etcd.stop(false)
}

func (etcd *ETCDClusterRunner) ComeBack() {
	etcd.start(false)
}

func (etcd *ETCDClusterRunner) NodeURLS() []string {
	urls := make([]string, etcd.numNodes)
	for i := 0; i < etcd.numNodes; i++ {
		urls[i] = "http://" + etcd.clientUrl(i)
	}
	return urls
}

func (etcd *ETCDClusterRunner) DiskUsage() (bytes int64, err error) {
	fi, err := os.Stat(etcd.tmpPathTo("log", 0))
	if err != nil {
		return 0, err
	}
	return fi.Size(), nil
}

func (etcd *ETCDClusterRunner) Reset() {
	etcd.mutex.RLock()
	running := etcd.running
	etcd.mutex.RUnlock()

	if running {
		response, err := etcd.client.Get("/", false, false)
		Ω(err).ShouldNot(HaveOccurred())
		for _, doomed := range response.Node.Nodes {
			etcd.client.Delete(doomed.Key, true)
		}
	}
}

func (etcd *ETCDClusterRunner) FastForwardTime(seconds int) {
	etcd.mutex.RLock()
	running := etcd.running
	etcd.mutex.RUnlock()

	if running {
		response, err := etcd.client.Get("/", false, true)
		Ω(err).ShouldNot(HaveOccurred())
		etcd.fastForwardTime(response.Node, seconds)
	}
}

func (etcd *ETCDClusterRunner) Adapter() storeadapter.StoreAdapter {
	pool := workerpool.NewWorkerPool(10)
	adapter := etcdstoreadapter.NewETCDStoreAdapter(etcd.NodeURLS(), pool)
	adapter.Connect()
	return adapter
}

func (etcd *ETCDClusterRunner) start(nuke bool) {
	etcd.mutex.RLock()
	running := etcd.running
	etcd.mutex.RUnlock()

	if running {
		return
	}

	etcd.mutex.Lock()
	defer etcd.mutex.Unlock()

	etcd.etcdSessions = make([]*gexec.Session, etcd.numNodes)

	for i := 0; i < etcd.numNodes; i++ {
		if nuke {
			etcd.nukeArtifacts(i)
		}

		if etcd.detectRunningEtcd(i) {
			log.Fatalf("Detected an ETCD already running on %s", etcd.clientUrl(i))
		}

		os.MkdirAll(etcd.tmpPath(i), 0700)
		args := []string{"-data-dir", etcd.tmpPath(i), "-addr", etcd.clientUrl(i), "-peer-addr", etcd.serverUrl(i), "-name", etcd.nodeName(i)}
		if i != 0 {
			args = append(args, "-peers", etcd.serverUrl(0))
		}

		session, err := gexec.Start(
			exec.Command("etcd", args...),
			gexec.NewPrefixedWriter("\x1b[32m[o]\x1b[33m[etcd_cluster]\x1b[0m ", ginkgo.GinkgoWriter),
			gexec.NewPrefixedWriter("\x1b[91m[e]\x1b[33m[etcd_cluster]\x1b[0m ", ginkgo.GinkgoWriter),
		)
		Ω(err).ShouldNot(HaveOccurred(), "Make sure etcd is compiled and on your $PATH.")

		etcd.etcdSessions[i] = session

		Eventually(func() bool {
			return etcd.detectRunningEtcd(i)
		}, 3, 0.05).Should(BeTrue(), "Expected ETCD to be up and running")
	}

	etcd.client = etcdclient.NewClient(etcd.NodeURLS())
	etcd.running = true
}

func (etcd *ETCDClusterRunner) stop(nuke bool) {
	etcd.mutex.Lock()
	defer etcd.mutex.Unlock()

	if etcd.running {
		for i := 0; i < etcd.numNodes; i++ {
			etcd.etcdSessions[i].Interrupt().Wait(5 * time.Second)
			if nuke {
				etcd.nukeArtifacts(i)
			}
		}
		etcd.markAsStopped()
	}
}

func (etcd *ETCDClusterRunner) kill() {
	etcd.mutex.Lock()
	defer etcd.mutex.Unlock()

	if etcd.running {
		for i := 0; i < etcd.numNodes; i++ {
			etcd.etcdSessions[i].Kill().Wait(5 * time.Second)
			etcd.nukeArtifacts(i)
		}
		etcd.markAsStopped()
	}
}

func (etcd *ETCDClusterRunner) markAsStopped() {
	etcd.etcdSessions = nil
	etcd.running = false
	etcd.client = nil
}

func (etcd *ETCDClusterRunner) detectRunningEtcd(index int) bool {
	client := etcdclient.NewClient([]string{})
	return client.SetCluster([]string{"http://" + etcd.clientUrl(index)})
}

func (etcd *ETCDClusterRunner) fastForwardTime(etcdNode *etcdclient.Node, seconds int) {
	if etcdNode.Dir == true {
		for _, child := range etcdNode.Nodes {
			etcd.fastForwardTime(child, seconds)
		}
	} else {
		if etcdNode.TTL == 0 {
			return
		}
		if etcdNode.TTL <= int64(seconds) {
			_, err := etcd.client.Delete(etcdNode.Key, true)
			Ω(err).ShouldNot(HaveOccurred())
		} else {
			_, err := etcd.client.Set(etcdNode.Key, etcdNode.Value, uint64(etcdNode.TTL-int64(seconds)))
			Ω(err).ShouldNot(HaveOccurred())
		}
	}
}

func (etcd *ETCDClusterRunner) clientUrl(index int) string {
	return fmt.Sprintf("127.0.0.1:%d", etcd.port(index))
}

func (etcd *ETCDClusterRunner) serverUrl(index int) string {
	return fmt.Sprintf("127.0.0.1:%d", etcd.port(index)+3000)
}

func (etcd *ETCDClusterRunner) nodeName(index int) string {
	return fmt.Sprintf("node%d", index)
}

func (etcd *ETCDClusterRunner) port(index int) int {
	return etcd.startingPort + index
}

func (etcd *ETCDClusterRunner) tmpPath(index int) string {
	return fmt.Sprintf("/tmp/ETCD_%d", etcd.port(index))
}

func (etcd *ETCDClusterRunner) tmpPathTo(subdir string, index int) string {
	return fmt.Sprintf("/%s/%s", etcd.tmpPath(index), subdir)
}

func (etcd *ETCDClusterRunner) nukeArtifacts(index int) {
	os.RemoveAll(etcd.tmpPath(index))
}
<|endoftext|>


<commit_before>package goasp

import (
	"testing"
)

func assertDecodedString(t *testing.T, encoded, expected string, token otype) {
	prs := newParserFromString(encoded)
	obj, err := decode_obj(&prs)

	if err != nil {
		t.Fatal(err)
	}

	if obj.token != token {
		gotTokenName, _ := otype_names[obj.token]
		t.Fatalf(
			"Expected %v (0x%02x), got %v (0x%02x)",
			otype_names[token],
			token,
			gotTokenName,
			obj.token,
		)
	}

	var val string
	var ok bool

	if val, ok = obj.value.(string); !ok {
		t.Fatalf("pobject(%v)'s value should be a castable to string.", otype_names[token])
	}

	if val != expected {
		t.Fatal("Expected \"" + expected + "\", got \"" + val + "\"")
	}
}

func TestDecode_str_nullterm(t *testing.T) {
	encoded := "\x0bA Test String\x00"
	decoded := "A Test String"
	assertDecodedString(t, encoded, decoded, tSTR_NULLTERM)
}

func TestDecode_length_string(t *testing.T) {

}
<commit_msg>test parse_length_string<commit_after>package goasp

import (
	"testing"
)

func assertDecodedString(t *testing.T, encoded, expected string, token otype) {
	prs := newParserFromString(encoded)
	obj, err := decode_obj(&prs)

	if err != nil {
		t.Fatal(err)
	}

	if obj.token != token {
		gotTokenName, _ := otype_names[obj.token]
		t.Fatalf(
			"Expected %v (0x%02x), got %v (0x%02x)",
			otype_names[token],
			token,
			gotTokenName,
			obj.token,
		)
	}

	var val string
	var ok bool

	if val, ok = obj.value.(string); !ok {
		t.Fatalf("pobject(%v)'s value should be a castable to string.", otype_names[token])
	}

	if val != expected {
		t.Fatal("Expected \"" + expected + "\", got \"" + val + "\"")
	}
}

func TestDecode_str_nullterm(t *testing.T) {
	encoded := "\x0bA Test String\x00"
	decoded := "A Test String"
	assertDecodedString(t, encoded, decoded, tSTR_NULLTERM)
}

func TestDecode_length_string(t *testing.T) {
	assertDecodedString(t, "\x05\x0CHello World!", "Hello World!", tSTRING)
	assertDecodedString(t, "\x1E\x0CHello World!", "Hello World!", tSTRING2)
	assertDecodedString(t, "\x05\x00", "", tSTRING)
	assertDecodedString(t, "\x05\x01\x07", "\x07", tSTRING)
}
<|endoftext|>


<commit_before>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
// vespa document command
// author: bratseth

package cmd

import (
	"fmt"
	"io"
	"io/ioutil"
	"strings"

	"github.com/spf13/cobra"
	"github.com/vespa-engine/vespa/client/go/util"
	"github.com/vespa-engine/vespa/client/go/vespa"
)

var printCurl bool

func init() {
	rootCmd.AddCommand(documentCmd)
	documentCmd.AddCommand(documentPutCmd)
	documentCmd.AddCommand(documentUpdateCmd)
	documentCmd.AddCommand(documentRemoveCmd)
	documentCmd.AddCommand(documentGetCmd)
	documentCmd.PersistentFlags().BoolVarP(&printCurl, "verbose", "v", false, "Print the equivalent curl command for the document operation")
}

var documentCmd = &cobra.Command{
	Use:   "document json-file",
	Short: "Issue a document operation to Vespa",
	Long: `Issue a document operation to Vespa.

The operation must be on the format documented in
https://docs.vespa.ai/en/reference/document-json-format.html#document-operations

When this returns successfully, the document is guaranteed to be visible in any
subsequent get or query operation.

To feed with high throughput, https://docs.vespa.ai/en/vespa-feed-client.html
should be used instead of this.`,
	Example:           `$ vespa document src/test/resources/A-Head-Full-of-Dreams.json`,
	DisableAutoGenTag: true,
	Args:              cobra.ExactArgs(1),
	Run: func(cmd *cobra.Command, args []string) {
		printResult(vespa.Send(args[0], documentService(), curlOutput()), false)
	},
}

var documentPutCmd = &cobra.Command{
	Use:   "put [id] json-file",
	Short: "Writes a document to Vespa",
	Long: `Writes the document in the given file to Vespa.
If the document already exists, all its values will be replaced by this document.
If the document id is specified both as an argument and in the file the argument takes precedence.`,
	Args: cobra.RangeArgs(1, 2),
	Example: `$ vespa document put src/test/resources/A-Head-Full-of-Dreams.json
$ vespa document put id:mynamespace:music::a-head-full-of-dreams src/test/resources/A-Head-Full-of-Dreams.json`,
	DisableAutoGenTag: true,
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) == 1 {
			printResult(vespa.Put("", args[0], documentService(), curlOutput()), false)
		} else {
			printResult(vespa.Put(args[0], args[1], documentService(), curlOutput()), false)
		}
	},
}

var documentUpdateCmd = &cobra.Command{
	Use:   "update [id] json-file",
	Short: "Modifies some fields of an existing document",
	Long: `Updates the values of the fields given in a json file as specified in the file.
If the document id is specified both as an argument and in the file the argument takes precedence.`,
	Args: cobra.RangeArgs(1, 2),
	Example: `$ vespa document update src/test/resources/A-Head-Full-of-Dreams-Update.json
$ vespa document update id:mynamespace:music::a-head-full-of-dreams src/test/resources/A-Head-Full-of-Dreams.json`,
	DisableAutoGenTag: true,
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) == 1 {
			printResult(vespa.Update("", args[0], documentService(), curlOutput()), false)
		} else {
			printResult(vespa.Update(args[0], args[1], documentService(), curlOutput()), false)
		}
	},
}

var documentRemoveCmd = &cobra.Command{
	Use:   "remove id | json-file",
	Short: "Removes a document from Vespa",
	Long: `Removes the document specified either as a document id or given in the json file.
If the document id is specified both as an argument and in the file the argument takes precedence.`,
	Args: cobra.ExactArgs(1),
	Example: `$ vespa document remove src/test/resources/A-Head-Full-of-Dreams-Remove.json
$ vespa document remove id:mynamespace:music::a-head-full-of-dreams`,
	DisableAutoGenTag: true,
	Run: func(cmd *cobra.Command, args []string) {
		if strings.HasPrefix(args[0], "id:") {
			printResult(vespa.RemoveId(args[0], documentService(), curlOutput()), false)
		} else {
			printResult(vespa.RemoveOperation(args[0], documentService(), curlOutput()), false)
		}
	},
}

var documentGetCmd = &cobra.Command{
	Use:               "get id",
	Short:             "Gets a document",
	Args:              cobra.ExactArgs(1),
	DisableAutoGenTag: true,
	Example:           `$ vespa document get id:mynamespace:music::a-head-full-of-dreams`,
	Run: func(cmd *cobra.Command, args []string) {
		printResult(vespa.Get(args[0], documentService(), curlOutput()), true)
	},
}

func documentService() *vespa.Service { return getService("document", 0) }

func curlOutput() io.Writer {
	if printCurl {
		return stderr
	}
	return ioutil.Discard
}

func printResult(result util.OperationResult, payloadOnlyOnSuccess bool) {
	out := stdout
	if !result.Success {
		out = stderr
	}

	if !result.Success {
		fmt.Fprintln(out, color.Red("Error:"), result.Message)
	} else if !(payloadOnlyOnSuccess && result.Payload != "") {
		fmt.Fprintln(out, color.Green("Success:"), result.Message)
	}

	if result.Detail != "" {
		fmt.Fprintln(out, color.Yellow(result.Detail))
	}

	if result.Payload != "" {
		if !payloadOnlyOnSuccess {
			fmt.Fprintln(out)
		}
		fmt.Fprintln(out, result.Payload)
	}
}
<commit_msg>Exit with non-zero on error<commit_after>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
// vespa document command
// author: bratseth

package cmd

import (
	"fmt"
	"io"
	"io/ioutil"
	"strings"

	"github.com/spf13/cobra"
	"github.com/vespa-engine/vespa/client/go/util"
	"github.com/vespa-engine/vespa/client/go/vespa"
)

var printCurl bool

func init() {
	rootCmd.AddCommand(documentCmd)
	documentCmd.AddCommand(documentPutCmd)
	documentCmd.AddCommand(documentUpdateCmd)
	documentCmd.AddCommand(documentRemoveCmd)
	documentCmd.AddCommand(documentGetCmd)
	documentCmd.PersistentFlags().BoolVarP(&printCurl, "verbose", "v", false, "Print the equivalent curl command for the document operation")
}

var documentCmd = &cobra.Command{
	Use:   "document json-file",
	Short: "Issue a document operation to Vespa",
	Long: `Issue a document operation to Vespa.

The operation must be on the format documented in
https://docs.vespa.ai/en/reference/document-json-format.html#document-operations

When this returns successfully, the document is guaranteed to be visible in any
subsequent get or query operation.

To feed with high throughput, https://docs.vespa.ai/en/vespa-feed-client.html
should be used instead of this.`,
	Example:           `$ vespa document src/test/resources/A-Head-Full-of-Dreams.json`,
	DisableAutoGenTag: true,
	Args:              cobra.ExactArgs(1),
	Run: func(cmd *cobra.Command, args []string) {
		printResult(vespa.Send(args[0], documentService(), curlOutput()), false)
	},
}

var documentPutCmd = &cobra.Command{
	Use:   "put [id] json-file",
	Short: "Writes a document to Vespa",
	Long: `Writes the document in the given file to Vespa.
If the document already exists, all its values will be replaced by this document.
If the document id is specified both as an argument and in the file the argument takes precedence.`,
	Args: cobra.RangeArgs(1, 2),
	Example: `$ vespa document put src/test/resources/A-Head-Full-of-Dreams.json
$ vespa document put id:mynamespace:music::a-head-full-of-dreams src/test/resources/A-Head-Full-of-Dreams.json`,
	DisableAutoGenTag: true,
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) == 1 {
			printResult(vespa.Put("", args[0], documentService(), curlOutput()), false)
		} else {
			printResult(vespa.Put(args[0], args[1], documentService(), curlOutput()), false)
		}
	},
}

var documentUpdateCmd = &cobra.Command{
	Use:   "update [id] json-file",
	Short: "Modifies some fields of an existing document",
	Long: `Updates the values of the fields given in a json file as specified in the file.
If the document id is specified both as an argument and in the file the argument takes precedence.`,
	Args: cobra.RangeArgs(1, 2),
	Example: `$ vespa document update src/test/resources/A-Head-Full-of-Dreams-Update.json
$ vespa document update id:mynamespace:music::a-head-full-of-dreams src/test/resources/A-Head-Full-of-Dreams.json`,
	DisableAutoGenTag: true,
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) == 1 {
			printResult(vespa.Update("", args[0], documentService(), curlOutput()), false)
		} else {
			printResult(vespa.Update(args[0], args[1], documentService(), curlOutput()), false)
		}
	},
}

var documentRemoveCmd = &cobra.Command{
	Use:   "remove id | json-file",
	Short: "Removes a document from Vespa",
	Long: `Removes the document specified either as a document id or given in the json file.
If the document id is specified both as an argument and in the file the argument takes precedence.`,
	Args: cobra.ExactArgs(1),
	Example: `$ vespa document remove src/test/resources/A-Head-Full-of-Dreams-Remove.json
$ vespa document remove id:mynamespace:music::a-head-full-of-dreams`,
	DisableAutoGenTag: true,
	Run: func(cmd *cobra.Command, args []string) {
		if strings.HasPrefix(args[0], "id:") {
			printResult(vespa.RemoveId(args[0], documentService(), curlOutput()), false)
		} else {
			printResult(vespa.RemoveOperation(args[0], documentService(), curlOutput()), false)
		}
	},
}

var documentGetCmd = &cobra.Command{
	Use:               "get id",
	Short:             "Gets a document",
	Args:              cobra.ExactArgs(1),
	DisableAutoGenTag: true,
	Example:           `$ vespa document get id:mynamespace:music::a-head-full-of-dreams`,
	Run: func(cmd *cobra.Command, args []string) {
		printResult(vespa.Get(args[0], documentService(), curlOutput()), true)
	},
}

func documentService() *vespa.Service { return getService("document", 0) }

func curlOutput() io.Writer {
	if printCurl {
		return stderr
	}
	return ioutil.Discard
}

func printResult(result util.OperationResult, payloadOnlyOnSuccess bool) {
	out := stdout
	if !result.Success {
		out = stderr
	}

	if !result.Success {
		fmt.Fprintln(out, color.Red("Error:"), result.Message)
	} else if !(payloadOnlyOnSuccess && result.Payload != "") {
		fmt.Fprintln(out, color.Green("Success:"), result.Message)
	}

	if result.Detail != "" {
		fmt.Fprintln(out, color.Yellow(result.Detail))
	}

	if result.Payload != "" {
		if !payloadOnlyOnSuccess {
			fmt.Fprintln(out)
		}
		fmt.Fprintln(out, result.Payload)
	}

	if !result.Success {
		exitFunc(1)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package isolate

import (
	"bytes"
	"encoding/json"
	"errors"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"go.chromium.org/luci/client/archiver"
	"go.chromium.org/luci/common/flag/stringlistflag"
	"go.chromium.org/luci/common/flag/stringmapflag"
	"go.chromium.org/luci/common/isolated"
	"go.chromium.org/luci/common/isolatedclient"
	"go.chromium.org/luci/common/runtime/tracer"
)

// IsolatedGenJSONVersion is used in the batcharchive json format.
//
// TODO(tandrii): Migrate to batch_archive.go.
const IsolatedGenJSONVersion = 1

// ValidVariable is the regexp of valid isolate variable name.
const ValidVariable = "[A-Za-z_][A-Za-z_0-9]*"

var validVariableMatcher = regexp.MustCompile(ValidVariable)
var variableSubstitutionMatcher = regexp.MustCompile("<\\(" + ValidVariable + "\\)")

// IsValidVariable returns true if the variable is a valid symbol name.
func IsValidVariable(variable string) bool {
	return validVariableMatcher.MatchString(variable)
}

// Tree to be isolated.
type Tree struct {
	Cwd  string
	Opts ArchiveOptions
}

// ArchiveOptions for achiving trees.
type ArchiveOptions struct {
	Isolate         string              `json:"isolate"`
	Isolated        string              `json:"isolated"`
	Blacklist       stringlistflag.Flag `json:"blacklist"`
	PathVariables   stringmapflag.Value `json:"path_variables"`
	ExtraVariables  stringmapflag.Value `json:"extra_variables"`
	ConfigVariables stringmapflag.Value `json:"config_variables"`
}

// Init initializes with non-nil values.
func (a *ArchiveOptions) Init() {
	a.Blacklist = stringlistflag.Flag{}
	a.PathVariables = map[string]string{}
	if IsWindows() {
		a.PathVariables["EXECUTABLE_SUFFIX"] = ".exe"
	} else {
		a.PathVariables["EXECUTABLE_SUFFIX"] = ""
	}
	a.ExtraVariables = map[string]string{}
	a.ConfigVariables = map[string]string{}
}

// PostProcess post-processes the flags to fix any compatibility issue.
func (a *ArchiveOptions) PostProcess(cwd string) {
	// Set default blacklist only if none is set.
	if len(a.Blacklist) == 0 {
		// This cannot be generalized as ".*" as there is known use that require
		// a ".pki" directory to be mapped.
		a.Blacklist = stringlistflag.Flag{
			// Temporary python files.
			"*.pyc",
			// Temporary vim files.
			"*.swp",
			".git",
			".hg",
			".svn",
		}
	}
	if !filepath.IsAbs(a.Isolate) {
		a.Isolate = filepath.Join(cwd, a.Isolate)
	}
	a.Isolate = filepath.Clean(a.Isolate)

	if !filepath.IsAbs(a.Isolated) {
		a.Isolated = filepath.Join(cwd, a.Isolated)
	}
	a.Isolated = filepath.Clean(a.Isolated)

	for k, v := range a.PathVariables {
		// This is due to a Windows + GYP specific issue, where double-quoted paths
		// would get mangled in a way that cannot be resolved unless a space is
		// injected.
		a.PathVariables[k] = strings.TrimSpace(v)
	}
}

// ReplaceVariables replaces any occurrences of '<(FOO)' in 'str' with the
// corresponding variable from 'opts'.
//
// If any substitution refers to a variable that is missing, the returned error will
// refer to the first such variable. In the case of errors, the returned string will
// still contain a valid result for any non-missing substitutions.
func ReplaceVariables(str string, opts *ArchiveOptions) (string, error) {
	var err error
	subst := variableSubstitutionMatcher.ReplaceAllStringFunc(str,
		func(match string) string {
			varName := match[2 : len(match)-1]
			if v, ok := opts.PathVariables[varName]; ok {
				return v
			}
			if v, ok := opts.ExtraVariables[varName]; ok {
				return v
			}
			if v, ok := opts.ConfigVariables[varName]; ok {
				return v
			}
			if err == nil {
				err = errors.New("no value for variable '" + varName + "'")
			}
			return match
		})
	return subst, err
}

// Archive processes a .isolate, generates a .isolated and archive it.
// Returns a *PendingItem to the .isolated.
func Archive(arch *archiver.Archiver, opts *ArchiveOptions) *archiver.PendingItem {
	displayName := filepath.Base(opts.Isolated)
	defer tracer.Span(arch, strings.SplitN(displayName, ".", 2)[0]+":archive", nil)(nil)
	f, err := archive(arch, opts, displayName)
	if err != nil {
		i := &archiver.PendingItem{DisplayName: displayName}
		i.SetErr(err)
		return i
	}
	return f
}

// ProcessIsolate parses an isolate file, returning the list of dependencies
// (both files and directories), the root directory and the initial Isolated struct.
func ProcessIsolate(opts *ArchiveOptions) ([]string, string, *isolated.Isolated, error) {
	content, err := ioutil.ReadFile(opts.Isolate)
	if err != nil {
		return nil, "", nil, err
	}
	cmd, deps, readOnly, isolateDir, err := LoadIsolateForConfig(filepath.Dir(opts.Isolate), content, opts.ConfigVariables)
	if err != nil {
		return nil, "", nil, err
	}

	// Expand variables in the commands.
	for i := range cmd {
		if cmd[i], err = ReplaceVariables(cmd[i], opts); err != nil {
			return nil, "", nil, err
		}
	}

	// Expand variables in the deps, and convert each path to a cleaned absolute form.
	for i := range deps {
		dep, err := ReplaceVariables(deps[i], opts)
		if err != nil {
			return nil, "", nil, err
		}
		deps[i] = join(isolateDir, dep)
	}

	// Find the root directory of all the files (the root might be above isolateDir).
	rootDir := isolateDir
	for _, dep := range deps {
		// Check if the dep is outside isolateDir.
		base := filepath.Dir(dep)
		for {
			rel, err := filepath.Rel(rootDir, base)
			if err != nil {
				return nil, "", nil, err
			}
			if !strings.HasPrefix(rel, "..") {
				break
			}
			newRootDir := filepath.Dir(rootDir)
			if newRootDir == rootDir {
				return nil, "", nil, errors.New("failed to find root dir")
			}
			rootDir = newRootDir
		}
	}
	if rootDir != isolateDir {
		log.Printf("Root: %s", rootDir)
	}

	// Prepare the .isolated struct.
	isol := &isolated.Isolated{
		Algo:     "sha-1",
		Files:    map[string]isolated.File{},
		ReadOnly: readOnly.ToIsolated(),
		Version:  isolated.IsolatedFormatVersion,
	}
	if len(cmd) != 0 {
		isol.Command = cmd
		// Only set RelativeCwd if a command was also specified. This reduce the
		// noise for Swarming tasks where the command is specified as part of the
		// Swarming task request and not thru the isolated file.
		if rootDir != isolateDir {
			relPath, err := filepath.Rel(rootDir, isolateDir)
			if err != nil {
				return nil, "", nil, err
			}
			isol.RelativeCwd = relPath
		}
	}
	return deps, rootDir, isol, nil
}

func processing(opts *ArchiveOptions) (filesCount, dirsCount int, deps []string, rootDir string, isol *isolated.Isolated, err error) {
	deps, rootDir, isol, err = ProcessIsolate(opts)
	if err != nil {
		return 0, 0, nil, "", nil, err
	}

	for i := range deps {
		if deps[i][len(deps[i])-1] == os.PathSeparator {
			dirsCount++
		} else {
			filesCount++
		}
	}

	// Processing of the .isolate file ended.
	return filesCount, dirsCount, deps, rootDir, isol, nil
}

// join joins the provided pair of path elements. Unlike filepath.Join, join
// will preserve the trailing slash of the second element, if any.
// TODO(djd): delete this function. Preserving the slash is fragile, and we
// can tell if a path is a dir by stat'ing it (which we need to do anyway).
func join(p1, p2 string) string {
	joined := filepath.Join(p1, p2)
	if p2[len(p2)-1] == os.PathSeparator {
		joined += osPathSeparator // Retain trailing slash.
	}
	return joined
}

func archive(arch *archiver.Archiver, opts *ArchiveOptions, displayName string) (*archiver.PendingItem, error) {
	end := tracer.Span(arch, strings.SplitN(displayName, ".", 2)[0]+":loading", nil)
	filesCount, dirsCount, deps, rootDir, i, err := processing(opts)
	end(tracer.Args{"err": err})
	if err != nil {
		return nil, err
	}
	// Handle each dependency, either a file or a directory..
	fileItems := make([]*archiver.PendingItem, 0, filesCount)
	dirItems := make([]*archiver.PendingItem, 0, dirsCount)
	for _, dep := range deps {
		relPath, err := filepath.Rel(rootDir, dep)
		if err != nil {
			return nil, err
		}
		if dep[len(dep)-1] == os.PathSeparator {
			relPath, err := filepath.Rel(rootDir, dep)
			if err != nil {
				return nil, err
			}
			dirItems = append(dirItems, archiver.PushDirectory(arch, dep, relPath, opts.Blacklist))
		} else {
			// Grab the stats right away.
			info, err := os.Lstat(dep)
			if err != nil {
				return nil, err
			}
			mode := info.Mode()
			if mode&os.ModeSymlink == os.ModeSymlink {
				l, err := os.Readlink(dep)
				if err != nil {
					// Kill the process: there's no reason to continue if a file is
					// unavailable.
					log.Fatalf("Unable to stat %q: %v", dep, err)
				}
				i.Files[relPath] = isolated.SymLink(l)
			} else {
				i.Files[relPath] = isolated.BasicFile("", int(mode.Perm()), info.Size())
				fileItems = append(fileItems, arch.PushFile(relPath, dep, -info.Size()))
			}
		}
	}

	for _, item := range fileItems {
		item.WaitForHashed()
		if err = item.Error(); err != nil {
			return nil, err
		}
		f := i.Files[item.DisplayName]
		f.Digest = item.Digest()
		i.Files[item.DisplayName] = f
	}
	// Avoid duplicated entries in includes.
	// TODO(tandrii): add test to reproduce the problem.
	includesSet := map[isolated.HexDigest]bool{}
	for _, item := range dirItems {
		item.WaitForHashed()
		if err = item.Error(); err != nil {
			return nil, err
		}
		includesSet[item.Digest()] = true
	}
	for digest := range includesSet {
		i.Includes = append(i.Includes, digest)
	}
	// Make the includes list deterministic.
	sort.Sort(i.Includes)

	raw := &bytes.Buffer{}
	if err = json.NewEncoder(raw).Encode(i); err != nil {
		return nil, err
	}

	if err := ioutil.WriteFile(opts.Isolated, raw.Bytes(), 0644); err != nil {
		return nil, err
	}
	return arch.Push(displayName, isolatedclient.NewBytesSource(raw.Bytes()), 0), nil
}
<commit_msg>[client] Use stat to check whether deps are directories<commit_after>// Copyright 2015 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package isolate

import (
	"bytes"
	"encoding/json"
	"errors"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"go.chromium.org/luci/client/archiver"
	"go.chromium.org/luci/common/flag/stringlistflag"
	"go.chromium.org/luci/common/flag/stringmapflag"
	"go.chromium.org/luci/common/isolated"
	"go.chromium.org/luci/common/isolatedclient"
	"go.chromium.org/luci/common/runtime/tracer"
)

// IsolatedGenJSONVersion is used in the batcharchive json format.
//
// TODO(tandrii): Migrate to batch_archive.go.
const IsolatedGenJSONVersion = 1

// ValidVariable is the regexp of valid isolate variable name.
const ValidVariable = "[A-Za-z_][A-Za-z_0-9]*"

var validVariableMatcher = regexp.MustCompile(ValidVariable)
var variableSubstitutionMatcher = regexp.MustCompile("<\\(" + ValidVariable + "\\)")

// IsValidVariable returns true if the variable is a valid symbol name.
func IsValidVariable(variable string) bool {
	return validVariableMatcher.MatchString(variable)
}

// Tree to be isolated.
type Tree struct {
	Cwd  string
	Opts ArchiveOptions
}

// ArchiveOptions for achiving trees.
type ArchiveOptions struct {
	Isolate         string              `json:"isolate"`
	Isolated        string              `json:"isolated"`
	Blacklist       stringlistflag.Flag `json:"blacklist"`
	PathVariables   stringmapflag.Value `json:"path_variables"`
	ExtraVariables  stringmapflag.Value `json:"extra_variables"`
	ConfigVariables stringmapflag.Value `json:"config_variables"`
}

// Init initializes with non-nil values.
func (a *ArchiveOptions) Init() {
	a.Blacklist = stringlistflag.Flag{}
	a.PathVariables = map[string]string{}
	if IsWindows() {
		a.PathVariables["EXECUTABLE_SUFFIX"] = ".exe"
	} else {
		a.PathVariables["EXECUTABLE_SUFFIX"] = ""
	}
	a.ExtraVariables = map[string]string{}
	a.ConfigVariables = map[string]string{}
}

// PostProcess post-processes the flags to fix any compatibility issue.
func (a *ArchiveOptions) PostProcess(cwd string) {
	// Set default blacklist only if none is set.
	if len(a.Blacklist) == 0 {
		// This cannot be generalized as ".*" as there is known use that require
		// a ".pki" directory to be mapped.
		a.Blacklist = stringlistflag.Flag{
			// Temporary python files.
			"*.pyc",
			// Temporary vim files.
			"*.swp",
			".git",
			".hg",
			".svn",
		}
	}
	if !filepath.IsAbs(a.Isolate) {
		a.Isolate = filepath.Join(cwd, a.Isolate)
	}
	a.Isolate = filepath.Clean(a.Isolate)

	if !filepath.IsAbs(a.Isolated) {
		a.Isolated = filepath.Join(cwd, a.Isolated)
	}
	a.Isolated = filepath.Clean(a.Isolated)

	for k, v := range a.PathVariables {
		// This is due to a Windows + GYP specific issue, where double-quoted paths
		// would get mangled in a way that cannot be resolved unless a space is
		// injected.
		a.PathVariables[k] = strings.TrimSpace(v)
	}
}

// ReplaceVariables replaces any occurrences of '<(FOO)' in 'str' with the
// corresponding variable from 'opts'.
//
// If any substitution refers to a variable that is missing, the returned error will
// refer to the first such variable. In the case of errors, the returned string will
// still contain a valid result for any non-missing substitutions.
func ReplaceVariables(str string, opts *ArchiveOptions) (string, error) {
	var err error
	subst := variableSubstitutionMatcher.ReplaceAllStringFunc(str,
		func(match string) string {
			varName := match[2 : len(match)-1]
			if v, ok := opts.PathVariables[varName]; ok {
				return v
			}
			if v, ok := opts.ExtraVariables[varName]; ok {
				return v
			}
			if v, ok := opts.ConfigVariables[varName]; ok {
				return v
			}
			if err == nil {
				err = errors.New("no value for variable '" + varName + "'")
			}
			return match
		})
	return subst, err
}

// Archive processes a .isolate, generates a .isolated and archive it.
// Returns a *PendingItem to the .isolated.
func Archive(arch *archiver.Archiver, opts *ArchiveOptions) *archiver.PendingItem {
	displayName := filepath.Base(opts.Isolated)
	defer tracer.Span(arch, strings.SplitN(displayName, ".", 2)[0]+":archive", nil)(nil)
	f, err := archive(arch, opts, displayName)
	if err != nil {
		i := &archiver.PendingItem{DisplayName: displayName}
		i.SetErr(err)
		return i
	}
	return f
}

// ProcessIsolate parses an isolate file, returning the list of dependencies
// (both files and directories), the root directory and the initial Isolated struct.
func ProcessIsolate(opts *ArchiveOptions) ([]string, string, *isolated.Isolated, error) {
	content, err := ioutil.ReadFile(opts.Isolate)
	if err != nil {
		return nil, "", nil, err
	}
	cmd, deps, readOnly, isolateDir, err := LoadIsolateForConfig(filepath.Dir(opts.Isolate), content, opts.ConfigVariables)
	if err != nil {
		return nil, "", nil, err
	}

	// Expand variables in the commands.
	for i := range cmd {
		if cmd[i], err = ReplaceVariables(cmd[i], opts); err != nil {
			return nil, "", nil, err
		}
	}

	// Expand variables in the deps, and convert each path to an absolute form.
	for i := range deps {
		dep, err := ReplaceVariables(deps[i], opts)
		if err != nil {
			return nil, "", nil, err
		}
		deps[i] = filepath.Join(isolateDir, dep)
	}

	// Find the root directory of all the files (the root might be above isolateDir).
	rootDir := isolateDir
	for _, dep := range deps {
		// Check if the dep is outside isolateDir.
		base := filepath.Dir(dep)
		for {
			rel, err := filepath.Rel(rootDir, base)
			if err != nil {
				return nil, "", nil, err
			}
			if !strings.HasPrefix(rel, "..") {
				break
			}
			newRootDir := filepath.Dir(rootDir)
			if newRootDir == rootDir {
				return nil, "", nil, errors.New("failed to find root dir")
			}
			rootDir = newRootDir
		}
	}
	if rootDir != isolateDir {
		log.Printf("Root: %s", rootDir)
	}

	// Prepare the .isolated struct.
	isol := &isolated.Isolated{
		Algo:     "sha-1",
		Files:    map[string]isolated.File{},
		ReadOnly: readOnly.ToIsolated(),
		Version:  isolated.IsolatedFormatVersion,
	}
	if len(cmd) != 0 {
		isol.Command = cmd
		// Only set RelativeCwd if a command was also specified. This reduce the
		// noise for Swarming tasks where the command is specified as part of the
		// Swarming task request and not thru the isolated file.
		if rootDir != isolateDir {
			relPath, err := filepath.Rel(rootDir, isolateDir)
			if err != nil {
				return nil, "", nil, err
			}
			isol.RelativeCwd = relPath
		}
	}
	return deps, rootDir, isol, nil
}

func archive(arch *archiver.Archiver, opts *ArchiveOptions, displayName string) (*archiver.PendingItem, error) {
	end := tracer.Span(arch, strings.SplitN(displayName, ".", 2)[0]+":loading", nil)
	deps, rootDir, i, err := ProcessIsolate(opts)
	end(tracer.Args{"err": err})
	if err != nil {
		return nil, err
	}
	// Handle each dependency, either a file or a directory.
	var fileItems []*archiver.PendingItem
	var dirItems []*archiver.PendingItem
	for _, dep := range deps {
		relPath, err := filepath.Rel(rootDir, dep)
		if err != nil {
			return nil, err
		}
		// Grab the stats right away; this can be used for both checking whether
		// it's a directory and checking whether it's a link.
		info, err := os.Lstat(dep)
		if err != nil {
			return nil, err
		}
		mode := info.Mode()
		if mode.IsDir() {
			relPath, err := filepath.Rel(rootDir, dep)
			if err != nil {
				return nil, err
			}
			dirItems = append(dirItems, archiver.PushDirectory(arch, dep, relPath, opts.Blacklist))
		} else {
			if mode&os.ModeSymlink == os.ModeSymlink {
				l, err := os.Readlink(dep)
				if err != nil {
					// Kill the process: there's no reason to continue if a file is
					// unavailable.
					log.Fatalf("Unable to stat %q: %v", dep, err)
				}
				i.Files[relPath] = isolated.SymLink(l)
			} else {
				i.Files[relPath] = isolated.BasicFile("", int(mode.Perm()), info.Size())
				fileItems = append(fileItems, arch.PushFile(relPath, dep, -info.Size()))
			}
		}
	}

	for _, item := range fileItems {
		item.WaitForHashed()
		if err = item.Error(); err != nil {
			return nil, err
		}
		f := i.Files[item.DisplayName]
		f.Digest = item.Digest()
		i.Files[item.DisplayName] = f
	}
	// Avoid duplicated entries in includes.
	// TODO(tandrii): add test to reproduce the problem.
	includesSet := map[isolated.HexDigest]bool{}
	for _, item := range dirItems {
		item.WaitForHashed()
		if err = item.Error(); err != nil {
			return nil, err
		}
		includesSet[item.Digest()] = true
	}
	for digest := range includesSet {
		i.Includes = append(i.Includes, digest)
	}
	// Make the includes list deterministic.
	sort.Sort(i.Includes)

	raw := &bytes.Buffer{}
	if err = json.NewEncoder(raw).Encode(i); err != nil {
		return nil, err
	}

	if err := ioutil.WriteFile(opts.Isolated, raw.Bytes(), 0644); err != nil {
		return nil, err
	}
	return arch.Push(displayName, isolatedclient.NewBytesSource(raw.Bytes()), 0), nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lease

import (
	"fmt"
	"io"

	"golang.org/x/net/context"
)

// A type used by read proxies to refresh their contents. See notes on
// NewReadProxy.
type Refresher interface {
	// Return the size of the underlying contents.
	Size() (size int64)

	// Return a read-closer for the contents. The same contents will always be
	// returned, and they will always be of length Size().
	Refresh(ctx context.Context) (rc io.ReadCloser, err error)
}

// A wrapper around a read lease, exposing a similar interface with the
// following differences:
//
//  *  Contents are fetched and re-fetched automatically when needed. Therefore
//     the user need not worry about lease expiration.
//
//  *  Methods that may involve fetching the contents (reading, seeking) accept
//     context arguments, so as to be cancellable.
//
//  *  Only random access reading is supported.
//
// External synchronization is required.
type ReadProxy interface {
	// Return the size of the proxied content. Guarantees to not block.
	Size() (size int64)

	// Semantics matching io.ReaderAt, except with context support.
	ReadAt(ctx context.Context, p []byte, off int64) (n int, err error)

	// Return a read/write lease for the proxied contents, destroying the read
	// proxy. The read proxy must not be used after calling this method.
	Upgrade(ctx context.Context) (rwl ReadWriteLease, err error)

	// Destroy any resources in use by the read proxy. It must not be used
	// further.
	Destroy()
}

// Create a read proxy.
//
// The supplied refresher will be used to obtain the proxy's contents whenever
// the file leaser decides to expire the temporary copy thus obtained.
//
// If rl is non-nil, it will be used as the first temporary copy of the
// contents, and must match what the refresher returns.
func NewReadProxy(
	fl FileLeaser,
	r Refresher,
	rl ReadLease) (rp ReadProxy) {
	rp = &readProxy{
		size:      r.Size(),
		leaser:    fl,
		refresher: r,
		lease:     rl,
	}

	return
}

// A wrapper around a read lease, exposing a similar interface with the
// following differences:
//
//  *  Contents are fetched and re-fetched automatically when needed. Therefore
//     the user need not worry about lease expiration.
//
//  *  Methods that may involve fetching the contents (reading, seeking) accept
//     context arguments, so as to be cancellable.
//
// External synchronization is required.
type readProxy struct {
	/////////////////////////
	// Constant data
	/////////////////////////

	size int64

	/////////////////////////
	// Dependencies
	/////////////////////////

	leaser    FileLeaser
	refresher Refresher

	/////////////////////////
	// Mutable state
	/////////////////////////

	// The current wrapped lease, or nil if one has never been issued.
	lease ReadLease
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func isRevokedErr(err error) bool {
	_, ok := err.(*RevokedError)
	return ok
}

// Set up a read/write lease and fill in our contents.
//
// REQUIRES: The caller has observed that rp.lease has expired.
func (rp *readProxy) getContents(
	ctx context.Context) (rwl ReadWriteLease, err error) {
	// Obtain some space to write the contents.
	rwl, err = rp.leaser.NewFile()
	if err != nil {
		err = fmt.Errorf("NewFile: %v", err)
		return
	}

	// Clean up if we exit early.
	defer func() {
		if err != nil {
			rwl.Downgrade().Revoke()
		}
	}()

	// Obtain the reader for our contents.
	rc, err := rp.refresher.Refresh(ctx)
	if err != nil {
		err = fmt.Errorf("User function: %v", err)
		return
	}

	defer func() {
		closeErr := rc.Close()
		if closeErr != nil && err == nil {
			err = fmt.Errorf("Close: %v", closeErr)
		}
	}()

	// Copy into the read/write lease.
	copied, err := io.Copy(rwl, rc)
	if err != nil {
		err = fmt.Errorf("Copy: %v", err)
		return
	}

	// Did the user lie about the size?
	if copied != rp.Size() {
		err = fmt.Errorf("Copied %v bytes; expected %v", copied, rp.Size())
		return
	}

	return
}

// Downgrade and save the supplied read/write lease obtained with getContents
// for later use.
func (rp *readProxy) saveContents(rwl ReadWriteLease) {
	rp.lease = rwl.Downgrade()
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// Semantics matching io.Reader, except with context support.
func (rp *readProxy) Read(
	ctx context.Context,
	p []byte) (n int, err error) {
	// Common case: is the existing lease still valid?
	if rp.lease != nil {
		n, err = rp.lease.Read(p)
		if !isRevokedErr(err) {
			return
		}

		// Clear the revoked error.
		err = nil
	}

	// Get hold of a read/write lease containing our contents.
	rwl, err := rp.getContents(ctx)
	if err != nil {
		err = fmt.Errorf("getContents: %v", err)
		return
	}

	defer rp.saveContents(rwl)

	// Serve from the read/write lease.
	n, err = rwl.Read(p)

	return
}

// Semantics matching io.Seeker, except with context support.
func (rp *readProxy) Seek(
	ctx context.Context,
	offset int64,
	whence int) (off int64, err error) {
	// Common case: is the existing lease still valid?
	if rp.lease != nil {
		off, err = rp.lease.Seek(offset, whence)
		if !isRevokedErr(err) {
			return
		}

		// Clear the revoked error.
		err = nil
	}

	// Get hold of a read/write lease containing our contents.
	rwl, err := rp.getContents(ctx)
	if err != nil {
		err = fmt.Errorf("getContents: %v", err)
		return
	}

	defer rp.saveContents(rwl)

	// Serve from the read/write lease.
	off, err = rwl.Seek(offset, whence)

	return
}

// Semantics matching io.ReaderAt, except with context support.
func (rp *readProxy) ReadAt(
	ctx context.Context,
	p []byte,
	off int64) (n int, err error) {
	// Common case: is the existing lease still valid?
	if rp.lease != nil {
		n, err = rp.lease.ReadAt(p, off)
		if !isRevokedErr(err) {
			return
		}

		// Clear the revoked error.
		err = nil
	}

	// Get hold of a read/write lease containing our contents.
	rwl, err := rp.getContents(ctx)
	if err != nil {
		err = fmt.Errorf("getContents: %v", err)
		return
	}

	defer rp.saveContents(rwl)

	// Serve from the read/write lease.
	n, err = rwl.ReadAt(p, off)

	return
}

// Return the size of the proxied content. Guarantees to not block.
func (rp *readProxy) Size() (size int64) {
	size = rp.size
	return
}

// Return a read/write lease for the proxied contents, destroying the read
// proxy. The read proxy must not be used after calling this method.
func (rp *readProxy) Upgrade(
	ctx context.Context) (rwl ReadWriteLease, err error) {
	// If we succeed, we are now destroyed.
	defer func() {
		if err == nil {
			rp.Destroy()
		}
	}()

	// Common case: is the existing lease still valid?
	if rp.lease != nil {
		rwl, err = rp.lease.Upgrade()
		if !isRevokedErr(err) {
			return
		}

		// Clear the revoked error.
		err = nil
	}

	// Build the read/write lease anew.
	rwl, err = rp.getContents(ctx)
	if err != nil {
		err = fmt.Errorf("getContents: %v", err)
		return
	}

	return
}

// Destroy any resources in use by the read proxy. It must not be used further.
func (rp *readProxy) Destroy() {
	if rp.lease != nil {
		rp.lease.Revoke()
	}

	// Make use-after-destroy errors obvious.
	rp.size = 0
	rp.leaser = nil
	rp.refresher = nil
	rp.lease = nil
}
<commit_msg>Killed methods not in the interface.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lease

import (
	"fmt"
	"io"

	"golang.org/x/net/context"
)

// A type used by read proxies to refresh their contents. See notes on
// NewReadProxy.
type Refresher interface {
	// Return the size of the underlying contents.
	Size() (size int64)

	// Return a read-closer for the contents. The same contents will always be
	// returned, and they will always be of length Size().
	Refresh(ctx context.Context) (rc io.ReadCloser, err error)
}

// A wrapper around a read lease, exposing a similar interface with the
// following differences:
//
//  *  Contents are fetched and re-fetched automatically when needed. Therefore
//     the user need not worry about lease expiration.
//
//  *  Methods that may involve fetching the contents (reading, seeking) accept
//     context arguments, so as to be cancellable.
//
//  *  Only random access reading is supported.
//
// External synchronization is required.
type ReadProxy interface {
	// Return the size of the proxied content. Guarantees to not block.
	Size() (size int64)

	// Semantics matching io.ReaderAt, except with context support.
	ReadAt(ctx context.Context, p []byte, off int64) (n int, err error)

	// Return a read/write lease for the proxied contents, destroying the read
	// proxy. The read proxy must not be used after calling this method.
	Upgrade(ctx context.Context) (rwl ReadWriteLease, err error)

	// Destroy any resources in use by the read proxy. It must not be used
	// further.
	Destroy()
}

// Create a read proxy.
//
// The supplied refresher will be used to obtain the proxy's contents whenever
// the file leaser decides to expire the temporary copy thus obtained.
//
// If rl is non-nil, it will be used as the first temporary copy of the
// contents, and must match what the refresher returns.
func NewReadProxy(
	fl FileLeaser,
	r Refresher,
	rl ReadLease) (rp ReadProxy) {
	rp = &readProxy{
		size:      r.Size(),
		leaser:    fl,
		refresher: r,
		lease:     rl,
	}

	return
}

// A wrapper around a read lease, exposing a similar interface with the
// following differences:
//
//  *  Contents are fetched and re-fetched automatically when needed. Therefore
//     the user need not worry about lease expiration.
//
//  *  Methods that may involve fetching the contents (reading, seeking) accept
//     context arguments, so as to be cancellable.
//
// External synchronization is required.
type readProxy struct {
	/////////////////////////
	// Constant data
	/////////////////////////

	size int64

	/////////////////////////
	// Dependencies
	/////////////////////////

	leaser    FileLeaser
	refresher Refresher

	/////////////////////////
	// Mutable state
	/////////////////////////

	// The current wrapped lease, or nil if one has never been issued.
	lease ReadLease
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func isRevokedErr(err error) bool {
	_, ok := err.(*RevokedError)
	return ok
}

// Set up a read/write lease and fill in our contents.
//
// REQUIRES: The caller has observed that rp.lease has expired.
func (rp *readProxy) getContents(
	ctx context.Context) (rwl ReadWriteLease, err error) {
	// Obtain some space to write the contents.
	rwl, err = rp.leaser.NewFile()
	if err != nil {
		err = fmt.Errorf("NewFile: %v", err)
		return
	}

	// Clean up if we exit early.
	defer func() {
		if err != nil {
			rwl.Downgrade().Revoke()
		}
	}()

	// Obtain the reader for our contents.
	rc, err := rp.refresher.Refresh(ctx)
	if err != nil {
		err = fmt.Errorf("User function: %v", err)
		return
	}

	defer func() {
		closeErr := rc.Close()
		if closeErr != nil && err == nil {
			err = fmt.Errorf("Close: %v", closeErr)
		}
	}()

	// Copy into the read/write lease.
	copied, err := io.Copy(rwl, rc)
	if err != nil {
		err = fmt.Errorf("Copy: %v", err)
		return
	}

	// Did the user lie about the size?
	if copied != rp.Size() {
		err = fmt.Errorf("Copied %v bytes; expected %v", copied, rp.Size())
		return
	}

	return
}

// Downgrade and save the supplied read/write lease obtained with getContents
// for later use.
func (rp *readProxy) saveContents(rwl ReadWriteLease) {
	rp.lease = rwl.Downgrade()
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// Semantics matching io.ReaderAt, except with context support.
func (rp *readProxy) ReadAt(
	ctx context.Context,
	p []byte,
	off int64) (n int, err error) {
	// Common case: is the existing lease still valid?
	if rp.lease != nil {
		n, err = rp.lease.ReadAt(p, off)
		if !isRevokedErr(err) {
			return
		}

		// Clear the revoked error.
		err = nil
	}

	// Get hold of a read/write lease containing our contents.
	rwl, err := rp.getContents(ctx)
	if err != nil {
		err = fmt.Errorf("getContents: %v", err)
		return
	}

	defer rp.saveContents(rwl)

	// Serve from the read/write lease.
	n, err = rwl.ReadAt(p, off)

	return
}

// Return the size of the proxied content. Guarantees to not block.
func (rp *readProxy) Size() (size int64) {
	size = rp.size
	return
}

// Return a read/write lease for the proxied contents, destroying the read
// proxy. The read proxy must not be used after calling this method.
func (rp *readProxy) Upgrade(
	ctx context.Context) (rwl ReadWriteLease, err error) {
	// If we succeed, we are now destroyed.
	defer func() {
		if err == nil {
			rp.Destroy()
		}
	}()

	// Common case: is the existing lease still valid?
	if rp.lease != nil {
		rwl, err = rp.lease.Upgrade()
		if !isRevokedErr(err) {
			return
		}

		// Clear the revoked error.
		err = nil
	}

	// Build the read/write lease anew.
	rwl, err = rp.getContents(ctx)
	if err != nil {
		err = fmt.Errorf("getContents: %v", err)
		return
	}

	return
}

// Destroy any resources in use by the read proxy. It must not be used further.
func (rp *readProxy) Destroy() {
	if rp.lease != nil {
		rp.lease.Revoke()
	}

	// Make use-after-destroy errors obvious.
	rp.size = 0
	rp.leaser = nil
	rp.refresher = nil
	rp.lease = nil
}
<|endoftext|>


<commit_before>package sobjects

type Opportunity struct {
	BaseSObject
	AccountId       string  `force:",omitempty"`
	Amount          float64 `force:",omitempty"`
	CloseDate       string  `force:",omitempty"`
	CurrencyIsoCode string  `force:",omitempty"`
	Description     string  `force:",omitempty"`
	ExpectedRevenue string  `force:",omitempty"`
	IsClosed        bool    `force:",omitempty"`
	IsDeleted       bool    `force:",omitempty"`
	IsSplit         bool    `force:",omitempty"`
	IsWon           bool    `force:",omitempty"`
	Name            string  `force:",omitempty"`
	OwnerId         string  `force:",omitempty"`
}

func (t *Opportunity) ApiName() string {
	return "Opportunity"
}

type OpportunityQueryResponse struct {
	BaseQuery
	Records []Opportunity `json:"Records" force:"records"`
}
<commit_msg>Added the StageName also since it is required for creation.<commit_after>package sobjects

type Opportunity struct {
	BaseSObject
	AccountId       string  `force:",omitempty"`
	Amount          float64 `force:",omitempty"`
	CloseDate       string  `force:",omitempty"`
	CurrencyIsoCode string  `force:",omitempty"`
	Description     string  `force:",omitempty"`
	ExpectedRevenue string  `force:",omitempty"`
	IsClosed        bool    `force:",omitempty"`
	IsDeleted       bool    `force:",omitempty"`
	IsSplit         bool    `force:",omitempty"`
	IsWon           bool    `force:",omitempty"`
	Name            string  `force:",omitempty"`
	OwnerId         string  `force:",omitempty"`
	StageName       string  `force:",omitempty"`
}

func (t *Opportunity) ApiName() string {
	return "Opportunity"
}

type OpportunityQueryResponse struct {
	BaseQuery
	Records []Opportunity `json:"Records" force:"records"`
}
<|endoftext|>


<commit_before>package now

import (
    "github.com/hoisie/web.go"
)

func Serve(http string) {
    web.SetStaticDir("human-friendly")
    web.Get("/now.json", jsonNext)
    web.Post("/later/", jsonLater)
    web.Get("/now", htmlNext)

    web.Run(http)
}

<commit_msg>Register new handlers.<commit_after>package now

import (
    "github.com/hoisie/web.go"
)

func Serve(http string) {
    web.SetStaticDir("human-friendly")
    web.Get("/now.json", jsonNext)
    web.Get("/now", htmlNext)
    web.Post("/done", jsonPop)
    web.Post("/now", jsonPushNext)
    web.Post("/later/", jsonLater)

    web.Run(http)
}

<|endoftext|>


<commit_before>package styx

import (
	"bytes"
	"fmt"
	"runtime"
	"strings"

	"aqwari.net/net/styx/internal/util"
	"aqwari.net/net/styx/styxproto"
)

func newQid(buf []byte, qtype uint8, version uint32, path uint64) styxproto.Qid {
	if len(buf) < styxproto.QidLen {
		buf = make([]byte, styxproto.QidLen)
	}
	q, _, err := styxproto.NewQid(buf, qtype, version, path)
	if err != nil {
		panic("styxproto.NewQid returned error despite sufficient buf size")
	}
	return q
}

var aqid = newQid(nil, styxproto.QTAUTH, 0, 0)

func (c *conn) serve() {
	defer func() {
		if err := recover(); err != nil {
			const size = 64 << 10
			buf := make([]byte, size)
			buf = buf[:runtime.Stack(buf, false)]
			c.srv.logf("panic serving: %v\n%s", err, buf)
		}
		c.close()
	}()

Loop:
	for c.Next() {
		for _, msg := range c.Messages() {
			c.srv.debugf("← %s", msg)
			if err := c.handleMessage(msg); err != nil {
				break Loop
			}
		}
		if err := c.bw.Flush(); err != nil {
			c.srv.logf("error flushing message buffer: %v", err)
			break
		}
	}
	if err := c.bw.Flush(); err != nil {
		c.srv.logf("error flushing message buffer: %v", err)
	}
	if c.Decoder.Err() != nil {
		c.srv.logf("error parsing messages: %v", c.Decoder.Err())
	}
}

func (c *conn) handleMessage(m styxproto.Msg) error {
	if m, ok := m.(styxproto.Tversion); ok {
		if c.state != stateNew {
			c.Rerror(m.Tag(), "late Tversion message")
			return nil
		}
	} else if c.state == stateNew {
		c.Rerror(m.Tag(), "protocol version not negotiated")
		return nil
	}

	switch m := m.(type) {
	case styxproto.Tversion:
		msize := c.srv.MaxSize
		if n := m.Msize(); n < msize {
			msize = n
		}
		if ver := m.Version(); !bytes.HasPrefix(ver, []byte("9P2000")) {
			c.srv.logf("received unknown version %s from %s", ver, c.remoteAddr)
			c.Rversion(uint32(msize), "unknown")
			break
		} else {
			c.Rversion(uint32(c.srv.MaxSize), "9P2000")
			c.state = stateActive
		}
	case styxproto.Tattach:
		rootQid := newQid(c.qidbuf, styxproto.QTDIR, 0, util.Hash64(m.Aname()))
		if afid := m.Afid(); afid != styxproto.NoFid {
			_, ok := c.getSession(afid)
			if !ok {
				c.Rerror(m.Tag(), "authentication failed")
				break
			}
			c.Rattach(m.Tag(), rootQid)
		} else if c.srv.Auth != nil {
			// transport-based auth methods can authenticate
			// Tattach requests as well. This lets users manage
			// authentication without modifying their clients. We
			// pass a dummy channel to the Auth callback.
			var (
				rw    = util.BlackHole{}
				uname = string(m.Uname())
				aname = string(m.Aname())
			)
			if err := c.srv.Auth.Auth(channel{c: c, rw: rw}, uname, aname); err != nil {
				c.Rerror(m.Tag(), "auth required", err)
				break
			}
			c.newSession(m)
			c.Rattach(m.Tag(), rootQid)
		} else {
			if _, inuse := c.getSession(m.Fid()); inuse {
				c.Rerror(m.Tag(), "fid already in use")
				break
			}
			c.newSession(m)
			c.Rattach(m.Tag(), rootQid)
		}
	case styxproto.Tauth:
		if c.srv.Auth == nil {
			c.Rerror(m.Tag(), "no auth required")
			break
		}
		c.Rauth(m.Tag(), aqid)
	case fcall:
		fid := m.Fid()
		_, ok := c.getSession(fid)
		if !ok {
			c.Rerror(m.Tag(), "unknown fid %d", m.Fid())
			break
		}
	case styxproto.Tflush:
		if cancel, ok := c.getPending(m.Oldtag()); ok {
			cancel()
		}
		c.Rflush(m.Tag())
	case styxproto.BadMessage:
		c.srv.logf("received bad message: %v", m.Err)
		c.Rerror(m.Tag(), "malformed message")
		break
	default:
		name := fmt.Sprintf("%T", m)
		name = name[strings.IndexByte(name, '.')+1:]
		c.Rerror(m.Tag(), "unexpected %s message", name)
	}
	return c.Encoder.Err()
}
<commit_msg>breakout authentication routine<commit_after>package styx

import (
	"bytes"
	"fmt"
	"runtime"
	"strings"
	"time"

	"aqwari.net/net/styx/internal/util"
	"aqwari.net/net/styx/styxproto"
)

func newQid(buf []byte, qtype uint8, version uint32, path uint64) styxproto.Qid {
	if len(buf) < styxproto.QidLen {
		buf = make([]byte, styxproto.QidLen)
	}
	q, _, err := styxproto.NewQid(buf, qtype, version, path)
	if err != nil {
		panic("styxproto.NewQid returned error despite sufficient buf size")
	}
	return q
}

var aqid = newQid(nil, styxproto.QTAUTH, 0, 0)

func (c *conn) serve() {
	defer func() {
		if err := recover(); err != nil {
			const size = 64 << 10
			buf := make([]byte, size)
			buf = buf[:runtime.Stack(buf, false)]
			c.srv.logf("panic serving: %v\n%s", err, buf)
		}
		c.close()
	}()

Loop:
	for c.Next() {
		for _, msg := range c.Messages() {
			c.srv.debugf("← %s", msg)
			if err := c.handleMessage(msg); err != nil {
				break Loop
			}
		}
		if err := c.bw.Flush(); err != nil {
			c.srv.logf("error flushing message buffer: %v", err)
			break
		}
	}
	if err := c.bw.Flush(); err != nil {
		c.srv.logf("error flushing message buffer: %v", err)
	}
	if c.Decoder.Err() != nil {
		c.srv.logf("error parsing messages: %v", c.Decoder.Err())
	}
}

// runs in its own goroutine. creates a session on succesful
// authentication.
func (c *conn) authenticate(afid uint32, user, access string) {
	// Without a timeout this goroutine could hang around
	// forever.
	const timeout = time.Second * 10
	deadline := time.NewTimer(timeout)
	select {
	case <-deadline.C:
		return
	}
}

func (c *conn) handleMessage(m styxproto.Msg) error {
	if m, ok := m.(styxproto.Tversion); ok {
		if c.state != stateNew {
			c.Rerror(m.Tag(), "late Tversion message")
			return nil
		}
	} else if c.state == stateNew {
		c.Rerror(m.Tag(), "protocol version not negotiated")
		return nil
	}

	switch m := m.(type) {
	case styxproto.Tversion:
		msize := c.srv.MaxSize
		if n := m.Msize(); n < msize {
			msize = n
		}
		if ver := m.Version(); !bytes.HasPrefix(ver, []byte("9P2000")) {
			c.srv.logf("received unknown version %s from %s", ver, c.remoteAddr)
			c.Rversion(uint32(msize), "unknown")
			break
		} else {
			c.Rversion(uint32(c.srv.MaxSize), "9P2000")
			c.state = stateActive
		}
	case styxproto.Tattach:
		rootQid := newQid(c.qidbuf, styxproto.QTDIR, 0, util.Hash64(m.Aname()))
		if afid := m.Afid(); afid != styxproto.NoFid {
			_, ok := c.getSession(afid)
			if !ok {
				c.Rerror(m.Tag(), "authentication failed")
				break
			}
			c.Rattach(m.Tag(), rootQid)
		} else if c.srv.Auth != nil {
			// transport-based auth methods can authenticate
			// Tattach requests as well. This lets users manage
			// authentication without modifying their clients. We
			// pass a dummy channel to the Auth callback.
			var (
				rw    = util.BlackHole{}
				uname = string(m.Uname())
				aname = string(m.Aname())
			)
			if err := c.srv.Auth.Auth(channel{c: c, rw: rw}, uname, aname); err != nil {
				c.Rerror(m.Tag(), "auth required", err)
				break
			}
			c.newSession(m)
			c.Rattach(m.Tag(), rootQid)
		} else {
			if _, inuse := c.getSession(m.Fid()); inuse {
				c.Rerror(m.Tag(), "fid already in use")
				break
			}
			c.newSession(m)
			c.Rattach(m.Tag(), rootQid)
		}
	case styxproto.Tauth:
		if c.srv.Auth == nil {
			c.Rerror(m.Tag(), "no auth required")
			break
		}
		go c.authenticate(m.Afid(), string(m.Uname()), string(m.Aname()))
		c.Rauth(m.Tag(), aqid)
	case fcall:
		fid := m.Fid()
		_, ok := c.getSession(fid)
		if !ok {
			c.Rerror(m.Tag(), "unknown fid %d", m.Fid())
			break
		}
	case styxproto.Tflush:
		if cancel, ok := c.getPending(m.Oldtag()); ok {
			cancel()
		}
		c.Rflush(m.Tag())
	case styxproto.BadMessage:
		c.srv.logf("received bad message: %v", m.Err)
		c.Rerror(m.Tag(), "malformed message")
		break
	default:
		name := fmt.Sprintf("%T", m)
		name = name[strings.IndexByte(name, '.')+1:]
		c.Rerror(m.Tag(), "unexpected %s message", name)
	}
	return c.Encoder.Err()
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"compress/gzip"
	"io"
	"net/http"
	"os"
	"path"
	"strconv"
	"strings"

	log "github.com/Sirupsen/logrus"
)

// GetDockerBinary ensures that we have the right version docker client
// for communicating with the Docker Daemon
func (d *Daemon) GetDockerBinary() error {
	// name of docker binary that is needed
	d.BinaryName = "docker-" + d.Version
	log.Infof("looking for docker binary named: %s", d.BinaryName)

	filename := path.Join("/bin", d.BinaryName)

	if _, err := os.Stat(filename); os.IsNotExist(err) {

		log.Infof("docker binary (version %s) not found.", d.Version)
		log.Infof("downloading %s...", d.BinaryName)

		out, err := os.Create(filename)
		if err != nil {
			return err
		}
		defer out.Close()

		// the method of downloading it is different for version >= 1.11.0
		// (in which case it is an archive containing multiple binaries)
		versionComp, err := compareVersions(d.Version, "1.11.0")
		if err != nil {
			return err
		}
		if versionComp >= 0 {
			err = getClient(out, "https://get.docker.com/builds/Linux/x86_64/docker-"+d.Version+".tgz", extractClient)
			if err != nil {
				return err
			}
		} else {
			err = getClient(out, "https://get.docker.com/builds/Linux/x86_64/docker-"+d.Version, copyClient)
			if err != nil {
				return err
			}
		}

		err = os.Chmod(filename, 0700)
		if err != nil {
			return err
		}
	} else {
		log.Infof("docker binary (version %s) found!", d.Version)
	}
	return nil
}

// Utility functions

type copier func(out *os.File, resp *http.Response) error

func compareVersions(v1 string, v2 string) (comp int, err error) {
	v1Parts := strings.Split(v1, ".")
	v2Parts := strings.Split(v2, ".")

	for i := 0; i < len(v1Parts) && i < len(v2Parts); i++ {
		v1int, err := strconv.Atoi(v1Parts[i])
		if err != nil {
			return -2, err
		}

		v2int, err := strconv.Atoi(v2Parts[i])
		if err != nil {
			return -2, err
		}

		if v1int < v2int {
			return -1, nil
		} else if v1int > v2int {
			return 1, nil
		}
	}

	if len(v1Parts) < len(v2Parts) {
		return -1, nil
	} else if len(v1Parts) > len(v2Parts) {
		return 1, nil
	}
	return 0, nil
}

func getClient(out *os.File, URL string, cp copier) error {
	resp, err := http.Get(URL)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	err = cp(out, resp)
	return err
}

func copyClient(out *os.File, resp *http.Response) error {
	_, err := io.Copy(out, resp.Body)
	return err
}

func extractClient(out *os.File, resp *http.Response) error {
	gr, err := gzip.NewReader(resp.Body)
	defer gr.Close()
	if err != nil {
		return err
	}

	tr := tar.NewReader(gr)
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		if hdr.Typeflag == tar.TypeReg && hdr.Name == "docker/docker" {
			_, err = io.Copy(out, tr)
			if err != nil {
				return err
			}
			break
		}
		// logrus.Println("not yet")
	}
	return nil
}
<commit_msg>Add support for Docker RC's<commit_after>package main

import (
	"archive/tar"
	"compress/gzip"
	"io"
	"net/http"
	"os"
	"path"
	"strconv"
	"strings"

	log "github.com/Sirupsen/logrus"
)

const (
	downloadURL   = "https://get.docker.com/builds/Linux/x86_64/docker-"
	rcDownloadURL = "https://test.docker.com/builds/Linux/x86_64/docker-"
)

// GetDockerBinary ensures that we have the right version docker client
// for communicating with the Docker Daemon
func (d *Daemon) GetDockerBinary() error {
	// name of docker binary that is needed
	d.BinaryName = "docker-" + d.Version
	log.Infof("looking for docker binary named: %s", d.BinaryName)

	filename := path.Join("/bin", d.BinaryName)

	if _, err := os.Stat(filename); os.IsNotExist(err) {

		log.Infof("docker binary (version %s) not found.", d.Version)
		log.Infof("downloading %s...", d.BinaryName)

		out, err := os.Create(filename)
		if err != nil {
			return err
		}
		defer out.Close()

		// determine if we're using an RC build of docker
		url := downloadURL
		if strings.Contains(d.Version, "rc") {
			url = rcDownloadURL
		}

		// the method of downloading it is different for version >= 1.11.0
		// (in which case it is an archive containing multiple binaries)
		versionComp, err := compareVersions(d.Version, "1.11.0")
		if err != nil {
			return err
		}
		if versionComp >= 0 {
			err = getClient(out, url+d.Version+".tgz", extractClient)
			if err != nil {
				return err
			}
		} else {
			err = getClient(out, url+d.Version, copyClient)
			if err != nil {
				return err
			}
		}

		err = os.Chmod(filename, 0700)
		if err != nil {
			return err
		}
	} else {
		log.Infof("docker binary (version %s) found!", d.Version)
	}
	return nil
}

// Utility functions

type copier func(out *os.File, resp *http.Response) error

func compareVersions(v1 string, v2 string) (comp int, err error) {
	v1Parts := strings.Split(v1, ".")
	v2Parts := strings.Split(v2, ".")

	for i := 0; i < len(v1Parts) && i < len(v2Parts); i++ {
		v1int, err := strconv.Atoi(v1Parts[i])
		if err != nil {
			return -2, err
		}

		v2int, err := strconv.Atoi(v2Parts[i])
		if err != nil {
			return -2, err
		}

		if v1int < v2int {
			return -1, nil
		} else if v1int > v2int {
			return 1, nil
		}
	}

	if len(v1Parts) < len(v2Parts) {
		return -1, nil
	} else if len(v1Parts) > len(v2Parts) {
		return 1, nil
	}
	return 0, nil
}

func getClient(out *os.File, URL string, cp copier) error {
	resp, err := http.Get(URL)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	err = cp(out, resp)
	return err
}

func copyClient(out *os.File, resp *http.Response) error {
	_, err := io.Copy(out, resp.Body)
	return err
}

func extractClient(out *os.File, resp *http.Response) error {
	gr, err := gzip.NewReader(resp.Body)
	defer gr.Close()
	if err != nil {
		return err
	}

	tr := tar.NewReader(gr)
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		if hdr.Typeflag == tar.TypeReg && hdr.Name == "docker/docker" {
			_, err = io.Copy(out, tr)
			if err != nil {
				return err
			}
			break
		}
		// logrus.Println("not yet")
	}
	return nil
}
<|endoftext|>