<commit_before>// Copyright 2017 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package serverutil

import (
	"sync"
	"time"
)

// The maximum number of visitors that a RateLimiter can track.
const rateMaxVisitors = 100000

// RateLimiter implements a rate limiter with exponential backoff,
// up to a specified maximum.
type RateLimiter struct {
	// Backoff specifies an initial backoff duration for a key.
	// After the first request for a given key the key will be denied until
	// the backoff has passed. If another request arrives after the backoff
	// but before Max, the backoff duration is doubled.
	Backoff time.Duration

	// Max specifies a maximum backoff duration.
	Max time.Duration

	mu          sync.Mutex // Guards the fields below.
	m           map[string]*visitor
	first, last *visitor
}

type visitor struct {
	key     string
	seen    time.Time
	backoff time.Duration

	prev, next *visitor
}

// Pass attempts to pass key through the rate limiter, returning true if key is
// within the rate limit. If it returns false it also returns the duration that
// must elapse before the key will be allowed to pass again.
func (r *RateLimiter) Pass(key string) (bool, time.Duration) {
	return r.pass(time.Now(), key)
}

func (r *RateLimiter) pass(now time.Time, key string) (bool, time.Duration) {
	r.mu.Lock()
	defer r.mu.Unlock()

	// Initialize the map lazily so that RateLimiter
	// may be useful in its zero form.
	if r.m == nil {
		r.m = map[string]*visitor{}
	}

	v, ok := r.m[key]
	if !ok {
		// We haven't seen this visitor before,
		// so add a map entry and permit it.
		v = &visitor{
			key:     key,
			seen:    now,
			backoff: r.Backoff,
		}
		r.m[key] = v

		// Add visitor to the end of the list.
		if r.last != nil {
			r.last.next = v
			v.prev = r.last
		}
		r.last = v

		// If the list is empty, add it at the start.
		if r.first == nil {
			r.first = v
		}
	} else {
		// We have seen this visitor before.
		// If MaxBackoff has passed since its last request,
		// permit it and reset the backoff to its initial state.
		// If v.backoff has passed, permit it but double the backoff.
		// Otherwise, deny it.
		resetTime := v.seen.Add(r.Max)
		passTime := v.seen.Add(v.backoff)
		switch {
		case now.After(resetTime):
			v.backoff = r.Backoff
		case now.After(passTime):
			v.backoff *= 2
			if v.backoff > r.Max {
				v.backoff = r.Max
			}
		default:
			return false, passTime.Sub(now)
		}

		// Mark that we've seen this visitor now.
		v.seen = now
		// Move v to the end of the list, if it's not there already.
		if r.last != v {
			// Remove v from the list.
			if v.prev != nil {
				v.prev.next = v.next
			} else {
				r.first = v.next
			}
			if v.next != nil {
				v.next.prev = v.prev
			}
			// Attach v to the end of the list.
			v.prev = r.last
			v.next = nil
			r.last.next = v
			r.last = v
		}
	}

	// Find and delete expired visitors.
	// Also check whether we have exceeded the maximum number of visitors
	// that we can track at once. If so, prune back to the maximum.
	drop := 0
	if len(r.m) >= rateMaxVisitors {
		drop = len(r.m) - rateMaxVisitors
	}
	for v, i := r.first, 0; v != nil; v, i = v.next, i+1 {
		if !now.After(v.seen.Add(r.Max)) && i >= drop {
			break
		}
		delete(r.m, v.key)
		r.first = v.next
		if v.next != nil {
			v.next.prev = nil
		}
	}

	return true, 0
}
<commit_msg>serverutil: optimize RateLimiter.pass<commit_after>// Copyright 2017 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package serverutil

import (
	"sync"
	"time"
)

// The maximum number of visitors that a RateLimiter can track.
const rateMaxVisitors = 100000

// RateLimiter implements a rate limiter with exponential backoff,
// up to a specified maximum.
type RateLimiter struct {
	// Backoff specifies an initial backoff duration for a key.
	// After the first request for a given key the key will be denied until
	// the backoff has passed. If another request arrives after the backoff
	// but before Max, the backoff duration is doubled.
	Backoff time.Duration

	// Max specifies a maximum backoff duration.
	Max time.Duration

	mu          sync.Mutex // Guards the fields below.
	m           map[string]*visitor
	first, last *visitor
}

type visitor struct {
	key     string
	seen    time.Time
	backoff time.Duration

	prev, next *visitor
}

// Pass attempts to pass key through the rate limiter, returning true if key is
// within the rate limit. If it returns false it also returns the duration that
// must elapse before the key will be allowed to pass again.
func (r *RateLimiter) Pass(key string) (bool, time.Duration) {
	return r.pass(time.Now(), key)
}

func (r *RateLimiter) pass(now time.Time, key string) (bool, time.Duration) {
	r.mu.Lock()
	defer r.mu.Unlock()

	// Initialize the map lazily so that RateLimiter
	// may be useful in its zero form.
	if r.m == nil {
		r.m = map[string]*visitor{}
	}

	v, ok := r.m[key]
	if !ok {
		// We haven't seen this visitor before,
		// so add a map entry and permit it.
		v = &visitor{
			key:     key,
			seen:    now,
			backoff: r.Backoff,
		}
		r.m[key] = v

		// Add visitor to the end of the list.
		if r.last != nil {
			r.last.next = v
			v.prev = r.last
		}
		r.last = v

		// If the list is empty, add it at the start.
		if r.first == nil {
			r.first = v
		}
	} else {
		// We have seen this visitor before.
		// If v.backoff has passed, permit it but double the backoff.
		// Otherwise, deny it.
		// If MaxBackoff has passed since its last request,
		// permit it and reset the backoff to its initial state.
		resetTime := v.seen.Add(r.Max)
		if now.After(resetTime) {
			v.backoff = r.Backoff
		} else {
			passTime := v.seen.Add(v.backoff)

			if now.After(passTime) {
				v.backoff *= 2
				if v.backoff > r.Max {
					v.backoff = r.Max
				}
			} else {
				return false, passTime.Sub(now)
			}
		}

		// Mark that we've seen this visitor now.
		v.seen = now
		// Move v to the end of the list, if it's not there already.
		if r.last != v {
			// Remove v from the list.
			if v.prev != nil {
				v.prev.next = v.next
			} else {
				r.first = v.next
			}
			if v.next != nil {
				v.next.prev = v.prev
			}
			// Attach v to the end of the list.
			v.prev = r.last
			v.next = nil
			r.last.next = v
			r.last = v
		}
	}

	// Find and delete expired visitors.
	// Also check whether we have exceeded the maximum number of visitors
	// that we can track at once. If so, prune back to the maximum.
	drop := 0
	if len(r.m) >= rateMaxVisitors {
		drop = len(r.m) - rateMaxVisitors
	}
	for v, i := r.first, 0; v != nil; v, i = v.next, i+1 {
		if !now.After(v.seen.Add(r.Max)) && i >= drop {
			break
		}
		delete(r.m, v.key)
		r.first = v.next
		if v.next != nil {
			v.next.prev = nil
		}
	}

	return true, 0
}
<|endoftext|>


<commit_before>/*
Copyright (c) 2019 the Octant contributors. All Rights Reserved.
SPDX-License-Identifier: Apache-2.0
*/

package overview

import (
	"context"
	"fmt"
	"net/http"
	"sync"

	"github.com/gorilla/mux"
	"github.com/pkg/errors"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"

	"github.com/vmware-tanzu/octant/internal/api"
	"github.com/vmware-tanzu/octant/internal/config"
	"github.com/vmware-tanzu/octant/internal/describer"
	"github.com/vmware-tanzu/octant/internal/generator"
	"github.com/vmware-tanzu/octant/internal/log"
	"github.com/vmware-tanzu/octant/internal/module"
	"github.com/vmware-tanzu/octant/internal/octant"
	"github.com/vmware-tanzu/octant/pkg/action"
	"github.com/vmware-tanzu/octant/pkg/icon"
	"github.com/vmware-tanzu/octant/pkg/navigation"
	"github.com/vmware-tanzu/octant/pkg/store"
	"github.com/vmware-tanzu/octant/pkg/view/component"
)

type Options struct {
	Namespace  string
	DashConfig config.Dash
}

// Overview is an API for generating a cluster overview.
type Overview struct {
	*octant.ObjectPath

	generator   generator.Interface
	dashConfig  config.Dash
	contextName string
	pathMatcher *describer.PathMatcher
	logger      log.Logger

	watchedCRDs []*unstructured.Unstructured

	mu sync.Mutex
}

var _ module.Module = (*Overview)(nil)
var _ module.ActionReceiver = (*Overview)(nil)

// New creates an instance of Overview.
func New(ctx context.Context, options Options) (*Overview, error) {
	if options.DashConfig == nil {
		return nil, errors.New("dash configuration is nil")
	}

	if err := options.DashConfig.Validate(); err != nil {
		return nil, errors.Wrap(err, "dash configuration")
	}

	co := &Overview{
		dashConfig: options.DashConfig,
		logger:     options.DashConfig.Logger().With("module", "overview"),
	}

	if err := co.bootstrap(ctx); err != nil {
		return nil, err
	}

	logger := log.From(ctx).With("module", "overview")

	co.dashConfig.ObjectStore().RegisterOnUpdate(func(newObjectStore store.Store) {
		logger.Debugf("object store was updated")
		if err := co.bootstrap(ctx); err != nil {
			logger.WithErr(err).Errorf("updating object store")
		}
	})

	return co, nil
}

func (co *Overview) SetContext(ctx context.Context, contextName string) error {
	co.mu.Lock()
	defer co.mu.Unlock()

	customResourcesDescriber := describer.NamespacedCRD()
	co.contextName = contextName
	for i := range co.watchedCRDs {
		describer.DeleteCRD(ctx, co.watchedCRDs[i], co.pathMatcher, customResourcesDescriber, co)
	}

	co.watchedCRDs = []*unstructured.Unstructured{}

	return nil
}

func (co *Overview) bootstrap(ctx context.Context) error {
	rootDescriber := describer.NamespacedOverview()

	if err := rootDescriber.Reset(ctx); err != nil {
		return err
	}

	pathMatcher := describer.NewPathMatcher("overview")
	for _, pf := range rootDescriber.PathFilters() {
		pathMatcher.Register(ctx, pf)
	}

	g, err := generator.NewGenerator(pathMatcher, co.dashConfig)
	if err != nil {
		return errors.Wrap(err, "create overview generator")
	}

	objectPathConfig := octant.ObjectPathConfig{
		ModuleName:     "overview",
		SupportedGVKs:  supportedGVKs,
		PathLookupFunc: gvkPath,
		CRDPathGenFunc: crdPath,
	}
	objectPath, err := octant.NewObjectPath(objectPathConfig)
	if err != nil {
		return errors.Wrap(err, "create module object path generator")
	}

	co.ObjectPath = objectPath
	co.generator = g

	crdWatcher := co.dashConfig.CRDWatcher()

	customResourcesDescriber := describer.NamespacedCRD()

	watchConfig := &config.CRDWatchConfig{
		Add: func(pathMatcher *describer.PathMatcher, sectionDescriber *describer.CRDSection) config.ObjectHandler {
			return func(ctx context.Context, object *unstructured.Unstructured) {
				co.mu.Lock()
				defer co.mu.Unlock()

				if object == nil {
					return
				}
				describer.AddCRD(ctx, object, pathMatcher, customResourcesDescriber, co)
				co.watchedCRDs = append(co.watchedCRDs, object)
			}
		}(pathMatcher, customResourcesDescriber),
		Delete: func(pathMatcher *describer.PathMatcher, csd *describer.CRDSection) config.ObjectHandler {
			return func(ctx context.Context, object *unstructured.Unstructured) {
				co.mu.Lock()
				defer co.mu.Unlock()

				if object == nil {
					return
				}
				describer.DeleteCRD(ctx, object, pathMatcher, customResourcesDescriber, co)
				var list []*unstructured.Unstructured
				for i := range co.watchedCRDs {
					if co.watchedCRDs[i].GetUID() == object.GetUID() {
						continue
					}
					list = append(list, co.watchedCRDs[i])
				}
				co.watchedCRDs = list
			}
		}(pathMatcher, customResourcesDescriber),
		IsNamespaced: true,
	}

	if err := crdWatcher.AddConfig(watchConfig); err != nil {
		return errors.Wrap(err, "create namespaced CRD watcher for overview")
	}

	co.pathMatcher = pathMatcher

	return nil
}

// Name returns the name for this module.
func (co *Overview) Name() string {
	return "overview"
}

func (co *Overview) ClientRequestHandlers() []octant.ClientRequestHandler {
	return nil
}

// ContentPath returns the content path for overview.
func (co *Overview) ContentPath() string {
	return co.Name()
}

// Navigation returns navigation entries for overview.
func (co *Overview) Navigation(ctx context.Context, namespace, root string) ([]navigation.Navigation, error) {
	navigationEntries := octant.NavigationEntries{
		Lookup: navPathLookup,
		EntriesFuncs: map[string]octant.EntriesFunc{
			"Workloads":                    workloadEntries,
			"Discovery and Load Balancing": discoAndLBEntries,
			"Config and Storage":           configAndStorageEntries,
			"Custom Resources":             navigation.CRDEntries,
			"RBAC":                         rbacEntries,
			"Events":                       nil,
		},
		Order: []string{
			"Workloads",
			"Discovery and Load Balancing",
			"Config and Storage",
			"Custom Resources",
			"RBAC",
			"Events",
		},
	}

	objectStore := co.dashConfig.ObjectStore()

	nf := octant.NewNavigationFactory(namespace, root, objectStore, navigationEntries)

	entries, err := nf.Generate(ctx, "Overview", icon.Overview, "", false)
	if err != nil {
		return nil, err
	}

	return []navigation.Navigation{
		*entries,
	}, nil
}

// Generators allow modules to send events to the frontend.
func (co *Overview) Generators() []octant.Generator {
	return []octant.Generator{}
}

// SetNamespace sets the current namespace.
func (co *Overview) SetNamespace(namespace string) error {
	co.dashConfig.Logger().With("namespace", namespace, "module", "overview").Debugf("setting namespace (noop)")
	return nil
}

// Start starts overview.
func (co *Overview) Start() error {
	return nil
}

// Stop stops overview.
func (co *Overview) Stop() {
	// NOOP
}

// Content serves content for overview.
func (co *Overview) Content(ctx context.Context, contentPath string, opts module.ContentOptions) (component.ContentResponse, error) {
	ctx = log.WithLoggerContext(ctx, co.dashConfig.Logger())
	genOpts := generator.Options{
		LabelSet:               opts.LabelSet,
		ExtensionDescriberFunc: co.extensionDescriber,
	}
	return co.generator.Generate(ctx, contentPath, genOpts)
}

func (co *Overview) extensionDescriber(path, namespace string, options describer.Options) (*component.Extension, error) {
	extension := component.NewExtension()
	terminals := options.TerminalManager().List(namespace)

	for _, t := range terminals {
		name := t.Command()
		if !t.Active() {
			name += " (exited)"
		}
		tfl := component.NewFlexLayout(name)
		tfl.SetAccessor(t.ID())

		details := component.TerminalDetails{
			Container: t.Container(),
			Command:   t.Command(),
			UUID:      t.ID(),
			CreatedAt: t.CreatedAt(),
			Active:    t.Active(),
		}

		tfl.AddSections([]component.FlexLayoutItem{
			{
				Width: component.WidthFull,
				View:  component.NewTerminal(t.Key().Namespace, t.Key().Name, details),
			},
		})

		extensionTab := component.ExtensionTab{
			Tab:          tfl,
			ClosePayload: action.CreatePayload("overview/deleteTerminal", action.Payload{"terminalID": t.ID()}),
		}

		extension.AddTab(extensionTab)
	}
	return extension, nil
}

func (co *Overview) portForwardsHandler() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		svc := co.dashConfig.PortForwarder()
		logger := co.dashConfig.Logger()

		if svc == nil {
			logger.Errorf("port forward service is nil")
			http.Error(w, "port forward service is nil", http.StatusInternalServerError)
			return
		}

		ctx := log.WithLoggerContext(r.Context(), logger)

		defer func() {
			if cErr := r.Body.Close(); cErr != nil {
				logger.With("err", cErr).Errorf("unable to close port forward request body")
			}
		}()

		switch r.Method {
		case http.MethodPost:
			err := createPortForward(ctx, r.Body, svc, w)
			handlePortForwardError(w, err, logger)
		default:
			api.RespondWithError(
				w,
				http.StatusNotFound,
				fmt.Sprintf("unhandled HTTP method %s", r.Method),
				logger,
			)
		}
	}
}

func (co *Overview) portForwardHandler() http.HandlerFunc {
	logger := co.dashConfig.Logger()

	return func(w http.ResponseWriter, r *http.Request) {
		svc := co.dashConfig.PortForwarder()
		if svc == nil {
			logger.Errorf("port forward service is nil")
			http.Error(w, "port forward service is nil", http.StatusInternalServerError)
			return
		}

		vars := mux.Vars(r)
		id := vars["id"]

		ctx := log.WithLoggerContext(r.Context(), logger)

		switch r.Method {
		case http.MethodDelete:
			err := deletePortForward(ctx, id, co.dashConfig.PortForwarder(), w)
			handlePortForwardError(w, err, logger)
		default:
			api.RespondWithError(
				w,
				http.StatusNotFound,
				fmt.Sprintf("unhandled HTTP method %s", r.Method),
				logger,
			)
		}
	}
}

// ActionPaths contain the actions this module is responsible for.
func (co *Overview) ActionPaths() map[string]action.DispatcherFunc {
	dispatchers := action.Dispatchers{
		octant.NewDeploymentConfigurationEditor(co.logger, co.dashConfig.ObjectStore()),
		octant.NewContainerEditor(co.dashConfig.ObjectStore()),
		octant.NewServiceConfigurationEditor(co.dashConfig.ObjectStore()),
		octant.NewPortForward(co.logger, co.dashConfig.ObjectStore(), co.dashConfig.PortForwarder()),
		octant.NewPortForwardDelete(co.logger, co.dashConfig.ObjectStore(), co.dashConfig.PortForwarder()),
		octant.NewTerminalCommandExec(co.logger, co.dashConfig.ObjectStore(), co.dashConfig.TerminalManager()),
		octant.NewTerminalDelete(co.logger, co.dashConfig.ObjectStore(), co.dashConfig.TerminalManager()),
		octant.NewCordon(co.dashConfig.ObjectStore(), co.dashConfig.ClusterClient()),
		octant.NewUncordon(co.dashConfig.ObjectStore(), co.dashConfig.ClusterClient()),
	}

	return dispatchers.ToActionPaths()
}
<commit_msg>remove unused portForward handlers<commit_after>/*
Copyright (c) 2019 the Octant contributors. All Rights Reserved.
SPDX-License-Identifier: Apache-2.0
*/

package overview

import (
	"context"
	"sync"

	"github.com/pkg/errors"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"

	"github.com/vmware-tanzu/octant/internal/config"
	"github.com/vmware-tanzu/octant/internal/describer"
	"github.com/vmware-tanzu/octant/internal/generator"
	"github.com/vmware-tanzu/octant/internal/log"
	"github.com/vmware-tanzu/octant/internal/module"
	"github.com/vmware-tanzu/octant/internal/octant"
	"github.com/vmware-tanzu/octant/pkg/action"
	"github.com/vmware-tanzu/octant/pkg/icon"
	"github.com/vmware-tanzu/octant/pkg/navigation"
	"github.com/vmware-tanzu/octant/pkg/store"
	"github.com/vmware-tanzu/octant/pkg/view/component"
)

type Options struct {
	Namespace  string
	DashConfig config.Dash
}

// Overview is an API for generating a cluster overview.
type Overview struct {
	*octant.ObjectPath

	generator   generator.Interface
	dashConfig  config.Dash
	contextName string
	pathMatcher *describer.PathMatcher
	logger      log.Logger

	watchedCRDs []*unstructured.Unstructured

	mu sync.Mutex
}

var _ module.Module = (*Overview)(nil)
var _ module.ActionReceiver = (*Overview)(nil)

// New creates an instance of Overview.
func New(ctx context.Context, options Options) (*Overview, error) {
	if options.DashConfig == nil {
		return nil, errors.New("dash configuration is nil")
	}

	if err := options.DashConfig.Validate(); err != nil {
		return nil, errors.Wrap(err, "dash configuration")
	}

	co := &Overview{
		dashConfig: options.DashConfig,
		logger:     options.DashConfig.Logger().With("module", "overview"),
	}

	if err := co.bootstrap(ctx); err != nil {
		return nil, err
	}

	logger := log.From(ctx).With("module", "overview")

	co.dashConfig.ObjectStore().RegisterOnUpdate(func(newObjectStore store.Store) {
		logger.Debugf("object store was updated")
		if err := co.bootstrap(ctx); err != nil {
			logger.WithErr(err).Errorf("updating object store")
		}
	})

	return co, nil
}

func (co *Overview) SetContext(ctx context.Context, contextName string) error {
	co.mu.Lock()
	defer co.mu.Unlock()

	customResourcesDescriber := describer.NamespacedCRD()
	co.contextName = contextName
	for i := range co.watchedCRDs {
		describer.DeleteCRD(ctx, co.watchedCRDs[i], co.pathMatcher, customResourcesDescriber, co)
	}

	co.watchedCRDs = []*unstructured.Unstructured{}

	return nil
}

func (co *Overview) bootstrap(ctx context.Context) error {
	rootDescriber := describer.NamespacedOverview()

	if err := rootDescriber.Reset(ctx); err != nil {
		return err
	}

	pathMatcher := describer.NewPathMatcher("overview")
	for _, pf := range rootDescriber.PathFilters() {
		pathMatcher.Register(ctx, pf)
	}

	g, err := generator.NewGenerator(pathMatcher, co.dashConfig)
	if err != nil {
		return errors.Wrap(err, "create overview generator")
	}

	objectPathConfig := octant.ObjectPathConfig{
		ModuleName:     "overview",
		SupportedGVKs:  supportedGVKs,
		PathLookupFunc: gvkPath,
		CRDPathGenFunc: crdPath,
	}
	objectPath, err := octant.NewObjectPath(objectPathConfig)
	if err != nil {
		return errors.Wrap(err, "create module object path generator")
	}

	co.ObjectPath = objectPath
	co.generator = g

	crdWatcher := co.dashConfig.CRDWatcher()

	customResourcesDescriber := describer.NamespacedCRD()

	watchConfig := &config.CRDWatchConfig{
		Add: func(pathMatcher *describer.PathMatcher, sectionDescriber *describer.CRDSection) config.ObjectHandler {
			return func(ctx context.Context, object *unstructured.Unstructured) {
				co.mu.Lock()
				defer co.mu.Unlock()

				if object == nil {
					return
				}
				describer.AddCRD(ctx, object, pathMatcher, customResourcesDescriber, co)
				co.watchedCRDs = append(co.watchedCRDs, object)
			}
		}(pathMatcher, customResourcesDescriber),
		Delete: func(pathMatcher *describer.PathMatcher, csd *describer.CRDSection) config.ObjectHandler {
			return func(ctx context.Context, object *unstructured.Unstructured) {
				co.mu.Lock()
				defer co.mu.Unlock()

				if object == nil {
					return
				}
				describer.DeleteCRD(ctx, object, pathMatcher, customResourcesDescriber, co)
				var list []*unstructured.Unstructured
				for i := range co.watchedCRDs {
					if co.watchedCRDs[i].GetUID() == object.GetUID() {
						continue
					}
					list = append(list, co.watchedCRDs[i])
				}
				co.watchedCRDs = list
			}
		}(pathMatcher, customResourcesDescriber),
		IsNamespaced: true,
	}

	if err := crdWatcher.AddConfig(watchConfig); err != nil {
		return errors.Wrap(err, "create namespaced CRD watcher for overview")
	}

	co.pathMatcher = pathMatcher

	return nil
}

// Name returns the name for this module.
func (co *Overview) Name() string {
	return "overview"
}

func (co *Overview) ClientRequestHandlers() []octant.ClientRequestHandler {
	return nil
}

// ContentPath returns the content path for overview.
func (co *Overview) ContentPath() string {
	return co.Name()
}

// Navigation returns navigation entries for overview.
func (co *Overview) Navigation(ctx context.Context, namespace, root string) ([]navigation.Navigation, error) {
	navigationEntries := octant.NavigationEntries{
		Lookup: navPathLookup,
		EntriesFuncs: map[string]octant.EntriesFunc{
			"Workloads":                    workloadEntries,
			"Discovery and Load Balancing": discoAndLBEntries,
			"Config and Storage":           configAndStorageEntries,
			"Custom Resources":             navigation.CRDEntries,
			"RBAC":                         rbacEntries,
			"Events":                       nil,
		},
		Order: []string{
			"Workloads",
			"Discovery and Load Balancing",
			"Config and Storage",
			"Custom Resources",
			"RBAC",
			"Events",
		},
	}

	objectStore := co.dashConfig.ObjectStore()

	nf := octant.NewNavigationFactory(namespace, root, objectStore, navigationEntries)

	entries, err := nf.Generate(ctx, "Overview", icon.Overview, "", false)
	if err != nil {
		return nil, err
	}

	return []navigation.Navigation{
		*entries,
	}, nil
}

// Generators allow modules to send events to the frontend.
func (co *Overview) Generators() []octant.Generator {
	return []octant.Generator{}
}

// SetNamespace sets the current namespace.
func (co *Overview) SetNamespace(namespace string) error {
	co.dashConfig.Logger().With("namespace", namespace, "module", "overview").Debugf("setting namespace (noop)")
	return nil
}

// Start starts overview.
func (co *Overview) Start() error {
	return nil
}

// Stop stops overview.
func (co *Overview) Stop() {
	// NOOP
}

// Content serves content for overview.
func (co *Overview) Content(ctx context.Context, contentPath string, opts module.ContentOptions) (component.ContentResponse, error) {
	ctx = log.WithLoggerContext(ctx, co.dashConfig.Logger())
	genOpts := generator.Options{
		LabelSet:               opts.LabelSet,
		ExtensionDescriberFunc: co.extensionDescriber,
	}
	return co.generator.Generate(ctx, contentPath, genOpts)
}

func (co *Overview) extensionDescriber(path, namespace string, options describer.Options) (*component.Extension, error) {
	extension := component.NewExtension()
	terminals := options.TerminalManager().List(namespace)

	for _, t := range terminals {
		name := t.Command()
		if !t.Active() {
			name += " (exited)"
		}
		tfl := component.NewFlexLayout(name)
		tfl.SetAccessor(t.ID())

		details := component.TerminalDetails{
			Container: t.Container(),
			Command:   t.Command(),
			UUID:      t.ID(),
			CreatedAt: t.CreatedAt(),
			Active:    t.Active(),
		}

		tfl.AddSections([]component.FlexLayoutItem{
			{
				Width: component.WidthFull,
				View:  component.NewTerminal(t.Key().Namespace, t.Key().Name, details),
			},
		})

		extensionTab := component.ExtensionTab{
			Tab:          tfl,
			ClosePayload: action.CreatePayload("overview/deleteTerminal", action.Payload{"terminalID": t.ID()}),
		}

		extension.AddTab(extensionTab)
	}
	return extension, nil
}

// ActionPaths contain the actions this module is responsible for.
func (co *Overview) ActionPaths() map[string]action.DispatcherFunc {
	dispatchers := action.Dispatchers{
		octant.NewDeploymentConfigurationEditor(co.logger, co.dashConfig.ObjectStore()),
		octant.NewContainerEditor(co.dashConfig.ObjectStore()),
		octant.NewServiceConfigurationEditor(co.dashConfig.ObjectStore()),
		octant.NewPortForward(co.logger, co.dashConfig.ObjectStore(), co.dashConfig.PortForwarder()),
		octant.NewPortForwardDelete(co.logger, co.dashConfig.ObjectStore(), co.dashConfig.PortForwarder()),
		octant.NewTerminalCommandExec(co.logger, co.dashConfig.ObjectStore(), co.dashConfig.TerminalManager()),
		octant.NewTerminalDelete(co.logger, co.dashConfig.ObjectStore(), co.dashConfig.TerminalManager()),
		octant.NewCordon(co.dashConfig.ObjectStore(), co.dashConfig.ClusterClient()),
		octant.NewUncordon(co.dashConfig.ObjectStore(), co.dashConfig.ClusterClient()),
	}

	return dispatchers.ToActionPaths()
}
<|endoftext|>


<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
nYou may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mysqlindexer

import (
	"crypto/sha1"
	"io"
	"json"
	"log"
	"os"

	mysql "camli/third_party/github.com/Philio/GoMySQL"

	"camli/blobref"
	"camli/blobserver"
	"camli/magic"
	"camli/schema"
)

const maxSniffSize = 1024 * 16

type blobSniffer struct {
	header   []byte
	written  int64
	camli    *schema.Superset
	mimeType *string
}

func (sn *blobSniffer) Write(d []byte) (int, os.Error) {
	sn.written += int64(len(d))
	if len(sn.header) < maxSniffSize {
		n := maxSniffSize - len(sn.header)
		if len(d) < n {
			n = len(d)
		}
		sn.header = append(sn.header, d[:n]...)
	}
	return len(d), nil
}

func (sn *blobSniffer) IsTruncated() bool {
	return sn.written > maxSniffSize
}

// returns content type (string) or nil if unknown
func (sn *blobSniffer) MimeType() interface{} {
	if sn.mimeType != nil {
		return *sn.mimeType
	}
	return nil
}

func (sn *blobSniffer) Parse() {
	// Try to parse it as JSON
	// TODO: move this into the magic library?  Is the magic library Camli-specific
	// or to be upstreamed elsewhere?
	if sn.bufferIsCamliJson() {
		str := "application/json; camliType=" + sn.camli.Type
		sn.mimeType = &str
	}

	if mime := magic.MimeType(sn.header); mime != "" {
		sn.mimeType = &mime
	}
}

func (sn *blobSniffer) bufferIsCamliJson() bool {
	buf := sn.header
	if len(buf) < 2 || buf[0] != '{' {
		return false
	}
	camli := new(schema.Superset)
	err := json.Unmarshal(buf, camli)
	if err != nil {
		return false
	}
	sn.camli = camli
	return true
}

func (mi *Indexer) ReceiveBlob(blobRef *blobref.BlobRef, source io.Reader) (retsb blobref.SizedBlobRef, err os.Error) {
	sniffer := new(blobSniffer)
	hash := blobRef.Hash()
	var written int64
	written, err = io.Copy(io.MultiWriter(hash, sniffer), source)
	log.Printf("mysqlindexer: hashed+sniffed %d bytes; err %v", written, err)
	if err != nil {
		return
	}

	if !blobRef.HashMatches(hash) {
		err = blobserver.ErrCorruptBlob
		return
	}

	sniffer.Parse()
	mimeType := sniffer.MimeType()
	log.Printf("mysqlindexer: type=%v; truncated=%v", mimeType, sniffer.IsTruncated())

	var client *mysql.Client
	if client, err = mi.getConnection(); err != nil {
		return
	}
	defer mi.releaseConnection(client)

	var stmt *mysql.Statement
	if stmt, err = client.Prepare("INSERT IGNORE INTO blobs (blobref, size, type) VALUES (?, ?, ?)"); err != nil {
		log.Printf("mysqlindexer: prepare error: %v", err)
		return
	}
	if err = stmt.BindParams(blobRef.String(), written, mimeType); err != nil {
		log.Printf("mysqlindexer: bind error: %v", err)
		return
	}
	if err = stmt.Execute(); err != nil {
		log.Printf("mysqlindexer: execute error: %v", err)
		return
	}

	if camli := sniffer.camli; camli != nil {
		switch camli.Type {
		case "claim":
			if err = populateClaim(client, blobRef, camli); err != nil {
				return
			}
		case "permanode":
			if err = populatePermanode(client, blobRef, camli); err != nil {
				return
			}
		case "file":
			if err = mi.populateFile(client, blobRef, camli); err != nil {
				return
			}
		}
	}

	retsb = blobref.SizedBlobRef{BlobRef: blobRef, Size: written}
	return
}

func execSQL(client *mysql.Client, sql string, args ...interface{}) (err os.Error) {
	var stmt *mysql.Statement
	if stmt, err = client.Prepare(sql); err != nil {
		log.Printf("mysqlindexer execSQL prepare: %v", err)
		return
	}
	if err = stmt.BindParams(args...); err != nil {
		log.Printf("mysqlindexer execSQL bind: %v", err)
		return
	}
	if err = stmt.Execute(); err != nil {
		log.Printf("mysqlindexer execSQL exe: %v", err)
		return
	}
	return
}

func populateClaim(client *mysql.Client, blobRef *blobref.BlobRef, camli *schema.Superset) (err os.Error) {
	pnBlobref := blobref.Parse(camli.Permanode)
	if pnBlobref == nil {
		// Skip bogus claim with malformed permanode.
		return
	}

	if err = execSQL(client,
		"INSERT IGNORE INTO claims (blobref, signer, date, unverified, claim, permanode, attr, value) "+
			"VALUES (?, ?, ?, 'Y', ?, ?, ?, ?)",
		blobRef.String(), camli.Signer, camli.ClaimDate,
		camli.ClaimType, camli.Permanode,
		camli.Attribute, camli.Value); err != nil {
		return
	}

	// And update the lastmod on the permanode row.
	if err = execSQL(client,
		"INSERT IGNORE INTO permanodes (blobref) VALUES (?)",
		pnBlobref.String()); err != nil {
		return
	}
	if err = execSQL(client,
		"UPDATE permanodes SET lastmod=? WHERE blobref=? AND ? > lastmod",
		camli.ClaimDate, pnBlobref.String(), camli.ClaimDate); err != nil {
		return
	}

	return nil
}

func populatePermanode(client *mysql.Client, blobRef *blobref.BlobRef, camli *schema.Superset) (err os.Error) {
	err = execSQL(client,
		"INSERT IGNORE INTO permanodes (blobref, unverified, signer, lastmod) "+
			"VALUES (?, 'Y', ?, '')",
		blobRef.String(), camli.Signer)
	return
}

func (mi *Indexer) populateFile(client *mysql.Client, blobRef *blobref.BlobRef, ss *schema.Superset) (err os.Error) {
	if ss.Fragment {
		return nil
	}
	seekFetcher, err := blobref.SeekerFromStreamingFetcher(mi.BlobSource)
	if err != nil {
		return err
	}

	sha1 := sha1.New()
	fr := ss.NewFileReader(seekFetcher)
	n, err := io.Copy(sha1, fr)
	if err != nil {
		return err
	}
	log.Printf("file %s blobref is %s, size %d", blobRef, blobref.FromHash("sha1", sha1), n)
	err = execSQL(client,
		"INSERT IGNORE INTO files (fileschemaref, bytesref, size) VALUES (?, ?, ?)",
		blobRef.String(),
		blobref.FromHash("sha1", sha1).String(),
		n)
	return
}
<commit_msg>don't retry forever indexing files.<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
nYou may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mysqlindexer

import (
	"crypto/sha1"
	"io"
	"json"
	"log"
	"os"

	mysql "camli/third_party/github.com/Philio/GoMySQL"

	"camli/blobref"
	"camli/blobserver"
	"camli/magic"
	"camli/schema"
)

const maxSniffSize = 1024 * 16

type blobSniffer struct {
	header   []byte
	written  int64
	camli    *schema.Superset
	mimeType *string
}

func (sn *blobSniffer) Write(d []byte) (int, os.Error) {
	sn.written += int64(len(d))
	if len(sn.header) < maxSniffSize {
		n := maxSniffSize - len(sn.header)
		if len(d) < n {
			n = len(d)
		}
		sn.header = append(sn.header, d[:n]...)
	}
	return len(d), nil
}

func (sn *blobSniffer) IsTruncated() bool {
	return sn.written > maxSniffSize
}

// returns content type (string) or nil if unknown
func (sn *blobSniffer) MimeType() interface{} {
	if sn.mimeType != nil {
		return *sn.mimeType
	}
	return nil
}

func (sn *blobSniffer) Parse() {
	// Try to parse it as JSON
	// TODO: move this into the magic library?  Is the magic library Camli-specific
	// or to be upstreamed elsewhere?
	if sn.bufferIsCamliJson() {
		str := "application/json; camliType=" + sn.camli.Type
		sn.mimeType = &str
	}

	if mime := magic.MimeType(sn.header); mime != "" {
		sn.mimeType = &mime
	}
}

func (sn *blobSniffer) bufferIsCamliJson() bool {
	buf := sn.header
	if len(buf) < 2 || buf[0] != '{' {
		return false
	}
	camli := new(schema.Superset)
	err := json.Unmarshal(buf, camli)
	if err != nil {
		return false
	}
	sn.camli = camli
	return true
}

func (mi *Indexer) ReceiveBlob(blobRef *blobref.BlobRef, source io.Reader) (retsb blobref.SizedBlobRef, err os.Error) {
	sniffer := new(blobSniffer)
	hash := blobRef.Hash()
	var written int64
	written, err = io.Copy(io.MultiWriter(hash, sniffer), source)
	log.Printf("mysqlindexer: hashed+sniffed %d bytes; err %v", written, err)
	if err != nil {
		return
	}

	if !blobRef.HashMatches(hash) {
		err = blobserver.ErrCorruptBlob
		return
	}

	sniffer.Parse()
	mimeType := sniffer.MimeType()
	log.Printf("mysqlindexer: type=%v; truncated=%v", mimeType, sniffer.IsTruncated())

	var client *mysql.Client
	if client, err = mi.getConnection(); err != nil {
		return
	}
	defer mi.releaseConnection(client)

	var stmt *mysql.Statement
	if stmt, err = client.Prepare("INSERT IGNORE INTO blobs (blobref, size, type) VALUES (?, ?, ?)"); err != nil {
		log.Printf("mysqlindexer: prepare error: %v", err)
		return
	}
	if err = stmt.BindParams(blobRef.String(), written, mimeType); err != nil {
		log.Printf("mysqlindexer: bind error: %v", err)
		return
	}
	if err = stmt.Execute(); err != nil {
		log.Printf("mysqlindexer: execute error: %v", err)
		return
	}

	if camli := sniffer.camli; camli != nil {
		switch camli.Type {
		case "claim":
			if err = populateClaim(client, blobRef, camli); err != nil {
				return
			}
		case "permanode":
			if err = populatePermanode(client, blobRef, camli); err != nil {
				return
			}
		case "file":
			if err = mi.populateFile(client, blobRef, camli); err != nil {
				return
			}
		}
	}

	retsb = blobref.SizedBlobRef{BlobRef: blobRef, Size: written}
	return
}

func execSQL(client *mysql.Client, sql string, args ...interface{}) (err os.Error) {
	var stmt *mysql.Statement
	if stmt, err = client.Prepare(sql); err != nil {
		log.Printf("mysqlindexer execSQL prepare: %v", err)
		return
	}
	if err = stmt.BindParams(args...); err != nil {
		log.Printf("mysqlindexer execSQL bind: %v", err)
		return
	}
	if err = stmt.Execute(); err != nil {
		log.Printf("mysqlindexer execSQL exe: %v", err)
		return
	}
	return
}

func populateClaim(client *mysql.Client, blobRef *blobref.BlobRef, camli *schema.Superset) (err os.Error) {
	pnBlobref := blobref.Parse(camli.Permanode)
	if pnBlobref == nil {
		// Skip bogus claim with malformed permanode.
		return
	}

	if err = execSQL(client,
		"INSERT IGNORE INTO claims (blobref, signer, date, unverified, claim, permanode, attr, value) "+
			"VALUES (?, ?, ?, 'Y', ?, ?, ?, ?)",
		blobRef.String(), camli.Signer, camli.ClaimDate,
		camli.ClaimType, camli.Permanode,
		camli.Attribute, camli.Value); err != nil {
		return
	}

	// And update the lastmod on the permanode row.
	if err = execSQL(client,
		"INSERT IGNORE INTO permanodes (blobref) VALUES (?)",
		pnBlobref.String()); err != nil {
		return
	}
	if err = execSQL(client,
		"UPDATE permanodes SET lastmod=? WHERE blobref=? AND ? > lastmod",
		camli.ClaimDate, pnBlobref.String(), camli.ClaimDate); err != nil {
		return
	}

	return nil
}

func populatePermanode(client *mysql.Client, blobRef *blobref.BlobRef, camli *schema.Superset) (err os.Error) {
	err = execSQL(client,
		"INSERT IGNORE INTO permanodes (blobref, unverified, signer, lastmod) "+
			"VALUES (?, 'Y', ?, '')",
		blobRef.String(), camli.Signer)
	return
}

func (mi *Indexer) populateFile(client *mysql.Client, blobRef *blobref.BlobRef, ss *schema.Superset) (err os.Error) {
	if ss.Fragment {
		return nil
	}
	seekFetcher, err := blobref.SeekerFromStreamingFetcher(mi.BlobSource)
	if err != nil {
		return err
	}

	sha1 := sha1.New()
	fr := ss.NewFileReader(seekFetcher)
	n, err := io.Copy(sha1, fr)
	if err != nil {
		// TODO: job scheduling system to retry this spaced
		// out max n times.  Right now our options are
		// ignoring this error (forever) or returning the
		// error and making the indexing try again (likely
		// forever failing).  Both options suck.  For now just
		// log and act like all's okay.
		log.Printf("mysqlindex: error indexing file %s: %v", blobRef, err)
		return nil
	}
	log.Printf("file %s blobref is %s, size %d", blobRef, blobref.FromHash("sha1", sha1), n)
	err = execSQL(client,
		"INSERT IGNORE INTO files (fileschemaref, bytesref, size) VALUES (?, ?, ?)",
		blobRef.String(),
		blobref.FromHash("sha1", sha1).String(),
		n)
	return
}
<|endoftext|>


<commit_before>// Package profile is for specific profiles
// @todo this package is the definition of cruft and
// should be rewritten in a more elegant way
package profile

import (
	"fmt"
	"os"

	"github.com/micro/cli/v2"
	"github.com/micro/go-micro/v3/auth/jwt"
	"github.com/micro/go-micro/v3/auth/noop"
	"github.com/micro/go-micro/v3/broker"
	"github.com/micro/go-micro/v3/broker/http"
	"github.com/micro/go-micro/v3/client"
	"github.com/micro/go-micro/v3/config"
	memStream "github.com/micro/go-micro/v3/events/stream/memory"
	"github.com/micro/go-micro/v3/registry"
	"github.com/micro/go-micro/v3/registry/mdns"
	"github.com/micro/go-micro/v3/registry/memory"
	"github.com/micro/go-micro/v3/router"
	regRouter "github.com/micro/go-micro/v3/router/registry"
	"github.com/micro/go-micro/v3/router/static"
	"github.com/micro/go-micro/v3/runtime/local"
	"github.com/micro/go-micro/v3/server"
	"github.com/micro/go-micro/v3/store/file"
	mem "github.com/micro/go-micro/v3/store/memory"
	"github.com/micro/micro/v3/service/logger"

	inAuth "github.com/micro/micro/v3/internal/auth"
	"github.com/micro/micro/v3/internal/user"
	microAuth "github.com/micro/micro/v3/service/auth"
	microBroker "github.com/micro/micro/v3/service/broker"
	microClient "github.com/micro/micro/v3/service/client"
	microConfig "github.com/micro/micro/v3/service/config"
	microEvents "github.com/micro/micro/v3/service/events"
	microRegistry "github.com/micro/micro/v3/service/registry"
	microRouter "github.com/micro/micro/v3/service/router"
	microRuntime "github.com/micro/micro/v3/service/runtime"
	microServer "github.com/micro/micro/v3/service/server"
	microStore "github.com/micro/micro/v3/service/store"
)

// profiles which when called will configure micro to run in that environment
var profiles = map[string]*Profile{
	// built in profiles
	"client":     Client,
	"service":    Service,
	"test":       Test,
	"local":      Local,
	"kubernetes": Kubernetes,
}

// Profile configures an environment
type Profile struct {
	// name of the profile
	Name string
	// function used for setup
	Setup func(*cli.Context) error
	// TODO: presetup dependencies
	// e.g start resources
}

// Register a profile
func Register(name string, p *Profile) error {
	if _, ok := profiles[name]; ok {
		return fmt.Errorf("profile %s already exists", name)
	}
	profiles[name] = p
	return nil
}

// Load a profile
func Load(name string) (*Profile, error) {
	v, ok := profiles[name]
	if !ok {
		return nil, fmt.Errorf("profile %s does not exist", name)
	}
	return v, nil
}

// Client profile is for any entrypoint that behaves as a client
var Client = &Profile{
	Name:  "client",
	Setup: func(ctx *cli.Context) error { return nil },
}

// Local profile to run locally
var Local = &Profile{
	Name: "local",
	Setup: func(ctx *cli.Context) error {
		microAuth.DefaultAuth = jwt.NewAuth()
		microRuntime.DefaultRuntime = local.NewRuntime()
		microStore.DefaultStore = file.NewStore()
		microConfig.DefaultConfig, _ = config.NewConfig()
		SetupBroker(http.NewBroker())
		SetupRegistry(mdns.NewRegistry())
		SetupJWT(ctx)

		var err error
		microEvents.DefaultStream, err = memStream.NewStream()
		if err != nil {
			logger.Fatalf("Error configuring stream: %v", err)
		}

		return nil
	},
}

// Kubernetes profile to run on kubernetes
var Kubernetes = &Profile{
	Name: "kubernetes",
	Setup: func(ctx *cli.Context) error {
		// TODO: implement
		// using a static router so queries are routed based on service name
		microRouter.DefaultRouter = static.NewRouter()
		// registry kubernetes
		// config configmap
		// store ...
		microAuth.DefaultAuth = jwt.NewAuth()
		SetupJWT(ctx)

		return nil
	},
}

// Service is the default for any services run
var Service = &Profile{
	Name:  "service",
	Setup: func(ctx *cli.Context) error { return nil },
}

// Test profile is used for the go test suite
var Test = &Profile{
	Name: "test",
	Setup: func(ctx *cli.Context) error {
		microAuth.DefaultAuth = noop.NewAuth()
		microStore.DefaultStore = mem.NewStore()
		microConfig.DefaultConfig, _ = config.NewConfig()
		SetupRegistry(memory.NewRegistry())
		return nil
	},
}

// SetupRegistry configures the registry
func SetupRegistry(reg registry.Registry) {
	microRegistry.DefaultRegistry = reg
	microRouter.DefaultRouter = regRouter.NewRouter(router.Registry(reg))
	microServer.DefaultServer.Init(server.Registry(reg))
	microClient.DefaultClient.Init(client.Registry(reg))
}

// SetupBroker configures the broker
func SetupBroker(b broker.Broker) {
	microBroker.DefaultBroker = b
	microClient.DefaultClient.Init(client.Broker(b))
	microServer.DefaultServer.Init(server.Broker(b))
}

// SetupJWTRules configures the default internal system rules
func SetupJWT(ctx *cli.Context) {
	for _, rule := range inAuth.SystemRules {
		if err := microAuth.DefaultAuth.Grant(rule); err != nil {
			logger.Fatal("Error creating default rule: %v", err)
		}
	}
	// Only set this up for core services
	// Won't work for multi node environments, could use
	// the file store for that.

	pubKey := ctx.String("auth_public_key")
	privKey := ctx.String("auth_private_key")
	if len(privKey) == 0 || len(pubKey) == 0 {
		privB, pubB, err := user.GetJWTCerts()
		if err != nil {
			logger.Fatalf("Error getting keys; %v", err)
		}
		os.Setenv("MICRO_AUTH_PRIVATE_KEY", string(privB))
		os.Setenv("MICRO_AUTH_PUBLIC_KEY", string(pubB))
	}

}
<commit_msg>Set k8s runtime in k8s profile (#1339)<commit_after>// Package profile is for specific profiles
// @todo this package is the definition of cruft and
// should be rewritten in a more elegant way
package profile

import (
	"fmt"
	"os"

	"github.com/micro/cli/v2"
	"github.com/micro/go-micro/v3/auth/jwt"
	"github.com/micro/go-micro/v3/auth/noop"
	"github.com/micro/go-micro/v3/broker"
	"github.com/micro/go-micro/v3/broker/http"
	"github.com/micro/go-micro/v3/client"
	"github.com/micro/go-micro/v3/config"
	memStream "github.com/micro/go-micro/v3/events/stream/memory"
	"github.com/micro/go-micro/v3/registry"
	"github.com/micro/go-micro/v3/registry/mdns"
	"github.com/micro/go-micro/v3/registry/memory"
	"github.com/micro/go-micro/v3/router"
	regRouter "github.com/micro/go-micro/v3/router/registry"
	"github.com/micro/go-micro/v3/router/static"
	"github.com/micro/go-micro/v3/runtime/kubernetes"
	"github.com/micro/go-micro/v3/runtime/local"
	"github.com/micro/go-micro/v3/server"
	"github.com/micro/go-micro/v3/store/file"
	mem "github.com/micro/go-micro/v3/store/memory"
	"github.com/micro/micro/v3/service/logger"

	inAuth "github.com/micro/micro/v3/internal/auth"
	"github.com/micro/micro/v3/internal/user"
	microAuth "github.com/micro/micro/v3/service/auth"
	microBroker "github.com/micro/micro/v3/service/broker"
	microClient "github.com/micro/micro/v3/service/client"
	microConfig "github.com/micro/micro/v3/service/config"
	microEvents "github.com/micro/micro/v3/service/events"
	microRegistry "github.com/micro/micro/v3/service/registry"
	microRouter "github.com/micro/micro/v3/service/router"
	microRuntime "github.com/micro/micro/v3/service/runtime"
	microServer "github.com/micro/micro/v3/service/server"
	microStore "github.com/micro/micro/v3/service/store"
)

// profiles which when called will configure micro to run in that environment
var profiles = map[string]*Profile{
	// built in profiles
	"client":     Client,
	"service":    Service,
	"test":       Test,
	"local":      Local,
	"kubernetes": Kubernetes,
}

// Profile configures an environment
type Profile struct {
	// name of the profile
	Name string
	// function used for setup
	Setup func(*cli.Context) error
	// TODO: presetup dependencies
	// e.g start resources
}

// Register a profile
func Register(name string, p *Profile) error {
	if _, ok := profiles[name]; ok {
		return fmt.Errorf("profile %s already exists", name)
	}
	profiles[name] = p
	return nil
}

// Load a profile
func Load(name string) (*Profile, error) {
	v, ok := profiles[name]
	if !ok {
		return nil, fmt.Errorf("profile %s does not exist", name)
	}
	return v, nil
}

// Client profile is for any entrypoint that behaves as a client
var Client = &Profile{
	Name:  "client",
	Setup: func(ctx *cli.Context) error { return nil },
}

// Local profile to run locally
var Local = &Profile{
	Name: "local",
	Setup: func(ctx *cli.Context) error {
		microAuth.DefaultAuth = jwt.NewAuth()
		microRuntime.DefaultRuntime = local.NewRuntime()
		microStore.DefaultStore = file.NewStore()
		microConfig.DefaultConfig, _ = config.NewConfig()
		SetupBroker(http.NewBroker())
		SetupRegistry(mdns.NewRegistry())
		SetupJWT(ctx)

		var err error
		microEvents.DefaultStream, err = memStream.NewStream()
		if err != nil {
			logger.Fatalf("Error configuring stream: %v", err)
		}

		return nil
	},
}

// Kubernetes profile to run on kubernetes
var Kubernetes = &Profile{
	Name: "kubernetes",
	Setup: func(ctx *cli.Context) error {
		// TODO: implement
		// using a static router so queries are routed based on service name
		microRouter.DefaultRouter = static.NewRouter()
		// Using the kubernetes runtime
		microRuntime.DefaultRuntime = kubernetes.NewRuntime()
		// registry kubernetes
		// config configmap
		// store ...
		microAuth.DefaultAuth = jwt.NewAuth()
		SetupJWT(ctx)

		return nil
	},
}

// Service is the default for any services run
var Service = &Profile{
	Name:  "service",
	Setup: func(ctx *cli.Context) error { return nil },
}

// Test profile is used for the go test suite
var Test = &Profile{
	Name: "test",
	Setup: func(ctx *cli.Context) error {
		microAuth.DefaultAuth = noop.NewAuth()
		microStore.DefaultStore = mem.NewStore()
		microConfig.DefaultConfig, _ = config.NewConfig()
		SetupRegistry(memory.NewRegistry())
		return nil
	},
}

// SetupRegistry configures the registry
func SetupRegistry(reg registry.Registry) {
	microRegistry.DefaultRegistry = reg
	microRouter.DefaultRouter = regRouter.NewRouter(router.Registry(reg))
	microServer.DefaultServer.Init(server.Registry(reg))
	microClient.DefaultClient.Init(client.Registry(reg))
}

// SetupBroker configures the broker
func SetupBroker(b broker.Broker) {
	microBroker.DefaultBroker = b
	microClient.DefaultClient.Init(client.Broker(b))
	microServer.DefaultServer.Init(server.Broker(b))
}

// SetupJWTRules configures the default internal system rules
func SetupJWT(ctx *cli.Context) {
	for _, rule := range inAuth.SystemRules {
		if err := microAuth.DefaultAuth.Grant(rule); err != nil {
			logger.Fatal("Error creating default rule: %v", err)
		}
	}
	// Only set this up for core services
	// Won't work for multi node environments, could use
	// the file store for that.

	pubKey := ctx.String("auth_public_key")
	privKey := ctx.String("auth_private_key")
	if len(privKey) == 0 || len(pubKey) == 0 {
		privB, pubB, err := user.GetJWTCerts()
		if err != nil {
			logger.Fatalf("Error getting keys; %v", err)
		}
		os.Setenv("MICRO_AUTH_PRIVATE_KEY", string(privB))
		os.Setenv("MICRO_AUTH_PUBLIC_KEY", string(pubB))
	}

}
<|endoftext|>


<commit_before>package shared

import (
	"bufio"
	"fmt"
	"os"
	"os/exec"
	"os/user"
	"path"
	"path/filepath"
	"strconv"
	"strings"
)

/*
 * One entry in id mapping set - a single range of either
 * uid or gid mappings.
 */
type IdmapEntry struct {
	Isuid    bool
	Isgid    bool
	Hostid   int // id as seen on the host - i.e. 100000
	Nsid     int // id as seen in the ns - i.e. 0
	Maprange int
}

func (e *IdmapEntry) ToLxcString() string {
	if e.Isuid {
		return fmt.Sprintf("u %d %d %d", e.Nsid, e.Hostid, e.Maprange)
	}
	return fmt.Sprintf("g %d %d %d", e.Nsid, e.Hostid, e.Maprange)
}

func is_between(x, low, high int) bool {
	return x >= low && x < high
}

func (e *IdmapEntry) Intersects(i IdmapEntry) bool {
	if (e.Isuid && i.Isuid) || (e.Isgid && i.Isgid) {
		switch {
		case is_between(e.Hostid, i.Hostid, i.Hostid+i.Maprange):
			return true
		case is_between(i.Hostid, e.Hostid, e.Hostid+e.Maprange):
			return true
		case is_between(e.Hostid+e.Maprange, i.Hostid, i.Hostid+i.Maprange):
			return true
		case is_between(i.Hostid+e.Maprange, e.Hostid, e.Hostid+e.Maprange):
			return true
		case is_between(e.Nsid, i.Nsid, i.Nsid+i.Maprange):
			return true
		case is_between(i.Nsid, e.Nsid, e.Nsid+e.Maprange):
			return true
		case is_between(e.Nsid+e.Maprange, i.Nsid, i.Nsid+i.Maprange):
			return true
		case is_between(i.Nsid+e.Maprange, e.Nsid, e.Nsid+e.Maprange):
			return true
		}
	}
	return false
}

func (e *IdmapEntry) parse(s string) error {
	split := strings.Split(s, ":")
	var err error
	if len(split) != 4 {
		return fmt.Errorf("Bad idmap: %q", s)
	}
	switch split[0] {
	case "u":
		e.Isuid = true
	case "g":
		e.Isgid = true
	case "b":
		e.Isuid = true
		e.Isgid = true
	default:
		return fmt.Errorf("Bad idmap type in %q", s)
	}
	e.Nsid, err = strconv.Atoi(split[1])
	if err != nil {
		return err
	}
	e.Hostid, err = strconv.Atoi(split[2])
	if err != nil {
		return err
	}
	e.Maprange, err = strconv.Atoi(split[3])
	if err != nil {
		return err
	}

	// wraparound
	if e.Hostid+e.Maprange < e.Hostid || e.Nsid+e.Maprange < e.Nsid {
		return fmt.Errorf("Bad mapping: id wraparound")
	}

	return nil
}

/*
 * Shift a uid from the host into the container
 * I.e. 0 -> 1000 -> 101000
 */
func (e *IdmapEntry) shift_into_ns(id int) (int, error) {
	if id < e.Nsid || id >= e.Nsid+e.Maprange {
		// this mapping doesn't apply
		return 0, fmt.Errorf("N/A")
	}

	return id - e.Nsid + e.Hostid, nil
}

/*
 * Shift a uid from the container back to the host
 * I.e. 101000 -> 1000
 */
func (e *IdmapEntry) shift_from_ns(id int) (int, error) {
	if id < e.Hostid || id >= e.Hostid+e.Maprange {
		// this mapping doesn't apply
		return 0, fmt.Errorf("N/A")
	}

	return id - e.Hostid + e.Nsid, nil
}

/* taken from http://blog.golang.org/slices (which is under BSD licence) */
func Extend(slice []IdmapEntry, element IdmapEntry) []IdmapEntry {
	n := len(slice)
	if n == cap(slice) {
		// Slice is full; must grow.
		// We double its size and add 1, so if the size is zero we still grow.
		newSlice := make([]IdmapEntry, len(slice), 2*len(slice)+1)
		copy(newSlice, slice)
		slice = newSlice
	}
	slice = slice[0 : n+1]
	slice[n] = element
	return slice
}

type IdmapSet struct {
	Idmap []IdmapEntry
}

func (m IdmapSet) Len() int {
	return len(m.Idmap)
}

func (m IdmapSet) Intersects(i IdmapEntry) bool {
	for _, e := range m.Idmap {
		if i.Intersects(e) {
			return true
		}
	}
	return false
}

func (m IdmapSet) ToLxcString() []string {
	var lines []string
	for _, e := range m.Idmap {
		lines = append(lines, e.ToLxcString()+"\n")
	}
	return lines
}

func (m IdmapSet) Append(s string) (IdmapSet, error) {
	e := IdmapEntry{}
	err := e.parse(s)
	if err != nil {
		return m, err
	}
	if m.Intersects(e) {
		return m, fmt.Errorf("Conflicting id mapping")
	}
	m.Idmap = Extend(m.Idmap, e)
	return m, nil
}

func (m IdmapSet) doShiftIntoNs(uid int, gid int, how string) (int, int) {
	u := -1
	g := -1
	for _, e := range m.Idmap {
		var err error
		var tmpu, tmpg int
		if e.Isuid && u == -1 {
			switch how {
			case "in":
				tmpu, err = e.shift_into_ns(uid)
			case "out":
				tmpu, err = e.shift_from_ns(uid)
			}
			if err == nil {
				u = tmpu
			}
		}
		if e.Isgid && g == -1 {
			switch how {
			case "in":
				tmpg, err = e.shift_into_ns(gid)
			case "out":
				tmpg, err = e.shift_from_ns(gid)
			}
			if err == nil {
				g = tmpg
			}
		}
	}

	return u, g
}

func (m IdmapSet) ShiftIntoNs(uid int, gid int) (int, int) {
	return m.doShiftIntoNs(uid, gid, "in")
}

func (m IdmapSet) ShiftFromNs(uid int, gid int) (int, int) {
	return m.doShiftIntoNs(uid, gid, "out")
}

func GetOwner(path string) (int, int, error) {
	uid, gid, _, _, _, _, err := GetFileStat(path)
	return uid, gid, err
}

func (set *IdmapSet) doUidshiftIntoContainer(dir string, testmode bool, how string) error {
	convert := func(path string, fi os.FileInfo, err error) (e error) {
		uid, gid, err := GetOwner(path)
		if err != nil {
			return err
		}
		var newuid, newgid int
		switch how {
		case "in":
			newuid, newgid = set.ShiftIntoNs(uid, gid)
		case "out":
			newuid, newgid = set.ShiftFromNs(uid, gid)
		}
		if testmode {
			fmt.Printf("I would shift %q to %d %d\n", path, newuid, newgid)
		} else {
			err = os.Lchown(path, int(newuid), int(newgid))
			if err == nil {
				m := fi.Mode()
				if m&os.ModeSymlink == 0 {
					err = os.Chmod(path, m)
					if err != nil {
						fmt.Printf("Error resetting mode on %q, continuing\n", path)
					}
				}
			}
		}
		return nil
	}

	if !PathExists(dir) {
		return fmt.Errorf("No such file or directory: %q", dir)
	}
	return filepath.Walk(dir, convert)
}

func (set *IdmapSet) UidshiftIntoContainer(dir string, testmode bool) error {
	return set.doUidshiftIntoContainer(dir, testmode, "in")
}

func (set *IdmapSet) UidshiftFromContainer(dir string, testmode bool) error {
	return set.doUidshiftIntoContainer(dir, testmode, "out")
}

func (set *IdmapSet) ShiftRootfs(p string) error {
	return set.doUidshiftIntoContainer(p, false, "in")
}

func (set *IdmapSet) UnshiftRootfs(p string) error {
	return set.doUidshiftIntoContainer(p, false, "out")
}

const (
	minIDRange = 65536
)

/*
 * get a uid or gid mapping from /etc/subxid
 */
func getFromMap(fname string, username string) (int, int, error) {
	f, err := os.Open(fname)
	var min int
	var idrange int
	if err != nil {
		return 0, 0, err
	}
	defer f.Close()
	scanner := bufio.NewScanner(f)
	min = 0
	idrange = 0
	for scanner.Scan() {
		/*
		 * /etc/sub{gu}id allow comments in the files, so ignore
		 * everything after a '#'
		 */
		s := strings.Split(scanner.Text(), "#")
		if len(s[0]) == 0 {
			continue
		}

		s = strings.Split(s[0], ":")
		if len(s) < 3 {
			return 0, 0, fmt.Errorf("unexpected values in %q: %q", fname, s)
		}
		if strings.EqualFold(s[0], username) {
			bigmin, err := strconv.ParseUint(s[1], 10, 32)
			if err != nil {
				continue
			}
			bigIdrange, err := strconv.ParseUint(s[2], 10, 32)
			if err != nil {
				continue
			}
			min = int(bigmin)
			idrange = int(bigIdrange)
			return min, idrange, nil
		}
	}

	return 0, 0, fmt.Errorf("User %q has no %ss.", username, path.Base(fname))
}

/*
 * Create a new default idmap
 */
func DefaultIdmapSet() (*IdmapSet, error) {
	me, err := user.Current()
	if err == nil {
		username = me.Username
        /* user.Current() requires cgo */
	} else {
		username = os.Getenv("USER")
		if username == "" {
			return nil, err
		}
	}

	umin := 1000000
	urange := 100000
	gmin := 1000000
	grange := 100000

	newuidmap, _ := exec.LookPath("newuidmap")
	newgidmap, _ := exec.LookPath("newgidmap")

	if newuidmap != "" && newgidmap != "" {
		umin, urange, err = getFromMap("/etc/subuid", username)
		if err != nil {
			return nil, err
		}

		gmin, grange, err = getFromMap("/etc/subgid", username)
		if err != nil {
			return nil, err
		}
	}

	if urange < minIDRange {
		return nil, fmt.Errorf("uidrange less than %d", minIDRange)
	}

	if grange < minIDRange {
		return nil, fmt.Errorf("gidrange less than %d", minIDRange)
	}

	m := new(IdmapSet)

	e := IdmapEntry{Isuid: true, Nsid: 0, Hostid: umin, Maprange: urange}
	m.Idmap = Extend(m.Idmap, e)
	e = IdmapEntry{Isgid: true, Nsid: 0, Hostid: gmin, Maprange: grange}
	m.Idmap = Extend(m.Idmap, e)

	return m, nil
}
<commit_msg>Move username getter into separate function<commit_after>package shared

import (
	"bufio"
	"fmt"
	"os"
	"os/exec"
	"os/user"
	"path"
	"path/filepath"
	"strconv"
	"strings"
)

/*
 * One entry in id mapping set - a single range of either
 * uid or gid mappings.
 */
type IdmapEntry struct {
	Isuid    bool
	Isgid    bool
	Hostid   int // id as seen on the host - i.e. 100000
	Nsid     int // id as seen in the ns - i.e. 0
	Maprange int
}

func (e *IdmapEntry) ToLxcString() string {
	if e.Isuid {
		return fmt.Sprintf("u %d %d %d", e.Nsid, e.Hostid, e.Maprange)
	}
	return fmt.Sprintf("g %d %d %d", e.Nsid, e.Hostid, e.Maprange)
}

func is_between(x, low, high int) bool {
	return x >= low && x < high
}

func (e *IdmapEntry) Intersects(i IdmapEntry) bool {
	if (e.Isuid && i.Isuid) || (e.Isgid && i.Isgid) {
		switch {
		case is_between(e.Hostid, i.Hostid, i.Hostid+i.Maprange):
			return true
		case is_between(i.Hostid, e.Hostid, e.Hostid+e.Maprange):
			return true
		case is_between(e.Hostid+e.Maprange, i.Hostid, i.Hostid+i.Maprange):
			return true
		case is_between(i.Hostid+e.Maprange, e.Hostid, e.Hostid+e.Maprange):
			return true
		case is_between(e.Nsid, i.Nsid, i.Nsid+i.Maprange):
			return true
		case is_between(i.Nsid, e.Nsid, e.Nsid+e.Maprange):
			return true
		case is_between(e.Nsid+e.Maprange, i.Nsid, i.Nsid+i.Maprange):
			return true
		case is_between(i.Nsid+e.Maprange, e.Nsid, e.Nsid+e.Maprange):
			return true
		}
	}
	return false
}

func (e *IdmapEntry) parse(s string) error {
	split := strings.Split(s, ":")
	var err error
	if len(split) != 4 {
		return fmt.Errorf("Bad idmap: %q", s)
	}
	switch split[0] {
	case "u":
		e.Isuid = true
	case "g":
		e.Isgid = true
	case "b":
		e.Isuid = true
		e.Isgid = true
	default:
		return fmt.Errorf("Bad idmap type in %q", s)
	}
	e.Nsid, err = strconv.Atoi(split[1])
	if err != nil {
		return err
	}
	e.Hostid, err = strconv.Atoi(split[2])
	if err != nil {
		return err
	}
	e.Maprange, err = strconv.Atoi(split[3])
	if err != nil {
		return err
	}

	// wraparound
	if e.Hostid+e.Maprange < e.Hostid || e.Nsid+e.Maprange < e.Nsid {
		return fmt.Errorf("Bad mapping: id wraparound")
	}

	return nil
}

/*
 * Shift a uid from the host into the container
 * I.e. 0 -> 1000 -> 101000
 */
func (e *IdmapEntry) shift_into_ns(id int) (int, error) {
	if id < e.Nsid || id >= e.Nsid+e.Maprange {
		// this mapping doesn't apply
		return 0, fmt.Errorf("N/A")
	}

	return id - e.Nsid + e.Hostid, nil
}

/*
 * Shift a uid from the container back to the host
 * I.e. 101000 -> 1000
 */
func (e *IdmapEntry) shift_from_ns(id int) (int, error) {
	if id < e.Hostid || id >= e.Hostid+e.Maprange {
		// this mapping doesn't apply
		return 0, fmt.Errorf("N/A")
	}

	return id - e.Hostid + e.Nsid, nil
}

/* taken from http://blog.golang.org/slices (which is under BSD licence) */
func Extend(slice []IdmapEntry, element IdmapEntry) []IdmapEntry {
	n := len(slice)
	if n == cap(slice) {
		// Slice is full; must grow.
		// We double its size and add 1, so if the size is zero we still grow.
		newSlice := make([]IdmapEntry, len(slice), 2*len(slice)+1)
		copy(newSlice, slice)
		slice = newSlice
	}
	slice = slice[0 : n+1]
	slice[n] = element
	return slice
}

type IdmapSet struct {
	Idmap []IdmapEntry
}

func (m IdmapSet) Len() int {
	return len(m.Idmap)
}

func (m IdmapSet) Intersects(i IdmapEntry) bool {
	for _, e := range m.Idmap {
		if i.Intersects(e) {
			return true
		}
	}
	return false
}

func (m IdmapSet) ToLxcString() []string {
	var lines []string
	for _, e := range m.Idmap {
		lines = append(lines, e.ToLxcString()+"\n")
	}
	return lines
}

func (m IdmapSet) Append(s string) (IdmapSet, error) {
	e := IdmapEntry{}
	err := e.parse(s)
	if err != nil {
		return m, err
	}
	if m.Intersects(e) {
		return m, fmt.Errorf("Conflicting id mapping")
	}
	m.Idmap = Extend(m.Idmap, e)
	return m, nil
}

func (m IdmapSet) doShiftIntoNs(uid int, gid int, how string) (int, int) {
	u := -1
	g := -1
	for _, e := range m.Idmap {
		var err error
		var tmpu, tmpg int
		if e.Isuid && u == -1 {
			switch how {
			case "in":
				tmpu, err = e.shift_into_ns(uid)
			case "out":
				tmpu, err = e.shift_from_ns(uid)
			}
			if err == nil {
				u = tmpu
			}
		}
		if e.Isgid && g == -1 {
			switch how {
			case "in":
				tmpg, err = e.shift_into_ns(gid)
			case "out":
				tmpg, err = e.shift_from_ns(gid)
			}
			if err == nil {
				g = tmpg
			}
		}
	}

	return u, g
}

func (m IdmapSet) ShiftIntoNs(uid int, gid int) (int, int) {
	return m.doShiftIntoNs(uid, gid, "in")
}

func (m IdmapSet) ShiftFromNs(uid int, gid int) (int, int) {
	return m.doShiftIntoNs(uid, gid, "out")
}

func GetOwner(path string) (int, int, error) {
	uid, gid, _, _, _, _, err := GetFileStat(path)
	return uid, gid, err
}

func (set *IdmapSet) doUidshiftIntoContainer(dir string, testmode bool, how string) error {
	convert := func(path string, fi os.FileInfo, err error) (e error) {
		uid, gid, err := GetOwner(path)
		if err != nil {
			return err
		}
		var newuid, newgid int
		switch how {
		case "in":
			newuid, newgid = set.ShiftIntoNs(uid, gid)
		case "out":
			newuid, newgid = set.ShiftFromNs(uid, gid)
		}
		if testmode {
			fmt.Printf("I would shift %q to %d %d\n", path, newuid, newgid)
		} else {
			err = os.Lchown(path, int(newuid), int(newgid))
			if err == nil {
				m := fi.Mode()
				if m&os.ModeSymlink == 0 {
					err = os.Chmod(path, m)
					if err != nil {
						fmt.Printf("Error resetting mode on %q, continuing\n", path)
					}
				}
			}
		}
		return nil
	}

	if !PathExists(dir) {
		return fmt.Errorf("No such file or directory: %q", dir)
	}
	return filepath.Walk(dir, convert)
}

func (set *IdmapSet) UidshiftIntoContainer(dir string, testmode bool) error {
	return set.doUidshiftIntoContainer(dir, testmode, "in")
}

func (set *IdmapSet) UidshiftFromContainer(dir string, testmode bool) error {
	return set.doUidshiftIntoContainer(dir, testmode, "out")
}

func (set *IdmapSet) ShiftRootfs(p string) error {
	return set.doUidshiftIntoContainer(p, false, "in")
}

func (set *IdmapSet) UnshiftRootfs(p string) error {
	return set.doUidshiftIntoContainer(p, false, "out")
}

const (
	minIDRange = 65536
)

/*
 * get a uid or gid mapping from /etc/subxid
 */
func getFromMap(fname string, username string) (int, int, error) {
	f, err := os.Open(fname)
	var min int
	var idrange int
	if err != nil {
		return 0, 0, err
	}
	defer f.Close()
	scanner := bufio.NewScanner(f)
	min = 0
	idrange = 0
	for scanner.Scan() {
		/*
		 * /etc/sub{gu}id allow comments in the files, so ignore
		 * everything after a '#'
		 */
		s := strings.Split(scanner.Text(), "#")
		if len(s[0]) == 0 {
			continue
		}

		s = strings.Split(s[0], ":")
		if len(s) < 3 {
			return 0, 0, fmt.Errorf("unexpected values in %q: %q", fname, s)
		}
		if strings.EqualFold(s[0], username) {
			bigmin, err := strconv.ParseUint(s[1], 10, 32)
			if err != nil {
				continue
			}
			bigIdrange, err := strconv.ParseUint(s[2], 10, 32)
			if err != nil {
				continue
			}
			min = int(bigmin)
			idrange = int(bigIdrange)
			return min, idrange, nil
		}
	}

	return 0, 0, fmt.Errorf("User %q has no %ss.", username, path.Base(fname))
}

/*
 * Get current username
 */
func getUsername() (string, error) {
	me, err := user.Current()
	if err == nil {
		return me.Username, nil
	} else {
		/* user.Current() requires cgo */
		username := os.Getenv("USER")
		if username == "" {
			return "", err
		}
		return username, nil
	}
}

/*
 * Create a new default idmap
 */
func DefaultIdmapSet() (*IdmapSet, error) {
	myname, err := getUsername()
	if err != nil {
		return nil, err
	}

	umin := 1000000
	urange := 100000
	gmin := 1000000
	grange := 100000

	newuidmap, _ := exec.LookPath("newuidmap")
	newgidmap, _ := exec.LookPath("newgidmap")

	if newuidmap != "" && newgidmap != "" {
		umin, urange, err = getFromMap("/etc/subuid", myname)
		if err != nil {
			return nil, err
		}

		gmin, grange, err = getFromMap("/etc/subgid", myname)
		if err != nil {
			return nil, err
		}
	}

	if urange < minIDRange {
		return nil, fmt.Errorf("uidrange less than %d", minIDRange)
	}

	if grange < minIDRange {
		return nil, fmt.Errorf("gidrange less than %d", minIDRange)
	}

	m := new(IdmapSet)

	e := IdmapEntry{Isuid: true, Nsid: 0, Hostid: umin, Maprange: urange}
	m.Idmap = Extend(m.Idmap, e)
	e = IdmapEntry{Isgid: true, Nsid: 0, Hostid: gmin, Maprange: grange}
	m.Idmap = Extend(m.Idmap, e)

	return m, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The following enables go generate to generate the doc.go file.
//go:generate go run $JIRI_ROOT/release/go/src/v.io/x/lib/cmdline/testdata/gendoc.go .

package main

import (
	"bytes"
	"fmt"
	"net/http"
	"net/url"
	"path"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/googleapi"
	storage "google.golang.org/api/storage/v1"

	"v.io/jiri/collect"
	"v.io/jiri/retry"
	"v.io/jiri/tool"
	"v.io/x/devtools/vbinary/exitcode"
	"v.io/x/lib/cmdline"
)

var (
	archFlag             string
	attemptsFlag         int
	datePrefixFlag       string
	keyFileFlag          string
	osFlag               string
	outputDirFlag        string
	releaseFlag          bool
	maxParallelDownloads int

	waitTimeBetweenAttempts = 3 * time.Minute
)

const (
	binariesBucketName        = "vanadium-binaries"
	releaseBinariesBucketName = "vanadium-release"
	gceUser                   = "veyron"
)

func bucketName() string {
	if releaseFlag {
		return releaseBinariesBucketName
	}
	return binariesBucketName
}

func dateLayout() string {
	if releaseFlag {
		return "2006-01-02.15:04"
	}
	return "2006-01-02T15:04:05-07:00"
}

func osArchDir() string {
	if releaseFlag {
		return ""
	}
	return fmt.Sprintf("%s_%s", osFlag, archFlag)
}

func stripOsArchDir(name string) string {
	if releaseFlag {
		return name
	}
	return strings.Split(name, "/")[1]
}

// TODO(suharshs): Add tests that mock out google.Storage.

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	cmdRoot.Flags.BoolVar(&releaseFlag, "release", false, "Operate on vanadium-release bucket instead of vanadium-binaries.")

	cmdRoot.Flags.StringVar(&archFlag, "arch", runtime.GOARCH, "Target architecture.  The default is the value of runtime.GOARCH.")
	cmdRoot.Flags.Lookup("arch").DefValue = "<runtime.GOARCH>"
	cmdRoot.Flags.StringVar(&osFlag, "os", runtime.GOOS, "Target operating system.  The default is the value of runtime.GOOS.")
	cmdRoot.Flags.Lookup("os").DefValue = "<runtime.GOOS>"

	cmdRoot.Flags.StringVar(&keyFileFlag, "key-file", "", "Google Developers service account JSON key file.")
	cmdRoot.Flags.StringVar(&datePrefixFlag, "date-prefix", "", "Date prefix to match daily build timestamps. Must be a prefix of YYYY-MM-DD.")
	cmdDownload.Flags.IntVar(&attemptsFlag, "attempts", 1, "Number of attempts before failing.")
	cmdDownload.Flags.StringVar(&outputDirFlag, "output-dir", "", "Directory for storing downloaded binaries.")
	cmdDownload.Flags.IntVar(&maxParallelDownloads, "max-parallel-downloads", 8, "Maximum number of downloads that can happen at the same time.")

	tool.InitializeRunFlags(&cmdRoot.Flags)
}

func main() {
	cmdline.Main(cmdRoot)
}

// cmdRoot represents the "vbinary" command.
var cmdRoot = &cmdline.Command{
	Name:  "vbinary",
	Short: "Access daily builds of Vanadium binaries",
	Long: `

Command vbinary retrieves daily builds of Vanadium binaries stored in
a Google Storage bucket.
`,
	Children: []*cmdline.Command{cmdList, cmdDownload},
}

// cmdList represents the "vbinary list" command.
var cmdList = &cmdline.Command{
	Runner: cmdline.RunnerFunc(runList),
	Name:   "list",
	Short:  "List existing daily builds of Vanadium binaries",
	Long: `
List existing daily builds of Vanadium binaries. The displayed dates
can be limited with the --date-prefix flag. An exit code of 3 indicates
that no snapshot was found.
`,
}

func runList(env *cmdline.Env, _ []string) error {
	ctx := tool.NewContextFromEnv(env)
	client, err := createClient(ctx)
	if err != nil {
		return err
	}
	service, err := storage.New(client)
	if err != nil {
		return err
	}
	binaries, err := binarySnapshots(ctx, service)
	if err != nil {
		return err
	}
	for _, name := range binaries {
		fmt.Fprintf(ctx.Stdout(), "%s\n", name)
	}
	return nil
}

// cmdDownload represents the "vbinary download" command.
var cmdDownload = &cmdline.Command{
	Runner: cmdline.RunnerFunc(runDownload),
	Name:   "download",
	Short:  "Download an existing daily build of Vanadium binaries",
	Long: `
Download an existing daily build of Vanadium binaries. The latest
snapshot within the --date-prefix range will be downloaded. If no
--date-prefix flag is provided, the overall latest snapshot will be
downloaded. An exit code of 3 indicates that no snapshot was found.
`,
}

func runDownload(env *cmdline.Env, args []string) error {
	ctx := tool.NewContextFromEnv(env)
	s := ctx.NewSeq()
	client, err := createClient(ctx)
	if err != nil {
		return err
	}

	binaries, timestamp, err := latestBinaries(ctx, client)
	if err != nil {
		return err
	}
	if len(outputDirFlag) == 0 {
		outputDirFlag = fmt.Sprintf("./v23_%s_%s_%s", osFlag, archFlag, timestamp)
	}
	if err := s.MkdirAll(outputDirFlag, 0755).Done(); err != nil {
		return err
	}

	numBinaries := len(binaries)
	downloadBinaries := func() error {
		downloadFn := func() error {
			errChan := make(chan error, numBinaries)
			downloadingChan := make(chan struct{}, maxParallelDownloads)
			for _, name := range binaries {
				downloadingChan <- struct{}{}
				go downloadBinary(ctx, client, name, errChan, downloadingChan)
			}
			gotError := false
			for i := 0; i < numBinaries; i++ {
				if err := <-errChan; err != nil {
					fmt.Fprintf(ctx.Stderr(), "failed to download binary: %v", err)
					gotError = true
				}
			}
			if gotError {
				if err := ctx.NewSeq().RemoveAll(outputDirFlag).Done(); err != nil {
					fmt.Fprintf(ctx.Stderr(), "%v", err)
				}
				return fmt.Errorf("Failed to download some binaries")
			}
			return nil
		}
		if err := retry.Function(ctx, downloadFn, retry.AttemptsOpt(attemptsFlag), retry.IntervalOpt(waitTimeBetweenAttempts)); err != nil {
			return fmt.Errorf("operation failed")
		}
		// Remove the .done file from the snapshot.
		if err := ctx.NewSeq().RemoveAll(path.Join(outputDirFlag, ".done")).Done(); err != nil {
			return err
		}
		return nil
	}
	return s.Call(downloadBinaries, "Downloading binaries to %s", outputDirFlag).Done()
}

// latestBinaries returns the binaries of the latest snapshot whose timestamp
// matches the datePrefixFlag, along with the matching timestamp.
func latestBinaries(ctx *tool.Context, client *http.Client) ([]string, string, error) {
	service, err := storage.New(client)
	if err != nil {
		return nil, "", err
	}
	timestamp, err := latestTimestamp(ctx, client, service)
	if err != nil {
		return nil, "", err
	}
	binaryPrefix := path.Join(osArchDir(), timestamp)
	res, err := service.Objects.List(bucketName()).Fields("nextPageToken", "items/name").Prefix(binaryPrefix).Do()
	if err != nil {
		return nil, "", err
	}
	objs := res.Items
	for res.NextPageToken != "" {
		res, err = service.Objects.List(bucketName()).PageToken(res.NextPageToken).Do()
		if err != nil {
			return nil, "", err
		}
		objs = append(objs, res.Items...)
	}
	if len(objs) == 0 {
		return nil, "", fmt.Errorf("no binaries found (OS: %s, Arch: %s, Date: %s)", osFlag, archFlag, timestamp)
	}
	ret := make([]string, len(objs))
	for i, obj := range objs {
		ret[i] = obj.Name
	}
	return ret, timestamp, nil
}

// latestTimestamp returns the time of the latest snapshot within the
// date-prefix range.
func latestTimestamp(ctx *tool.Context, client *http.Client, service *storage.Service) (string, error) {
	// If no datePrefixFlag is provided, we just want to get the latest snapshot.
	if datePrefixFlag == "" {
		latestFile := path.Join(osArchDir(), "latest")
		b, err := downloadFileBytes(client, latestFile)
		if err != nil {
			return "", err
		}
		return string(b), nil
	}
	// Otherwise, we get the snapshots that match datePrefixFlag and choose the latest.
	snapshots, err := binarySnapshots(ctx, service)
	if err != nil {
		return "", err
	}
	layout := dateLayout()
	var latest string
	var latestTime time.Time
	for _, name := range snapshots {
		timestamp := stripOsArchDir(name)
		t, err := time.Parse(layout, timestamp)
		if err != nil {
			return "", err
		}
		if t.After(latestTime) {
			latest = timestamp
			latestTime = t
		}
	}
	return latest, nil
}

func binarySnapshots(ctx *tool.Context, service *storage.Service) ([]string, error) {
	filterSnapshots := func(call *storage.ObjectsListCall) (*storage.Objects, error) {
		binaryPrefix := path.Join(osArchDir(), datePrefixFlag)
		// We delimit results by the ".done" file to ensure that only successfully completed snapshots are considered.
		return call.Fields("nextPageToken", "prefixes").Prefix(binaryPrefix).Delimiter("/.done").Do()
	}
	res, err := filterSnapshots(service.Objects.List(bucketName()))
	if err != nil {
		return nil, err
	}
	snapshots := res.Prefixes
	for res.NextPageToken != "" {
		res, err = filterSnapshots(service.Objects.List(bucketName()).PageToken(res.NextPageToken))
		if err != nil {
			return nil, err
		}
		snapshots = append(snapshots, res.Prefixes...)
	}
	if len(snapshots) == 0 {
		fmt.Fprintf(ctx.Stderr(), "no snapshots found (OS: %s, Arch: %s, Date: %s)\n", osFlag, archFlag, datePrefixFlag)
		return nil, cmdline.ErrExitCode(exitcode.NoSnapshotExitCode)
	}
	ret := make([]string, len(snapshots))
	for i, snapshot := range snapshots {
		ret[i] = strings.TrimSuffix(snapshot, "/.done")
	}
	return ret, nil
}

func createClient(ctx *tool.Context) (*http.Client, error) {
	if len(keyFileFlag) > 0 {
		data, err := ctx.NewSeq().ReadFile(keyFileFlag)
		if err != nil {
			return nil, err
		}
		conf, err := google.JWTConfigFromJSON(data, storage.CloudPlatformScope)
		if err != nil {
			return nil, fmt.Errorf("failed to create JWT config file: %v", err)
		}
		return conf.Client(oauth2.NoContext), nil
	}

	var defaultClient *http.Client
	createDefaultClientFn := func() error {
		var err error
		defaultClient, err = google.DefaultClient(oauth2.NoContext, storage.CloudPlatformScope)
		if err != nil {
			return err
		}
		return nil
	}
	if err := retry.Function(ctx, createDefaultClientFn); err != nil {
		return nil, fmt.Errorf("failed to create default client")
	}
	return defaultClient, nil
}

func downloadBinary(ctx *tool.Context, client *http.Client, binaryPath string, errChan chan<- error, downloadingChan chan struct{}) {
	helper := func() error {
		b, err := downloadFileBytes(client, binaryPath)
		if err != nil {
			return fmt.Errorf("failed to download file %v: %v", binaryPath, err)
		}
		fileName := filepath.Join(outputDirFlag, path.Base(binaryPath))
		if err := ctx.NewSeq().WriteFile(fileName, b, 0755).Done(); err != nil {
			return err
		}
		return nil
	}
	errChan <- helper()
	<-downloadingChan
}

func downloadFileBytes(client *http.Client, filePath string) (b []byte, e error) {
	// This roundabout request is required because of the issue detailed here:
	// https://plus.sandbox.google.com/+IanRose/posts/Tzw3QZqEQZk
	// and here:
	// https://groups.google.com/forum/#!msg/Golang-nuts/juguXl-ss2Q/oOVFvHYqoSgJ.
	urls := "https://www.googleapis.com/download/storage/v1/b/{bucket}/o/{object}?alt=media"
	req, err := http.NewRequest("GET", urls, nil)
	if err != nil {
		return nil, fmt.Errorf("failed to create request to %s: %v\n", urls, err)
	}
	req.URL.Path = strings.Replace(req.URL.Path, "{bucket}", url.QueryEscape(bucketName()), 1)
	req.URL.Path = strings.Replace(req.URL.Path, "{object}", url.QueryEscape(filePath), 1)
	googleapi.SetOpaque(req.URL)
	res, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("failed to download %v: %v\n", req.URL.RequestURI(), err)
	}
	if res.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("got StatusCode %v for download %v", req.URL.RequestURI(), res.StatusCode)
	}
	defer collect.Error(func() error { return res.Body.Close() }, &e)

	var buf bytes.Buffer
	if _, err := buf.ReadFrom(res.Body); err != nil {
		return nil, fmt.Errorf("failed to read response body: %v", err)
	}
	return buf.Bytes(), nil
}
<commit_msg>vbinary: Fix retry loop<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The following enables go generate to generate the doc.go file.
//go:generate go run $JIRI_ROOT/release/go/src/v.io/x/lib/cmdline/testdata/gendoc.go .

package main

import (
	"bytes"
	"fmt"
	"net/http"
	"net/url"
	"path"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/googleapi"
	storage "google.golang.org/api/storage/v1"

	"v.io/jiri/collect"
	"v.io/jiri/retry"
	"v.io/jiri/tool"
	"v.io/x/devtools/vbinary/exitcode"
	"v.io/x/lib/cmdline"
)

var (
	archFlag             string
	attemptsFlag         int
	datePrefixFlag       string
	keyFileFlag          string
	osFlag               string
	outputDirFlag        string
	releaseFlag          bool
	maxParallelDownloads int

	waitTimeBetweenAttempts = 3 * time.Minute
)

const (
	binariesBucketName        = "vanadium-binaries"
	releaseBinariesBucketName = "vanadium-release"
	gceUser                   = "veyron"
)

func bucketName() string {
	if releaseFlag {
		return releaseBinariesBucketName
	}
	return binariesBucketName
}

func dateLayout() string {
	if releaseFlag {
		return "2006-01-02.15:04"
	}
	return "2006-01-02T15:04:05-07:00"
}

func osArchDir() string {
	if releaseFlag {
		return ""
	}
	return fmt.Sprintf("%s_%s", osFlag, archFlag)
}

func stripOsArchDir(name string) string {
	if releaseFlag {
		return name
	}
	return strings.Split(name, "/")[1]
}

// TODO(suharshs): Add tests that mock out google.Storage.

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	cmdRoot.Flags.BoolVar(&releaseFlag, "release", false, "Operate on vanadium-release bucket instead of vanadium-binaries.")

	cmdRoot.Flags.StringVar(&archFlag, "arch", runtime.GOARCH, "Target architecture.  The default is the value of runtime.GOARCH.")
	cmdRoot.Flags.Lookup("arch").DefValue = "<runtime.GOARCH>"
	cmdRoot.Flags.StringVar(&osFlag, "os", runtime.GOOS, "Target operating system.  The default is the value of runtime.GOOS.")
	cmdRoot.Flags.Lookup("os").DefValue = "<runtime.GOOS>"

	cmdRoot.Flags.StringVar(&keyFileFlag, "key-file", "", "Google Developers service account JSON key file.")
	cmdRoot.Flags.StringVar(&datePrefixFlag, "date-prefix", "", "Date prefix to match daily build timestamps. Must be a prefix of YYYY-MM-DD.")
	cmdDownload.Flags.IntVar(&attemptsFlag, "attempts", 1, "Number of attempts before failing.")
	cmdDownload.Flags.StringVar(&outputDirFlag, "output-dir", "", "Directory for storing downloaded binaries.")
	cmdDownload.Flags.IntVar(&maxParallelDownloads, "max-parallel-downloads", 8, "Maximum number of downloads that can happen at the same time.")

	tool.InitializeRunFlags(&cmdRoot.Flags)
}

func main() {
	cmdline.Main(cmdRoot)
}

// cmdRoot represents the "vbinary" command.
var cmdRoot = &cmdline.Command{
	Name:  "vbinary",
	Short: "Access daily builds of Vanadium binaries",
	Long: `

Command vbinary retrieves daily builds of Vanadium binaries stored in
a Google Storage bucket.
`,
	Children: []*cmdline.Command{cmdList, cmdDownload},
}

// cmdList represents the "vbinary list" command.
var cmdList = &cmdline.Command{
	Runner: cmdline.RunnerFunc(runList),
	Name:   "list",
	Short:  "List existing daily builds of Vanadium binaries",
	Long: `
List existing daily builds of Vanadium binaries. The displayed dates
can be limited with the --date-prefix flag. An exit code of 3 indicates
that no snapshot was found.
`,
}

func runList(env *cmdline.Env, _ []string) error {
	ctx := tool.NewContextFromEnv(env)
	client, err := createClient(ctx)
	if err != nil {
		return err
	}
	service, err := storage.New(client)
	if err != nil {
		return err
	}
	binaries, err := binarySnapshots(ctx, service)
	if err != nil {
		return err
	}
	for _, name := range binaries {
		fmt.Fprintf(ctx.Stdout(), "%s\n", name)
	}
	return nil
}

// cmdDownload represents the "vbinary download" command.
var cmdDownload = &cmdline.Command{
	Runner: cmdline.RunnerFunc(runDownload),
	Name:   "download",
	Short:  "Download an existing daily build of Vanadium binaries",
	Long: `
Download an existing daily build of Vanadium binaries. The latest
snapshot within the --date-prefix range will be downloaded. If no
--date-prefix flag is provided, the overall latest snapshot will be
downloaded. An exit code of 3 indicates that no snapshot was found.
`,
}

func runDownload(env *cmdline.Env, args []string) error {
	ctx := tool.NewContextFromEnv(env)
	s := ctx.NewSeq()
	client, err := createClient(ctx)
	if err != nil {
		return err
	}

	binaries, timestamp, err := latestBinaries(ctx, client)
	if err != nil {
		return err
	}
	if len(outputDirFlag) == 0 {
		outputDirFlag = fmt.Sprintf("./v23_%s_%s_%s", osFlag, archFlag, timestamp)
	}

	numBinaries := len(binaries)
	downloadBinaries := func() error {
		downloadFn := func() error {
			if err := ctx.NewSeq().MkdirAll(outputDirFlag, 0755).Done(); err != nil {
				return err
			}
			errChan := make(chan error, numBinaries)
			downloadingChan := make(chan struct{}, maxParallelDownloads)
			for _, name := range binaries {
				downloadingChan <- struct{}{}
				go downloadBinary(ctx, client, name, errChan, downloadingChan)
			}
			gotError := false
			for i := 0; i < numBinaries; i++ {
				if err := <-errChan; err != nil {
					fmt.Fprintf(ctx.Stderr(), "failed to download binary: %v\n", err)
					gotError = true
				}
			}
			if gotError {
				if err := ctx.NewSeq().RemoveAll(outputDirFlag).Done(); err != nil {
					fmt.Fprintf(ctx.Stderr(), "%v", err)
				}
				return fmt.Errorf("Failed to download some binaries")
			}
			return nil
		}
		if err := retry.Function(ctx, downloadFn, retry.AttemptsOpt(attemptsFlag), retry.IntervalOpt(waitTimeBetweenAttempts)); err != nil {
			return fmt.Errorf("operation failed")
		}
		// Remove the .done file from the snapshot.
		if err := ctx.NewSeq().RemoveAll(path.Join(outputDirFlag, ".done")).Done(); err != nil {
			return err
		}
		return nil
	}
	return s.Call(downloadBinaries, "Downloading binaries to %s", outputDirFlag).Done()
}

// latestBinaries returns the binaries of the latest snapshot whose timestamp
// matches the datePrefixFlag, along with the matching timestamp.
func latestBinaries(ctx *tool.Context, client *http.Client) ([]string, string, error) {
	service, err := storage.New(client)
	if err != nil {
		return nil, "", err
	}
	timestamp, err := latestTimestamp(ctx, client, service)
	if err != nil {
		return nil, "", err
	}
	binaryPrefix := path.Join(osArchDir(), timestamp)
	res, err := service.Objects.List(bucketName()).Fields("nextPageToken", "items/name").Prefix(binaryPrefix).Do()
	if err != nil {
		return nil, "", err
	}
	objs := res.Items
	for res.NextPageToken != "" {
		res, err = service.Objects.List(bucketName()).PageToken(res.NextPageToken).Do()
		if err != nil {
			return nil, "", err
		}
		objs = append(objs, res.Items...)
	}
	if len(objs) == 0 {
		return nil, "", fmt.Errorf("no binaries found (OS: %s, Arch: %s, Date: %s)", osFlag, archFlag, timestamp)
	}
	ret := make([]string, len(objs))
	for i, obj := range objs {
		ret[i] = obj.Name
	}
	return ret, timestamp, nil
}

// latestTimestamp returns the time of the latest snapshot within the
// date-prefix range.
func latestTimestamp(ctx *tool.Context, client *http.Client, service *storage.Service) (string, error) {
	// If no datePrefixFlag is provided, we just want to get the latest snapshot.
	if datePrefixFlag == "" {
		latestFile := path.Join(osArchDir(), "latest")
		b, err := downloadFileBytes(client, latestFile)
		if err != nil {
			return "", err
		}
		return string(b), nil
	}
	// Otherwise, we get the snapshots that match datePrefixFlag and choose the latest.
	snapshots, err := binarySnapshots(ctx, service)
	if err != nil {
		return "", err
	}
	layout := dateLayout()
	var latest string
	var latestTime time.Time
	for _, name := range snapshots {
		timestamp := stripOsArchDir(name)
		t, err := time.Parse(layout, timestamp)
		if err != nil {
			return "", err
		}
		if t.After(latestTime) {
			latest = timestamp
			latestTime = t
		}
	}
	return latest, nil
}

func binarySnapshots(ctx *tool.Context, service *storage.Service) ([]string, error) {
	filterSnapshots := func(call *storage.ObjectsListCall) (*storage.Objects, error) {
		binaryPrefix := path.Join(osArchDir(), datePrefixFlag)
		// We delimit results by the ".done" file to ensure that only successfully completed snapshots are considered.
		return call.Fields("nextPageToken", "prefixes").Prefix(binaryPrefix).Delimiter("/.done").Do()
	}
	res, err := filterSnapshots(service.Objects.List(bucketName()))
	if err != nil {
		return nil, err
	}
	snapshots := res.Prefixes
	for res.NextPageToken != "" {
		res, err = filterSnapshots(service.Objects.List(bucketName()).PageToken(res.NextPageToken))
		if err != nil {
			return nil, err
		}
		snapshots = append(snapshots, res.Prefixes...)
	}
	if len(snapshots) == 0 {
		fmt.Fprintf(ctx.Stderr(), "no snapshots found (OS: %s, Arch: %s, Date: %s)\n", osFlag, archFlag, datePrefixFlag)
		return nil, cmdline.ErrExitCode(exitcode.NoSnapshotExitCode)
	}
	ret := make([]string, len(snapshots))
	for i, snapshot := range snapshots {
		ret[i] = strings.TrimSuffix(snapshot, "/.done")
	}
	return ret, nil
}

func createClient(ctx *tool.Context) (*http.Client, error) {
	if len(keyFileFlag) > 0 {
		data, err := ctx.NewSeq().ReadFile(keyFileFlag)
		if err != nil {
			return nil, err
		}
		conf, err := google.JWTConfigFromJSON(data, storage.CloudPlatformScope)
		if err != nil {
			return nil, fmt.Errorf("failed to create JWT config file: %v", err)
		}
		return conf.Client(oauth2.NoContext), nil
	}

	var defaultClient *http.Client
	createDefaultClientFn := func() error {
		var err error
		defaultClient, err = google.DefaultClient(oauth2.NoContext, storage.CloudPlatformScope)
		if err != nil {
			return err
		}
		return nil
	}
	if err := retry.Function(ctx, createDefaultClientFn); err != nil {
		return nil, fmt.Errorf("failed to create default client")
	}
	return defaultClient, nil
}

func downloadBinary(ctx *tool.Context, client *http.Client, binaryPath string, errChan chan<- error, downloadingChan chan struct{}) {
	helper := func() error {
		b, err := downloadFileBytes(client, binaryPath)
		if err != nil {
			return fmt.Errorf("failed to download file %v: %v", binaryPath, err)
		}
		fileName := filepath.Join(outputDirFlag, path.Base(binaryPath))
		if err := ctx.NewSeq().WriteFile(fileName, b, 0755).Done(); err != nil {
			return err
		}
		return nil
	}
	errChan <- helper()
	<-downloadingChan
}

func downloadFileBytes(client *http.Client, filePath string) (b []byte, e error) {
	// This roundabout request is required because of the issue detailed here:
	// https://plus.sandbox.google.com/+IanRose/posts/Tzw3QZqEQZk
	// and here:
	// https://groups.google.com/forum/#!msg/Golang-nuts/juguXl-ss2Q/oOVFvHYqoSgJ.
	urls := "https://www.googleapis.com/download/storage/v1/b/{bucket}/o/{object}?alt=media"
	req, err := http.NewRequest("GET", urls, nil)
	if err != nil {
		return nil, fmt.Errorf("failed to create request to %s: %v\n", urls, err)
	}
	req.URL.Path = strings.Replace(req.URL.Path, "{bucket}", url.QueryEscape(bucketName()), 1)
	req.URL.Path = strings.Replace(req.URL.Path, "{object}", url.QueryEscape(filePath), 1)
	googleapi.SetOpaque(req.URL)
	res, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("failed to download %v: %v\n", req.URL.RequestURI(), err)
	}
	if res.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("got StatusCode %v for download %v", req.URL.RequestURI(), res.StatusCode)
	}
	defer collect.Error(func() error { return res.Body.Close() }, &e)

	var buf bytes.Buffer
	if _, err := buf.ReadFrom(res.Body); err != nil {
		return nil, fmt.Errorf("failed to read response body: %v", err)
	}
	return buf.Bytes(), nil
}
<|endoftext|>