<commit_before>package main

import (
	"archive/zip"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/tormoder/fit/cmd/fitgen/internal/fitstringer"
	"github.com/tormoder/fit/cmd/fitgen/internal/profile"
)

const (
	workbookNameXLS  = "Profile.xls"
	workbookNameXLSX = "Profile.xlsx"
)

func main() {
	sdkOverride := flag.String(
		"sdk",
		"",
		"provide or override SDK version printed in generated code",
	)
	timestamp := flag.Bool(
		"timestamp",
		false,
		"add generation timestamp to generated code",
	)
	runTests := flag.Bool(
		"test",
		false,
		"run all tests in fit repository after code has been generated",
	)
	verbose := flag.Bool(
		"verbose",
		false,
		"print verbose debugging output for profile parsing and code generation",
	)

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "usage: fitgen [flags] [path to sdk zip, xls or xlsx file] [output directory]\n")
		flag.PrintDefaults()
	}

	flag.Parse()
	if flag.NArg() != 2 {
		flag.Usage()
		os.Exit(2)
	}

	l := log.New(os.Stdout, "fitgen:\t", 0)

	fitSrcDir := flag.Arg(1)
	l.Println("fit source output directory:", fitSrcDir)

	var (
		messagesOut    = filepath.Join(fitSrcDir, "messages.go")
		typesOut       = filepath.Join(fitSrcDir, "types.go")
		profileOut     = filepath.Join(fitSrcDir, "profile.go")
		typesStringOut = filepath.Join(fitSrcDir, "types_string.go")
	)

	var (
		inputData []byte
		input     = flag.Arg(0)
		inputExt  = filepath.Ext(input)
		err       error
	)

	switch inputExt {
	case ".zip":
		inputData, err = readDataFromZIP(input)
	case ".xls", ".xlsx":
		inputData, err = readDataFromXLSX(input)
		if *sdkOverride == "" {
			log.Fatal("-sdk flag required if input is .xls(x)")
		}
	default:
		l.Fatalln("input file must be of type [.zip | .xls | .xlsx], got:", inputExt)
	}
	if err != nil {
		l.Fatal(err)
	}

	genOptions := []profile.GeneratorOption{
		profile.WithGenerationTimestamp(*timestamp),
		profile.WithLogger(l),
	}
	if *verbose {
		genOptions = append(genOptions, profile.WithDebugOutput())
	}

	var sdkString string
	if *sdkOverride != "" {
		sdkString = *sdkOverride
	} else {
		sdkString = parseSDKVersionStringFromZipFilePath(input)
	}

	sdkMaj, sdkMin, err := parseMajorAndMinorSDKVersion(sdkString)
	if err != nil {
		l.Fatalln("error parsing sdk version:", err)
	}

	generator, err := profile.NewGenerator(sdkMaj, sdkMin, inputData, genOptions...)
	if err != nil {
		l.Fatal(err)
	}

	fitProfile, err := generator.GenerateProfile()
	if err != nil {
		l.Fatal(err)
	}

	if err = ioutil.WriteFile(typesOut, fitProfile.TypesSource, 0644); err != nil {
		l.Fatalf("typegen: error writing types output file: %v", err)
	}

	if err = ioutil.WriteFile(messagesOut, fitProfile.MessagesSource, 0644); err != nil {
		l.Fatalf("typegen: error writing messages output file: %v", err)
	}

	if err = ioutil.WriteFile(profileOut, fitProfile.ProfileSource, 0644); err != nil {
		l.Fatalf("typegen: error writing profile output file: %v", err)
	}

	l.Println("fitstringer: generating string methods for types")
	err = runStringerOnTypes(typesOut, typesStringOut, fitProfile.StringerInput)
	if err != nil {
		l.Fatal(err)
	}
	l.Println("fitstringer: done")

	logMesgNumVsMessages(fitProfile.MesgNumsWithoutMessage, l)

	if *runTests {
		err = runAllTests(fitSrcDir)
		if err != nil {
			l.Fatal(err)
		}
		l.Println("go test: pass")
	}

	l.Println("done")
}

func runStringerOnTypes(typesIn, typesStringOut string, fitTypes []string) error {
	output, err := fitstringer.Generate(fitTypes, typesIn)
	if err != nil {
		return fmt.Errorf("fitstringer: generation failed: %v", err)
	}

	if err := ioutil.WriteFile(typesStringOut, output, 0644); err != nil {
		return fmt.Errorf("error writing fitstringer output: %v", err)
	}

	return nil
}

func runAllTests(pkgDir string) error {
	goTestArgs := []string{"test", pkgDir}
	testCmd := exec.Command("go", goTestArgs...)
	output, err := testCmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("go test: fail: %v\n%s", err, output)
	}

	return nil
}

func logMesgNumVsMessages(msgs []string, l *log.Logger) {
	if len(msgs) == 0 {
		return
	}
	l.Println("mesgnum-vs-msgs: implementation detail below, this may be automated in the future")
	l.Println("mesgnum-vs-msgs: #mesgnum values != #generated messages, diff:", len(msgs))
	l.Println("mesgnum-vs-msgs: remember to add/verify map entries for sdk in sdk.go for the following message(s):")
	for _, msg := range msgs {
		l.Printf("mesgnum-vs-msgs: ----> mesgnum %q has no corresponding message\n", msg)
	}
}

func readDataFromZIP(path string) ([]byte, error) {
	r, err := zip.OpenReader(path)
	if err != nil {
		return nil, fmt.Errorf("error opening sdk zip file: %v", err)
	}
	defer r.Close()

	var wfile *zip.File
	for _, f := range r.File {
		if f.Name == workbookNameXLS {
			wfile = f
			break
		}
		if f.Name == workbookNameXLSX {
			wfile = f
			break
		}
	}
	if wfile == nil {
		return nil, fmt.Errorf(
			"no file named %q or %q found in zip archive",
			workbookNameXLS, workbookNameXLSX,
		)
	}

	rc, err := wfile.Open()
	if err != nil {
		return nil, fmt.Errorf("error opening zip archive: %v", err)
	}
	defer rc.Close()

	data, err := ioutil.ReadAll(rc)
	if err != nil {
		return nil, fmt.Errorf("error reading %q from archive: %v", wfile.Name, err)
	}

	return data, nil
}

func readDataFromXLSX(path string) ([]byte, error) {
	return ioutil.ReadFile(path)
}

func parseSDKVersionStringFromZipFilePath(path string) string {
	_, file := filepath.Split(path)
	ver := strings.TrimSuffix(file, ".zip")
	return strings.TrimPrefix(ver, "FitSDKRelease_")
}

func parseMajorAndMinorSDKVersion(sdkString string) (int, int, error) {
	splitted := strings.Split(sdkString, ".")
	if len(splitted) < 2 {
		return 0, 0, fmt.Errorf("could not parse major/minor version from input: %q", sdkString)
	}

	maj, err := strconv.Atoi(splitted[0])
	if err != nil {
		return 0, 0, fmt.Errorf("could not parse major version from input: %q", splitted[0])
	}

	min, err := strconv.Atoi(splitted[1])
	if err != nil {
		return 0, 0, fmt.Errorf("could not parse minor version from input: %q", splitted[1])
	}

	return maj, min, nil
}
<commit_msg>cmd/fitgen: update description for 'test' flag<commit_after>package main

import (
	"archive/zip"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/tormoder/fit/cmd/fitgen/internal/fitstringer"
	"github.com/tormoder/fit/cmd/fitgen/internal/profile"
)

const (
	workbookNameXLS  = "Profile.xls"
	workbookNameXLSX = "Profile.xlsx"
)

func main() {
	sdkOverride := flag.String(
		"sdk",
		"",
		"provide or override SDK version printed in generated code",
	)
	timestamp := flag.Bool(
		"timestamp",
		false,
		"add generation timestamp to generated code",
	)
	runTests := flag.Bool(
		"test",
		false,
		"run all tests in output directory after code has been generated",
	)
	verbose := flag.Bool(
		"verbose",
		false,
		"print verbose debugging output for profile parsing and code generation",
	)

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "usage: fitgen [flags] [path to sdk zip, xls or xlsx file] [output directory]\n")
		flag.PrintDefaults()
	}

	flag.Parse()
	if flag.NArg() != 2 {
		flag.Usage()
		os.Exit(2)
	}

	l := log.New(os.Stdout, "fitgen:\t", 0)

	fitSrcDir := flag.Arg(1)
	l.Println("fit source output directory:", fitSrcDir)

	var (
		messagesOut    = filepath.Join(fitSrcDir, "messages.go")
		typesOut       = filepath.Join(fitSrcDir, "types.go")
		profileOut     = filepath.Join(fitSrcDir, "profile.go")
		typesStringOut = filepath.Join(fitSrcDir, "types_string.go")
	)

	var (
		inputData []byte
		input     = flag.Arg(0)
		inputExt  = filepath.Ext(input)
		err       error
	)

	switch inputExt {
	case ".zip":
		inputData, err = readDataFromZIP(input)
	case ".xls", ".xlsx":
		inputData, err = readDataFromXLSX(input)
		if *sdkOverride == "" {
			log.Fatal("-sdk flag required if input is .xls(x)")
		}
	default:
		l.Fatalln("input file must be of type [.zip | .xls | .xlsx], got:", inputExt)
	}
	if err != nil {
		l.Fatal(err)
	}

	genOptions := []profile.GeneratorOption{
		profile.WithGenerationTimestamp(*timestamp),
		profile.WithLogger(l),
	}
	if *verbose {
		genOptions = append(genOptions, profile.WithDebugOutput())
	}

	var sdkString string
	if *sdkOverride != "" {
		sdkString = *sdkOverride
	} else {
		sdkString = parseSDKVersionStringFromZipFilePath(input)
	}

	sdkMaj, sdkMin, err := parseMajorAndMinorSDKVersion(sdkString)
	if err != nil {
		l.Fatalln("error parsing sdk version:", err)
	}

	generator, err := profile.NewGenerator(sdkMaj, sdkMin, inputData, genOptions...)
	if err != nil {
		l.Fatal(err)
	}

	fitProfile, err := generator.GenerateProfile()
	if err != nil {
		l.Fatal(err)
	}

	if err = ioutil.WriteFile(typesOut, fitProfile.TypesSource, 0644); err != nil {
		l.Fatalf("typegen: error writing types output file: %v", err)
	}

	if err = ioutil.WriteFile(messagesOut, fitProfile.MessagesSource, 0644); err != nil {
		l.Fatalf("typegen: error writing messages output file: %v", err)
	}

	if err = ioutil.WriteFile(profileOut, fitProfile.ProfileSource, 0644); err != nil {
		l.Fatalf("typegen: error writing profile output file: %v", err)
	}

	l.Println("fitstringer: generating string methods for types")
	err = runStringerOnTypes(typesOut, typesStringOut, fitProfile.StringerInput)
	if err != nil {
		l.Fatal(err)
	}
	l.Println("fitstringer: done")

	logMesgNumVsMessages(fitProfile.MesgNumsWithoutMessage, l)

	if *runTests {
		err = runAllTests(fitSrcDir)
		if err != nil {
			l.Fatal(err)
		}
		l.Println("go test: pass")
	}

	l.Println("done")
}

func runStringerOnTypes(typesIn, typesStringOut string, fitTypes []string) error {
	output, err := fitstringer.Generate(fitTypes, typesIn)
	if err != nil {
		return fmt.Errorf("fitstringer: generation failed: %v", err)
	}

	if err := ioutil.WriteFile(typesStringOut, output, 0644); err != nil {
		return fmt.Errorf("error writing fitstringer output: %v", err)
	}

	return nil
}

func runAllTests(pkgDir string) error {
	goTestArgs := []string{"test", pkgDir}
	testCmd := exec.Command("go", goTestArgs...)
	output, err := testCmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("go test: fail: %v\n%s", err, output)
	}

	return nil
}

func logMesgNumVsMessages(msgs []string, l *log.Logger) {
	if len(msgs) == 0 {
		return
	}
	l.Println("mesgnum-vs-msgs: implementation detail below, this may be automated in the future")
	l.Println("mesgnum-vs-msgs: #mesgnum values != #generated messages, diff:", len(msgs))
	l.Println("mesgnum-vs-msgs: remember to add/verify map entries for sdk in sdk.go for the following message(s):")
	for _, msg := range msgs {
		l.Printf("mesgnum-vs-msgs: ----> mesgnum %q has no corresponding message\n", msg)
	}
}

func readDataFromZIP(path string) ([]byte, error) {
	r, err := zip.OpenReader(path)
	if err != nil {
		return nil, fmt.Errorf("error opening sdk zip file: %v", err)
	}
	defer r.Close()

	var wfile *zip.File
	for _, f := range r.File {
		if f.Name == workbookNameXLS {
			wfile = f
			break
		}
		if f.Name == workbookNameXLSX {
			wfile = f
			break
		}
	}
	if wfile == nil {
		return nil, fmt.Errorf(
			"no file named %q or %q found in zip archive",
			workbookNameXLS, workbookNameXLSX,
		)
	}

	rc, err := wfile.Open()
	if err != nil {
		return nil, fmt.Errorf("error opening zip archive: %v", err)
	}
	defer rc.Close()

	data, err := ioutil.ReadAll(rc)
	if err != nil {
		return nil, fmt.Errorf("error reading %q from archive: %v", wfile.Name, err)
	}

	return data, nil
}

func readDataFromXLSX(path string) ([]byte, error) {
	return ioutil.ReadFile(path)
}

func parseSDKVersionStringFromZipFilePath(path string) string {
	_, file := filepath.Split(path)
	ver := strings.TrimSuffix(file, ".zip")
	return strings.TrimPrefix(ver, "FitSDKRelease_")
}

func parseMajorAndMinorSDKVersion(sdkString string) (int, int, error) {
	splitted := strings.Split(sdkString, ".")
	if len(splitted) < 2 {
		return 0, 0, fmt.Errorf("could not parse major/minor version from input: %q", sdkString)
	}

	maj, err := strconv.Atoi(splitted[0])
	if err != nil {
		return 0, 0, fmt.Errorf("could not parse major version from input: %q", splitted[0])
	}

	min, err := strconv.Atoi(splitted[1])
	if err != nil {
		return 0, 0, fmt.Errorf("could not parse minor version from input: %q", splitted[1])
	}

	return maj, min, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"context"
	"fmt"
	"sort"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/minio/madmin-go"
	"github.com/minio/minio/internal/color"
	"github.com/minio/minio/internal/config/storageclass"
	"github.com/minio/minio/internal/logger"
	"github.com/minio/pkg/console"
	"github.com/minio/pkg/wildcard"
)

const (
	bgHealingUUID = "0000-0000-0000-0000"
)

// NewBgHealSequence creates a background healing sequence
// operation which scans all objects and heal them.
func newBgHealSequence() *healSequence {
	reqInfo := &logger.ReqInfo{API: "BackgroundHeal"}
	ctx, cancelCtx := context.WithCancel(logger.SetReqInfo(GlobalContext, reqInfo))

	hs := madmin.HealOpts{
		// Remove objects that do not have read-quorum
		Remove: healDeleteDangling,
	}

	return &healSequence{
		respCh:      make(chan healResult),
		startTime:   UTCNow(),
		clientToken: bgHealingUUID,
		// run-background heal with reserved bucket
		bucket:   minioReservedBucket,
		settings: hs,
		currentStatus: healSequenceStatus{
			Summary:      healNotStartedStatus,
			HealSettings: hs,
		},
		cancelCtx:          cancelCtx,
		ctx:                ctx,
		reportProgress:     false,
		scannedItemsMap:    make(map[madmin.HealItemType]int64),
		healedItemsMap:     make(map[madmin.HealItemType]int64),
		healFailedItemsMap: make(map[string]int64),
	}
}

// getBackgroundHealStatus will return the
func getBackgroundHealStatus(ctx context.Context, o ObjectLayer) (madmin.BgHealState, bool) {
	if globalBackgroundHealState == nil {
		return madmin.BgHealState{}, false
	}

	bgSeq, ok := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
	if !ok {
		return madmin.BgHealState{}, false
	}

	status := madmin.BgHealState{
		ScannedItemsCount: bgSeq.getScannedItemsCount(),
	}

	if globalMRFState.initialized() {
		status.MRF = map[string]madmin.MRFStatus{
			globalLocalNodeName: globalMRFState.getCurrentMRFRoundInfo(),
		}
	}

	healDisksMap := map[string]struct{}{}
	for _, ep := range getLocalDisksToHeal() {
		healDisksMap[ep.String()] = struct{}{}
	}

	if o == nil {
		healing := globalBackgroundHealState.getLocalHealingDisks()
		for _, disk := range healing {
			status.HealDisks = append(status.HealDisks, disk.Endpoint)
		}

		return status, true
	}

	// ignores any errors here.
	si, _ := o.StorageInfo(ctx)

	indexed := make(map[string][]madmin.Disk)
	for _, disk := range si.Disks {
		setIdx := fmt.Sprintf("%d-%d", disk.PoolIndex, disk.SetIndex)
		indexed[setIdx] = append(indexed[setIdx], disk)
	}

	for id, disks := range indexed {
		ss := madmin.SetStatus{
			ID:        id,
			SetIndex:  disks[0].SetIndex,
			PoolIndex: disks[0].PoolIndex,
		}
		for _, disk := range disks {
			ss.Disks = append(ss.Disks, disk)
			if disk.Healing {
				ss.HealStatus = "Healing"
				ss.HealPriority = "high"
				status.HealDisks = append(status.HealDisks, disk.Endpoint)
			}
		}
		sortDisks(ss.Disks)
		status.Sets = append(status.Sets, ss)
	}
	sort.Slice(status.Sets, func(i, j int) bool {
		return status.Sets[i].ID < status.Sets[j].ID
	})

	backendInfo := o.BackendInfo()
	status.SCParity = make(map[string]int)
	status.SCParity[storageclass.STANDARD] = backendInfo.StandardSCParity
	status.SCParity[storageclass.RRS] = backendInfo.RRSCParity

	return status, true
}

func mustGetHealSequence(ctx context.Context) *healSequence {
	// Get background heal sequence to send elements to heal
	for {
		globalHealStateLK.RLock()
		hstate := globalBackgroundHealState
		globalHealStateLK.RUnlock()

		if hstate == nil {
			time.Sleep(time.Second)
			continue
		}

		bgSeq, ok := hstate.getHealSequenceByToken(bgHealingUUID)
		if !ok {
			time.Sleep(time.Second)
			continue
		}
		return bgSeq
	}
}

// healErasureSet lists and heals all objects in a specific erasure set
func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string, tracker *healingTracker) error {
	bgSeq := mustGetHealSequence(ctx)
	scanMode := madmin.HealNormalScan

	// Make sure to copy since `buckets slice`
	// is modified in place by tracker.
	healBuckets := make([]string, len(buckets))
	copy(healBuckets, buckets)

	// Heal all buckets first in this erasure set - this is useful
	// for new objects upload in different buckets to be successful
	for _, bucket := range healBuckets {
		_, err := er.HealBucket(ctx, bucket, madmin.HealOpts{ScanMode: scanMode})
		if err != nil {
			// Log bucket healing error if any, we shall retry again.
			logger.LogIf(ctx, err)
		}
	}

	var retErr error
	// Heal all buckets with all objects
	for _, bucket := range healBuckets {
		if tracker.isHealed(bucket) {
			continue
		}
		var forwardTo string
		// If we resume to the same bucket, forward to last known item.
		if tracker.Bucket != "" {
			if tracker.Bucket == bucket {
				forwardTo = tracker.Object
			} else {
				// Reset to where last bucket ended if resuming.
				tracker.resume()
			}
		}
		tracker.Object = ""
		tracker.Bucket = bucket
		// Heal current bucket again in case if it is failed
		// in the  being of erasure set healing
		if _, err := er.HealBucket(ctx, bucket, madmin.HealOpts{
			ScanMode: scanMode,
		}); err != nil {
			logger.LogIf(ctx, err)
			continue
		}

		if serverDebugLog {
			console.Debugf(color.Green("healDisk:")+" healing bucket %s content on %s erasure set\n",
				bucket, humanize.Ordinal(tracker.SetIndex+1))
		}

		disks, _ := er.getOnlineDisksWithHealing()
		if len(disks) == 0 {
			// all disks are healing in this set, this is allowed
			// so we simply proceed to next bucket, marking the bucket
			// as done as there are no objects to heal.
			tracker.bucketDone(bucket)
			logger.LogIf(ctx, tracker.update(ctx))
			continue
		}

		// Limit listing to 3 drives.
		if len(disks) > 3 {
			disks = disks[:3]
		}

		healEntry := func(entry metaCacheEntry) {
			if entry.name == "" && len(entry.metadata) == 0 {
				// ignore entries that don't have metadata.
				return
			}
			if entry.isDir() {
				// ignore healing entry.name's with `/` suffix.
				return
			}
			// We might land at .metacache, .trash, .multipart
			// no need to heal them skip, only when bucket
			// is '.minio.sys'
			if bucket == minioMetaBucket {
				if wildcard.Match("buckets/*/.metacache/*", entry.name) {
					return
				}
				if wildcard.Match("tmp/.trash/*", entry.name) {
					return
				}
				if wildcard.Match("multipart/*", entry.name) {
					return
				}
			}

			fivs, err := entry.fileInfoVersions(bucket)
			if err != nil {
				err := bgSeq.queueHealTask(healSource{
					bucket:    bucket,
					object:    entry.name,
					versionID: "",
				}, madmin.HealItemObject)
				if err != nil {
					tracker.ItemsFailed++
					logger.LogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", bucket, entry.name, err))
				} else {
					tracker.ItemsHealed++
				}
				bgSeq.logHeal(madmin.HealItemObject)
				return
			}

			for _, version := range fivs.Versions {
				if _, err := er.HealObject(ctx, bucket, version.Name,
					version.VersionID, madmin.HealOpts{
						ScanMode: scanMode,
						Remove:   healDeleteDangling,
					}); err != nil {
					// If not deleted, assume they failed.
					tracker.ItemsFailed++
					tracker.BytesFailed += uint64(version.Size)
					if version.VersionID != "" {
						logger.LogIf(ctx, fmt.Errorf("unable to heal object %s/%s-v(%s): %w", bucket, version.Name, version.VersionID, err))
					} else {
						logger.LogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", bucket, version.Name, err))
					}
				} else {
					tracker.ItemsHealed++
					tracker.BytesDone += uint64(version.Size)
				}
				bgSeq.logHeal(madmin.HealItemObject)
			}
			tracker.Object = entry.name
			if time.Since(tracker.LastUpdate) > time.Minute {
				logger.LogIf(ctx, tracker.update(ctx))
			}

			// Wait and proceed if there are active requests
			waitForLowHTTPReq()
		}

		// How to resolve partial results.
		resolver := metadataResolutionParams{
			dirQuorum: 1,
			objQuorum: 1,
			bucket:    bucket,
		}

		err := listPathRaw(ctx, listPathRawOptions{
			disks:          disks,
			bucket:         bucket,
			recursive:      true,
			forwardTo:      forwardTo,
			minDisks:       1,
			reportNotFound: false,
			agreed:         healEntry,
			partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
				entry, ok := entries.resolve(&resolver)
				if !ok {
					// check if we can get one entry atleast
					// proceed to heal nonetheless.
					entry, _ = entries.firstFound()
				}
				healEntry(*entry)
			},
			finished: nil,
		})
		if err != nil {
			// Set this such that when we return this function
			// we let the caller retry this disk again for the
			// buckets it failed to list.
			retErr = err
			logger.LogIf(ctx, err)
			continue
		}

		select {
		// If context is canceled don't mark as done...
		case <-ctx.Done():
			return ctx.Err()
		default:
			tracker.bucketDone(bucket)
			logger.LogIf(ctx, tracker.update(ctx))
		}
	}
	tracker.Object = ""
	tracker.Bucket = ""

	return retErr
}

// healObject heals given object path in deep to fix bitrot.
func healObject(bucket, object, versionID string, scan madmin.HealScanMode) {
	// Get background heal sequence to send elements to heal
	globalHealStateLK.Lock()
	bgSeq, ok := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
	globalHealStateLK.Unlock()
	if ok {
		bgSeq.queueHealTask(healSource{
			bucket:    bucket,
			object:    object,
			versionID: versionID,
			opts: &madmin.HealOpts{
				Remove:   healDeleteDangling, // if found dangling purge it.
				ScanMode: scan,
			},
		}, madmin.HealItemObject)
	}
}
<commit_msg>heal: Include dir markers when healing a fresh disk (#15158)<commit_after>// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"context"
	"fmt"
	"sort"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/minio/madmin-go"
	"github.com/minio/minio/internal/color"
	"github.com/minio/minio/internal/config/storageclass"
	"github.com/minio/minio/internal/logger"
	"github.com/minio/pkg/console"
	"github.com/minio/pkg/wildcard"
)

const (
	bgHealingUUID = "0000-0000-0000-0000"
)

// NewBgHealSequence creates a background healing sequence
// operation which scans all objects and heal them.
func newBgHealSequence() *healSequence {
	reqInfo := &logger.ReqInfo{API: "BackgroundHeal"}
	ctx, cancelCtx := context.WithCancel(logger.SetReqInfo(GlobalContext, reqInfo))

	hs := madmin.HealOpts{
		// Remove objects that do not have read-quorum
		Remove: healDeleteDangling,
	}

	return &healSequence{
		respCh:      make(chan healResult),
		startTime:   UTCNow(),
		clientToken: bgHealingUUID,
		// run-background heal with reserved bucket
		bucket:   minioReservedBucket,
		settings: hs,
		currentStatus: healSequenceStatus{
			Summary:      healNotStartedStatus,
			HealSettings: hs,
		},
		cancelCtx:          cancelCtx,
		ctx:                ctx,
		reportProgress:     false,
		scannedItemsMap:    make(map[madmin.HealItemType]int64),
		healedItemsMap:     make(map[madmin.HealItemType]int64),
		healFailedItemsMap: make(map[string]int64),
	}
}

// getBackgroundHealStatus will return the
func getBackgroundHealStatus(ctx context.Context, o ObjectLayer) (madmin.BgHealState, bool) {
	if globalBackgroundHealState == nil {
		return madmin.BgHealState{}, false
	}

	bgSeq, ok := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
	if !ok {
		return madmin.BgHealState{}, false
	}

	status := madmin.BgHealState{
		ScannedItemsCount: bgSeq.getScannedItemsCount(),
	}

	if globalMRFState.initialized() {
		status.MRF = map[string]madmin.MRFStatus{
			globalLocalNodeName: globalMRFState.getCurrentMRFRoundInfo(),
		}
	}

	healDisksMap := map[string]struct{}{}
	for _, ep := range getLocalDisksToHeal() {
		healDisksMap[ep.String()] = struct{}{}
	}

	if o == nil {
		healing := globalBackgroundHealState.getLocalHealingDisks()
		for _, disk := range healing {
			status.HealDisks = append(status.HealDisks, disk.Endpoint)
		}

		return status, true
	}

	// ignores any errors here.
	si, _ := o.StorageInfo(ctx)

	indexed := make(map[string][]madmin.Disk)
	for _, disk := range si.Disks {
		setIdx := fmt.Sprintf("%d-%d", disk.PoolIndex, disk.SetIndex)
		indexed[setIdx] = append(indexed[setIdx], disk)
	}

	for id, disks := range indexed {
		ss := madmin.SetStatus{
			ID:        id,
			SetIndex:  disks[0].SetIndex,
			PoolIndex: disks[0].PoolIndex,
		}
		for _, disk := range disks {
			ss.Disks = append(ss.Disks, disk)
			if disk.Healing {
				ss.HealStatus = "Healing"
				ss.HealPriority = "high"
				status.HealDisks = append(status.HealDisks, disk.Endpoint)
			}
		}
		sortDisks(ss.Disks)
		status.Sets = append(status.Sets, ss)
	}
	sort.Slice(status.Sets, func(i, j int) bool {
		return status.Sets[i].ID < status.Sets[j].ID
	})

	backendInfo := o.BackendInfo()
	status.SCParity = make(map[string]int)
	status.SCParity[storageclass.STANDARD] = backendInfo.StandardSCParity
	status.SCParity[storageclass.RRS] = backendInfo.RRSCParity

	return status, true
}

func mustGetHealSequence(ctx context.Context) *healSequence {
	// Get background heal sequence to send elements to heal
	for {
		globalHealStateLK.RLock()
		hstate := globalBackgroundHealState
		globalHealStateLK.RUnlock()

		if hstate == nil {
			time.Sleep(time.Second)
			continue
		}

		bgSeq, ok := hstate.getHealSequenceByToken(bgHealingUUID)
		if !ok {
			time.Sleep(time.Second)
			continue
		}
		return bgSeq
	}
}

// healErasureSet lists and heals all objects in a specific erasure set
func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string, tracker *healingTracker) error {
	bgSeq := mustGetHealSequence(ctx)
	scanMode := madmin.HealNormalScan

	// Make sure to copy since `buckets slice`
	// is modified in place by tracker.
	healBuckets := make([]string, len(buckets))
	copy(healBuckets, buckets)

	// Heal all buckets first in this erasure set - this is useful
	// for new objects upload in different buckets to be successful
	for _, bucket := range healBuckets {
		_, err := er.HealBucket(ctx, bucket, madmin.HealOpts{ScanMode: scanMode})
		if err != nil {
			// Log bucket healing error if any, we shall retry again.
			logger.LogIf(ctx, err)
		}
	}

	var retErr error
	// Heal all buckets with all objects
	for _, bucket := range healBuckets {
		if tracker.isHealed(bucket) {
			continue
		}
		var forwardTo string
		// If we resume to the same bucket, forward to last known item.
		if tracker.Bucket != "" {
			if tracker.Bucket == bucket {
				forwardTo = tracker.Object
			} else {
				// Reset to where last bucket ended if resuming.
				tracker.resume()
			}
		}
		tracker.Object = ""
		tracker.Bucket = bucket
		// Heal current bucket again in case if it is failed
		// in the  being of erasure set healing
		if _, err := er.HealBucket(ctx, bucket, madmin.HealOpts{
			ScanMode: scanMode,
		}); err != nil {
			logger.LogIf(ctx, err)
			continue
		}

		if serverDebugLog {
			console.Debugf(color.Green("healDisk:")+" healing bucket %s content on %s erasure set\n",
				bucket, humanize.Ordinal(tracker.SetIndex+1))
		}

		disks, _ := er.getOnlineDisksWithHealing()
		if len(disks) == 0 {
			// all disks are healing in this set, this is allowed
			// so we simply proceed to next bucket, marking the bucket
			// as done as there are no objects to heal.
			tracker.bucketDone(bucket)
			logger.LogIf(ctx, tracker.update(ctx))
			continue
		}

		// Limit listing to 3 drives.
		if len(disks) > 3 {
			disks = disks[:3]
		}

		healEntry := func(entry metaCacheEntry) {
			if entry.name == "" && len(entry.metadata) == 0 {
				// ignore entries that don't have metadata.
				return
			}
			if entry.isDir() {
				// ignore healing entry.name's with `/` suffix.
				return
			}
			// We might land at .metacache, .trash, .multipart
			// no need to heal them skip, only when bucket
			// is '.minio.sys'
			if bucket == minioMetaBucket {
				if wildcard.Match("buckets/*/.metacache/*", entry.name) {
					return
				}
				if wildcard.Match("tmp/.trash/*", entry.name) {
					return
				}
				if wildcard.Match("multipart/*", entry.name) {
					return
				}
			}

			fivs, err := entry.fileInfoVersions(bucket)
			if err != nil {
				err := bgSeq.queueHealTask(healSource{
					bucket:    bucket,
					object:    entry.name,
					versionID: "",
				}, madmin.HealItemObject)
				if err != nil {
					tracker.ItemsFailed++
					logger.LogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", bucket, entry.name, err))
				} else {
					tracker.ItemsHealed++
				}
				bgSeq.logHeal(madmin.HealItemObject)
				return
			}

			// erasureObjects layer needs object names to be encoded
			encodedEntryName := encodeDirObject(entry.name)

			for _, version := range fivs.Versions {
				if _, err := er.HealObject(ctx, bucket, encodedEntryName,
					version.VersionID, madmin.HealOpts{
						ScanMode: scanMode,
						Remove:   healDeleteDangling,
					}); err != nil {
					// If not deleted, assume they failed.
					tracker.ItemsFailed++
					tracker.BytesFailed += uint64(version.Size)
					if version.VersionID != "" {
						logger.LogIf(ctx, fmt.Errorf("unable to heal object %s/%s-v(%s): %w", bucket, version.Name, version.VersionID, err))
					} else {
						logger.LogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", bucket, version.Name, err))
					}
				} else {
					tracker.ItemsHealed++
					tracker.BytesDone += uint64(version.Size)
				}
				bgSeq.logHeal(madmin.HealItemObject)
			}
			tracker.Object = entry.name
			if time.Since(tracker.LastUpdate) > time.Minute {
				logger.LogIf(ctx, tracker.update(ctx))
			}

			// Wait and proceed if there are active requests
			waitForLowHTTPReq()
		}

		// How to resolve partial results.
		resolver := metadataResolutionParams{
			dirQuorum: 1,
			objQuorum: 1,
			bucket:    bucket,
		}

		err := listPathRaw(ctx, listPathRawOptions{
			disks:          disks,
			bucket:         bucket,
			recursive:      true,
			forwardTo:      forwardTo,
			minDisks:       1,
			reportNotFound: false,
			agreed:         healEntry,
			partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
				entry, ok := entries.resolve(&resolver)
				if !ok {
					// check if we can get one entry atleast
					// proceed to heal nonetheless.
					entry, _ = entries.firstFound()
				}
				healEntry(*entry)
			},
			finished: nil,
		})
		if err != nil {
			// Set this such that when we return this function
			// we let the caller retry this disk again for the
			// buckets it failed to list.
			retErr = err
			logger.LogIf(ctx, err)
			continue
		}

		select {
		// If context is canceled don't mark as done...
		case <-ctx.Done():
			return ctx.Err()
		default:
			tracker.bucketDone(bucket)
			logger.LogIf(ctx, tracker.update(ctx))
		}
	}
	tracker.Object = ""
	tracker.Bucket = ""

	return retErr
}

// healObject heals given object path in deep to fix bitrot.
func healObject(bucket, object, versionID string, scan madmin.HealScanMode) {
	// Get background heal sequence to send elements to heal
	globalHealStateLK.Lock()
	bgSeq, ok := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
	globalHealStateLK.Unlock()
	if ok {
		bgSeq.queueHealTask(healSource{
			bucket:    bucket,
			object:    object,
			versionID: versionID,
			opts: &madmin.HealOpts{
				Remove:   healDeleteDangling, // if found dangling purge it.
				ScanMode: scan,
			},
		}, madmin.HealItemObject)
	}
}
<|endoftext|>


<commit_before>// Copyright (C) 2018 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// gofuse is a utility program to help go tools work with bazel generated files.
//
// gofuse will create a new 'fused' directory in the project root which
// contains:
//  • Symlinks to authored .go source files in the GAPID source tree.
//  • Symlinks to bazel-generated .go files.
//  • Symlinks to external 3rd-party .go files.
// These symlinks are 'fused' into a single, common directory structure that
// is expected by the typical GOPATH rules used by go tooling.
//
// Note: Currently gofuse only works on macOS. It should be simple to get this
// working on Linux, but the extensive use of symlinks makes Windows support
// unlikely.
package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
)

// Map of bazel external package names to the expected import names.
var externals = map[string]string{
	"com_github_golang_protobuf": filepath.Join("github.com", "golang", "protobuf"),
	"com_github_google_protobuf": filepath.Join("github.com", "google", "protobuf"),
	"com_github_pkg_errors":      filepath.Join("github.com", "pkg", "errors"),
	"llvm":                       "llvm",
}

var (
	fuseDir = flag.String("dir", "", "directory to use as the fuse root")
)

func main() {
	flag.Parse()

	if err := run(); err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}
}

func run() error {
	// Locate the root of the GAPID project.
	projectRoot, err := projectRoot()
	if err != nil {
		return err
	}

	// fusedRoot is the root of the generated directory.
	fusedRoot := filepath.Join(projectRoot, "fused")
	if *fuseDir != "" {
		fusedRoot = *fuseDir
	}

	fmt.Println("Updating fused directory at:", fusedRoot)

	// Collect all the existing symlinks under the fused root.
	fusedFiles := collect(fusedRoot, always).ifTrue(and(hasSuffix(".go"), isFile, isSymlink))

	// Collect all the source (authored) file mappings.
	srcMapping := collect(projectRoot,
		and(
			hasPrefix(fusedRoot).not(),                            // Don't traverse the fused root
			hasPrefix(filepath.Join(projectRoot, "bazel-")).not(), // Don't traverse the bazel directories
		)).
		ifTrue(and(isFile, hasSuffix(".go"))). // Only consider .go files
		mapping(func(path string) string {
			return filepath.Join(fusedRoot, "src", "github.com", "google", "gapid", rel(projectRoot, path))
		})

	// Collect all the bazel generated file mappings.
	genRoot := filepath.Join(projectRoot, "bazel-out", "darwin-fastbuild", "bin")
	genMapping := collect(genRoot, always).ifTrue(and(
		isFile,
		contains("github.com"),
		hasSuffix(".go"),
	)).mapping(func(path string) string {
		return filepath.Join(fusedRoot, "src", trimUpTo(rel(projectRoot, path), "github.com"))
	})

	// Collect all the external package file mappings.
	extMapping := mappings{}
	for pkg, imp := range externals {
		src := filepath.Join(projectRoot, "bazel-gapid", "external", pkg)
		dst := filepath.Join(fusedRoot, "src", imp)
		m := collect(src, always).ifTrue(and(isFile, hasSuffix(".go"))).
			mapping(func(path string) string {
				return filepath.Join(dst, rel(src, path))
			})
		extMapping = append(extMapping, m...)
	}

	// Every mapping we're going to deal with.
	allMappings := join(srcMapping, genMapping, extMapping)

	// Remove all existing symlinks in the fused directory that part of the
	// mappings. This may never happen if the OS automatically deletes deleted
	// symlink targets.
	if err := fusedFiles.ifFalse(allMappings.dsts().set().contains).foreach(remove); err != nil {
		return err
	}

	// Create symlinks for all of the missing mappings.
	if err := allMappings.ifDstFalse(fusedFiles.set().contains).foreach(mapping.symlink); err != nil {
		return err
	}

	// Finally remove any empty directories
	dirs := collect(fusedRoot, isDir).
		ifFalse(contains(".git")) // Incase you go-get your go tools into the fused dir
	for len(dirs) > 0 { // Reverse loop to delete child directories first
		dir := dirs[len(dirs)-1]
		dirs = dirs[:len(dirs)-1]
		if isEmpty(dir) {
			fmt.Println("Removing empty directory", dir)
			os.Remove(dir)
		}
	}

	return nil
}

// A predicate function.
type pred func(string) bool

// A string transform.
type transform func(string) string

// A collection of file paths.
type paths []string

// A mapping from source to destination path.
type mapping struct{ src, dst string }

// A collection of mappings.
type mappings []mapping

// A unique set of strings.
type set map[string]struct{}

// mapping returns a new list of mappings by transforming the source paths in
// l to destination paths using the transform t.
func (l paths) mapping(t transform) mappings {
	out := make(mappings, len(l))
	for i, p := range l {
		out[i] = mapping{p, t(p)}
	}
	return out
}

// ifTrue returns all the paths in l where the predicate p returns true.
func (l paths) ifTrue(p pred) paths {
	out := make(paths, 0, len(l))
	for _, s := range l {
		if p(s) {
			out = append(out, s)
		}
	}
	return out
}

// ifTrue returns all the paths in l where the predicate p returns false.
func (l paths) ifFalse(p pred) paths {
	return l.ifTrue(p.not())
}

// set returns a new set from the list of paths.
func (l paths) set() set {
	out := set{}
	for _, s := range l {
		out[s] = struct{}{}
	}
	return out
}

// foreach calls f for each path in l.
func (l paths) foreach(f func(string) error) error {
	for _, m := range l {
		if err := f(m); err != nil {
			return err
		}
	}
	return nil
}

// join returns the concatenated list of mappings in l.
func join(l ...mappings) mappings {
	out := mappings{}
	for _, m := range l {
		out = append(out, m...)
	}
	return out
}

// dsts returns the destination paths of all the mappings in l.
func (l mappings) dsts() paths {
	out := make(paths, len(l))
	for i, p := range l {
		out[i] = p.dst
	}
	return out
}

// ifDstTrue returns all the mappings in l where the predicate p returns true
// for the destination path.
func (l mappings) ifDstTrue(p pred) mappings {
	out := make(mappings, 0, len(l))
	for _, m := range l {
		if p(m.dst) {
			out = append(out, m)
		}
	}
	return out
}

// ifDstTrue returns all the mappings in l where the predicate p returns false
// for the destination path.
func (l mappings) ifDstFalse(p pred) mappings {
	return l.ifDstTrue(p.not())
}

// foreach calls f for each mapping in l.
func (l mappings) foreach(f func(mapping) error) error {
	for _, m := range l {
		if err := f(m); err != nil {
			return err
		}
	}
	return nil
}

// symlink creates a symlink from the mapping source to the mapping destination.
func (m mapping) symlink() error {
	fmt.Println("--- Symlinking source file:\n", m.src, "->", m.dst)
	dir, _ := filepath.Split(m.dst)
	if err := os.MkdirAll(dir, 0755); err != nil {
		return err
	}
	return os.Symlink(m.src, m.dst)
}

// contains returns true if the set s contains str.
func (s set) contains(str string) bool {
	_, ok := s[str]
	return ok
}

// remove deletes the file or directory at p.
func remove(p string) error {
	fmt.Println("--- Removing symlink:\n", p)
	return os.Remove(p)
}

// hasPrefix returns a prediacte that returns true iff the string begins with
// str.
func hasPrefix(str string) pred {
	return func(p string) bool { return strings.HasPrefix(p, str) }
}

// hasSuffix returns a prediacte that returns true iff the string ends with str.
func hasSuffix(str string) pred {
	return func(p string) bool { return strings.HasSuffix(p, str) }
}

// contains returns a predicate that returns true iff the string contains the
// substring s.
func contains(s string) pred {
	return func(p string) bool { return strings.Contains(p, s) }
}

// not inverses the predicate.
func (f pred) not() pred {
	return func(p string) bool { return !f(p) }
}

// always is a predicate that always returns true.
func always(string) bool { return true }

// or returns a predicate that returns true if any of the predicates in l
// return true.
func or(l ...pred) pred {
	return func(path string) bool {
		for _, f := range l {
			if f(path) {
				return true
			}
		}
		return false
	}
}

// or returns a predicate that returns true if all of the predicates in l
// return true.
func and(list ...pred) pred {
	return func(path string) bool {
		for _, f := range list {
			if !f(path) {
				return false
			}
		}
		return true
	}
}

// trimUpTo returns a string with all runes in str omitted up to (but not
// including) the first occurance of pat. If str does not contain pat, then str
// is returned.
func trimUpTo(str, pat string) string {
	i := strings.Index(str, pat)
	if i > 0 {
		return str[i:]
	}
	return str
}

// rel returns the relative path of target from base. rel panics on error.
func rel(base, target string) string {
	rel, err := filepath.Rel(base, target)
	if err != nil {
		panic(err)
	}
	return rel
}

// collect returns all the file paths under root that pass the predicate p.
// Unlike filepath.Walk, symlinked directories are also traversed.
func collect(root string, p pred) paths {
	out := paths{}
	filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		if !p(path) {
			return nil
		}
		if info != nil && info.Mode()&os.ModeSymlink != 0 && isDir(path) {
			pred := and(p, func(s string) bool { return s != path })

			target, err := os.Readlink(path)
			if err != nil {
				return err
			}
			for _, p := range collect(target, pred) {
				out = append(out, filepath.Join(path, rel(target, p)))
			}
		}
		out = append(out, path)
		return nil
	})
	return out
}

// projectRoot searches upwards from the current working directory for the first
// directory containing a file called 'WORKSPACE'.
func projectRoot() (string, error) {
	root, err := os.Getwd()
	if err != nil {
		return "", fmt.Errorf("Could not get the current working directory: %v", err)
	}
	for isDir(root) {
		if isFile(filepath.Join(root, "WORKSPACE")) {
			return root, nil
		}
		root, _ = filepath.Split(root)
	}
	return "", fmt.Errorf("Couldn't find project root from CWD")
}

// isFile is a predicate that returns true if there is a file at path that is
// not a directory.
func isFile(path string) bool {
	fi, err := os.Stat(path)
	if err != nil {
		return false
	}
	return !fi.IsDir()
}

// isFile is a predicate that returns true if there is a directory at path.
func isDir(path string) bool {
	fi, err := os.Stat(path)
	if err != nil {
		return false
	}
	return fi.IsDir()
}

// isSymlink is a predicate that returns true if there is a symlink.
func isSymlink(path string) bool {
	fi, err := os.Lstat(path)
	if err != nil {
		return false
	}
	return (fi.Mode() & os.ModeSymlink) != 0
}

// isEmpty is a predicate that returns true if the directory at path is empty.
func isEmpty(path string) bool {
	f, err := os.Open(path)
	if err != nil {
		return false
	}
	defer f.Close()
	_, err = f.Readdirnames(1)
	if err == io.EOF {
		return true
	}
	return false
}
<commit_msg>Improve gofuse. (#2326)<commit_after>// Copyright (C) 2018 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// gofuse is a utility program to help go tools work with bazel generated files.
//
// gofuse will create a new 'fused' directory in the project root which
// contains:
//  • Symlinks to authored .go (and optionally, .proto) source files in the GAPID source tree.
//  • Symlinks to bazel-generated .go files (bazel-out/[config]/{bin,genfiles}).
//  • Symlinks to external 3rd-party .go files.
// These symlinks are 'fused' into a single, common directory structure that
// is expected by the typical GOPATH rules used by go tooling.
//
// Note: the extensive use of symlinks makes Windows support unlikely.
//
// Examples:
//   bazel run //cmd/gofuse
//   bazel run //cmd/gofuse -- --bazelout=k8-fastbuild
//   bazel run //cmd/gofuse -- --bazelout=k8-dbg
//   bazel run //cmd/gofuse -- --bazelout=darwin-fastbuild --linkproto=false
package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"strings"
)

// Map of bazel external package names to the expected import names.
var externals = map[string]string{
	"com_github_golang_protobuf": filepath.Join("github.com", "golang", "protobuf"),
	"com_github_google_protobuf": filepath.Join("github.com", "google", "protobuf"),
	"com_github_pkg_errors":      filepath.Join("github.com", "pkg", "errors"),
	"llvm":                       "llvm",
}

var (
	fuseDir = flag.String("dir", "", "directory to use as the fuse root")

	bazelOutDirectory = flag.String("bazelout", "",
		"The bazel-out/X directory name from which to include .go files. E.g. k8-fastbuild, darwin-fastbuild, k8-dbg, etc.")

	linkProtoFiles = flag.Bool("linkproto", true,
		"Whether to symlink .proto files. This is useful if you want to work in the fuse directory exclusively.")
)

func main() {
	flag.Parse()

	if err := run(); err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}
}

func run() error {

	if len(*bazelOutDirectory) == 0 {
		switch runtime.GOOS {
		case "linux":
			*bazelOutDirectory = "k8-fastbuild"
			break
		case "darwin":
			*bazelOutDirectory = "darwin-fastbuild"
		default:
		}

		if len(*bazelOutDirectory) == 0 {
			fmt.Println("\nFailed to guess bazel-out/X directory. Please pass --bazelout=X")
			return fmt.Errorf("need bazelout flag")
		}

		fmt.Printf("\nWARNING: guessed bazel-out/X directory as: %s. Please pass --bazelout=X to be more explicit.\n", *bazelOutDirectory)
		fmt.Println()
	}

	// If run via 'bazel run', use the workspace directory.
	if cwd := os.Getenv("BUILD_WORKSPACE_DIRECTORY"); cwd != "" {
		os.Chdir(cwd)
	}

	// Locate the root of the GAPID project.
	projectRoot, err := projectRoot()
	if err != nil {
		return err
	}

	// fusedRoot is the root of the generated directory.
	fusedRoot := filepath.Join(projectRoot, "fused")
	if *fuseDir != "" {
		fusedRoot = *fuseDir
	}

	fmt.Println("Updating fused directory at:", fusedRoot)

	// Collect all the existing symlinks under the fused root.
	fusedFiles := collect(fusedRoot, always).ifTrue(and(or(hasSuffix(".go"), hasSuffix(".proto")), isFile, isSymlink))

	fmt.Print("Collecting .go ")
	if *linkProtoFiles {
		fmt.Println("and .proto files from:", projectRoot)
	} else {
		fmt.Println("files from:", projectRoot)
	}
	srcMapping := collect(projectRoot,
		and(
			hasPrefix(fusedRoot).not(),                            // Don't traverse the fused root
			hasPrefix(filepath.Join(projectRoot, "bazel-")).not(), // Don't traverse the bazel directories
		)).
		ifTrue(and(isFile, or(hasSuffix(".go"), and(fromBool(*linkProtoFiles), hasSuffix(".proto"))))). // Only consider .go and .proto files
		mapping(func(path string) string {
			return filepath.Join(fusedRoot, "src", "github.com", "google", "gapid", rel(projectRoot, path))
		})

	// E.g. bazel-out/k8-dbg/genfiles
	genfilesOut := filepath.Join(projectRoot, "bazel-out", *bazelOutDirectory, "genfiles")
	fmt.Println("Collecting generated .go files from:", genfilesOut)
	genfilesMappingOut := collect(genfilesOut, always).
		ifTrue(and(isFile, hasSuffix(".go"))). // Only consider .go files
		mapping(func(path string) string {
			return filepath.Join(fusedRoot, "src", "github.com", "google", "gapid", rel(genfilesOut, path))
		})

	// E.g. bazel-out/k8-dbg/bin
	binOut := filepath.Join(projectRoot, "bazel-out", *bazelOutDirectory, "bin")
	fmt.Println("Collecting generated .go files from:", binOut)
	binMappingOut := collect(binOut, always).ifTrue(and(
		isFile,
		contains("github.com"),
		hasSuffix(".go"),
	)).mapping(func(path string) string {
		return filepath.Join(fusedRoot, "src", trimUpTo(rel(projectRoot, path), "github.com"))
	})

	// Collect all the external package file mappings.
	extMapping := mappings{}
	for pkg, imp := range externals {
		src := filepath.Join(projectRoot, "bazel-gapid", "external", pkg)
		fmt.Println("Collecting .go files from:", src)
		dst := filepath.Join(fusedRoot, "src", imp)
		m := collect(src, always).ifTrue(and(isFile, hasSuffix(".go"))).
			mapping(func(path string) string {
				return filepath.Join(dst, rel(src, path))
			})
		extMapping = append(extMapping, m...)
	}

	// Every mapping we're going to deal with.
	allMappings := join(srcMapping, genfilesMappingOut, binMappingOut, extMapping)

	// Remove all existing symlinks in the fused directory that part of the
	// mappings. This may never happen if the OS automatically deletes deleted
	// symlink targets.
	if err := fusedFiles.ifFalse(allMappings.dsts().set().contains).foreach(remove); err != nil {
		return err
	}

	// Create symlinks for all of the missing mappings.
	if err := allMappings.ifDstFalse(fusedFiles.set().contains).foreach(mapping.symlink); err != nil {
		return err
	}

	// Finally remove any empty directories
	dirs := collect(fusedRoot, isDir).
		ifFalse(contains(".git")) // In case you go-get your go tools into the fused dir
	for len(dirs) > 0 { // Reverse loop to delete child directories first
		dir := dirs[len(dirs)-1]
		dirs = dirs[:len(dirs)-1]
		if isEmpty(dir) {
			fmt.Println("Removing empty directory", dir)
			os.Remove(dir)
		}
	}

	return nil
}

// A predicate function.
type pred func(string) bool

// A string transform.
type transform func(string) string

// A collection of file paths.
type paths []string

// A mapping from source to destination path.
type mapping struct{ src, dst string }

// A collection of mappings.
type mappings []mapping

// A unique set of strings.
type set map[string]struct{}

// mapping returns a new list of mappings by transforming the source paths in
// l to destination paths using the transform t.
func (l paths) mapping(t transform) mappings {
	out := make(mappings, len(l))
	for i, p := range l {
		out[i] = mapping{p, t(p)}
	}
	return out
}

// ifTrue returns all the paths in l where the predicate p returns true.
func (l paths) ifTrue(p pred) paths {
	out := make(paths, 0, len(l))
	for _, s := range l {
		if p(s) {
			out = append(out, s)
		}
	}
	return out
}

// ifTrue returns all the paths in l where the predicate p returns false.
func (l paths) ifFalse(p pred) paths {
	return l.ifTrue(p.not())
}

// set returns a new set from the list of paths.
func (l paths) set() set {
	out := set{}
	for _, s := range l {
		out[s] = struct{}{}
	}
	return out
}

// foreach calls f for each path in l.
func (l paths) foreach(f func(string) error) error {
	for _, m := range l {
		if err := f(m); err != nil {
			return err
		}
	}
	return nil
}

// join returns the concatenated list of mappings in l.
func join(l ...mappings) mappings {
	out := mappings{}
	for _, m := range l {
		out = append(out, m...)
	}
	return out
}

// dsts returns the destination paths of all the mappings in l.
func (l mappings) dsts() paths {
	out := make(paths, len(l))
	for i, p := range l {
		out[i] = p.dst
	}
	return out
}

// ifDstTrue returns all the mappings in l where the predicate p returns true
// for the destination path.
func (l mappings) ifDstTrue(p pred) mappings {
	out := make(mappings, 0, len(l))
	for _, m := range l {
		if p(m.dst) {
			out = append(out, m)
		}
	}
	return out
}

// ifDstTrue returns all the mappings in l where the predicate p returns false
// for the destination path.
func (l mappings) ifDstFalse(p pred) mappings {
	return l.ifDstTrue(p.not())
}

// foreach calls f for each mapping in l.
func (l mappings) foreach(f func(mapping) error) error {
	for _, m := range l {
		if err := f(m); err != nil {
			return err
		}
	}
	return nil
}

// symlink creates a symlink from the mapping source to the mapping destination.
func (m mapping) symlink() error {
	fmt.Println("--- Symlinking source file:\n", m.src, "->", m.dst)
	dir, _ := filepath.Split(m.dst)
	if err := os.MkdirAll(dir, 0755); err != nil {
		return err
	}
	return os.Symlink(m.src, m.dst)
}

// contains returns true if the set s contains str.
func (s set) contains(str string) bool {
	_, ok := s[str]
	return ok
}

// remove deletes the file or directory at p.
func remove(p string) error {
	fmt.Println("--- Removing symlink:\n", p)
	return os.Remove(p)
}

// hasPrefix returns a prediacte that returns true iff the string begins with
// str.
func hasPrefix(str string) pred {
	return func(p string) bool { return strings.HasPrefix(p, str) }
}

// hasSuffix returns a prediacte that returns true iff the string ends with str.
func hasSuffix(str string) pred {
	return func(p string) bool { return strings.HasSuffix(p, str) }
}

// contains returns a predicate that returns true iff the string contains the
// substring s.
func contains(s string) pred {
	return func(p string) bool { return strings.Contains(p, s) }
}

// not inverses the predicate.
func (f pred) not() pred {
	return func(p string) bool { return !f(p) }
}

// always is a predicate that always returns true.
func always(string) bool { return true }

// or returns a predicate that returns true if any of the predicates in l
// return true.
func or(l ...pred) pred {
	return func(path string) bool {
		for _, f := range l {
			if f(path) {
				return true
			}
		}
		return false
	}
}

// or returns a predicate that returns true if all of the predicates in l
// return true.
func and(list ...pred) pred {
	return func(path string) bool {
		for _, f := range list {
			if !f(path) {
				return false
			}
		}
		return true
	}
}

func fromBool(b bool) pred {
	return func(path string) bool {
		return b
	}
}

// trimUpTo returns a string with all runes in str omitted up to (but not
// including) the first occurance of pat. If str does not contain pat, then str
// is returned.
func trimUpTo(str, pat string) string {
	i := strings.Index(str, pat)
	if i > 0 {
		return str[i:]
	}
	return str
}

// rel returns the relative path of target from base. rel panics on error.
func rel(base, target string) string {
	rel, err := filepath.Rel(base, target)
	if err != nil {
		panic(err)
	}
	return rel
}

// collect returns all the file paths under root that pass the predicate p.
// Unlike filepath.Walk, symlinked directories are also traversed.
func collect(root string, p pred) paths {
	out := paths{}
	filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		if !p(path) {
			return nil
		}
		if info != nil && info.Mode()&os.ModeSymlink != 0 && isDir(path) {
			pred := and(p, func(s string) bool { return s != path })

			target, err := os.Readlink(path)
			if err != nil {
				return err
			}
			for _, p := range collect(target, pred) {
				out = append(out, filepath.Join(path, rel(target, p)))
			}
		}
		out = append(out, path)
		return nil
	})
	return out
}

// projectRoot searches upwards from the current working directory for the first
// directory containing a file called 'WORKSPACE'.
func projectRoot() (string, error) {
	root, err := os.Getwd()
	if err != nil {
		return "", fmt.Errorf("Could not get the current working directory: %v", err)
	}
	for isDir(root) {
		if isFile(filepath.Join(root, "WORKSPACE")) {
			return root, nil
		}
		root, _ = filepath.Split(root)
	}
	return "", fmt.Errorf("Couldn't find project root from CWD")
}

// isFile is a predicate that returns true if there is a file at path that is
// not a directory.
func isFile(path string) bool {
	fi, err := os.Stat(path)
	if err != nil {
		return false
	}
	return !fi.IsDir()
}

// isFile is a predicate that returns true if there is a directory at path.
func isDir(path string) bool {
	fi, err := os.Stat(path)
	if err != nil {
		return false
	}
	return fi.IsDir()
}

// isSymlink is a predicate that returns true if there is a symlink.
func isSymlink(path string) bool {
	fi, err := os.Lstat(path)
	if err != nil {
		return false
	}
	return (fi.Mode() & os.ModeSymlink) != 0
}

// isEmpty is a predicate that returns true if the directory at path is empty.
func isEmpty(path string) bool {
	f, err := os.Open(path)
	if err != nil {
		return false
	}
	defer f.Close()
	_, err = f.Readdirnames(1)
	if err == io.EOF {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/nsf/termbox-go"
	"github.com/nsf/tulib"
	"log"
	"os"
	"time"
)

const (
	NumFPSSamples = 64
	FPSLimit      = 60
)

var (
	fpsSamples    [64]float64
	currentSample = 0
	logfile       = flag.String("log", "goland.log", "log file")
	debug         = flag.Bool("debug", false, "print debugging info")
)

type Game struct {
	P *Player

	Terminal
	logfile *os.File

	CloseChan chan bool

	// unexported
	fps float64

	Objects []Object

	Map *MapChunk
}

func NewGame() *Game {
	g := Game{}

	g.CloseChan = make(chan bool, 1)

	g.Map = NewMapChunk()
	g.Map.Tiles[30][22].Blocked = true
	g.Map.Tiles[30][22].SightBlocked = true
	g.Map.Tiles[30][22].Ch = MAP_WALL
	g.Map.Tiles[50][22].Blocked = true
	g.Map.Tiles[50][22].SightBlocked = true
	g.Map.Tiles[50][22].Ch = MAP_WALL

	g.P = NewPlayer(&g)
	g.P.Pos = Vector{10, 10}

	g.Objects = append(g.Objects, g.P)

	u := NewUnit(&g)
	u.Ch.Ch = '@'
	u.Pos = Vector{15, 15}

	g.Objects = append(g.Objects, &u)

	return &g
}

func (g *Game) Run() {

	g.Start()

	timer := NewDeltaTimer()
	ticker := time.NewTicker(time.Second / FPSLimit)

	run := true

	for run {
		select {
		case <-ticker.C:
			// frame tick
			delta := timer.DeltaTime()

			if delta.Seconds() > 0.25 {
				delta = time.Duration(250 * time.Millisecond)
			}

			g.Update(delta)
			g.Draw()

			g.Flush()

		case <-g.CloseChan:
			run = false
		}
	}

	g.End()

}

func (g *Game) Start() {
	f, err := os.OpenFile(*logfile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
	if err != nil {
		log.Fatal(err)
	}

	log.SetOutput(f)
	log.Print("Logging started")

	g.logfile = f

	g.Terminal.Start()

	g.HandleKey(termbox.KeyEsc, func(ev termbox.Event) { g.CloseChan <- false })

	scale := 2

	for k, v := range CARDINALS {
		func (c rune, d Direction) {
			g.HandleRune(c, func(_ termbox.Event) {
				g.P.Move(d)
			})
		}(k, v)
	}

	g.HandleRune('W', func(_ termbox.Event) {
		for i := 0; i < scale; i++ {
			g.P.Move(DIR_UP)
		}
	})
	g.HandleRune('A', func(_ termbox.Event) {
		for i := 0; i < scale; i++ {
			g.P.Move(DIR_LEFT)
		}
	})
	g.HandleRune('S', func(_ termbox.Event) {
		for i := 0; i < scale; i++ {
			g.P.Move(DIR_DOWN)
		}
	})
	g.HandleRune('D', func(_ termbox.Event) {
		for i := 0; i < scale; i++ {
			g.P.Move(DIR_RIGHT)
		}
	})

}

func (g *Game) End() {
	log.Print("Logging ended")
	g.logfile.Close()
	g.Terminal.End()
}

func (g *Game) Update(delta time.Duration) {
	// update fps
	g.fps = g.calcFPS(delta)

	g.RunInputHandlers()

	for _, o := range g.Objects {
		o.Update(delta)
	}

}

func (g *Game) Draw() {
	//	g.Terminal.Draw()
	for x, col := range g.Map.Tiles {
		for y, tile := range col {
			if tile.CanSee() {
				termbox.SetCell(x, y, tile.Ch.Ch, tile.Ch.Fg, tile.Ch.Bg)
			} else {
				termbox.SetCell(x, y, tile.Ch.Ch, termbox.ColorBlack, termbox.ColorWhite)
			}
		}
	}

	labelparams := &tulib.LabelParams{termbox.ColorRed, termbox.ColorBlack, tulib.AlignCenter, '»', false}
	labelrect := tulib.Rect{1, 0, 12, 1}

	g.Terminal.DrawLabel(labelrect, labelparams, []byte(fmt.Sprintf(" FPS: %5.2f ", g.fps)))

	fpsrect := tulib.Rect{14, 0, 9, 1}

	g.Terminal.DrawLabel(fpsrect, labelparams, []byte(fmt.Sprintf(" %dx%d ", g.Terminal.Rect.Width, g.Terminal.Rect.Height)))

	for _, o := range g.Objects {
		o.Draw(g)
	}

	//fps := g.CalcFPS(delta)
	//g.Printf(0, 0, termbox.ColorRed, termbox.ColorBlack, "FPS: %f", g.fps)

}

func (g *Game) calcFPS(delta time.Duration) float64 {
	fpsSamples[currentSample%NumFPSSamples] = 1.0 / delta.Seconds()
	currentSample++
	fps := 0.0

	for i := 0; i < NumFPSSamples; i++ {
		fps += fpsSamples[i]
	}

	fps /= NumFPSSamples

	return fps
}
<commit_msg>Added upercase support for directions<commit_after>package main

import (
	"flag"
	"fmt"
	"unicode"
	"github.com/nsf/termbox-go"
	"github.com/nsf/tulib"
	"log"
	"os"
	"time"
)

const (
	NumFPSSamples = 64
	FPSLimit      = 60
)

var (
	fpsSamples    [64]float64
	currentSample = 0
	logfile       = flag.String("log", "goland.log", "log file")
	debug         = flag.Bool("debug", false, "print debugging info")
)

type Game struct {
	P *Player

	Terminal
	logfile *os.File

	CloseChan chan bool

	// unexported
	fps float64

	Objects []Object

	Map *MapChunk
}

func NewGame() *Game {
	g := Game{}

	g.CloseChan = make(chan bool, 1)

	g.Map = NewMapChunk()
	g.Map.Tiles[30][22].Blocked = true
	g.Map.Tiles[30][22].SightBlocked = true
	g.Map.Tiles[30][22].Ch = MAP_WALL
	g.Map.Tiles[50][22].Blocked = true
	g.Map.Tiles[50][22].SightBlocked = true
	g.Map.Tiles[50][22].Ch = MAP_WALL

	g.P = NewPlayer(&g)
	g.P.Pos = Vector{10, 10}

	g.Objects = append(g.Objects, g.P)

	u := NewUnit(&g)
	u.Ch.Ch = '@'
	u.Pos = Vector{15, 15}

	g.Objects = append(g.Objects, &u)

	return &g
}

func (g *Game) Run() {

	g.Start()

	timer := NewDeltaTimer()
	ticker := time.NewTicker(time.Second / FPSLimit)

	run := true

	for run {
		select {
		case <-ticker.C:
			// frame tick
			delta := timer.DeltaTime()

			if delta.Seconds() > 0.25 {
				delta = time.Duration(250 * time.Millisecond)
			}

			g.Update(delta)
			g.Draw()

			g.Flush()

		case <-g.CloseChan:
			run = false
		}
	}

	g.End()

}

func (g *Game) Start() {
	f, err := os.OpenFile(*logfile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
	if err != nil {
		log.Fatal(err)
	}

	log.SetOutput(f)
	log.Print("Logging started")

	g.logfile = f

	g.Terminal.Start()

	g.HandleKey(termbox.KeyEsc, func(ev termbox.Event) { g.CloseChan <- false })

	scale := 2

	// convert to func SetupDirections()
	for k, v := range CARDINALS {
		func (c rune, d Direction) {
			g.HandleRune(c, func(_ termbox.Event) {
				g.P.Move(d)
			})

			upperc := unicode.ToUpper(c)
			g.HandleRune(upperc, func(_ termbox.Event) {
				for i := 0; i < scale; i++ {
					g.P.Move(d)
				}
			})
		}(k, v)
	}
}

func (g *Game) End() {
	log.Print("Logging ended")
	g.logfile.Close()
	g.Terminal.End()
}

func (g *Game) Update(delta time.Duration) {
	// update fps
	g.fps = g.calcFPS(delta)

	g.RunInputHandlers()

	for _, o := range g.Objects {
		o.Update(delta)
	}

}

func (g *Game) Draw() {
	//	g.Terminal.Draw()
	for x, col := range g.Map.Tiles {
		for y, tile := range col {
			if tile.CanSee() {
				termbox.SetCell(x, y, tile.Ch.Ch, tile.Ch.Fg, tile.Ch.Bg)
			} else {
				termbox.SetCell(x, y, tile.Ch.Ch, termbox.ColorBlack, termbox.ColorWhite)
			}
		}
	}

	labelparams := &tulib.LabelParams{termbox.ColorRed, termbox.ColorBlack, tulib.AlignCenter, '»', false}
	labelrect := tulib.Rect{1, 0, 12, 1}

	g.Terminal.DrawLabel(labelrect, labelparams, []byte(fmt.Sprintf(" FPS: %5.2f ", g.fps)))

	fpsrect := tulib.Rect{14, 0, 9, 1}

	g.Terminal.DrawLabel(fpsrect, labelparams, []byte(fmt.Sprintf(" %dx%d ", g.Terminal.Rect.Width, g.Terminal.Rect.Height)))

	for _, o := range g.Objects {
		o.Draw(g)
	}

	//fps := g.CalcFPS(delta)
	//g.Printf(0, 0, termbox.ColorRed, termbox.ColorBlack, "FPS: %f", g.fps)

}

func (g *Game) calcFPS(delta time.Duration) float64 {
	fpsSamples[currentSample%NumFPSSamples] = 1.0 / delta.Seconds()
	currentSample++
	fps := 0.0

	for i := 0; i < NumFPSSamples; i++ {
		fps += fpsSamples[i]
	}

	fps /= NumFPSSamples

	return fps
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/influxdata/influxdb"
	"github.com/influxdata/influxdb/bolt"
	"github.com/influxdata/influxdb/cmd/influx/internal"
	"github.com/influxdata/influxdb/http"
	"github.com/influxdata/influxdb/internal/fs"
	"github.com/influxdata/influxdb/kit/cli"
	"github.com/influxdata/influxdb/kv"
	"github.com/influxdata/influxdb/pkg/httpc"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"go.uber.org/zap"
)

const maxTCPConnections = 128

func main() {
	influxCmd := influxCmd()
	if err := influxCmd.Execute(); err != nil {
		seeHelp(influxCmd, nil)
		os.Exit(1)
	}
}

var (
	httpClient *httpc.Client
)

func newHTTPClient() (*httpc.Client, error) {
	if httpClient != nil {
		return httpClient, nil
	}

	c, err := http.NewHTTPClient(flags.host, flags.token, flags.skipVerify)
	if err != nil {
		return nil, err
	}

	httpClient = c
	return httpClient, nil
}

type (
	runEWrapFn func(fn func(*cobra.Command, []string) error) func(*cobra.Command, []string) error

	genericCLIOptFn func(*genericCLIOpts)
)

type genericCLIOpts struct {
	in         io.Reader
	w          io.Writer
	runEWrapFn runEWrapFn
}

func (o genericCLIOpts) newCmd(use string, runE func(*cobra.Command, []string) error) *cobra.Command {
	cmd := &cobra.Command{
		Use:  use,
		RunE: runE,
	}
	if runE != nil && o.runEWrapFn != nil {
		cmd.RunE = o.runEWrapFn(runE)
	}
	cmd.SetOut(o.w)
	return cmd
}

func in(r io.Reader) genericCLIOptFn {
	return func(o *genericCLIOpts) {
		o.in = r
	}
}

func out(w io.Writer) genericCLIOptFn {
	return func(o *genericCLIOpts) {
		o.w = w
	}
}

func runEWrap(fn runEWrapFn) genericCLIOptFn {
	return func(opts *genericCLIOpts) {
		opts.runEWrapFn = fn
	}
}

var flags struct {
	token      string
	host       string
	local      bool
	skipVerify bool
}

func influxCmd(opts ...genericCLIOptFn) *cobra.Command {
	opt := genericCLIOpts{
		in: os.Stdin,
		w:  os.Stdout,
	}
	for _, o := range opts {
		o(&opt)
	}

	cmd := opt.newCmd("influx", nil)
	cmd.Short = "Influx Client"
	cmd.SilenceUsage = true

	setViperOptions()

	runEWrapper := runEWrap(wrapCheckSetup)

	cmd.AddCommand(
		cmdAuth(),
		cmdBackup(),
		cmdBucket(runEWrapper),
		cmdDelete(),
		cmdOrganization(runEWrapper),
		cmdPing(),
		cmdPkg(runEWrapper),
		cmdQuery(),
		cmdTranspile(),
		cmdREPL(),
		cmdSetup(),
		cmdTask(),
		cmdUser(runEWrapper),
		cmdWrite(),
	)

	fOpts := flagOpts{
		{
			DestP:      &flags.token,
			Flag:       "token",
			Short:      't',
			Desc:       "API token to be used throughout client calls",
			Persistent: true,
		},
		{
			DestP:      &flags.host,
			Flag:       "host",
			Default:    "http://localhost:9999",
			Desc:       "HTTP address of Influx",
			Persistent: true,
		},
	}
	fOpts.mustRegister(cmd)

	// this is after the flagOpts register b/c we don't want to show the default value
	// in the usage display. This will add it as the token value, then if a token flag
	// is provided too, the flag will take precedence.
	flags.token = getTokenFromDefaultPath()

	cmd.PersistentFlags().BoolVar(&flags.local, "local", false, "Run commands locally against the filesystem")
	cmd.PersistentFlags().BoolVar(&flags.skipVerify, "skip-verify", false, "SkipVerify controls whether a client verifies the server's certificate chain and host name.")

	// Update help description for all commands in command tree
	walk(cmd, func(c *cobra.Command) {
		c.Flags().BoolP("help", "h", false, fmt.Sprintf("Help for the %s command ", c.Name()))
	})

	return cmd
}

func fetchSubCommand(parent *cobra.Command, args []string) *cobra.Command {
	var err error
	var cmd *cobra.Command

	// Workaround FAIL with "go test -v" or "cobra.test -test.v", see #155
	if args == nil && filepath.Base(os.Args[0]) != "cobra.test" {
		args = os.Args[1:]
	}

	if parent.TraverseChildren {
		cmd, _, err = parent.Traverse(args)
	} else {
		cmd, _, err = parent.Find(args)
	}
	// return nil if any errs
	if err != nil {
		return nil
	}
	return cmd
}

func seeHelp(c *cobra.Command, args []string) {
	if c = fetchSubCommand(c, args); c == nil {
		return //return here, since cobra already handles the error
	}
	c.Printf("See '%s -h' for help\n", c.CommandPath())
}

func defaultTokenPath() (string, string, error) {
	dir, err := fs.InfluxDir()
	if err != nil {
		return "", "", err
	}
	return filepath.Join(dir, http.DefaultTokenFile), dir, nil
}

func getTokenFromDefaultPath() string {
	path, _, err := defaultTokenPath()
	if err != nil {
		return ""
	}
	b, err := ioutil.ReadFile(path)
	if err != nil {
		return ""
	}
	return strings.TrimSpace(string(b))
}

func writeTokenToPath(tok, path, dir string) error {
	if err := os.MkdirAll(dir, os.ModePerm); err != nil {
		return err
	}
	return ioutil.WriteFile(path, []byte(tok), 0600)
}

func checkSetup(host string) error {
	s := &http.SetupService{
		Addr:               flags.host,
		InsecureSkipVerify: flags.skipVerify,
	}

	isOnboarding, err := s.IsOnboarding(context.Background())
	if err != nil {
		return err
	}

	if isOnboarding {
		return fmt.Errorf("the instance at %q has not been setup. Please run `influx setup` before issuing any additional commands", host)
	}

	return nil
}

func wrapCheckSetup(fn func(*cobra.Command, []string) error) func(*cobra.Command, []string) error {
	return wrapErrorFmt(func(cmd *cobra.Command, args []string) error {
		err := fn(cmd, args)
		if err == nil {
			return nil
		}

		if setupErr := checkSetup(flags.host); setupErr != nil && influxdb.EUnauthorized != influxdb.ErrorCode(setupErr) {
			return setupErr
		}

		return err
	})
}

func wrapErrorFmt(fn func(*cobra.Command, []string) error) func(*cobra.Command, []string) error {
	return func(cmd *cobra.Command, args []string) error {
		err := fn(cmd, args)
		if err == nil {
			return nil
		}

		return internal.ErrorFmt(err)
	}
}

// walk calls f for c and all of its children.
func walk(c *cobra.Command, f func(*cobra.Command)) {
	f(c)
	for _, c := range c.Commands() {
		walk(c, f)
	}
}

func newLocalKVService() (*kv.Service, error) {
	boltFile, err := fs.BoltFile()
	if err != nil {
		return nil, err
	}

	store := bolt.NewKVStore(zap.NewNop(), boltFile)
	if err := store.Open(context.Background()); err != nil {
		return nil, err
	}

	return kv.NewService(zap.NewNop(), store), nil
}

type organization struct {
	id, name string
}

func (o *organization) register(cmd *cobra.Command, persistent bool) {
	opts := flagOpts{
		{
			DestP:      &o.id,
			Flag:       "org-id",
			Desc:       "The ID of the organization",
			Persistent: persistent,
		},
		{
			DestP:      &o.name,
			Flag:       "org",
			Short:      'o',
			Desc:       "The name of the organization",
			Persistent: persistent,
		},
	}
	opts.mustRegister(cmd)
}

func (o *organization) getID(orgSVC influxdb.OrganizationService) (influxdb.ID, error) {
	if o.id != "" {
		influxOrgID, err := influxdb.IDFromString(o.id)
		if err != nil {
			return 0, fmt.Errorf("invalid org ID provided: %s", err.Error())
		}
		return *influxOrgID, nil
	} else if o.name != "" {
		org, err := orgSVC.FindOrganization(context.Background(), influxdb.OrganizationFilter{
			Name: &o.name,
		})
		if err != nil {
			return 0, fmt.Errorf("%v", err)
		}
		return org.ID, nil
	}
	return 0, fmt.Errorf("failed to locate an organization id")
}

func (o *organization) validOrgFlags() error {
	if o.id == "" && o.name == "" {
		return fmt.Errorf("must specify org-id, or org name")
	} else if o.id != "" && o.name != "" {
		return fmt.Errorf("must specify org-id, or org name not both")
	}
	return nil
}

type flagOpts []cli.Opt

func (f flagOpts) mustRegister(cmd *cobra.Command) {
	for i := range f {
		envVar := f[i].Flag
		if e := f[i].EnvVar; e != "" {
			envVar = e
		}

		f[i].Desc = fmt.Sprintf(
			"%s; Maps to env var $INFLUX_%s",
			f[i].Desc,
			strings.ToUpper(strings.Replace(envVar, "-", "_", -1)),
		)
	}
	cli.BindOptions(cmd, f)
}

func setViperOptions() {
	viper.SetEnvPrefix("INFLUX")
	viper.AutomaticEnv()
	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
}

func newBucketService() (influxdb.BucketService, error) {
	if flags.local {
		return newLocalKVService()
	}

	client, err := newHTTPClient()
	if err != nil {
		return nil, err
	}

	return &http.BucketService{
		Client: client,
	}, nil
}
<commit_msg>chore(pkg): remove ambiguity in wrong cmd calls within influx pkg cmd<commit_after>package main

import (
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/influxdata/influxdb"
	"github.com/influxdata/influxdb/bolt"
	"github.com/influxdata/influxdb/cmd/influx/internal"
	"github.com/influxdata/influxdb/http"
	"github.com/influxdata/influxdb/internal/fs"
	"github.com/influxdata/influxdb/kit/cli"
	"github.com/influxdata/influxdb/kv"
	"github.com/influxdata/influxdb/pkg/httpc"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"go.uber.org/zap"
)

const maxTCPConnections = 128

func main() {
	influxCmd := influxCmd()
	if err := influxCmd.Execute(); err != nil {
		seeHelp(influxCmd, nil)
		os.Exit(1)
	}
}

var (
	httpClient *httpc.Client
)

func newHTTPClient() (*httpc.Client, error) {
	if httpClient != nil {
		return httpClient, nil
	}

	c, err := http.NewHTTPClient(flags.host, flags.token, flags.skipVerify)
	if err != nil {
		return nil, err
	}

	httpClient = c
	return httpClient, nil
}

type (
	runEWrapFn func(fn func(*cobra.Command, []string) error) func(*cobra.Command, []string) error

	genericCLIOptFn func(*genericCLIOpts)
)

type genericCLIOpts struct {
	in         io.Reader
	w          io.Writer
	runEWrapFn runEWrapFn
}

func (o genericCLIOpts) newCmd(use string, runE func(*cobra.Command, []string) error) *cobra.Command {
	cmd := &cobra.Command{
		Args: cobra.NoArgs,
		Use:  use,
		RunE: runE,
	}
	if runE != nil && o.runEWrapFn != nil {
		cmd.RunE = o.runEWrapFn(runE)
	}
	cmd.SetOut(o.w)
	return cmd
}

func in(r io.Reader) genericCLIOptFn {
	return func(o *genericCLIOpts) {
		o.in = r
	}
}

func out(w io.Writer) genericCLIOptFn {
	return func(o *genericCLIOpts) {
		o.w = w
	}
}

func runEWrap(fn runEWrapFn) genericCLIOptFn {
	return func(opts *genericCLIOpts) {
		opts.runEWrapFn = fn
	}
}

var flags struct {
	token      string
	host       string
	local      bool
	skipVerify bool
}

func influxCmd(opts ...genericCLIOptFn) *cobra.Command {
	opt := genericCLIOpts{
		in: os.Stdin,
		w:  os.Stdout,
	}
	for _, o := range opts {
		o(&opt)
	}

	cmd := opt.newCmd("influx", nil)
	cmd.Short = "Influx Client"
	cmd.SilenceUsage = true

	setViperOptions()

	runEWrapper := runEWrap(wrapCheckSetup)

	cmd.AddCommand(
		cmdAuth(),
		cmdBackup(),
		cmdBucket(runEWrapper),
		cmdDelete(),
		cmdOrganization(runEWrapper),
		cmdPing(),
		cmdPkg(runEWrapper),
		cmdQuery(),
		cmdTranspile(),
		cmdREPL(),
		cmdSetup(),
		cmdTask(),
		cmdUser(runEWrapper),
		cmdWrite(),
	)

	fOpts := flagOpts{
		{
			DestP:      &flags.token,
			Flag:       "token",
			Short:      't',
			Desc:       "API token to be used throughout client calls",
			Persistent: true,
		},
		{
			DestP:      &flags.host,
			Flag:       "host",
			Default:    "http://localhost:9999",
			Desc:       "HTTP address of Influx",
			Persistent: true,
		},
	}
	fOpts.mustRegister(cmd)

	// this is after the flagOpts register b/c we don't want to show the default value
	// in the usage display. This will add it as the token value, then if a token flag
	// is provided too, the flag will take precedence.
	flags.token = getTokenFromDefaultPath()

	cmd.PersistentFlags().BoolVar(&flags.local, "local", false, "Run commands locally against the filesystem")
	cmd.PersistentFlags().BoolVar(&flags.skipVerify, "skip-verify", false, "SkipVerify controls whether a client verifies the server's certificate chain and host name.")

	// Update help description for all commands in command tree
	walk(cmd, func(c *cobra.Command) {
		c.Flags().BoolP("help", "h", false, fmt.Sprintf("Help for the %s command ", c.Name()))
	})

	return cmd
}

func fetchSubCommand(parent *cobra.Command, args []string) *cobra.Command {
	var err error
	var cmd *cobra.Command

	// Workaround FAIL with "go test -v" or "cobra.test -test.v", see #155
	if args == nil && filepath.Base(os.Args[0]) != "cobra.test" {
		args = os.Args[1:]
	}

	if parent.TraverseChildren {
		cmd, _, err = parent.Traverse(args)
	} else {
		cmd, _, err = parent.Find(args)
	}
	// return nil if any errs
	if err != nil {
		return nil
	}
	return cmd
}

func seeHelp(c *cobra.Command, args []string) {
	if c = fetchSubCommand(c, args); c == nil {
		return //return here, since cobra already handles the error
	}
	c.Printf("See '%s -h' for help\n", c.CommandPath())
}

func defaultTokenPath() (string, string, error) {
	dir, err := fs.InfluxDir()
	if err != nil {
		return "", "", err
	}
	return filepath.Join(dir, http.DefaultTokenFile), dir, nil
}

func getTokenFromDefaultPath() string {
	path, _, err := defaultTokenPath()
	if err != nil {
		return ""
	}
	b, err := ioutil.ReadFile(path)
	if err != nil {
		return ""
	}
	return strings.TrimSpace(string(b))
}

func writeTokenToPath(tok, path, dir string) error {
	if err := os.MkdirAll(dir, os.ModePerm); err != nil {
		return err
	}
	return ioutil.WriteFile(path, []byte(tok), 0600)
}

func checkSetup(host string) error {
	s := &http.SetupService{
		Addr:               flags.host,
		InsecureSkipVerify: flags.skipVerify,
	}

	isOnboarding, err := s.IsOnboarding(context.Background())
	if err != nil {
		return err
	}

	if isOnboarding {
		return fmt.Errorf("the instance at %q has not been setup. Please run `influx setup` before issuing any additional commands", host)
	}

	return nil
}

func wrapCheckSetup(fn func(*cobra.Command, []string) error) func(*cobra.Command, []string) error {
	return wrapErrorFmt(func(cmd *cobra.Command, args []string) error {
		err := fn(cmd, args)
		if err == nil {
			return nil
		}

		if setupErr := checkSetup(flags.host); setupErr != nil && influxdb.EUnauthorized != influxdb.ErrorCode(setupErr) {
			return setupErr
		}

		return err
	})
}

func wrapErrorFmt(fn func(*cobra.Command, []string) error) func(*cobra.Command, []string) error {
	return func(cmd *cobra.Command, args []string) error {
		err := fn(cmd, args)
		if err == nil {
			return nil
		}

		return internal.ErrorFmt(err)
	}
}

// walk calls f for c and all of its children.
func walk(c *cobra.Command, f func(*cobra.Command)) {
	f(c)
	for _, c := range c.Commands() {
		walk(c, f)
	}
}

func newLocalKVService() (*kv.Service, error) {
	boltFile, err := fs.BoltFile()
	if err != nil {
		return nil, err
	}

	store := bolt.NewKVStore(zap.NewNop(), boltFile)
	if err := store.Open(context.Background()); err != nil {
		return nil, err
	}

	return kv.NewService(zap.NewNop(), store), nil
}

type organization struct {
	id, name string
}

func (o *organization) register(cmd *cobra.Command, persistent bool) {
	opts := flagOpts{
		{
			DestP:      &o.id,
			Flag:       "org-id",
			Desc:       "The ID of the organization",
			Persistent: persistent,
		},
		{
			DestP:      &o.name,
			Flag:       "org",
			Short:      'o',
			Desc:       "The name of the organization",
			Persistent: persistent,
		},
	}
	opts.mustRegister(cmd)
}

func (o *organization) getID(orgSVC influxdb.OrganizationService) (influxdb.ID, error) {
	if o.id != "" {
		influxOrgID, err := influxdb.IDFromString(o.id)
		if err != nil {
			return 0, fmt.Errorf("invalid org ID provided: %s", err.Error())
		}
		return *influxOrgID, nil
	} else if o.name != "" {
		org, err := orgSVC.FindOrganization(context.Background(), influxdb.OrganizationFilter{
			Name: &o.name,
		})
		if err != nil {
			return 0, fmt.Errorf("%v", err)
		}
		return org.ID, nil
	}
	return 0, fmt.Errorf("failed to locate an organization id")
}

func (o *organization) validOrgFlags() error {
	if o.id == "" && o.name == "" {
		return fmt.Errorf("must specify org-id, or org name")
	} else if o.id != "" && o.name != "" {
		return fmt.Errorf("must specify org-id, or org name not both")
	}
	return nil
}

type flagOpts []cli.Opt

func (f flagOpts) mustRegister(cmd *cobra.Command) {
	for i := range f {
		envVar := f[i].Flag
		if e := f[i].EnvVar; e != "" {
			envVar = e
		}

		f[i].Desc = fmt.Sprintf(
			"%s; Maps to env var $INFLUX_%s",
			f[i].Desc,
			strings.ToUpper(strings.Replace(envVar, "-", "_", -1)),
		)
	}
	cli.BindOptions(cmd, f)
}

func setViperOptions() {
	viper.SetEnvPrefix("INFLUX")
	viper.AutomaticEnv()
	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
}

func newBucketService() (influxdb.BucketService, error) {
	if flags.local {
		return newLocalKVService()
	}

	client, err := newHTTPClient()
	if err != nil {
		return nil, err
	}

	return &http.BucketService{
		Client: client,
	}, nil
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"os"

	"github.com/loadimpact/k6/stats/cloud"
	"github.com/loadimpact/k6/ui"
	"github.com/pkg/errors"
        "github.com/spf13/cobra"
)

// loginCloudCommand represents the 'login cloud' command
var loginCloudCommand = &cobra.Command{
        Use:   "cloud",
	Short: "Authenticate with Load Impact",
        Long: `Authenticate with Load Impact.

This will set the default token used when just "k6 run -o cloud" is passed.`,
        Example: `
  # Show the stored token.
  k6 login cloud -s

  # Store a token.
  k6 login cloud -t YOUR_TOKEN

  # Log in with an email/password.
  k6 login cloud`[1:],
        Args: cobra.NoArgs,
        RunE: func(cmd *cobra.Command, args []string) error {
                config, cdir, err := readDiskConfig()
                if err != nil {
                        return err
                }

                show := getNullBool(cmd.Flags(), "show")
                token := getNullString(cmd.Flags(), "token")

                conf := config.Collectors.Cloud

                switch {
                case show.Bool:
                case token.Valid:
                        conf.Token = token.String
                default:
                        form := ui.Form{
                                Fields: []ui.Field{
                                        ui.StringField{
                                                Key:   "Email",
                                                Label: "Email",
					},
                                        ui.StringField{
                                                Key:   "Password",
                                                Label: "Password",
                                        },
                                },
                        }
                        vals, err := form.Run(os.Stdin, stdout)
                        if err != nil {
                                return err
                        }
                        email := vals["Email"].(string)
                        password := vals["Password"].(string)

                        client := cloud.NewClient("", conf.Host, Version)
                        res, err := client.Login(email, password)
                        if err != nil {
                                return err
                        }

                        if res.Token == "" {
                                return errors.New("Your account has no API token, please generate one: \"https://app.loadimpact.com/account/token\".")
                        }

                        conf.Token = res.Token
                }

                config.Collectors.Cloud = conf
                if err := writeDiskConfig(cdir, config); err != nil {
                        return err
                }

                fmt.Fprintf(stdout, "  token: %s\n", ui.ValueColor.Sprint(conf.Token))
                return nil
        },
}

func init() {
	loginCmd.AddCommand(loginCloudCommand)
	loginCloudCommand.Flags().StringP("token", "t", "", "specify `token` to use")
	loginCloudCommand.Flags().BoolP("show", "s", false, "display saved token and exit")
}
<commit_msg>Use backticks instead of quotes"<commit_after>package cmd

import (
	"fmt"
	"os"

	"github.com/loadimpact/k6/stats/cloud"
	"github.com/loadimpact/k6/ui"
	"github.com/pkg/errors"
        "github.com/spf13/cobra"
)

// loginCloudCommand represents the 'login cloud' command
var loginCloudCommand = &cobra.Command{
        Use:   "cloud",
	Short: "Authenticate with Load Impact",
        Long: `Authenticate with Load Impact.

This will set the default token used when just "k6 run -o cloud" is passed.`,
        Example: `
  # Show the stored token.
  k6 login cloud -s

  # Store a token.
  k6 login cloud -t YOUR_TOKEN

  # Log in with an email/password.
  k6 login cloud`[1:],
        Args: cobra.NoArgs,
        RunE: func(cmd *cobra.Command, args []string) error {
                config, cdir, err := readDiskConfig()
                if err != nil {
                        return err
                }

                show := getNullBool(cmd.Flags(), "show")
                token := getNullString(cmd.Flags(), "token")

                conf := config.Collectors.Cloud

                switch {
                case show.Bool:
                case token.Valid:
                        conf.Token = token.String
                default:
                        form := ui.Form{
                                Fields: []ui.Field{
                                        ui.StringField{
                                                Key:   "Email",
                                                Label: "Email",
					},
                                        ui.StringField{
                                                Key:   "Password",
                                                Label: "Password",
                                        },
                                },
                        }
                        vals, err := form.Run(os.Stdin, stdout)
                        if err != nil {
                                return err
                        }
                        email := vals["Email"].(string)
                        password := vals["Password"].(string)

                        client := cloud.NewClient("", conf.Host, Version)
                        res, err := client.Login(email, password)
                        if err != nil {
                                return err
                        }

                        if res.Token == "" {
                                return errors.New(`Your account has no API token, please generate one: "https://app.loadimpact.com/account/token".`)
                        }

                        conf.Token = res.Token
                }

                config.Collectors.Cloud = conf
                if err := writeDiskConfig(cdir, config); err != nil {
                        return err
                }

                fmt.Fprintf(stdout, "  token: %s\n", ui.ValueColor.Sprint(conf.Token))
                return nil
        },
}

func init() {
	loginCmd.AddCommand(loginCloudCommand)
	loginCloudCommand.Flags().StringP("token", "t", "", "specify `token` to use")
	loginCloudCommand.Flags().BoolP("show", "s", false, "display saved token and exit")
}
<|endoftext|>