<commit_before>package cmd

import (
	"encoding/json"
	"fmt"
	"log"
	"net"
	"os"
	"reflect"
	"strings"

	"github.com/exoscale/egoscale"
	"github.com/spf13/cobra"
)

// apiCmd represents the api command
var apiCmd = &cobra.Command{
	Use:   "api",
	Short: "Exoscale api",
}

const userDocumentationURL = "http://cloudstack.apache.org/api/apidocs-4.4/user/%s.html"

// global flags
var apiDebug bool
var apiDryRun bool

func init() {
	RootCmd.AddCommand(apiCmd)
	buildCommands(methods)
	apiCmd.PersistentFlags().BoolVarP(&apiDebug, "debug", "d", false, "debug mode on")
	apiCmd.PersistentFlags().BoolVarP(&apiDryRun, "dry-run", "D", false, "produce a cURL ready URL")
	if err := apiCmd.PersistentFlags().MarkHidden("dry-run"); err != nil {
		log.Fatal(err)
	}
}

func buildCommands(methods []category) {
	for _, category := range methods {

		cmd := cobra.Command{
			Use:     category.name,
			Aliases: category.alias,
			Short:   category.doc,
		}

		apiCmd.AddCommand(&cmd)

		for i := range category.cmd {
			s := category.cmd[i]

			realName := cs.APIName(s.command)
			description := cs.APIDescription(s.command)

			url := userDocumentationURL

			name := realName
			if s.name != "" {
				name = s.name
			}

			hiddenCMD := cobra.Command{
				Use:    realName,
				Short:  description,
				Long:   fmt.Sprintf("%s <%s>", description, fmt.Sprintf(url, realName)),
				Hidden: true,
			}

			subCMD := cobra.Command{
				Use:     name,
				Short:   description,
				Long:    fmt.Sprintf("%s <%s>", description, fmt.Sprintf(url, realName)),
				Aliases: append(s.alias, realName),
			}

			buildFlags(s.command, &subCMD)
			buildFlags(s.command, &hiddenCMD)

			runCMD := func(cmd *cobra.Command, args []string) error {

				// Show request and quit DEBUG
				if apiDebug {
					payload, err := cs.Payload(s.command)
					if err != nil {
						log.Fatal(err)
					}
					qs := payload.Encode()
					if _, err = fmt.Fprintf(os.Stdout, "%s\\\n?%s", cs.Endpoint, strings.Replace(qs, "&", "\\\n&", -1)); err != nil {
						log.Fatal(err)
					}

					if _, err := fmt.Fprintln(os.Stdout); err != nil {
						log.Fatal(err)
					}
					os.Exit(0)
				}

				if apiDryRun {
					payload, err := cs.Payload(s.command)
					if err != nil {
						log.Fatal(err)
					}
					signature, err := cs.Sign(payload)
					if err != nil {
						log.Fatal(err)
					}

					if _, err := fmt.Fprintf(os.Stdout, "%s?%s\n", cs.Endpoint, signature); err != nil {
						log.Fatal(err)
					}
					os.Exit(0)
				}

				// End debug section

				resp, err := cs.RequestWithContext(gContext, s.command)
				if err != nil {
					return err
				}

				data, err := json.MarshalIndent(&resp, "", "  ")
				if err != nil {
					return err
				}

				fmt.Println(string(data))

				return nil
			}

			subCMD.RunE = runCMD
			hiddenCMD.RunE = runCMD

			subCMD.Flags().SortFlags = false
			hiddenCMD.Flags().SortFlags = false

			cmd.AddCommand(&subCMD)
			apiCmd.AddCommand(&hiddenCMD)
		}
	}
}

func buildFlags(method egoscale.Command, cmd *cobra.Command) {
	val := reflect.ValueOf(method)
	// we've got a pointer
	value := val.Elem()

	if value.Kind() != reflect.Struct {
		log.Fatalf("struct was expected")
		return
	}

	ty := value.Type()
	for i := 0; i < value.NumField(); i++ {
		field := ty.Field(i)

		if field.Name == "_" {
			continue
		}

		// XXX refactor with request.go
		var argName string
		required := false
		if json, ok := field.Tag.Lookup("json"); ok {
			tags := strings.Split(json, ",")
			argName = tags[0]
			required = true
			for _, tag := range tags {
				if tag == "omitempty" {
					required = false
				}
			}
			if argName == "" || argName == "omitempty" {
				continue
			}
		}

		description := ""
		if required {
			description = "required"
		}

		if doc, ok := field.Tag.Lookup("doc"); ok {
			if description != "" {
				description = fmt.Sprintf("[%s] %s", description, doc)
			} else {
				description = doc
			}
		}

		val := value.Field(i)
		addr := val.Addr().Interface()
		switch val.Kind() {
		case reflect.Bool:
			cmd.Flags().BoolVarP(addr.(*bool), argName, "", false, description)
		case reflect.Int:
			cmd.Flags().IntVarP(addr.(*int), argName, "", 0, description)
		case reflect.Int64:
			cmd.Flags().Int64VarP(addr.(*int64), argName, "", 0, description)
		case reflect.Uint:
			cmd.Flags().UintVarP(addr.(*uint), argName, "", 0, description)
		case reflect.Uint64:
			cmd.Flags().Uint64VarP(addr.(*uint64), argName, "", 0, description)
		case reflect.Float64:
			cmd.Flags().Float64VarP(addr.(*float64), argName, "", 0, description)
		case reflect.Int16:
			typeName := field.Type.Name()
			if typeName != "int16" {
				cmd.Flags().VarP(&intTypeGeneric{addr: addr, base: 10, bitSize: 16, typ: field.Type}, argName, "", description)
			} else {
				cmd.Flags().Int16VarP(addr.(*int16), argName, "", 0, description)
			}
		case reflect.Uint8:
			cmd.Flags().Uint8VarP(addr.(*uint8), argName, "", 0, description)
		case reflect.Uint16:
			cmd.Flags().Uint16VarP(addr.(*uint16), argName, "", 0, description)
		case reflect.String:
			typeName := field.Type.Name()
			if typeName != "string" {
				cmd.Flags().VarP(&stringerTypeGeneric{addr: addr, typ: field.Type}, argName, "", description)
			} else {
				cmd.Flags().StringVarP(addr.(*string), argName, "", "", description)
			}
		case reflect.Slice:
			switch field.Type.Elem().Kind() {
			case reflect.Uint8:
				ip := addr.(*net.IP)
				if *ip == nil || (*ip).Equal(net.IPv4zero) || (*ip).Equal(net.IPv6zero) {
					cmd.Flags().IPP(argName, "", *ip, description)
				}
			case reflect.String:
				cmd.Flags().StringSliceP(argName, "", *addr.(*[]string), description)
			default:
				switch field.Type.Elem() {
				case reflect.TypeOf(egoscale.ResourceTag{}):
					cmd.Flags().VarP(&tagGeneric{addr.(*[]egoscale.ResourceTag)}, argName, "", description)
				case reflect.TypeOf(egoscale.CIDR{}):
					cmd.Flags().VarP(&cidrListGeneric{addr.(*[]egoscale.CIDR)}, argName, "", description)
				case reflect.TypeOf(egoscale.UUID{}):
					cmd.Flags().VarP(&uuidListGeneric{addr.(*[]egoscale.UUID)}, argName, "", description)
				default:
					//log.Printf("[SKIP] Slice of %s is not supported!", field.Name)
				}
			}
		case reflect.Map:
			key := reflect.TypeOf(val.Interface()).Key()
			switch key.Kind() {
			case reflect.String:
				cmd.Flags().VarP(&mapGeneric{addr.(*map[string]string)}, argName, "", description)
			default:
				log.Printf("[SKIP] Type map for %s is not supported!", field.Name)
			}
		case reflect.Ptr:
			switch field.Type.Elem() {
			case reflect.TypeOf(true):
				cmd.Flags().VarP(&boolFlag{(addr.(**bool))}, argName, "", description)
			case reflect.TypeOf(egoscale.CIDR{}):
				cmd.Flags().VarP(&cidr{addr.(**egoscale.CIDR)}, argName, "", description)
			case reflect.TypeOf(egoscale.UUID{}):
				cmd.Flags().VarP(&uuid{addr.(**egoscale.UUID)}, argName, "", description)

			default:
				log.Printf("[SKIP] Ptr type of %s is not supported!", field.Name)
			}
		default:
			log.Printf("[SKIP] Type of %s is not supported! %v", field.Name, val.Kind())
		}
	}
}
<commit_msg>Fix #303 (exo api -D does strange things)<commit_after>package cmd

import (
	"encoding/json"
	"fmt"
	"log"
	"net"
	"os"
	"reflect"
	"strings"

	"github.com/exoscale/egoscale"
	"github.com/spf13/cobra"
)

// apiCmd represents the api command
var apiCmd = &cobra.Command{
	Use:   "api",
	Short: "Exoscale api",
}

const userDocumentationURL = "http://cloudstack.apache.org/api/apidocs-4.4/user/%s.html"

// global flags
var apiDebug bool
var apiDryRun bool

func init() {
	RootCmd.AddCommand(apiCmd)
	buildCommands(methods)
	apiCmd.PersistentFlags().BoolVarP(&apiDebug, "debug", "d", false, "debug mode on")
	apiCmd.PersistentFlags().BoolVarP(&apiDryRun, "dry-run", "D", false, "produce a cURL ready URL")
	if err := apiCmd.PersistentFlags().MarkHidden("dry-run"); err != nil {
		log.Fatal(err)
	}
}

func buildCommands(methods []category) {
	for _, category := range methods {

		cmd := cobra.Command{
			Use:     category.name,
			Aliases: category.alias,
			Short:   category.doc,
		}

		apiCmd.AddCommand(&cmd)

		for i := range category.cmd {
			s := category.cmd[i]

			realName := cs.APIName(s.command)
			description := cs.APIDescription(s.command)

			url := userDocumentationURL

			name := realName
			if s.name != "" {
				name = s.name
			}

			hiddenCMD := cobra.Command{
				Use:    realName,
				Short:  description,
				Long:   fmt.Sprintf("%s <%s>", description, fmt.Sprintf(url, realName)),
				Hidden: true,
			}

			subCMD := cobra.Command{
				Use:     name,
				Short:   description,
				Long:    fmt.Sprintf("%s <%s>", description, fmt.Sprintf(url, realName)),
				Aliases: append(s.alias, realName),
			}

			buildFlags(s.command, &subCMD)
			buildFlags(s.command, &hiddenCMD)

			runCMD := func(cmd *cobra.Command, args []string) error {

				// Show request and quit DEBUG
				if apiDebug {
					payload, err := cs.Payload(s.command)
					if err != nil {
						log.Fatal(err)
					}
					qs := payload.Encode()
					if _, err = fmt.Fprintf(os.Stdout, "%s\\\n?%s", cs.Endpoint, strings.Replace(qs, "&", "\\\n&", -1)); err != nil {
						log.Fatal(err)
					}

					if _, err := fmt.Fprintln(os.Stdout); err != nil {
						log.Fatal(err)
					}
					os.Exit(0)
				}

				if apiDryRun {
					payload, err := cs.Payload(s.command)
					if err != nil {
						log.Fatal(err)
					}
					signature, err := cs.Sign(payload)
					if err != nil {
						log.Fatal(err)
					}

					payload.Add("signature", signature)

					if _, err := fmt.Fprintf(os.Stdout, "%s?%s\n", cs.Endpoint, payload.Encode()); err != nil {
						log.Fatal(err)
					}
					os.Exit(0)
				}

				// End debug section

				resp, err := cs.RequestWithContext(gContext, s.command)
				if err != nil {
					return err
				}

				data, err := json.MarshalIndent(&resp, "", "  ")
				if err != nil {
					return err
				}

				fmt.Println(string(data))

				return nil
			}

			subCMD.RunE = runCMD
			hiddenCMD.RunE = runCMD

			subCMD.Flags().SortFlags = false
			hiddenCMD.Flags().SortFlags = false

			cmd.AddCommand(&subCMD)
			apiCmd.AddCommand(&hiddenCMD)
		}
	}
}

func buildFlags(method egoscale.Command, cmd *cobra.Command) {
	val := reflect.ValueOf(method)
	// we've got a pointer
	value := val.Elem()

	if value.Kind() != reflect.Struct {
		log.Fatalf("struct was expected")
		return
	}

	ty := value.Type()
	for i := 0; i < value.NumField(); i++ {
		field := ty.Field(i)

		if field.Name == "_" {
			continue
		}

		// XXX refactor with request.go
		var argName string
		required := false
		if json, ok := field.Tag.Lookup("json"); ok {
			tags := strings.Split(json, ",")
			argName = tags[0]
			required = true
			for _, tag := range tags {
				if tag == "omitempty" {
					required = false
				}
			}
			if argName == "" || argName == "omitempty" {
				continue
			}
		}

		description := ""
		if required {
			description = "required"
		}

		if doc, ok := field.Tag.Lookup("doc"); ok {
			if description != "" {
				description = fmt.Sprintf("[%s] %s", description, doc)
			} else {
				description = doc
			}
		}

		val := value.Field(i)
		addr := val.Addr().Interface()
		switch val.Kind() {
		case reflect.Bool:
			cmd.Flags().BoolVarP(addr.(*bool), argName, "", false, description)
		case reflect.Int:
			cmd.Flags().IntVarP(addr.(*int), argName, "", 0, description)
		case reflect.Int64:
			cmd.Flags().Int64VarP(addr.(*int64), argName, "", 0, description)
		case reflect.Uint:
			cmd.Flags().UintVarP(addr.(*uint), argName, "", 0, description)
		case reflect.Uint64:
			cmd.Flags().Uint64VarP(addr.(*uint64), argName, "", 0, description)
		case reflect.Float64:
			cmd.Flags().Float64VarP(addr.(*float64), argName, "", 0, description)
		case reflect.Int16:
			typeName := field.Type.Name()
			if typeName != "int16" {
				cmd.Flags().VarP(&intTypeGeneric{addr: addr, base: 10, bitSize: 16, typ: field.Type}, argName, "", description)
			} else {
				cmd.Flags().Int16VarP(addr.(*int16), argName, "", 0, description)
			}
		case reflect.Uint8:
			cmd.Flags().Uint8VarP(addr.(*uint8), argName, "", 0, description)
		case reflect.Uint16:
			cmd.Flags().Uint16VarP(addr.(*uint16), argName, "", 0, description)
		case reflect.String:
			typeName := field.Type.Name()
			if typeName != "string" {
				cmd.Flags().VarP(&stringerTypeGeneric{addr: addr, typ: field.Type}, argName, "", description)
			} else {
				cmd.Flags().StringVarP(addr.(*string), argName, "", "", description)
			}
		case reflect.Slice:
			switch field.Type.Elem().Kind() {
			case reflect.Uint8:
				ip := addr.(*net.IP)
				if *ip == nil || (*ip).Equal(net.IPv4zero) || (*ip).Equal(net.IPv6zero) {
					cmd.Flags().IPP(argName, "", *ip, description)
				}
			case reflect.String:
				cmd.Flags().StringSliceP(argName, "", *addr.(*[]string), description)
			default:
				switch field.Type.Elem() {
				case reflect.TypeOf(egoscale.ResourceTag{}):
					cmd.Flags().VarP(&tagGeneric{addr.(*[]egoscale.ResourceTag)}, argName, "", description)
				case reflect.TypeOf(egoscale.CIDR{}):
					cmd.Flags().VarP(&cidrListGeneric{addr.(*[]egoscale.CIDR)}, argName, "", description)
				case reflect.TypeOf(egoscale.UUID{}):
					cmd.Flags().VarP(&uuidListGeneric{addr.(*[]egoscale.UUID)}, argName, "", description)
				default:
					//log.Printf("[SKIP] Slice of %s is not supported!", field.Name)
				}
			}
		case reflect.Map:
			key := reflect.TypeOf(val.Interface()).Key()
			switch key.Kind() {
			case reflect.String:
				cmd.Flags().VarP(&mapGeneric{addr.(*map[string]string)}, argName, "", description)
			default:
				log.Printf("[SKIP] Type map for %s is not supported!", field.Name)
			}
		case reflect.Ptr:
			switch field.Type.Elem() {
			case reflect.TypeOf(true):
				cmd.Flags().VarP(&boolFlag{(addr.(**bool))}, argName, "", description)
			case reflect.TypeOf(egoscale.CIDR{}):
				cmd.Flags().VarP(&cidr{addr.(**egoscale.CIDR)}, argName, "", description)
			case reflect.TypeOf(egoscale.UUID{}):
				cmd.Flags().VarP(&uuid{addr.(**egoscale.UUID)}, argName, "", description)

			default:
				log.Printf("[SKIP] Ptr type of %s is not supported!", field.Name)
			}
		default:
			log.Printf("[SKIP] Type of %s is not supported! %v", field.Name, val.Kind())
		}
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"io/ioutil"
	"net/http"
	"runtime"
	"strconv"
	"sync"
	"time"

	"github.com/shirou/gopsutil/v3/mem"

	"github.com/minio/minio/internal/config/api"
	xioutil "github.com/minio/minio/internal/ioutil"
	"github.com/minio/minio/internal/logger"
)

type apiConfig struct {
	mu sync.RWMutex

	requestsDeadline time.Duration
	requestsPool     chan struct{}
	clusterDeadline  time.Duration
	listQuorum       string
	corsAllowOrigins []string
	// total drives per erasure set across pools.
	totalDriveCount          int
	replicationWorkers       int
	replicationFailedWorkers int
	transitionWorkers        int

	staleUploadsExpiry          time.Duration
	staleUploadsCleanupInterval time.Duration
	deleteCleanupInterval       time.Duration
	disableODirect              bool
	gzipObjects                 bool
}

const cgroupLimitFile = "/sys/fs/cgroup/memory/memory.limit_in_bytes"

func cgroupLimit(limitFile string) (limit uint64) {
	buf, err := ioutil.ReadFile(limitFile)
	if err != nil {
		return 9223372036854771712
	}
	limit, err = strconv.ParseUint(string(buf), 10, 64)
	if err != nil {
		return 9223372036854771712
	}
	return limit
}

func availableMemory() (available uint64) {
	available = 8 << 30 // Default to 8 GiB when we can't find the limits.

	if runtime.GOOS == "linux" {
		available = cgroupLimit(cgroupLimitFile)

		// No limit set, It's the highest positive signed 64-bit
		// integer (2^63-1), rounded down to multiples of 4096 (2^12),
		// the most common page size on x86 systems - for cgroup_limits.
		if available != 9223372036854771712 {
			// This means cgroup memory limit is configured.
			return
		} // no-limit set proceed to set the limits based on virtual memory.

	} // for all other platforms limits are based on virtual memory.

	memStats, err := mem.VirtualMemory()
	if err != nil {
		return
	}

	available = memStats.Available / 2
	return
}

func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
	t.mu.Lock()
	defer t.mu.Unlock()

	t.clusterDeadline = cfg.ClusterDeadline
	t.corsAllowOrigins = cfg.CorsAllowOrigin
	maxSetDrives := 0
	for _, setDriveCount := range setDriveCounts {
		t.totalDriveCount += setDriveCount
		if setDriveCount > maxSetDrives {
			maxSetDrives = setDriveCount
		}
	}

	var apiRequestsMaxPerNode int
	if cfg.RequestsMax <= 0 {
		maxMem := availableMemory()

		// max requests per node is calculated as
		// total_ram / ram_per_request
		// ram_per_request is (2MiB+128KiB) * driveCount \
		//    + 2 * 10MiB (default erasure block size v1) + 2 * 1MiB (default erasure block size v2)
		blockSize := xioutil.BlockSizeLarge + xioutil.BlockSizeSmall
		apiRequestsMaxPerNode = int(maxMem / uint64(maxSetDrives*blockSize+int(blockSizeV1*2+blockSizeV2*2)))
		logger.Info("Automatically configured API requests per node based on available memory on the system: %d", apiRequestsMaxPerNode)
	} else {
		apiRequestsMaxPerNode = cfg.RequestsMax
		if len(globalEndpoints.Hostnames()) > 0 {
			apiRequestsMaxPerNode /= len(globalEndpoints.Hostnames())
		}
	}

	if cap(t.requestsPool) != apiRequestsMaxPerNode {
		// Only replace if needed.
		// Existing requests will use the previous limit,
		// but new requests will use the new limit.
		// There will be a short overlap window,
		// but this shouldn't last long.
		t.requestsPool = make(chan struct{}, apiRequestsMaxPerNode)
	}
	t.requestsDeadline = cfg.RequestsDeadline
	t.listQuorum = cfg.ListQuorum
	if globalReplicationPool != nil &&
		cfg.ReplicationWorkers != t.replicationWorkers {
		globalReplicationPool.ResizeFailedWorkers(cfg.ReplicationFailedWorkers)
		globalReplicationPool.ResizeWorkers(cfg.ReplicationWorkers)
	}
	t.replicationFailedWorkers = cfg.ReplicationFailedWorkers
	t.replicationWorkers = cfg.ReplicationWorkers
	if globalTransitionState != nil && cfg.TransitionWorkers != t.transitionWorkers {
		globalTransitionState.UpdateWorkers(cfg.TransitionWorkers)
	}
	t.transitionWorkers = cfg.TransitionWorkers

	t.staleUploadsExpiry = cfg.StaleUploadsExpiry
	t.staleUploadsCleanupInterval = cfg.StaleUploadsCleanupInterval
	t.deleteCleanupInterval = cfg.DeleteCleanupInterval
	t.disableODirect = cfg.DisableODirect
	t.gzipObjects = cfg.GzipObjects
}

func (t *apiConfig) isDisableODirect() bool {
	t.mu.RLock()
	defer t.mu.RUnlock()

	return t.disableODirect
}

func (t *apiConfig) shouldGzipObjects() bool {
	t.mu.RLock()
	defer t.mu.RUnlock()

	return t.gzipObjects
}

func (t *apiConfig) getListQuorum() string {
	t.mu.RLock()
	defer t.mu.RUnlock()

	return t.listQuorum
}

func (t *apiConfig) getCorsAllowOrigins() []string {
	t.mu.RLock()
	defer t.mu.RUnlock()

	corsAllowOrigins := make([]string, len(t.corsAllowOrigins))
	copy(corsAllowOrigins, t.corsAllowOrigins)
	return corsAllowOrigins
}

func (t *apiConfig) getStaleUploadsCleanupInterval() time.Duration {
	t.mu.RLock()
	defer t.mu.RUnlock()

	if t.staleUploadsCleanupInterval == 0 {
		return 6 * time.Hour // default 6 hours
	}

	return t.staleUploadsCleanupInterval
}

func (t *apiConfig) getStaleUploadsExpiry() time.Duration {
	t.mu.RLock()
	defer t.mu.RUnlock()

	if t.staleUploadsExpiry == 0 {
		return 24 * time.Hour // default 24 hours
	}

	return t.staleUploadsExpiry
}

func (t *apiConfig) getDeleteCleanupInterval() time.Duration {
	t.mu.RLock()
	defer t.mu.RUnlock()

	if t.deleteCleanupInterval == 0 {
		return 5 * time.Minute // every 5 minutes
	}

	return t.deleteCleanupInterval
}

func (t *apiConfig) getClusterDeadline() time.Duration {
	t.mu.RLock()
	defer t.mu.RUnlock()

	if t.clusterDeadline == 0 {
		return 10 * time.Second
	}

	return t.clusterDeadline
}

func (t *apiConfig) getRequestsPool() (chan struct{}, time.Duration) {
	t.mu.RLock()
	defer t.mu.RUnlock()

	if t.requestsPool == nil {
		return nil, time.Duration(0)
	}

	return t.requestsPool, t.requestsDeadline
}

// maxClients throttles the S3 API calls
func maxClients(f http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		globalHTTPStats.incS3RequestsIncoming()

		if r.Header.Get(globalObjectPerfUserMetadata) == "" {
			if val := globalServiceFreeze.Load(); val != nil {
				if unlock, ok := val.(chan struct{}); ok && unlock != nil {
					// Wait until unfrozen.
					<-unlock
				}
			}
		}

		pool, deadline := globalAPIConfig.getRequestsPool()
		if pool == nil {
			f.ServeHTTP(w, r)
			return
		}

		globalHTTPStats.addRequestsInQueue(1)

		deadlineTimer := time.NewTimer(deadline)
		defer deadlineTimer.Stop()

		select {
		case pool <- struct{}{}:
			defer func() { <-pool }()
			globalHTTPStats.addRequestsInQueue(-1)
			f.ServeHTTP(w, r)
		case <-deadlineTimer.C:
			// Send a http timeout message
			writeErrorResponse(r.Context(), w,
				errorCodes.ToAPIErr(ErrOperationMaxedOut),
				r.URL)
			globalHTTPStats.addRequestsInQueue(-1)
			return
		case <-r.Context().Done():
			globalHTTPStats.addRequestsInQueue(-1)
			return
		}
	}
}

func (t *apiConfig) getReplicationFailedWorkers() int {
	t.mu.RLock()
	defer t.mu.RUnlock()

	return t.replicationFailedWorkers
}

func (t *apiConfig) getReplicationWorkers() int {
	t.mu.RLock()
	defer t.mu.RUnlock()

	return t.replicationWorkers
}

func (t *apiConfig) getTransitionWorkers() int {
	t.mu.RLock()
	defer t.mu.RUnlock()

	return t.transitionWorkers
}
<commit_msg>fix: for frozen calls return if client disconnects (#15062)<commit_after>// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"io/ioutil"
	"net/http"
	"runtime"
	"strconv"
	"sync"
	"time"

	"github.com/shirou/gopsutil/v3/mem"

	"github.com/minio/minio/internal/config/api"
	xioutil "github.com/minio/minio/internal/ioutil"
	"github.com/minio/minio/internal/logger"
)

type apiConfig struct {
	mu sync.RWMutex

	requestsDeadline time.Duration
	requestsPool     chan struct{}
	clusterDeadline  time.Duration
	listQuorum       string
	corsAllowOrigins []string
	// total drives per erasure set across pools.
	totalDriveCount          int
	replicationWorkers       int
	replicationFailedWorkers int
	transitionWorkers        int

	staleUploadsExpiry          time.Duration
	staleUploadsCleanupInterval time.Duration
	deleteCleanupInterval       time.Duration
	disableODirect              bool
	gzipObjects                 bool
}

const cgroupLimitFile = "/sys/fs/cgroup/memory/memory.limit_in_bytes"

func cgroupLimit(limitFile string) (limit uint64) {
	buf, err := ioutil.ReadFile(limitFile)
	if err != nil {
		return 9223372036854771712
	}
	limit, err = strconv.ParseUint(string(buf), 10, 64)
	if err != nil {
		return 9223372036854771712
	}
	return limit
}

func availableMemory() (available uint64) {
	available = 8 << 30 // Default to 8 GiB when we can't find the limits.

	if runtime.GOOS == "linux" {
		available = cgroupLimit(cgroupLimitFile)

		// No limit set, It's the highest positive signed 64-bit
		// integer (2^63-1), rounded down to multiples of 4096 (2^12),
		// the most common page size on x86 systems - for cgroup_limits.
		if available != 9223372036854771712 {
			// This means cgroup memory limit is configured.
			return
		} // no-limit set proceed to set the limits based on virtual memory.

	} // for all other platforms limits are based on virtual memory.

	memStats, err := mem.VirtualMemory()
	if err != nil {
		return
	}

	available = memStats.Available / 2
	return
}

func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
	t.mu.Lock()
	defer t.mu.Unlock()

	t.clusterDeadline = cfg.ClusterDeadline
	t.corsAllowOrigins = cfg.CorsAllowOrigin
	maxSetDrives := 0
	for _, setDriveCount := range setDriveCounts {
		t.totalDriveCount += setDriveCount
		if setDriveCount > maxSetDrives {
			maxSetDrives = setDriveCount
		}
	}

	var apiRequestsMaxPerNode int
	if cfg.RequestsMax <= 0 {
		maxMem := availableMemory()

		// max requests per node is calculated as
		// total_ram / ram_per_request
		// ram_per_request is (2MiB+128KiB) * driveCount \
		//    + 2 * 10MiB (default erasure block size v1) + 2 * 1MiB (default erasure block size v2)
		blockSize := xioutil.BlockSizeLarge + xioutil.BlockSizeSmall
		apiRequestsMaxPerNode = int(maxMem / uint64(maxSetDrives*blockSize+int(blockSizeV1*2+blockSizeV2*2)))
		logger.Info("Automatically configured API requests per node based on available memory on the system: %d", apiRequestsMaxPerNode)
	} else {
		apiRequestsMaxPerNode = cfg.RequestsMax
		if len(globalEndpoints.Hostnames()) > 0 {
			apiRequestsMaxPerNode /= len(globalEndpoints.Hostnames())
		}
	}

	if cap(t.requestsPool) != apiRequestsMaxPerNode {
		// Only replace if needed.
		// Existing requests will use the previous limit,
		// but new requests will use the new limit.
		// There will be a short overlap window,
		// but this shouldn't last long.
		t.requestsPool = make(chan struct{}, apiRequestsMaxPerNode)
	}
	t.requestsDeadline = cfg.RequestsDeadline
	t.listQuorum = cfg.ListQuorum
	if globalReplicationPool != nil &&
		cfg.ReplicationWorkers != t.replicationWorkers {
		globalReplicationPool.ResizeFailedWorkers(cfg.ReplicationFailedWorkers)
		globalReplicationPool.ResizeWorkers(cfg.ReplicationWorkers)
	}
	t.replicationFailedWorkers = cfg.ReplicationFailedWorkers
	t.replicationWorkers = cfg.ReplicationWorkers
	if globalTransitionState != nil && cfg.TransitionWorkers != t.transitionWorkers {
		globalTransitionState.UpdateWorkers(cfg.TransitionWorkers)
	}
	t.transitionWorkers = cfg.TransitionWorkers

	t.staleUploadsExpiry = cfg.StaleUploadsExpiry
	t.staleUploadsCleanupInterval = cfg.StaleUploadsCleanupInterval
	t.deleteCleanupInterval = cfg.DeleteCleanupInterval
	t.disableODirect = cfg.DisableODirect
	t.gzipObjects = cfg.GzipObjects
}

func (t *apiConfig) isDisableODirect() bool {
	t.mu.RLock()
	defer t.mu.RUnlock()

	return t.disableODirect
}

func (t *apiConfig) shouldGzipObjects() bool {
	t.mu.RLock()
	defer t.mu.RUnlock()

	return t.gzipObjects
}

func (t *apiConfig) getListQuorum() string {
	t.mu.RLock()
	defer t.mu.RUnlock()

	return t.listQuorum
}

func (t *apiConfig) getCorsAllowOrigins() []string {
	t.mu.RLock()
	defer t.mu.RUnlock()

	corsAllowOrigins := make([]string, len(t.corsAllowOrigins))
	copy(corsAllowOrigins, t.corsAllowOrigins)
	return corsAllowOrigins
}

func (t *apiConfig) getStaleUploadsCleanupInterval() time.Duration {
	t.mu.RLock()
	defer t.mu.RUnlock()

	if t.staleUploadsCleanupInterval == 0 {
		return 6 * time.Hour // default 6 hours
	}

	return t.staleUploadsCleanupInterval
}

func (t *apiConfig) getStaleUploadsExpiry() time.Duration {
	t.mu.RLock()
	defer t.mu.RUnlock()

	if t.staleUploadsExpiry == 0 {
		return 24 * time.Hour // default 24 hours
	}

	return t.staleUploadsExpiry
}

func (t *apiConfig) getDeleteCleanupInterval() time.Duration {
	t.mu.RLock()
	defer t.mu.RUnlock()

	if t.deleteCleanupInterval == 0 {
		return 5 * time.Minute // every 5 minutes
	}

	return t.deleteCleanupInterval
}

func (t *apiConfig) getClusterDeadline() time.Duration {
	t.mu.RLock()
	defer t.mu.RUnlock()

	if t.clusterDeadline == 0 {
		return 10 * time.Second
	}

	return t.clusterDeadline
}

func (t *apiConfig) getRequestsPool() (chan struct{}, time.Duration) {
	t.mu.RLock()
	defer t.mu.RUnlock()

	if t.requestsPool == nil {
		return nil, time.Duration(0)
	}

	return t.requestsPool, t.requestsDeadline
}

// maxClients throttles the S3 API calls
func maxClients(f http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		globalHTTPStats.incS3RequestsIncoming()

		if r.Header.Get(globalObjectPerfUserMetadata) == "" {
			if val := globalServiceFreeze.Load(); val != nil {
				if unlock, ok := val.(chan struct{}); ok && unlock != nil {
					// Wait until unfrozen.
					select {
					case <-unlock:
					case <-r.Context().Done():
						// if client canceled we don't need to wait here forever.
						return
					}
				}
			}
		}

		pool, deadline := globalAPIConfig.getRequestsPool()
		if pool == nil {
			f.ServeHTTP(w, r)
			return
		}

		globalHTTPStats.addRequestsInQueue(1)

		deadlineTimer := time.NewTimer(deadline)
		defer deadlineTimer.Stop()

		select {
		case pool <- struct{}{}:
			defer func() { <-pool }()
			globalHTTPStats.addRequestsInQueue(-1)
			f.ServeHTTP(w, r)
		case <-deadlineTimer.C:
			// Send a http timeout message
			writeErrorResponse(r.Context(), w,
				errorCodes.ToAPIErr(ErrOperationMaxedOut),
				r.URL)
			globalHTTPStats.addRequestsInQueue(-1)
			return
		case <-r.Context().Done():
			globalHTTPStats.addRequestsInQueue(-1)
			return
		}
	}
}

func (t *apiConfig) getReplicationFailedWorkers() int {
	t.mu.RLock()
	defer t.mu.RUnlock()

	return t.replicationFailedWorkers
}

func (t *apiConfig) getReplicationWorkers() int {
	t.mu.RLock()
	defer t.mu.RUnlock()

	return t.replicationWorkers
}

func (t *apiConfig) getTransitionWorkers() int {
	t.mu.RLock()
	defer t.mu.RUnlock()

	return t.transitionWorkers
}
<|endoftext|>


<commit_before>package shell

import (
	"log"
	"os"
	"os/exec"
	"os/signal"
	"reflect"
	"strings"
	"syscall"

	"bytes"

	"fmt"
	"github.com/gruntwork-io/terragrunt/errors"
	"github.com/gruntwork-io/terragrunt/options"
)

// Run the given Terraform command
func RunTerraformCommand(terragruntOptions *options.TerragruntOptions, args ...string) error {
	return RunShellCommand(terragruntOptions, terragruntOptions.TerraformPath, args...)
}

// Run the given Terraform command and return the stdout as a string
func RunTerraformCommandAndCaptureOutput(terragruntOptions *options.TerragruntOptions, args ...string) (string, error) {
	return RunShellCommandAndCaptureOutput(terragruntOptions, terragruntOptions.TerraformPath, args...)
}

// Run the specified shell command with the specified arguments. Connect the command's stdin, stdout, and stderr to
// the currently running app.
func RunShellCommand(terragruntOptions *options.TerragruntOptions, command string, args ...string) error {
	terragruntOptions.Logger.Printf("Running command: %s %s", command, strings.Join(args, " "))

	cmd := exec.Command(command, args...)

	// TODO: consider adding prefix from terragruntOptions logger to stdout and stderr
	cmd.Stdin = os.Stdin
	cmd.Stdout = terragruntOptions.Writer
	cmd.Stderr = terragruntOptions.ErrWriter
	cmd.Env = toEnvVarsList(terragruntOptions.Env)

	// Terragrunt can run some commands (such as terraform remote config) before running the actual terraform
	// command requested by the user. The output of these other commands should not end up on stdout as this
	// breaks scripts relying on terraform's output.
	if !reflect.DeepEqual(terragruntOptions.TerraformCliArgs, args) {
		cmd.Stdout = cmd.Stderr
	}

	cmd.Dir = terragruntOptions.WorkingDir

	cmdChannel := make(chan error)
	signalChannel := NewSignalsForwarder(forwardSignals, cmd, terragruntOptions.Logger, cmdChannel)
	defer signalChannel.Close()

	err := cmd.Run()
	cmdChannel <- err

	return errors.WithStackTrace(err)
}

func toEnvVarsList(envVarsAsMap map[string]string) []string {
	envVarsAsList := []string{}
	for key, value := range envVarsAsMap {
		envVarsAsList = append(envVarsAsList, fmt.Sprintf("%s=%s", key, value))
	}
	return envVarsAsList
}

// Run the specified shell command with the specified arguments. Capture the command's stdout and return it as a
// string.
func RunShellCommandAndCaptureOutput(terragruntOptions *options.TerragruntOptions, command string, args ...string) (string, error) {
	stdout := new(bytes.Buffer)

	terragruntOptionsCopy := terragruntOptions.Clone(terragruntOptions.TerragruntConfigPath)
	terragruntOptionsCopy.Writer = stdout
	terragruntOptionsCopy.ErrWriter = stdout

	err := RunShellCommand(terragruntOptionsCopy, command, args...)
	return stdout.String(), err
}

// Return the exit code of a command. If the error does not implement errors.IErrorCode or is not an exec.ExitError type,
// the error is returned.
func GetExitCode(err error) (int, error) {
	if exiterr, ok := errors.Unwrap(err).(errors.IErrorCode); ok {
		return exiterr.ExitStatus()
	}

	if exiterr, ok := errors.Unwrap(err).(*exec.ExitError); ok {
		status := exiterr.Sys().(syscall.WaitStatus)
		return status.ExitStatus(), nil
	}
	return 0, err
}

type SignalsForwarder chan os.Signal

// Forwards signals to a command, waiting for the command to finish.
func NewSignalsForwarder(signals []os.Signal, c *exec.Cmd, logger *log.Logger, cmdChannel chan error) SignalsForwarder {
	signalChannel := make(chan os.Signal, 1)
	signal.Notify(signalChannel, signals...)

	go func() {
		for {
			select {
			case s := <-signalChannel:
				logger.Printf("Forward signal %v to terraform.", s)
				err := c.Process.Signal(s)
				if err != nil {
					logger.Printf("Error forwarding signal: %v", err)
				}
			case <-cmdChannel:
				return
			}
		}
	}()

	return signalChannel
}

func (signalChannel *SignalsForwarder) Close() error {
	signal.Stop(*signalChannel)
	*signalChannel <- nil
	close(*signalChannel)
	return nil
}
<commit_msg>Fix data race<commit_after>package shell

import (
	"bytes"
	"fmt"
	"log"
	"os"
	"os/exec"
	"os/signal"
	"reflect"
	"strings"
	"syscall"

	"github.com/gruntwork-io/terragrunt/errors"
	"github.com/gruntwork-io/terragrunt/options"
)

// Run the given Terraform command
func RunTerraformCommand(terragruntOptions *options.TerragruntOptions, args ...string) error {
	return RunShellCommand(terragruntOptions, terragruntOptions.TerraformPath, args...)
}

// Run the given Terraform command and return the stdout as a string
func RunTerraformCommandAndCaptureOutput(terragruntOptions *options.TerragruntOptions, args ...string) (string, error) {
	return RunShellCommandAndCaptureOutput(terragruntOptions, terragruntOptions.TerraformPath, args...)
}

// Run the specified shell command with the specified arguments. Connect the command's stdin, stdout, and stderr to
// the currently running app.
func RunShellCommand(terragruntOptions *options.TerragruntOptions, command string, args ...string) error {
	terragruntOptions.Logger.Printf("Running command: %s %s", command, strings.Join(args, " "))

	cmd := exec.Command(command, args...)

	// TODO: consider adding prefix from terragruntOptions logger to stdout and stderr
	cmd.Stdin = os.Stdin
	cmd.Stdout = terragruntOptions.Writer
	cmd.Stderr = terragruntOptions.ErrWriter
	cmd.Env = toEnvVarsList(terragruntOptions.Env)

	// Terragrunt can run some commands (such as terraform remote config) before running the actual terraform
	// command requested by the user. The output of these other commands should not end up on stdout as this
	// breaks scripts relying on terraform's output.
	if !reflect.DeepEqual(terragruntOptions.TerraformCliArgs, args) {
		cmd.Stdout = cmd.Stderr
	}

	cmd.Dir = terragruntOptions.WorkingDir

	if err := cmd.Start(); err != nil {
		// bad path, binary not executable, &c
		return errors.WithStackTrace(err)
	}
	cmdChannel := make(chan error)
	signalChannel := NewSignalsForwarder(forwardSignals, cmd, terragruntOptions.Logger, cmdChannel)
	defer signalChannel.Close()

	err := cmd.Wait()
	cmdChannel <- err

	return errors.WithStackTrace(err)
}

func toEnvVarsList(envVarsAsMap map[string]string) []string {
	envVarsAsList := []string{}
	for key, value := range envVarsAsMap {
		envVarsAsList = append(envVarsAsList, fmt.Sprintf("%s=%s", key, value))
	}
	return envVarsAsList
}

// Run the specified shell command with the specified arguments. Capture the command's stdout and return it as a
// string.
func RunShellCommandAndCaptureOutput(terragruntOptions *options.TerragruntOptions, command string, args ...string) (string, error) {
	stdout := new(bytes.Buffer)

	terragruntOptionsCopy := terragruntOptions.Clone(terragruntOptions.TerragruntConfigPath)
	terragruntOptionsCopy.Writer = stdout
	terragruntOptionsCopy.ErrWriter = stdout

	err := RunShellCommand(terragruntOptionsCopy, command, args...)
	return stdout.String(), err
}

// Return the exit code of a command. If the error does not implement errors.IErrorCode or is not an exec.ExitError type,
// the error is returned.
func GetExitCode(err error) (int, error) {
	if exiterr, ok := errors.Unwrap(err).(errors.IErrorCode); ok {
		return exiterr.ExitStatus()
	}

	if exiterr, ok := errors.Unwrap(err).(*exec.ExitError); ok {
		status := exiterr.Sys().(syscall.WaitStatus)
		return status.ExitStatus(), nil
	}
	return 0, err
}

type SignalsForwarder chan os.Signal

// Forwards signals to a command, waiting for the command to finish.
func NewSignalsForwarder(signals []os.Signal, c *exec.Cmd, logger *log.Logger, cmdChannel chan error) SignalsForwarder {
	signalChannel := make(chan os.Signal, 1)
	signal.Notify(signalChannel, signals...)

	go func() {
		for {
			select {
			case s := <-signalChannel:
				logger.Printf("Forward signal %v to terraform.", s)
				err := c.Process.Signal(s)
				if err != nil {
					logger.Printf("Error forwarding signal: %v", err)
				}
			case <-cmdChannel:
				return
			}
		}
	}()

	return signalChannel
}

func (signalChannel *SignalsForwarder) Close() error {
	signal.Stop(*signalChannel)
	*signalChannel <- nil
	close(*signalChannel)
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"compress/gzip"
	"crypto/sha256"
	"encoding/hex"
	"io"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/superp00t/etc"
	"github.com/superp00t/etc/yo"
)

type diskStatus struct {
	All  uint64 `json:"all"`
	Used uint64 `json:"used"`
	Free uint64 `json:"free"`
}

type cacher struct {
	Handler http.Handler

	sync.Mutex
}

func hashString(name string) string {
	s := sha256.New()
	s.Write([]byte(name))
	return strings.ToUpper(hex.EncodeToString(s.Sum(nil)))
}

func (c *cacher) Available() uint64 {
	return directory.Concat("c").Free()
}

func (c *cacher) serveContent(rw http.ResponseWriter, r *http.Request, name, path string) {
	if strings.Contains(r.Header.Get("Accept-Ranges"), "-") {
		// Cannot serve compressed in this fashion
		http.ServeFile(rw, r, path)
		return
	}

	if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
		file, err := etc.FileController(path, true)
		if err != nil {
			yo.Warn("Cannot open file", path, err)
			return
		}

		tp := http.DetectContentType(file.ReadBytes(512))

		if strings.HasPrefix(tp, "text") {
			typeMap := map[string]string{
				"svg": "image/svg+xml; charset=utf-8",
				"css": "text/css; charset=utf-8",
			}

			s := strings.Split(name, ".")
			typ := s[len(s)-1]
			if typeMap[typ] != "" {
				tp = typeMap[typ]
			}
		}

		yo.Ok("type == ", tp)
		yo.Ok("content == ", path)

		rw.Header().Set("Content-Type", tp)
		rw.Header().Set("Content-Encoding", "gzip")

		file.SeekR(0)
		rw.WriteHeader(200)

		gz := gzip.NewWriter(rw)
		_, err = io.Copy(gz, file)
		if err != nil {
			yo.Warn(err)
		}
		gz.Close()
		file.Close()

		return
	}

	http.ServeFile(rw, r, path)
}

func (c *cacher) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
	pth := r.URL.Path[1:]
	yo.Ok("Serving", pth)
	hash := hashString(pth)
	pCachePath := directory.Concat("c").Concat(hash)
	pSrcPath := directory.Concat("i").GetSub(etc.ParseUnixPath(pth))

	if pCachePath.IsExtant() && time.Since(pCachePath.Time()) < Config.CacheDuration.Duration {
		// cached file exists.
		c.serveContent(rw, r, pth, pCachePath.Render())
		return
	}

	// Backend may be down. serve cached file in its place.
	if !pSrcPath.IsExtant() && pCachePath.IsExtant() {
		c.serveContent(rw, r, pth, pCachePath.Render())
		return
	}

	if pSrcPath.IsExtant() == false {
		http.Error(rw, "file not found", 404)
		return
	}

	// do not cache large files.
	if pSrcPath.IsExtant() && pSrcPath.Size() > 250*etc.MB {
		http.ServeFile(rw, r, pSrcPath.Render())
		return
	}

	cacheDir := directory.Concat("c")

	// delete oldest item in cache if we have not enough space.
	for cacheDir.Free() < pSrcPath.Size() || cacheDir.Size() > Config.MaxCacheBytes {
		yo.Ok("erasing until bytes free are more than", cacheDir.Free())

		lru, err := cacheDir.LRU()
		if err != nil {
			yo.Warn(err)
			break
		}

		cacheDir.Concat(lru).Remove()
	}

	pCachePath.Remove()

	f, err := etc.FileController(pCachePath.Render())
	if err != nil {
		yo.Fatal(err)
	}

	if err = f.Flush(); err != nil {
		yo.Fatal(err)
	}

	s, err := etc.FileController(pSrcPath.Render(), true)
	if err != nil {
		yo.Fatal(err)
	}

	if _, err = io.Copy(f, s); err != nil {
		yo.Fatal(err)
	}

	f.Close()
	s.Close()

	c.serveContent(rw, r, pth, pCachePath.Render())
}
<commit_msg>debug<commit_after>package main

import (
	"compress/gzip"
	"crypto/sha256"
	"encoding/hex"
	"io"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/superp00t/etc"
	"github.com/superp00t/etc/yo"
)

type diskStatus struct {
	All  uint64 `json:"all"`
	Used uint64 `json:"used"`
	Free uint64 `json:"free"`
}

type cacher struct {
	Handler http.Handler

	sync.Mutex
}

func hashString(name string) string {
	s := sha256.New()
	s.Write([]byte(name))
	return strings.ToUpper(hex.EncodeToString(s.Sum(nil)))
}

func (c *cacher) Available() uint64 {
	return directory.Concat("c").Free()
}

func (c *cacher) serveContent(rw http.ResponseWriter, r *http.Request, name, path string) {
	if strings.Contains(r.Header.Get("Accept-Ranges"), "-") {
		// Cannot serve compressed in this fashion
		http.ServeFile(rw, r, path)
		return
	}

	if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
		file, err := etc.FileController(path, true)
		if err != nil {
			yo.Warn("Cannot open file", path, err)
			return
		}

		tp := http.DetectContentType(file.ReadBytes(512))

		if strings.HasPrefix(tp, "text") {
			typeMap := map[string]string{
				"svg": "image/svg+xml; charset=utf-8",
				"css": "text/css; charset=utf-8",
			}

			s := strings.Split(name, ".")
			typ := s[len(s)-1]
			if typeMap[typ] != "" {
				tp = typeMap[typ]
			} else {
				yo.Warn(s)
				yo.Warn(typ)
				yo.Warn(tp)
			}
		}

		yo.Ok("type == ", tp)
		yo.Ok("content == ", path)

		rw.Header().Set("Content-Type", tp)
		rw.Header().Set("Content-Encoding", "gzip")

		file.SeekR(0)
		rw.WriteHeader(200)

		gz := gzip.NewWriter(rw)
		_, err = io.Copy(gz, file)
		if err != nil {
			yo.Warn(err)
		}
		gz.Close()
		file.Close()

		return
	}

	http.ServeFile(rw, r, path)
}

func (c *cacher) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
	pth := r.URL.Path[1:]
	yo.Ok("Serving", pth)
	hash := hashString(pth)
	pCachePath := directory.Concat("c").Concat(hash)
	pSrcPath := directory.Concat("i").GetSub(etc.ParseUnixPath(pth))

	if pCachePath.IsExtant() && time.Since(pCachePath.Time()) < Config.CacheDuration.Duration {
		// cached file exists.
		c.serveContent(rw, r, pth, pCachePath.Render())
		return
	}

	// Backend may be down. serve cached file in its place.
	if !pSrcPath.IsExtant() && pCachePath.IsExtant() {
		c.serveContent(rw, r, pth, pCachePath.Render())
		return
	}

	if pSrcPath.IsExtant() == false {
		http.Error(rw, "file not found", 404)
		return
	}

	// do not cache large files.
	if pSrcPath.IsExtant() && pSrcPath.Size() > 250*etc.MB {
		http.ServeFile(rw, r, pSrcPath.Render())
		return
	}

	cacheDir := directory.Concat("c")

	// delete oldest item in cache if we have not enough space.
	for cacheDir.Free() < pSrcPath.Size() || cacheDir.Size() > Config.MaxCacheBytes {
		yo.Ok("erasing until bytes free are more than", cacheDir.Free())

		lru, err := cacheDir.LRU()
		if err != nil {
			yo.Warn(err)
			break
		}

		cacheDir.Concat(lru).Remove()
	}

	pCachePath.Remove()

	f, err := etc.FileController(pCachePath.Render())
	if err != nil {
		yo.Fatal(err)
	}

	if err = f.Flush(); err != nil {
		yo.Fatal(err)
	}

	s, err := etc.FileController(pSrcPath.Render(), true)
	if err != nil {
		yo.Fatal(err)
	}

	if _, err = io.Copy(f, s); err != nil {
		yo.Fatal(err)
	}

	f.Close()
	s.Close()

	c.serveContent(rw, r, pth, pCachePath.Render())
}
<|endoftext|>


<commit_before>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2016 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package cmd

import (
	"os"
	"sync"

	"github.com/fatih/color"
	"github.com/mattn/go-isatty"
	log "github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
)

var Version = "0.17.1"
var Banner = `
          /\      |‾‾|  /‾‾/  /‾/
     /\  /  \     |  |_/  /  / /
    /  \/    \    |      |  /  ‾‾\
   /          \   |  |‾\  \ | (_) |
  / __________ \  |__|  \__\ \___/ .io`

var BannerColor = color.New(color.FgCyan)

var (
	outMutex  = &sync.Mutex{}
	stdoutTTY = isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd())
	stderrTTY = isatty.IsTerminal(os.Stderr.Fd()) || isatty.IsCygwinTerminal(os.Stderr.Fd())
	stdout    = consoleWriter{os.Stdout, stdoutTTY, outMutex}
	stderr    = consoleWriter{os.Stderr, stderrTTY, outMutex}
)

var (
	cfgFile string

	verbose bool
	quiet   bool
	address string
)

// RootCmd represents the base command when called without any subcommands.
var RootCmd = &cobra.Command{
	Use:           "k6",
	Short:         "a next-generation load generator",
	Long:          Banner,
	SilenceUsage:  true,
	SilenceErrors: true,
	PersistentPreRun: func(cmd *cobra.Command, args []string) {
		l := log.StandardLogger()
		l.Out = stderr
		l.Formatter = &log.TextFormatter{ForceColors: stderrTTY}
		if verbose {
			l.SetLevel(log.DebugLevel)
		}
	},
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		log.Error(err.Error())
		if e, ok := err.(ExitCode); ok {
			os.Exit(e.Code)
		}
		os.Exit(-1)
	}
}

func init() {
	RootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "enable debug logging")
	RootCmd.PersistentFlags().BoolVarP(&quiet, "quiet", "q", false, "disable progress updates")
	RootCmd.PersistentFlags().StringVarP(&address, "address", "a", "localhost:6565", "address for the api server")
	RootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "c", "", "config file (default ./k6.yaml or ~/.config/k6.yaml)")
	must(cobra.MarkFlagFilename(RootCmd.PersistentFlags(), "config"))
}
<commit_msg>Coloured banner, why not<commit_after>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2016 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package cmd

import (
	"os"
	"sync"

	"github.com/fatih/color"
	"github.com/mattn/go-isatty"
	log "github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
)

var Version = "0.17.1"
var Banner = `
          /\      |‾‾|  /‾‾/  /‾/
     /\  /  \     |  |_/  /  / /
    /  \/    \    |      |  /  ‾‾\
   /          \   |  |‾\  \ | (_) |
  / __________ \  |__|  \__\ \___/ .io`

var BannerColor = color.New(color.FgCyan)

var (
	outMutex  = &sync.Mutex{}
	stdoutTTY = isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd())
	stderrTTY = isatty.IsTerminal(os.Stderr.Fd()) || isatty.IsCygwinTerminal(os.Stderr.Fd())
	stdout    = consoleWriter{os.Stdout, stdoutTTY, outMutex}
	stderr    = consoleWriter{os.Stderr, stderrTTY, outMutex}
)

var (
	cfgFile string

	verbose bool
	quiet   bool
	address string
)

// RootCmd represents the base command when called without any subcommands.
var RootCmd = &cobra.Command{
	Use:           "k6",
	Short:         "a next-generation load generator",
	Long:          BannerColor.Sprint(Banner),
	SilenceUsage:  true,
	SilenceErrors: true,
	PersistentPreRun: func(cmd *cobra.Command, args []string) {
		l := log.StandardLogger()
		l.Out = stderr
		l.Formatter = &log.TextFormatter{ForceColors: stderrTTY}
		if verbose {
			l.SetLevel(log.DebugLevel)
		}
	},
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		log.Error(err.Error())
		if e, ok := err.(ExitCode); ok {
			os.Exit(e.Code)
		}
		os.Exit(-1)
	}
}

func init() {
	RootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "enable debug logging")
	RootCmd.PersistentFlags().BoolVarP(&quiet, "quiet", "q", false, "disable progress updates")
	RootCmd.PersistentFlags().StringVarP(&address, "address", "a", "localhost:6565", "address for the api server")
	RootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "c", "", "config file (default ./k6.yaml or ~/.config/k6.yaml)")
	must(cobra.MarkFlagFilename(RootCmd.PersistentFlags(), "config"))
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"io"

	"bytes"
	"github.com/golang/glog"
	"github.com/spf13/cobra"
	"k8s.io/kops/cmd/kops/util"
	kopsapi "k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/apis/kops/v1alpha1"
	"k8s.io/kops/upup/pkg/fi/cloudup"
	"k8s.io/kops/util/pkg/vfs"
	k8sapi "k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/errors"
	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
	"k8s.io/kubernetes/pkg/kubectl/resource"
	"k8s.io/kubernetes/pkg/runtime/schema"
)

type CreateOptions struct {
	resource.FilenameOptions
}

func NewCmdCreate(f *util.Factory, out io.Writer) *cobra.Command {
	options := &CreateOptions{}

	cmd := &cobra.Command{
		Use:   "create -f FILENAME",
		Short: "Create a resource by filename or stdin",
		Run: func(cmd *cobra.Command, args []string) {
			if cmdutil.IsFilenameEmpty(options.Filenames) {
				cmd.Help()
				return
			}
			//cmdutil.CheckErr(ValidateArgs(cmd, args))
			//cmdutil.CheckErr(cmdutil.ValidateOutputArgs(cmd))
			cmdutil.CheckErr(RunCreate(f, out, options))
		},
	}

	cmd.Flags().StringSliceVarP(&options.Filenames, "filename", "f", options.Filenames, "Filename to use to create the resource")
	//usage := "to use to create the resource"
	//cmdutil.AddFilenameOptionFlags(cmd, options, usage)
	cmd.MarkFlagRequired("filename")
	//cmdutil.AddValidateFlags(cmd)
	//cmdutil.AddOutputFlagsForMutation(cmd)
	//cmdutil.AddApplyAnnotationFlags(cmd)
	//cmdutil.AddRecordFlag(cmd)
	//cmdutil.AddInclude3rdPartyFlags(cmd)

	// create subcommands
	cmd.AddCommand(NewCmdCreateCluster(f, out))
	cmd.AddCommand(NewCmdCreateInstanceGroup(f, out))
	cmd.AddCommand(NewCmdCreateSecret(f, out))
	return cmd
}

func RunCreate(f *util.Factory, out io.Writer, c *CreateOptions) error {
	clientset, err := f.Clientset()
	if err != nil {
		return err
	}

	// Codecs provides access to encoding and decoding for the scheme
	codecs := k8sapi.Codecs //serializer.NewCodecFactory(scheme)

	codec := codecs.UniversalDecoder(kopsapi.SchemeGroupVersion)

	var clusterName = ""
	var cSpec = false
	for _, f := range c.Filenames {
		contents, err := vfs.Context.ReadFile(f)
		if err != nil {
			return fmt.Errorf("error reading file %q: %v", f, err)
		}

		sections := bytes.Split(contents, []byte("\n---\n"))
		for _, section := range sections {
			defaults := &schema.GroupVersionKind{
				Group:   v1alpha1.SchemeGroupVersion.Group,
				Version: v1alpha1.SchemeGroupVersion.Version,
			}
			o, gvk, err := codec.Decode(section, defaults, nil)
			if err != nil {
				return fmt.Errorf("error parsing file %q: %v", f, err)
			}

			switch v := o.(type) {
			case *kopsapi.Federation:
				_, err = clientset.Federations().Create(v)
				if err != nil {
					if errors.IsAlreadyExists(err) {
						return fmt.Errorf("federation %q already exists", v.ObjectMeta.Name)
					}
					return fmt.Errorf("error creating federation: %v", err)
				}

			case *kopsapi.Cluster:
				// Adding a PerformAssignments() call here as the user might be trying to use
				// the new `-f` feature, with an old cluster definition.
				err = cloudup.PerformAssignments(v)
				if err != nil {
					return fmt.Errorf("error populating configuration: %v", err)
				}
				_, err = clientset.Clusters().Create(v)
				if err != nil {
					if errors.IsAlreadyExists(err) {
						return fmt.Errorf("cluster %q already exists", v.ObjectMeta.Name)
					}
					return fmt.Errorf("error creating cluster: %v", err)
				} else {
					cSpec = true
				}

			case *kopsapi.InstanceGroup:
				clusterName = v.ObjectMeta.Labels[kopsapi.LabelClusterName]
				if clusterName == "" {
					return fmt.Errorf("must specify %q label with cluster name to create instanceGroup", kopsapi.LabelClusterName)
				}
				_, err = clientset.InstanceGroups(clusterName).Create(v)
				if err != nil {
					if errors.IsAlreadyExists(err) {
						return fmt.Errorf("instanceGroup %q already exists", v.ObjectMeta.Name)
					}
					return fmt.Errorf("error creating instanceGroup: %v", err)
				}

			default:
				glog.V(2).Infof("Type of object was %T", v)
				return fmt.Errorf("Unhandled kind %q in %q", gvk, f)
			}
		}

	}
	{
		var sb bytes.Buffer
		fmt.Fprintf(&sb, "\n")
		fmt.Fprintf(&sb, "Your input has been successfully parsed.\n")
		fmt.Fprintf(&sb, "\n")

		// This isn't pretty.
		// The point is to give some sort of feedback if the input was successfully parsed.
		// And if this was a full cluster spec, let's show how to deploy the cluster.
		if clusterName != "" && cSpec {
			fmt.Fprintf(&sb, "To deploy these resources, run: kops update cluster %s --yes\n", clusterName)
			fmt.Fprintf(&sb, "\n")
		}
		_, err := out.Write(sb.Bytes())
		if err != nil {
			return fmt.Errorf("error writing to output: %v", err)
		}
	}
	return nil
}
<commit_msg>List the resources that we create with -f and advise the user how to deploy them.<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"io"

	"bytes"
	"github.com/golang/glog"
	"github.com/spf13/cobra"
	"k8s.io/kops/cmd/kops/util"
	kopsapi "k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/apis/kops/v1alpha1"
	"k8s.io/kops/upup/pkg/fi/cloudup"
	"k8s.io/kops/util/pkg/vfs"
	k8sapi "k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/errors"
	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
	"k8s.io/kubernetes/pkg/kubectl/resource"
	"k8s.io/kubernetes/pkg/runtime/schema"
)

type CreateOptions struct {
	resource.FilenameOptions
}

func NewCmdCreate(f *util.Factory, out io.Writer) *cobra.Command {
	options := &CreateOptions{}

	cmd := &cobra.Command{
		Use:   "create -f FILENAME",
		Short: "Create a resource by filename or stdin",
		Run: func(cmd *cobra.Command, args []string) {
			if cmdutil.IsFilenameEmpty(options.Filenames) {
				cmd.Help()
				return
			}
			//cmdutil.CheckErr(ValidateArgs(cmd, args))
			//cmdutil.CheckErr(cmdutil.ValidateOutputArgs(cmd))
			cmdutil.CheckErr(RunCreate(f, out, options))
		},
	}

	cmd.Flags().StringSliceVarP(&options.Filenames, "filename", "f", options.Filenames, "Filename to use to create the resource")
	//usage := "to use to create the resource"
	//cmdutil.AddFilenameOptionFlags(cmd, options, usage)
	cmd.MarkFlagRequired("filename")
	//cmdutil.AddValidateFlags(cmd)
	//cmdutil.AddOutputFlagsForMutation(cmd)
	//cmdutil.AddApplyAnnotationFlags(cmd)
	//cmdutil.AddRecordFlag(cmd)
	//cmdutil.AddInclude3rdPartyFlags(cmd)

	// create subcommands
	cmd.AddCommand(NewCmdCreateCluster(f, out))
	cmd.AddCommand(NewCmdCreateInstanceGroup(f, out))
	cmd.AddCommand(NewCmdCreateSecret(f, out))
	return cmd
}

func RunCreate(f *util.Factory, out io.Writer, c *CreateOptions) error {
	clientset, err := f.Clientset()
	if err != nil {
		return err
	}

	// Codecs provides access to encoding and decoding for the scheme
	codecs := k8sapi.Codecs //serializer.NewCodecFactory(scheme)

	codec := codecs.UniversalDecoder(kopsapi.SchemeGroupVersion)

	var clusterName = ""
	//var cSpec = false
	var sb bytes.Buffer
	fmt.Fprintf(&sb, "\n")
	for _, f := range c.Filenames {
		contents, err := vfs.Context.ReadFile(f)
		if err != nil {
			return fmt.Errorf("error reading file %q: %v", f, err)
		}

		sections := bytes.Split(contents, []byte("\n---\n"))
		for _, section := range sections {
			defaults := &schema.GroupVersionKind{
				Group:   v1alpha1.SchemeGroupVersion.Group,
				Version: v1alpha1.SchemeGroupVersion.Version,
			}
			o, gvk, err := codec.Decode(section, defaults, nil)
			if err != nil {
				return fmt.Errorf("error parsing file %q: %v", f, err)
			}

			switch v := o.(type) {
			case *kopsapi.Federation:
				_, err = clientset.Federations().Create(v)
				if err != nil {
					if errors.IsAlreadyExists(err) {
						return fmt.Errorf("federation %q already exists", v.ObjectMeta.Name)
					}
					return fmt.Errorf("error creating federation: %v", err)
				}
				fmt.Fprintf(&sb, "Created federation/%q\n", v.ObjectMeta.Name)

			case *kopsapi.Cluster:
				// Adding a PerformAssignments() call here as the user might be trying to use
				// the new `-f` feature, with an old cluster definition.
				err = cloudup.PerformAssignments(v)
				if err != nil {
					return fmt.Errorf("error populating configuration: %v", err)
				}
				_, err = clientset.Clusters().Create(v)
				if err != nil {
					if errors.IsAlreadyExists(err) {
						return fmt.Errorf("cluster %q already exists", v.ObjectMeta.Name)
					}
					return fmt.Errorf("error creating cluster: %v", err)
				} else {
					fmt.Fprintf(&sb, "Created cluster/%s\n", v.ObjectMeta.Name)
					//cSpec = true
				}

			case *kopsapi.InstanceGroup:
				clusterName = v.ObjectMeta.Labels[kopsapi.LabelClusterName]
				if clusterName == "" {
					return fmt.Errorf("must specify %q label with cluster name to create instanceGroup", kopsapi.LabelClusterName)
				}
				_, err = clientset.InstanceGroups(clusterName).Create(v)
				if err != nil {
					if errors.IsAlreadyExists(err) {
						return fmt.Errorf("instanceGroup %q already exists", v.ObjectMeta.Name)
					}
					return fmt.Errorf("error creating instanceGroup: %v", err)
				} else {
					fmt.Fprintf(&sb, "Created instancegroup/%s\n", v.ObjectMeta.Name)
				}

			default:
				glog.V(2).Infof("Type of object was %T", v)
				return fmt.Errorf("Unhandled kind %q in %s", gvk, f)
			}
		}

	}
	{
		// If there is a value in this sb, this should mean that we have something to deploy
		// so let's advise the user how to engage the cloud provider and deploy
		if sb.String() != "" {
			fmt.Fprintf(&sb, "\n")
			fmt.Fprintf(&sb, "To deploy these resources, run: kops update cluster %s --yes\n", clusterName)
			fmt.Fprintf(&sb, "\n")
		}
		_, err := out.Write(sb.Bytes())
		if err != nil {
			return fmt.Errorf("error writing to output: %v", err)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/pkg/errors"
)

// HelpOp describes printing help.
type HelpOp struct{}

func (_ HelpOp) Run(stdout, _ io.Writer) error {
	return printUsage(stdout)
}

func printUsage(out io.Writer) error {
	help := `USAGE:
  %PROG%                    : list the namespaces in the current context
  %PROG% <NAME>             : change the active namespace of current context
  %PROG% -                  : switch to the previous namespace in this context
  %PROG% -c, --current      : show the current namespace
  %PROG% -h,--help          : show this message
`
	// TODO this replace logic is duplicated between this and kubectx
	help = strings.ReplaceAll(help, "%PROG%", selfName())

	_, err := fmt.Fprintf(out, "%s\n", help)
	return errors.Wrap(err, "write error")
}

// selfName guesses how the user invoked the program.
func selfName() string {
	// TODO this method is duplicated between this and kubectx
	me := filepath.Base(os.Args[0])
	pluginPrefix := "kubectl-"
	if strings.HasPrefix(me, pluginPrefix) {
		return "kubectl " + strings.TrimPrefix(me, pluginPrefix)
	}
	return "kubectx"
}
<commit_msg>changed help in kubens, return had a typo (#210)<commit_after>package main

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/pkg/errors"
)

// HelpOp describes printing help.
type HelpOp struct{}

func (_ HelpOp) Run(stdout, _ io.Writer) error {
	return printUsage(stdout)
}

func printUsage(out io.Writer) error {
	help := `USAGE:
  %PROG%                    : list the namespaces in the current context
  %PROG% <NAME>             : change the active namespace of current context
  %PROG% -                  : switch to the previous namespace in this context
  %PROG% -c, --current      : show the current namespace
  %PROG% -h,--help          : show this message
`
	// TODO this replace logic is duplicated between this and kubectx
	help = strings.ReplaceAll(help, "%PROG%", selfName())

	_, err := fmt.Fprintf(out, "%s\n", help)
	return errors.Wrap(err, "write error")
}

// selfName guesses how the user invoked the program.
func selfName() string {
	// TODO this method is duplicated between this and kubectx
	me := filepath.Base(os.Args[0])
	pluginPrefix := "kubectl-"
	if strings.HasPrefix(me, pluginPrefix) {
		return "kubectl " + strings.TrimPrefix(me, pluginPrefix)
	}
	return "kubens"
}
<|endoftext|>


<commit_before>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2017 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package cmd

import (
	"fmt"
	"os"

	"gopkg.in/guregu/null.v3"

	"github.com/loadimpact/k6/stats/cloud"
	"github.com/loadimpact/k6/ui"
	"github.com/pkg/errors"
	"github.com/spf13/afero"
	"github.com/spf13/cobra"
)

// loginCloudCommand represents the 'login cloud' command
var loginCloudCommand = &cobra.Command{
	Use:   "cloud",
	Short: "Authenticate with Load Impact",
	Long: `Authenticate with Load Impact.

This will set the default token used when just "k6 run -o cloud" is passed.`,
	Example: `
  # Show the stored token.
  k6 login cloud -s

  # Store a token.
  k6 login cloud -t YOUR_TOKEN

  # Log in with an email/password.
  k6 login cloud`[1:],
	Args: cobra.NoArgs,
	RunE: func(cmd *cobra.Command, args []string) error {
		fs := afero.NewOsFs()
		config, cdir, err := readDiskConfig(fs)
		if err != nil {
			return err
		}

		show := getNullBool(cmd.Flags(), "show")
		token := getNullString(cmd.Flags(), "token")

		conf := cloud.NewConfig().Apply(config.Collectors.Cloud)

		switch {
		case show.Bool:
		case token.Valid:
			conf.Token = token
		default:
			form := ui.Form{
				Fields: []ui.Field{
					ui.StringField{
						Key:   "Email",
						Label: "Email",
					},
					ui.StringField{
						Key:   "Password",
						Label: "Password",
					},
				},
			}
			vals, err := form.Run(os.Stdin, stdout)
			if err != nil {
				return err
			}
			email := vals["Email"].(string)
			password := vals["Password"].(string)

			client := cloud.NewClient("", conf.Host.String, Version)
			res, err := client.Login(email, password)
			if err != nil {
				return err
			}

			if res.Token == "" {
				return errors.New(`Your account has no API token, please generate one: "https://app.loadimpact.com/account/token".`)
			}

			conf.Token = null.StringFrom(res.Token)
		}

		config.Collectors.Cloud = conf
		if err := writeDiskConfig(fs, cdir, config); err != nil {
			return err
		}

		fmt.Fprintf(stdout, "  token: %s\n", ui.ValueColor.Sprint(conf.Token.String))
		return nil
	},
}

func init() {
	loginCmd.AddCommand(loginCloudCommand)
	loginCloudCommand.Flags().StringP("token", "t", "", "specify `token` to use")
	loginCloudCommand.Flags().BoolP("show", "s", false, "display saved token and exit")
}
<commit_msg>Adding option to reset cloud token<commit_after>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2017 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package cmd

import (
	"fmt"
	"os"

	"gopkg.in/guregu/null.v3"

	"github.com/loadimpact/k6/stats/cloud"
	"github.com/loadimpact/k6/ui"
	"github.com/pkg/errors"
	"github.com/spf13/afero"
	"github.com/spf13/cobra"
)

// loginCloudCommand represents the 'login cloud' command
var loginCloudCommand = &cobra.Command{
	Use:   "cloud",
	Short: "Authenticate with Load Impact",
	Long: `Authenticate with Load Impact.

This will set the default token used when just "k6 run -o cloud" is passed.`,
	Example: `
  # Show the stored token.
  k6 login cloud -s

  # Store a token.
  k6 login cloud -t YOUR_TOKEN

  # Log in with an email/password.
  k6 login cloud`[1:],
	Args: cobra.NoArgs,
	RunE: func(cmd *cobra.Command, args []string) error {
		fs := afero.NewOsFs()
		config, cdir, err := readDiskConfig(fs)
		if err != nil {
			return err
		}

		show := getNullBool(cmd.Flags(), "show")
		reset := getNullBool(cmd.Flags(), "reset")
		token := getNullString(cmd.Flags(), "token")

		conf := cloud.NewConfig().Apply(config.Collectors.Cloud)

		switch {
		case reset.Valid:
			conf.Token = null.StringFromPtr(nil)
		case show.Bool:
		case token.Valid:
			conf.Token = token
		default:
			form := ui.Form{
				Fields: []ui.Field{
					ui.StringField{
						Key:   "Email",
						Label: "Email",
					},
					ui.StringField{
						Key:   "Password",
						Label: "Password",
					},
				},
			}
			vals, err := form.Run(os.Stdin, stdout)
			if err != nil {
				return err
			}
			email := vals["Email"].(string)
			password := vals["Password"].(string)

			client := cloud.NewClient("", conf.Host.String, Version)
			res, err := client.Login(email, password)
			if err != nil {
				return err
			}

			if res.Token == "" {
				return errors.New(`Your account has no API token, please generate one: "https://app.loadimpact.com/account/token".`)
			}

			conf.Token = null.StringFrom(res.Token)
		}

		config.Collectors.Cloud = conf
		if err := writeDiskConfig(fs, cdir, config); err != nil {
			return err
		}

		if conf.Token.Valid {
			fmt.Fprintf(stdout, "  token: %s\n", ui.ValueColor.Sprint(conf.Token.String))
		}
		return nil
	},
}

func init() {
	loginCmd.AddCommand(loginCloudCommand)
	loginCloudCommand.Flags().StringP("token", "t", "", "specify `token` to use")
	loginCloudCommand.Flags().BoolP("show", "s", false, "display saved token and exit")
	loginCloudCommand.Flags().BoolP("reset", "r", false, "reset token")
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"encoding/gob"
	"encoding/json"
	"errors"
	"github.com/Symantec/Dominator/lib/mdb"
	"log"
	"os"
	"path"
	"regexp"
	"sort"
	"time"
)

type genericEncoder interface {
	Encode(v interface{}) error
}

func runDaemon(driverFunc driverFunc, url, mdbFileName, hostnameRegex string,
	fetchInterval uint, logger *log.Logger) {
	var prevMdb *mdb.Mdb
	var hostnameRE *regexp.Regexp
	var err error
	if hostnameRegex != ".*" {
		hostnameRE, err = regexp.Compile("^" + hostnameRegex)
		if err != nil {
			logger.Println(err)
			os.Exit(1)
		}
	}
	var cycleStopTime time.Time
	for ; ; sleepUntil(cycleStopTime) {
		cycleStopTime = time.Now().Add(time.Duration(fetchInterval))
		if newMdb := loadMdb(driverFunc, url, logger); newMdb != nil {
			newMdb := selectHosts(newMdb, hostnameRE)
			sort.Sort(newMdb)
			if newMdbIsDifferent(prevMdb, newMdb) {
				if err := writeMdb(newMdb, mdbFileName); err != nil {
					logger.Println(err)
				} else {
					prevMdb = newMdb
				}
			}
		}
	}
}

func sleepUntil(wakeTime time.Time) {
	sleepTime := wakeTime.Sub(time.Now())
	if sleepTime < time.Second {
		sleepTime = time.Second
	}
	time.Sleep(sleepTime)
}

func loadMdb(driverFunc driverFunc, url string, logger *log.Logger) *mdb.Mdb {
	file, err := os.Open(url)
	if err != nil {
		logger.Println("Error opening file " + err.Error())
		return nil
	}
	defer file.Close()
	return driverFunc(bufio.NewReader(file), logger)
}

func selectHosts(inMdb *mdb.Mdb, hostnameRE *regexp.Regexp) *mdb.Mdb {
	if hostnameRE == nil {
		return inMdb
	}
	var outMdb mdb.Mdb
	for _, machine := range inMdb.Machines {
		if hostnameRE.MatchString(machine.Hostname) {
			outMdb.Machines = append(outMdb.Machines, machine)
		}
	}
	return &outMdb
}

func newMdbIsDifferent(prevMdb, newMdb *mdb.Mdb) bool {
	if prevMdb == nil {
		return true
	}
	if len(prevMdb.Machines) != len(newMdb.Machines) {
		return true
	}
	for index, prevMachine := range prevMdb.Machines {
		if prevMachine != newMdb.Machines[index] {
			return true
		}
	}
	return false
}

func writeMdb(mdb *mdb.Mdb, mdbFileName string) error {
	tmpFileName := mdbFileName + "~"
	file, err := os.Create(tmpFileName)
	if err != nil {
		return errors.New("Error opening file " + err.Error())
	}
	defer os.Remove(tmpFileName)
	defer file.Close()
	writer := bufio.NewWriter(file)
	defer writer.Flush()
	switch path.Ext(mdbFileName) {
	case ".gob":
		if err := gob.NewEncoder(writer).Encode(mdb.Machines); err != nil {
			return err
		}
	default:
		b, err := json.Marshal(mdb.Machines)
		if err != nil {
			return err
		}
		var out bytes.Buffer
		json.Indent(&out, b, "", "    ")
		_, err = out.WriteTo(writer)
		if err != nil {
			return err
		}
		writer.Write([]byte("\n"))
	}
	return os.Rename(tmpFileName, mdbFileName)
}
<commit_msg>Add http/https URL support to mdbd.<commit_after>package main

import (
	"bufio"
	"bytes"
	"encoding/gob"
	"encoding/json"
	"errors"
	"github.com/Symantec/Dominator/lib/mdb"
	"log"
	"net/http"
	"os"
	"path"
	"regexp"
	"sort"
	"strings"
	"time"
)

type genericEncoder interface {
	Encode(v interface{}) error
}

func runDaemon(driverFunc driverFunc, url, mdbFileName, hostnameRegex string,
	fetchInterval uint, logger *log.Logger) {
	var prevMdb *mdb.Mdb
	var hostnameRE *regexp.Regexp
	var err error
	if hostnameRegex != ".*" {
		hostnameRE, err = regexp.Compile("^" + hostnameRegex)
		if err != nil {
			logger.Println(err)
			os.Exit(1)
		}
	}
	var cycleStopTime time.Time
	for ; ; sleepUntil(cycleStopTime) {
		cycleStopTime = time.Now().Add(time.Duration(fetchInterval))
		if newMdb := loadMdb(driverFunc, url, logger); newMdb != nil {
			newMdb := selectHosts(newMdb, hostnameRE)
			sort.Sort(newMdb)
			if newMdbIsDifferent(prevMdb, newMdb) {
				if err := writeMdb(newMdb, mdbFileName); err != nil {
					logger.Println(err)
				} else {
					prevMdb = newMdb
				}
			}
		}
	}
}

func sleepUntil(wakeTime time.Time) {
	sleepTime := wakeTime.Sub(time.Now())
	if sleepTime < time.Second {
		sleepTime = time.Second
	}
	time.Sleep(sleepTime)
}

func loadMdb(driverFunc driverFunc, url string, logger *log.Logger) *mdb.Mdb {
	if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") {
		return loadHttpMdb(driverFunc, url, logger)
	}
	file, err := os.Open(url)
	if err != nil {
		logger.Println("Error opening file " + err.Error())
		return nil
	}
	defer file.Close()
	return driverFunc(bufio.NewReader(file), logger)
}

func loadHttpMdb(driverFunc driverFunc, url string,
	logger *log.Logger) *mdb.Mdb {
	response, err := http.Get(url)
	if err != nil {
		logger.Println(err)
		return nil
	}
	defer response.Body.Close()
	if response.StatusCode != http.StatusOK {
		logger.Println("HTTP get failed: " + err.Error())
		return nil
	}
	return driverFunc(response.Body, logger)
}

func selectHosts(inMdb *mdb.Mdb, hostnameRE *regexp.Regexp) *mdb.Mdb {
	if hostnameRE == nil {
		return inMdb
	}
	var outMdb mdb.Mdb
	for _, machine := range inMdb.Machines {
		if hostnameRE.MatchString(machine.Hostname) {
			outMdb.Machines = append(outMdb.Machines, machine)
		}
	}
	return &outMdb
}

func newMdbIsDifferent(prevMdb, newMdb *mdb.Mdb) bool {
	if prevMdb == nil {
		return true
	}
	if len(prevMdb.Machines) != len(newMdb.Machines) {
		return true
	}
	for index, prevMachine := range prevMdb.Machines {
		if prevMachine != newMdb.Machines[index] {
			return true
		}
	}
	return false
}

func writeMdb(mdb *mdb.Mdb, mdbFileName string) error {
	tmpFileName := mdbFileName + "~"
	file, err := os.Create(tmpFileName)
	if err != nil {
		return errors.New("Error opening file " + err.Error())
	}
	defer os.Remove(tmpFileName)
	defer file.Close()
	writer := bufio.NewWriter(file)
	defer writer.Flush()
	switch path.Ext(mdbFileName) {
	case ".gob":
		if err := gob.NewEncoder(writer).Encode(mdb.Machines); err != nil {
			return err
		}
	default:
		b, err := json.Marshal(mdb.Machines)
		if err != nil {
			return err
		}
		var out bytes.Buffer
		json.Indent(&out, b, "", "    ")
		_, err = out.WriteTo(writer)
		if err != nil {
			return err
		}
		writer.Write([]byte("\n"))
	}
	return os.Rename(tmpFileName, mdbFileName)
}
<|endoftext|>


<commit_before>package print

import (
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/cavaliercoder/grab"
	"github.com/dustin/go-humanize"
	"github.com/markelog/curse"
	"github.com/mgutz/ansi"
)

var (
	gray  = ansi.ColorCode("240")
	reset = ansi.ColorCode("reset")
)

func InStyle(name, entity string) {
	name = ansi.Color(name, "white+b")
	entity = ansi.Color(" "+entity+" ", "cyan+h")

	fmt.Print(name, entity)
}

func InStyleln(name, entity string) {
	InStyle(name, entity)
	fmt.Println()
}

func Version(version string) {
	fmt.Println(gray, "  ", version, reset)
}

func CurrentVersion(version string) {
	fmt.Println(ansi.Color("  ♥ "+version, "cyan"))
}

func Error(err error) {
	if err == nil {
		return
	}

	fmt.Println()
	fmt.Print(ansi.Color("> ", "red"))

	fmt.Fprintf(os.Stderr, "%v", err)
	fmt.Println()
	fmt.Println()

	os.Exit(1)
}

func Download(response *grab.Response, version string) string {
	Error(response.Error)

	c, _ := curse.New()

	before := func() {
		time.Sleep(500 * time.Millisecond)
	}

	started := false
	prefix := func() {
		Error(response.Error)
		size := humanize.Bytes(response.Size)
		transfered := humanize.Bytes(response.BytesTransferred())
		transfered = strings.Replace(transfered, " MB", "", 1)

		c.MoveUp(1)

		if started {
			c.EraseCurrentLine()
		}
		started = true
		text := fmt.Sprintf("(%s/%s ", transfered, size)

		InStyle("Version", version)
		fmt.Print(gray, text, reset)
	}

	postfix := func() {
		progress := int(100 * response.Progress())
		text := fmt.Sprintf("%d%%)", progress)

		fmt.Println(gray, text, reset)

		time.Sleep(200 * time.Millisecond)
	}

	after := func() {
		c.EraseCurrentLine()
		InStyle("Version", version)
		fmt.Println()
	}

	s := &Spinner{
		Before:  before,
		After:   after,
		Prefix:  prefix,
		Postfix: postfix,
	}

	s.Start()
	for response.IsComplete() == false {
		time.Sleep(200 * time.Millisecond)
	}
	s.Stop()

	return response.Filename
}

func CustomSpin(header, item, message string) *Spinner {
	c, _ := curse.New()

	before := func() {}

	started := false
	prefix := func() {
		c.MoveUp(1)

		if started {
			c.EraseCurrentLine()
		}
		started = true

		InStyle(header, item)
	}

	postfix := func() {
		fmt.Println(gray, message, reset)

		time.Sleep(300 * time.Millisecond)
	}

	after := func() {
		c.EraseCurrentLine()
		InStyle(header, item)
		fmt.Println()
	}

	s := &Spinner{
		Before:  before,
		After:   after,
		Prefix:  prefix,
		Postfix: postfix,
	}

	return s
}

func Warning(message, command string) {
	fmt.Println()
	fmt.Print(ansi.Color("> ", "red"))
	fmt.Print(message)

	if command != "" {
		fmt.Println()
		fmt.Println()

		fmt.Print(ansi.Color("> ", "green") + command)
	}

	fmt.Println()
	fmt.Println()
}
<commit_msg>Add debug info to error output if desired<commit_after>package print

import (
	"fmt"
	"os"
	"strings"
	"time"
	"runtime/debug"

	"github.com/cavaliercoder/grab"
	"github.com/dustin/go-humanize"
	"github.com/markelog/curse"
	"github.com/markelog/eclectica/variables"
	"github.com/mgutz/ansi"
)

var (
	gray  = ansi.ColorCode("240")
	reset = ansi.ColorCode("reset")
)

func InStyle(name, entity string) {
	name = ansi.Color(name, "white+b")
	entity = ansi.Color(" "+entity+" ", "cyan+h")

	fmt.Print(name, entity)
}

func InStyleln(name, entity string) {
	InStyle(name, entity)
	fmt.Println()
}

func Version(version string) {
	fmt.Println(gray, "  ", version, reset)
}

func CurrentVersion(version string) {
	fmt.Println(ansi.Color("  ♥ "+version, "cyan"))
}

func Error(err error) {
	if err == nil {
		return
	}

	fmt.Println()
	fmt.Print(ansi.Color("> ", "red"))

	fmt.Fprintf(os.Stderr, "%v", err)

	if variables.IsDebug() {
		debug.PrintStack()
	}

	fmt.Println()
	fmt.Println()

	os.Exit(1)
}

func Download(response *grab.Response, version string) string {
	Error(response.Error)

	c, _ := curse.New()

	before := func() {
		time.Sleep(500 * time.Millisecond)
	}

	started := false
	prefix := func() {
		Error(response.Error)
		size := humanize.Bytes(response.Size)
		transfered := humanize.Bytes(response.BytesTransferred())
		transfered = strings.Replace(transfered, " MB", "", 1)

		c.MoveUp(1)

		if started {
			c.EraseCurrentLine()
		}
		started = true
		text := fmt.Sprintf("(%s/%s ", transfered, size)

		InStyle("Version", version)
		fmt.Print(gray, text, reset)
	}

	postfix := func() {
		progress := int(100 * response.Progress())
		text := fmt.Sprintf("%d%%)", progress)

		fmt.Println(gray, text, reset)

		time.Sleep(200 * time.Millisecond)
	}

	after := func() {
		c.EraseCurrentLine()
		InStyle("Version", version)
		fmt.Println()
	}

	s := &Spinner{
		Before:  before,
		After:   after,
		Prefix:  prefix,
		Postfix: postfix,
	}

	s.Start()
	for response.IsComplete() == false {
		time.Sleep(200 * time.Millisecond)
	}
	s.Stop()

	return response.Filename
}

func CustomSpin(header, item, message string) *Spinner {
	c, _ := curse.New()

	before := func() {}

	started := false
	prefix := func() {
		c.MoveUp(1)

		if started {
			c.EraseCurrentLine()
		}
		started = true

		InStyle(header, item)
	}

	postfix := func() {
		fmt.Println(gray, message, reset)

		time.Sleep(300 * time.Millisecond)
	}

	after := func() {
		c.EraseCurrentLine()
		InStyle(header, item)
		fmt.Println()
	}

	s := &Spinner{
		Before:  before,
		After:   after,
		Prefix:  prefix,
		Postfix: postfix,
	}

	return s
}

func Warning(message, command string) {
	fmt.Println()
	fmt.Print(ansi.Color("> ", "red"))
	fmt.Print(message)

	if command != "" {
		fmt.Println()
		fmt.Println()

		fmt.Print(ansi.Color("> ", "green") + command)
	}

	fmt.Println()
	fmt.Println()
}
<|endoftext|>


<commit_before>package main
<commit_msg>Add server's main<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"

	"github.com/otoolep/go-grpc-pg/service"
)

// Command line defaults
const (
	DefaultgRPCAddr       = "localhost:11000"
	DefaultPostgreSQLAddr = "localhost:5432"
)

// Command line parameters
var gRPCAddr string
var pgAddr string

func init() {
	flag.StringVar(&gRPCAddr, "grpc-addr", DefaultgRPCAddr, "Set the gRPC bind address")
	flag.StringVar(&pgAddr, "pg-addr", DefaultPostgreSQLAddr, "Set PostgreSQL address")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [options]\n", os.Args[0])
		flag.PrintDefaults()
	}
}

func main() {
	flag.Parse()

	// Create the service.
	srv := service.New(gRPCAddr, nil)

	// Start the service.
	if err := srv.Open(); err != nil {
		fmt.Fprintf(os.Stderr, "failed to start service: %s", err.Error())
		os.Exit(1)
	}
	log.Println("service started successfully")

	// Block until a signal is received.
	terminate := make(chan os.Signal, 1)
	signal.Notify(terminate, os.Interrupt)
	<-terminate
	log.Println("service exiting")
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"go/ast"
	"go/format"
	"go/parser"
	"go/token"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"text/template"
)

var (
	gopath        = os.Getenv("GOPATH")
	runner string = "slurp."
	cwd    string

	install   = flag.Bool("install", false, "install current slurp.Go as slurp.PKG.")
	bare      = flag.Bool("bare", false, "Run/Install the slurp.go file without any other files.")
	slurpfile = flag.String("slurpfile", "slurp.go", "The file that includes the Slurp(*s.Build) function, use by -bare")
)

func main() {

	flag.Parse()

	if gopath == "" {
		log.Fatal("$GOPATH must be set.")
	}

	err := run(*install)
	if err != nil {
		log.Fatal(err)
	}
}

func run(install bool) error {
	path, err := generate()
	if err != nil {
		return err
	}

	//Don't forget to clean up.
	defer os.RemoveAll(path)

	var args []string

	//if len(params) > 0 && params[0] == "init"
	get := exec.Command("go", "get", "-tags=slurp", "-v")
	get.Dir = filepath.Join(path, "tmp")
	get.Stdin = os.Stdin
	get.Stdout = os.Stdout
	get.Stderr = os.Stderr

	if install {
		err := get.Run()
		if err != nil {
			return err
		}

		runnerpkg, err := filepath.Rel(filepath.Join(gopath, "src"), filepath.Join(filepath.Join(path, runner)))
		if err != nil {
			return err
		}
		args = []string{"install", "-tags=slurp", runnerpkg}

	} else {
		params := flag.Args()

		if len(params) > 0 && params[0] == "init" {
			err := get.Run()
			if err != nil {
				return err
			}
		}

		args = []string{"run", "-tags=slurp", filepath.Join(filepath.Join(path, runner, "main.go"))}
		args = append(args, params...)
	}

	cmd := exec.Command("go", args...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	err = cmd.Run()

	if err != nil {
		return err
	}

	return nil
}

func generate() (string, error) {

	//Let's grab a temp folder.
	path, err := ioutil.TempDir(filepath.Join(gopath, "src"), "slurp-run-")
	if err != nil {
		return "", err
	}

	tmp := filepath.Join(path, "tmp")
	err = os.Mkdir(tmp, 0700)
	if err != nil {
		return path, err
	}

	cwd, err = os.Getwd()
	if err != nil {
		return path, err
	}

	runner = runner + filepath.Base(cwd)
	runnerpkg := filepath.Join(path, runner)
	err = os.Mkdir(runnerpkg, 0700)
	if err != nil {
		return path, err
	}

	//TODO, copy [*.go !_test.go] files into tmp first,
	// this would allow slurp to work for broken packages
	// with "-bare" as the package files will be excluded.
	fset := token.NewFileSet() // positions are relative to fset

	var pkgs map[string]*ast.Package

	if *bare {
		pkgs = make(map[string]*ast.Package)
		src, err := parser.ParseFile(fset, *slurpfile, nil, parser.ParseComments)
		if err != nil {
			return path, err
		}
		pkgs[src.Name.Name] = &ast.Package{
			Name:  src.Name.Name,
			Files: map[string]*ast.File{filepath.Join(cwd, *slurpfile): src},
		}
	} else {
		pkgs, err = parser.ParseDir(fset, cwd, nil, parser.ParseComments)
		if err != nil {
			return path, err
		}
	}

	if len(pkgs) > 1 {
		return path, errors.New("Error: Multiple packages detected.")
	}

	for _, pkg := range pkgs {
		//This loop always runs once. I don't know of any other way to get the pkg out of pkgs
		// witout understanding the names.
		for name, f := range pkg.Files {
			f.Name.Name = "tmp" //Change package name

			name, err = filepath.Rel(cwd, name)
			if err != nil {
				//Should never get error. But just incase.
				return path, err
			}
			err = writeFileSet(filepath.Join(tmp, name), fset, f)
			if err != nil {
				return path, err
			}
		}
	}

	file, err := os.Create(filepath.Join(runnerpkg, "main.go"))
	if err != nil {
		return path, err
	}

	tmp, err = filepath.Rel(filepath.Join(gopath, "src"), path)
	if err != nil {
		return path, err
	}

	err = runnerSrc.Execute(file, tmp)
	if err != nil {
		return path, err
	}

	err = file.Close()
	if err != nil {
		return path, err
	}

	return path, nil

}

func writeFileSet(filepath string, fset *token.FileSet, node interface{}) error {
	// Print the modified AST.
	file, err := os.Create(filepath)
	if err != nil {
		return err
	}
	defer file.Close()

	return format.Node(file, fset, node)
}

var runnerSrc = template.Must(template.New("main").Parse(`
package main

import (
  "flag"
  "strings"
  "os"
  "os/signal"

  "github.com/omeid/slurp"

  client "{{ . }}/tmp"
)

func main() {

  flag.Parse()

  interrupts := make(chan os.Signal, 1)
  signal.Notify(interrupts, os.Interrupt)

  slurp := slurp.NewBuild()

  go func() {
	sig := <-interrupts
	// stop watches and clean up.
	slurp.Printf("captured %v, stopping build and exiting..\n", sig)
	slurp.Close()
	os.Exit(1)
  }()

  client.Slurp(slurp)

  tasks := flag.Args()
  if len(tasks) == 0 {
	tasks = []string{"default"}
  }

  slurp.Printf("Running: %s", strings.Join(tasks, "," ))
  slurp.Run(slurp.C, tasks...)
  slurp.Close()
}
`))
<commit_msg>cmd/slurp: introduce -install<commit_after>package main

import (
	"errors"
	"flag"
	"go/ast"
	"go/format"
	"go/parser"
	"go/token"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"text/template"
)

var (
	gopath        = os.Getenv("GOPATH")
	runner string = "slurp."
	cwd    string

	build = flag.Bool("build", false, "build the current build as slurp-bin")
	install   = flag.Bool("install", false, "install current slurp.Go as slurp.PKG.")
	bare      = flag.Bool("bare", false, "Run/Install the slurp.go file without any other files.")
	slurpfile = flag.String("slurpfile", "slurp.go", "The file that includes the Slurp(*s.Build) function, use by -bare")
)

func main() {

	flag.Parse()

	if gopath == "" {
		log.Fatal("$GOPATH must be set.")
	}

	err := run()
	if err != nil {
		log.Fatal(err)
	}
}

func run() error {
	path, err := generate()
	if err != nil {
		return err
	}

	//Don't forget to clean up.
	defer os.RemoveAll(path)

	var args []string

	//if len(params) > 0 && params[0] == "init"
	get := exec.Command("go", "get", "-tags=slurp", "-v")
	get.Dir = filepath.Join(path, "tmp")
	get.Stdin = os.Stdin
	get.Stdout = os.Stdout
	get.Stderr = os.Stderr

	if *build {
		err := get.Run()
		if err != nil {
			return err
		}

		runnerpkg, err := filepath.Rel(filepath.Join(gopath, "src"), filepath.Join(filepath.Join(path, runner)))
		if err != nil {
			return err
		}
		args = []string{"build", "-tags=slurp", "-o=slurp-bin", runnerpkg}

	} else if *install {
		err := get.Run()
		if err != nil {
			return err
		}

		runnerpkg, err := filepath.Rel(filepath.Join(gopath, "src"), filepath.Join(filepath.Join(path, runner)))
		if err != nil {
			return err
		}
		args = []string{"install", "-tags=slurp", runnerpkg}

	} else {
		params := flag.Args()

		if len(params) > 0 && params[0] == "init" {
			err := get.Run()
			if err != nil {
				return err
			}
		}

		args = []string{"run", "-tags=slurp", filepath.Join(filepath.Join(path, runner, "main.go"))}
		args = append(args, params...)
	}

	cmd := exec.Command("go", args...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	err = cmd.Run()

	if err != nil {
		return err
	}

	return nil
}

func generate() (string, error) {

	//Let's grab a temp folder.
	path, err := ioutil.TempDir(filepath.Join(gopath, "src"), "slurp-run-")
	if err != nil {
		return "", err
	}

	tmp := filepath.Join(path, "tmp")
	err = os.Mkdir(tmp, 0700)
	if err != nil {
		return path, err
	}

	cwd, err = os.Getwd()
	if err != nil {
		return path, err
	}

	runner = runner + filepath.Base(cwd)
	runnerpkg := filepath.Join(path, runner)
	err = os.Mkdir(runnerpkg, 0700)
	if err != nil {
		return path, err
	}

	//TODO, copy [*.go !_test.go] files into tmp first,
	// this would allow slurp to work for broken packages
	// with "-bare" as the package files will be excluded.
	fset := token.NewFileSet() // positions are relative to fset

	var pkgs map[string]*ast.Package

	if *bare {
		pkgs = make(map[string]*ast.Package)
		src, err := parser.ParseFile(fset, *slurpfile, nil, parser.ParseComments)
		if err != nil {
			return path, err
		}
		pkgs[src.Name.Name] = &ast.Package{
			Name:  src.Name.Name,
			Files: map[string]*ast.File{filepath.Join(cwd, *slurpfile): src},
		}
	} else {
		pkgs, err = parser.ParseDir(fset, cwd, nil, parser.ParseComments)
		if err != nil {
			return path, err
		}
	}

	if len(pkgs) > 1 {
		return path, errors.New("Error: Multiple packages detected.")
	}

	for _, pkg := range pkgs {
		//This loop always runs once. I don't know of any other way to get the pkg out of pkgs
		// witout understanding the names.
		for name, f := range pkg.Files {
			f.Name.Name = "tmp" //Change package name

			name, err = filepath.Rel(cwd, name)
			if err != nil {
				//Should never get error. But just incase.
				return path, err
			}
			err = writeFileSet(filepath.Join(tmp, name), fset, f)
			if err != nil {
				return path, err
			}
		}
	}

	file, err := os.Create(filepath.Join(runnerpkg, "main.go"))
	if err != nil {
		return path, err
	}

	tmp, err = filepath.Rel(filepath.Join(gopath, "src"), path)
	if err != nil {
		return path, err
	}

	err = runnerSrc.Execute(file, tmp)
	if err != nil {
		return path, err
	}

	err = file.Close()
	if err != nil {
		return path, err
	}

	return path, nil

}

func writeFileSet(filepath string, fset *token.FileSet, node interface{}) error {
	// Print the modified AST.
	file, err := os.Create(filepath)
	if err != nil {
		return err
	}
	defer file.Close()

	return format.Node(file, fset, node)
}

var runnerSrc = template.Must(template.New("main").Parse(`
package main

import (
  "flag"
  "strings"
  "os"
  "os/signal"

  "github.com/omeid/slurp"

  client "{{ . }}/tmp"
)

func main() {

  flag.Parse()

  interrupts := make(chan os.Signal, 1)
  signal.Notify(interrupts, os.Interrupt)

  slurp := slurp.NewBuild()

  go func() {
	sig := <-interrupts
	// stop watches and clean up.
	slurp.Printf("captured %v, stopping build and exiting..\n", sig)
	slurp.Close()
	os.Exit(1)
  }()

  client.Slurp(slurp)

  tasks := flag.Args()
  if len(tasks) == 0 {
	tasks = []string{"default"}
  }

  slurp.Printf("Running: %s", strings.Join(tasks, "," ))
  slurp.Run(slurp.C, tasks...)
  slurp.Close()
}
`))
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/jackc/sqlfmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
)

const Version = "0.1.0"

var options struct {
	write   bool
	version bool
}

type job struct {
	name string
	r    io.ReadCloser
	w    io.WriteCloser
}

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "usage:  %s [options] [path ...]\n", os.Args[0])
		flag.PrintDefaults()
	}

	flag.BoolVar(&options.write, "w", false, "write result to (source) file instead of stdout")
	flag.BoolVar(&options.version, "version", false, "print version and exit")
	flag.Parse()

	if options.version {
		fmt.Printf("sqlfmt v%v\n", Version)
		os.Exit(0)
	}

	var jobs []job

	if len(flag.Args()) > 0 {
		for _, fp := range flag.Args() {
			j := job{name: fp}
			if !options.write {
				j.w = os.Stdout
			}
			jobs = append(jobs, j)
		}
	} else {
		jobs = append(jobs, job{r: os.Stdin, w: os.Stdout})
	}

	var errors []error

	for _, j := range jobs {
		var err error
		if j.r == nil {
			j.r, err = os.Open(j.name)
			if err != nil {
				errors = append(errors, err)
				continue
			}
		}

		input, err := ioutil.ReadAll(j.r)
		if err != nil {
			errors = append(errors, err)
			continue
		}

		err = j.r.Close()
		if err != nil {
			errors = append(errors, err)
			continue
		}

		lexer := sqlfmt.NewSqlLexer(string(input))
		stmt, err := sqlfmt.Parse(lexer)
		if err != nil {
			errors = append(errors, err)
			continue
		}

		var inPlace bool
		var tmpPath string

		if j.w == nil {
			dir := filepath.Dir(j.name)
			base := filepath.Base(j.name)
			tmpPath = path.Join(dir, "."+base+".sqlfmt")
			j.w, err = os.Create(tmpPath)
			if err != nil {
				errors = append(errors, err)
				continue
			}
			inPlace = true
		}

		r := sqlfmt.NewTextRenderer(j.w)
		stmt.RenderTo(r)
		if r.Error() != nil {
			errors = append(errors, err)
			continue
		}

		if inPlace {
			err = j.w.Close()
			if err != nil {
				errors = append(errors, err)
				continue
			}
			err = os.Rename(tmpPath, j.name)
			if err != nil {
				errors = append(errors, err)
				continue
			}
		}
	}

	if len(errors) > 0 {
		for _, e := range errors {
			fmt.Fprintln(os.Stderr, e)
		}
		os.Exit(1)
	}
}
<commit_msg>Extract job.run<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/jackc/sqlfmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
)

const Version = "0.1.0"

var options struct {
	write   bool
	version bool
}

type job struct {
	name string
	r    io.ReadCloser
	w    io.WriteCloser
}

func (j *job) run() error {
	if j.r == nil {
		var err error
		j.r, err = os.Open(j.name)
		if err != nil {
			return err
		}
	}

	input, err := ioutil.ReadAll(j.r)
	if err != nil {
		return err
	}

	err = j.r.Close()
	if err != nil {
		return err
	}

	lexer := sqlfmt.NewSqlLexer(string(input))
	stmt, err := sqlfmt.Parse(lexer)
	if err != nil {
		return err
	}

	var inPlace bool
	var tmpPath string

	if j.w == nil {
		dir := filepath.Dir(j.name)
		base := filepath.Base(j.name)
		tmpPath = path.Join(dir, "."+base+".sqlfmt")
		j.w, err = os.Create(tmpPath)
		if err != nil {
			return err
		}
		inPlace = true
	}

	r := sqlfmt.NewTextRenderer(j.w)
	stmt.RenderTo(r)
	if r.Error() != nil {
		return err
	}

	if inPlace {
		err = j.w.Close()
		if err != nil {
			return err
		}
		err = os.Rename(tmpPath, j.name)
		if err != nil {
			return err
		}
	}

	return nil
}

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "usage:  %s [options] [path ...]\n", os.Args[0])
		flag.PrintDefaults()
	}

	flag.BoolVar(&options.write, "w", false, "write result to (source) file instead of stdout")
	flag.BoolVar(&options.version, "version", false, "print version and exit")
	flag.Parse()

	if options.version {
		fmt.Printf("sqlfmt v%v\n", Version)
		os.Exit(0)
	}

	var jobs []job

	if len(flag.Args()) > 0 {
		for _, fp := range flag.Args() {
			j := job{name: fp}
			if !options.write {
				j.w = os.Stdout
			}
			jobs = append(jobs, j)
		}
	} else {
		jobs = append(jobs, job{r: os.Stdin, w: os.Stdout})
	}

	var errors []error

	for _, j := range jobs {
		if err := j.run(); err != nil {
			errors = append(errors, err)
		}
	}

	if len(errors) > 0 {
		for _, e := range errors {
			fmt.Fprintln(os.Stderr, e)
		}
		os.Exit(1)
	}
}
<|endoftext|>