<commit_before>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package request

import (
	"math"
	"net/http"
	"time"

	apirequest "k8s.io/apiserver/pkg/endpoints/request"
	"k8s.io/apiserver/pkg/util/flowcontrol/metrics"
)

const (
	watchesPerSeat          = 10.0
	eventAdditionalDuration = 5 * time.Millisecond
	// TODO(wojtekt): Remove it once we tune the algorithm to not fail
	// scalability tests.
	enableMutatingWorkEstimator = false
)

func newMutatingWorkEstimator(countFn watchCountGetterFunc) WorkEstimatorFunc {
	return newTestMutatingWorkEstimator(countFn, enableMutatingWorkEstimator)
}

func newTestMutatingWorkEstimator(countFn watchCountGetterFunc, enabled bool) WorkEstimatorFunc {
	estimator := &mutatingWorkEstimator{
		countFn: countFn,
		enabled: enabled,
	}
	return estimator.estimate
}

type mutatingWorkEstimator struct {
	countFn watchCountGetterFunc
	enabled bool
}

func (e *mutatingWorkEstimator) estimate(r *http.Request, flowSchemaName, priorityLevelName string) WorkEstimate {
	// TODO(wojtekt): Remove once we tune the algorithm to not fail
	// scalability tests.
	if !e.enabled {
		return WorkEstimate{
			InitialSeats: 1,
		}
	}

	requestInfo, ok := apirequest.RequestInfoFrom(r.Context())
	if !ok {
		// no RequestInfo should never happen, but to be on the safe side
		// let's return a large value.
		return WorkEstimate{
			InitialSeats:      1,
			FinalSeats:        maximumSeats,
			AdditionalLatency: eventAdditionalDuration,
		}
	}
	watchCount := e.countFn(requestInfo)
	metrics.ObserveWatchCount(r.Context(), priorityLevelName, flowSchemaName, watchCount)

	// The cost of the request associated with the watchers of that event
	// consists of three parts:
	// - cost of going through the event change logic
	// - cost of serialization of the event
	// - cost of processing an event object for each watcher (e.g. filtering,
	//     sending data over network)
	// We're starting simple to get some operational experience with it and
	// we will work on tuning the algorithm later. Given that the actual work
	// associated with processing watch events is happening in multiple
	// goroutines (proportional to the number of watchers) that are all
	// resumed at once, as a starting point we assume that each such goroutine
	// is taking 1/Nth of a seat for M milliseconds.
	// We allow the accounting of that work in P&F to be reshaped into another
	// rectangle of equal area for practical reasons.
	var finalSeats uint
	var additionalLatency time.Duration

	// TODO: Make this unconditional after we tune the algorithm better.
	//   Technically, there is an overhead connected to processing an event after
	//   the request finishes even if there is a small number of watches.
	//   However, until we tune the estimation we want to stay on the safe side
	//   an avoid introducing additional latency for almost every single request.
	if watchCount >= watchesPerSeat {
		// TODO: As described in the KEP, we should take into account that not all
		//   events are equal and try to estimate the cost of a single event based on
		//   some historical data about size of events.
		finalSeats = uint(math.Ceil(float64(watchCount) / watchesPerSeat))
		finalWork := SeatsTimesDuration(float64(finalSeats), eventAdditionalDuration)

		// While processing individual events is highly parallel,
		// the design/implementation of P&F has a couple limitations that
		// make using this assumption in the P&F implementation very
		// inefficient because:
		// - we reserve max(initialSeats, finalSeats) for time of executing
		//   both phases of the request
		// - even more importantly, when a given `wide` request is the one to
		//   be dispatched, we are not dispatching any other request until
		//   we accumulate enough seats to dispatch the nominated one, even
		//   if currently unoccupied seats would allow for dispatching some
		//   other requests in the meantime
		// As a consequence of these, the wider the request, the more capacity
		// will effectively be blocked and unused during dispatching and
		// executing this request.
		//
		// To mitigate the impact of it, we're capping the maximum number of
		// seats that can be assigned to a given request. Thanks to it:
		// 1) we reduce the amount of seat-seconds that are "wasted" during
		//    dispatching and executing initial phase of the request
		// 2) we are not changing the finalWork estimate - just potentially
		//    reshaping it to be narrower and longer. As long as the maximum
		//    seats setting will prevent dispatching too many requests at once
		//    to prevent overloading kube-apiserver (and/or etcd or the VM or
		//    a physical machine it is running on), we believe the relaxed
		//    version should be good enough to achieve the P&F goals.
		//
		// TODO: Confirm that the current cap of maximumSeats allow us to
		//   achieve the above.
		if finalSeats > maximumSeats {
			finalSeats = maximumSeats
		}
		additionalLatency = finalWork.DurationPerSeat(float64(finalSeats))
	}

	return WorkEstimate{
		InitialSeats:      1,
		FinalSeats:        finalSeats,
		AdditionalLatency: additionalLatency,
	}
}
<commit_msg>P&F: enable mutating work estimator<commit_after>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package request

import (
	"math"
	"net/http"
	"time"

	apirequest "k8s.io/apiserver/pkg/endpoints/request"
	"k8s.io/apiserver/pkg/util/flowcontrol/metrics"
)

const (
	watchesPerSeat          = 10.0
	eventAdditionalDuration = 5 * time.Millisecond
	// TODO(wojtekt): Remove it once we tune the algorithm to not fail
	// scalability tests.
	enableMutatingWorkEstimator = true
)

func newMutatingWorkEstimator(countFn watchCountGetterFunc) WorkEstimatorFunc {
	return newTestMutatingWorkEstimator(countFn, enableMutatingWorkEstimator)
}

func newTestMutatingWorkEstimator(countFn watchCountGetterFunc, enabled bool) WorkEstimatorFunc {
	estimator := &mutatingWorkEstimator{
		countFn: countFn,
		enabled: enabled,
	}
	return estimator.estimate
}

type mutatingWorkEstimator struct {
	countFn watchCountGetterFunc
	enabled bool
}

func (e *mutatingWorkEstimator) estimate(r *http.Request, flowSchemaName, priorityLevelName string) WorkEstimate {
	// TODO(wojtekt): Remove once we tune the algorithm to not fail
	// scalability tests.
	if !e.enabled {
		return WorkEstimate{
			InitialSeats: 1,
		}
	}

	requestInfo, ok := apirequest.RequestInfoFrom(r.Context())
	if !ok {
		// no RequestInfo should never happen, but to be on the safe side
		// let's return a large value.
		return WorkEstimate{
			InitialSeats:      1,
			FinalSeats:        maximumSeats,
			AdditionalLatency: eventAdditionalDuration,
		}
	}
	watchCount := e.countFn(requestInfo)
	metrics.ObserveWatchCount(r.Context(), priorityLevelName, flowSchemaName, watchCount)

	// The cost of the request associated with the watchers of that event
	// consists of three parts:
	// - cost of going through the event change logic
	// - cost of serialization of the event
	// - cost of processing an event object for each watcher (e.g. filtering,
	//     sending data over network)
	// We're starting simple to get some operational experience with it and
	// we will work on tuning the algorithm later. Given that the actual work
	// associated with processing watch events is happening in multiple
	// goroutines (proportional to the number of watchers) that are all
	// resumed at once, as a starting point we assume that each such goroutine
	// is taking 1/Nth of a seat for M milliseconds.
	// We allow the accounting of that work in P&F to be reshaped into another
	// rectangle of equal area for practical reasons.
	var finalSeats uint
	var additionalLatency time.Duration

	// TODO: Make this unconditional after we tune the algorithm better.
	//   Technically, there is an overhead connected to processing an event after
	//   the request finishes even if there is a small number of watches.
	//   However, until we tune the estimation we want to stay on the safe side
	//   an avoid introducing additional latency for almost every single request.
	if watchCount >= watchesPerSeat {
		// TODO: As described in the KEP, we should take into account that not all
		//   events are equal and try to estimate the cost of a single event based on
		//   some historical data about size of events.
		finalSeats = uint(math.Ceil(float64(watchCount) / watchesPerSeat))
		finalWork := SeatsTimesDuration(float64(finalSeats), eventAdditionalDuration)

		// While processing individual events is highly parallel,
		// the design/implementation of P&F has a couple limitations that
		// make using this assumption in the P&F implementation very
		// inefficient because:
		// - we reserve max(initialSeats, finalSeats) for time of executing
		//   both phases of the request
		// - even more importantly, when a given `wide` request is the one to
		//   be dispatched, we are not dispatching any other request until
		//   we accumulate enough seats to dispatch the nominated one, even
		//   if currently unoccupied seats would allow for dispatching some
		//   other requests in the meantime
		// As a consequence of these, the wider the request, the more capacity
		// will effectively be blocked and unused during dispatching and
		// executing this request.
		//
		// To mitigate the impact of it, we're capping the maximum number of
		// seats that can be assigned to a given request. Thanks to it:
		// 1) we reduce the amount of seat-seconds that are "wasted" during
		//    dispatching and executing initial phase of the request
		// 2) we are not changing the finalWork estimate - just potentially
		//    reshaping it to be narrower and longer. As long as the maximum
		//    seats setting will prevent dispatching too many requests at once
		//    to prevent overloading kube-apiserver (and/or etcd or the VM or
		//    a physical machine it is running on), we believe the relaxed
		//    version should be good enough to achieve the P&F goals.
		//
		// TODO: Confirm that the current cap of maximumSeats allow us to
		//   achieve the above.
		if finalSeats > maximumSeats {
			finalSeats = maximumSeats
		}
		additionalLatency = finalWork.DurationPerSeat(float64(finalSeats))
	}

	return WorkEstimate{
		InitialSeats:      1,
		FinalSeats:        finalSeats,
		AdditionalLatency: additionalLatency,
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/urfave/cli"
	"log"
	"net/url"
	"strconv"
	"strings"
)

// ByAlphabet is for sorting
type ByAlphabet []cli.Command

func (a ByAlphabet) Len() int           { return len(a) }
func (a ByAlphabet) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByAlphabet) Less(i, j int) bool { return a[i].Name < a[j].Name }

// checkGlobalFlags is the place to check global parameters
func checkGlobalFlags(o map[string]string) (opts map[string]string) {
	opts = o

	for k, v := range map[string]string{
		"fields":          fFieldList,
		"format":          fFormat,
		"include":         fInclude,
		"optional_fields": fOptFields,
		"page":            fPageNum,
		"page_size":       fPageSize,
		"sort":            fSortOrder,
	} {
		if v != "" {
			client.SetOption(k, v)
		}
	}

	if fFormat != "" && validateFormat(fFormat) {
		client.SetOption("format", fFormat)
	}
	return opts
}

// validateFormat allows only supported formats
func validateFormat(fmt string) bool {
	f := strings.ToLower(fmt)
	if f == "json" || f == "xml" || f == "api" || f == "txt" || f == "jsonp" {
		return true
	}
	return false
}

func displayOptions(opts map[string]string) {
	log.Println("Options:")
	for key, val := range opts {
		log.Printf("  %s: %s", key, val)
	}
}

func boolToString(k bool) string {
	if k {
		return "true"
	} else {
		return "false"
	}
}

// analyzeTarget breaks up an url into its components
func analyzeTarget(target string) (proto, site, path string, port int) {
	uri, err := url.Parse(target)
	if err != nil {
		// This is no URL, make one
		proto = "https"
		site = target
		path = "/"
		port = 443
	} else {
		proto = uri.Scheme

		// might be host:port
		sp := strings.Split(uri.Host, ":")
		if len(sp) == 2 {
			port, _ = strconv.Atoi(sp[1])
			site = sp[0]
		} else {
			site = uri.Host
		}

		path = uri.Path
		// Path can't be null
		if path == "" {
			path = "/"
		}
	}
	return
}
<commit_msg>Add code to identify IP(4,6) address vs hostnames.<commit_after>package main

import (
	"github.com/urfave/cli"
	"log"
	"net"
	"net/url"
	"strconv"
	"strings"
)

// ByAlphabet is for sorting
type ByAlphabet []cli.Command

func (a ByAlphabet) Len() int           { return len(a) }
func (a ByAlphabet) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByAlphabet) Less(i, j int) bool { return a[i].Name < a[j].Name }

// checkGlobalFlags is the place to check global parameters
func checkGlobalFlags(o map[string]string) (opts map[string]string) {
	opts = o

	for k, v := range map[string]string{
		"fields":          fFieldList,
		"format":          fFormat,
		"include":         fInclude,
		"optional_fields": fOptFields,
		"page":            fPageNum,
		"page_size":       fPageSize,
		"sort":            fSortOrder,
	} {
		if v != "" {
			client.SetOption(k, v)
		}
	}

	if fFormat != "" && validateFormat(fFormat) {
		client.SetOption("format", fFormat)
	}
	return opts
}

// validateFormat allows only supported formats
func validateFormat(fmt string) bool {
	f := strings.ToLower(fmt)
	if f == "json" || f == "xml" || f == "api" || f == "txt" || f == "jsonp" {
		return true
	}
	return false
}

func displayOptions(opts map[string]string) {
	log.Println("Options:")
	for key, val := range opts {
		log.Printf("  %s: %s", key, val)
	}
}

func boolToString(k bool) string {
	if k {
		return "true"
	} else {
		return "false"
	}
}

// analyzeTarget breaks up an url into its components
func analyzeTarget(target string) (proto, site, path string, port int) {
	uri, err := url.Parse(target)
	if err != nil {
		// This is no URL, make one
		proto = "https"
		site = target
		path = "/"
		port = 443
	} else {
		proto = uri.Scheme

		// might be host:port
		sp := strings.Split(uri.Host, ":")
		if len(sp) == 2 {
			port, _ = strconv.Atoi(sp[1])
			site = sp[0]
		} else {
			site = uri.Host
		}

		path = uri.Path
		// Path can't be null
		if path == "" {
			path = "/"
		}
	}
	return
}

const (
	Hostname = iota
	IPv4
	IPv6
)

// checkArgumentType checks whether we have a domain/host or an IPv4/v6 address
func checkArgumentType(arg string) int {
	ip := net.ParseIP(arg)
	if ip != nil {
		if ip.To4() == nil {
			return IPv6
		}
		return IPv4
	}
	return Hostname
}

// prepareFamily sets Want4 and/or Want6 depending on the argument
func prepareFamily(arg string) {
	switch checkArgumentType(arg) {
	case IPv4:
		mycnf.WantAF = Want4
	case IPv6:
		mycnf.WantAF = Want6
	default:
		mycnf.WantAF = WantBoth
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/harbur/captain"
	"github.com/harbur/captain/Godeps/_workspace/src/github.com/fatih/color"
	"github.com/harbur/captain/Godeps/_workspace/src/github.com/spf13/cobra"
)

// Options that are passed by CLI are mapped here for consumption
type Options struct {
	debug        bool
	force        bool
	all_branches bool
	namespace    string
	config       string
	images       []string
}

var options Options

func handleCmd() {

	var cmdBuild = &cobra.Command{
		Use:   "build [image]",
		Short: "Builds the docker image(s) of your repository",
		Long:  `It will build the docker image(s) described on captain.yml in order they appear on file.`,
		Run: func(cmd *cobra.Command, args []string) {
			config := captain.NewConfig(options.namespace, options.config, true)

			if len(args) == 1 {
				config.FilterConfig(args[0])
			}

			buildOpts := captain.BuildOptions{
				Config: config,
				Force:  options.force,
				All_branches:  options.all_branches,
			}

			captain.Build(buildOpts)
		},
	}

	var cmdTest = &cobra.Command{
		Use:   "test",
		Short: "Runs the tests",
		Long:  `It will execute the commands described on test section in order they appear on file.`,
		Run: func(cmd *cobra.Command, args []string) {
			config := captain.NewConfig(options.namespace, options.config, true)

			if len(args) == 1 {
				config.FilterConfig(args[0])
			}

			buildOpts := captain.BuildOptions{
				Config: config,
				Force:  options.force,
				All_branches:  options.all_branches,
			}

			// Build everything before testing
			captain.Build(buildOpts)
			captain.Test(buildOpts)
		},
	}

	var cmdPush = &cobra.Command{
		Use:   "push",
		Short: "Pushes the images to remote registry",
		Long:  `It will push the generated images to the remote registry.`,
		Run: func(cmd *cobra.Command, args []string) {
			config := captain.NewConfig(options.namespace, options.config, true)

			if len(args) == 1 {
				config.FilterConfig(args[0])
			}

			buildOpts := captain.BuildOptions{
				Config: config,
				Force:  options.force,
				All_branches:  options.all_branches,
			}

			// Build everything before pushing
			captain.Build(buildOpts)
			captain.Push(buildOpts)
		},
	}

	var cmdPull = &cobra.Command{
		Use:   "pull",
		Short: "Pulls the images from remote registry",
		Long:  `It will pull the images from the remote registry.`,
		Run: func(cmd *cobra.Command, args []string) {
			config := captain.NewConfig(options.namespace, options.config, true)

			if len(args) == 1 {
				config.FilterConfig(args[0])
			}

			buildOpts := captain.BuildOptions{
				Config: config,
				Force:  options.force,
				All_branches:  options.all_branches,
			}

			captain.Pull(buildOpts)
		},
	}

	var cmdPurge = &cobra.Command{
		Use:   "purge",
		Short: "Purges the stale images",
		Long:  `It will purge the stale images. Stale image is an image that is not the latest of at least one branch.`,
		Run: func(cmd *cobra.Command, args []string) {
			config := captain.NewConfig(options.namespace, options.config, true)

			if len(args) == 1 {
				config.FilterConfig(args[0])
			}

			buildOpts := captain.BuildOptions{
				Config: config,
				Force:  options.force,
				All_branches:  options.all_branches,
			}

			captain.Purge(buildOpts)
		},
	}

	var cmdVersion = &cobra.Command{
		Use:   "version",
		Short: "Display version",
		Long:  `Displays the version of Captain.`,
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Println("v0.8.0")
		},
	}

	var captainCmd = &cobra.Command{
		Use:   "captain",
		Short: "captain - build tool for Docker focused on CI/CD",
		Long: `
Captain, the CLI build tool for Docker made for Continuous Integration / Continuous Delivery.

It works by reading captain.yaml file which describes how to build, test, push and release the docker image(s) of your repository.`,
	}

	captainCmd.PersistentFlags().BoolVarP(&captain.Debug, "debug", "D", false, "Enable debug mode")
	captainCmd.PersistentFlags().StringVarP(&options.namespace, "namespace", "N", getNamespace(), "Set default image namespace")
	captainCmd.PersistentFlags().BoolVarP(&color.NoColor, "no-color", "n", false, "Disable color output")
	captainCmd.PersistentFlags().BoolVarP(&options.all_branches, "all-branches", "B", false, "Build all branches on specific commit instead of just working branch")
	cmdBuild.Flags().BoolVarP(&options.force, "force", "f", false, "Force build even if image is already built")
	captainCmd.AddCommand(cmdBuild, cmdTest, cmdPush, cmdPull, cmdVersion, cmdPurge)
	captainCmd.Execute()
}

func getNamespace() string {
	return os.Getenv("USER")
}
<commit_msg>Added dangling image flag<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/harbur/captain"
	"github.com/harbur/captain/Godeps/_workspace/src/github.com/fatih/color"
	"github.com/harbur/captain/Godeps/_workspace/src/github.com/spf13/cobra"
)

// Options that are passed by CLI are mapped here for consumption
type Options struct {
	debug        bool
	force        bool
	all_branches bool
	namespace    string
	config       string
	images       []string
}

var options Options

func handleCmd() {

	var cmdBuild = &cobra.Command{
		Use:   "build [image]",
		Short: "Builds the docker image(s) of your repository",
		Long:  `It will build the docker image(s) described on captain.yml in order they appear on file.`,
		Run: func(cmd *cobra.Command, args []string) {
			config := captain.NewConfig(options.namespace, options.config, true)

			if len(args) == 1 {
				config.FilterConfig(args[0])
			}

			buildOpts := captain.BuildOptions{
				Config: config,
				Force:  options.force,
				All_branches:  options.all_branches,
			}

			captain.Build(buildOpts)
		},
	}

	var cmdTest = &cobra.Command{
		Use:   "test",
		Short: "Runs the tests",
		Long:  `It will execute the commands described on test section in order they appear on file.`,
		Run: func(cmd *cobra.Command, args []string) {
			config := captain.NewConfig(options.namespace, options.config, true)

			if len(args) == 1 {
				config.FilterConfig(args[0])
			}

			buildOpts := captain.BuildOptions{
				Config: config,
				Force:  options.force,
				All_branches:  options.all_branches,
			}

			// Build everything before testing
			captain.Build(buildOpts)
			captain.Test(buildOpts)
		},
	}

	var cmdPush = &cobra.Command{
		Use:   "push",
		Short: "Pushes the images to remote registry",
		Long:  `It will push the generated images to the remote registry.`,
		Run: func(cmd *cobra.Command, args []string) {
			config := captain.NewConfig(options.namespace, options.config, true)

			if len(args) == 1 {
				config.FilterConfig(args[0])
			}

			buildOpts := captain.BuildOptions{
				Config: config,
				Force:  options.force,
				All_branches:  options.all_branches,
			}

			// Build everything before pushing
			captain.Build(buildOpts)
			captain.Push(buildOpts)
		},
	}

	var cmdPull = &cobra.Command{
		Use:   "pull",
		Short: "Pulls the images from remote registry",
		Long:  `It will pull the images from the remote registry.`,
		Run: func(cmd *cobra.Command, args []string) {
			config := captain.NewConfig(options.namespace, options.config, true)

			if len(args) == 1 {
				config.FilterConfig(args[0])
			}

			buildOpts := captain.BuildOptions{
				Config: config,
				Force:  options.force,
				All_branches:  options.all_branches,
			}

			captain.Pull(buildOpts)
		},
	}

	var cmdPurge = &cobra.Command{
		Use:   "purge",
		Short: "Purges the stale images",
		Long:  `It will purge the stale images. Stale image is an image that is not the latest of at least one branch.`,
		Run: func(cmd *cobra.Command, args []string) {
			config := captain.NewConfig(options.namespace, options.config, true)

			if len(args) == 1 {
				config.FilterConfig(args[0])
			}

			buildOpts := captain.BuildOptions{
				Config: config,
				Force:  options.force,
				All_branches:  options.all_branches,
			}

			captain.Purge(buildOpts)
		},
	}

	var cmdVersion = &cobra.Command{
		Use:   "version",
		Short: "Display version",
		Long:  `Displays the version of Captain.`,
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Println("v0.8.0")
		},
	}

	var captainCmd = &cobra.Command{
		Use:   "captain",
		Short: "captain - build tool for Docker focused on CI/CD",
		Long: `
Captain, the CLI build tool for Docker made for Continuous Integration / Continuous Delivery.

It works by reading captain.yaml file which describes how to build, test, push and release the docker image(s) of your repository.`,
	}

	captainCmd.PersistentFlags().BoolVarP(&captain.Debug, "debug", "D", false, "Enable debug mode")
	captainCmd.PersistentFlags().StringVarP(&options.namespace, "namespace", "N", getNamespace(), "Set default image namespace")
	captainCmd.PersistentFlags().BoolVarP(&color.NoColor, "no-color", "n", false, "Disable color output")
	captainCmd.PersistentFlags().BoolVarP(&options.all_branches, "all-branches", "B", false, "Build all branches on specific commit instead of just working branch")
	cmdBuild.Flags().BoolVarP(&options.force, "force", "f", false, "Force build even if image is already built")
	cmdPurge.Flags().BoolVarP(&options.force, "dangling", "d", false, "Remove dangling images")
	captainCmd.AddCommand(cmdBuild, cmdTest, cmdPush, cmdPull, cmdVersion, cmdPurge)
	captainCmd.Execute()
}

func getNamespace() string {
	return os.Getenv("USER")
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"gnd.la/admin"
	"gnd.la/app"
	"gnd.la/internal/project"
	"gnd.la/log"
	"gnd.la/net/urlutil"
	"gnd.la/util/fileutil"
	"gnd.la/util/generic"
	"gnd.la/util/stringutil"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"text/tabwriter"
	"text/template"
)

const (
	serverUrl = "http://www.gondolaweb.com/api/v1"
)

func getAvailableTemplates() ([]*project.Template, error) {
	ep := serverUrl + "/templates"
	resp, err := http.Get(ep)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("unexpect HTTP response code %d", resp.StatusCode)
	}
	dec := json.NewDecoder(resp.Body)
	var templates []*project.Template
	if err := dec.Decode(&templates); err != nil {
		return nil, err
	}
	return templates, nil
}

func getTemplateReader(url string, gae bool) (*tar.Reader, error) {
	ep, err := urlutil.Join(serverUrl, url)
	if err != nil {
		return nil, err
	}
	if gae {
		ep += "?gae=1"
	}
	resp, err := http.Get(ep)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("unexpect HTTP response code %d", resp.StatusCode)
	}
	zr, err := gzip.NewReader(resp.Body)
	if err != nil {
		return nil, err
	}
	defer zr.Close()
	data, err := ioutil.ReadAll(zr)
	if err != nil {
		return nil, err
	}
	return tar.NewReader(bytes.NewReader(data)), nil
}

func isEmptyDir(dir string) bool {
	files, err := ioutil.ReadDir(dir)
	return err == nil && len(files) == 0
}

func NewCmd(ctx *app.Context) {
	tmpls, err := getAvailableTemplates()
	if err != nil {
		panic(err)
	}
	var list bool
	var ptemplate string
	var gae bool
	ctx.ParseParamValue("list", &list)
	ctx.ParseParamValue("template", &ptemplate)
	ctx.ParseParamValue("gae", &gae)
	if list {
		w := tabwriter.NewWriter(os.Stdout, 8, 4, 2, ' ', 0)
		for _, v := range tmpls {
			fmt.Fprintf(w, "%s:\t%s\n", v.Name, v.Description)
		}
		if err := w.Flush(); err != nil {
			panic(err)
		}
	}
	name := ctx.IndexValue(0)
	if name == "" {
		admin.UsageError("missing directory name")
	}
	for _, v := range tmpls {
		if v.Name == ptemplate {
			if exists, _ := fileutil.Exists(name); exists && !isEmptyDir(name) {
				admin.Errorf("%s already exists", name)
			}
			if err := os.MkdirAll(name, 0755); err != nil {
				panic(err)
			}
			r, err := getTemplateReader(v.URL, gae)
			if err != nil {
				panic(err)
			}
			hdr, err := r.Next()
			if err != nil {
				panic(err)
			}
			// Data to pass to templates
			tmplData := map[string]interface{}{
				"Port":      10000 + rand.Intn(20001), // random port between 10k and 30k
				"AppSecret": stringutil.RandomPrintable(64),
				"DevSecret": stringutil.RandomPrintable(64),
			}
			for hdr != nil {
				p := filepath.Join(name, filepath.FromSlash(hdr.Name))
				info := hdr.FileInfo()
				if info.IsDir() {
					log.Debugf("creating directory %s", p)
					if err := os.MkdirAll(p, info.Mode()); err != nil {
						panic(err)
					}
				} else if filepath.Base(p) != ".keep" {
					// .keep is used to make git keep the directory
					data, err := ioutil.ReadAll(r)
					if err != nil {
						panic(err)
					}
					if ext := filepath.Ext(p); ext == ".gtmpl" {
						p = p[:len(p)-len(ext)]
						tmpl, err := template.New(filepath.Base(p)).Parse(string(data))
						if err != nil {
							panic(err)
						}
						var buf bytes.Buffer
						if err := tmpl.Execute(&buf, tmplData); err != nil {
							panic(err)
						}
						data = buf.Bytes()
					}
					log.Debugf("writing file %s", p)
					if err := ioutil.WriteFile(p, data, info.Mode()); err != nil {
						panic(err)
					}
				}
				hdr, err = r.Next()
				if err != nil && err != io.EOF {
					panic(err)
				}
			}
			return
		}
	}
	available := generic.Map(tmpls, func(t *project.Template) string { return t.Name }).([]string)
	admin.Errorf("template %s not found, availble ones are: %s", ptemplate, strings.Join(available, ", "))
}

func init() {
	admin.Register(NewCmd, &admin.Options{
		Name:  "New",
		Help:  "Create a new Gondola project",
		Usage: "<dir_name>",
		Flags: admin.Flags(
			admin.StringFlag("template", "hello", "Project template to use"),
			admin.BoolFlag("list", false, "List available project templates"),
			admin.BoolFlag("gae", false, "Create an App Engine hybrid project"),
		),
	})
}
<commit_msg>Return after listing available templates<commit_after>package main

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"gnd.la/admin"
	"gnd.la/app"
	"gnd.la/internal/project"
	"gnd.la/log"
	"gnd.la/net/urlutil"
	"gnd.la/util/fileutil"
	"gnd.la/util/generic"
	"gnd.la/util/stringutil"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"text/tabwriter"
	"text/template"
)

const (
	serverUrl = "http://www.gondolaweb.com/api/v1"
)

func getAvailableTemplates() ([]*project.Template, error) {
	ep := serverUrl + "/templates"
	resp, err := http.Get(ep)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("unexpect HTTP response code %d", resp.StatusCode)
	}
	dec := json.NewDecoder(resp.Body)
	var templates []*project.Template
	if err := dec.Decode(&templates); err != nil {
		return nil, err
	}
	return templates, nil
}

func getTemplateReader(url string, gae bool) (*tar.Reader, error) {
	ep, err := urlutil.Join(serverUrl, url)
	if err != nil {
		return nil, err
	}
	if gae {
		ep += "?gae=1"
	}
	resp, err := http.Get(ep)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("unexpect HTTP response code %d", resp.StatusCode)
	}
	zr, err := gzip.NewReader(resp.Body)
	if err != nil {
		return nil, err
	}
	defer zr.Close()
	data, err := ioutil.ReadAll(zr)
	if err != nil {
		return nil, err
	}
	return tar.NewReader(bytes.NewReader(data)), nil
}

func isEmptyDir(dir string) bool {
	files, err := ioutil.ReadDir(dir)
	return err == nil && len(files) == 0
}

func NewCmd(ctx *app.Context) {
	tmpls, err := getAvailableTemplates()
	if err != nil {
		panic(err)
	}
	var list bool
	var ptemplate string
	var gae bool
	ctx.ParseParamValue("list", &list)
	ctx.ParseParamValue("template", &ptemplate)
	ctx.ParseParamValue("gae", &gae)
	if list {
		w := tabwriter.NewWriter(os.Stdout, 8, 4, 2, ' ', 0)
		for _, v := range tmpls {
			fmt.Fprintf(w, "%s:\t%s\n", v.Name, v.Description)
		}
		if err := w.Flush(); err != nil {
			panic(err)
		}
		return
	}
	name := ctx.IndexValue(0)
	if name == "" {
		admin.UsageError("missing directory name")
	}
	for _, v := range tmpls {
		if v.Name == ptemplate {
			if exists, _ := fileutil.Exists(name); exists && !isEmptyDir(name) {
				admin.Errorf("%s already exists", name)
			}
			if err := os.MkdirAll(name, 0755); err != nil {
				panic(err)
			}
			r, err := getTemplateReader(v.URL, gae)
			if err != nil {
				panic(err)
			}
			hdr, err := r.Next()
			if err != nil {
				panic(err)
			}
			// Data to pass to templates
			tmplData := map[string]interface{}{
				"Port":      10000 + rand.Intn(20001), // random port between 10k and 30k
				"AppSecret": stringutil.RandomPrintable(64),
				"DevSecret": stringutil.RandomPrintable(64),
			}
			for hdr != nil {
				p := filepath.Join(name, filepath.FromSlash(hdr.Name))
				info := hdr.FileInfo()
				if info.IsDir() {
					log.Debugf("creating directory %s", p)
					if err := os.MkdirAll(p, info.Mode()); err != nil {
						panic(err)
					}
				} else if filepath.Base(p) != ".keep" {
					// .keep is used to make git keep the directory
					data, err := ioutil.ReadAll(r)
					if err != nil {
						panic(err)
					}
					if ext := filepath.Ext(p); ext == ".gtmpl" {
						p = p[:len(p)-len(ext)]
						tmpl, err := template.New(filepath.Base(p)).Parse(string(data))
						if err != nil {
							panic(err)
						}
						var buf bytes.Buffer
						if err := tmpl.Execute(&buf, tmplData); err != nil {
							panic(err)
						}
						data = buf.Bytes()
					}
					log.Debugf("writing file %s", p)
					if err := ioutil.WriteFile(p, data, info.Mode()); err != nil {
						panic(err)
					}
				}
				hdr, err = r.Next()
				if err != nil && err != io.EOF {
					panic(err)
				}
			}
			return
		}
	}
	available := generic.Map(tmpls, func(t *project.Template) string { return t.Name }).([]string)
	admin.Errorf("template %s not found, availble ones are: %s", ptemplate, strings.Join(available, ", "))
}

func init() {
	admin.Register(NewCmd, &admin.Options{
		Name:  "New",
		Help:  "Create a new Gondola project",
		Usage: "<dir_name>",
		Flags: admin.Flags(
			admin.StringFlag("template", "hello", "Project template to use"),
			admin.BoolFlag("list", false, "List available project templates"),
			admin.BoolFlag("gae", false, "Create an App Engine hybrid project"),
		),
	})
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"math/rand"
	"net"
	"os"
	"strings"
	"sync/atomic"
	"time"
)

func main() {
	opts := parseArgs(os.Args[1:])

	pendingWorkers := make(chan struct{}, opts.Workers)
	metricGenerators := make([]*metricGenerator, 0, opts.Workers)
	for i := uint(0); i < opts.Workers; i++ {
		generator := &metricGenerator{
			rnd: rand.New(rand.NewSource(rand.Int63())),
			counters: metricData{
				nameFormat:      fmt.Sprintf("%scounter%s", opts.MetricPrefix, opts.MetricSuffix),
				count:           opts.Counts.Counter / uint64(opts.Workers),
				nameCardinality: opts.NameCard.Counter,
				tagCardinality:  opts.TagCard.Counter,
				valueLimit:      opts.ValueRange.Counter,
			},
			gauges: metricData{
				nameFormat:      fmt.Sprintf("%sgauge%s", opts.MetricPrefix, opts.MetricSuffix),
				count:           opts.Counts.Gauge / uint64(opts.Workers),
				nameCardinality: opts.NameCard.Gauge,
				tagCardinality:  opts.TagCard.Gauge,
				valueLimit:      opts.ValueRange.Gauge,
			},
			sets: metricData{
				nameFormat:      fmt.Sprintf("%sset%s", opts.MetricPrefix, opts.MetricSuffix),
				count:           opts.Counts.Set / uint64(opts.Workers),
				nameCardinality: opts.NameCard.Set,
				tagCardinality:  opts.TagCard.Set,
				valueLimit:      opts.ValueRange.Set,
			},
			timers: metricData{
				nameFormat:      fmt.Sprintf("%stimer%s", opts.MetricPrefix, opts.MetricSuffix),
				count:           opts.Counts.Timer / uint64(opts.Workers),
				nameCardinality: opts.NameCard.Timer,
				tagCardinality:  opts.TagCard.Timer,
				valueLimit:      opts.ValueRange.Timer,
			},
		}
		metricGenerators = append(metricGenerators, generator)
		go sendMetricsWorker(
			opts.Target,
			opts.DatagramSize,
			opts.Rate/opts.Workers,
			generator,
			pendingWorkers,
		)
	}

	runningWorkers := opts.Workers
	statusTicker := time.NewTicker(1 * time.Second)
	for runningWorkers > 0 {
		select {
		case <-pendingWorkers:
			runningWorkers--
		case <-statusTicker.C:
			counters := uint64(0)
			gauges := uint64(0)
			sets := uint64(0)
			timers := uint64(0)
			for _, mg := range metricGenerators {
				counters += atomic.LoadUint64(&mg.counters.count)
				gauges += atomic.LoadUint64(&mg.gauges.count)
				sets += atomic.LoadUint64(&mg.sets.count)
				timers += atomic.LoadUint64(&mg.timers.count)
			}
			fmt.Printf("%d counters, %d gauges, %d sets, %d timers\n", counters, gauges, sets, timers)
		}
	}
}

func sendMetricsWorker(
	address string,
	bufSize uint,
	rate uint,
	generator *metricGenerator,
	chDone chan<- struct{},
) {
	s, err := net.DialTimeout("udp", address, 1*time.Second)
	if err != nil {
		panic(err)
	}

	b := &bytes.Buffer{}

	interval := time.Second / time.Duration(rate)

	next := time.Now().Add(interval)

	sb := &strings.Builder{}
	for generator.next(sb) {
		if uint(b.Len()+sb.Len()) > bufSize {
			timeToFlush := time.Until(next)
			if timeToFlush > 0 {
				time.Sleep(timeToFlush)
			}
			_, err := s.Write(b.Bytes())
			if err != nil {
				panic(err)
			}
			b.Reset()
			next = next.Add(interval)
		}
		b.WriteString(sb.String())
		sb.Reset()
	}

	if b.Len() > 0 {
		_, err := s.Write(b.Bytes())
		if err != nil {
			panic(err)
		}
	}
	chDone <- struct{}{}
}
<commit_msg>Don't panic after failing to send.<commit_after>package main

import (
	"bytes"
	"fmt"
	"math/rand"
	"net"
	"os"
	"strings"
	"sync/atomic"
	"time"
)

func main() {
	opts := parseArgs(os.Args[1:])

	pendingWorkers := make(chan struct{}, opts.Workers)
	metricGenerators := make([]*metricGenerator, 0, opts.Workers)
	for i := uint(0); i < opts.Workers; i++ {
		generator := &metricGenerator{
			rnd: rand.New(rand.NewSource(rand.Int63())),
			counters: metricData{
				nameFormat:      fmt.Sprintf("%scounter%s", opts.MetricPrefix, opts.MetricSuffix),
				count:           opts.Counts.Counter / uint64(opts.Workers),
				nameCardinality: opts.NameCard.Counter,
				tagCardinality:  opts.TagCard.Counter,
				valueLimit:      opts.ValueRange.Counter,
			},
			gauges: metricData{
				nameFormat:      fmt.Sprintf("%sgauge%s", opts.MetricPrefix, opts.MetricSuffix),
				count:           opts.Counts.Gauge / uint64(opts.Workers),
				nameCardinality: opts.NameCard.Gauge,
				tagCardinality:  opts.TagCard.Gauge,
				valueLimit:      opts.ValueRange.Gauge,
			},
			sets: metricData{
				nameFormat:      fmt.Sprintf("%sset%s", opts.MetricPrefix, opts.MetricSuffix),
				count:           opts.Counts.Set / uint64(opts.Workers),
				nameCardinality: opts.NameCard.Set,
				tagCardinality:  opts.TagCard.Set,
				valueLimit:      opts.ValueRange.Set,
			},
			timers: metricData{
				nameFormat:      fmt.Sprintf("%stimer%s", opts.MetricPrefix, opts.MetricSuffix),
				count:           opts.Counts.Timer / uint64(opts.Workers),
				nameCardinality: opts.NameCard.Timer,
				tagCardinality:  opts.TagCard.Timer,
				valueLimit:      opts.ValueRange.Timer,
			},
		}
		metricGenerators = append(metricGenerators, generator)
		go sendMetricsWorker(
			opts.Target,
			opts.DatagramSize,
			opts.Rate/opts.Workers,
			generator,
			pendingWorkers,
		)
	}

	runningWorkers := opts.Workers
	statusTicker := time.NewTicker(1 * time.Second)
	for runningWorkers > 0 {
		select {
		case <-pendingWorkers:
			runningWorkers--
		case <-statusTicker.C:
			counters := uint64(0)
			gauges := uint64(0)
			sets := uint64(0)
			timers := uint64(0)
			for _, mg := range metricGenerators {
				counters += atomic.LoadUint64(&mg.counters.count)
				gauges += atomic.LoadUint64(&mg.gauges.count)
				sets += atomic.LoadUint64(&mg.sets.count)
				timers += atomic.LoadUint64(&mg.timers.count)
			}
			fmt.Printf("%d counters, %d gauges, %d sets, %d timers\n", counters, gauges, sets, timers)
		}
	}
}

func sendMetricsWorker(
	address string,
	bufSize uint,
	rate uint,
	generator *metricGenerator,
	chDone chan<- struct{},
) {
	s, err := net.DialTimeout("udp", address, 1*time.Second)
	if err != nil {
		panic(err)
	}

	b := &bytes.Buffer{}

	interval := time.Second / time.Duration(rate)

	next := time.Now().Add(interval)

	sb := &strings.Builder{}
	for generator.next(sb) {
		if uint(b.Len()+sb.Len()) > bufSize {
			timeToFlush := time.Until(next)
			if timeToFlush > 0 {
				time.Sleep(timeToFlush)
			}
			_, err := s.Write(b.Bytes())
			if err != nil {
				fmt.Printf("Pausing for 1 second, error sending packet: %v\n", err)
				time.Sleep(1*time.Second)
			}
			b.Reset()
			next = next.Add(interval)
		}
		b.WriteString(sb.String())
		sb.Reset()
	}

	if b.Len() > 0 {
		_, err := s.Write(b.Bytes())
		if err != nil {
			panic(err)
		}
	}
	chDone <- struct{}{}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"encoding/gob"
	"encoding/json"
	"errors"
	"github.com/Symantec/Dominator/lib/mdb"
	"log"
	"net/http"
	"os"
	"path"
	"regexp"
	"runtime"
	"sort"
	"strings"
	"time"
)

type genericEncoder interface {
	Encode(v interface{}) error
}

func runDaemon(sources []source, mdbFileName, hostnameRegex string,
	datacentre string, fetchInterval uint, logger *log.Logger, debug bool) {
	var prevMdb *mdb.Mdb
	var hostnameRE *regexp.Regexp
	var err error
	if hostnameRegex != ".*" {
		hostnameRE, err = regexp.Compile("^" + hostnameRegex)
		if err != nil {
			logger.Println(err)
			os.Exit(1)
		}
	}
	var cycleStopTime time.Time
	fetchIntervalDuration := time.Duration(fetchInterval) * time.Second
	for ; ; sleepUntil(cycleStopTime) {
		cycleStopTime = time.Now().Add(fetchIntervalDuration)
		newMdb, err := loadFromAll(sources, datacentre, logger)
		if err != nil {
			logger.Println(err)
			continue
		}
		newMdb = selectHosts(newMdb, hostnameRE)
		sort.Sort(newMdb)
		if newMdbIsDifferent(prevMdb, newMdb) {
			if err := writeMdb(newMdb, mdbFileName); err != nil {
				logger.Println(err)
			} else {
				if debug {
					logger.Printf("Wrote new MDB data, %d machines\n",
						len(newMdb.Machines))
				}
				prevMdb = newMdb
			}
		} else if debug {
			logger.Printf("Refreshed MDB data, same %d machines\n",
				len(newMdb.Machines))
		}
	}
}

func sleepUntil(wakeTime time.Time) {
	runtime.GC() // An opportune time to take out the garbage.
	sleepTime := wakeTime.Sub(time.Now())
	if sleepTime < time.Second {
		sleepTime = time.Second
	}
	time.Sleep(sleepTime)
}

func loadFromAll(sources []source, datacentre string,
	logger *log.Logger) (*mdb.Mdb, error) {
	var newMdb mdb.Mdb
	hostMap := make(map[string]struct{})
	for _, source := range sources {
		mdb, err := loadMdb(source.driverFunc, source.url, datacentre, logger)
		if err != nil {
			return nil, err
		}
		for _, machine := range mdb.Machines {
			if _, ok := hostMap[machine.Hostname]; !ok {
				newMdb.Machines = append(newMdb.Machines, machine)
				hostMap[machine.Hostname] = struct{}{}
			}
		}
	}
	return &newMdb, nil
}

func loadMdb(driverFunc driverFunc, url string, datacentre string,
	logger *log.Logger) (
	*mdb.Mdb, error) {
	if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") {
		return loadHttpMdb(driverFunc, url, datacentre, logger)
	}
	file, err := os.Open(url)
	if err != nil {
		return nil, errors.New(("Error opening file " + err.Error()))
	}
	defer file.Close()
	return driverFunc(bufio.NewReader(file), datacentre, logger)
}

func loadHttpMdb(driverFunc driverFunc, url string, datacentre string,
	logger *log.Logger) (
	*mdb.Mdb, error) {
	response, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer response.Body.Close()
	if response.StatusCode != http.StatusOK {
		return nil, errors.New("HTTP get failed")
	}
	return driverFunc(response.Body, datacentre, logger)
}

func selectHosts(inMdb *mdb.Mdb, hostnameRE *regexp.Regexp) *mdb.Mdb {
	if hostnameRE == nil {
		return inMdb
	}
	var outMdb mdb.Mdb
	for _, machine := range inMdb.Machines {
		if hostnameRE.MatchString(machine.Hostname) {
			outMdb.Machines = append(outMdb.Machines, machine)
		}
	}
	return &outMdb
}

func newMdbIsDifferent(prevMdb, newMdb *mdb.Mdb) bool {
	if prevMdb == nil {
		return true
	}
	if len(prevMdb.Machines) != len(newMdb.Machines) {
		return true
	}
	for index, prevMachine := range prevMdb.Machines {
		if prevMachine != newMdb.Machines[index] {
			return true
		}
	}
	return false
}

func writeMdb(mdb *mdb.Mdb, mdbFileName string) error {
	tmpFileName := mdbFileName + "~"
	file, err := os.Create(tmpFileName)
	if err != nil {
		return errors.New("Error opening file " + err.Error())
	}
	defer os.Remove(tmpFileName)
	defer file.Close()
	writer := bufio.NewWriter(file)
	defer writer.Flush()
	switch path.Ext(mdbFileName) {
	case ".gob":
		if err := gob.NewEncoder(writer).Encode(mdb.Machines); err != nil {
			return err
		}
	default:
		b, err := json.Marshal(mdb.Machines)
		if err != nil {
			return err
		}
		var out bytes.Buffer
		json.Indent(&out, b, "", "    ")
		_, err = out.WriteTo(writer)
		if err != nil {
			return err
		}
		writer.Write([]byte("\n"))
	}
	return os.Rename(tmpFileName, mdbFileName)
}
<commit_msg>Change mdbd to merge machine data.<commit_after>package main

import (
	"bufio"
	"bytes"
	"encoding/gob"
	"encoding/json"
	"errors"
	"github.com/Symantec/Dominator/lib/mdb"
	"log"
	"net/http"
	"os"
	"path"
	"regexp"
	"runtime"
	"sort"
	"strings"
	"time"
)

type genericEncoder interface {
	Encode(v interface{}) error
}

func runDaemon(sources []source, mdbFileName, hostnameRegex string,
	datacentre string, fetchInterval uint, logger *log.Logger, debug bool) {
	var prevMdb *mdb.Mdb
	var hostnameRE *regexp.Regexp
	var err error
	if hostnameRegex != ".*" {
		hostnameRE, err = regexp.Compile("^" + hostnameRegex)
		if err != nil {
			logger.Println(err)
			os.Exit(1)
		}
	}
	var cycleStopTime time.Time
	fetchIntervalDuration := time.Duration(fetchInterval) * time.Second
	for ; ; sleepUntil(cycleStopTime) {
		cycleStopTime = time.Now().Add(fetchIntervalDuration)
		newMdb, err := loadFromAll(sources, datacentre, logger)
		if err != nil {
			logger.Println(err)
			continue
		}
		newMdb = selectHosts(newMdb, hostnameRE)
		sort.Sort(newMdb)
		if newMdbIsDifferent(prevMdb, newMdb) {
			if err := writeMdb(newMdb, mdbFileName); err != nil {
				logger.Println(err)
			} else {
				if debug {
					logger.Printf("Wrote new MDB data, %d machines\n",
						len(newMdb.Machines))
				}
				prevMdb = newMdb
			}
		} else if debug {
			logger.Printf("Refreshed MDB data, same %d machines\n",
				len(newMdb.Machines))
		}
	}
}

func sleepUntil(wakeTime time.Time) {
	runtime.GC() // An opportune time to take out the garbage.
	sleepTime := wakeTime.Sub(time.Now())
	if sleepTime < time.Second {
		sleepTime = time.Second
	}
	time.Sleep(sleepTime)
}

func loadFromAll(sources []source, datacentre string,
	logger *log.Logger) (*mdb.Mdb, error) {
	machineMap := make(map[string]mdb.Machine)
	for _, source := range sources {
		mdb, err := loadMdb(source.driverFunc, source.url, datacentre, logger)
		if err != nil {
			return nil, err
		}
		for _, machine := range mdb.Machines {
			if oldMachine, ok := machineMap[machine.Hostname]; ok {
				if machine.RequiredImage != "" {
					oldMachine.RequiredImage = machine.RequiredImage
				}
				if machine.PlannedImage != "" {
					oldMachine.PlannedImage = machine.PlannedImage
				}
				machineMap[machine.Hostname] = oldMachine
			} else {
				machineMap[machine.Hostname] = machine
			}
		}
	}
	var newMdb mdb.Mdb
	for _, machine := range machineMap {
		newMdb.Machines = append(newMdb.Machines, machine)
	}
	return &newMdb, nil
}

func loadMdb(driverFunc driverFunc, url string, datacentre string,
	logger *log.Logger) (
	*mdb.Mdb, error) {
	if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") {
		return loadHttpMdb(driverFunc, url, datacentre, logger)
	}
	file, err := os.Open(url)
	if err != nil {
		return nil, errors.New(("Error opening file " + err.Error()))
	}
	defer file.Close()
	return driverFunc(bufio.NewReader(file), datacentre, logger)
}

func loadHttpMdb(driverFunc driverFunc, url string, datacentre string,
	logger *log.Logger) (
	*mdb.Mdb, error) {
	response, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer response.Body.Close()
	if response.StatusCode != http.StatusOK {
		return nil, errors.New("HTTP get failed")
	}
	return driverFunc(response.Body, datacentre, logger)
}

func selectHosts(inMdb *mdb.Mdb, hostnameRE *regexp.Regexp) *mdb.Mdb {
	if hostnameRE == nil {
		return inMdb
	}
	var outMdb mdb.Mdb
	for _, machine := range inMdb.Machines {
		if hostnameRE.MatchString(machine.Hostname) {
			outMdb.Machines = append(outMdb.Machines, machine)
		}
	}
	return &outMdb
}

func newMdbIsDifferent(prevMdb, newMdb *mdb.Mdb) bool {
	if prevMdb == nil {
		return true
	}
	if len(prevMdb.Machines) != len(newMdb.Machines) {
		return true
	}
	for index, prevMachine := range prevMdb.Machines {
		if prevMachine != newMdb.Machines[index] {
			return true
		}
	}
	return false
}

func writeMdb(mdb *mdb.Mdb, mdbFileName string) error {
	tmpFileName := mdbFileName + "~"
	file, err := os.Create(tmpFileName)
	if err != nil {
		return errors.New("Error opening file " + err.Error())
	}
	defer os.Remove(tmpFileName)
	defer file.Close()
	writer := bufio.NewWriter(file)
	defer writer.Flush()
	switch path.Ext(mdbFileName) {
	case ".gob":
		if err := gob.NewEncoder(writer).Encode(mdb.Machines); err != nil {
			return err
		}
	default:
		b, err := json.Marshal(mdb.Machines)
		if err != nil {
			return err
		}
		var out bytes.Buffer
		json.Indent(&out, b, "", "    ")
		_, err = out.WriteTo(writer)
		if err != nil {
			return err
		}
		writer.Write([]byte("\n"))
	}
	return os.Rename(tmpFileName, mdbFileName)
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"encoding/pem"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path"
	"regexp"
	"sort"
	"strings"
)

const (
	ServerCertificateName = "server"
)

var clientVersion = strings.Replace(buildVersion, "v", "", 1)

type Client struct {
	host   string
	client *http.Client
}

type ListItem struct {
	Name        string
	Description string
	Vpnc        bool
	OpenConnect bool
	Socks       bool
	Hosts       []string
	LocalOnly   bool
	Version     float64
	IntVersion  int
}

type upgradeManifest struct {
	URL string
}

var obfuscatedRe = regexp.MustCompile(`\$mole\$[0-9a-f-]{36}`)

func caCert() *x509.Certificate {
	file, err := os.Open(path.Join(globalOpts.Home, "ca-cert.pem"))
	if err != nil {
		return nil
	}
	pemdata, err := ioutil.ReadAll(file)
	if err != nil {
		return nil
	}

	block, _ := pem.Decode(pemdata)
	if block.Type != "CERTIFICATE" {
		return nil
	}

	cert, err := x509.ParseCertificate(block.Bytes)
	if err != nil {
		return nil
	}

	return cert
}

func NewClient(host string, cert tls.Certificate) *Client {
	crt := caCert()
	if crt != nil {
		pool := x509.NewCertPool()
		pool.AddCert(crt)
		transport := &http.Transport{
			Dial: func(n, a string) (net.Conn, error) {
				return net.Dial(n, host)
			},
			TLSClientConfig: &tls.Config{
				ServerName:   ServerCertificateName,
				RootCAs:      pool,
				Certificates: []tls.Certificate{cert},
			},
		}
		client := &http.Client{Transport: transport}
		return &Client{ServerCertificateName, client}
	} else {
		debugln(msgWarnNoCert)
		transport := &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
				Certificates:       []tls.Certificate{cert},
			},
		}
		client := &http.Client{Transport: transport}
		return &Client{host, client}
	}
}

func (c *Client) request(method, path string, content io.Reader) *http.Response {
	url := "https://" + c.host + path
	debugln(method, url)

	req, err := http.NewRequest(method, url, content)
	fatalErr(err)
	req.Header.Add("X-Mole-Version", clientVersion)

	resp, err := c.client.Do(req)
	fatalErr(err)

	if resp.StatusCode != 200 {
		data, _ := ioutil.ReadAll(resp.Body)
		warnln(resp.Status)
		fatalln(string(data))
	}

	debugln(resp.Status, resp.Header.Get("Content-type"), resp.ContentLength)

	return resp
}

func (c *Client) List() []ListItem {
	resp := c.request("GET", "/store", nil)
	defer resp.Body.Close()

	data, err := ioutil.ReadAll(resp.Body)
	fatalErr(err)
	var items []ListItem
	err = json.Unmarshal(data, &items)
	fatalErr(err)
	for i := range items {
		items[i].IntVersion = int(100 * items[i].Version)
	}
	sort.Sort(listItems(items))
	return items
}

func (c *Client) Get(tunnel string) string {
	resp := c.request("GET", "/store/"+tunnel+".ini", nil)
	defer resp.Body.Close()

	data, err := ioutil.ReadAll(resp.Body)
	fatalErr(err)
	res := string(data)

	return res
}

func (c *Client) Put(tunnel string, data io.Reader) {
	resp := c.request("PUT", "/store/"+tunnel+".ini", data)
	defer resp.Body.Close()
}

func (c *Client) Deobfuscate(tunnel string) string {
	matches := obfuscatedRe.FindAllString(tunnel, -1)
	for _, o := range matches {
		tunnel = strings.Replace(tunnel, o, c.getToken(o[6:]), -1)
	}

	return tunnel
}

func (c *Client) UpgradesURL() string {
	url := "https://" + c.host + "/extra/upgrades.json"
	debugln("GET", url)

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return ""
	}
	req.Header.Add("X-Mole-Version", clientVersion)

	resp, err := c.client.Do(req)
	if err != nil {
		return ""
	}

	defer resp.Body.Close()

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return ""
	}

	var manifest upgradeManifest
	err = json.Unmarshal(data, &manifest)
	if err != nil {
		return ""
	}
	return manifest.URL
}

func (c *Client) getToken(token string) string {
	resp := c.request("GET", "/key/"+token, nil)
	defer resp.Body.Close()

	data, err := ioutil.ReadAll(resp.Body)
	fatalErr(err)

	var res map[string]string
	err = json.Unmarshal(data, &res)
	fatalErr(err)
	return fmt.Sprintf("%q", res["key"])
}

type listItems []ListItem

func (l listItems) Len() int {
	return len(l)
}

func (l listItems) Less(i, j int) bool {
	return l[i].Name < l[j].Name
}

func (l listItems) Swap(i, j int) {
	l[i], l[j] = l[j], l[i]
}
<commit_msg>Minor refactor (function name)<commit_after>package main

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"encoding/pem"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path"
	"regexp"
	"sort"
	"strings"
)

const (
	ServerCertificateName = "server"
)

var clientVersion = strings.Replace(buildVersion, "v", "", 1)

type Client struct {
	host   string
	client *http.Client
}

type ListItem struct {
	Name        string
	Description string
	Vpnc        bool
	OpenConnect bool
	Socks       bool
	Hosts       []string
	LocalOnly   bool
	Version     float64
	IntVersion  int
}

type upgradeManifest struct {
	URL string
}

var obfuscatedRe = regexp.MustCompile(`\$mole\$[0-9a-f-]{36}`)

func caCert() *x509.Certificate {
	file, err := os.Open(path.Join(globalOpts.Home, "ca-cert.pem"))
	if err != nil {
		return nil
	}
	pemdata, err := ioutil.ReadAll(file)
	if err != nil {
		return nil
	}

	block, _ := pem.Decode(pemdata)
	if block.Type != "CERTIFICATE" {
		return nil
	}

	cert, err := x509.ParseCertificate(block.Bytes)
	if err != nil {
		return nil
	}

	return cert
}

func NewClient(host string, cert tls.Certificate) *Client {
	crt := caCert()
	if crt != nil {
		pool := x509.NewCertPool()
		pool.AddCert(crt)
		transport := &http.Transport{
			Dial: func(n, a string) (net.Conn, error) {
				return net.Dial(n, host)
			},
			TLSClientConfig: &tls.Config{
				ServerName:   ServerCertificateName,
				RootCAs:      pool,
				Certificates: []tls.Certificate{cert},
			},
		}
		client := &http.Client{Transport: transport}
		return &Client{ServerCertificateName, client}
	} else {
		debugln(msgWarnNoCert)
		transport := &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
				Certificates:       []tls.Certificate{cert},
			},
		}
		client := &http.Client{Transport: transport}
		return &Client{host, client}
	}
}

func (c *Client) request(method, path string, content io.Reader) *http.Response {
	url := "https://" + c.host + path
	debugln(method, url)

	req, err := http.NewRequest(method, url, content)
	fatalErr(err)
	req.Header.Add("X-Mole-Version", clientVersion)

	resp, err := c.client.Do(req)
	fatalErr(err)

	if resp.StatusCode != 200 {
		data, _ := ioutil.ReadAll(resp.Body)
		warnln(resp.Status)
		fatalln(string(data))
	}

	debugln(resp.Status, resp.Header.Get("Content-type"), resp.ContentLength)

	return resp
}

func (c *Client) List() []ListItem {
	resp := c.request("GET", "/store", nil)
	defer resp.Body.Close()

	data, err := ioutil.ReadAll(resp.Body)
	fatalErr(err)
	var items []ListItem
	err = json.Unmarshal(data, &items)
	fatalErr(err)
	for i := range items {
		items[i].IntVersion = int(100 * items[i].Version)
	}
	sort.Sort(listItems(items))
	return items
}

func (c *Client) Get(tunnel string) string {
	resp := c.request("GET", "/store/"+tunnel+".ini", nil)
	defer resp.Body.Close()

	data, err := ioutil.ReadAll(resp.Body)
	fatalErr(err)
	res := string(data)

	return res
}

func (c *Client) Put(tunnel string, data io.Reader) {
	resp := c.request("PUT", "/store/"+tunnel+".ini", data)
	defer resp.Body.Close()
}

func (c *Client) Deobfuscate(tunnel string) string {
	matches := obfuscatedRe.FindAllString(tunnel, -1)
	for _, o := range matches {
		tunnel = strings.Replace(tunnel, o, c.resolveKey(o[6:]), -1)
	}

	return tunnel
}

func (c *Client) UpgradesURL() string {
	url := "https://" + c.host + "/extra/upgrades.json"
	debugln("GET", url)

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return ""
	}
	req.Header.Add("X-Mole-Version", clientVersion)

	resp, err := c.client.Do(req)
	if err != nil {
		return ""
	}

	defer resp.Body.Close()

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return ""
	}

	var manifest upgradeManifest
	err = json.Unmarshal(data, &manifest)
	if err != nil {
		return ""
	}
	return manifest.URL
}

func (c *Client) resolveKey(key string) string {
	resp := c.request("GET", "/key/"+key, nil)
	defer resp.Body.Close()

	data, err := ioutil.ReadAll(resp.Body)
	fatalErr(err)

	var res map[string]string
	err = json.Unmarshal(data, &res)
	fatalErr(err)
	return fmt.Sprintf("%q", res["key"])
}

type listItems []ListItem

func (l listItems) Len() int {
	return len(l)
}

func (l listItems) Less(i, j int) bool {
	return l[i].Name < l[j].Name
}

func (l listItems) Swap(i, j int) {
	l[i], l[j] = l[j], l[i]
}
<|endoftext|>


<commit_before>// +build go1.14

package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sync"

	"github.com/ulikunitz/xz/lzma"
	"harvey-os.org/ninep/protocol"
	"harvey-os.org/ninep/tmpfs"
	"harvey-os.org/sys"
)

var (
	srv   = flag.String("s", "tmpfs", "srv name")
	debug = flag.Bool("d", false, "Print debug messages")
	ext   = flag.String("ext", "cpio", "file type")
)

// Constant error messages to match those found in the linux 9p source.
// These strings are used by linux to map to particular error codes.
// (See net/9p/error.c in the Linux code)
const (
	ErrorAuthFailed   = "authentication failed"
	ErrorReadOnlyFs   = "Read-only file system"
	ErrorFileNotFound = "file not found"
	ErrorFidInUse     = "fid already in use"
	ErrorFidNotFound  = "fid unknown or out of range"
)

type fileServer struct {
	sync.Mutex

	Versioned bool

	ioUnit  protocol.MaxSize
	archive *tmpfs.Archive
	files   map[protocol.FID]*FidEntry
}

// FidEntry wraps an Entry with the instance data required for a fid reference
type FidEntry struct {
	tmpfs.Entry

	// Username to be used for all entries in this hierarchy
	uname string

	// We can't know how big a serialized dentry is until we serialize it.
	// At that point it might be too big. We save it here if that happens,
	// and on the next directory read we start with that.
	oflow []byte

	// Index of next child to return when reading a directory
	nextChildIdx int
}

func newFidEntry(entry tmpfs.Entry, uname string) *FidEntry {
	return &FidEntry{Entry: entry, uname: uname, oflow: nil, nextChildIdx: -1}
}

// Rversion initiates the session
func (fs *fileServer) Rversion(msize protocol.MaxSize, version string) (protocol.MaxSize, string, error) {
	if version != "9P2000" {
		return 0, "", fmt.Errorf("%v not supported; only 9P2000", version)
	}
	return msize, version, nil
}

// Rattach attaches a fid to the root for the given user.  aname and afid are not used.
func (fs *fileServer) Rattach(fid protocol.FID, afid protocol.FID, uname string, aname string) (protocol.QID, error) {
	if afid != protocol.NOFID {
		return protocol.QID{}, fmt.Errorf(ErrorAuthFailed)
	}

	root := fs.archive.Root()
	fs.setFile(fid, root, uname)

	return root.Qid(), nil
}

// Rflush does nothing in tmpfs
func (fs *fileServer) Rflush(o protocol.Tag) error {
	return nil
}

// Rwalk walks the hierarchy from fid, with the walk path determined by paths
func (fs *fileServer) Rwalk(fid protocol.FID, newfid protocol.FID, paths []string) ([]protocol.QID, error) {
	// Lookup the parent fid
	parentEntry, err := fs.getFile(fid)
	if err != nil {
		return nil, fmt.Errorf(ErrorFileNotFound)
	}

	if len(paths) == 0 {
		// Clone fid - point to same entry
		fs.setFile(newfid, parentEntry.Entry, parentEntry.uname)
		return []protocol.QID{}, nil
	}

	walkQids := make([]protocol.QID, len(paths))

	currEntry := parentEntry.Entry
	for i, pathcmp := range paths {
		var ok bool
		var dir *tmpfs.Directory
		if dir, ok = currEntry.(*tmpfs.Directory); ok {
			if pathcmp == ".." {
				currEntry = dir.Parent()
			} else {
				currEntry, err = dir.ChildByName(pathcmp)
				if err != nil {
					// From the RFC: If the first element cannot be walked for any
					// reason, Rerror is returned. Otherwise, the walk will return an
					// Rwalk message containing nwqid qids corresponding, in order, to
					// the files that are visited by the nwqid successful elementwise
					// walks; nwqid is therefore either nwname or the index of the
					// first elementwise walk that failed. The value of nwqid cannot be
					// zero unless nwname is zero. Also, nwqid will always be less than
					// or equal to nwname. Only if it is equal, however, will newfid be
					// affected, in which case newfid will represent the file reached
					// by the final elementwise walk requested in the message.
					//
					// to sum up: if any walks have succeeded, you return the QIDS for
					// one more than the last successful walk
					if i == 0 {
						return nil, err
					}
					// we only get here if i is > 0 and less than nwname,
					// so the i should be safe.
					return walkQids[:i], nil
				}
			}
		}
		walkQids[i] = currEntry.Qid()
	}

	fs.setFile(newfid, currEntry, parentEntry.uname)
	return walkQids, nil
}

// Ropen opens the file associated with fid
func (fs *fileServer) Ropen(fid protocol.FID, mode protocol.Mode) (protocol.QID, protocol.MaxSize, error) {
	if mode&(protocol.OTRUNC|protocol.ORCLOSE|protocol.OAPPEND) != 0 {
		return protocol.QID{}, 0, fmt.Errorf(ErrorReadOnlyFs)
	}
	switch mode & 3 {
	case protocol.OWRITE, protocol.ORDWR:
		return protocol.QID{}, 0, fmt.Errorf(ErrorReadOnlyFs)
	}

	// Lookup the parent fid
	f, err := fs.getFile(fid)
	if err != nil {
		return protocol.QID{}, 0, fmt.Errorf(ErrorFileNotFound)
	}

	// TODO Check executable

	return f.Qid(), fs.ioUnit, nil
}

// Rcreate not supported since it's a read-only filesystem
func (fs *fileServer) Rcreate(fid protocol.FID, name string, perm protocol.Perm, mode protocol.Mode) (protocol.QID, protocol.MaxSize, error) {
	return protocol.QID{}, 0, fmt.Errorf(ErrorReadOnlyFs)
}

// Rclunk drops the fid association in the file system
func (fs *fileServer) Rclunk(fid protocol.FID) error {
	_, err := fs.clunk(fid)
	return err
}

// Rstat returns stat message for the file associated with fid
func (fs *fileServer) Rstat(fid protocol.FID) ([]byte, error) {
	f, err := fs.getFile(fid)
	if err != nil {
		return []byte{}, err
	}

	d := f.P9Dir(f.uname)

	var b bytes.Buffer
	protocol.Marshaldir(&b, *d)
	return b.Bytes(), nil
}

// Rwstat not supported since it's a read-only filesystem
func (fs *fileServer) Rwstat(fid protocol.FID, b []byte) error {
	return fmt.Errorf(ErrorReadOnlyFs)
}

// Rremove not supported since it's a read-only filesystem
func (fs *fileServer) Rremove(fid protocol.FID) error {
	return fmt.Errorf(ErrorReadOnlyFs)
}

// Rread returns up to c bytes from file fid starting at offset o
func (fs *fileServer) Rread(fid protocol.FID, o protocol.Offset, c protocol.Count) ([]byte, error) {
	f, err := fs.getFile(fid)
	if err != nil {
		return nil, err
	}

	if dir, ok := f.Entry.(*tmpfs.Directory); ok {
		if o == 0 {
			f.oflow = nil
		}

		// We make the assumption that they can always fit at least one
		// directory entry into a read. If that assumption does not hold
		// so many things are broken that we can't fix them here.
		// But we'll drop out of the loop below having returned nothing
		// anyway.
		b := bytes.NewBuffer(f.oflow)
		f.oflow = nil
		pos := 0

		for {
			if b.Len() > int(c) {
				f.oflow = b.Bytes()[pos:]
				return b.Bytes()[:pos], nil
			}

			f.nextChildIdx++
			pos += b.Len()

			if f.nextChildIdx >= dir.NumChildren() {
				return b.Bytes(), nil
			}
			d9p := dir.Child(f.nextChildIdx).P9Dir(f.uname)
			protocol.Marshaldir(b, *d9p)

			// Seen on linux clients: sometimes the math is wrong and
			// they end up asking for the last element with not enough data.
			// Linux bug or bug with this server? Not sure yet.
			if b.Len() > int(c) {
				log.Printf("Warning: Server bug? %v, need %d bytes;count is %d: skipping", d9p, b.Len(), c)
				return nil, nil
			}
			// TODO handle more than one entry at a time
			// We're not quite doing the array right.
			// What does work is returning one thing so, for now, do that.
			return b.Bytes(), nil
		}

	} else if file, ok := f.Entry.(*tmpfs.File); ok {
		end := int(o) + int(c)
		maxEnd := len(file.Data())
		if end > maxEnd {
			end = maxEnd
		}
		return file.Data()[o:end], nil
	}
	log.Fatalf("Unrecognised FidEntry")
	return nil, nil
}

// Rwrite not supported since it's a read-only filesystem
func (fs *fileServer) Rwrite(fid protocol.FID, o protocol.Offset, b []byte) (protocol.Count, error) {
	return -1, fmt.Errorf(ErrorReadOnlyFs)
}

func (fs *fileServer) getFile(fid protocol.FID) (*FidEntry, error) {
	fs.Lock()
	defer fs.Unlock()

	f, ok := fs.files[fid]
	if !ok {
		return nil, fmt.Errorf(ErrorFileNotFound)
	}
	return f, nil
}

// Associate newfid with the entry, assuming newfid isn't already in use
func (fs *fileServer) setFile(newfid protocol.FID, entry tmpfs.Entry, uname string) error {
	fs.Lock()
	defer fs.Unlock()
	if _, ok := fs.files[newfid]; ok {
		return fmt.Errorf(ErrorFidInUse)
	}
	fs.files[newfid] = newFidEntry(entry, uname)
	return nil
}

func (fs *fileServer) clunk(fid protocol.FID) (tmpfs.Entry, error) {
	fs.Lock()
	defer fs.Unlock()

	f, ok := fs.files[fid]
	if !ok {
		return nil, fmt.Errorf(ErrorFidNotFound)
	}
	delete(fs.files, fid)

	return f, nil
}

var usage = func() {
	fmt.Fprintf(flag.CommandLine.Output(), "Usage: tmpfs [options] <tarfile>\n")
	flag.PrintDefaults()
	os.Exit(1)
}

func readImage(n string) (*tmpfs.Archive, error) {
	f, err := os.Open(n)
	if err != nil {
		log.Fatal(err)
	}
	defer func() {
		if err := f.Close(); err != nil {
			log.Print(err)
		}
	}()
	r := io.Reader(f)

	if arch, err := lzma.NewReader(f); err == nil {
		r = arch
	} else {
		if _, err := f.Seek(0, 0); err != nil {
			return nil, err
		}
	}

	if len(*ext) == 0 {
		*ext = filepath.Ext(n)
	}
	if *ext == "tgz" || *ext == "gz" {
		return tmpfs.ReadImageTar(r)
	} else if *ext == "cpio" {
		// This is where Unix people say "but mmap!!!"
		b, err := ioutil.ReadAll(r)
		if err != nil {
			return nil, err
		}
		return tmpfs.ReadImageCpio(bytes.NewReader(b))
	}
	return nil, fmt.Errorf("No reader matches suffix %s", *ext)
}

func main() {
	flag.Usage = usage
	flag.Parse()

	a := flag.Args()
	// TODO: serve more than one archive.
	if len(a) != 1 {
		flag.Usage()
	}
	if *debug {
		protocol.Debug = log.Printf
		tmpfs.Debug = log.Printf
	}

	arch, err := readImage(a[0])
	if err != nil {
		log.Fatal(err)
	}
	if *debug {
		arch.DumpArchive()
	}
	fd, err := sys.PostPipe(*srv)
	if err != nil {
		log.Fatal(err)
	}

	fs := &fileServer{archive: arch, files: make(map[protocol.FID]*FidEntry), ioUnit: 1 * 1024 * 1024}

	// TODO: get the tracing back in.
	// The ninep package was from a long time ago and it's
	// awkward at best.
	protocol.ServeFromRWC(os.NewFile(uintptr(fd), *srv), fs, *srv)
}
<commit_msg>cmd/tmpfs: add listener for non-plan9 systems<commit_after>// +build go1.14

package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"path/filepath"
	"sync"

	"github.com/ulikunitz/xz/lzma"
	"harvey-os.org/ninep/protocol"
	"harvey-os.org/ninep/tmpfs"
	"harvey-os.org/sys"
)

var (
	srv     = flag.String("s", "tmpfs", "srv name")
	debug   = flag.Bool("d", false, "Print debug messages")
	ext     = flag.String("ext", "cpio", "file type")
	network = flag.String("net", "tcp", "network used to listen")
	addr    = flag.String("addr", "", "network address to listen on")
)

// Constant error messages to match those found in the linux 9p source.
// These strings are used by linux to map to particular error codes.
// (See net/9p/error.c in the Linux code)
const (
	ErrorAuthFailed   = "authentication failed"
	ErrorReadOnlyFs   = "Read-only file system"
	ErrorFileNotFound = "file not found"
	ErrorFidInUse     = "fid already in use"
	ErrorFidNotFound  = "fid unknown or out of range"
)

type fileServer struct {
	sync.Mutex

	Versioned bool

	ioUnit  protocol.MaxSize
	archive *tmpfs.Archive
	files   map[protocol.FID]*FidEntry
}

// FidEntry wraps an Entry with the instance data required for a fid reference
type FidEntry struct {
	tmpfs.Entry

	// Username to be used for all entries in this hierarchy
	uname string

	// We can't know how big a serialized dentry is until we serialize it.
	// At that point it might be too big. We save it here if that happens,
	// and on the next directory read we start with that.
	oflow []byte

	// Index of next child to return when reading a directory
	nextChildIdx int
}

func newFidEntry(entry tmpfs.Entry, uname string) *FidEntry {
	return &FidEntry{Entry: entry, uname: uname, oflow: nil, nextChildIdx: -1}
}

// Rversion initiates the session
func (fs *fileServer) Rversion(msize protocol.MaxSize, version string) (protocol.MaxSize, string, error) {
	if version != "9P2000" {
		return 0, "", fmt.Errorf("%v not supported; only 9P2000", version)
	}
	return msize, version, nil
}

// Rattach attaches a fid to the root for the given user.  aname and afid are not used.
func (fs *fileServer) Rattach(fid protocol.FID, afid protocol.FID, uname string, aname string) (protocol.QID, error) {
	if afid != protocol.NOFID {
		return protocol.QID{}, fmt.Errorf(ErrorAuthFailed)
	}

	root := fs.archive.Root()
	fs.setFile(fid, root, uname)

	return root.Qid(), nil
}

// Rflush does nothing in tmpfs
func (fs *fileServer) Rflush(o protocol.Tag) error {
	return nil
}

// Rwalk walks the hierarchy from fid, with the walk path determined by paths
func (fs *fileServer) Rwalk(fid protocol.FID, newfid protocol.FID, paths []string) ([]protocol.QID, error) {
	// Lookup the parent fid
	parentEntry, err := fs.getFile(fid)
	if err != nil {
		return nil, fmt.Errorf(ErrorFileNotFound)
	}

	if len(paths) == 0 {
		// Clone fid - point to same entry
		fs.setFile(newfid, parentEntry.Entry, parentEntry.uname)
		return []protocol.QID{}, nil
	}

	walkQids := make([]protocol.QID, len(paths))

	currEntry := parentEntry.Entry
	for i, pathcmp := range paths {
		var ok bool
		var dir *tmpfs.Directory
		if dir, ok = currEntry.(*tmpfs.Directory); ok {
			if pathcmp == ".." {
				currEntry = dir.Parent()
			} else {
				currEntry, err = dir.ChildByName(pathcmp)
				if err != nil {
					// From the RFC: If the first element cannot be walked for any
					// reason, Rerror is returned. Otherwise, the walk will return an
					// Rwalk message containing nwqid qids corresponding, in order, to
					// the files that are visited by the nwqid successful elementwise
					// walks; nwqid is therefore either nwname or the index of the
					// first elementwise walk that failed. The value of nwqid cannot be
					// zero unless nwname is zero. Also, nwqid will always be less than
					// or equal to nwname. Only if it is equal, however, will newfid be
					// affected, in which case newfid will represent the file reached
					// by the final elementwise walk requested in the message.
					//
					// to sum up: if any walks have succeeded, you return the QIDS for
					// one more than the last successful walk
					if i == 0 {
						return nil, err
					}
					// we only get here if i is > 0 and less than nwname,
					// so the i should be safe.
					return walkQids[:i], nil
				}
			}
		}
		walkQids[i] = currEntry.Qid()
	}

	fs.setFile(newfid, currEntry, parentEntry.uname)
	return walkQids, nil
}

// Ropen opens the file associated with fid
func (fs *fileServer) Ropen(fid protocol.FID, mode protocol.Mode) (protocol.QID, protocol.MaxSize, error) {
	if mode&(protocol.OTRUNC|protocol.ORCLOSE|protocol.OAPPEND) != 0 {
		return protocol.QID{}, 0, fmt.Errorf(ErrorReadOnlyFs)
	}
	switch mode & 3 {
	case protocol.OWRITE, protocol.ORDWR:
		return protocol.QID{}, 0, fmt.Errorf(ErrorReadOnlyFs)
	}

	// Lookup the parent fid
	f, err := fs.getFile(fid)
	if err != nil {
		return protocol.QID{}, 0, fmt.Errorf(ErrorFileNotFound)
	}

	// TODO Check executable

	return f.Qid(), fs.ioUnit, nil
}

// Rcreate not supported since it's a read-only filesystem
func (fs *fileServer) Rcreate(fid protocol.FID, name string, perm protocol.Perm, mode protocol.Mode) (protocol.QID, protocol.MaxSize, error) {
	return protocol.QID{}, 0, fmt.Errorf(ErrorReadOnlyFs)
}

// Rclunk drops the fid association in the file system
func (fs *fileServer) Rclunk(fid protocol.FID) error {
	_, err := fs.clunk(fid)
	return err
}

// Rstat returns stat message for the file associated with fid
func (fs *fileServer) Rstat(fid protocol.FID) ([]byte, error) {
	f, err := fs.getFile(fid)
	if err != nil {
		return []byte{}, err
	}

	d := f.P9Dir(f.uname)

	var b bytes.Buffer
	protocol.Marshaldir(&b, *d)
	return b.Bytes(), nil
}

// Rwstat not supported since it's a read-only filesystem
func (fs *fileServer) Rwstat(fid protocol.FID, b []byte) error {
	return fmt.Errorf(ErrorReadOnlyFs)
}

// Rremove not supported since it's a read-only filesystem
func (fs *fileServer) Rremove(fid protocol.FID) error {
	return fmt.Errorf(ErrorReadOnlyFs)
}

// Rread returns up to c bytes from file fid starting at offset o
func (fs *fileServer) Rread(fid protocol.FID, o protocol.Offset, c protocol.Count) ([]byte, error) {
	f, err := fs.getFile(fid)
	if err != nil {
		return nil, err
	}

	if dir, ok := f.Entry.(*tmpfs.Directory); ok {
		if o == 0 {
			f.oflow = nil
		}

		// We make the assumption that they can always fit at least one
		// directory entry into a read. If that assumption does not hold
		// so many things are broken that we can't fix them here.
		// But we'll drop out of the loop below having returned nothing
		// anyway.
		b := bytes.NewBuffer(f.oflow)
		f.oflow = nil
		pos := 0

		for {
			if b.Len() > int(c) {
				f.oflow = b.Bytes()[pos:]
				return b.Bytes()[:pos], nil
			}

			f.nextChildIdx++
			pos += b.Len()

			if f.nextChildIdx >= dir.NumChildren() {
				return b.Bytes(), nil
			}
			d9p := dir.Child(f.nextChildIdx).P9Dir(f.uname)
			protocol.Marshaldir(b, *d9p)

			// Seen on linux clients: sometimes the math is wrong and
			// they end up asking for the last element with not enough data.
			// Linux bug or bug with this server? Not sure yet.
			if b.Len() > int(c) {
				log.Printf("Warning: Server bug? %v, need %d bytes;count is %d: skipping", d9p, b.Len(), c)
				return nil, nil
			}
			// TODO handle more than one entry at a time
			// We're not quite doing the array right.
			// What does work is returning one thing so, for now, do that.
			return b.Bytes(), nil
		}

	} else if file, ok := f.Entry.(*tmpfs.File); ok {
		end := int(o) + int(c)
		maxEnd := len(file.Data())
		if end > maxEnd {
			end = maxEnd
		}
		return file.Data()[o:end], nil
	}
	log.Fatalf("Unrecognised FidEntry")
	return nil, nil
}

// Rwrite not supported since it's a read-only filesystem
func (fs *fileServer) Rwrite(fid protocol.FID, o protocol.Offset, b []byte) (protocol.Count, error) {
	return -1, fmt.Errorf(ErrorReadOnlyFs)
}

func (fs *fileServer) getFile(fid protocol.FID) (*FidEntry, error) {
	fs.Lock()
	defer fs.Unlock()

	f, ok := fs.files[fid]
	if !ok {
		return nil, fmt.Errorf(ErrorFileNotFound)
	}
	return f, nil
}

// Associate newfid with the entry, assuming newfid isn't already in use
func (fs *fileServer) setFile(newfid protocol.FID, entry tmpfs.Entry, uname string) error {
	fs.Lock()
	defer fs.Unlock()
	if _, ok := fs.files[newfid]; ok {
		return fmt.Errorf(ErrorFidInUse)
	}
	fs.files[newfid] = newFidEntry(entry, uname)
	return nil
}

func (fs *fileServer) clunk(fid protocol.FID) (tmpfs.Entry, error) {
	fs.Lock()
	defer fs.Unlock()

	f, ok := fs.files[fid]
	if !ok {
		return nil, fmt.Errorf(ErrorFidNotFound)
	}
	delete(fs.files, fid)

	return f, nil
}

var usage = func() {
	fmt.Fprintf(flag.CommandLine.Output(), "Usage: tmpfs [options] <tarfile>\n")
	flag.PrintDefaults()
	os.Exit(1)
}

func readImage(n string) (*tmpfs.Archive, error) {
	f, err := os.Open(n)
	if err != nil {
		log.Fatal(err)
	}
	defer func() {
		if err := f.Close(); err != nil {
			log.Print(err)
		}
	}()
	r := io.Reader(f)

	if arch, err := lzma.NewReader(f); err == nil {
		r = arch
	} else {
		if _, err := f.Seek(0, 0); err != nil {
			return nil, err
		}
	}

	if len(*ext) == 0 {
		*ext = filepath.Ext(n)
	}
	if *ext == "tgz" || *ext == "gz" {
		return tmpfs.ReadImageTar(r)
	} else if *ext == "cpio" {
		// This is where Unix people say "but mmap!!!"
		b, err := ioutil.ReadAll(r)
		if err != nil {
			return nil, err
		}
		return tmpfs.ReadImageCpio(bytes.NewReader(b))
	}
	return nil, fmt.Errorf("No reader matches suffix %s", *ext)
}

func main() {
	flag.Usage = usage
	flag.Parse()

	a := flag.Args()
	// TODO: serve more than one archive.
	if len(a) != 1 {
		flag.Usage()
	}
	if *debug {
		protocol.Debug = log.Printf
		tmpfs.Debug = log.Printf
	}

	arch, err := readImage(a[0])
	if err != nil {
		log.Fatal(err)
	}
	if *debug {
		arch.DumpArchive()
	}

	nsCreator := func() protocol.NineServer {
		return &fileServer{archive: arch, files: make(map[protocol.FID]*FidEntry), ioUnit: 1 * 1024 * 1024}
	}

	// TODO: get the tracing back in.
	// The ninep package was from a long time ago and it's
	// awkward at best.
	if *addr != "" {
		ln, err := net.Listen(*network, *addr)
		if err != nil {
			log.Fatal(err)
		}
		netListener, err := protocol.NewNetListener(nsCreator)
		if err != nil {
			log.Fatal(err)
		}
		if err := netListener.Serve(ln); err != nil {
			log.Fatal(err)
		}
	} else {
		fd, err := sys.PostPipe(*srv)
		if err != nil {
			log.Fatal(err)
		}
		protocol.ServeFromRWC(os.NewFile(uintptr(fd), *srv), nsCreator(), *srv)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"os"

	cf_debug_server "github.com/cloudfoundry-incubator/cf-debug-server"
	cf_lager "github.com/cloudfoundry-incubator/cf-lager"
	. "github.com/cloudfoundry-incubator/volman/delegate"
	"github.com/cloudfoundry-incubator/volman/handlers"
	"github.com/pivotal-golang/lager"
	. "github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"
)

var atAddress = flag.String(
	"listenAddr",
	"0.0.0.0:8750",
	"host:port to serve volume management functions",
)

func main() {
	parseCommandLine()
	withLogger, logTap := logger()
	servers := grouper.Members{}
	servers = createAndAppendServer(*atAddress, servers)
	servers = createAndAppendDebugServer(cf_debug_server.DebugAddress(flag.CommandLine), logTap, servers)
	untilTerminated(Invoke(processRunnerFor(servers)), withLogger)
}

func exitOnFailure(err error) {
	if err != nil {
		os.Exit(1)
	}
}

func untilTerminated(process Process, logger lager.Logger) {
	logger.Info("started")
	err := <-process.Wait()
	exitOnFailure(err)
	logger.Info("exited")
}

func processRunnerFor(servers grouper.Members) Runner {
	return sigmon.New(grouper.NewOrdered(os.Interrupt, servers))
}

func createAndAppendDebugServer(atDebugAddress string, logTap *lager.ReconfigurableSink, servers grouper.Members) grouper.Members {
	if atDebugAddress == "" {
		return servers
	}

	return append(grouper.Members{
		{"debug-server", cf_debug_server.Runner(atDebugAddress, logTap)},
	}, servers...)
}

func createAndAppendServer(atAddress string, servers grouper.Members) grouper.Members {
	handler, err := handlers.Generate()
	exitOnFailure(err)
	server := http_server.New(atAddress, handler)
	return append(grouper.Members{
		{"volman-server", server},
	}, servers...)
}

func logger() (lager.Logger, *lager.ReconfigurableSink) {

	logger, reconfigurableSink := cf_lager.New("volman")
	return logger, reconfigurableSink
}

func parseCommandLine() {
	cf_lager.AddFlags(flag.CommandLine)
	cf_debug_server.AddFlags(flag.CommandLine)
	flag.Parse()
}
<commit_msg>[#112021157] - Fix imports<commit_after>package main

import (
	"flag"
	"os"

	cf_debug_server "github.com/cloudfoundry-incubator/cf-debug-server"
	cf_lager "github.com/cloudfoundry-incubator/cf-lager"
	handlers "github.com/cloudfoundry-incubator/volman/handlers"
	"github.com/pivotal-golang/lager"
	. "github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"
)

var atAddress = flag.String(
	"listenAddr",
	"0.0.0.0:8750",
	"host:port to serve volume management functions",
)

func main() {
	parseCommandLine()
	withLogger, logTap := logger()
	servers := grouper.Members{}
	servers = createAndAppendServer(*atAddress, servers)
	servers = createAndAppendDebugServer(cf_debug_server.DebugAddress(flag.CommandLine), logTap, servers)
	untilTerminated(Invoke(processRunnerFor(servers)), withLogger)
}

func exitOnFailure(err error) {
	if err != nil {
		os.Exit(1)
	}
}

func untilTerminated(process Process, logger lager.Logger) {
	logger.Info("started")
	err := <-process.Wait()
	exitOnFailure(err)
	logger.Info("exited")
}

func processRunnerFor(servers grouper.Members) Runner {
	return sigmon.New(grouper.NewOrdered(os.Interrupt, servers))
}

func createAndAppendDebugServer(atDebugAddress string, logTap *lager.ReconfigurableSink, servers grouper.Members) grouper.Members {
	if atDebugAddress == "" {
		return servers
	}

	return append(grouper.Members{
		{"debug-server", cf_debug_server.Runner(atDebugAddress, logTap)},
	}, servers...)
}

func createAndAppendServer(atAddress string, servers grouper.Members) grouper.Members {
	handler, err := handlers.Generate()
	exitOnFailure(err)
	server := http_server.New(atAddress, handler)
	return append(grouper.Members{
		{"volman-server", server},
	}, servers...)
}

func logger() (lager.Logger, *lager.ReconfigurableSink) {

	logger, reconfigurableSink := cf_lager.New("volman")
	return logger, reconfigurableSink
}

func parseCommandLine() {
	cf_lager.AddFlags(flag.CommandLine)
	cf_debug_server.AddFlags(flag.CommandLine)
	flag.Parse()
}
<|endoftext|>


<commit_before>package gps

import (
	"fmt"
	"strconv"
	"strings"
)

const (
	successChar   = "✓"
	successCharSp = successChar + " "
	failChar      = "✗"
	failCharSp    = failChar + " "
	backChar      = "←"
	innerIndent   = "  "
)

func (s *solver) traceCheckPkgs(bmi bimodalIdentifier) {
	if s.tl == nil {
		return
	}

	prefix := getprei(len(s.vqs) + 1)
	s.tl.Printf("%s\n", tracePrefix(fmt.Sprintf("? revisit %s to add %v pkgs", bmi.id.errString(), len(bmi.pl)), prefix, prefix))
}

func (s *solver) traceCheckQueue(q *versionQueue, bmi bimodalIdentifier, cont bool, offset int) {
	if s.tl == nil {
		return
	}

	prefix := getprei(len(s.vqs) + offset)
	vlen := strconv.Itoa(len(q.pi))
	if !q.allLoaded {
		vlen = "at least " + vlen
	}

	// TODO(sdboyer) how...to list the packages in the limited space we have?
	var verb string
	indent := ""
	if cont {
		// Continue is an "inner" message.. indenting
		verb = "continue"
		vlen = vlen + " more"
		indent = innerIndent
	} else {
		verb = "attempt"
	}

	s.tl.Printf("%s\n", tracePrefix(fmt.Sprintf("%s? %s %s with %v pkgs; %s versions to try",indent, verb, bmi.id.errString(), len(bmi.pl), vlen), prefix, prefix))
}

// traceStartBacktrack is called with the bmi that first failed, thus initiating
// backtracking
func (s *solver) traceStartBacktrack(bmi bimodalIdentifier, err error, pkgonly bool) {
	if s.tl == nil {
		return
	}

	var msg string
	if pkgonly {
		msg = fmt.Sprintf("%s%s could not add %v pkgs to %s; begin backtrack",innerIndent, backChar, len(bmi.pl), bmi.id.errString())
	} else {
		msg = fmt.Sprintf("%s%s no more versions of %s to try; begin backtrack",innerIndent, backChar, bmi.id.errString())
	}

	prefix := getprei(len(s.sel.projects))
	s.tl.Printf("%s\n", tracePrefix(msg, prefix, prefix))
}

// traceBacktrack is called when a package or project is poppped off during
// backtracking
func (s *solver) traceBacktrack(bmi bimodalIdentifier, pkgonly bool) {
	if s.tl == nil {
		return
	}

	var msg string
	if pkgonly {
		msg = fmt.Sprintf("%s backtrack: popped %v pkgs from %s", backChar, len(bmi.pl), bmi.id.errString())
	} else {
		msg = fmt.Sprintf("%s backtrack: no more versions of %s to try", backChar, bmi.id.errString())
	}

	prefix := getprei(len(s.sel.projects))
	s.tl.Printf("%s\n", tracePrefix(msg, prefix, prefix))
}

// Called just once after solving has finished, whether success or not
func (s *solver) traceFinish(sol solution, err error) {
	if s.tl == nil {
		return
	}

	if err == nil {
		var pkgcount int
		for _, lp := range sol.Projects() {
			pkgcount += len(lp.pkgs)
		}
		s.tl.Printf("%s%s found solution with %v packages from %v projects",innerIndent, successChar, pkgcount, len(sol.Projects()))
	} else {
		s.tl.Printf("%s%s solving failed",innerIndent, failChar)
	}
}

// traceSelectRoot is called just once, when the root project is selected
func (s *solver) traceSelectRoot(ptree PackageTree, cdeps []completeDep) {
	if s.tl == nil {
		return
	}

	// This duplicates work a bit, but we're in trace mode and it's only once,
	// so who cares
	rm := ptree.ExternalReach(true, true, s.rd.ig)

	s.tl.Printf("Root project is %q", s.rd.rpt.ImportRoot)

	var expkgs int
	for _, cdep := range cdeps {
		expkgs += len(cdep.pl)
	}

	// TODO(sdboyer) include info on ignored pkgs/imports, etc.
	s.tl.Printf(" %v transitively valid internal packages", len(rm))
	s.tl.Printf(" %v external packages imported from %v projects", expkgs, len(cdeps))
	s.tl.Printf("(0)   " + successCharSp + "select (root)")
}

// traceSelect is called when an atom is successfully selected
func (s *solver) traceSelect(awp atomWithPackages, pkgonly bool) {
	if s.tl == nil {
		return
	}

	var msg string
	if pkgonly {
		msg = fmt.Sprintf("%s%s include %v more pkgs from %s",innerIndent, successChar, len(awp.pl), a2vs(awp.a))
	} else {
		msg = fmt.Sprintf("%s select %s w/%v pkgs", successChar, a2vs(awp.a), len(awp.pl))
	}

	prefix := getprei(len(s.sel.projects) - 1)
	s.tl.Printf("%s\n", tracePrefix(msg, prefix, prefix))
}

func (s *solver) traceInfo(args ...interface{}) {
	if s.tl == nil {
		return
	}

	if len(args) == 0 {
		panic("must pass at least one param to traceInfo")
	}

	preflen := len(s.sel.projects)
	var msg string
	switch data := args[0].(type) {
	case string:
		msg = tracePrefix(innerIndent + fmt.Sprintf(data, args[1:]...), "  ", "  ")
	case traceError:
		preflen++
		// We got a special traceError, use its custom method
		msg = tracePrefix(innerIndent + data.traceString(), "  ", failCharSp)
	case error:
		// Regular error; still use the x leader but default Error() string
		msg = tracePrefix(innerIndent + data.Error(), "  ", failCharSp)
	default:
		// panic here because this can *only* mean a stupid internal bug
		panic(fmt.Sprintf("canary - unknown type passed as first param to traceInfo %T", data))
	}

	prefix := getprei(preflen)
	s.tl.Printf("%s\n", tracePrefix(msg, prefix, prefix))
}

func getprei(i int) string {
	var s string
	if i < 10 {
		s = fmt.Sprintf("(%d)	", i)
	} else if i < 100 {
		s = fmt.Sprintf("(%d)  ", i)
	} else {
		s = fmt.Sprintf("(%d) ", i)
	}
	return s
}

func tracePrefix(msg, sep, fsep string) string {
	parts := strings.Split(strings.TrimSuffix(msg, "\n"), "\n")
	for k, str := range parts {
		if k == 0 {
			parts[k] = fsep + str
		} else {
			parts[k] = sep + str
		}
	}

	return strings.Join(parts, "\n")
}
<commit_msg>gofmt<commit_after>package gps

import (
	"fmt"
	"strconv"
	"strings"
)

const (
	successChar   = "✓"
	successCharSp = successChar + " "
	failChar      = "✗"
	failCharSp    = failChar + " "
	backChar      = "←"
	innerIndent   = "  "
)

func (s *solver) traceCheckPkgs(bmi bimodalIdentifier) {
	if s.tl == nil {
		return
	}

	prefix := getprei(len(s.vqs) + 1)
	s.tl.Printf("%s\n", tracePrefix(fmt.Sprintf("? revisit %s to add %v pkgs", bmi.id.errString(), len(bmi.pl)), prefix, prefix))
}

func (s *solver) traceCheckQueue(q *versionQueue, bmi bimodalIdentifier, cont bool, offset int) {
	if s.tl == nil {
		return
	}

	prefix := getprei(len(s.vqs) + offset)
	vlen := strconv.Itoa(len(q.pi))
	if !q.allLoaded {
		vlen = "at least " + vlen
	}

	// TODO(sdboyer) how...to list the packages in the limited space we have?
	var verb string
	indent := ""
	if cont {
		// Continue is an "inner" message.. indenting
		verb = "continue"
		vlen = vlen + " more"
		indent = innerIndent
	} else {
		verb = "attempt"
	}

	s.tl.Printf("%s\n", tracePrefix(fmt.Sprintf("%s? %s %s with %v pkgs; %s versions to try", indent, verb, bmi.id.errString(), len(bmi.pl), vlen), prefix, prefix))
}

// traceStartBacktrack is called with the bmi that first failed, thus initiating
// backtracking
func (s *solver) traceStartBacktrack(bmi bimodalIdentifier, err error, pkgonly bool) {
	if s.tl == nil {
		return
	}

	var msg string
	if pkgonly {
		msg = fmt.Sprintf("%s%s could not add %v pkgs to %s; begin backtrack", innerIndent, backChar, len(bmi.pl), bmi.id.errString())
	} else {
		msg = fmt.Sprintf("%s%s no more versions of %s to try; begin backtrack", innerIndent, backChar, bmi.id.errString())
	}

	prefix := getprei(len(s.sel.projects))
	s.tl.Printf("%s\n", tracePrefix(msg, prefix, prefix))
}

// traceBacktrack is called when a package or project is poppped off during
// backtracking
func (s *solver) traceBacktrack(bmi bimodalIdentifier, pkgonly bool) {
	if s.tl == nil {
		return
	}

	var msg string
	if pkgonly {
		msg = fmt.Sprintf("%s backtrack: popped %v pkgs from %s", backChar, len(bmi.pl), bmi.id.errString())
	} else {
		msg = fmt.Sprintf("%s backtrack: no more versions of %s to try", backChar, bmi.id.errString())
	}

	prefix := getprei(len(s.sel.projects))
	s.tl.Printf("%s\n", tracePrefix(msg, prefix, prefix))
}

// Called just once after solving has finished, whether success or not
func (s *solver) traceFinish(sol solution, err error) {
	if s.tl == nil {
		return
	}

	if err == nil {
		var pkgcount int
		for _, lp := range sol.Projects() {
			pkgcount += len(lp.pkgs)
		}
		s.tl.Printf("%s%s found solution with %v packages from %v projects", innerIndent, successChar, pkgcount, len(sol.Projects()))
	} else {
		s.tl.Printf("%s%s solving failed", innerIndent, failChar)
	}
}

// traceSelectRoot is called just once, when the root project is selected
func (s *solver) traceSelectRoot(ptree PackageTree, cdeps []completeDep) {
	if s.tl == nil {
		return
	}

	// This duplicates work a bit, but we're in trace mode and it's only once,
	// so who cares
	rm := ptree.ExternalReach(true, true, s.rd.ig)

	s.tl.Printf("Root project is %q", s.rd.rpt.ImportRoot)

	var expkgs int
	for _, cdep := range cdeps {
		expkgs += len(cdep.pl)
	}

	// TODO(sdboyer) include info on ignored pkgs/imports, etc.
	s.tl.Printf(" %v transitively valid internal packages", len(rm))
	s.tl.Printf(" %v external packages imported from %v projects", expkgs, len(cdeps))
	s.tl.Printf("(0)   " + successCharSp + "select (root)")
}

// traceSelect is called when an atom is successfully selected
func (s *solver) traceSelect(awp atomWithPackages, pkgonly bool) {
	if s.tl == nil {
		return
	}

	var msg string
	if pkgonly {
		msg = fmt.Sprintf("%s%s include %v more pkgs from %s", innerIndent, successChar, len(awp.pl), a2vs(awp.a))
	} else {
		msg = fmt.Sprintf("%s select %s w/%v pkgs", successChar, a2vs(awp.a), len(awp.pl))
	}

	prefix := getprei(len(s.sel.projects) - 1)
	s.tl.Printf("%s\n", tracePrefix(msg, prefix, prefix))
}

func (s *solver) traceInfo(args ...interface{}) {
	if s.tl == nil {
		return
	}

	if len(args) == 0 {
		panic("must pass at least one param to traceInfo")
	}

	preflen := len(s.sel.projects)
	var msg string
	switch data := args[0].(type) {
	case string:
		msg = tracePrefix(innerIndent+fmt.Sprintf(data, args[1:]...), "  ", "  ")
	case traceError:
		preflen++
		// We got a special traceError, use its custom method
		msg = tracePrefix(innerIndent+data.traceString(), "  ", failCharSp)
	case error:
		// Regular error; still use the x leader but default Error() string
		msg = tracePrefix(innerIndent+data.Error(), "  ", failCharSp)
	default:
		// panic here because this can *only* mean a stupid internal bug
		panic(fmt.Sprintf("canary - unknown type passed as first param to traceInfo %T", data))
	}

	prefix := getprei(preflen)
	s.tl.Printf("%s\n", tracePrefix(msg, prefix, prefix))
}

func getprei(i int) string {
	var s string
	if i < 10 {
		s = fmt.Sprintf("(%d)	", i)
	} else if i < 100 {
		s = fmt.Sprintf("(%d)  ", i)
	} else {
		s = fmt.Sprintf("(%d) ", i)
	}
	return s
}

func tracePrefix(msg, sep, fsep string) string {
	parts := strings.Split(strings.TrimSuffix(msg, "\n"), "\n")
	for k, str := range parts {
		if k == 0 {
			parts[k] = fsep + str
		} else {
			parts[k] = sep + str
		}
	}

	return strings.Join(parts, "\n")
}
<|endoftext|>


<commit_before>package trace

import "time"

type Span struct {
	name string

	begin time.Time
	end   time.Time
}

func (s *Span) Finish() {
	panic("not implemented")
}

type Trace struct {
	name  string
	spans []Span
}
<commit_msg>remove trace<commit_after>package trace

import (
	"log"
	"time"
)

var Logger log.Logger

type Span struct {
	name  string
	spans []*Span

	begin time.Time
	end   time.Time
}

func NewSpan(name string) *Span {
	return &Span{
		name:  name,
		spans: make([]*Span, 0),
		begin: time.Now(),
	}
}

func (s *Span) NewSpan(name string) *Span {
	child := NewSpan(name)
	s.spans = append(s.spans, child)
	return child
}

func (s *Span) Finish() {
	s.end = time.Now()
}
<|endoftext|>