<commit_before>package client

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"math"
	"net/http"
	"net/url"
	"sync"
	"time"

	"code.cloudfoundry.org/clock"
	"code.cloudfoundry.org/lager"
	"github.com/tedsuo/rata"

	"github.com/concourse/baggageclaim"
	"github.com/concourse/baggageclaim/api"
	"github.com/concourse/baggageclaim/volume"
	"github.com/concourse/retryhttp"
)

type Client interface {
	baggageclaim.Client
}

type client struct {
	requestGenerator *rata.RequestGenerator

	sleeper            clock.Clock
	retryPolicy        retryhttp.RetryPolicy
	nestedRoundTripper http.RoundTripper
}

func New(apiURL string) Client {
	return &client{
		requestGenerator: rata.NewRequestGenerator(apiURL, baggageclaim.Routes),

		sleeper: clock.NewClock(),

		retryPolicy: retryhttp.ExponentialRetryPolicy{
			Timeout: 60 * time.Minute,
		},

		nestedRoundTripper: &http.Transport{
			DisableKeepAlives: true,
		},
	}
}

func (c *client) httpClient(logger lager.Logger) *http.Client {
	return &http.Client{
		Transport: &retryhttp.RetryRoundTripper{
			Logger:       logger.Session("retry-round-tripper"),
			Sleeper:      c.sleeper,
			RetryPolicy:  c.retryPolicy,
			RoundTripper: c.nestedRoundTripper,
		},
	}
}

func (c *client) CreateVolume(logger lager.Logger, volumeSpec baggageclaim.VolumeSpec) (baggageclaim.Volume, error) {
	strategy := volumeSpec.Strategy
	if strategy == nil {
		strategy = baggageclaim.EmptyStrategy{}
	}

	buffer := &bytes.Buffer{}
	json.NewEncoder(buffer).Encode(baggageclaim.VolumeRequest{
		Strategy:     strategy.Encode(),
		TTLInSeconds: uint(math.Ceil(volumeSpec.TTL.Seconds())),
		Properties:   volumeSpec.Properties,
		Privileged:   volumeSpec.Privileged,
	})

	request, _ := c.requestGenerator.CreateRequest(baggageclaim.CreateVolume, nil, buffer)
	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return nil, err
	}

	defer response.Body.Close()

	if response.StatusCode != 201 {
		return nil, getError(response)
	}

	if header := response.Header.Get("Content-Type"); header != "application/json" {
		return nil, fmt.Errorf("unexpected content-type of: %s", header)
	}

	var volumeResponse baggageclaim.VolumeResponse
	err = json.NewDecoder(response.Body).Decode(&volumeResponse)
	if err != nil {
		return nil, err
	}

	v, initialHeartbeatSuccess := c.newVolume(logger, volumeResponse)
	if !initialHeartbeatSuccess {
		return nil, volume.ErrVolumeDoesNotExist
	}
	return v, nil
}

func (c *client) ListVolumes(logger lager.Logger, properties baggageclaim.VolumeProperties) (baggageclaim.Volumes, error) {
	if properties == nil {
		properties = baggageclaim.VolumeProperties{}
	}

	request, err := c.requestGenerator.CreateRequest(baggageclaim.ListVolumes, nil, nil)
	if err != nil {
		return nil, err
	}

	queryString := request.URL.Query()
	for key, val := range properties {
		queryString.Add(key, val)
	}

	request.URL.RawQuery = queryString.Encode()

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return nil, err
	}

	defer response.Body.Close()

	if response.StatusCode != 200 {
		return nil, getError(response)
	}

	if header := response.Header.Get("Content-Type"); header != "application/json" {
		return nil, fmt.Errorf("unexpected content-type of: %s", header)
	}

	var volumesResponse []baggageclaim.VolumeResponse
	err = json.NewDecoder(response.Body).Decode(&volumesResponse)
	if err != nil {
		return nil, err
	}

	var volumes baggageclaim.Volumes
	for _, vr := range volumesResponse {
		v, initialHeartbeatSuccess := c.newVolume(logger, vr)
		if initialHeartbeatSuccess {
			volumes = append(volumes, v)
		}
	}

	return volumes, nil
}

func (c *client) LookupVolume(logger lager.Logger, handle string) (baggageclaim.Volume, bool, error) {

	volumeResponse, found, err := c.getVolumeResponse(logger, handle)
	if err != nil {
		return nil, false, err
	}
	if !found {
		return nil, found, nil
	}

	v, initialHeartbeatSuccess := c.newVolume(logger, volumeResponse)
	if !initialHeartbeatSuccess {
		return nil, false, nil
	}
	return v, true, nil
}

func (c *client) newVolume(logger lager.Logger, apiVolume baggageclaim.VolumeResponse) (baggageclaim.Volume, bool) {
	volume := &clientVolume{
		logger: logger,

		handle: apiVolume.Handle,
		path:   apiVolume.Path,

		bcClient:     c,
		heartbeating: new(sync.WaitGroup),
		release:      make(chan *time.Duration, 1),
	}

	initialHeartbeatSuccess := volume.startHeartbeating(logger, time.Duration(apiVolume.TTLInSeconds)*time.Second)

	return volume, initialHeartbeatSuccess
}

func (c *client) streamIn(logger lager.Logger, destHandle string, path string, tarContent io.Reader) error {
	request, err := c.requestGenerator.CreateRequest(baggageclaim.StreamIn, rata.Params{
		"handle": destHandle,
	}, tarContent)

	request.URL.RawQuery = url.Values{"path": []string{path}}.Encode()
	if err != nil {
		return err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return err
	}

	defer response.Body.Close()
	if response.StatusCode == http.StatusNoContent {
		return nil
	}
	return getError(response)
}

func (c *client) streamOut(logger lager.Logger, srcHandle string, path string) (io.ReadCloser, error) {
	request, err := c.requestGenerator.CreateRequest(baggageclaim.StreamOut, rata.Params{
		"handle": srcHandle,
	}, nil)

	request.URL.RawQuery = url.Values{"path": []string{path}}.Encode()
	if err != nil {
		return nil, err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return nil, err
	}

	if response.StatusCode != http.StatusOK {
		return nil, getError(response)
	}

	return response.Body, nil
}

func getError(response *http.Response) error {
	var errorResponse *api.ErrorResponse
	err := json.NewDecoder(response.Body).Decode(&errorResponse)
	if err != nil {
		return err
	}

	if errorResponse.Message == api.ErrStreamOutNotFound.Error() {
		return baggageclaim.ErrFileNotFound
	}

	if response.StatusCode == 404 {
		return baggageclaim.ErrVolumeNotFound
	}

	return errors.New(errorResponse.Message)
}

func (c *client) getVolumeResponse(logger lager.Logger, handle string) (baggageclaim.VolumeResponse, bool, error) {
	request, err := c.requestGenerator.CreateRequest(baggageclaim.GetVolume, rata.Params{
		"handle": handle,
	}, nil)
	if err != nil {
		return baggageclaim.VolumeResponse{}, false, err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return baggageclaim.VolumeResponse{}, false, err
	}

	defer response.Body.Close()

	if response.StatusCode != http.StatusOK {
		if response.StatusCode == http.StatusNotFound {
			return baggageclaim.VolumeResponse{}, false, nil
		}

		return baggageclaim.VolumeResponse{}, false, getError(response)
	}

	if header := response.Header.Get("Content-Type"); header != "application/json" {
		return baggageclaim.VolumeResponse{}, false, fmt.Errorf("unexpected content-type of: %s", header)
	}

	var volumeResponse baggageclaim.VolumeResponse
	err = json.NewDecoder(response.Body).Decode(&volumeResponse)
	if err != nil {
		return baggageclaim.VolumeResponse{}, false, err
	}

	return volumeResponse, true, nil
}

func (c *client) getVolumeStatsResponse(logger lager.Logger, handle string) (baggageclaim.VolumeStatsResponse, error) {
	request, err := c.requestGenerator.CreateRequest(baggageclaim.GetVolumeStats, rata.Params{
		"handle": handle,
	}, nil)
	if err != nil {
		return baggageclaim.VolumeStatsResponse{}, err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return baggageclaim.VolumeStatsResponse{}, err
	}

	defer response.Body.Close()

	if response.StatusCode != http.StatusOK {
		if response.StatusCode == http.StatusNotFound {
			return baggageclaim.VolumeStatsResponse{}, nil
		}
		return baggageclaim.VolumeStatsResponse{}, getError(response)
	}

	if header := response.Header.Get("Content-Type"); header != "application/json" {
		return baggageclaim.VolumeStatsResponse{}, fmt.Errorf("unexpected content-type of: %s", header)
	}

	var volumeStatsResponse baggageclaim.VolumeStatsResponse
	err = json.NewDecoder(response.Body).Decode(&volumeStatsResponse)
	if err != nil {
		return baggageclaim.VolumeStatsResponse{}, err
	}

	return volumeStatsResponse, nil
}

func (c *client) setTTL(logger lager.Logger, handle string, ttl time.Duration) error {
	buffer := &bytes.Buffer{}
	json.NewEncoder(buffer).Encode(baggageclaim.TTLRequest{
		Value: uint(math.Ceil(ttl.Seconds())),
	})

	request, err := c.requestGenerator.CreateRequest(baggageclaim.SetTTL, rata.Params{
		"handle": handle,
	}, buffer)
	if err != nil {
		return err
	}

	request.Header.Add("Content-type", "application/json")

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return err
	}

	defer response.Body.Close()

	if response.StatusCode != 204 {
		return getError(response)
	}

	return nil
}

func (c *client) setProperty(logger lager.Logger, handle string, propertyName string, propertyValue string) error {
	buffer := &bytes.Buffer{}
	json.NewEncoder(buffer).Encode(baggageclaim.PropertyRequest{
		Value: propertyValue,
	})

	request, err := c.requestGenerator.CreateRequest(baggageclaim.SetProperty, rata.Params{
		"handle":   handle,
		"property": propertyName,
	}, buffer)
	if err != nil {
		return err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return err
	}

	defer response.Body.Close()

	if response.StatusCode != 204 {
		return getError(response)
	}

	return nil
}
<commit_msg>use updated retryhttp<commit_after>package client

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"math"
	"net/http"
	"net/url"
	"sync"
	"time"

	"code.cloudfoundry.org/lager"
	"github.com/tedsuo/rata"

	"github.com/concourse/baggageclaim"
	"github.com/concourse/baggageclaim/api"
	"github.com/concourse/baggageclaim/volume"
	"github.com/concourse/retryhttp"
)

type Client interface {
	baggageclaim.Client
}

type client struct {
	requestGenerator *rata.RequestGenerator

	retryBackOffFactory retryhttp.BackOffFactory
	nestedRoundTripper  http.RoundTripper
}

func New(apiURL string) Client {
	return &client{
		requestGenerator: rata.NewRequestGenerator(apiURL, baggageclaim.Routes),

		retryBackOffFactory: retryhttp.NewExponentialBackOffFactory(60 * time.Minute),

		nestedRoundTripper: &http.Transport{
			DisableKeepAlives: true,
		},
	}
}

func (c *client) httpClient(logger lager.Logger) *http.Client {
	return &http.Client{
		Transport: &retryhttp.RetryRoundTripper{
			Logger:         logger.Session("retry-round-tripper"),
			BackOffFactory: c.retryBackOffFactory,
			RoundTripper:   c.nestedRoundTripper,
		},
	}
}

func (c *client) CreateVolume(logger lager.Logger, volumeSpec baggageclaim.VolumeSpec) (baggageclaim.Volume, error) {
	strategy := volumeSpec.Strategy
	if strategy == nil {
		strategy = baggageclaim.EmptyStrategy{}
	}

	buffer := &bytes.Buffer{}
	json.NewEncoder(buffer).Encode(baggageclaim.VolumeRequest{
		Strategy:     strategy.Encode(),
		TTLInSeconds: uint(math.Ceil(volumeSpec.TTL.Seconds())),
		Properties:   volumeSpec.Properties,
		Privileged:   volumeSpec.Privileged,
	})

	request, _ := c.requestGenerator.CreateRequest(baggageclaim.CreateVolume, nil, buffer)
	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return nil, err
	}

	defer response.Body.Close()

	if response.StatusCode != 201 {
		return nil, getError(response)
	}

	if header := response.Header.Get("Content-Type"); header != "application/json" {
		return nil, fmt.Errorf("unexpected content-type of: %s", header)
	}

	var volumeResponse baggageclaim.VolumeResponse
	err = json.NewDecoder(response.Body).Decode(&volumeResponse)
	if err != nil {
		return nil, err
	}

	v, initialHeartbeatSuccess := c.newVolume(logger, volumeResponse)
	if !initialHeartbeatSuccess {
		return nil, volume.ErrVolumeDoesNotExist
	}
	return v, nil
}

func (c *client) ListVolumes(logger lager.Logger, properties baggageclaim.VolumeProperties) (baggageclaim.Volumes, error) {
	if properties == nil {
		properties = baggageclaim.VolumeProperties{}
	}

	request, err := c.requestGenerator.CreateRequest(baggageclaim.ListVolumes, nil, nil)
	if err != nil {
		return nil, err
	}

	queryString := request.URL.Query()
	for key, val := range properties {
		queryString.Add(key, val)
	}

	request.URL.RawQuery = queryString.Encode()

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return nil, err
	}

	defer response.Body.Close()

	if response.StatusCode != 200 {
		return nil, getError(response)
	}

	if header := response.Header.Get("Content-Type"); header != "application/json" {
		return nil, fmt.Errorf("unexpected content-type of: %s", header)
	}

	var volumesResponse []baggageclaim.VolumeResponse
	err = json.NewDecoder(response.Body).Decode(&volumesResponse)
	if err != nil {
		return nil, err
	}

	var volumes baggageclaim.Volumes
	for _, vr := range volumesResponse {
		v, initialHeartbeatSuccess := c.newVolume(logger, vr)
		if initialHeartbeatSuccess {
			volumes = append(volumes, v)
		}
	}

	return volumes, nil
}

func (c *client) LookupVolume(logger lager.Logger, handle string) (baggageclaim.Volume, bool, error) {

	volumeResponse, found, err := c.getVolumeResponse(logger, handle)
	if err != nil {
		return nil, false, err
	}
	if !found {
		return nil, found, nil
	}

	v, initialHeartbeatSuccess := c.newVolume(logger, volumeResponse)
	if !initialHeartbeatSuccess {
		return nil, false, nil
	}
	return v, true, nil
}

func (c *client) newVolume(logger lager.Logger, apiVolume baggageclaim.VolumeResponse) (baggageclaim.Volume, bool) {
	volume := &clientVolume{
		logger: logger,

		handle: apiVolume.Handle,
		path:   apiVolume.Path,

		bcClient:     c,
		heartbeating: new(sync.WaitGroup),
		release:      make(chan *time.Duration, 1),
	}

	initialHeartbeatSuccess := volume.startHeartbeating(logger, time.Duration(apiVolume.TTLInSeconds)*time.Second)

	return volume, initialHeartbeatSuccess
}

func (c *client) streamIn(logger lager.Logger, destHandle string, path string, tarContent io.Reader) error {
	request, err := c.requestGenerator.CreateRequest(baggageclaim.StreamIn, rata.Params{
		"handle": destHandle,
	}, tarContent)

	request.URL.RawQuery = url.Values{"path": []string{path}}.Encode()
	if err != nil {
		return err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return err
	}

	defer response.Body.Close()
	if response.StatusCode == http.StatusNoContent {
		return nil
	}
	return getError(response)
}

func (c *client) streamOut(logger lager.Logger, srcHandle string, path string) (io.ReadCloser, error) {
	request, err := c.requestGenerator.CreateRequest(baggageclaim.StreamOut, rata.Params{
		"handle": srcHandle,
	}, nil)

	request.URL.RawQuery = url.Values{"path": []string{path}}.Encode()
	if err != nil {
		return nil, err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return nil, err
	}

	if response.StatusCode != http.StatusOK {
		return nil, getError(response)
	}

	return response.Body, nil
}

func getError(response *http.Response) error {
	var errorResponse *api.ErrorResponse
	err := json.NewDecoder(response.Body).Decode(&errorResponse)
	if err != nil {
		return err
	}

	if errorResponse.Message == api.ErrStreamOutNotFound.Error() {
		return baggageclaim.ErrFileNotFound
	}

	if response.StatusCode == 404 {
		return baggageclaim.ErrVolumeNotFound
	}

	return errors.New(errorResponse.Message)
}

func (c *client) getVolumeResponse(logger lager.Logger, handle string) (baggageclaim.VolumeResponse, bool, error) {
	request, err := c.requestGenerator.CreateRequest(baggageclaim.GetVolume, rata.Params{
		"handle": handle,
	}, nil)
	if err != nil {
		return baggageclaim.VolumeResponse{}, false, err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return baggageclaim.VolumeResponse{}, false, err
	}

	defer response.Body.Close()

	if response.StatusCode != http.StatusOK {
		if response.StatusCode == http.StatusNotFound {
			return baggageclaim.VolumeResponse{}, false, nil
		}

		return baggageclaim.VolumeResponse{}, false, getError(response)
	}

	if header := response.Header.Get("Content-Type"); header != "application/json" {
		return baggageclaim.VolumeResponse{}, false, fmt.Errorf("unexpected content-type of: %s", header)
	}

	var volumeResponse baggageclaim.VolumeResponse
	err = json.NewDecoder(response.Body).Decode(&volumeResponse)
	if err != nil {
		return baggageclaim.VolumeResponse{}, false, err
	}

	return volumeResponse, true, nil
}

func (c *client) getVolumeStatsResponse(logger lager.Logger, handle string) (baggageclaim.VolumeStatsResponse, error) {
	request, err := c.requestGenerator.CreateRequest(baggageclaim.GetVolumeStats, rata.Params{
		"handle": handle,
	}, nil)
	if err != nil {
		return baggageclaim.VolumeStatsResponse{}, err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return baggageclaim.VolumeStatsResponse{}, err
	}

	defer response.Body.Close()

	if response.StatusCode != http.StatusOK {
		if response.StatusCode == http.StatusNotFound {
			return baggageclaim.VolumeStatsResponse{}, nil
		}
		return baggageclaim.VolumeStatsResponse{}, getError(response)
	}

	if header := response.Header.Get("Content-Type"); header != "application/json" {
		return baggageclaim.VolumeStatsResponse{}, fmt.Errorf("unexpected content-type of: %s", header)
	}

	var volumeStatsResponse baggageclaim.VolumeStatsResponse
	err = json.NewDecoder(response.Body).Decode(&volumeStatsResponse)
	if err != nil {
		return baggageclaim.VolumeStatsResponse{}, err
	}

	return volumeStatsResponse, nil
}

func (c *client) setTTL(logger lager.Logger, handle string, ttl time.Duration) error {
	buffer := &bytes.Buffer{}
	json.NewEncoder(buffer).Encode(baggageclaim.TTLRequest{
		Value: uint(math.Ceil(ttl.Seconds())),
	})

	request, err := c.requestGenerator.CreateRequest(baggageclaim.SetTTL, rata.Params{
		"handle": handle,
	}, buffer)
	if err != nil {
		return err
	}

	request.Header.Add("Content-type", "application/json")

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return err
	}

	defer response.Body.Close()

	if response.StatusCode != 204 {
		return getError(response)
	}

	return nil
}

func (c *client) setProperty(logger lager.Logger, handle string, propertyName string, propertyValue string) error {
	buffer := &bytes.Buffer{}
	json.NewEncoder(buffer).Encode(baggageclaim.PropertyRequest{
		Value: propertyValue,
	})

	request, err := c.requestGenerator.CreateRequest(baggageclaim.SetProperty, rata.Params{
		"handle":   handle,
		"property": propertyName,
	}, buffer)
	if err != nil {
		return err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return err
	}

	defer response.Body.Close()

	if response.StatusCode != 204 {
		return getError(response)
	}

	return nil
}
<|endoftext|>


<commit_before>package z

import (
	"fmt"
	"log"
	"strconv"
	"strings"

	"github.com/pkg/errors"
)

func parseFlag(flag string) map[string]string {
	kvm := make(map[string]string)
	for _, kv := range strings.Split(flag, ";") {
		if strings.TrimSpace(kv) == "" {
			continue
		}
		splits := strings.SplitN(kv, "=", 2)
		k := strings.TrimSpace(splits[0])
		k = strings.ToLower(k)
		k = strings.ReplaceAll(k, "_", "-")
		kvm[k] = strings.TrimSpace(splits[1])
	}
	return kvm
}

type SuperFlag struct {
	m map[string]string
}

func NewSuperFlag(flag string) *SuperFlag {
	return &SuperFlag{
		m: parseFlag(flag),
	}
}
func (sf *SuperFlag) String() string {
	if sf == nil {
		return ""
	}
	var kvs []string
	for k, v := range sf.m {
		kvs = append(kvs, fmt.Sprintf("%s=%s", k, v))
	}
	return strings.Join(kvs, "; ")
}
func (sf *SuperFlag) MergeAndCheckDefault(flag string) *SuperFlag {
	if sf == nil {
		sf = &SuperFlag{
			m: parseFlag(flag),
		}
		return sf
	}
	numKeys := len(sf.m)
	src := parseFlag(flag)
	for k := range src {
		if _, ok := sf.m[k]; ok {
			numKeys--
		}
	}
	if numKeys != 0 {
		msg := fmt.Sprintf("Found invalid options in %s. Valid options: %v", sf, flag)
		panic(msg)
	}
	for k, v := range src {
		if _, ok := sf.m[k]; !ok {
			sf.m[k] = v
		}
	}
	return sf
}
func (sf *SuperFlag) GetBool(opt string) bool {
	val := sf.GetString(opt)
	if val == "" {
		return false
	}
	b, err := strconv.ParseBool(val)
	if err != nil {
		err = errors.Wrapf(err,
			"Unable to parse %s as bool for key: %s. Options: %s\n",
			val, opt, sf)
		log.Fatalf("%+v", err)
	}
	return b
}
func (sf *SuperFlag) GetUint64(opt string) uint64 {
	val := sf.GetString(opt)
	if val == "" {
		return 0
	}
	u, err := strconv.ParseUint(val, 0, 64)
	if err != nil {
		err = errors.Wrapf(err,
			"Unable to parse %s as uint64 for key: %s. Options: %s\n",
			val, opt, sf)
		log.Fatalf("%+v", err)
	}
	return u
}
func (sf *SuperFlag) GetUint32(opt string) uint32 {
	val := sf.GetString(opt)
	if val == "" {
		return 0
	}
	u, err := strconv.ParseUint(val, 0, 32)
	if err != nil {
		err = errors.Wrapf(err,
			"Unable to parse %s as uint32 for key: %s. Options: %s\n",
			val, opt, sf)
		log.Fatalf("%+v", err)
	}
	return uint32(u)
}
func (sf *SuperFlag) GetString(opt string) string {
	if sf == nil {
		return ""
	}
	return sf.m[opt]
}
<commit_msg>add Has, GetFloat64, and GetInt64 to SuperFlag (#247)<commit_after>package z

import (
	"fmt"
	"log"
	"strconv"
	"strings"

	"github.com/pkg/errors"
)

func parseFlag(flag string) map[string]string {
	kvm := make(map[string]string)
	for _, kv := range strings.Split(flag, ";") {
		if strings.TrimSpace(kv) == "" {
			continue
		}
		splits := strings.SplitN(kv, "=", 2)
		k := strings.TrimSpace(splits[0])
		k = strings.ToLower(k)
		k = strings.ReplaceAll(k, "_", "-")
		kvm[k] = strings.TrimSpace(splits[1])
	}
	return kvm
}

type SuperFlag struct {
	m map[string]string
}

func NewSuperFlag(flag string) *SuperFlag {
	return &SuperFlag{
		m: parseFlag(flag),
	}
}

func (sf *SuperFlag) String() string {
	if sf == nil {
		return ""
	}
	var kvs []string
	for k, v := range sf.m {
		kvs = append(kvs, fmt.Sprintf("%s=%s", k, v))
	}
	return strings.Join(kvs, "; ")
}

func (sf *SuperFlag) MergeAndCheckDefault(flag string) *SuperFlag {
	if sf == nil {
		sf = &SuperFlag{
			m: parseFlag(flag),
		}
		return sf
	}
	numKeys := len(sf.m)
	src := parseFlag(flag)
	for k := range src {
		if _, ok := sf.m[k]; ok {
			numKeys--
		}
	}
	if numKeys != 0 {
		msg := fmt.Sprintf("Found invalid options in %s. Valid options: %v", sf, flag)
		panic(msg)
	}
	for k, v := range src {
		if _, ok := sf.m[k]; !ok {
			sf.m[k] = v
		}
	}
	return sf
}

func (sf *SuperFlag) Has(opt string) bool {
	val := sf.GetString(opt)
	return val != ""
}

func (sf *SuperFlag) GetBool(opt string) bool {
	val := sf.GetString(opt)
	if val == "" {
		return false
	}
	b, err := strconv.ParseBool(val)
	if err != nil {
		err = errors.Wrapf(err,
			"Unable to parse %s as bool for key: %s. Options: %s\n",
			val, opt, sf)
		log.Fatalf("%+v", err)
	}
	return b
}

func (sf *SuperFlag) GetFloat64(opt string) float64 {
	val := sf.GetString(opt)
	if val == "" {
		return 0
	}
	f, err := strconv.ParseFloat(val, 64)
	if err != nil {
		err = errors.Wrapf(err,
			"Unable to parse %s as float64 for key: %s. Options: %s\n",
			val, opt, sf)
		log.Fatalf("%+v", err)
	}
	return f
}

func (sf *SuperFlag) GetInt64(opt string) int64 {
	val := sf.GetString(opt)
	if val == "" {
		return 0
	}
	i, err := strconv.ParseInt(val, 0, 64)
	if err != nil {
		err = errors.Wrapf(err,
			"Unable to parse %s as int64 for key: %s. Options: %s\n",
			val, opt, sf)
		log.Fatalf("%+v", err)
	}
	return i
}

func (sf *SuperFlag) GetUint64(opt string) uint64 {
	val := sf.GetString(opt)
	if val == "" {
		return 0
	}
	u, err := strconv.ParseUint(val, 0, 64)
	if err != nil {
		err = errors.Wrapf(err,
			"Unable to parse %s as uint64 for key: %s. Options: %s\n",
			val, opt, sf)
		log.Fatalf("%+v", err)
	}
	return u
}

func (sf *SuperFlag) GetUint32(opt string) uint32 {
	val := sf.GetString(opt)
	if val == "" {
		return 0
	}
	u, err := strconv.ParseUint(val, 0, 32)
	if err != nil {
		err = errors.Wrapf(err,
			"Unable to parse %s as uint32 for key: %s. Options: %s\n",
			val, opt, sf)
		log.Fatalf("%+v", err)
	}
	return uint32(u)
}

func (sf *SuperFlag) GetString(opt string) string {
	if sf == nil {
		return ""
	}
	return sf.m[opt]
}
<|endoftext|>


<commit_before>package nvd_search

import (
	"log"
	"fmt"
	"time"
	"path"

	"github.com/levigross/grequests"
)

const (
	NVDFeedBaseUrl = "https://static.nvd.nist.gov/feeds/"
	NVDJsonFeedUrl = "json/cve/%.1[1]f/nvdcve-%.1[1]f-"
	NVDFeedVersion = 1.0
)

var NVDUrl string = fmt.Sprintf("%v%v", NVDFeedBaseUrl, fmt.Sprintf(NVDJsonFeedUrl, NVDFeedVersion))

func checkFatal(e error) {
	if e != nil {
		log.Fatal(e)
	}
}

func checkError(e error) bool {
	if e != nil {
		log.Println(e)
		return false
	}
	return true
}

func downloadFile(uri, filename string) bool {
	log.Println("Downloading license file from", uri)
	response, err := grequests.Get(uri, nil)
	if !checkError(err) {
		log.Println("Couldn't download file, maybe it's not valid URL?")
		return false
	}
	err = response.DownloadToFile(filename)
	checkFatal(err)
	log.Println("Saved content from", uri, "to", filename)
	return true
}

func generateFileList() []string {
	fileList := append(make([]string, 20), "recent", "modified")
	for year := 2002; year <= time.Now().Year(); year++ {
		fileList = append(fileList, fmt.Sprintf("%v", year))
	}
	return fileList
}

func getMeta(variety string) {
	url := fmt.Sprintf("%v%v.meta", NVDUrl, variety)
	response, err := grequests.Get(url, nil)
	checkFatal(err)
	fmt.Println(response.String())
}

func getJsonGz(variety, filepath string) {
	filename := fmt.Sprintf("%v.json.gz", variety)
	url := fmt.Sprintf("%v%v", NVDUrl, filename)
	if !downloadFile(url, path.Join(filepath, filename)) {
		log.Fatal("oops")
	}
}
<commit_msg>Fix fileList init in generateFileList func<commit_after>package nvd_search

import (
	"log"
	"fmt"
	"time"
	"path"

	"github.com/levigross/grequests"
)

const (
	NVDFeedBaseUrl = "https://static.nvd.nist.gov/feeds/"
	NVDJsonFeedUrl = "json/cve/%.1[1]f/nvdcve-%.1[1]f-"
	NVDFeedVersion = 1.0
)

var NVDUrl string = fmt.Sprintf("%v%v", NVDFeedBaseUrl, fmt.Sprintf(NVDJsonFeedUrl, NVDFeedVersion))

func checkFatal(e error) {
	if e != nil {
		log.Fatal(e)
	}
}

func checkError(e error) bool {
	if e != nil {
		log.Println(e)
		return false
	}
	return true
}

func downloadFile(uri, filename string) bool {
	log.Println("Downloading license file from", uri)
	response, err := grequests.Get(uri, nil)
	if !checkError(err) {
		log.Println("Couldn't download file, maybe it's not valid URL?")
		return false
	}
	err = response.DownloadToFile(filename)
	checkFatal(err)
	log.Println("Saved content from", uri, "to", filename)
	return true
}

func generateFileList() []string {
	fileList := []string{"modified"}
	for year := 2002; year <= time.Now().Year(); year++ {
		fileList = append(fileList, fmt.Sprintf("%v", year))
	}
	return fileList
}

func getMeta(variety string) {
	url := fmt.Sprintf("%v%v.meta", NVDUrl, variety)
	response, err := grequests.Get(url, nil)
	checkFatal(err)
	fmt.Println(response.String())
}

func getJsonGz(variety, filepath string) {
	filename := fmt.Sprintf("%v.json.gz", variety)
	url := fmt.Sprintf("%v%v", NVDUrl, filename)
	if !downloadFile(url, path.Join(filepath, filename)) {
		log.Fatal("oops")
	}
}
<|endoftext|>


<commit_before>package restrict

import (
	"fmt"
	"github.com/dotcloud/docker/pkg/system"
	"path/filepath"
	"syscall"
)

const flags = syscall.MS_BIND | syscall.MS_REC | syscall.MS_RDONLY

var restrictions = map[string]string{
	// dirs
	"/proc/sys":  "",
	"/proc/irq":  "",
	"/proc/acpi": "",

	// files
	"/proc/sysrq-trigger": "/dev/null",
	"/proc/kcore":         "/dev/null",
}

// Restrict locks down access to many areas of proc
// by using the asumption that the user does not have mount caps to
// revert the changes made here
func Restrict(rootfs, empty string) error {
	for dest, source := range restrictions {
		dest = filepath.Join(rootfs, dest)

		// we don't have a "/dev/null" for dirs so have the requester pass a dir
		// for us to bind mount
		switch source {
		case "":
			source = empty
		default:
			source = filepath.Join(rootfs, source)
		}
		if err := system.Mount(source, dest, "bind", flags, ""); err != nil {
			return fmt.Errorf("unable to mount %s over %s %s", source, dest, err)
		}
		if err := system.Mount("", dest, "bind", flags|syscall.MS_REMOUNT, ""); err != nil {
			return fmt.Errorf("unable to mount %s over %s %s", source, dest, err)
		}
	}
	return nil
}
<commit_msg>Ignore isnot exists errors for proc paths Docker-DCO-1.1-Signed-off-by: Michael Crosby <michael@crosbymichael.com> (github: crosbymichael)<commit_after>package restrict

import (
	"fmt"
	"os"
	"path/filepath"
	"syscall"

	"github.com/dotcloud/docker/pkg/system"
)

const flags = syscall.MS_BIND | syscall.MS_REC | syscall.MS_RDONLY

var restrictions = map[string]string{
	// dirs
	"/proc/sys":  "",
	"/proc/irq":  "",
	"/proc/acpi": "",

	// files
	"/proc/sysrq-trigger": "/dev/null",
	"/proc/kcore":         "/dev/null",
}

// Restrict locks down access to many areas of proc
// by using the asumption that the user does not have mount caps to
// revert the changes made here
func Restrict(rootfs, empty string) error {
	for dest, source := range restrictions {
		dest = filepath.Join(rootfs, dest)

		// we don't have a "/dev/null" for dirs so have the requester pass a dir
		// for us to bind mount
		switch source {
		case "":
			source = empty
		default:
			source = filepath.Join(rootfs, source)
		}
		if err := system.Mount(source, dest, "bind", flags, ""); err != nil {
			if os.IsNotExist(err) {
				continue
			}
			return fmt.Errorf("unable to mount %s over %s %s", source, dest, err)
		}
		if err := system.Mount("", dest, "bind", flags|syscall.MS_REMOUNT, ""); err != nil {
			return fmt.Errorf("unable to mount %s over %s %s", source, dest, err)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strings"
	"syscall"
)

var passTests bool = true

func main() {
	err := os.Setenv("FRUNDIS", "ok")
	if err != nil {
		fmt.Fprint(os.Stderr, "could not set environment variable")
	}
	for _, f := range []func() error{doFragments, doStandalones} {
		err := f()
		if err != nil {
			fmt.Fprint(os.Stderr, err)
			os.Exit(1)
		}
	}
	if !passTests {
		os.Exit(1)
	}
}

func doFragments() error {
	dataDir, err := os.Open("t/data")
	if err != nil {
		return errors.New(fmt.Sprintf("Error reading t/data:%v", err))
	}
	defer func() {
		err := dataDir.Close()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error closing t/data:%v", err)
		}
	}()
	names, err := dataDir.Readdirnames(-1)
	for _, f := range names {
		if b, _ := path.Match("*.frundis", f); !b {
			continue
		}
		fullPath := path.Join("t", "data", f)
		for _, format := range []string{"latex", "mom", "xhtml", "markdown"} {
			err := testFile(fullPath, format)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func doStandalones() error {
	dataDir, err := os.Open("t/data-dirs")
	if err != nil {
		return errors.New(fmt.Sprintf("Error reading t/data-dirs:%v", err))
	}
	defer func() {
		err = dataDir.Close()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error closing t/data-dirs:%v", err)
		}
	}()
	names, err := dataDir.Readdirnames(-1)
	for _, f := range names {
		if b, _ := path.Match("*.frundis", f); !b {
			continue
		}
		fullPath := path.Join("t", "data-dirs", f)
		if b, _ := path.Match("*-epub*", f); b {
			err := testStandalone(fullPath, "epub", false)
			if err != nil {
				return err
			}
			continue
		}
		if b, _ := path.Match("*-xhtml*", f); b {
			err := testStandalone(fullPath, "xhtml", false)
			if err != nil {
				return err
			}
			err = testStandalone(fullPath, "xhtml", true)
			if err != nil {
				return err
			}
			continue
		}
		if b, _ := path.Match("*-latex*", f); b {
			err := testStandalone(fullPath, "latex", true)
			if err != nil {
				return err
			}
			continue
		}
		err := testStandalone(fullPath, "xhtml", false)
		if err != nil {
			return err
		}
		for _, format := range []string{"xhtml", "latex", "mom"} {
			err = testStandalone(fullPath, format, true)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

var outputFile = ".gofrundistest.out"
var outputDir = ".gofrundistestdir.out"

func getBinPath() (string, error) {
	binPath, okEnv := os.LookupEnv("GOPATH")
	if !okEnv {
		return "", errors.New("no GOPATH")
	}
	binPath = path.Join(binPath, "bin", "frundis")
	return binPath, nil
}

func testFile(file string, format string) error {
	binPath, err := getBinPath()
	if err != nil {
		return err
	}
	name := strings.TrimSuffix(file, ".frundis")
	suffix := strings.Replace(format, "xhtml", "html", -1)
	suffix = strings.Replace(suffix, "latex", "tex", -1)
	cmd := exec.Command(binPath, "-T", format, "-x", "-a", "-o", outputFile, file)
	cmdout, err := cmd.CombinedOutput()
	fmt.Fprint(os.Stderr, string(cmdout))
	ref := name + "." + suffix
	if err != nil {
		ok(false, ref)
		fmt.Fprint(os.Stderr, err)
		return nil
	}
	cmd = exec.Command("diff", "-u", ref, outputFile)
	_, e := os.Stat(ref)
	var diff []byte
	if e != nil {
		diff = []byte("Reference file does not exist yet.\n")
	} else {
		var err error
		diff, err = cmd.CombinedOutput()
		if err != nil {
			switch err := err.(type) {
			case *exec.ExitError:
				s := err.Sys().(syscall.WaitStatus)
				ok(false, ref)
				if s.ExitStatus() >= 2 {
					fmt.Fprintf(os.Stderr, "Error executing command:%s:%s", strings.Join(cmd.Args, " "), string(diff))
					return nil
				}
			default:
				ok(false, ref)
				return err
			}
		}
	}
	if !ok(string(diff) == "", ref) {
		fmt.Fprint(os.Stderr, string(diff))
		input := readLine()
		if input == "Y" {
			b, err := ioutil.ReadFile(outputFile)
			if err != nil {
				fmt.Fprintf(os.Stderr, "Error reading:%s", outputFile)
				return nil
			}
			err = ioutil.WriteFile(ref, b, 0644)
			if err != nil {
				fmt.Fprintf(os.Stderr, "Error writing:%s", ref)
				return nil
			}

		}
	}
	return nil
}

func testStandalone(file string, format string, toFile bool) error {
	binPath, err := getBinPath()
	if err != nil {
		return err
	}
	var suffix string
	switch format {
	case "epub":
		suffix = "-epub"
	case "xhtml":
		if toFile {
			suffix = ".html"
		} else {
			suffix = "-html"
		}
	case "latex":
		suffix = ".tex"
	case "mom":
		suffix = ".mom"
	default:
		return errors.New(fmt.Sprintf("internal error:unknown format:%s", format))
	}
	name := strings.TrimSuffix(file, ".frundis")
	info, err := os.Stat(outputDir)
	if err == nil {
		if info.IsDir() {
			err = os.RemoveAll(outputDir)
			if err != nil {
				return errors.New(fmt.Sprintf("removing outputDir:%v", err))
			}
		} else {
			os.Remove(outputDir)
		}
	}
	var cmd *exec.Cmd
	if toFile {
		cmd = exec.Command(binPath, "-T", format, "-a", "-s", "-o", outputDir, file)
	} else {
		cmd = exec.Command(binPath, "-T", format, "-o", outputDir, file)
	}
	cmdout, err := cmd.CombinedOutput()
	fmt.Fprint(os.Stderr, string(cmdout))
	ref := name + suffix
	cmdExpression := strings.Join(cmd.Args, " ")
	if err != nil {
		ok(false, ref)
		fmt.Fprintf(os.Stderr, "Error executing command:%s\n", cmdExpression)
		fmt.Fprint(os.Stderr, err)
		return nil
	}
	_, e := os.Stat(ref)
	var diff []byte
	if e != nil {
		diff = []byte("Reference file does not exist yet\n")
	} else {
		cmd = exec.Command("diff", "-ru", ref, outputDir)
		var err error
		diff, err = cmd.CombinedOutput()
		if err != nil {
			switch err := err.(type) {
			case *exec.ExitError:
				s := err.Sys().(syscall.WaitStatus)
				if s.ExitStatus() >= 2 {
					ok(false, ref)
					fmt.Fprintf(os.Stderr, "Error for command:%s\n", cmdExpression)
					fmt.Fprintf(os.Stderr, "Error executing command:%s\n", strings.Join(cmd.Args, " "))
					fmt.Fprint(os.Stderr, string(diff))
					fmt.Fprintln(os.Stderr, "^^^^^^^ END OF ERROR ^^^^^^^^^")
					return nil
				}
			default:
				return err
			}
		}
	}
	if !ok(string(diff) == "", ref) {
		fmt.Fprintf(os.Stderr, "Diff for command:%s\n", cmdExpression)
		fmt.Fprint(os.Stderr, string(diff))
		input := readLine()
		if input == "Y" {
			fmt.Fprintf(os.Stderr, "replacing %s with %s\n", ref, outputFile)
			err := os.RemoveAll(ref)
			if err != nil {
				return errors.New(fmt.Sprintf("Error removing directory %s:%v", ref, err))
			}
			err = os.Rename(outputDir, ref)
			if err != nil {
				return errors.New(fmt.Sprintf("Error renaming to %s", ref))
			}
		}
	}
	return nil
}

func readLine() string {
	fmt.Fprintf(os.Stderr, "Files differ. Put new [Y/n]?")
	in := bufio.NewScanner(os.Stdin)
	okScan := in.Scan()
	if !okScan {
		return ""
	}
	return in.Text()
}

var testNum int

func ok(b bool, msg string) bool {
	testNum++
	if b {
		fmt.Fprintf(os.Stderr, "ok %d - %s\n", testNum, msg)
	} else {
		fmt.Fprintf(os.Stderr, "not ok %d - %s\n", testNum, msg)
		passTests = false
	}
	return b
}
<commit_msg>golint stuff in tests<commit_after>package main

import (
	"bufio"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strings"
	"syscall"
)

var passTests = true

func main() {
	err := os.Setenv("FRUNDIS", "ok")
	if err != nil {
		fmt.Fprint(os.Stderr, "could not set environment variable")
	}
	for _, f := range []func() error{doFragments, doStandalones} {
		err := f()
		if err != nil {
			fmt.Fprint(os.Stderr, err)
			os.Exit(1)
		}
	}
	if !passTests {
		os.Exit(1)
	}
}

func doFragments() error {
	dataDir, err := os.Open("t/data")
	if err != nil {
		return fmt.Errorf("Error reading t/data:%v", err)
	}
	defer func() {
		err := dataDir.Close()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error closing t/data:%v", err)
		}
	}()
	names, err := dataDir.Readdirnames(-1)
	for _, f := range names {
		if b, _ := path.Match("*.frundis", f); !b {
			continue
		}
		fullPath := path.Join("t", "data", f)
		for _, format := range []string{"latex", "mom", "xhtml", "markdown"} {
			err := testFile(fullPath, format)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func doStandalones() error {
	dataDir, err := os.Open("t/data-dirs")
	if err != nil {
		return fmt.Errorf("Error reading t/data-dirs:%v", err)
	}
	defer func() {
		err = dataDir.Close()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error closing t/data-dirs:%v", err)
		}
	}()
	names, err := dataDir.Readdirnames(-1)
	for _, f := range names {
		if b, _ := path.Match("*.frundis", f); !b {
			continue
		}
		fullPath := path.Join("t", "data-dirs", f)
		if b, _ := path.Match("*-epub*", f); b {
			err := testStandalone(fullPath, "epub", false)
			if err != nil {
				return err
			}
			continue
		}
		if b, _ := path.Match("*-xhtml*", f); b {
			err := testStandalone(fullPath, "xhtml", false)
			if err != nil {
				return err
			}
			err = testStandalone(fullPath, "xhtml", true)
			if err != nil {
				return err
			}
			continue
		}
		if b, _ := path.Match("*-latex*", f); b {
			err := testStandalone(fullPath, "latex", true)
			if err != nil {
				return err
			}
			continue
		}
		err := testStandalone(fullPath, "xhtml", false)
		if err != nil {
			return err
		}
		for _, format := range []string{"xhtml", "latex", "mom"} {
			err = testStandalone(fullPath, format, true)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

var outputFile = ".gofrundistest.out"
var outputDir = ".gofrundistestdir.out"

func getBinPath() (string, error) {
	binPath, okEnv := os.LookupEnv("GOPATH")
	if !okEnv {
		return "", errors.New("no GOPATH")
	}
	binPath = path.Join(binPath, "bin", "frundis")
	return binPath, nil
}

func testFile(file string, format string) error {
	binPath, err := getBinPath()
	if err != nil {
		return err
	}
	name := strings.TrimSuffix(file, ".frundis")
	suffix := strings.Replace(format, "xhtml", "html", -1)
	suffix = strings.Replace(suffix, "latex", "tex", -1)
	cmd := exec.Command(binPath, "-T", format, "-x", "-a", "-o", outputFile, file)
	cmdout, err := cmd.CombinedOutput()
	fmt.Fprint(os.Stderr, string(cmdout))
	ref := name + "." + suffix
	if err != nil {
		ok(false, ref)
		fmt.Fprint(os.Stderr, err)
		return nil
	}
	cmd = exec.Command("diff", "-u", ref, outputFile)
	_, e := os.Stat(ref)
	var diff []byte
	if e != nil {
		diff = []byte("Reference file does not exist yet.\n")
	} else {
		var err error
		diff, err = cmd.CombinedOutput()
		if err != nil {
			switch err := err.(type) {
			case *exec.ExitError:
				s := err.Sys().(syscall.WaitStatus)
				ok(false, ref)
				if s.ExitStatus() >= 2 {
					fmt.Fprintf(os.Stderr, "Error executing command:%s:%s", strings.Join(cmd.Args, " "), string(diff))
					return nil
				}
			default:
				ok(false, ref)
				return err
			}
		}
	}
	if !ok(string(diff) == "", ref) {
		fmt.Fprint(os.Stderr, string(diff))
		input := readLine()
		if input == "Y" {
			b, err := ioutil.ReadFile(outputFile)
			if err != nil {
				fmt.Fprintf(os.Stderr, "Error reading:%s", outputFile)
				return nil
			}
			err = ioutil.WriteFile(ref, b, 0644)
			if err != nil {
				fmt.Fprintf(os.Stderr, "Error writing:%s", ref)
				return nil
			}

		}
	}
	return nil
}

func testStandalone(file string, format string, toFile bool) error {
	binPath, err := getBinPath()
	if err != nil {
		return err
	}
	var suffix string
	switch format {
	case "epub":
		suffix = "-epub"
	case "xhtml":
		if toFile {
			suffix = ".html"
		} else {
			suffix = "-html"
		}
	case "latex":
		suffix = ".tex"
	case "mom":
		suffix = ".mom"
	default:
		return fmt.Errorf("internal error:unknown format:%s", format)
	}
	name := strings.TrimSuffix(file, ".frundis")
	info, err := os.Stat(outputDir)
	if err == nil {
		if info.IsDir() {
			err = os.RemoveAll(outputDir)
			if err != nil {
				return fmt.Errorf("removing outputDir:%v", err)
			}
		} else {
			os.Remove(outputDir)
		}
	}
	var cmd *exec.Cmd
	if toFile {
		cmd = exec.Command(binPath, "-T", format, "-a", "-s", "-o", outputDir, file)
	} else {
		cmd = exec.Command(binPath, "-T", format, "-o", outputDir, file)
	}
	cmdout, err := cmd.CombinedOutput()
	fmt.Fprint(os.Stderr, string(cmdout))
	ref := name + suffix
	cmdExpression := strings.Join(cmd.Args, " ")
	if err != nil {
		ok(false, ref)
		fmt.Fprintf(os.Stderr, "Error executing command:%s\n", cmdExpression)
		fmt.Fprint(os.Stderr, err)
		return nil
	}
	_, e := os.Stat(ref)
	var diff []byte
	if e != nil {
		diff = []byte("Reference file does not exist yet\n")
	} else {
		cmd = exec.Command("diff", "-ru", ref, outputDir)
		var err error
		diff, err = cmd.CombinedOutput()
		if err != nil {
			switch err := err.(type) {
			case *exec.ExitError:
				s := err.Sys().(syscall.WaitStatus)
				if s.ExitStatus() >= 2 {
					ok(false, ref)
					fmt.Fprintf(os.Stderr, "Error for command:%s\n", cmdExpression)
					fmt.Fprintf(os.Stderr, "Error executing command:%s\n", strings.Join(cmd.Args, " "))
					fmt.Fprint(os.Stderr, string(diff))
					fmt.Fprintln(os.Stderr, "^^^^^^^ END OF ERROR ^^^^^^^^^")
					return nil
				}
			default:
				return err
			}
		}
	}
	if !ok(string(diff) == "", ref) {
		fmt.Fprintf(os.Stderr, "Diff for command:%s\n", cmdExpression)
		fmt.Fprint(os.Stderr, string(diff))
		input := readLine()
		if input == "Y" {
			fmt.Fprintf(os.Stderr, "replacing %s with %s\n", ref, outputFile)
			err := os.RemoveAll(ref)
			if err != nil {
				return fmt.Errorf("Error removing directory %s:%v", ref, err)
			}
			err = os.Rename(outputDir, ref)
			if err != nil {
				return fmt.Errorf("Error renaming to %s", ref)
			}
		}
	}
	return nil
}

func readLine() string {
	fmt.Fprintf(os.Stderr, "Files differ. Put new [Y/n]?")
	in := bufio.NewScanner(os.Stdin)
	okScan := in.Scan()
	if !okScan {
		return ""
	}
	return in.Text()
}

var testNum int

func ok(b bool, msg string) bool {
	testNum++
	if b {
		fmt.Fprintf(os.Stderr, "ok %d - %s\n", testNum, msg)
	} else {
		fmt.Fprintf(os.Stderr, "not ok %d - %s\n", testNum, msg)
		passTests = false
	}
	return b
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"os"
	"reflect"
	"strings"
	"unicode"
	"utf8"
)

func initRewrite() {
	if *rewriteRule == "" {
		return
	}
	f := strings.Split(*rewriteRule, "->", -1)
	if len(f) != 2 {
		fmt.Fprintf(os.Stderr, "rewrite rule must be of the form 'pattern -> replacement'\n")
		os.Exit(2)
	}
	pattern := parseExpr(f[0], "pattern")
	replace := parseExpr(f[1], "replacement")
	rewrite = func(p *ast.File) *ast.File { return rewriteFile(pattern, replace, p) }
}

// parseExpr parses s as an expression.
// It might make sense to expand this to allow statement patterns,
// but there are problems with preserving formatting and also
// with what a wildcard for a statement looks like.
func parseExpr(s string, what string) ast.Expr {
	x, err := parser.ParseExpr(fset, "input", s)
	if err != nil {
		fmt.Fprintf(os.Stderr, "parsing %s %s: %s\n", what, s, err)
		os.Exit(2)
	}
	return x
}

// rewriteFile applies the rewrite rule 'pattern -> replace' to an entire file.
func rewriteFile(pattern, replace ast.Expr, p *ast.File) *ast.File {
	m := make(map[string]reflect.Value)
	pat := reflect.NewValue(pattern)
	repl := reflect.NewValue(replace)
	var f func(val reflect.Value) reflect.Value // f is recursive
	f = func(val reflect.Value) reflect.Value {
		for k := range m {
			m[k] = nil, false
		}
		val = apply(f, val)
		if match(m, pat, val) {
			val = subst(m, repl, reflect.NewValue(val.Interface().(ast.Node).Pos()))
		}
		return val
	}
	return apply(f, reflect.NewValue(p)).Interface().(*ast.File)
}

var positionType = reflect.Typeof(token.NoPos)
var identType = reflect.Typeof((*ast.Ident)(nil))

func isWildcard(s string) bool {
	rune, size := utf8.DecodeRuneInString(s)
	return size == len(s) && unicode.IsLower(rune)
}

// apply replaces each AST field x in val with f(x), returning val.
// To avoid extra conversions, f operates on the reflect.Value form.
func apply(f func(reflect.Value) reflect.Value, val reflect.Value) reflect.Value {
	if val == nil {
		return nil
	}
	switch v := reflect.Indirect(val).(type) {
	case *reflect.SliceValue:
		for i := 0; i < v.Len(); i++ {
			e := v.Elem(i)
			e.SetValue(f(e))
		}
	case *reflect.StructValue:
		for i := 0; i < v.NumField(); i++ {
			e := v.Field(i)
			e.SetValue(f(e))
		}
	case *reflect.InterfaceValue:
		e := v.Elem()
		v.SetValue(f(e))
	}
	return val
}

// match returns true if pattern matches val,
// recording wildcard submatches in m.
// If m == nil, match checks whether pattern == val.
func match(m map[string]reflect.Value, pattern, val reflect.Value) bool {
	// Wildcard matches any expression.  If it appears multiple
	// times in the pattern, it must match the same expression
	// each time.
	if m != nil && pattern.Type() == identType {
		name := pattern.Interface().(*ast.Ident).Name
		if isWildcard(name) {
			// wildcards only match expressions
			if _, ok := val.Interface().(ast.Expr); ok {
				if old, ok := m[name]; ok {
					return match(nil, old, val)
				}
				m[name] = val
				return true
			}
		}
	}

	// Otherwise, pattern and val must match recursively.
	if pattern == nil || val == nil {
		return pattern == nil && val == nil
	}
	if pattern.Type() != val.Type() {
		return false
	}

	// Special cases.
	switch pattern.Type() {
	case positionType:
		// token positions don't need to match
		return true
	case identType:
		// For identifiers, only the names need to match
		// (and none of the other *ast.Object information).
		// This is a common case, handle it all here instead
		// of recursing down any further via reflection.
		p := pattern.Interface().(*ast.Ident)
		v := val.Interface().(*ast.Ident)
		return p == nil && v == nil || p != nil && v != nil && p.Name == v.Name
	}

	p := reflect.Indirect(pattern)
	v := reflect.Indirect(val)
	if p == nil || v == nil {
		return p == nil && v == nil
	}

	switch p := p.(type) {
	case *reflect.SliceValue:
		v := v.(*reflect.SliceValue)
		if p.Len() != v.Len() {
			return false
		}
		for i := 0; i < p.Len(); i++ {
			if !match(m, p.Elem(i), v.Elem(i)) {
				return false
			}
		}
		return true

	case *reflect.StructValue:
		v := v.(*reflect.StructValue)
		if p.NumField() != v.NumField() {
			return false
		}
		for i := 0; i < p.NumField(); i++ {
			if !match(m, p.Field(i), v.Field(i)) {
				return false
			}
		}
		return true

	case *reflect.InterfaceValue:
		v := v.(*reflect.InterfaceValue)
		return match(m, p.Elem(), v.Elem())
	}

	// Handle token integers, etc.
	return p.Interface() == v.Interface()
}

// subst returns a copy of pattern with values from m substituted in place
// of wildcards and pos used as the position of tokens from the pattern.
// if m == nil, subst returns a copy of pattern and doesn't change the line
// number information.
func subst(m map[string]reflect.Value, pattern reflect.Value, pos reflect.Value) reflect.Value {
	if pattern == nil {
		return nil
	}

	// Wildcard gets replaced with map value.
	if m != nil && pattern.Type() == identType {
		name := pattern.Interface().(*ast.Ident).Name
		if isWildcard(name) {
			if old, ok := m[name]; ok {
				return subst(nil, old, nil)
			}
		}
	}

	if pos != nil && pattern.Type() == positionType {
		// use new position only if old position was valid in the first place
		if old := pattern.Interface().(token.Pos); !old.IsValid() {
			return pattern
		}
		return pos
	}

	// Otherwise copy.
	switch p := pattern.(type) {
	case *reflect.SliceValue:
		v := reflect.MakeSlice(p.Type().(*reflect.SliceType), p.Len(), p.Len())
		for i := 0; i < p.Len(); i++ {
			v.Elem(i).SetValue(subst(m, p.Elem(i), pos))
		}
		return v

	case *reflect.StructValue:
		v := reflect.MakeZero(p.Type()).(*reflect.StructValue)
		for i := 0; i < p.NumField(); i++ {
			v.Field(i).SetValue(subst(m, p.Field(i), pos))
		}
		return v

	case *reflect.PtrValue:
		v := reflect.MakeZero(p.Type()).(*reflect.PtrValue)
		v.PointTo(subst(m, p.Elem(), pos))
		return v

	case *reflect.InterfaceValue:
		v := reflect.MakeZero(p.Type()).(*reflect.InterfaceValue)
		v.SetValue(subst(m, p.Elem(), pos))
		return v
	}

	return pattern
}
<commit_msg>gofmt: don't attempt certain illegal rewrites<commit_after>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"os"
	"reflect"
	"strings"
	"unicode"
	"utf8"
)

func initRewrite() {
	if *rewriteRule == "" {
		return
	}
	f := strings.Split(*rewriteRule, "->", -1)
	if len(f) != 2 {
		fmt.Fprintf(os.Stderr, "rewrite rule must be of the form 'pattern -> replacement'\n")
		os.Exit(2)
	}
	pattern := parseExpr(f[0], "pattern")
	replace := parseExpr(f[1], "replacement")
	rewrite = func(p *ast.File) *ast.File { return rewriteFile(pattern, replace, p) }
}

// parseExpr parses s as an expression.
// It might make sense to expand this to allow statement patterns,
// but there are problems with preserving formatting and also
// with what a wildcard for a statement looks like.
func parseExpr(s string, what string) ast.Expr {
	x, err := parser.ParseExpr(fset, "input", s)
	if err != nil {
		fmt.Fprintf(os.Stderr, "parsing %s %s: %s\n", what, s, err)
		os.Exit(2)
	}
	return x
}

// rewriteFile applies the rewrite rule 'pattern -> replace' to an entire file.
func rewriteFile(pattern, replace ast.Expr, p *ast.File) *ast.File {
	m := make(map[string]reflect.Value)
	pat := reflect.NewValue(pattern)
	repl := reflect.NewValue(replace)
	var f func(val reflect.Value) reflect.Value // f is recursive
	f = func(val reflect.Value) reflect.Value {
		for k := range m {
			m[k] = nil, false
		}
		val = apply(f, val)
		if match(m, pat, val) {
			val = subst(m, repl, reflect.NewValue(val.Interface().(ast.Node).Pos()))
		}
		return val
	}
	return apply(f, reflect.NewValue(p)).Interface().(*ast.File)
}

// setValue is a wrapper for x.SetValue(y); it protects
// the caller from panics if x cannot be changed to y.
func setValue(x, y reflect.Value) {
	defer func() {
		if x := recover(); x != nil {
			if s, ok := x.(string); ok && strings.HasPrefix(s, "type mismatch") {
				// x cannot be set to y - ignore this rewrite
				return
			}
			panic(x)
		}
	}()
	x.SetValue(y)
}

// apply replaces each AST field x in val with f(x), returning val.
// To avoid extra conversions, f operates on the reflect.Value form.
func apply(f func(reflect.Value) reflect.Value, val reflect.Value) reflect.Value {
	if val == nil {
		return nil
	}
	switch v := reflect.Indirect(val).(type) {
	case *reflect.SliceValue:
		for i := 0; i < v.Len(); i++ {
			e := v.Elem(i)
			setValue(e, f(e))
		}
	case *reflect.StructValue:
		for i := 0; i < v.NumField(); i++ {
			e := v.Field(i)
			setValue(e, f(e))
		}
	case *reflect.InterfaceValue:
		e := v.Elem()
		setValue(v, f(e))
	}
	return val
}

var positionType = reflect.Typeof(token.NoPos)
var identType = reflect.Typeof((*ast.Ident)(nil))

func isWildcard(s string) bool {
	rune, size := utf8.DecodeRuneInString(s)
	return size == len(s) && unicode.IsLower(rune)
}

// match returns true if pattern matches val,
// recording wildcard submatches in m.
// If m == nil, match checks whether pattern == val.
func match(m map[string]reflect.Value, pattern, val reflect.Value) bool {
	// Wildcard matches any expression.  If it appears multiple
	// times in the pattern, it must match the same expression
	// each time.
	if m != nil && pattern.Type() == identType {
		name := pattern.Interface().(*ast.Ident).Name
		if isWildcard(name) {
			// wildcards only match expressions
			if _, ok := val.Interface().(ast.Expr); ok {
				if old, ok := m[name]; ok {
					return match(nil, old, val)
				}
				m[name] = val
				return true
			}
		}
	}

	// Otherwise, pattern and val must match recursively.
	if pattern == nil || val == nil {
		return pattern == nil && val == nil
	}
	if pattern.Type() != val.Type() {
		return false
	}

	// Special cases.
	switch pattern.Type() {
	case positionType:
		// token positions don't need to match
		return true
	case identType:
		// For identifiers, only the names need to match
		// (and none of the other *ast.Object information).
		// This is a common case, handle it all here instead
		// of recursing down any further via reflection.
		p := pattern.Interface().(*ast.Ident)
		v := val.Interface().(*ast.Ident)
		return p == nil && v == nil || p != nil && v != nil && p.Name == v.Name
	}

	p := reflect.Indirect(pattern)
	v := reflect.Indirect(val)
	if p == nil || v == nil {
		return p == nil && v == nil
	}

	switch p := p.(type) {
	case *reflect.SliceValue:
		v := v.(*reflect.SliceValue)
		if p.Len() != v.Len() {
			return false
		}
		for i := 0; i < p.Len(); i++ {
			if !match(m, p.Elem(i), v.Elem(i)) {
				return false
			}
		}
		return true

	case *reflect.StructValue:
		v := v.(*reflect.StructValue)
		if p.NumField() != v.NumField() {
			return false
		}
		for i := 0; i < p.NumField(); i++ {
			if !match(m, p.Field(i), v.Field(i)) {
				return false
			}
		}
		return true

	case *reflect.InterfaceValue:
		v := v.(*reflect.InterfaceValue)
		return match(m, p.Elem(), v.Elem())
	}

	// Handle token integers, etc.
	return p.Interface() == v.Interface()
}

// subst returns a copy of pattern with values from m substituted in place
// of wildcards and pos used as the position of tokens from the pattern.
// if m == nil, subst returns a copy of pattern and doesn't change the line
// number information.
func subst(m map[string]reflect.Value, pattern reflect.Value, pos reflect.Value) reflect.Value {
	if pattern == nil {
		return nil
	}

	// Wildcard gets replaced with map value.
	if m != nil && pattern.Type() == identType {
		name := pattern.Interface().(*ast.Ident).Name
		if isWildcard(name) {
			if old, ok := m[name]; ok {
				return subst(nil, old, nil)
			}
		}
	}

	if pos != nil && pattern.Type() == positionType {
		// use new position only if old position was valid in the first place
		if old := pattern.Interface().(token.Pos); !old.IsValid() {
			return pattern
		}
		return pos
	}

	// Otherwise copy.
	switch p := pattern.(type) {
	case *reflect.SliceValue:
		v := reflect.MakeSlice(p.Type().(*reflect.SliceType), p.Len(), p.Len())
		for i := 0; i < p.Len(); i++ {
			v.Elem(i).SetValue(subst(m, p.Elem(i), pos))
		}
		return v

	case *reflect.StructValue:
		v := reflect.MakeZero(p.Type()).(*reflect.StructValue)
		for i := 0; i < p.NumField(); i++ {
			v.Field(i).SetValue(subst(m, p.Field(i), pos))
		}
		return v

	case *reflect.PtrValue:
		v := reflect.MakeZero(p.Type()).(*reflect.PtrValue)
		v.PointTo(subst(m, p.Elem(), pos))
		return v

	case *reflect.InterfaceValue:
		v := reflect.MakeZero(p.Type()).(*reflect.InterfaceValue)
		v.SetValue(subst(m, p.Elem(), pos))
		return v
	}

	return pattern
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Goinstall is an experiment in automatic package installation.
It installs packages, possibly downloading them from the internet.
It maintains a list of public Go packages at http://godashboard.appspot.com/package.

Usage:
	goinstall [flags] importpath...
	goinstall [flags] -a

Flags and default settings:
        -a=false          install all previously installed packages
	-clean=false      clean the package directory before installing
	-dashboard=true   tally public packages on godashboard.appspot.com
	-install=true     build and install the package and its dependencies
	-log=true         log installed packages to $GOROOT/goinstall.log for use by -a
	-nuke=false       remove the target object and clean before installing
	-u=false          update already-downloaded packages
	-v=false          verbose operation

Goinstall installs each of the packages identified on the command line.  It
installs a package's prerequisites before trying to install the package
itself. Unless -log=false is specified, goinstall logs the import path of each
installed package to $GOROOT/goinstall.log for use by goinstall -a.

If the -a flag is given, goinstall reinstalls all previously installed
packages, reading the list from $GOROOT/goinstall.log.  After updating to a
new Go release, which deletes all package binaries, running

	goinstall -a

will recompile and reinstall goinstalled packages.

Another common idiom is to use

	goinstall -a -u

to update, recompile, and reinstall all goinstalled packages.

The source code for a package with import path foo/bar is expected
to be in the directory $GOROOT/src/pkg/foo/bar/.  If the import
path refers to a code hosting site, goinstall will download the code
if necessary.  The recognized code hosting sites are:

	BitBucket (Mercurial)

		import "bitbucket.org/user/project"
		import "bitbucket.org/user/project/sub/directory"

	GitHub (Git)

		import "github.com/user/project"
		import "github.com/user/project/sub/directory"

	Google Code Project Hosting (Mercurial, Subversion)

		import "project.googlecode.com/hg"
		import "project.googlecode.com/hg/sub/directory"

		import "project.googlecode.com/svn/trunk"
		import "project.googlecode.com/svn/trunk/sub/directory"

	Launchpad

		import "launchpad.net/project"
		import "launchpad.net/project/series"
		import "launchpad.net/project/series/sub/directory"

		import "launchpad.net/~user/project/branch"
		import "launchpad.net/~user/project/branch/sub/directory"

If the destination directory (e.g., $GOROOT/src/pkg/bitbucket.org/user/project)
already exists and contains an appropriate checkout, goinstall will not
attempt to fetch updates.  The -u flag changes this behavior,
causing goinstall to update all remote packages encountered during
the installation.

When downloading or updating, goinstall first looks for a tag or branch
named "release".  If there is one, it uses that version of the code.
Otherwise it uses the default version selected by the version control
system, typically HEAD for git, tip for Mercurial.

After a successful download and installation of a publicly accessible
remote package, goinstall reports the installation to godashboard.appspot.com,
which increments a count associated with the package and the time
of its most recent installation.  This mechanism powers the package list
at http://godashboard.appspot.com/package, allowing Go programmers
to learn about popular packages that might be worth looking at.
The -dashboard=false flag disables this reporting.

By default, goinstall prints output only when it encounters an error.
The -v flag causes goinstall to print information about packages
being considered and installed.

Goinstall does not attempt to be a replacement for make.
Instead, it invokes "make install" after locating the package sources.
For local packages without a Makefile and all remote packages,
goinstall creates and uses a temporary Makefile constructed from
the import path and the list of Go files in the package.

The GOPATH Environment Variable

GOPATH may be set to a colon-separated list of paths inside which Go code,
package objects, and executables may be found.

Set a GOPATH to use goinstall to build and install your own code and
external libraries outside of the Go tree (and to avoid writing Makefiles).

The top-level directory structure of a GOPATH is prescribed:

The 'src' directory is for source code. The directory naming inside 'src'
determines the package import path or executable name.

The 'pkg' directory is for package objects. Like the Go tree, package objects
are stored inside a directory named after the target operating system and
processor architecture ('pkg/$GOOS_$GOARCH').
A package whose source is located at '$GOPATH/src/foo/bar' would be imported
as 'foo/bar' and installed as '$GOPATH/pkg/$GOOS_$GOARCH/foo/bar.a'.

The 'bin' directory is for executable files.
Goinstall installs program binaries using the name of the source folder.
A binary whose source is at 'src/foo/qux' would be built and installed to
'$GOPATH/bin/qux'. (Note 'bin/qux', not 'bin/foo/qux' - this is such that
you can put the bin directory in your PATH.)

Here's an example directory layout:

	GOPATH=/home/user/gocode

	/home/user/gocode/
		src/foo/
			bar/               (go code in package bar)
			qux/               (go code in package main)
		bin/qux                    (executable file)
		pkg/linux_amd64/foo/bar.a  (object file)

Run 'goinstall foo/bar' to build and install the package 'foo/bar'
(and its dependencies).
Goinstall will search each GOPATH (in order) for 'src/foo/bar'.
If the directory cannot be found, goinstall will attempt to fetch the
source from a remote repository and write it to the 'src' directory of the
first GOPATH (or $GOROOT/src/pkg if GOPATH is not set).

Goinstall recognizes relative and absolute paths (paths beginning with / or .).
The following commands would build our example packages:

	goinstall /home/user/gocode/src/foo/bar  # build and install foo/bar
	cd /home/user/gocode/src/foo
	goinstall ./bar  # build and install foo/bar (again)
	cd qux
	goinstall .      # build and install foo/qux

*/
package documentation
<commit_msg>goinstall: update doc.go<commit_after>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Goinstall is an experiment in automatic package installation.
It installs packages, possibly downloading them from the internet.
It maintains a list of public Go packages at http://godashboard.appspot.com/package.

Usage:
	goinstall [flags] importpath...
	goinstall [flags] -a

Flags and default settings:
        -a=false          install all previously installed packages
	-clean=false      clean the package directory before installing
	-dashboard=true   tally public packages on godashboard.appspot.com
	-install=true     build and install the package and its dependencies
	-log=true         log installed packages to $GOROOT/goinstall.log for use by -a
	-nuke=false       remove the target object and clean before installing
	-u=false          update already-downloaded packages
	-v=false          verbose operation

Goinstall installs each of the packages identified on the command line.  It
installs a package's prerequisites before trying to install the package
itself. Unless -log=false is specified, goinstall logs the import path of each
installed package to $GOROOT/goinstall.log for use by goinstall -a.

If the -a flag is given, goinstall reinstalls all previously installed
packages, reading the list from $GOROOT/goinstall.log.  After updating to a
new Go release, which deletes all package binaries, running

	goinstall -a

will recompile and reinstall goinstalled packages.

Another common idiom is to use

	goinstall -a -u

to update, recompile, and reinstall all goinstalled packages.

The source code for a package with import path foo/bar is expected
to be in the directory $GOPATH/src/foo/bar/ or $GOROOT/src/pkg/foo/bar/.
(See the discussion of GOPATH below for more detail.)

If the package source is not found locally and the import path begins
with a domain name, goinstall attempts to detect a remote source repository
(Bazaar, Git, Mercurial, or Subversion). If a supported repository is found,
goinstall uses the appropriate tool to download the source code.

If the import path refers to a known code hosting site, goinstall skips the
repository detection and downloads the code directly.
The recognized code hosting sites are:

	BitBucket (Mercurial)

		import "bitbucket.org/user/project"
		import "bitbucket.org/user/project/sub/directory"

	GitHub (Git)

		import "github.com/user/project"
		import "github.com/user/project/sub/directory"

	Google Code Project Hosting (Mercurial, Subversion)

		import "project.googlecode.com/hg"
		import "project.googlecode.com/hg/sub/directory"

		import "project.googlecode.com/svn/trunk"
		import "project.googlecode.com/svn/trunk/sub/directory"

	Launchpad (Bazaar)

		import "launchpad.net/project"
		import "launchpad.net/project/series"
		import "launchpad.net/project/series/sub/directory"

		import "launchpad.net/~user/project/branch"
		import "launchpad.net/~user/project/branch/sub/directory"

If the destination directory (e.g., $GOROOT/src/pkg/bitbucket.org/user/project)
already exists and contains an appropriate checkout, goinstall will not
attempt to fetch updates.  The -u flag changes this behavior,
causing goinstall to update all remote packages encountered during
the installation.

When downloading or updating, goinstall first looks for a tag or branch
named "release".  If there is one, it uses that version of the code.
Otherwise it uses the default version selected by the version control
system, typically HEAD for git, tip for Mercurial.

After a successful download and installation of a publicly accessible
remote package, goinstall reports the installation to godashboard.appspot.com,
which increments a count associated with the package and the time
of its most recent installation. This mechanism powers the package list
at http://godashboard.appspot.com/package, allowing Go programmers
to learn about popular packages that might be worth looking at.
The -dashboard=false flag disables this reporting.

By default, goinstall prints output only when it encounters an error.
The -v flag causes goinstall to print information about packages
being considered and installed.

Goinstall does not use make. Makefiles are ignored by goinstall.

The GOPATH Environment Variable

GOPATH may be set to a colon-separated list of paths inside which Go code,
package objects, and executables may be found.

Set a GOPATH to use goinstall to build and install your own code and
external libraries outside of the Go tree (and to avoid writing Makefiles).

The top-level directory structure of a GOPATH is prescribed:

The 'src' directory is for source code. The directory naming inside 'src'
determines the package import path or executable name.

The 'pkg' directory is for package objects. Like the Go tree, package objects
are stored inside a directory named after the target operating system and
processor architecture ('pkg/$GOOS_$GOARCH').
A package whose source is located at '$GOPATH/src/foo/bar' would be imported
as 'foo/bar' and installed as '$GOPATH/pkg/$GOOS_$GOARCH/foo/bar.a'.

The 'bin' directory is for executable files.
Goinstall installs program binaries using the name of the source folder.
A binary whose source is at 'src/foo/qux' would be built and installed to
'$GOPATH/bin/qux'. (Note 'bin/qux', not 'bin/foo/qux' - this is such that
you can put the bin directory in your PATH.)

Here's an example directory layout:

	GOPATH=/home/user/gocode

	/home/user/gocode/
		src/foo/
			bar/               (go code in package bar)
			qux/               (go code in package main)
		bin/qux                    (executable file)
		pkg/linux_amd64/foo/bar.a  (object file)

Run 'goinstall foo/bar' to build and install the package 'foo/bar'
(and its dependencies).
Goinstall will search each GOPATH (in order) for 'src/foo/bar'.
If the directory cannot be found, goinstall will attempt to fetch the
source from a remote repository and write it to the 'src' directory of the
first GOPATH (or $GOROOT/src/pkg if GOPATH is not set).

Goinstall recognizes relative and absolute paths (paths beginning with / or .).
The following commands would build our example packages:

	goinstall /home/user/gocode/src/foo/bar  # build and install foo/bar
	cd /home/user/gocode/src/foo
	goinstall ./bar  # build and install foo/bar (again)
	cd qux
	goinstall .      # build and install foo/qux

*/
package documentation
<|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build dragonfly freebsd linux netbsd openbsd

package main

import (
	"cmd/internal/sys"
	"debug/elf"
	"fmt"
	"internal/testenv"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"testing"
	"text/template"
)

func getCCAndCCFLAGS(t *testing.T, env []string) (string, []string) {
	goTool := testenv.GoToolPath(t)
	cmd := exec.Command(goTool, "env", "CC")
	cmd.Env = env
	ccb, err := cmd.Output()
	if err != nil {
		t.Fatal(err)
	}
	cc := strings.TrimSpace(string(ccb))

	cmd = exec.Command(goTool, "env", "GOGCCFLAGS")
	cmd.Env = env
	cflagsb, err := cmd.Output()
	if err != nil {
		t.Fatal(err)
	}
	cflags := strings.Fields(string(cflagsb))

	return cc, cflags
}

var asmSource = `
	.section .text1,"ax"
s1:
	.byte 0
	.section .text2,"ax"
s2:
	.byte 0
`

var goSource = `
package main
func main() {}
`

// The linker used to crash if an ELF input file had multiple text sections
// with the same name.
func TestSectionsWithSameName(t *testing.T) {
	testenv.MustHaveGoBuild(t)
	testenv.MustHaveCGO(t)
	t.Parallel()

	objcopy, err := exec.LookPath("objcopy")
	if err != nil {
		t.Skipf("can't find objcopy: %v", err)
	}

	dir, err := ioutil.TempDir("", "go-link-TestSectionsWithSameName")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	gopath := filepath.Join(dir, "GOPATH")
	env := append(os.Environ(), "GOPATH="+gopath)

	if err := ioutil.WriteFile(filepath.Join(dir, "go.mod"), []byte("module elf_test\n"), 0666); err != nil {
		t.Fatal(err)
	}

	asmFile := filepath.Join(dir, "x.s")
	if err := ioutil.WriteFile(asmFile, []byte(asmSource), 0444); err != nil {
		t.Fatal(err)
	}

	goTool := testenv.GoToolPath(t)
	cc, cflags := getCCAndCCFLAGS(t, env)

	asmObj := filepath.Join(dir, "x.o")
	t.Logf("%s %v -c -o %s %s", cc, cflags, asmObj, asmFile)
	if out, err := exec.Command(cc, append(cflags, "-c", "-o", asmObj, asmFile)...).CombinedOutput(); err != nil {
		t.Logf("%s", out)
		t.Fatal(err)
	}

	asm2Obj := filepath.Join(dir, "x2.syso")
	t.Logf("%s --rename-section .text2=.text1 %s %s", objcopy, asmObj, asm2Obj)
	if out, err := exec.Command(objcopy, "--rename-section", ".text2=.text1", asmObj, asm2Obj).CombinedOutput(); err != nil {
		t.Logf("%s", out)
		t.Fatal(err)
	}

	for _, s := range []string{asmFile, asmObj} {
		if err := os.Remove(s); err != nil {
			t.Fatal(err)
		}
	}

	goFile := filepath.Join(dir, "main.go")
	if err := ioutil.WriteFile(goFile, []byte(goSource), 0444); err != nil {
		t.Fatal(err)
	}

	cmd := exec.Command(goTool, "build")
	cmd.Dir = dir
	cmd.Env = env
	t.Logf("%s build", goTool)
	if out, err := cmd.CombinedOutput(); err != nil {
		t.Logf("%s", out)
		t.Fatal(err)
	}
}

var cSources35779 = []string{`
static int blah() { return 42; }
int Cfunc1() { return blah(); }
`, `
static int blah() { return 42; }
int Cfunc2() { return blah(); }
`,
}

// TestMinusRSymsWithSameName tests a corner case in the new
// loader. Prior to the fix this failed with the error 'loadelf:
// $WORK/b001/_pkg_.a(ldr.syso): duplicate symbol reference: blah in
// both main(.text) and main(.text)'. See issue #35779.
func TestMinusRSymsWithSameName(t *testing.T) {
	testenv.MustHaveGoBuild(t)
	testenv.MustHaveCGO(t)
	t.Parallel()

	dir, err := ioutil.TempDir("", "go-link-TestMinusRSymsWithSameName")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	gopath := filepath.Join(dir, "GOPATH")
	env := append(os.Environ(), "GOPATH="+gopath)

	if err := ioutil.WriteFile(filepath.Join(dir, "go.mod"), []byte("module elf_test\n"), 0666); err != nil {
		t.Fatal(err)
	}

	goTool := testenv.GoToolPath(t)
	cc, cflags := getCCAndCCFLAGS(t, env)

	objs := []string{}
	csrcs := []string{}
	for i, content := range cSources35779 {
		csrcFile := filepath.Join(dir, fmt.Sprintf("x%d.c", i))
		csrcs = append(csrcs, csrcFile)
		if err := ioutil.WriteFile(csrcFile, []byte(content), 0444); err != nil {
			t.Fatal(err)
		}

		obj := filepath.Join(dir, fmt.Sprintf("x%d.o", i))
		objs = append(objs, obj)
		t.Logf("%s %v -c -o %s %s", cc, cflags, obj, csrcFile)
		if out, err := exec.Command(cc, append(cflags, "-c", "-o", obj, csrcFile)...).CombinedOutput(); err != nil {
			t.Logf("%s", out)
			t.Fatal(err)
		}
	}

	sysoObj := filepath.Join(dir, "ldr.syso")
	t.Logf("%s %v -nostdlib -r -o %s %v", cc, cflags, sysoObj, objs)
	if out, err := exec.Command(cc, append(cflags, "-nostdlib", "-r", "-o", sysoObj, objs[0], objs[1])...).CombinedOutput(); err != nil {
		t.Logf("%s", out)
		t.Fatal(err)
	}

	cruft := [][]string{objs, csrcs}
	for _, sl := range cruft {
		for _, s := range sl {
			if err := os.Remove(s); err != nil {
				t.Fatal(err)
			}
		}
	}

	goFile := filepath.Join(dir, "main.go")
	if err := ioutil.WriteFile(goFile, []byte(goSource), 0444); err != nil {
		t.Fatal(err)
	}

	t.Logf("%s build", goTool)
	cmd := exec.Command(goTool, "build")
	cmd.Dir = dir
	cmd.Env = env
	if out, err := cmd.CombinedOutput(); err != nil {
		t.Logf("%s", out)
		t.Fatal(err)
	}
}

const pieSourceTemplate = `
package main

import "fmt"

// Force the creation of a lot of type descriptors that will go into
// the .data.rel.ro section.
{{range $index, $element := .}}var V{{$index}} interface{} = [{{$index}}]int{}
{{end}}

func main() {
{{range $index, $element := .}}	fmt.Println(V{{$index}})
{{end}}
}
`

func TestPIESize(t *testing.T) {
	testenv.MustHaveGoBuild(t)
	if !sys.BuildModeSupported(runtime.Compiler, "pie", runtime.GOOS, runtime.GOARCH) {
		t.Skip("-buildmode=pie not supported")
	}

	t.Parallel()

	tmpl := template.Must(template.New("pie").Parse(pieSourceTemplate))

	writeGo := func(t *testing.T, dir string) {
		f, err := os.Create(filepath.Join(dir, "pie.go"))
		if err != nil {
			t.Fatal(err)
		}

		// Passing a 100-element slice here will cause
		// pieSourceTemplate to create 100 variables with
		// different types.
		if err := tmpl.Execute(f, make([]byte, 100)); err != nil {
			t.Fatal(err)
		}

		if err := f.Close(); err != nil {
			t.Fatal(err)
		}
	}

	for _, external := range []bool{false, true} {
		external := external

		name := "TestPieSize-"
		if external {
			name += "external"
		} else {
			name += "internal"
		}
		t.Run(name, func(t *testing.T) {
			t.Parallel()

			dir, err := ioutil.TempDir("", "go-link-"+name)
			if err != nil {
				t.Fatal(err)
			}
			defer os.RemoveAll(dir)

			writeGo(t, dir)

			binexe := filepath.Join(dir, "exe")
			binpie := filepath.Join(dir, "pie")
			if external {
				binexe += "external"
				binpie += "external"
			}

			build := func(bin, mode string) error {
				cmd := exec.Command(testenv.GoToolPath(t), "build", "-o", bin, "-buildmode="+mode)
				if external {
					cmd.Args = append(cmd.Args, "-ldflags=-linkmode=external")
				}
				cmd.Args = append(cmd.Args, "pie.go")
				cmd.Dir = dir
				t.Logf("%v", cmd.Args)
				out, err := cmd.CombinedOutput()
				if len(out) > 0 {
					t.Logf("%s", out)
				}
				if err != nil {
					t.Error(err)
				}
				return err
			}

			var errexe, errpie error
			var wg sync.WaitGroup
			wg.Add(2)
			go func() {
				defer wg.Done()
				errexe = build(binexe, "exe")
			}()
			go func() {
				defer wg.Done()
				errpie = build(binpie, "pie")
			}()
			wg.Wait()
			if errexe != nil || errpie != nil {
				t.Fatal("link failed")
			}

			var sizeexe, sizepie uint64
			if fi, err := os.Stat(binexe); err != nil {
				t.Fatal(err)
			} else {
				sizeexe = uint64(fi.Size())
			}
			if fi, err := os.Stat(binpie); err != nil {
				t.Fatal(err)
			} else {
				sizepie = uint64(fi.Size())
			}

			elfexe, err := elf.Open(binexe)
			if err != nil {
				t.Fatal(err)
			}
			defer elfexe.Close()

			elfpie, err := elf.Open(binpie)
			if err != nil {
				t.Fatal(err)
			}
			defer elfpie.Close()

			// The difference in size between exe and PIE
			// should be approximately the difference in
			// size of the .text section plus the size of
			// the PIE dynamic data sections plus the
			// difference in size of the .got and .plt
			// sections if they exist.
			// We ignore unallocated sections.
			// There may be gaps between non-writeable and
			// writable PT_LOAD segments. We also skip those
			// gaps (see issue #36023).

			textsize := func(ef *elf.File, name string) uint64 {
				for _, s := range ef.Sections {
					if s.Name == ".text" {
						return s.Size
					}
				}
				t.Fatalf("%s: no .text section", name)
				return 0
			}
			textexe := textsize(elfexe, binexe)
			textpie := textsize(elfpie, binpie)

			dynsize := func(ef *elf.File) uint64 {
				var ret uint64
				for _, s := range ef.Sections {
					if s.Flags&elf.SHF_ALLOC == 0 {
						continue
					}
					switch s.Type {
					case elf.SHT_DYNSYM, elf.SHT_STRTAB, elf.SHT_REL, elf.SHT_RELA, elf.SHT_HASH, elf.SHT_GNU_HASH, elf.SHT_GNU_VERDEF, elf.SHT_GNU_VERNEED, elf.SHT_GNU_VERSYM:
						ret += s.Size
					}
					if s.Flags&elf.SHF_WRITE != 0 && (strings.Contains(s.Name, ".got") || strings.Contains(s.Name, ".plt")) {
						ret += s.Size
					}
				}
				return ret
			}

			dynexe := dynsize(elfexe)
			dynpie := dynsize(elfpie)

			extrasize := func(ef *elf.File) uint64 {
				var ret uint64
				// skip unallocated sections
				for _, s := range ef.Sections {
					if s.Flags&elf.SHF_ALLOC == 0 {
						ret += s.Size
					}
				}
				// also skip gaps between PT_LOAD segments
				var prev *elf.Prog
				for _, seg := range ef.Progs {
					if seg.Type != elf.PT_LOAD {
						continue
					}
					if prev != nil {
						ret += seg.Off - prev.Off - prev.Filesz
					}
					prev = seg
				}
				return ret
			}

			extraexe := extrasize(elfexe)
			extrapie := extrasize(elfpie)

			diffReal := (sizepie - extrapie) - (sizeexe - extraexe)
			diffExpected := (textpie + dynpie) - (textexe + dynexe)

			t.Logf("real size difference %#x, expected %#x", diffReal, diffExpected)

			if diffReal > (diffExpected + diffExpected/10) {
				t.Errorf("PIE unexpectedly large: got difference of %d (%d - %d), expected difference %d", diffReal, sizepie, sizeexe, diffExpected)
			}
		})
	}
}
<commit_msg>cmd/link: disable TestPIESize if CGO isn't enabled<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build dragonfly freebsd linux netbsd openbsd

package main

import (
	"cmd/internal/sys"
	"debug/elf"
	"fmt"
	"internal/testenv"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"testing"
	"text/template"
)

func getCCAndCCFLAGS(t *testing.T, env []string) (string, []string) {
	goTool := testenv.GoToolPath(t)
	cmd := exec.Command(goTool, "env", "CC")
	cmd.Env = env
	ccb, err := cmd.Output()
	if err != nil {
		t.Fatal(err)
	}
	cc := strings.TrimSpace(string(ccb))

	cmd = exec.Command(goTool, "env", "GOGCCFLAGS")
	cmd.Env = env
	cflagsb, err := cmd.Output()
	if err != nil {
		t.Fatal(err)
	}
	cflags := strings.Fields(string(cflagsb))

	return cc, cflags
}

var asmSource = `
	.section .text1,"ax"
s1:
	.byte 0
	.section .text2,"ax"
s2:
	.byte 0
`

var goSource = `
package main
func main() {}
`

// The linker used to crash if an ELF input file had multiple text sections
// with the same name.
func TestSectionsWithSameName(t *testing.T) {
	testenv.MustHaveGoBuild(t)
	testenv.MustHaveCGO(t)
	t.Parallel()

	objcopy, err := exec.LookPath("objcopy")
	if err != nil {
		t.Skipf("can't find objcopy: %v", err)
	}

	dir, err := ioutil.TempDir("", "go-link-TestSectionsWithSameName")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	gopath := filepath.Join(dir, "GOPATH")
	env := append(os.Environ(), "GOPATH="+gopath)

	if err := ioutil.WriteFile(filepath.Join(dir, "go.mod"), []byte("module elf_test\n"), 0666); err != nil {
		t.Fatal(err)
	}

	asmFile := filepath.Join(dir, "x.s")
	if err := ioutil.WriteFile(asmFile, []byte(asmSource), 0444); err != nil {
		t.Fatal(err)
	}

	goTool := testenv.GoToolPath(t)
	cc, cflags := getCCAndCCFLAGS(t, env)

	asmObj := filepath.Join(dir, "x.o")
	t.Logf("%s %v -c -o %s %s", cc, cflags, asmObj, asmFile)
	if out, err := exec.Command(cc, append(cflags, "-c", "-o", asmObj, asmFile)...).CombinedOutput(); err != nil {
		t.Logf("%s", out)
		t.Fatal(err)
	}

	asm2Obj := filepath.Join(dir, "x2.syso")
	t.Logf("%s --rename-section .text2=.text1 %s %s", objcopy, asmObj, asm2Obj)
	if out, err := exec.Command(objcopy, "--rename-section", ".text2=.text1", asmObj, asm2Obj).CombinedOutput(); err != nil {
		t.Logf("%s", out)
		t.Fatal(err)
	}

	for _, s := range []string{asmFile, asmObj} {
		if err := os.Remove(s); err != nil {
			t.Fatal(err)
		}
	}

	goFile := filepath.Join(dir, "main.go")
	if err := ioutil.WriteFile(goFile, []byte(goSource), 0444); err != nil {
		t.Fatal(err)
	}

	cmd := exec.Command(goTool, "build")
	cmd.Dir = dir
	cmd.Env = env
	t.Logf("%s build", goTool)
	if out, err := cmd.CombinedOutput(); err != nil {
		t.Logf("%s", out)
		t.Fatal(err)
	}
}

var cSources35779 = []string{`
static int blah() { return 42; }
int Cfunc1() { return blah(); }
`, `
static int blah() { return 42; }
int Cfunc2() { return blah(); }
`,
}

// TestMinusRSymsWithSameName tests a corner case in the new
// loader. Prior to the fix this failed with the error 'loadelf:
// $WORK/b001/_pkg_.a(ldr.syso): duplicate symbol reference: blah in
// both main(.text) and main(.text)'. See issue #35779.
func TestMinusRSymsWithSameName(t *testing.T) {
	testenv.MustHaveGoBuild(t)
	testenv.MustHaveCGO(t)
	t.Parallel()

	dir, err := ioutil.TempDir("", "go-link-TestMinusRSymsWithSameName")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	gopath := filepath.Join(dir, "GOPATH")
	env := append(os.Environ(), "GOPATH="+gopath)

	if err := ioutil.WriteFile(filepath.Join(dir, "go.mod"), []byte("module elf_test\n"), 0666); err != nil {
		t.Fatal(err)
	}

	goTool := testenv.GoToolPath(t)
	cc, cflags := getCCAndCCFLAGS(t, env)

	objs := []string{}
	csrcs := []string{}
	for i, content := range cSources35779 {
		csrcFile := filepath.Join(dir, fmt.Sprintf("x%d.c", i))
		csrcs = append(csrcs, csrcFile)
		if err := ioutil.WriteFile(csrcFile, []byte(content), 0444); err != nil {
			t.Fatal(err)
		}

		obj := filepath.Join(dir, fmt.Sprintf("x%d.o", i))
		objs = append(objs, obj)
		t.Logf("%s %v -c -o %s %s", cc, cflags, obj, csrcFile)
		if out, err := exec.Command(cc, append(cflags, "-c", "-o", obj, csrcFile)...).CombinedOutput(); err != nil {
			t.Logf("%s", out)
			t.Fatal(err)
		}
	}

	sysoObj := filepath.Join(dir, "ldr.syso")
	t.Logf("%s %v -nostdlib -r -o %s %v", cc, cflags, sysoObj, objs)
	if out, err := exec.Command(cc, append(cflags, "-nostdlib", "-r", "-o", sysoObj, objs[0], objs[1])...).CombinedOutput(); err != nil {
		t.Logf("%s", out)
		t.Fatal(err)
	}

	cruft := [][]string{objs, csrcs}
	for _, sl := range cruft {
		for _, s := range sl {
			if err := os.Remove(s); err != nil {
				t.Fatal(err)
			}
		}
	}

	goFile := filepath.Join(dir, "main.go")
	if err := ioutil.WriteFile(goFile, []byte(goSource), 0444); err != nil {
		t.Fatal(err)
	}

	t.Logf("%s build", goTool)
	cmd := exec.Command(goTool, "build")
	cmd.Dir = dir
	cmd.Env = env
	if out, err := cmd.CombinedOutput(); err != nil {
		t.Logf("%s", out)
		t.Fatal(err)
	}
}

const pieSourceTemplate = `
package main

import "fmt"

// Force the creation of a lot of type descriptors that will go into
// the .data.rel.ro section.
{{range $index, $element := .}}var V{{$index}} interface{} = [{{$index}}]int{}
{{end}}

func main() {
{{range $index, $element := .}}	fmt.Println(V{{$index}})
{{end}}
}
`

func TestPIESize(t *testing.T) {
	testenv.MustHaveGoBuild(t)

	// We don't want to test -linkmode=external if cgo is not supported.
	// On some systems -buildmode=pie implies -linkmode=external, so just
	// always skip the test if cgo is not supported.
	testenv.MustHaveCGO(t)

	if !sys.BuildModeSupported(runtime.Compiler, "pie", runtime.GOOS, runtime.GOARCH) {
		t.Skip("-buildmode=pie not supported")
	}

	t.Parallel()

	tmpl := template.Must(template.New("pie").Parse(pieSourceTemplate))

	writeGo := func(t *testing.T, dir string) {
		f, err := os.Create(filepath.Join(dir, "pie.go"))
		if err != nil {
			t.Fatal(err)
		}

		// Passing a 100-element slice here will cause
		// pieSourceTemplate to create 100 variables with
		// different types.
		if err := tmpl.Execute(f, make([]byte, 100)); err != nil {
			t.Fatal(err)
		}

		if err := f.Close(); err != nil {
			t.Fatal(err)
		}
	}

	for _, external := range []bool{false, true} {
		external := external

		name := "TestPieSize-"
		if external {
			name += "external"
		} else {
			name += "internal"
		}
		t.Run(name, func(t *testing.T) {
			t.Parallel()

			dir, err := ioutil.TempDir("", "go-link-"+name)
			if err != nil {
				t.Fatal(err)
			}
			defer os.RemoveAll(dir)

			writeGo(t, dir)

			binexe := filepath.Join(dir, "exe")
			binpie := filepath.Join(dir, "pie")
			if external {
				binexe += "external"
				binpie += "external"
			}

			build := func(bin, mode string) error {
				cmd := exec.Command(testenv.GoToolPath(t), "build", "-o", bin, "-buildmode="+mode)
				if external {
					cmd.Args = append(cmd.Args, "-ldflags=-linkmode=external")
				}
				cmd.Args = append(cmd.Args, "pie.go")
				cmd.Dir = dir
				t.Logf("%v", cmd.Args)
				out, err := cmd.CombinedOutput()
				if len(out) > 0 {
					t.Logf("%s", out)
				}
				if err != nil {
					t.Error(err)
				}
				return err
			}

			var errexe, errpie error
			var wg sync.WaitGroup
			wg.Add(2)
			go func() {
				defer wg.Done()
				errexe = build(binexe, "exe")
			}()
			go func() {
				defer wg.Done()
				errpie = build(binpie, "pie")
			}()
			wg.Wait()
			if errexe != nil || errpie != nil {
				t.Fatal("link failed")
			}

			var sizeexe, sizepie uint64
			if fi, err := os.Stat(binexe); err != nil {
				t.Fatal(err)
			} else {
				sizeexe = uint64(fi.Size())
			}
			if fi, err := os.Stat(binpie); err != nil {
				t.Fatal(err)
			} else {
				sizepie = uint64(fi.Size())
			}

			elfexe, err := elf.Open(binexe)
			if err != nil {
				t.Fatal(err)
			}
			defer elfexe.Close()

			elfpie, err := elf.Open(binpie)
			if err != nil {
				t.Fatal(err)
			}
			defer elfpie.Close()

			// The difference in size between exe and PIE
			// should be approximately the difference in
			// size of the .text section plus the size of
			// the PIE dynamic data sections plus the
			// difference in size of the .got and .plt
			// sections if they exist.
			// We ignore unallocated sections.
			// There may be gaps between non-writeable and
			// writable PT_LOAD segments. We also skip those
			// gaps (see issue #36023).

			textsize := func(ef *elf.File, name string) uint64 {
				for _, s := range ef.Sections {
					if s.Name == ".text" {
						return s.Size
					}
				}
				t.Fatalf("%s: no .text section", name)
				return 0
			}
			textexe := textsize(elfexe, binexe)
			textpie := textsize(elfpie, binpie)

			dynsize := func(ef *elf.File) uint64 {
				var ret uint64
				for _, s := range ef.Sections {
					if s.Flags&elf.SHF_ALLOC == 0 {
						continue
					}
					switch s.Type {
					case elf.SHT_DYNSYM, elf.SHT_STRTAB, elf.SHT_REL, elf.SHT_RELA, elf.SHT_HASH, elf.SHT_GNU_HASH, elf.SHT_GNU_VERDEF, elf.SHT_GNU_VERNEED, elf.SHT_GNU_VERSYM:
						ret += s.Size
					}
					if s.Flags&elf.SHF_WRITE != 0 && (strings.Contains(s.Name, ".got") || strings.Contains(s.Name, ".plt")) {
						ret += s.Size
					}
				}
				return ret
			}

			dynexe := dynsize(elfexe)
			dynpie := dynsize(elfpie)

			extrasize := func(ef *elf.File) uint64 {
				var ret uint64
				// skip unallocated sections
				for _, s := range ef.Sections {
					if s.Flags&elf.SHF_ALLOC == 0 {
						ret += s.Size
					}
				}
				// also skip gaps between PT_LOAD segments
				var prev *elf.Prog
				for _, seg := range ef.Progs {
					if seg.Type != elf.PT_LOAD {
						continue
					}
					if prev != nil {
						ret += seg.Off - prev.Off - prev.Filesz
					}
					prev = seg
				}
				return ret
			}

			extraexe := extrasize(elfexe)
			extrapie := extrasize(elfpie)

			diffReal := (sizepie - extrapie) - (sizeexe - extraexe)
			diffExpected := (textpie + dynpie) - (textexe + dynexe)

			t.Logf("real size difference %#x, expected %#x", diffReal, diffExpected)

			if diffReal > (diffExpected + diffExpected/10) {
				t.Errorf("PIE unexpectedly large: got difference of %d (%d - %d), expected difference %d", diffReal, sizepie, sizeexe, diffExpected)
			}
		})
	}
}
<|endoftext|>


<commit_before>package main

import (
	"model"
	"runner"
	"runner/sshrunner"

	"fmt"

	"util"

	"github.com/codegangsta/cli"
)

var (
	// ErrCmdRequired require cmd option
	ErrCmdRequired = fmt.Errorf("option -c/--cmd is required")
	// ErrNoNodeToExec no more node to execute
	ErrNoNodeToExec = fmt.Errorf("found no node to execute")
)

type execParams struct {
	GroupName string
	NodeName  string
	User      string
	Cmd       string
}

func initExecSubCmd(app *cli.App) {
	execSubCmd := cli.Command{
		Name:        "exec",
		Usage:       "exec <options>",
		Description: "exec command on groups or nodes",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "g,group",
				Value: "*",
				Usage: "exec command on group",
			},
			cli.StringFlag{
				Name:  "n,node",
				Value: "",
				Usage: "exec command on node",
			},
			cli.StringFlag{
				Name:  "u,user",
				Value: "root",
				Usage: "user who exec the command",
			},
			cli.StringFlag{
				Name:  "c,cmd",
				Value: "",
				Usage: "command for exec",
			},
		},
		Action: func(c *cli.Context) {
			var ep, err = checkExecParams(c)
			if err != nil {
				fmt.Println(util.FgRed(err))
				cli.ShowCommandHelp(c, "exec")
				return
			}
			if err = execCmd(ep); err != nil {
				fmt.Println(util.FgRed(err))
			}
		},
	}

	if app.Commands == nil {
		app.Commands = cli.Commands{execSubCmd}
	} else {
		app.Commands = append(app.Commands, execSubCmd)
	}
}

func checkExecParams(c *cli.Context) (execParams, error) {
	var ep = execParams{
		GroupName: c.String("group"),
		NodeName:  c.String("node"),
		User:      c.String("user"),
		Cmd:       c.String("cmd"),
	}

	if ep.Cmd == "" {
		return ep, ErrCmdRequired
	}

	return ep, nil
}

func execCmd(ep execParams) error {
	// TODO should use sshrunner from config

	// get node info for exec
	repo := GetRepo()
	var nodes, err = repo.FilterNodes(ep.GroupName, ep.NodeName)
	if err != nil {
		return err
	}

	if len(nodes) == 0 {
		return ErrNoNodeToExec
	}

	if !confirmExec(nodes, ep.User, ep.Cmd) {
		return nil
	}

	// exec cmd on node
	for _, n := range nodes {
		fmt.Printf("Start to excute \"%s\" on %s(%s):\n", util.FgBoldGreen(ep.Cmd), util.FgBoldGreen(n.Name), util.FgBoldGreen(n.Host))
		var runCmd = sshrunner.New(n.User, n.Password, n.KeyPath, n.Host, n.Port)
		var input = runner.Input{
			ExecHost: n.Host,
			ExecUser: ep.User,
			Command:  ep.Cmd,
		}

		// display result
		output, err := runCmd.SyncExec(input)
		displayExecResult(output, err)
	}
	return nil
}

func displayExecResult(output *runner.Output, err error) {
	if err != nil {
		fmt.Printf("Command exec failed: %s\n", util.FgRed(err))
	}

	if output == nil {
		return
	}

	fmt.Printf(">>>>>>>>>>>>>>>>>>>> STDOUT >>>>>>>>>>>>>>>>>>>>\n%s\n", output.StdOutput)
	if output.StdError != "" {
		fmt.Printf(">>>>>>>>>>>>>>>>>>>> STDERR >>>>>>>>>>>>>>>>>>>>\n%s\n", output.StdError)
	}
	fmt.Printf("time costs: %v\n", output.ExecEnd.Sub(output.ExecStart))
	fmt.Println(util.FgBoldBlue("==========================================================\n"))
}

func confirmExec(nodes []model.Node, user, cmd string) bool {
	fmt.Printf("%-3s\t%-10s\t%-10s\n", "No.", "Name", "IP")
	fmt.Println("----------------------------------------------------------------------")
	for index, n := range nodes {
		fmt.Printf("%-3d\t%-10s\t%-10s\n", index+1, n.Name, n.Host)
	}

	fmt.Println()
	return util.Confirm(fmt.Sprintf("You want to exec COMMAND(%s) by UESR(%s) at the above nodes, yes/no(y/n) ?",
		util.FgBoldRed(cmd), util.FgBoldRed(user)))
}
<commit_msg>add -y option for subcommand exec<commit_after>package main

import (
	"model"
	"runner"
	"runner/sshrunner"

	"fmt"

	"util"

	"github.com/codegangsta/cli"
)

var (
	// ErrCmdRequired require cmd option
	ErrCmdRequired = fmt.Errorf("option -c/--cmd is required")
	// ErrNoNodeToExec no more node to execute
	ErrNoNodeToExec = fmt.Errorf("found no node to execute")
)

type execParams struct {
	GroupName string
	NodeName  string
	User      string
	Cmd       string
	Yes       bool
}

func initExecSubCmd(app *cli.App) {
	execSubCmd := cli.Command{
		Name:        "exec",
		Usage:       "exec <options>",
		Description: "exec command on groups or nodes",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "g,group",
				Value: "*",
				Usage: "exec command on group",
			},
			cli.StringFlag{
				Name:  "n,node",
				Value: "",
				Usage: "exec command on node",
			},
			cli.StringFlag{
				Name:  "u,user",
				Value: "root",
				Usage: "user who exec the command",
			},
			cli.StringFlag{
				Name:  "c,cmd",
				Value: "",
				Usage: "command for exec",
			},
			cli.BoolFlag{
				Name:  "y,yes",
				Usage: "excute command without confirm",
			},
		},
		Action: func(c *cli.Context) {
			var ep, err = checkExecParams(c)
			if err != nil {
				fmt.Println(util.FgRed(err))
				cli.ShowCommandHelp(c, "exec")
				return
			}
			if err = execCmd(ep); err != nil {
				fmt.Println(util.FgRed(err))
			}
		},
	}

	if app.Commands == nil {
		app.Commands = cli.Commands{execSubCmd}
	} else {
		app.Commands = append(app.Commands, execSubCmd)
	}
}

func checkExecParams(c *cli.Context) (execParams, error) {
	var ep = execParams{
		GroupName: c.String("group"),
		NodeName:  c.String("node"),
		User:      c.String("user"),
		Cmd:       c.String("cmd"),
		Yes:       c.Bool("yes"),
	}

	if ep.Cmd == "" {
		return ep, ErrCmdRequired
	}

	return ep, nil
}

func execCmd(ep execParams) error {
	// TODO should use sshrunner from config

	// get node info for exec
	repo := GetRepo()
	var nodes, err = repo.FilterNodes(ep.GroupName, ep.NodeName)
	if err != nil {
		return err
	}

	if len(nodes) == 0 {
		return ErrNoNodeToExec
	}

	if !ep.Yes && !confirmExec(nodes, ep.User, ep.Cmd) {
		return nil
	}

	// exec cmd on node
	for _, n := range nodes {
		fmt.Printf("Start to excute \"%s\" on %s(%s):\n", util.FgBoldGreen(ep.Cmd), util.FgBoldGreen(n.Name), util.FgBoldGreen(n.Host))
		var runCmd = sshrunner.New(n.User, n.Password, n.KeyPath, n.Host, n.Port)
		var input = runner.Input{
			ExecHost: n.Host,
			ExecUser: ep.User,
			Command:  ep.Cmd,
		}

		// display result
		output, err := runCmd.SyncExec(input)
		displayExecResult(output, err)
	}
	return nil
}

func displayExecResult(output *runner.Output, err error) {
	if err != nil {
		fmt.Printf("Command exec failed: %s\n", util.FgRed(err))
	}

	if output == nil {
		return
	}

	fmt.Printf(">>>>>>>>>>>>>>>>>>>> STDOUT >>>>>>>>>>>>>>>>>>>>\n%s\n", output.StdOutput)
	if output.StdError != "" {
		fmt.Printf(">>>>>>>>>>>>>>>>>>>> STDERR >>>>>>>>>>>>>>>>>>>>\n%s\n", output.StdError)
	}
	fmt.Printf("time costs: %v\n", output.ExecEnd.Sub(output.ExecStart))
	fmt.Println(util.FgBoldBlue("==========================================================\n"))
}

func confirmExec(nodes []model.Node, user, cmd string) bool {
	fmt.Printf("%-3s\t%-10s\t%-10s\n", "No.", "Name", "IP")
	fmt.Println("----------------------------------------------------------------------")
	for index, n := range nodes {
		fmt.Printf("%-3d\t%-10s\t%-10s\n", index+1, n.Name, n.Host)
	}

	fmt.Println()
	return util.Confirm(fmt.Sprintf("You want to exec COMMAND(%s) by UESR(%s) at the above nodes, yes/no(y/n) ?",
		util.FgBoldRed(cmd), util.FgBoldRed(user)))
}
<|endoftext|>