<commit_before>// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package stun

import (
	"net"
	"sync/atomic"
	"time"

	"github.com/AudriusButkevicius/pfilter"
	"github.com/ccding/go-stun/stun"
	"github.com/thejerf/suture"

	"github.com/syncthing/syncthing/lib/config"
	"github.com/syncthing/syncthing/lib/util"
)

const stunRetryInterval = 5 * time.Minute

type Host = stun.Host
type NATType = stun.NATType

// NAT types.

const (
	NATError                = stun.NATError
	NATUnknown              = stun.NATUnknown
	NATNone                 = stun.NATNone
	NATBlocked              = stun.NATBlocked
	NATFull                 = stun.NATFull
	NATSymmetric            = stun.NATSymmetric
	NATRestricted           = stun.NATRestricted
	NATPortRestricted       = stun.NATPortRestricted
	NATSymmetricUDPFirewall = stun.NATSymmetricUDPFirewall
)

type writeTrackingPacketConn struct {
	lastWrite int64 // atomic, must remain 64-bit aligned
	net.PacketConn
}

func (c *writeTrackingPacketConn) WriteTo(p []byte, addr net.Addr) (n int, err error) {
	atomic.StoreInt64(&c.lastWrite, time.Now().Unix())
	return c.PacketConn.WriteTo(p, addr)
}

func (c *writeTrackingPacketConn) getLastWrite() time.Time {
	unix := atomic.LoadInt64(&c.lastWrite)
	return time.Unix(unix, 0)
}

type Subscriber interface {
	OnNATTypeChanged(natType NATType)
	OnExternalAddressChanged(address *Host, via string)
}

type Service struct {
	suture.Service

	name       string
	cfg        config.Wrapper
	subscriber Subscriber
	stunConn   net.PacketConn
	client     *stun.Client

	writeTrackingPacketConn *writeTrackingPacketConn

	natType NATType
	addr    *Host
}

func New(cfg config.Wrapper, subscriber Subscriber, conn net.PacketConn) (*Service, net.PacketConn) {
	// Wrap the original connection to track writes on it
	writeTrackingPacketConn := &writeTrackingPacketConn{lastWrite: 0, PacketConn: conn}

	// Wrap it in a filter and split it up, so that stun packets arrive on stun conn, others arrive on the data conn
	filterConn := pfilter.NewPacketFilter(writeTrackingPacketConn)
	otherDataConn := filterConn.NewConn(otherDataPriority, nil)
	stunConn := filterConn.NewConn(stunFilterPriority, &stunFilter{
		ids: make(map[string]time.Time),
	})

	filterConn.Start()

	// Construct the client to use the stun conn
	client := stun.NewClientWithConnection(stunConn)
	client.SetSoftwareName("") // Explicitly unset this, seems to freak some servers out.

	// Return the service and the other conn to the client
	s := &Service{
		name: "Stun@" + conn.LocalAddr().Network() + "://" + conn.LocalAddr().String(),

		cfg:        cfg,
		subscriber: subscriber,
		stunConn:   stunConn,
		client:     client,

		writeTrackingPacketConn: writeTrackingPacketConn,

		natType: NATUnknown,
		addr:    nil,
	}
	s.Service = util.AsService(s.serve)
	return s, otherDataConn
}

func (s *Service) Stop() {
	_ = s.stunConn.Close()
	s.Service.Stop()
}

func (s *Service) serve(stop chan struct{}) {
	for {
	disabled:
		s.setNATType(NATUnknown)
		s.setExternalAddress(nil, "")

		if s.cfg.Options().IsStunDisabled() {
			select {
			case <-stop:
				return
			case <-time.After(time.Second):
				continue
			}
		}

		l.Debugf("Starting stun for %s", s)

		for _, addr := range s.cfg.StunServers() {
			// This blocks until we hit an exit condition or there are issues with the STUN server.
			// This returns a boolean signifying if a different STUN server should be tried (oppose to the whole thing
			// shutting down and this winding itself down.
			if !s.runStunForServer(addr, stop) {
				// Check exit conditions.

				// Have we been asked to stop?
				select {
				case <-stop:
					return
				default:
				}

				// Are we disabled?
				if s.cfg.Options().IsStunDisabled() {
					l.Infoln("STUN disabled")
					goto disabled
				}

				// Unpunchable NAT? Chillout for some time.
				if !s.isCurrentNATTypePunchable() {
					break
				}
			}
		}

		// Failed all servers, sad.
		s.setNATType(NATUnknown)
		s.setExternalAddress(nil, "")

		// We failed to contact all provided stun servers or the nat is not punchable.
		// Chillout for a while.
		select {
		case <-time.After(stunRetryInterval):
		case <-stop:
			return
		}
	}
}

func (s *Service) runStunForServer(addr string, stop chan struct{}) (tryNext bool) {
	l.Debugf("Running stun for %s via %s", s, addr)

	// Resolve the address, so that in case the server advertises two
	// IPs, we always hit the same one, as otherwise, the mapping might
	// expire as we hit the other address, and cause us to flip flop
	// between servers/external addresses, as a result flooding discovery
	// servers.
	udpAddr, err := net.ResolveUDPAddr("udp", addr)
	if err != nil {
		l.Debugf("%s stun addr resolution on %s: %s", s, addr, err)
		return true
	}
	s.client.SetServerAddr(udpAddr.String())

	natType, extAddr, err := s.client.Discover()
	if err != nil || extAddr == nil {
		l.Debugf("%s stun discovery on %s: %s", s, addr, err)
		return true
	}

	// The stun server is most likely borked, try another one.
	if natType == NATError || natType == NATUnknown || natType == NATBlocked {
		l.Debugf("%s stun discovery on %s resolved to %s", s, addr, natType)
		return true
	}

	s.setNATType(natType)
	s.setExternalAddress(extAddr, addr)
	l.Debugf("%s detected NAT type: %s via %s", s, natType, addr)

	// We can't punch through this one, so no point doing keepalives
	// and such, just let the caller check the nat type and work it out themselves.
	if !s.isCurrentNATTypePunchable() {
		l.Debugf("%s cannot punch %s, skipping", s, natType)
		return false
	}

	return s.stunKeepAlive(addr, extAddr, stop)
}

func (s *Service) stunKeepAlive(addr string, extAddr *Host, stop chan struct{}) (tryNext bool) {
	var err error
	nextSleep := time.Duration(s.cfg.Options().StunKeepaliveStartS) * time.Second

	l.Debugf("%s starting stun keepalive via %s, next sleep %s", s, addr, nextSleep)

	for {
		if areDifferent(s.addr, extAddr) {
			// If the port has changed (addresses are not equal but the hosts are equal),
			// we're probably spending too much time between keepalives, reduce the sleep.
			if s.addr != nil && extAddr != nil && s.addr.IP() == extAddr.IP() {
				nextSleep /= 2
				l.Debugf("%s stun port change (%s to %s), next sleep %s", s, s.addr.TransportAddr(), extAddr.TransportAddr(), nextSleep)
			}

			s.setExternalAddress(extAddr, addr)

			// The stun server is probably stuffed, we've gone beyond min timeout, yet the address keeps changing.
			minSleep := time.Duration(s.cfg.Options().StunKeepaliveMinS) * time.Second
			if nextSleep < minSleep {
				l.Debugf("%s keepalive aborting, sleep below min: %s < %s", s, nextSleep, minSleep)
				return true
			}
		}

		// Adjust the keepalives to fire only nextSleep after last write.
		lastWrite := s.writeTrackingPacketConn.getLastWrite()
		minSleep := time.Duration(s.cfg.Options().StunKeepaliveMinS) * time.Second
		if nextSleep < minSleep {
			nextSleep = minSleep
		}
	tryLater:
		sleepFor := nextSleep

		timeUntilNextKeepalive := time.Until(lastWrite.Add(sleepFor))
		if timeUntilNextKeepalive > 0 {
			sleepFor = timeUntilNextKeepalive
		}

		l.Debugf("%s stun sleeping for %s", s, sleepFor)

		select {
		case <-time.After(sleepFor):
		case <-stop:
			l.Debugf("%s stopping, aborting stun", s)
			return false
		}

		if s.cfg.Options().IsStunDisabled() {
			// Disabled, give up
			l.Debugf("%s disabled, aborting stun ", s)
			return false
		}

		// Check if any writes happened while we were sleeping, if they did, sleep again
		lastWrite = s.writeTrackingPacketConn.getLastWrite()
		if gap := time.Since(lastWrite); gap < nextSleep {
			l.Debugf("%s stun last write gap less than next sleep: %s < %s. Will try later", s, gap, nextSleep)
			goto tryLater
		}

		l.Debugf("%s stun keepalive", s)

		extAddr, err = s.client.Keepalive()
		if err != nil {
			l.Debugf("%s stun keepalive on %s: %s (%v)", s, addr, err, extAddr)
			return true
		}
	}
}

func (s *Service) setNATType(natType NATType) {
	if natType != s.natType {
		l.Debugf("Notifying %s of NAT type change: %s", s.subscriber, natType)
		s.subscriber.OnNATTypeChanged(natType)
	}
	s.natType = natType
}

func (s *Service) setExternalAddress(addr *Host, via string) {
	if areDifferent(s.addr, addr) {
		l.Debugf("Notifying %s of address change: %s via %s", s.subscriber, addr, via)
		s.subscriber.OnExternalAddressChanged(addr, via)
	}
	s.addr = addr
}

func (s *Service) String() string {
	return s.name
}

func (s *Service) isCurrentNATTypePunchable() bool {
	return s.natType == NATNone || s.natType == NATPortRestricted || s.natType == NATRestricted || s.natType == NATFull
}

func areDifferent(first, second *Host) bool {
	if (first == nil) != (second == nil) {
		return true
	}
	if first != nil {
		return first.TransportAddr() != second.TransportAddr()
	}
	return false
}
<commit_msg>Add NATSymmetricUDPFirewall to punchable NATs (#5931)<commit_after>// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package stun

import (
	"net"
	"sync/atomic"
	"time"

	"github.com/AudriusButkevicius/pfilter"
	"github.com/ccding/go-stun/stun"
	"github.com/thejerf/suture"

	"github.com/syncthing/syncthing/lib/config"
	"github.com/syncthing/syncthing/lib/util"
)

const stunRetryInterval = 5 * time.Minute

type Host = stun.Host
type NATType = stun.NATType

// NAT types.

const (
	NATError                = stun.NATError
	NATUnknown              = stun.NATUnknown
	NATNone                 = stun.NATNone
	NATBlocked              = stun.NATBlocked
	NATFull                 = stun.NATFull
	NATSymmetric            = stun.NATSymmetric
	NATRestricted           = stun.NATRestricted
	NATPortRestricted       = stun.NATPortRestricted
	NATSymmetricUDPFirewall = stun.NATSymmetricUDPFirewall
)

type writeTrackingPacketConn struct {
	lastWrite int64 // atomic, must remain 64-bit aligned
	net.PacketConn
}

func (c *writeTrackingPacketConn) WriteTo(p []byte, addr net.Addr) (n int, err error) {
	atomic.StoreInt64(&c.lastWrite, time.Now().Unix())
	return c.PacketConn.WriteTo(p, addr)
}

func (c *writeTrackingPacketConn) getLastWrite() time.Time {
	unix := atomic.LoadInt64(&c.lastWrite)
	return time.Unix(unix, 0)
}

type Subscriber interface {
	OnNATTypeChanged(natType NATType)
	OnExternalAddressChanged(address *Host, via string)
}

type Service struct {
	suture.Service

	name       string
	cfg        config.Wrapper
	subscriber Subscriber
	stunConn   net.PacketConn
	client     *stun.Client

	writeTrackingPacketConn *writeTrackingPacketConn

	natType NATType
	addr    *Host
}

func New(cfg config.Wrapper, subscriber Subscriber, conn net.PacketConn) (*Service, net.PacketConn) {
	// Wrap the original connection to track writes on it
	writeTrackingPacketConn := &writeTrackingPacketConn{lastWrite: 0, PacketConn: conn}

	// Wrap it in a filter and split it up, so that stun packets arrive on stun conn, others arrive on the data conn
	filterConn := pfilter.NewPacketFilter(writeTrackingPacketConn)
	otherDataConn := filterConn.NewConn(otherDataPriority, nil)
	stunConn := filterConn.NewConn(stunFilterPriority, &stunFilter{
		ids: make(map[string]time.Time),
	})

	filterConn.Start()

	// Construct the client to use the stun conn
	client := stun.NewClientWithConnection(stunConn)
	client.SetSoftwareName("") // Explicitly unset this, seems to freak some servers out.

	// Return the service and the other conn to the client
	s := &Service{
		name: "Stun@" + conn.LocalAddr().Network() + "://" + conn.LocalAddr().String(),

		cfg:        cfg,
		subscriber: subscriber,
		stunConn:   stunConn,
		client:     client,

		writeTrackingPacketConn: writeTrackingPacketConn,

		natType: NATUnknown,
		addr:    nil,
	}
	s.Service = util.AsService(s.serve)
	return s, otherDataConn
}

func (s *Service) Stop() {
	_ = s.stunConn.Close()
	s.Service.Stop()
}

func (s *Service) serve(stop chan struct{}) {
	for {
	disabled:
		s.setNATType(NATUnknown)
		s.setExternalAddress(nil, "")

		if s.cfg.Options().IsStunDisabled() {
			select {
			case <-stop:
				return
			case <-time.After(time.Second):
				continue
			}
		}

		l.Debugf("Starting stun for %s", s)

		for _, addr := range s.cfg.StunServers() {
			// This blocks until we hit an exit condition or there are issues with the STUN server.
			// This returns a boolean signifying if a different STUN server should be tried (oppose to the whole thing
			// shutting down and this winding itself down.
			if !s.runStunForServer(addr, stop) {
				// Check exit conditions.

				// Have we been asked to stop?
				select {
				case <-stop:
					return
				default:
				}

				// Are we disabled?
				if s.cfg.Options().IsStunDisabled() {
					l.Infoln("STUN disabled")
					goto disabled
				}

				// Unpunchable NAT? Chillout for some time.
				if !s.isCurrentNATTypePunchable() {
					break
				}
			}
		}

		// Failed all servers, sad.
		s.setNATType(NATUnknown)
		s.setExternalAddress(nil, "")

		// We failed to contact all provided stun servers or the nat is not punchable.
		// Chillout for a while.
		select {
		case <-time.After(stunRetryInterval):
		case <-stop:
			return
		}
	}
}

func (s *Service) runStunForServer(addr string, stop chan struct{}) (tryNext bool) {
	l.Debugf("Running stun for %s via %s", s, addr)

	// Resolve the address, so that in case the server advertises two
	// IPs, we always hit the same one, as otherwise, the mapping might
	// expire as we hit the other address, and cause us to flip flop
	// between servers/external addresses, as a result flooding discovery
	// servers.
	udpAddr, err := net.ResolveUDPAddr("udp", addr)
	if err != nil {
		l.Debugf("%s stun addr resolution on %s: %s", s, addr, err)
		return true
	}
	s.client.SetServerAddr(udpAddr.String())

	natType, extAddr, err := s.client.Discover()
	if err != nil || extAddr == nil {
		l.Debugf("%s stun discovery on %s: %s", s, addr, err)
		return true
	}

	// The stun server is most likely borked, try another one.
	if natType == NATError || natType == NATUnknown || natType == NATBlocked {
		l.Debugf("%s stun discovery on %s resolved to %s", s, addr, natType)
		return true
	}

	s.setNATType(natType)
	s.setExternalAddress(extAddr, addr)
	l.Debugf("%s detected NAT type: %s via %s", s, natType, addr)

	// We can't punch through this one, so no point doing keepalives
	// and such, just let the caller check the nat type and work it out themselves.
	if !s.isCurrentNATTypePunchable() {
		l.Debugf("%s cannot punch %s, skipping", s, natType)
		return false
	}

	return s.stunKeepAlive(addr, extAddr, stop)
}

func (s *Service) stunKeepAlive(addr string, extAddr *Host, stop chan struct{}) (tryNext bool) {
	var err error
	nextSleep := time.Duration(s.cfg.Options().StunKeepaliveStartS) * time.Second

	l.Debugf("%s starting stun keepalive via %s, next sleep %s", s, addr, nextSleep)

	for {
		if areDifferent(s.addr, extAddr) {
			// If the port has changed (addresses are not equal but the hosts are equal),
			// we're probably spending too much time between keepalives, reduce the sleep.
			if s.addr != nil && extAddr != nil && s.addr.IP() == extAddr.IP() {
				nextSleep /= 2
				l.Debugf("%s stun port change (%s to %s), next sleep %s", s, s.addr.TransportAddr(), extAddr.TransportAddr(), nextSleep)
			}

			s.setExternalAddress(extAddr, addr)

			// The stun server is probably stuffed, we've gone beyond min timeout, yet the address keeps changing.
			minSleep := time.Duration(s.cfg.Options().StunKeepaliveMinS) * time.Second
			if nextSleep < minSleep {
				l.Debugf("%s keepalive aborting, sleep below min: %s < %s", s, nextSleep, minSleep)
				return true
			}
		}

		// Adjust the keepalives to fire only nextSleep after last write.
		lastWrite := s.writeTrackingPacketConn.getLastWrite()
		minSleep := time.Duration(s.cfg.Options().StunKeepaliveMinS) * time.Second
		if nextSleep < minSleep {
			nextSleep = minSleep
		}
	tryLater:
		sleepFor := nextSleep

		timeUntilNextKeepalive := time.Until(lastWrite.Add(sleepFor))
		if timeUntilNextKeepalive > 0 {
			sleepFor = timeUntilNextKeepalive
		}

		l.Debugf("%s stun sleeping for %s", s, sleepFor)

		select {
		case <-time.After(sleepFor):
		case <-stop:
			l.Debugf("%s stopping, aborting stun", s)
			return false
		}

		if s.cfg.Options().IsStunDisabled() {
			// Disabled, give up
			l.Debugf("%s disabled, aborting stun ", s)
			return false
		}

		// Check if any writes happened while we were sleeping, if they did, sleep again
		lastWrite = s.writeTrackingPacketConn.getLastWrite()
		if gap := time.Since(lastWrite); gap < nextSleep {
			l.Debugf("%s stun last write gap less than next sleep: %s < %s. Will try later", s, gap, nextSleep)
			goto tryLater
		}

		l.Debugf("%s stun keepalive", s)

		extAddr, err = s.client.Keepalive()
		if err != nil {
			l.Debugf("%s stun keepalive on %s: %s (%v)", s, addr, err, extAddr)
			return true
		}
	}
}

func (s *Service) setNATType(natType NATType) {
	if natType != s.natType {
		l.Debugf("Notifying %s of NAT type change: %s", s.subscriber, natType)
		s.subscriber.OnNATTypeChanged(natType)
	}
	s.natType = natType
}

func (s *Service) setExternalAddress(addr *Host, via string) {
	if areDifferent(s.addr, addr) {
		l.Debugf("Notifying %s of address change: %s via %s", s.subscriber, addr, via)
		s.subscriber.OnExternalAddressChanged(addr, via)
	}
	s.addr = addr
}

func (s *Service) String() string {
	return s.name
}

func (s *Service) isCurrentNATTypePunchable() bool {
	return s.natType == NATNone || s.natType == NATPortRestricted || s.natType == NATRestricted || s.natType == NATFull || s.natType == NATSymmetricUDPFirewall
}

func areDifferent(first, second *Host) bool {
	if (first == nil) != (second == nil) {
		return true
	}
	if first != nil {
		return first.TransportAddr() != second.TransportAddr()
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// No testdata on Android.

//go:build !android && go1.11
// +build !android,go1.11

package main

import (
	"bytes"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"golang.org/x/tools/internal/testenv"
)

func init() {
	// This test currently requires GOPATH mode.
	// Explicitly disabling module mode should suffix, but
	// we'll also turn off GOPROXY just for good measure.
	if err := os.Setenv("GO111MODULE", "off"); err != nil {
		log.Fatal(err)
	}
	if err := os.Setenv("GOPROXY", "off"); err != nil {
		log.Fatal(err)
	}
}

func TestCallgraph(t *testing.T) {
	testenv.NeedsTool(t, "go")

	gopath, err := filepath.Abs("testdata")
	if err != nil {
		t.Fatal(err)
	}

	for _, test := range []struct {
		algo  string
		tests bool
		want  []string
	}{
		{"rta", false, []string{
			// rta imprecisely shows cross product of {main,main2} x {C,D}
			`pkg.main --> (pkg.C).f`,
			`pkg.main --> (pkg.D).f`,
			`pkg.main --> pkg.main2`,
			`pkg.main2 --> (pkg.C).f`,
			`pkg.main2 --> (pkg.D).f`,
		}},
		{"pta", false, []string{
			// pta distinguishes main->C, main2->D.  Also has a root node.
			`<root> --> pkg.init`,
			`<root> --> pkg.main`,
			`pkg.main --> (pkg.C).f`,
			`pkg.main --> pkg.main2`,
			`pkg.main2 --> (pkg.D).f`,
		}},
		// tests: both the package's main and the test's main are called.
		// The callgraph includes all the guts of the "testing" package.
		{"rta", true, []string{
			`pkg.test.main --> testing.MainStart`,
			`testing.runExample --> pkg.Example`,
			`pkg.Example --> (pkg.C).f`,
			`pkg.main --> (pkg.C).f`,
		}},
		{"pta", true, []string{
			`<root> --> pkg.test.main`,
			`<root> --> pkg.main`,
			`pkg.test.main --> testing.MainStart`,
			`testing.runExample --> pkg.Example`,
			`pkg.Example --> (pkg.C).f`,
			`pkg.main --> (pkg.C).f`,
		}},
	} {
		const format = "{{.Caller}} --> {{.Callee}}"
		stdout = new(bytes.Buffer)
		if err := doCallgraph("testdata/src", gopath, test.algo, format, test.tests, []string{"pkg"}); err != nil {
			t.Error(err)
			continue
		}

		edges := make(map[string]bool)
		for _, line := range strings.Split(fmt.Sprint(stdout), "\n") {
			edges[line] = true
		}
		for _, edge := range test.want {
			if !edges[edge] {
				t.Errorf("callgraph(%q, %t): missing edge: %s",
					test.algo, test.tests, edge)
			}
		}
		if t.Failed() {
			t.Log("got:\n", stdout)
		}
	}
}
<commit_msg>cmd/callgraph: skip TestCallgraph on the windows-arm64-10 builder<commit_after>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// No testdata on Android.

//go:build !android && go1.11
// +build !android,go1.11

package main

import (
	"bytes"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"golang.org/x/tools/internal/testenv"
)

func init() {
	// This test currently requires GOPATH mode.
	// Explicitly disabling module mode should suffix, but
	// we'll also turn off GOPROXY just for good measure.
	if err := os.Setenv("GO111MODULE", "off"); err != nil {
		log.Fatal(err)
	}
	if err := os.Setenv("GOPROXY", "off"); err != nil {
		log.Fatal(err)
	}
}

func TestCallgraph(t *testing.T) {
	if b := os.Getenv("GO_BUILDER_NAME"); b == "windows-arm64-10" {
		t.Skipf("skipping due to suspected file corruption bug on %s builder (https://go.dev/issue/50706)", b)
	}

	testenv.NeedsTool(t, "go")

	gopath, err := filepath.Abs("testdata")
	if err != nil {
		t.Fatal(err)
	}

	for _, test := range []struct {
		algo  string
		tests bool
		want  []string
	}{
		{"rta", false, []string{
			// rta imprecisely shows cross product of {main,main2} x {C,D}
			`pkg.main --> (pkg.C).f`,
			`pkg.main --> (pkg.D).f`,
			`pkg.main --> pkg.main2`,
			`pkg.main2 --> (pkg.C).f`,
			`pkg.main2 --> (pkg.D).f`,
		}},
		{"pta", false, []string{
			// pta distinguishes main->C, main2->D.  Also has a root node.
			`<root> --> pkg.init`,
			`<root> --> pkg.main`,
			`pkg.main --> (pkg.C).f`,
			`pkg.main --> pkg.main2`,
			`pkg.main2 --> (pkg.D).f`,
		}},
		// tests: both the package's main and the test's main are called.
		// The callgraph includes all the guts of the "testing" package.
		{"rta", true, []string{
			`pkg.test.main --> testing.MainStart`,
			`testing.runExample --> pkg.Example`,
			`pkg.Example --> (pkg.C).f`,
			`pkg.main --> (pkg.C).f`,
		}},
		{"pta", true, []string{
			`<root> --> pkg.test.main`,
			`<root> --> pkg.main`,
			`pkg.test.main --> testing.MainStart`,
			`testing.runExample --> pkg.Example`,
			`pkg.Example --> (pkg.C).f`,
			`pkg.main --> (pkg.C).f`,
		}},
	} {
		const format = "{{.Caller}} --> {{.Callee}}"
		stdout = new(bytes.Buffer)
		if err := doCallgraph("testdata/src", gopath, test.algo, format, test.tests, []string{"pkg"}); err != nil {
			t.Error(err)
			continue
		}

		edges := make(map[string]bool)
		for _, line := range strings.Split(fmt.Sprint(stdout), "\n") {
			edges[line] = true
		}
		for _, edge := range test.want {
			if !edges[edge] {
				t.Errorf("callgraph(%q, %t): missing edge: %s",
					test.algo, test.tests, edge)
			}
		}
		if t.Failed() {
			t.Log("got:\n", stdout)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"

	"github.com/fsouza/go-dockerclient"
	"github.com/robbertkl/docker-ipv6nat"
)

const buildVersion = "0.4.3"

var (
	cleanup       bool
	retry         bool
	userlandProxy bool
	version       bool
	debug         bool
)

func usage() {
	fmt.Fprintln(os.Stderr, `Usage: docker-ipv6 [options]

Automatically configure IPv6 NAT for running docker containers

Options:`)
	flag.PrintDefaults()

	fmt.Fprintln(os.Stderr, `
Environment Variables:
  DOCKER_HOST - default value for -endpoint
  DOCKER_CERT_PATH - directory path containing key.pem, cert.pem and ca.pem
  DOCKER_TLS_VERIFY - enable client TLS verification
`)

	fmt.Fprintln(os.Stderr, `For more information, see https://github.com/robbertkl/docker-ipv6nat`)
}

func initFlags() {
	flag.BoolVar(&cleanup, "cleanup", false, "remove rules when shutting down")
	flag.BoolVar(&retry, "retry", false, "keep retrying to reconnect after a disconnect")
	flag.BoolVar(&version, "version", false, "show version")
	flag.BoolVar(&debug, "debug", false, "log ruleset changes to stdout")

	flag.Usage = usage
	flag.Parse()
}

func main() {
	initFlags()

	if version {
		fmt.Println(buildVersion)
		return
	}

	if flag.NArg() > 0 {
		usage()
		os.Exit(1)
	}

	if err := run(); err != nil {
		log.Fatalf("%v", err)
	}
}

func run() error {
	if debug {
		log.Println("docker-ipv6nat is running in debug mode")
	}

	client, err := docker.NewClientFromEnv()
	if err != nil {
		return err
	}

	state, err := dockeripv6nat.NewState(debug)
	if err != nil {
		return err
	}

	if cleanup {
		defer func() {
			if err := state.Cleanup(); err != nil {
				log.Printf("%v", err)
			}
		}()
	}

	watcher := dockeripv6nat.NewWatcher(client, state, retry)
	if err := watcher.Watch(); err != nil {
		return err
	}

	return nil
}
<commit_msg>Release v0.4.4<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"

	"github.com/fsouza/go-dockerclient"
	"github.com/robbertkl/docker-ipv6nat"
)

const buildVersion = "0.4.4"

var (
	cleanup       bool
	retry         bool
	userlandProxy bool
	version       bool
	debug         bool
)

func usage() {
	fmt.Fprintln(os.Stderr, `Usage: docker-ipv6 [options]

Automatically configure IPv6 NAT for running docker containers

Options:`)
	flag.PrintDefaults()

	fmt.Fprintln(os.Stderr, `
Environment Variables:
  DOCKER_HOST - default value for -endpoint
  DOCKER_CERT_PATH - directory path containing key.pem, cert.pem and ca.pem
  DOCKER_TLS_VERIFY - enable client TLS verification
`)

	fmt.Fprintln(os.Stderr, `For more information, see https://github.com/robbertkl/docker-ipv6nat`)
}

func initFlags() {
	flag.BoolVar(&cleanup, "cleanup", false, "remove rules when shutting down")
	flag.BoolVar(&retry, "retry", false, "keep retrying to reconnect after a disconnect")
	flag.BoolVar(&version, "version", false, "show version")
	flag.BoolVar(&debug, "debug", false, "log ruleset changes to stdout")

	flag.Usage = usage
	flag.Parse()
}

func main() {
	initFlags()

	if version {
		fmt.Println(buildVersion)
		return
	}

	if flag.NArg() > 0 {
		usage()
		os.Exit(1)
	}

	if err := run(); err != nil {
		log.Fatalf("%v", err)
	}
}

func run() error {
	if debug {
		log.Println("docker-ipv6nat is running in debug mode")
	}

	client, err := docker.NewClientFromEnv()
	if err != nil {
		return err
	}

	state, err := dockeripv6nat.NewState(debug)
	if err != nil {
		return err
	}

	if cleanup {
		defer func() {
			if err := state.Cleanup(); err != nil {
				log.Printf("%v", err)
			}
		}()
	}

	watcher := dockeripv6nat.NewWatcher(client, state, retry)
	if err := watcher.Watch(); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package framework

import (
	"flag"
	"testing"
)

func RegisterCommonFlags() {
	flag.StringVar(&TestContext.KubeConfig, "kubeconfig", "", "kubernetes config path")
	flag.StringVar(&TestContext.CloudConfig, "cloud-config", "", "cloud config")
	flag.StringVar(&TestContext.LoadBalancerID, "lbid", "", "reused slb id")
	flag.StringVar(&TestContext.MasterZoneID, "MasterZoneID", "", "master zone id")
	flag.StringVar(&TestContext.SlaveZoneID, "SlaveZoneID", "", "slave zone id")
	flag.StringVar(&TestContext.BackendLabel, "BackendLabel", "", "backend label: k1=v1,k2=v2")
	flag.StringVar(&TestContext.AclID, "aclid", "", "acl id")
	flag.StringVar(&TestContext.VSwitchID, "vswitchid", "", "vswitch id")
	flag.StringVar(&TestContext.CertID, "certid", "", "cert id")
	flag.StringVar(&TestContext.PrivateZoneID, "PrivateZoneID", "", "private zone id")
	flag.StringVar(&TestContext.PrivateZoneName, "PrivateZoneName", "", "private zone name")
	flag.StringVar(&TestContext.PrivateZoneRecordName, "PrivateZoneRecordName", "", "private zone record id")
	flag.StringVar(&TestContext.PrivateZoneRecordTTL, "PrivateZoneRecordTTL", "", "private zone record ttl")
	flag.StringVar(&TestContext.ResourceGroupID, "ResourceGroupID", "", "resource group id")
}

// ViperizeFlags sets up all flag and config processing. Future configuration info should be added to viper, not to flags.
func ViperizeFlags() {

	testing.Init()
	// Part 1: Set regular flags.
	// TODO: Future, lets eliminate e2e 'flag' deps entirely in favor of viper only,
	// since go test 'flag's are sort of incompatible w/ flag, klog, etc.
	RegisterCommonFlags()
	flag.Parse()
}
<commit_msg>update e2etest common flags<commit_after>package framework

import (
	"flag"
	"testing"
)

func RegisterCommonFlags() {
	flag.StringVar(&TestContext.KubeConfig, "kubeconfig", "", "kubernetes config path")
	flag.StringVar(&TestContext.CloudConfig, "cloud-config", "", "cloud config")
	flag.StringVar(&TestContext.LoadBalancerID, "lb-id", "", "reused slb id")
	flag.StringVar(&TestContext.MasterZoneID, "master-zone-id", "", "master zone id")
	flag.StringVar(&TestContext.SlaveZoneID, "slave-zone-id", "", "slave zone id")
	flag.StringVar(&TestContext.BackendLabel, "backend-label", "", "backend label: k1=v1,k2=v2")
	flag.StringVar(&TestContext.AclID, "acl-id", "", "acl id")
	flag.StringVar(&TestContext.VSwitchID, "vswitch-id", "", "vswitch id")
	flag.StringVar(&TestContext.CertID, "cert-id", "", "cert id")
	flag.StringVar(&TestContext.PrivateZoneID, "private-zone-id", "", "private zone id")
	flag.StringVar(&TestContext.PrivateZoneName, "private-zone-mame", "", "private zone name")
	flag.StringVar(&TestContext.PrivateZoneRecordName, "private-zone-record-name", "", "private zone record id")
	flag.StringVar(&TestContext.PrivateZoneRecordTTL, "private-zone-record-ttl", "", "private zone record ttl")
	flag.StringVar(&TestContext.ResourceGroupID, "resource-group-id", "", "resource group id")
}

// ViperizeFlags sets up all flag and config processing. Future configuration info should be added to viper, not to flags.
func ViperizeFlags() {

	testing.Init()
	// Part 1: Set regular flags.
	// TODO: Future, lets eliminate e2e 'flag' deps entirely in favor of viper only,
	// since go test 'flag's are sort of incompatible w/ flag, klog, etc.
	RegisterCommonFlags()
	flag.Parse()
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"errors"
	"flag"
	"fmt"
	"go/build"
	"go/scanner"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"strings"

	"golang.org/x/tools/internal/imports"
)

var (
	// main operation modes
	list   = flag.Bool("l", false, "list files whose formatting differs from goimport's")
	write  = flag.Bool("w", false, "write result to (source) file instead of stdout")
	doDiff = flag.Bool("d", false, "display diffs instead of rewriting files")
	srcdir = flag.String("srcdir", "", "choose imports as if source code is from `dir`. When operating on a single file, dir may instead be the complete file name.")

	verbose bool // verbose logging

	cpuProfile     = flag.String("cpuprofile", "", "CPU profile output")
	memProfile     = flag.String("memprofile", "", "memory profile output")
	memProfileRate = flag.Int("memrate", 0, "if > 0, sets runtime.MemProfileRate")

	options = &imports.Options{
		TabWidth:  8,
		TabIndent: true,
		Comments:  true,
		Fragment:  true,
		// This environment, and its caches, will be reused for the whole run.
		Env: &imports.ProcessEnv{
			GOPATH:      build.Default.GOPATH,
			GOROOT:      build.Default.GOROOT,
			GOFLAGS:     os.Getenv("GOFLAGS"),
			GO111MODULE: os.Getenv("GO111MODULE"),
			GOPROXY:     os.Getenv("GOPROXY"),
			GOSUMDB:     os.Getenv("GOSUMDB"),
		},
	}
	exitCode = 0
)

func init() {
	flag.BoolVar(&options.AllErrors, "e", false, "report all errors (not just the first 10 on different lines)")
	flag.StringVar(&options.Env.LocalPrefix, "local", "", "put imports beginning with this string after 3rd-party packages; comma-separated list")
	flag.BoolVar(&options.FormatOnly, "format-only", false, "if true, don't fix imports and only format. In this mode, goimports is effectively gofmt, with the addition that imports are grouped into sections.")
}

func report(err error) {
	scanner.PrintError(os.Stderr, err)
	exitCode = 2
}

func usage() {
	fmt.Fprintf(os.Stderr, "usage: goimports [flags] [path ...]\n")
	flag.PrintDefaults()
	os.Exit(2)
}

func isGoFile(f os.FileInfo) bool {
	// ignore non-Go files
	name := f.Name()
	return !f.IsDir() && !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".go")
}

// argumentType is which mode goimports was invoked as.
type argumentType int

const (
	// fromStdin means the user is piping their source into goimports.
	fromStdin argumentType = iota

	// singleArg is the common case from editors, when goimports is run on
	// a single file.
	singleArg

	// multipleArg is when the user ran "goimports file1.go file2.go"
	// or ran goimports on a directory tree.
	multipleArg
)

func processFile(filename string, in io.Reader, out io.Writer, argType argumentType) error {
	opt := options
	if argType == fromStdin {
		nopt := *options
		nopt.Fragment = true
		opt = &nopt
	}

	if in == nil {
		f, err := os.Open(filename)
		if err != nil {
			return err
		}
		defer f.Close()
		in = f
	}

	src, err := ioutil.ReadAll(in)
	if err != nil {
		return err
	}

	target := filename
	if *srcdir != "" {
		// Determine whether the provided -srcdirc is a directory or file
		// and then use it to override the target.
		//
		// See https://github.com/dominikh/go-mode.el/issues/146
		if isFile(*srcdir) {
			if argType == multipleArg {
				return errors.New("-srcdir value can't be a file when passing multiple arguments or when walking directories")
			}
			target = *srcdir
		} else if argType == singleArg && strings.HasSuffix(*srcdir, ".go") && !isDir(*srcdir) {
			// For a file which doesn't exist on disk yet, but might shortly.
			// e.g. user in editor opens $DIR/newfile.go and newfile.go doesn't yet exist on disk.
			// The goimports on-save hook writes the buffer to a temp file
			// first and runs goimports before the actual save to newfile.go.
			// The editor's buffer is named "newfile.go" so that is passed to goimports as:
			//      goimports -srcdir=/gopath/src/pkg/newfile.go /tmp/gofmtXXXXXXXX.go
			// and then the editor reloads the result from the tmp file and writes
			// it to newfile.go.
			target = *srcdir
		} else {
			// Pretend that file is from *srcdir in order to decide
			// visible imports correctly.
			target = filepath.Join(*srcdir, filepath.Base(filename))
		}
	}

	res, err := imports.Process(target, src, opt)
	if err != nil {
		return err
	}

	if !bytes.Equal(src, res) {
		// formatting has changed
		if *list {
			fmt.Fprintln(out, filename)
			exitCode = 1
		}
		if *write {
			if argType == fromStdin {
				// filename is "<standard input>"
				return errors.New("can't use -w on stdin")
			}
			// On Windows, we need to re-set the permissions from the file. See golang/go#38225.
			var perms os.FileMode
			if fi, err := os.Stat(filename); err == nil {
				perms = fi.Mode() & os.ModePerm
			}
			err = ioutil.WriteFile(filename, res, perms)
			if err != nil {
				return err
			}
		}
		if *doDiff {
			if argType == fromStdin {
				filename = "stdin.go" // because <standard input>.orig looks silly
			}
			data, err := diff(src, res, filename)
			if err != nil {
				return fmt.Errorf("computing diff: %s", err)
			}
			fmt.Printf("diff -u %s %s\n", filepath.ToSlash(filename+".orig"), filepath.ToSlash(filename))
			out.Write(data)
		}
	}

	if !*list && !*write && !*doDiff {
		_, err = out.Write(res)
	}

	return err
}

func visitFile(path string, f os.FileInfo, err error) error {
	if err == nil && isGoFile(f) {
		err = processFile(path, nil, os.Stdout, multipleArg)
	}
	if err != nil {
		report(err)
	}
	return nil
}

func walkDir(path string) {
	filepath.Walk(path, visitFile)
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	// call gofmtMain in a separate function
	// so that it can use defer and have them
	// run before the exit.
	gofmtMain()
	os.Exit(exitCode)
}

// parseFlags parses command line flags and returns the paths to process.
// It's a var so that custom implementations can replace it in other files.
var parseFlags = func() []string {
	flag.BoolVar(&verbose, "v", false, "verbose logging")

	flag.Parse()
	return flag.Args()
}

func bufferedFileWriter(dest string) (w io.Writer, close func()) {
	f, err := os.Create(dest)
	if err != nil {
		log.Fatal(err)
	}
	bw := bufio.NewWriter(f)
	return bw, func() {
		if err := bw.Flush(); err != nil {
			log.Fatalf("error flushing %v: %v", dest, err)
		}
		if err := f.Close(); err != nil {
			log.Fatal(err)
		}
	}
}

func gofmtMain() {
	flag.Usage = usage
	paths := parseFlags()

	if *cpuProfile != "" {
		bw, flush := bufferedFileWriter(*cpuProfile)
		pprof.StartCPUProfile(bw)
		defer flush()
		defer pprof.StopCPUProfile()
	}
	// doTrace is a conditionally compiled wrapper around runtime/trace. It is
	// used to allow goimports to compile under gccgo, which does not support
	// runtime/trace. See https://golang.org/issue/15544.
	defer doTrace()()
	if *memProfileRate > 0 {
		runtime.MemProfileRate = *memProfileRate
		bw, flush := bufferedFileWriter(*memProfile)
		defer func() {
			runtime.GC() // materialize all statistics
			if err := pprof.WriteHeapProfile(bw); err != nil {
				log.Fatal(err)
			}
			flush()
		}()
	}

	if verbose {
		log.SetFlags(log.LstdFlags | log.Lmicroseconds)
		options.Env.Logf = log.Printf
	}
	if options.TabWidth < 0 {
		fmt.Fprintf(os.Stderr, "negative tabwidth %d\n", options.TabWidth)
		exitCode = 2
		return
	}

	if len(paths) == 0 {
		if err := processFile("<standard input>", os.Stdin, os.Stdout, fromStdin); err != nil {
			report(err)
		}
		return
	}

	argType := singleArg
	if len(paths) > 1 {
		argType = multipleArg
	}

	for _, path := range paths {
		switch dir, err := os.Stat(path); {
		case err != nil:
			report(err)
		case dir.IsDir():
			walkDir(path)
		default:
			if err := processFile(path, nil, os.Stdout, argType); err != nil {
				report(err)
			}
		}
	}
}

func writeTempFile(dir, prefix string, data []byte) (string, error) {
	file, err := ioutil.TempFile(dir, prefix)
	if err != nil {
		return "", err
	}
	_, err = file.Write(data)
	if err1 := file.Close(); err == nil {
		err = err1
	}
	if err != nil {
		os.Remove(file.Name())
		return "", err
	}
	return file.Name(), nil
}

func diff(b1, b2 []byte, filename string) (data []byte, err error) {
	f1, err := writeTempFile("", "gofmt", b1)
	if err != nil {
		return
	}
	defer os.Remove(f1)

	f2, err := writeTempFile("", "gofmt", b2)
	if err != nil {
		return
	}
	defer os.Remove(f2)

	cmd := "diff"
	if runtime.GOOS == "plan9" {
		cmd = "/bin/ape/diff"
	}

	data, err = exec.Command(cmd, "-u", f1, f2).CombinedOutput()
	if len(data) > 0 {
		// diff exits with a non-zero status when the files don't match.
		// Ignore that failure as long as we get output.
		return replaceTempFilename(data, filename)
	}
	return
}

// replaceTempFilename replaces temporary filenames in diff with actual one.
//
// --- /tmp/gofmt316145376	2017-02-03 19:13:00.280468375 -0500
// +++ /tmp/gofmt617882815	2017-02-03 19:13:00.280468375 -0500
// ...
// ->
// --- path/to/file.go.orig	2017-02-03 19:13:00.280468375 -0500
// +++ path/to/file.go	2017-02-03 19:13:00.280468375 -0500
// ...
func replaceTempFilename(diff []byte, filename string) ([]byte, error) {
	bs := bytes.SplitN(diff, []byte{'\n'}, 3)
	if len(bs) < 3 {
		return nil, fmt.Errorf("got unexpected diff for %s", filename)
	}
	// Preserve timestamps.
	var t0, t1 []byte
	if i := bytes.LastIndexByte(bs[0], '\t'); i != -1 {
		t0 = bs[0][i:]
	}
	if i := bytes.LastIndexByte(bs[1], '\t'); i != -1 {
		t1 = bs[1][i:]
	}
	// Always print filepath with slash separator.
	f := filepath.ToSlash(filename)
	bs[0] = []byte(fmt.Sprintf("--- %s%s", f+".orig", t0))
	bs[1] = []byte(fmt.Sprintf("+++ %s%s", f, t1))
	return bytes.Join(bs, []byte{'\n'}), nil
}

// isFile reports whether name is a file.
func isFile(name string) bool {
	fi, err := os.Stat(name)
	return err == nil && fi.Mode().IsRegular()
}

// isDir reports whether name is a directory.
func isDir(name string) bool {
	fi, err := os.Stat(name)
	return err == nil && fi.IsDir()
}
<commit_msg>Revert "cmd/goimports: return exit code 1 when -l finds files to change"<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"errors"
	"flag"
	"fmt"
	"go/build"
	"go/scanner"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"strings"

	"golang.org/x/tools/internal/imports"
)

var (
	// main operation modes
	list   = flag.Bool("l", false, "list files whose formatting differs from goimport's")
	write  = flag.Bool("w", false, "write result to (source) file instead of stdout")
	doDiff = flag.Bool("d", false, "display diffs instead of rewriting files")
	srcdir = flag.String("srcdir", "", "choose imports as if source code is from `dir`. When operating on a single file, dir may instead be the complete file name.")

	verbose bool // verbose logging

	cpuProfile     = flag.String("cpuprofile", "", "CPU profile output")
	memProfile     = flag.String("memprofile", "", "memory profile output")
	memProfileRate = flag.Int("memrate", 0, "if > 0, sets runtime.MemProfileRate")

	options = &imports.Options{
		TabWidth:  8,
		TabIndent: true,
		Comments:  true,
		Fragment:  true,
		// This environment, and its caches, will be reused for the whole run.
		Env: &imports.ProcessEnv{
			GOPATH:      build.Default.GOPATH,
			GOROOT:      build.Default.GOROOT,
			GOFLAGS:     os.Getenv("GOFLAGS"),
			GO111MODULE: os.Getenv("GO111MODULE"),
			GOPROXY:     os.Getenv("GOPROXY"),
			GOSUMDB:     os.Getenv("GOSUMDB"),
		},
	}
	exitCode = 0
)

func init() {
	flag.BoolVar(&options.AllErrors, "e", false, "report all errors (not just the first 10 on different lines)")
	flag.StringVar(&options.Env.LocalPrefix, "local", "", "put imports beginning with this string after 3rd-party packages; comma-separated list")
	flag.BoolVar(&options.FormatOnly, "format-only", false, "if true, don't fix imports and only format. In this mode, goimports is effectively gofmt, with the addition that imports are grouped into sections.")
}

func report(err error) {
	scanner.PrintError(os.Stderr, err)
	exitCode = 2
}

func usage() {
	fmt.Fprintf(os.Stderr, "usage: goimports [flags] [path ...]\n")
	flag.PrintDefaults()
	os.Exit(2)
}

func isGoFile(f os.FileInfo) bool {
	// ignore non-Go files
	name := f.Name()
	return !f.IsDir() && !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".go")
}

// argumentType is which mode goimports was invoked as.
type argumentType int

const (
	// fromStdin means the user is piping their source into goimports.
	fromStdin argumentType = iota

	// singleArg is the common case from editors, when goimports is run on
	// a single file.
	singleArg

	// multipleArg is when the user ran "goimports file1.go file2.go"
	// or ran goimports on a directory tree.
	multipleArg
)

func processFile(filename string, in io.Reader, out io.Writer, argType argumentType) error {
	opt := options
	if argType == fromStdin {
		nopt := *options
		nopt.Fragment = true
		opt = &nopt
	}

	if in == nil {
		f, err := os.Open(filename)
		if err != nil {
			return err
		}
		defer f.Close()
		in = f
	}

	src, err := ioutil.ReadAll(in)
	if err != nil {
		return err
	}

	target := filename
	if *srcdir != "" {
		// Determine whether the provided -srcdirc is a directory or file
		// and then use it to override the target.
		//
		// See https://github.com/dominikh/go-mode.el/issues/146
		if isFile(*srcdir) {
			if argType == multipleArg {
				return errors.New("-srcdir value can't be a file when passing multiple arguments or when walking directories")
			}
			target = *srcdir
		} else if argType == singleArg && strings.HasSuffix(*srcdir, ".go") && !isDir(*srcdir) {
			// For a file which doesn't exist on disk yet, but might shortly.
			// e.g. user in editor opens $DIR/newfile.go and newfile.go doesn't yet exist on disk.
			// The goimports on-save hook writes the buffer to a temp file
			// first and runs goimports before the actual save to newfile.go.
			// The editor's buffer is named "newfile.go" so that is passed to goimports as:
			//      goimports -srcdir=/gopath/src/pkg/newfile.go /tmp/gofmtXXXXXXXX.go
			// and then the editor reloads the result from the tmp file and writes
			// it to newfile.go.
			target = *srcdir
		} else {
			// Pretend that file is from *srcdir in order to decide
			// visible imports correctly.
			target = filepath.Join(*srcdir, filepath.Base(filename))
		}
	}

	res, err := imports.Process(target, src, opt)
	if err != nil {
		return err
	}

	if !bytes.Equal(src, res) {
		// formatting has changed
		if *list {
			fmt.Fprintln(out, filename)
		}
		if *write {
			if argType == fromStdin {
				// filename is "<standard input>"
				return errors.New("can't use -w on stdin")
			}
			// On Windows, we need to re-set the permissions from the file. See golang/go#38225.
			var perms os.FileMode
			if fi, err := os.Stat(filename); err == nil {
				perms = fi.Mode() & os.ModePerm
			}
			err = ioutil.WriteFile(filename, res, perms)
			if err != nil {
				return err
			}
		}
		if *doDiff {
			if argType == fromStdin {
				filename = "stdin.go" // because <standard input>.orig looks silly
			}
			data, err := diff(src, res, filename)
			if err != nil {
				return fmt.Errorf("computing diff: %s", err)
			}
			fmt.Printf("diff -u %s %s\n", filepath.ToSlash(filename+".orig"), filepath.ToSlash(filename))
			out.Write(data)
		}
	}

	if !*list && !*write && !*doDiff {
		_, err = out.Write(res)
	}

	return err
}

func visitFile(path string, f os.FileInfo, err error) error {
	if err == nil && isGoFile(f) {
		err = processFile(path, nil, os.Stdout, multipleArg)
	}
	if err != nil {
		report(err)
	}
	return nil
}

func walkDir(path string) {
	filepath.Walk(path, visitFile)
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	// call gofmtMain in a separate function
	// so that it can use defer and have them
	// run before the exit.
	gofmtMain()
	os.Exit(exitCode)
}

// parseFlags parses command line flags and returns the paths to process.
// It's a var so that custom implementations can replace it in other files.
var parseFlags = func() []string {
	flag.BoolVar(&verbose, "v", false, "verbose logging")

	flag.Parse()
	return flag.Args()
}

func bufferedFileWriter(dest string) (w io.Writer, close func()) {
	f, err := os.Create(dest)
	if err != nil {
		log.Fatal(err)
	}
	bw := bufio.NewWriter(f)
	return bw, func() {
		if err := bw.Flush(); err != nil {
			log.Fatalf("error flushing %v: %v", dest, err)
		}
		if err := f.Close(); err != nil {
			log.Fatal(err)
		}
	}
}

func gofmtMain() {
	flag.Usage = usage
	paths := parseFlags()

	if *cpuProfile != "" {
		bw, flush := bufferedFileWriter(*cpuProfile)
		pprof.StartCPUProfile(bw)
		defer flush()
		defer pprof.StopCPUProfile()
	}
	// doTrace is a conditionally compiled wrapper around runtime/trace. It is
	// used to allow goimports to compile under gccgo, which does not support
	// runtime/trace. See https://golang.org/issue/15544.
	defer doTrace()()
	if *memProfileRate > 0 {
		runtime.MemProfileRate = *memProfileRate
		bw, flush := bufferedFileWriter(*memProfile)
		defer func() {
			runtime.GC() // materialize all statistics
			if err := pprof.WriteHeapProfile(bw); err != nil {
				log.Fatal(err)
			}
			flush()
		}()
	}

	if verbose {
		log.SetFlags(log.LstdFlags | log.Lmicroseconds)
		options.Env.Logf = log.Printf
	}
	if options.TabWidth < 0 {
		fmt.Fprintf(os.Stderr, "negative tabwidth %d\n", options.TabWidth)
		exitCode = 2
		return
	}

	if len(paths) == 0 {
		if err := processFile("<standard input>", os.Stdin, os.Stdout, fromStdin); err != nil {
			report(err)
		}
		return
	}

	argType := singleArg
	if len(paths) > 1 {
		argType = multipleArg
	}

	for _, path := range paths {
		switch dir, err := os.Stat(path); {
		case err != nil:
			report(err)
		case dir.IsDir():
			walkDir(path)
		default:
			if err := processFile(path, nil, os.Stdout, argType); err != nil {
				report(err)
			}
		}
	}
}

func writeTempFile(dir, prefix string, data []byte) (string, error) {
	file, err := ioutil.TempFile(dir, prefix)
	if err != nil {
		return "", err
	}
	_, err = file.Write(data)
	if err1 := file.Close(); err == nil {
		err = err1
	}
	if err != nil {
		os.Remove(file.Name())
		return "", err
	}
	return file.Name(), nil
}

func diff(b1, b2 []byte, filename string) (data []byte, err error) {
	f1, err := writeTempFile("", "gofmt", b1)
	if err != nil {
		return
	}
	defer os.Remove(f1)

	f2, err := writeTempFile("", "gofmt", b2)
	if err != nil {
		return
	}
	defer os.Remove(f2)

	cmd := "diff"
	if runtime.GOOS == "plan9" {
		cmd = "/bin/ape/diff"
	}

	data, err = exec.Command(cmd, "-u", f1, f2).CombinedOutput()
	if len(data) > 0 {
		// diff exits with a non-zero status when the files don't match.
		// Ignore that failure as long as we get output.
		return replaceTempFilename(data, filename)
	}
	return
}

// replaceTempFilename replaces temporary filenames in diff with actual one.
//
// --- /tmp/gofmt316145376	2017-02-03 19:13:00.280468375 -0500
// +++ /tmp/gofmt617882815	2017-02-03 19:13:00.280468375 -0500
// ...
// ->
// --- path/to/file.go.orig	2017-02-03 19:13:00.280468375 -0500
// +++ path/to/file.go	2017-02-03 19:13:00.280468375 -0500
// ...
func replaceTempFilename(diff []byte, filename string) ([]byte, error) {
	bs := bytes.SplitN(diff, []byte{'\n'}, 3)
	if len(bs) < 3 {
		return nil, fmt.Errorf("got unexpected diff for %s", filename)
	}
	// Preserve timestamps.
	var t0, t1 []byte
	if i := bytes.LastIndexByte(bs[0], '\t'); i != -1 {
		t0 = bs[0][i:]
	}
	if i := bytes.LastIndexByte(bs[1], '\t'); i != -1 {
		t1 = bs[1][i:]
	}
	// Always print filepath with slash separator.
	f := filepath.ToSlash(filename)
	bs[0] = []byte(fmt.Sprintf("--- %s%s", f+".orig", t0))
	bs[1] = []byte(fmt.Sprintf("+++ %s%s", f, t1))
	return bytes.Join(bs, []byte{'\n'}), nil
}

// isFile reports whether name is a file.
func isFile(name string) bool {
	fi, err := os.Stat(name)
	return err == nil && fi.Mode().IsRegular()
}

// isDir reports whether name is a directory.
func isDir(name string) bool {
	fi, err := os.Stat(name)
	return err == nil && fi.IsDir()
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/constraints"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/agent"
	"launchpad.net/juju-core/environs/config"
	"launchpad.net/juju-core/environs/dummy"
	"launchpad.net/juju-core/testing"
	"launchpad.net/juju-core/version"
	"net/http"
	"os"
	"strings"
)

type BootstrapSuite struct {
	testing.LoggingSuite
	testing.MgoSuite
}

var _ = Suite(&BootstrapSuite{})

func (s *BootstrapSuite) SetUpSuite(c *C) {
	s.LoggingSuite.SetUpSuite(c)
	s.MgoSuite.SetUpSuite(c)
}

func (s *BootstrapSuite) SetUpTest(c *C) {
	s.LoggingSuite.SetUpTest(c)
	s.MgoSuite.SetUpTest(c)
}

func (s *BootstrapSuite) TearDownSuite(c *C) {
	s.MgoSuite.TearDownSuite(c)
	s.LoggingSuite.TearDownSuite(c)
}

func (s *BootstrapSuite) TearDownTest(c *C) {
	s.MgoSuite.TearDownTest(c)
	s.LoggingSuite.TearDownTest(c)
	dummy.Reset()
}

func (*BootstrapSuite) TestBasic(c *C) {
	defer testing.MakeFakeHome(c, envConfig).Restore()
	opc, errc := runCommand(new(BootstrapCommand))
	c.Check(<-errc, IsNil)
	opBootstrap := (<-opc).(dummy.OpBootstrap)
	c.Check(opBootstrap.Env, Equals, "peckham")
	c.Check(opBootstrap.Constraints, DeepEquals, constraints.Value{})
}

func (*BootstrapSuite) TestRunGeneratesCertificate(c *C) {
	defer testing.MakeFakeHome(c, envConfig).Restore()
	envName := "peckham"
	_, err := testing.RunCommand(c, new(BootstrapCommand), nil)
	c.Assert(err, IsNil)

	// Check that the CA certificate and key have been automatically generated
	// for the environment.
	info, err := os.Stat(config.JujuHomePath(envName + "-cert.pem"))
	c.Assert(err, IsNil)
	c.Assert(info.Size() > 0, Equals, true)
	info, err = os.Stat(config.JujuHomePath(envName + "-private-key.pem"))
	c.Assert(err, IsNil)
	c.Assert(info.Size() > 0, Equals, true)

	// Check that the environment validates the cert and key.
	_, err = environs.NewFromName(envName)
	c.Assert(err, IsNil)
}

func (*BootstrapSuite) TestConstraints(c *C) {
	defer testing.MakeFakeHome(c, envConfig, "brokenenv").Restore()
	scons := " cpu-cores=2   mem=4G"
	opc, errc := runCommand(new(BootstrapCommand), "--constraints", scons)
	c.Check(<-errc, IsNil)
	opBootstrap := (<-opc).(dummy.OpBootstrap)
	c.Check(opBootstrap.Env, Equals, "peckham")
	c.Check(opBootstrap.Constraints, DeepEquals, constraints.MustParse(scons))
}

func (*BootstrapSuite) TestUploadTools(c *C) {
	defer testing.MakeFakeHome(c, envConfig).Restore()
	origCurrent := version.Current
	version.Current.Series = "hostseries"
	defer func() { version.Current = origCurrent }()

	opc, errc := runCommand(new(BootstrapCommand), "--upload-tools")
	c.Check(<-errc, IsNil)
	c.Check((<-opc).(dummy.OpPutFile).Env, Equals, "peckham") // hostseries from version.Current
	c.Check((<-opc).(dummy.OpPutFile).Env, Equals, "peckham") // defaultseries from env config
	c.Check((<-opc).(dummy.OpPutFile).Env, Equals, "peckham") // precise from config.DefaultSeries
	opBootstrap := (<-opc).(dummy.OpBootstrap)
	c.Check(opBootstrap.Env, Equals, "peckham")
	c.Check(opBootstrap.Constraints, DeepEquals, constraints.Value{})

	vers := version.Current
	assertUploadedSomething(c, vers)
	vers.Series = "defaultseries"
	assertUploadedSomething(c, vers)
	vers.Series = "precise"
	assertUploadedSomething(c, vers)
}

func assertUploadedSomething(c *C, vers version.Binary) {
	// Check that some file was uploaded and can be unpacked; detailed
	// semantics tested elsewhere.
	envs, err := environs.ReadEnvirons("")
	c.Assert(err, IsNil)
	env, err := envs.Open("peckham")
	c.Assert(err, IsNil)
	tools, err := environs.FindTools(env, vers, environs.CompatVersion)
	c.Assert(err, IsNil)
	c.Assert(tools.Binary, Equals, vers)
	resp, err := http.Get(tools.URL)
	c.Assert(err, IsNil)
	defer resp.Body.Close()
	err = agent.UnpackTools(c.MkDir(), tools, resp.Body)
	c.Assert(err, IsNil)
}

func (*BootstrapSuite) TestUploadToolsSeries(c *C) {
	defer testing.MakeFakeHome(c, envConfig).Restore()
	opc, errc := runCommand(new(BootstrapCommand), "--upload-tools", "--series=good,great")
	c.Check(<-errc, IsNil)
	c.Check((<-opc).(dummy.OpPutFile).Env, Equals, "peckham")
	c.Check((<-opc).(dummy.OpPutFile).Env, Equals, "peckham")
	c.Check((<-opc).(dummy.OpPutFile).Env, Equals, "peckham")
	c.Check((<-opc).(dummy.OpBootstrap).Env, Equals, "peckham")

	vers := version.Current
	assertUploadedSomething(c, vers)
	vers.Series = "good"
	assertUploadedSomething(c, vers)
	vers.Series = "great"
	assertUploadedSomething(c, vers)
}

func (*BootstrapSuite) TestSeriesBadParams(c *C) {
	defer testing.MakeFakeHome(c, envConfig).Restore()
	opc, errc := runCommand(new(BootstrapCommand), "--series=bad1")
	c.Check(<-errc, ErrorMatches, `invalid value "bad1" for flag --series: invalid series name "bad1"`)
	c.Check(<-opc, IsNil)
}

func (*BootstrapSuite) TestSeriesNoUploadTools(c *C) {
	defer testing.MakeFakeHome(c, envConfig).Restore()
	opc, errc := runCommand(new(BootstrapCommand), "--series=good,great")
	c.Check(<-errc, ErrorMatches, `--series requires --upload-tools`)
	c.Check(<-opc, IsNil)
}

func (*BootstrapSuite) TestBrokenEnvironment(c *C) {
	defer testing.MakeFakeHome(c, envConfig).Restore()
	opc, errc := runCommand(new(BootstrapCommand), "-e", "brokenenv")
	c.Check(<-errc, ErrorMatches, "dummy.Bootstrap is broken")
	c.Check(<-opc, IsNil)
}

func (*BootstrapSuite) TestMissingEnvironment(c *C) {
	defer testing.MakeFakeHomeNoEnvironments(c, "empty").Restore()
	ctx := testing.Context(c)
	code := cmd.Main(&BootstrapCommand{}, ctx, nil)
	c.Check(code, Equals, 1)
	errStr := ctx.Stderr.(*bytes.Buffer).String()
	strippedErr := strings.Replace(errStr, "\n", "", -1)
	c.Assert(strippedErr, Matches, ".*No juju environment configuration file exists.*")
}
<commit_msg>actually mock out uploadTools in BootstrapSuite<commit_after>package main

import (
	"bytes"
	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/constraints"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/agent"
	"launchpad.net/juju-core/environs/config"
	"launchpad.net/juju-core/environs/dummy"
	"launchpad.net/juju-core/environs/tools"
	"launchpad.net/juju-core/testing"
	"launchpad.net/juju-core/version"
	"net/http"
	"os"
	"strings"
)

type BootstrapSuite struct {
	testing.LoggingSuite
	testing.MgoSuite
}

var _ = Suite(&BootstrapSuite{})

func (s *BootstrapSuite) SetUpSuite(c *C) {
	s.LoggingSuite.SetUpSuite(c)
	s.MgoSuite.SetUpSuite(c)
	uploadTools = mockUploadTools
}

func (s *BootstrapSuite) SetUpTest(c *C) {
	s.LoggingSuite.SetUpTest(c)
	s.MgoSuite.SetUpTest(c)
}

func (s *BootstrapSuite) TearDownSuite(c *C) {
	s.MgoSuite.TearDownSuite(c)
	s.LoggingSuite.TearDownSuite(c)
}

func (s *BootstrapSuite) TearDownTest(c *C) {
	uploadTools = tools.Upload
	s.MgoSuite.TearDownTest(c)
	s.LoggingSuite.TearDownTest(c)
	dummy.Reset()
}

func (*BootstrapSuite) TestBasic(c *C) {
	defer testing.MakeFakeHome(c, envConfig).Restore()
	opc, errc := runCommand(new(BootstrapCommand))
	c.Check(<-errc, IsNil)
	opBootstrap := (<-opc).(dummy.OpBootstrap)
	c.Check(opBootstrap.Env, Equals, "peckham")
	c.Check(opBootstrap.Constraints, DeepEquals, constraints.Value{})
}

func (*BootstrapSuite) TestRunGeneratesCertificate(c *C) {
	defer testing.MakeFakeHome(c, envConfig).Restore()
	envName := "peckham"
	_, err := testing.RunCommand(c, new(BootstrapCommand), nil)
	c.Assert(err, IsNil)

	// Check that the CA certificate and key have been automatically generated
	// for the environment.
	info, err := os.Stat(config.JujuHomePath(envName + "-cert.pem"))
	c.Assert(err, IsNil)
	c.Assert(info.Size() > 0, Equals, true)
	info, err = os.Stat(config.JujuHomePath(envName + "-private-key.pem"))
	c.Assert(err, IsNil)
	c.Assert(info.Size() > 0, Equals, true)

	// Check that the environment validates the cert and key.
	_, err = environs.NewFromName(envName)
	c.Assert(err, IsNil)
}

func (*BootstrapSuite) TestConstraints(c *C) {
	defer testing.MakeFakeHome(c, envConfig, "brokenenv").Restore()
	scons := " cpu-cores=2   mem=4G"
	opc, errc := runCommand(new(BootstrapCommand), "--constraints", scons)
	c.Check(<-errc, IsNil)
	opBootstrap := (<-opc).(dummy.OpBootstrap)
	c.Check(opBootstrap.Env, Equals, "peckham")
	c.Check(opBootstrap.Constraints, DeepEquals, constraints.MustParse(scons))
}

func (*BootstrapSuite) TestUploadTools(c *C) {
	defer testing.MakeFakeHome(c, envConfig).Restore()
	origCurrent := version.Current
	version.Current.Series = "hostseries"
	defer func() { version.Current = origCurrent }()

	opc, errc := runCommand(new(BootstrapCommand), "--upload-tools")
	c.Check(<-errc, IsNil)
	c.Check((<-opc).(dummy.OpPutFile).Env, Equals, "peckham") // hostseries from version.Current
	c.Check((<-opc).(dummy.OpPutFile).Env, Equals, "peckham") // defaultseries from env config
	c.Check((<-opc).(dummy.OpPutFile).Env, Equals, "peckham") // precise from config.DefaultSeries
	opBootstrap := (<-opc).(dummy.OpBootstrap)
	c.Check(opBootstrap.Env, Equals, "peckham")
	c.Check(opBootstrap.Constraints, DeepEquals, constraints.Value{})

	vers := version.Current
	assertUploadedSomething(c, vers)
	vers.Series = "defaultseries"
	assertUploadedSomething(c, vers)
	vers.Series = "precise"
	assertUploadedSomething(c, vers)
}

func assertUploadedSomething(c *C, vers version.Binary) {
	// Check that some file was uploaded and can be unpacked; detailed
	// semantics tested elsewhere.
	envs, err := environs.ReadEnvirons("")
	c.Assert(err, IsNil)
	env, err := envs.Open("peckham")
	c.Assert(err, IsNil)
	tools, err := environs.FindTools(env, vers, environs.CompatVersion)
	c.Assert(err, IsNil)
	c.Assert(tools.Binary, Equals, vers)
	resp, err := http.Get(tools.URL)
	c.Assert(err, IsNil)
	defer resp.Body.Close()
	err = agent.UnpackTools(c.MkDir(), tools, resp.Body)
	c.Assert(err, IsNil)
}

func (*BootstrapSuite) TestUploadToolsSeries(c *C) {
	defer testing.MakeFakeHome(c, envConfig).Restore()
	opc, errc := runCommand(new(BootstrapCommand), "--upload-tools", "--series=good,great")
	c.Check(<-errc, IsNil)
	c.Check((<-opc).(dummy.OpPutFile).Env, Equals, "peckham")
	c.Check((<-opc).(dummy.OpPutFile).Env, Equals, "peckham")
	c.Check((<-opc).(dummy.OpPutFile).Env, Equals, "peckham")
	c.Check((<-opc).(dummy.OpBootstrap).Env, Equals, "peckham")

	vers := version.Current
	assertUploadedSomething(c, vers)
	vers.Series = "good"
	assertUploadedSomething(c, vers)
	vers.Series = "great"
	assertUploadedSomething(c, vers)
}

func (*BootstrapSuite) TestSeriesBadParams(c *C) {
	defer testing.MakeFakeHome(c, envConfig).Restore()
	opc, errc := runCommand(new(BootstrapCommand), "--series=bad1")
	c.Check(<-errc, ErrorMatches, `invalid value "bad1" for flag --series: invalid series name "bad1"`)
	c.Check(<-opc, IsNil)
}

func (*BootstrapSuite) TestSeriesNoUploadTools(c *C) {
	defer testing.MakeFakeHome(c, envConfig).Restore()
	opc, errc := runCommand(new(BootstrapCommand), "--series=good,great")
	c.Check(<-errc, ErrorMatches, `--series requires --upload-tools`)
	c.Check(<-opc, IsNil)
}

func (*BootstrapSuite) TestBrokenEnvironment(c *C) {
	defer testing.MakeFakeHome(c, envConfig).Restore()
	opc, errc := runCommand(new(BootstrapCommand), "-e", "brokenenv")
	c.Check(<-errc, ErrorMatches, "dummy.Bootstrap is broken")
	c.Check(<-opc, IsNil)
}

func (*BootstrapSuite) TestMissingEnvironment(c *C) {
	defer testing.MakeFakeHomeNoEnvironments(c, "empty").Restore()
	ctx := testing.Context(c)
	code := cmd.Main(&BootstrapCommand{}, ctx, nil)
	c.Check(code, Equals, 1)
	errStr := ctx.Stderr.(*bytes.Buffer).String()
	strippedErr := strings.Replace(errStr, "\n", "", -1)
	c.Assert(strippedErr, Matches, ".*No juju environment configuration file exists.*")
}
<|endoftext|>


<commit_before>package main // import "github.com/prometheus/prombench/cmd/prombench"

import (
	"fmt"
	"log"
	"os"
	"path/filepath"

	"github.com/pkg/errors"
	"github.com/prometheus/prombench/pkg/provider/gke"
	"gopkg.in/alecthomas/kingpin.v2"
)

func main() {
	log.SetFlags(log.LTime | log.Lshortfile)

	app := kingpin.New(filepath.Base(os.Args[0]), "The Prometheus benchmarking tool")
	app.HelpFlag.Short('h')

	g := gke.New()
	k8sGKE := app.Command("gke", `Google container engine provider - https://cloud.google.com/kubernetes-engine/`).
		Action(g.NewGKEClient)
	k8sGKE.Flag("auth", "json authentication file for the project - https://cloud.google.com/iam/docs/creating-managing-service-account-keys. If not set the tool will use the GOOGLE_APPLICATION_CREDENTIALS env variable (export GOOGLE_APPLICATION_CREDENTIALS=service-account.json)").
		PlaceHolder("service-account.json").
		Short('a').
		ExistingFileVar(&g.AuthFile)
	k8sGKE.Flag("file", "yaml file or folder  that describes the parameters for the object that will be deployed.").
		Required().
		Short('f').
		ExistingFilesOrDirsVar(&g.DeploymentFiles)
	k8sGKE.Flag("vars", "When provided it will substitute the token holders in the yaml file. Follows the standard golang template formating - {{ .hashStable }}.").
		Short('v').
		StringMapVar(&g.DeploymentVars)

	// Cluster operations.
	k8sGKECluster := k8sGKE.Command("cluster", "manage GKE clusters").
		Action(g.GKEDeploymentsParse)
	k8sGKECluster.Command("create", "gke cluster create -a service-account.json -f FileOrFolder").
		Action(g.ClusterCreate)
	k8sGKECluster.Command("delete", "gke cluster delete -a service-account.json -f FileOrFolder").
		Action(g.ClusterDelete)

	// Cluster node-pool operations
	k8sGKENodePool := k8sGKE.Command("nodepool", "manage GKE clusters nodepools").
		Action(g.GKEDeploymentsParse)
	k8sGKENodePool.Command("create", "gke nodepool create -a service-account.json -f FileOrFolder").
		Action(g.NodePoolCreate)
	k8sGKENodePool.Command("delete", "gke nodepool delete -a service-account.json -f FileOrFolder").
		Action(g.NodePoolDelete)

	// K8s resource operations.
	k8sGKEResource := k8sGKE.Command("resource", `Apply and delete different k8s resources - deployments, services, config maps etc.Required variables -v PROJECT_ID, -v ZONE: -west1-b -v CLUSTER_NAME`).
		Action(g.NewK8sProvider).
		Action(g.K8SDeploymentsParse)
	k8sGKEResource.Command("apply", "gke resource apply -a service-account.json -f manifestsFileOrFolder -v PROJECT_ID:test -v ZONE:europe-west1-b -v CLUSTER_NAME:test -v hashStable:COMMIT1 -v hashTesting:COMMIT2").
		Action(g.ResourceApply)
	k8sGKEResource.Command("delete", "gke resource delete -a service-account.json -f manifestsFileOrFolder -v PROJECT_ID:test -v ZONE:europe-west1-b -v CLUSTER_NAME:test -v hashStable:COMMIT1 -v hashTesting:COMMIT2").
		Action(g.ResourceDelete)

	if _, err := app.Parse(os.Args[1:]); err != nil {
		fmt.Fprintln(os.Stderr, errors.Wrapf(err, "Error parsing commandline arguments"))
		app.Usage(os.Args[1:])
		os.Exit(2)
	}

}
<commit_msg>fix<commit_after>package main // import "github.com/prometheus/prombench/cmd/prombench"

import (
	"fmt"
	"log"
	"os"
	"path/filepath"

	"github.com/pkg/errors"
	"github.com/prometheus/prombench/pkg/provider/gke"
	"gopkg.in/alecthomas/kingpin.v2"
)

func main() {
	log.SetFlags(log.Ltime | log.Lshortfile)

	app := kingpin.New(filepath.Base(os.Args[0]), "The Prometheus benchmarking tool")
	app.HelpFlag.Short('h')

	g := gke.New()
	k8sGKE := app.Command("gke", `Google container engine provider - https://cloud.google.com/kubernetes-engine/`).
		Action(g.NewGKEClient)
	k8sGKE.Flag("auth", "json authentication file for the project - https://cloud.google.com/iam/docs/creating-managing-service-account-keys. If not set the tool will use the GOOGLE_APPLICATION_CREDENTIALS env variable (export GOOGLE_APPLICATION_CREDENTIALS=service-account.json)").
		PlaceHolder("service-account.json").
		Short('a').
		ExistingFileVar(&g.AuthFile)
	k8sGKE.Flag("file", "yaml file or folder  that describes the parameters for the object that will be deployed.").
		Required().
		Short('f').
		ExistingFilesOrDirsVar(&g.DeploymentFiles)
	k8sGKE.Flag("vars", "When provided it will substitute the token holders in the yaml file. Follows the standard golang template formating - {{ .hashStable }}.").
		Short('v').
		StringMapVar(&g.DeploymentVars)

	// Cluster operations.
	k8sGKECluster := k8sGKE.Command("cluster", "manage GKE clusters").
		Action(g.GKEDeploymentsParse)
	k8sGKECluster.Command("create", "gke cluster create -a service-account.json -f FileOrFolder").
		Action(g.ClusterCreate)
	k8sGKECluster.Command("delete", "gke cluster delete -a service-account.json -f FileOrFolder").
		Action(g.ClusterDelete)

	// Cluster node-pool operations
	k8sGKENodePool := k8sGKE.Command("nodepool", "manage GKE clusters nodepools").
		Action(g.GKEDeploymentsParse)
	k8sGKENodePool.Command("create", "gke nodepool create -a service-account.json -f FileOrFolder").
		Action(g.NodePoolCreate)
	k8sGKENodePool.Command("delete", "gke nodepool delete -a service-account.json -f FileOrFolder").
		Action(g.NodePoolDelete)

	// K8s resource operations.
	k8sGKEResource := k8sGKE.Command("resource", `Apply and delete different k8s resources - deployments, services, config maps etc.Required variables -v PROJECT_ID, -v ZONE: -west1-b -v CLUSTER_NAME`).
		Action(g.NewK8sProvider).
		Action(g.K8SDeploymentsParse)
	k8sGKEResource.Command("apply", "gke resource apply -a service-account.json -f manifestsFileOrFolder -v PROJECT_ID:test -v ZONE:europe-west1-b -v CLUSTER_NAME:test -v hashStable:COMMIT1 -v hashTesting:COMMIT2").
		Action(g.ResourceApply)
	k8sGKEResource.Command("delete", "gke resource delete -a service-account.json -f manifestsFileOrFolder -v PROJECT_ID:test -v ZONE:europe-west1-b -v CLUSTER_NAME:test -v hashStable:COMMIT1 -v hashTesting:COMMIT2").
		Action(g.ResourceDelete)

	if _, err := app.Parse(os.Args[1:]); err != nil {
		fmt.Fprintln(os.Stderr, errors.Wrapf(err, "Error parsing commandline arguments"))
		app.Usage(os.Args[1:])
		os.Exit(2)
	}

}
<|endoftext|>


<commit_before>package ginDoi

import (
	"fmt"
	log "github.com/Sirupsen/logrus"
	"io/ioutil"
	"net/http"
	"os/exec"
	"crypto/rsa"
	"os"
	"crypto/md5"
	"encoding/hex"
	"gopkg.in/yaml.v2"
)

type GogsDataSource struct {
	GinURL    string
	GinGitURL string
	pubKey    string
}

func (s *GogsDataSource) getDoiFile(URI string, user OauthIdentity) ([]byte, error) {
	//git archive --remote=git://git.foo.com/project.git HEAD:path/to/directory filename
	//https://github.com/go-yaml/yaml.git
	//git@github.com:go-yaml/yaml.git
	fetchRepoPath := fmt.Sprintf("%s/raw/master/datacite.yml", URI)
	client := &http.Client{}
	req, _ := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/%s", s.GinURL, fetchRepoPath), nil)
	req.Header.Add("Cookie", fmt.Sprintf("i_like_gogits=%s", user.Token))
	resp, err := client.Do(req)
	if err != nil {
		// todo Try to infer what went wrong
		log.WithFields(log.Fields{
			"path":   fetchRepoPath,
			"source": DSOURCELOGPREFIX,
			"error":  err,
		}).Debug("Could not get doifile")
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("could not get doifile:%+v,%s", resp.Status)
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.WithFields(log.Fields{
			"path":   fetchRepoPath,
			"source": DSOURCELOGPREFIX,
			"error":  err,
		}).Debug("Could not read from received Clodberry")
		return nil, err
	}
	return body, nil
}

func (s *GogsDataSource) Get(URI string, To string, key *rsa.PrivateKey) (string, error) {
	gin_uri := fmt.Sprintf("%s/%s.git", s.GinGitURL, strings.ToLower(URI))
	log.WithFields(log.Fields{
		"URI":     URI,
		"gin_uri": gin_uri,
		"to":      To,
		"source":  DSOURCELOGPREFIX,
	}).Debug("Start cloning")

	//Create tmp ssh keys files from the key provided
	tmpDir, err := ioutil.TempDir("", "")
	if err != nil {
		log.WithFields(log.Fields{
			"source": DSOURCELOGPREFIX,
			"error":  err,
		}).Error("SSH key tmp dir not created")
		return "", err
	}

	cmd := exec.Command("git", "clone", gin_uri, To)
	env := os.Environ()
	// If a key was provided we need to use that with nthe ssh
	if key != nil {
		_, priv_path, err := WriteSSHKeyPair(tmpDir, key)
		if err != nil {
			log.WithFields(log.Fields{
				"source": DSOURCELOGPREFIX,
				"error":  err,
			}).Error("SSH key storing failed")
			return "", err
		}
		env = append(env, fmt.Sprintf("GIT_SSH_COMMAND=ssh -i %s", priv_path))
		env = append(env, "GIT_COMMITTER_NAME=GINDOI")
		env = append(env, "GIT_COMMITTER_EMAIL=doi@g-node.org")
		cmd.Env = env
	}
	out, err := cmd.CombinedOutput()
	log.WithFields(log.Fields{
		"URI":     URI,
		"gin_uri": gin_uri,
		"to":      To,
		"out":     string(out),
		"source":  DSOURCELOGPREFIX,
	}).Debug("Done with cloning")
	if err != nil {
		log.WithFields(log.Fields{
			"URI":     URI,
			"gin_uri": gin_uri,
			"to":      To,
			"source":  DSOURCELOGPREFIX,
			"error":   string(out),
		}).Debug("Cloning did not work")
		return string(out), err
	}

	cmd = exec.Command("git-annex", "get")
	cmd.Dir = To
	cmd.Env = env
	out, err = cmd.CombinedOutput()
	if err != nil {
		// Workaround for uninitilaizes git annexes (-> return nil)
		// todo
		log.WithFields(log.Fields{
			"source": DSOURCELOGPREFIX,
			"error":  string(out),
		}).Debug("Annex get failed")
	}
	cmd = exec.Command("git-annex", "uninit")
	cmd.Dir = To
	cmd.Env = env
	out, err = cmd.CombinedOutput()
	if err != nil {
		// Workaround for uninitilaizes git annexes (-> return nil)
		// todo
		log.WithFields(log.Fields{
			"source": DSOURCELOGPREFIX,
			"error":  string(out),
		}).Debug("Anex unlock failed")
	}
	return string(out), nil
}

func (s *GogsDataSource) MakeUUID(URI string, user OauthIdentity) (string, error) {
	fetchRepoPath := fmt.Sprintf("%s", URI)
	client := &http.Client{}
	req, _ := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/%s", s.GinURL, fetchRepoPath), nil)
	req.Header.Add("Cookie", fmt.Sprintf("i_like_gogits=%s", user.Token))
	resp, err := client.Do(req)
	// todo error checking
	if err != nil {
		return "", err
	}
	if bd, err := ioutil.ReadAll(resp.Body); err != nil {
		return "", err
	} else {
		currMd5 := md5.Sum(bd)
		return hex.EncodeToString(currMd5[:]), nil
	}
}

// Return true if the specifies URI "has" a doi File containing all nec. information
func (s *GogsDataSource) ValidDoiFile(URI string, user OauthIdentity) (bool, *CBerry) {
	in, err := s.getDoiFile(URI, user)
	if err != nil {
		log.WithFields(log.Fields{
			"data":   string(in),
			"source": DSOURCELOGPREFIX,
			"error":  err,
		}).Error("Could not get the doi file")
		return false, nil
	}
	doiInfo := CBerry{}
	err = yaml.Unmarshal(in, &doiInfo)
	if err != nil {
		log.WithFields(log.Fields{
			"data":   string(in),
			"source": DSOURCELOGPREFIX,
			"error":  err,
		}).Error("Could not unmarshal doifile")
		return false, &CBerry{}
	}
	if !hasValues(&doiInfo) {
		log.WithFields(log.Fields{
			"data":    in,
			"doiInfo": doiInfo,
			"source":  DSOURCELOGPREFIX,
			"error":   err,
		}).Debug("Doi File misses entries")
		return false, &doiInfo
	}
	return true, &doiInfo
}
<commit_msg>Doi data UUID should be generated from repo path<commit_after>package ginDoi

import (
	"fmt"
	log "github.com/Sirupsen/logrus"
	"io/ioutil"
	"net/http"
	"os/exec"
	"crypto/rsa"
	"os"
	"crypto/md5"
	"encoding/hex"
	"gopkg.in/yaml.v2"
	"strings"
	"encoding/json"
	"github.com/gogits/go-gogs-client"
)

type GogsDataSource struct {
	GinURL    string
	GinGitURL string
	pubKey    string
}

func (s *GogsDataSource) getDoiFile(URI string, user OauthIdentity) ([]byte, error) {
	//git archive --remote=git://git.foo.com/project.git HEAD:path/to/directory filename
	//https://github.com/go-yaml/yaml.git
	//git@github.com:go-yaml/yaml.git
	fetchRepoPath := fmt.Sprintf("%s/raw/master/datacite.yml", URI)
	client := &http.Client{}
	req, _ := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/%s", s.GinURL, fetchRepoPath), nil)
	req.Header.Add("Cookie", fmt.Sprintf("i_like_gogits=%s", user.Token))
	resp, err := client.Do(req)
	if err != nil {
		// todo Try to infer what went wrong
		log.WithFields(log.Fields{
			"path":   fetchRepoPath,
			"source": DSOURCELOGPREFIX,
			"error":  err,
		}).Debug("Could not get doifile")
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("could not get doifile:%+v,%s", resp.Status)
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.WithFields(log.Fields{
			"path":   fetchRepoPath,
			"source": DSOURCELOGPREFIX,
			"error":  err,
		}).Debug("Could not read from received Clodberry")
		return nil, err
	}
	return body, nil
}

func (s *GogsDataSource) Get(URI string, To string, key *rsa.PrivateKey) (string, error) {
	gin_uri := fmt.Sprintf("%s/%s.git", s.GinGitURL, strings.ToLower(URI))
	log.WithFields(log.Fields{
		"URI":     URI,
		"gin_uri": gin_uri,
		"to":      To,
		"source":  DSOURCELOGPREFIX,
	}).Debug("Start cloning")

	//Create tmp ssh keys files from the key provided
	tmpDir, err := ioutil.TempDir("", "")
	if err != nil {
		log.WithFields(log.Fields{
			"source": DSOURCELOGPREFIX,
			"error":  err,
		}).Error("SSH key tmp dir not created")
		return "", err
	}

	cmd := exec.Command("git", "clone", gin_uri, To)
	env := os.Environ()
	// If a key was provided we need to use that with nthe ssh
	if key != nil {
		_, priv_path, err := WriteSSHKeyPair(tmpDir, key)
		if err != nil {
			log.WithFields(log.Fields{
				"source": DSOURCELOGPREFIX,
				"error":  err,
			}).Error("SSH key storing failed")
			return "", err
		}
		env = append(env, fmt.Sprintf("GIT_SSH_COMMAND=ssh -i %s", priv_path))
		env = append(env, "GIT_COMMITTER_NAME=GINDOI")
		env = append(env, "GIT_COMMITTER_EMAIL=doi@g-node.org")
		cmd.Env = env
	}
	out, err := cmd.CombinedOutput()
	log.WithFields(log.Fields{
		"URI":     URI,
		"gin_uri": gin_uri,
		"to":      To,
		"out":     string(out),
		"source":  DSOURCELOGPREFIX,
	}).Debug("Done with cloning")
	if err != nil {
		log.WithFields(log.Fields{
			"URI":     URI,
			"gin_uri": gin_uri,
			"to":      To,
			"source":  DSOURCELOGPREFIX,
			"error":   string(out),
		}).Debug("Cloning did not work")
		return string(out), err
	}

	cmd = exec.Command("git-annex", "get")
	cmd.Dir = To
	cmd.Env = env
	out, err = cmd.CombinedOutput()
	if err != nil {
		// Workaround for uninitilaizes git annexes (-> return nil)
		// todo
		log.WithFields(log.Fields{
			"source": DSOURCELOGPREFIX,
			"error":  string(out),
		}).Debug("Annex get failed")
	}
	cmd = exec.Command("git-annex", "uninit")
	cmd.Dir = To
	cmd.Env = env
	out, err = cmd.CombinedOutput()
	if err != nil {
		// Workaround for uninitilaizes git annexes (-> return nil)
		// todo
		log.WithFields(log.Fields{
			"source": DSOURCELOGPREFIX,
			"error":  string(out),
		}).Debug("Anex unlock failed")
	}
	return string(out), nil
}

func (s *GogsDataSource) MakeUUID(URI string, user OauthIdentity) (string, error) {
	currMd5 := md5.Sum([]byte(URI))
	return hex.EncodeToString(currMd5[:]), nil
}

// Determine the lates commit id of the master branch
func (s *GogsDataSource) GetMasterCommit(URI string, user OauthIdentity) (string, error) {
	fetchRepoPath := fmt.Sprintf("%s", URI)
	client := &http.Client{}
	req, _ := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/repos/%s/branches", s.GinURL, fetchRepoPath), nil)
	req.Header.Add("Cookie", fmt.Sprintf("i_like_gogits=%s", user.Token))
	resp, err := client.Do(req)
	defer resp.Body.Close()
	if err != nil {
		return "", err
	}
	if resp.StatusCode != http.StatusOK {
		return "", fmt.Errorf("Could not get repo branches: %s", resp.Status)
	}

	branches := []gogs.Branch{}
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	json.Unmarshal(data, &branches)
	for _, branch := range branches {
		if branch.Name == "master" {
			return branch.Commit.ID, nil
		}
	}
	return "", fmt.Errorf("Could not locate master branch")
}

// Return true if the specifies URI "has" a doi File containing all nec. information
func (s *GogsDataSource) ValidDoiFile(URI string, user OauthIdentity) (bool, *CBerry) {
	in, err := s.getDoiFile(URI, user)
	if err != nil {
		log.WithFields(log.Fields{
			"data":   string(in),
			"source": DSOURCELOGPREFIX,
			"error":  err,
		}).Error("Could not get the doi file")
		return false, nil
	}
	doiInfo := CBerry{}
	err = yaml.Unmarshal(in, &doiInfo)
	if err != nil {
		log.WithFields(log.Fields{
			"data":   string(in),
			"source": DSOURCELOGPREFIX,
			"error":  err,
		}).Error("Could not unmarshal doifile")
		return false, &CBerry{}
	}
	if !hasValues(&doiInfo) {
		log.WithFields(log.Fields{
			"data":    in,
			"doiInfo": doiInfo,
			"source":  DSOURCELOGPREFIX,
			"error":   err,
		}).Debug("Doi File misses entries")
		return false, &doiInfo
	}
	return true, &doiInfo
}
<|endoftext|>


<commit_before>package graph

import (
	"http"
	"io/ioutil"
	"json"
	"os"
	"strconv"
	"fmt"
	"time"
)

type Object struct {
	ID   string
	Name string
}

func parseObject(value map[string]interface{}) (obj Object) {
	obj.ID = value["id"].(string)
	obj.Name = value["name"].(string)
	return
}

func parseObjects(value []interface{}) (objs []Object) {
	objs = make([]Object, len(value))
	for i, v := range value {
		objs[i] = parseObject(v.(map[string]interface{}))
	}
	return
}

func getData(url string) (value []interface{}, err os.Error) {
	resp, _, err := http.Get(url) // Response, final URL, error
	if err != nil {
		return
	}
	defer resp.Body.Close()
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}
	data, err := getJsonMap(b)
	if err != nil {
		return
	}
	value, ok := data["data"].([]interface{})
	if !ok {
		err = os.NewError("getData: Couldn't parse data,")
	}
	return
}

type URL struct {
	Name string
	URL  string
}

func parseURL(value map[string]interface{}) (url URL) {
	url.Name = value["name"].(string)
	url.URL = value["link"].(string)
	return
}

func parseURLs(value []interface{}) (urls []URL) {
	urls = make([]URL, len(value))
	for i, v := range value {
		urls[i] = parseURL(v.(map[string]interface{}))
	}
	return
}

func getObject(id string) (data map[string]interface{}, err os.Error) {
	return getObjByURL(GRAPHURL + id + "?metadata=1")
}

func getObjByURL(url string) (data map[string]interface{}, err os.Error) {
	// TODO: Check for valid ID
	resp, _, err := http.Get(url) // Response, final URL, error
	if err != nil {
		return
	}
	defer resp.Body.Close()
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}
	data, err = getJsonMap(b)
	return
}

func getJsonMap(body []byte) (data map[string]interface{}, err os.Error) {
	var values interface{}
	if err = json.Unmarshal(body, &values); err != nil {
		return
	}
	data = values.(map[string]interface{})
	if e, ok := data["error"]; ok == true {
		error := e.(map[string]interface{})
		t := error["type"].(string)
		message := error["message"].(string)
		err = os.NewError(t + ": " + message)
	}
	return
}

func parseTime(value string) (t *time.Time, err os.Error) {
	t, err = time.Parse("RFC3339", value)
	if err != nil {
		t, err = time.Parse("2006-01-02T15:04:05+0000", value)
		if err != nil {
			t, err = time.Parse("January 2, 2006", value)
			if err != nil {
				t, err = time.Parse("Jan 2006", value)
			}
		}
	}
	return
}

func post(url string, data map[string]string) (err os.Error) {
	_, err = http.PostForm(url, data)
	return
}

func debugInterface(value interface{}, key, funcName string) {
	var str string
	switch value.(type) {
	case float64:
		str = strconv.Ftoa64(value.(float64), 'e', -1)
	}
	fmt.Printf("%s: Unknown pair: %s : %s\n", funcName, key, str)
}
<commit_msg>More error checking.<commit_after>package graph

import (
	"http"
	"io/ioutil"
	"json"
	"os"
	"strconv"
	"fmt"
	"time"
)

type Object struct {
	ID   string
	Name string
}

func parseObject(value map[string]interface{}) (obj Object) {
	obj.ID = value["id"].(string)
	obj.Name = value["name"].(string)
	return
}

func parseObjects(value []interface{}) (objs []Object) {
	objs = make([]Object, len(value))
	for i, v := range value {
		if v == nil {
			// TODO: Do something about this
		} else {
			objs[i] = parseObject(v.(map[string]interface{}))
		}
	}
	return
}

func getData(url string) (value []interface{}, err os.Error) {
	if len(url) == 0 {
		return value, os.NewError("getData: url is empty.")
	}
	resp, _, err := http.Get(url) // Response, final URL, error
	if err != nil {
		return
	}
	defer resp.Body.Close()
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}
	data, err := getJsonMap(b)
	if err != nil {
		return
	}
	value, ok := data["data"].([]interface{})
	if !ok {
		err = os.NewError("getData: Couldn't parse data,")
	}
	return
}

type URL struct {
	Name string
	URL  string
}

func parseURL(value map[string]interface{}) (url URL) {
	url.Name = value["name"].(string)
	url.URL = value["link"].(string)
	return
}

func parseURLs(value []interface{}) (urls []URL) {
	urls = make([]URL, len(value))
	for i, v := range value {
		urls[i] = parseURL(v.(map[string]interface{}))
	}
	return
}

func getObject(id string) (data map[string]interface{}, err os.Error) {
	return getObjByURL(GRAPHURL + id + "?metadata=1")
}

func getObjByURL(url string) (data map[string]interface{}, err os.Error) {
	if url == "" {
		return data, os.NewError("getObjByURL: url is empty.")
	}
	resp, _, err := http.Get(url) // Response, final URL, error
	if err != nil {
		return
	}
	defer resp.Body.Close()
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}
	data, err = getJsonMap(b)
	return
}

func getJsonMap(body []byte) (data map[string]interface{}, err os.Error) {
	var values interface{}
	if err = json.Unmarshal(body, &values); err != nil {
		return
	}
	data = values.(map[string]interface{})
	if e, ok := data["error"]; ok == true {
		error := e.(map[string]interface{})
		t := error["type"].(string)
		message := error["message"].(string)
		err = os.NewError(t + ": " + message)
	}
	return
}

func parseTime(value string) (t *time.Time, err os.Error) {
	t, err = time.Parse("RFC3339", value)
	if err != nil {
		t, err = time.Parse("2006-01-02T15:04:05+0000", value)
		if err != nil {
			t, err = time.Parse("January 2, 2006", value)
			if err != nil {
				t, err = time.Parse("Jan 2006", value)
			}
		}
	}
	return
}

func post(url string, data map[string]string) (err os.Error) {
	_, err = http.PostForm(url, data)
	return
}

func debugInterface(value interface{}, key, funcName string) {
	var str string
	switch value.(type) {
	case float64:
		str = strconv.Ftoa64(value.(float64), 'e', -1)
	}
	fmt.Printf("%s: Unknown pair: %s : %s\n", funcName, key, str)
}
<|endoftext|>


<commit_before>package catalog

import (
	"errors"
	"fmt"
	"regexp"
	"strings"

	yarn "github.com/omeid/go-yarn"
)

var (
	// ErrUnknownCommentTag is return when a line that is expected to be
	// a starting line starts with an unsupported comment type.
	// i.e: it is not `--`, `#`, `//`, or `/*`.
	ErrUnknownCommentTag = errors.New("Uknown comment pair")

	// ErrYarnNotFound is returned when the requested file does not
	// exist in the provided yarn.
	ErrYarnNotFound = errors.New("Yarn not found")
)

// MustParse is like Parse, but panics if the named string
// is not found in the provied Yarn or fails to parse
// the catalog.
func MustParse(y yarn.Yarn, name string) yarn.Yarn {
	// files, err := parse(f, Pattern)
	var err error
	if err != nil {
		panic(err)
	}

	return nil
}

const nl = "\n"

const (
	commentAdaLike  = "--"
	commentBashLike = "##"
	commentCLike    = "//"

	commentCBlockStart = "/*"
	commentCBlockEnd   = "*/"
)

func endTag(start string) (string, error) {
	switch start {
	case commentAdaLike,
		commentBashLike,
		commentCLike:
		return "", nil // requires no endtag.
	case commentCBlockStart:
		return commentCBlockEnd, nil
	default:
		return "", ErrUnknownCommentTag
	}
}

// Parse the value of catalog from the provided yarn.
func Parse(y yarn.Yarn, catalog string) (yarn.Yarn, error) {
	source, ok := y.Get(catalog)

	if !ok {
		return nil, ErrYarnNotFound
	}

	return parse(source)
}

// ParseString is like Parse, but accept a string as source.
func ParseString(source string) (yarn.Yarn, error) {
	return parse(source)
}

// Parse loads the value of name from y into a new Yarn.
func parse(source string) (yarn.Yarn, error) {

	files := map[string]string{}
	var (
		// per catalog.
		start *regexp.Regexp
		end   *regexp.Regexp

		// per entry.
		entryName    string
		entryContent string
		err          error
	)

	entryAdd := func() {
		files[entryName] = entryContent
		entryName = ""
		entryContent = ""
	}

	entryReadName := func(i int, line string) error {

		// only once, we only support one comment type per catalog.
		if start == nil {
			var err error
			start, end, err = getDelims(line)
			if err != nil {
				return err
			}
		}

		segs := start.FindStringSubmatch(line)

		if len(segs) < 2 {
			return fmt.Errorf("Invalid start tag at line %v", i)
		}

		entryName = segs[1]
		return nil
	}

	entryEndline := func(i int, line string) (bool, error) {

		segs := end.FindStringSubmatch(line)
		if len(segs) == 0 {
			return false, nil
		}

		if len(segs) < 2 {
			return true, fmt.Errorf("Invalid end tag at line %v", i)
		}

		if segs[1] != entryName {
			return true, fmt.Errorf("Invalid end tag name %v expected %v at line %v", segs[1], entryName, i)
		}

		return true, nil
	}

	for i, line := range strings.Split(source, nl) {

		if entryName != "" {
			end, err := entryEndline(i, line)

			if err != nil {
				return nil, err
			}

			if end {
				entryAdd()
				continue
			}

			// Don't add newline to the start of the content.
			if entryContent != "" {
				line = nl + line
			}

			entryContent += line
			continue
		}

		// whitespace only.
		if len(strings.Fields(line)) == 0 {
			continue
		}

		// then it must be starting tag.
		err = entryReadName(i, line)
		if err != nil {
			return nil, err
		}
	}

	if entryName != "" {
		// Unexpected end.
	}

	return yarn.NewFromMap(files), nil
}

func getDelims(line string) (*regexp.Regexp, *regexp.Regexp, error) {
	if len(line) < 2 {
		return nil, nil, ErrUnknownCommentTag
	}

	labels := []string{"start", "end"}

	start := line[:2]
	end, err := endTag(start)

	if err != nil {
		return nil, nil, err
	}

	regs, err := makeRegexs(labels, start, end)
	if err != nil {
		return nil, nil, err
	}

	return regs[0], regs[1], nil
}

func makeRegexs(labels []string, start, end string) ([]*regexp.Regexp, error) {
	regs := make([]*regexp.Regexp, len(labels))

	var err error

	start = regexp.QuoteMeta(start)
	end = regexp.QuoteMeta(end)
	for i, label := range labels {

		reg := start + `\s+` + label + `\s?:\s*(.*)?`
		if end != "" {
			reg = reg + `\s+` + end + `\s*$`
		} else {
			reg = reg + `\s*$`
		}

		regs[i], err = regexp.Compile(reg)
		if err != nil {
			return nil, err
		}
	}
	return regs, nil
}
<commit_msg>whops.<commit_after>package catalog

import (
	"errors"
	"fmt"
	"regexp"
	"strings"

	yarn "github.com/omeid/go-yarn"
)

var (
	// ErrUnknownCommentTag is return when a line that is expected to be
	// a starting line starts with an unsupported comment type.
	// i.e: it is not `--`, `#`, `//`, or `/*`.
	ErrUnknownCommentTag = errors.New("Uknown comment pair")

	// ErrYarnNotFound is returned when the requested file does not
	// exist in the provided yarn.
	ErrYarnNotFound = errors.New("Yarn not found")

	// ErrUnexpectedEnd is returned when a catalog is malformed.
	ErrUnexpectedEnd = errors.New("Unexpected end")
)

// MustParse is like Parse, but panics if the named string
// is not found in the provied Yarn or fails to parse
// the catalog.
func MustParse(y yarn.Yarn, name string) yarn.Yarn {
	yarn, err := Parse(y, name)
	if err != nil {
		panic(err)
	}

	return yarn
}

const nl = "\n"

const (
	commentAdaLike  = "--"
	commentBashLike = "##"
	commentCLike    = "//"

	commentCBlockStart = "/*"
	commentCBlockEnd   = "*/"
)

func endTag(start string) (string, error) {
	switch start {
	case commentAdaLike,
		commentBashLike,
		commentCLike:
		return "", nil // requires no endtag.
	case commentCBlockStart:
		return commentCBlockEnd, nil
	default:
		return "", ErrUnknownCommentTag
	}
}

// Parse the value of catalog from the provided yarn.
func Parse(y yarn.Yarn, catalog string) (yarn.Yarn, error) {
	source, ok := y.Get(catalog)

	if !ok {
		return nil, ErrYarnNotFound
	}

	return parse(source)
}

// ParseString is like Parse, but accept a string as source.
func ParseString(source string) (yarn.Yarn, error) {
	return parse(source)
}

// Parse loads the value of name from y into a new Yarn.
func parse(source string) (yarn.Yarn, error) {

	files := map[string]string{}
	var (
		// per catalog.
		start *regexp.Regexp
		end   *regexp.Regexp

		// per entry.
		entryName    string
		entryContent string
		err          error
	)

	entryAdd := func() {
		files[entryName] = entryContent
		entryName = ""
		entryContent = ""
	}

	entryReadName := func(i int, line string) error {

		// only once, we only support one comment type per catalog.
		if start == nil {
			var err error
			start, end, err = getDelims(line)
			if err != nil {
				return err
			}
		}

		segs := start.FindStringSubmatch(line)

		if len(segs) < 2 {
			return fmt.Errorf("Invalid start tag at line %v", i)
		}

		entryName = segs[1]
		return nil
	}

	entryEndline := func(i int, line string) (bool, error) {

		segs := end.FindStringSubmatch(line)
		if len(segs) == 0 {
			return false, nil
		}

		if len(segs) < 2 {
			return true, fmt.Errorf("Invalid end tag at line %v", i)
		}

		if segs[1] != entryName {
			return true, fmt.Errorf("Invalid end tag name %v expected %v at line %v", segs[1], entryName, i)
		}

		return true, nil
	}

	for i, line := range strings.Split(source, nl) {

		if entryName != "" {
			end, err := entryEndline(i, line)

			if err != nil {
				return nil, err
			}

			if end {
				entryAdd()
				continue
			}

			// Don't add newline to the start of the content.
			if entryContent != "" {
				line = nl + line
			}

			entryContent += line
			continue
		}

		// whitespace only.
		if len(strings.Fields(line)) == 0 {
			continue
		}

		// then it must be starting tag.
		err = entryReadName(i, line)
		if err != nil {
			return nil, err
		}
	}

	if entryName != "" {
		// Unexpected end.
		return nil, ErrUnexpectedEnd
	}

	return yarn.NewFromMap(files), nil
}

func getDelims(line string) (*regexp.Regexp, *regexp.Regexp, error) {
	if len(line) < 2 {
		return nil, nil, ErrUnknownCommentTag
	}

	labels := []string{"start", "end"}

	start := line[:2]
	end, err := endTag(start)

	if err != nil {
		return nil, nil, err
	}

	regs, err := makeRegexs(labels, start, end)
	if err != nil {
		return nil, nil, err
	}

	return regs[0], regs[1], nil
}

func makeRegexs(labels []string, start, end string) ([]*regexp.Regexp, error) {
	regs := make([]*regexp.Regexp, len(labels))

	var err error

	start = regexp.QuoteMeta(start)
	end = regexp.QuoteMeta(end)
	for i, label := range labels {

		reg := start + `\s+` + label + `\s?:\s*(.*)?`
		if end != "" {
			reg = reg + `\s+` + end + `\s*$`
		} else {
			reg = reg + `\s*$`
		}

		regs[i], err = regexp.Compile(reg)
		if err != nil {
			return nil, err
		}
	}
	return regs, nil
}
<|endoftext|>