<commit_before>package main

import (
	"crypto/rand"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"runtime"
	"strconv"
	"time"

	"golang.org/x/net/context"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"

	flags "github.com/btcsuite/go-flags"
	proxy "github.com/grpc-ecosystem/grpc-gateway/runtime"
	"github.com/lightningnetwork/lnd/channeldb"
	"github.com/lightningnetwork/lnd/lnrpc"
	"github.com/lightningnetwork/lnd/lnwallet"
	"github.com/lightningnetwork/lnd/lnwire"
	"github.com/roasbeef/btcd/btcec"
	"github.com/roasbeef/btcutil"
)

const (
	autogenCertValidity = 10 * 365 * 24 * time.Hour
)

var (
	cfg              *config
	shutdownChannel  = make(chan struct{})
	registeredChains = newChainRegistry()
)

// lndMain is the true entry point for lnd. This function is required since
// defers created in the top-level scope of a main method aren't executed if
// os.Exit() is called.
func lndMain() error {
	// Load the configuration, and parse any command line options. This
	// function will also set up logging properly.
	loadedConfig, err := loadConfig()
	if err != nil {
		return err
	}
	cfg = loadedConfig
	defer func() {
		if logRotator != nil {
			logRotator.Close()
		}
	}()

	// Show version at startup.
	ltndLog.Infof("Version %s", version())

	// Enable http profiling server if requested.
	if cfg.Profile != "" {
		go func() {
			listenAddr := net.JoinHostPort("", cfg.Profile)
			profileRedirect := http.RedirectHandler("/debug/pprof",
				http.StatusSeeOther)
			http.Handle("/", profileRedirect)
			fmt.Println(http.ListenAndServe(listenAddr, nil))
		}()
	}

	// Open the channeldb, which is dedicated to storing channel, and
	// network related metadata.
	chanDB, err := channeldb.Open(cfg.DataDir)
	if err != nil {
		ltndLog.Errorf("unable to open channeldb: %v", err)
		return err
	}
	defer chanDB.Close()

	// With the information parsed from the configuration, create valid
	// instances of the paertinent interfaces required to operate the
	// Lightning Network Daemon.
	activeChainControl, chainCleanUp, err := newChainControlFromConfig(cfg, chanDB)
	if err != nil {
		fmt.Printf("unable to create chain control: %v\n", err)
		return err
	}
	if chainCleanUp != nil {
		defer chainCleanUp()
	}

	// Finally before we start the server, we'll register the "holy
	// trinity" of interface for our current "home chain" with the active
	// chainRegistry interface.
	primaryChain := registeredChains.PrimaryChain()
	registeredChains.RegisterChain(primaryChain, activeChainControl)

	idPrivKey, err := activeChainControl.wallet.GetIdentitykey()
	if err != nil {
		return err
	}
	idPrivKey.Curve = btcec.S256()

	// Set up the core server which will listen for incoming peer
	// connections.
	defaultListenAddrs := []string{
		net.JoinHostPort("", strconv.Itoa(cfg.PeerPort)),
	}
	server, err := newServer(defaultListenAddrs, chanDB, activeChainControl,
		idPrivKey)
	if err != nil {
		srvrLog.Errorf("unable to create server: %v\n", err)
		return err
	}

	// Next, we'll initialize the funding manager itself so it can answer
	// queries while the wallet+chain are still syncing.
	nodeSigner := newNodeSigner(idPrivKey)
	var chanIDSeed [32]byte
	if _, err := rand.Read(chanIDSeed[:]); err != nil {
		return err
	}
	fundingMgr, err := newFundingManager(fundingConfig{
		IDKey:        idPrivKey.PubKey(),
		Wallet:       activeChainControl.wallet,
		Notifier:     activeChainControl.chainNotifier,
		FeeEstimator: activeChainControl.feeEstimator,
		SignMessage: func(pubKey *btcec.PublicKey,
			msg []byte) (*btcec.Signature, error) {

			if pubKey.IsEqual(idPrivKey.PubKey()) {
				return nodeSigner.SignMessage(pubKey, msg)
			}

			return activeChainControl.msgSigner.SignMessage(
				pubKey, msg,
			)
		},
		CurrentNodeAnnouncement: func() (lnwire.NodeAnnouncement, error) {
			return server.genNodeAnnouncement(true)
		},
		SendAnnouncement: func(msg lnwire.Message) error {
			errChan := server.discoverSrv.ProcessLocalAnnouncement(msg,
				idPrivKey.PubKey())
			return <-errChan
		},
		ArbiterChan:    server.breachArbiter.newContracts,
		SendToPeer:     server.SendToPeer,
		FindPeer:       server.FindPeer,
		TempChanIDSeed: chanIDSeed,
		FindChannel: func(chanID lnwire.ChannelID) (*lnwallet.LightningChannel, error) {
			dbChannels, err := chanDB.FetchAllChannels()
			if err != nil {
				return nil, err
			}

			for _, channel := range dbChannels {
				if chanID.IsChanPoint(&channel.FundingOutpoint) {
					return lnwallet.NewLightningChannel(
						activeChainControl.signer,
						activeChainControl.chainNotifier,
						activeChainControl.feeEstimator,
						channel)
				}
			}

			return nil, fmt.Errorf("unable to find channel")
		},
		DefaultRoutingPolicy: activeChainControl.routingPolicy,
		NumRequiredConfs: func(chanAmt btcutil.Amount, pushAmt btcutil.Amount) uint16 {
			// TODO(roasbeef): add configurable mapping
			//  * simple switch initially
			//  * assign coefficient, etc
			return uint16(cfg.DefaultNumChanConfs)
		},
		RequiredRemoteDelay: func(chanAmt btcutil.Amount) uint16 {
			// TODO(roasbeef): add additional hooks for
			// configuration
			return 4
		},
	})
	if err != nil {
		return err
	}
	if err := fundingMgr.Start(); err != nil {
		return err
	}
	server.fundingMgr = fundingMgr

	// Ensure we create TLS key and certificate if they don't exist
	if !fileExists(cfg.TLSCertPath) && !fileExists(cfg.TLSKeyPath) {
		if err := genCertPair(cfg.TLSCertPath, cfg.TLSKeyPath); err != nil {
			return err
		}
	}

	// Initialize, and register our implementation of the gRPC interface
	// exported by the rpcServer.
	rpcServer := newRPCServer(server)
	if err := rpcServer.Start(); err != nil {
		return err
	}
	sCreds, err := credentials.NewServerTLSFromFile(cfg.TLSCertPath,
		cfg.TLSKeyPath)
	if err != nil {
		return err
	}
	opts := []grpc.ServerOption{grpc.Creds(sCreds)}
	grpcServer := grpc.NewServer(opts...)
	lnrpc.RegisterLightningServer(grpcServer, rpcServer)

	// Next, Start the gRPC server listening for HTTP/2 connections.
	grpcEndpoint := fmt.Sprintf("localhost:%d", loadedConfig.RPCPort)
	lis, err := net.Listen("tcp", grpcEndpoint)
	if err != nil {
		fmt.Printf("failed to listen: %v", err)
		return err
	}
	defer lis.Close()
	go func() {
		rpcsLog.Infof("RPC server listening on %s", lis.Addr())
		grpcServer.Serve(lis)
	}()
	cCreds, err := credentials.NewClientTLSFromFile(cfg.TLSCertPath, "")
	if err != nil {
		return err
	}
	// Finally, start the REST proxy for our gRPC server above.
	ctx := context.Background()
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	mux := proxy.NewServeMux()
	proxyOpts := []grpc.DialOption{grpc.WithTransportCredentials(cCreds)}
	err = lnrpc.RegisterLightningHandlerFromEndpoint(ctx, mux, grpcEndpoint,
		proxyOpts)
	if err != nil {
		return err
	}
	go func() {
		restEndpoint := fmt.Sprintf(":%d", loadedConfig.RESTPort)
		rpcsLog.Infof("gRPC proxy started at localhost%s", restEndpoint)
		http.ListenAndServeTLS(restEndpoint, cfg.TLSCertPath,
			cfg.TLSKeyPath, mux)
	}()

	// If we're not in simnet mode, We'll wait until we're fully synced to
	// continue the start up of the remainder of the daemon. This ensures
	// that we don't accept any possibly invalid state transitions, or
	// accept channels with spent funds.
	if !(cfg.Bitcoin.SimNet || cfg.Litecoin.SimNet) {
		_, bestHeight, err := activeChainControl.chainIO.GetBestBlock()
		if err != nil {
			return err
		}

		ltndLog.Infof("Waiting for chain backend to finish sync, "+
			"start_height=%v", bestHeight)

		for {
			synced, err := activeChainControl.wallet.IsSynced()
			if err != nil {
				return err
			}

			if synced {
				break
			}

			time.Sleep(time.Second * 1)
		}

		_, bestHeight, err = activeChainControl.chainIO.GetBestBlock()
		if err != nil {
			return err
		}

		ltndLog.Infof("Chain backend is fully synced (end_height=%v)!",
			bestHeight)
	}

	// With all the relevant chains initialized, we can finally start the
	// server itself.
	if err := server.Start(); err != nil {
		srvrLog.Errorf("unable to create to start server: %v\n", err)
		return err
	}

	addInterruptHandler(func() {
		ltndLog.Infof("Gracefully shutting down the server...")
		rpcServer.Stop()
		fundingMgr.Stop()
		server.Stop()
		server.WaitForShutdown()
	})

	// Wait for shutdown signal from either a graceful server stop or from
	// the interrupt handler.
	<-shutdownChannel
	ltndLog.Info("Shutdown complete")
	return nil
}

func main() {
	// Use all processor cores.
	// TODO(roasbeef): remove this if required version # is > 1.6?
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Call the "real" main in a nested manner so the defers will properly
	// be executed in the case of a graceful shutdown.
	if err := lndMain(); err != nil {
		if e, ok := err.(*flags.Error); ok && e.Type == flags.ErrHelp {
		} else {
			fmt.Fprintln(os.Stderr, err)
		}
		os.Exit(1)
	}
}

// fileExists reports whether the named file or directory exists.
// This function is taken from https://github.com/btcsuite/btcd
func fileExists(name string) bool {
	if _, err := os.Stat(name); err != nil {
		if os.IsNotExist(err) {
			return false
		}
	}
	return true
}

// genCertPair generates a key/cert pair to the paths provided.
// This function is adapted from https://github.com/btcsuite/btcd
func genCertPair(certFile, keyFile string) error {
	rpcsLog.Infof("Generating TLS certificates...")

	org := "lnd autogenerated cert"
	validUntil := time.Now().Add(autogenCertValidity)
	cert, key, err := btcutil.NewTLSCertPair(org, validUntil, nil)
	if err != nil {
		return err
	}

	// Write cert and key files.
	if err = ioutil.WriteFile(certFile, cert, 0644); err != nil {
		return err
	}
	if err = ioutil.WriteFile(keyFile, key, 0600); err != nil {
		os.Remove(certFile)
		return err
	}

	rpcsLog.Infof("Done generating TLS certificates")
	return nil
}
<commit_msg>lnd: initialize an autopilot.Agent instance if set in the config file<commit_after>package main

import (
	"crypto/rand"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"runtime"
	"strconv"
	"time"

	"golang.org/x/net/context"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"

	flags "github.com/btcsuite/go-flags"
	proxy "github.com/grpc-ecosystem/grpc-gateway/runtime"
	"github.com/lightningnetwork/lnd/autopilot"
	"github.com/lightningnetwork/lnd/channeldb"
	"github.com/lightningnetwork/lnd/lnrpc"
	"github.com/lightningnetwork/lnd/lnwallet"
	"github.com/lightningnetwork/lnd/lnwire"
	"github.com/roasbeef/btcd/btcec"
	"github.com/roasbeef/btcutil"
)

const (
	autogenCertValidity = 10 * 365 * 24 * time.Hour
)

var (
	cfg              *config
	shutdownChannel  = make(chan struct{})
	registeredChains = newChainRegistry()
)

// lndMain is the true entry point for lnd. This function is required since
// defers created in the top-level scope of a main method aren't executed if
// os.Exit() is called.
func lndMain() error {
	// Load the configuration, and parse any command line options. This
	// function will also set up logging properly.
	loadedConfig, err := loadConfig()
	if err != nil {
		return err
	}
	cfg = loadedConfig
	defer func() {
		if logRotator != nil {
			logRotator.Close()
		}
	}()

	// Show version at startup.
	ltndLog.Infof("Version %s", version())

	// Enable http profiling server if requested.
	if cfg.Profile != "" {
		go func() {
			listenAddr := net.JoinHostPort("", cfg.Profile)
			profileRedirect := http.RedirectHandler("/debug/pprof",
				http.StatusSeeOther)
			http.Handle("/", profileRedirect)
			fmt.Println(http.ListenAndServe(listenAddr, nil))
		}()
	}

	// Open the channeldb, which is dedicated to storing channel, and
	// network related metadata.
	chanDB, err := channeldb.Open(cfg.DataDir)
	if err != nil {
		ltndLog.Errorf("unable to open channeldb: %v", err)
		return err
	}
	defer chanDB.Close()

	// With the information parsed from the configuration, create valid
	// instances of the paertinent interfaces required to operate the
	// Lightning Network Daemon.
	activeChainControl, chainCleanUp, err := newChainControlFromConfig(cfg, chanDB)
	if err != nil {
		fmt.Printf("unable to create chain control: %v\n", err)
		return err
	}
	if chainCleanUp != nil {
		defer chainCleanUp()
	}

	// Finally before we start the server, we'll register the "holy
	// trinity" of interface for our current "home chain" with the active
	// chainRegistry interface.
	primaryChain := registeredChains.PrimaryChain()
	registeredChains.RegisterChain(primaryChain, activeChainControl)

	idPrivKey, err := activeChainControl.wallet.GetIdentitykey()
	if err != nil {
		return err
	}
	idPrivKey.Curve = btcec.S256()

	// Set up the core server which will listen for incoming peer
	// connections.
	defaultListenAddrs := []string{
		net.JoinHostPort("", strconv.Itoa(cfg.PeerPort)),
	}
	server, err := newServer(defaultListenAddrs, chanDB, activeChainControl,
		idPrivKey)
	if err != nil {
		srvrLog.Errorf("unable to create server: %v\n", err)
		return err
	}

	// Next, we'll initialize the funding manager itself so it can answer
	// queries while the wallet+chain are still syncing.
	nodeSigner := newNodeSigner(idPrivKey)
	var chanIDSeed [32]byte
	if _, err := rand.Read(chanIDSeed[:]); err != nil {
		return err
	}
	fundingMgr, err := newFundingManager(fundingConfig{
		IDKey:        idPrivKey.PubKey(),
		Wallet:       activeChainControl.wallet,
		Notifier:     activeChainControl.chainNotifier,
		FeeEstimator: activeChainControl.feeEstimator,
		SignMessage: func(pubKey *btcec.PublicKey,
			msg []byte) (*btcec.Signature, error) {

			if pubKey.IsEqual(idPrivKey.PubKey()) {
				return nodeSigner.SignMessage(pubKey, msg)
			}

			return activeChainControl.msgSigner.SignMessage(
				pubKey, msg,
			)
		},
		CurrentNodeAnnouncement: func() (lnwire.NodeAnnouncement, error) {
			return server.genNodeAnnouncement(true)
		},
		SendAnnouncement: func(msg lnwire.Message) error {
			errChan := server.discoverSrv.ProcessLocalAnnouncement(msg,
				idPrivKey.PubKey())
			return <-errChan
		},
		ArbiterChan:    server.breachArbiter.newContracts,
		SendToPeer:     server.SendToPeer,
		FindPeer:       server.FindPeer,
		TempChanIDSeed: chanIDSeed,
		FindChannel: func(chanID lnwire.ChannelID) (*lnwallet.LightningChannel, error) {
			dbChannels, err := chanDB.FetchAllChannels()
			if err != nil {
				return nil, err
			}

			for _, channel := range dbChannels {
				if chanID.IsChanPoint(&channel.FundingOutpoint) {
					return lnwallet.NewLightningChannel(
						activeChainControl.signer,
						activeChainControl.chainNotifier,
						activeChainControl.feeEstimator,
						channel)
				}
			}

			return nil, fmt.Errorf("unable to find channel")
		},
		DefaultRoutingPolicy: activeChainControl.routingPolicy,
		NumRequiredConfs: func(chanAmt btcutil.Amount, pushAmt btcutil.Amount) uint16 {
			// TODO(roasbeef): add configurable mapping
			//  * simple switch initially
			//  * assign coefficient, etc
			return uint16(cfg.DefaultNumChanConfs)
		},
		RequiredRemoteDelay: func(chanAmt btcutil.Amount) uint16 {
			// TODO(roasbeef): add additional hooks for
			// configuration
			return 4
		},
	})
	if err != nil {
		return err
	}
	if err := fundingMgr.Start(); err != nil {
		return err
	}
	server.fundingMgr = fundingMgr

	// Ensure we create TLS key and certificate if they don't exist
	if !fileExists(cfg.TLSCertPath) && !fileExists(cfg.TLSKeyPath) {
		if err := genCertPair(cfg.TLSCertPath, cfg.TLSKeyPath); err != nil {
			return err
		}
	}

	// Initialize, and register our implementation of the gRPC interface
	// exported by the rpcServer.
	rpcServer := newRPCServer(server)
	if err := rpcServer.Start(); err != nil {
		return err
	}
	sCreds, err := credentials.NewServerTLSFromFile(cfg.TLSCertPath,
		cfg.TLSKeyPath)
	if err != nil {
		return err
	}
	opts := []grpc.ServerOption{grpc.Creds(sCreds)}
	grpcServer := grpc.NewServer(opts...)
	lnrpc.RegisterLightningServer(grpcServer, rpcServer)

	// Next, Start the gRPC server listening for HTTP/2 connections.
	grpcEndpoint := fmt.Sprintf("localhost:%d", loadedConfig.RPCPort)
	lis, err := net.Listen("tcp", grpcEndpoint)
	if err != nil {
		fmt.Printf("failed to listen: %v", err)
		return err
	}
	defer lis.Close()
	go func() {
		rpcsLog.Infof("RPC server listening on %s", lis.Addr())
		grpcServer.Serve(lis)
	}()
	cCreds, err := credentials.NewClientTLSFromFile(cfg.TLSCertPath, "")
	if err != nil {
		return err
	}
	// Finally, start the REST proxy for our gRPC server above.
	ctx := context.Background()
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	mux := proxy.NewServeMux()
	proxyOpts := []grpc.DialOption{grpc.WithTransportCredentials(cCreds)}
	err = lnrpc.RegisterLightningHandlerFromEndpoint(ctx, mux, grpcEndpoint,
		proxyOpts)
	if err != nil {
		return err
	}
	go func() {
		restEndpoint := fmt.Sprintf(":%d", loadedConfig.RESTPort)
		rpcsLog.Infof("gRPC proxy started at localhost%s", restEndpoint)
		http.ListenAndServeTLS(restEndpoint, cfg.TLSCertPath,
			cfg.TLSKeyPath, mux)
	}()

	// If we're not in simnet mode, We'll wait until we're fully synced to
	// continue the start up of the remainder of the daemon. This ensures
	// that we don't accept any possibly invalid state transitions, or
	// accept channels with spent funds.
	if !(cfg.Bitcoin.SimNet || cfg.Litecoin.SimNet) {
		_, bestHeight, err := activeChainControl.chainIO.GetBestBlock()
		if err != nil {
			return err
		}

		ltndLog.Infof("Waiting for chain backend to finish sync, "+
			"start_height=%v", bestHeight)

		for {
			synced, err := activeChainControl.wallet.IsSynced()
			if err != nil {
				return err
			}

			if synced {
				break
			}

			time.Sleep(time.Second * 1)
		}

		_, bestHeight, err = activeChainControl.chainIO.GetBestBlock()
		if err != nil {
			return err
		}

		ltndLog.Infof("Chain backend is fully synced (end_height=%v)!",
			bestHeight)
	}

	// With all the relevant chains initialized, we can finally start the
	// server itself.
	if err := server.Start(); err != nil {
		srvrLog.Errorf("unable to create to start server: %v\n", err)
		return err
	}

	// Now that the server has started, if the autopilot mode is currently
	// active, then we'll initialize a fresh instance of it and start it.
	var pilot *autopilot.Agent
	if cfg.Autopilot.Active {
		pilot, err := initAutoPilot(server, cfg.Autopilot)
		if err != nil {
			ltndLog.Errorf("unable to create autopilot agent: %v",
				err)
			return err
		}
		if err := pilot.Start(); err != nil {
			ltndLog.Errorf("unable to start autopilot agent: %v",
				err)
			return err
		}
	}

	addInterruptHandler(func() {
		ltndLog.Infof("Gracefully shutting down the server...")
		rpcServer.Stop()
		fundingMgr.Stop()
		server.Stop()

		if pilot != nil {
			pilot.Stop()
		}

		server.WaitForShutdown()
	})

	// Wait for shutdown signal from either a graceful server stop or from
	// the interrupt handler.
	<-shutdownChannel
	ltndLog.Info("Shutdown complete")
	return nil
}

func main() {
	// Use all processor cores.
	// TODO(roasbeef): remove this if required version # is > 1.6?
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Call the "real" main in a nested manner so the defers will properly
	// be executed in the case of a graceful shutdown.
	if err := lndMain(); err != nil {
		if e, ok := err.(*flags.Error); ok && e.Type == flags.ErrHelp {
		} else {
			fmt.Fprintln(os.Stderr, err)
		}
		os.Exit(1)
	}
}

// fileExists reports whether the named file or directory exists.
// This function is taken from https://github.com/btcsuite/btcd
func fileExists(name string) bool {
	if _, err := os.Stat(name); err != nil {
		if os.IsNotExist(err) {
			return false
		}
	}
	return true
}

// genCertPair generates a key/cert pair to the paths provided.
// This function is adapted from https://github.com/btcsuite/btcd
func genCertPair(certFile, keyFile string) error {
	rpcsLog.Infof("Generating TLS certificates...")

	org := "lnd autogenerated cert"
	validUntil := time.Now().Add(autogenCertValidity)
	cert, key, err := btcutil.NewTLSCertPair(org, validUntil, nil)
	if err != nil {
		return err
	}

	// Write cert and key files.
	if err = ioutil.WriteFile(certFile, cert, 0644); err != nil {
		return err
	}
	if err = ioutil.WriteFile(keyFile, key, 0600); err != nil {
		os.Remove(certFile)
		return err
	}

	rpcsLog.Infof("Done generating TLS certificates")
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"github.com/bgentry/heroku-go"
	"io"
	"log"
	"net/http"
	"os"
	"regexp"
)

var (
	lines  int
	source string
	dyno   string
)

var cmdLog = &Command{
	Run:      runLog,
	Usage:    "log [-n <lines>] [-s <source>] [-d <dyno>]",
	NeedsApp: true,
	Category: "app",
	Short:    "stream app log lines",
	Long: `
Log prints the streaming application log.

Options:

    -n <N>       print at most N log lines
    -s <source>  filter log source
    -d <dyno>    filter dyno or process type

Examples:

    $ hk log
    2013-10-17T00:17:35.066089+00:00 app[web.1]: Completed 302 Found in 0ms
    2013-10-17T00:17:35.079095+00:00 heroku[router]: at=info method=GET path=/ host=www.heroku.com fwd="1.2.3.4" dyno=web.1 connect=1ms service=6ms status=302 bytes=95
    2013-10-17T00:17:35.505389+00:00 heroku[nginx]: 1.2.3.4 - - [17/Oct/2013:00:17:35 +0000] "GET / HTTP/1.1" 301 5 "-" "Amazon Route 53 Health Check Service" www.heroku.com
		...

    $ hk log -n 2 -s app -d web
		2013-10-17T00:17:34.288521+00:00 app[web.1]: Completed 200 OK in 10ms (Views: 10.0ms)
    2013-10-17T00:17:33.918946+00:00 heroku[web.5]: Started GET "/" for 1.2.3.4 at 2013-10-17 00:17:32 +0000
		2013-10-17T00:17:34.667654+00:00 heroku[router]: at=info method=GET path=/ host=www.heroku.com fwd="1.2.3.4" dyno=web.5 connect=3ms service=8ms status=301 bytes=0
    2013-10-17T00:17:35.079095+00:00 heroku[router]: at=info method=GET path=/ host=www.heroku.com fwd="1.2.3.4" dyno=web.1 connect=1ms service=6ms status=302 bytes=95
		...

    $ hk log -d web.5
    2013-10-17T00:17:33.918946+00:00 app[web.5]: Started GET "/" for 1.2.3.4 at 2013-10-17 00:17:32 +0000
    2013-10-17T00:17:33.918658+00:00 app[web.5]: Processing by PagesController#root as HTML
		...
`,
}

func init() {
	cmdLog.Flag.IntVar(&lines, "n", -1, "max number of log lines to request")
	cmdLog.Flag.StringVar(&source, "s", "", "only display logs from the given source")
	cmdLog.Flag.StringVar(&dyno, "d", "", "only display logs from the given dyno or process type")
}

func runLog(cmd *Command, args []string) {
	opts := heroku.LogSessionCreateOpts{}
	if dyno != "" {
		opts.Dyno = &dyno
	}
	if source != "" {
		opts.Source = &source
	}

	if lines != -1 {
		opts.Lines = &lines
	} else {
		tailopt := true
		lineopt := 10
		opts.Tail = &tailopt
		opts.Lines = &lineopt
	}

	session, err := client.LogSessionCreate(mustApp(), opts)
	if err != nil {
		log.Fatal(err)
	}
	resp, err := http.Get(session.LogplexURL)
	if err != nil {
		log.Fatal(err)
	}
	if resp.StatusCode/100 != 2 {
		if resp.StatusCode/100 == 4 {
			log.Fatal("Unauthorized")
		} else {
			log.Fatal("Unexpected error: " + resp.Status)
		}
	}

	writer := LineWriter(WriterAdapter{os.Stdout})

	scanner := bufio.NewScanner(resp.Body)
	scanner.Split(bufio.ScanLines)

	for scanner.Scan() {
		if _, err = writer.Writeln(scanner.Text()); err != nil {
			log.Fatal(err)
		}
	}

	resp.Body.Close()
}

type LineWriter interface {
	Writeln(p string) (int, error)
}

type WriterAdapter struct {
	io.Writer
}

func (w WriterAdapter) Writeln(p string) (n int, err error) {
	return fmt.Fprintln(w, p)
}

type colorizer struct {
	colors      map[string]string
	colorScheme []string
	filter      *regexp.Regexp
	writer      LineWriter
}

func newColorizer(writer LineWriter) *colorizer {
	return &colorizer{
		colors: make(map[string]string),
		colorScheme: []string{
			"36", //cyan
			"33", //yellow
			"32", //green
			"35", //magenta
			"31", //red
		},
		filter: regexp.MustCompile(`(?s)^(.*?\[([\w-]+)(?:[\d\.]+)?\]:)(.*)?$`),
		writer: writer,
	}
}

func (c *colorizer) resolve(p string) string {
	if color, ok := c.colors[p]; ok {
		return color
	}

	color := c.colorScheme[len(c.colors)%len(c.colorScheme)]
	c.colors[p] = color
	return color
}
<commit_msg>Define method for writing a line<commit_after>package main

import (
	"bufio"
	"fmt"
	"github.com/bgentry/heroku-go"
	"io"
	"log"
	"net/http"
	"os"
	"regexp"
)

var (
	lines  int
	source string
	dyno   string
)

var cmdLog = &Command{
	Run:      runLog,
	Usage:    "log [-n <lines>] [-s <source>] [-d <dyno>]",
	NeedsApp: true,
	Category: "app",
	Short:    "stream app log lines",
	Long: `
Log prints the streaming application log.

Options:

    -n <N>       print at most N log lines
    -s <source>  filter log source
    -d <dyno>    filter dyno or process type

Examples:

    $ hk log
    2013-10-17T00:17:35.066089+00:00 app[web.1]: Completed 302 Found in 0ms
    2013-10-17T00:17:35.079095+00:00 heroku[router]: at=info method=GET path=/ host=www.heroku.com fwd="1.2.3.4" dyno=web.1 connect=1ms service=6ms status=302 bytes=95
    2013-10-17T00:17:35.505389+00:00 heroku[nginx]: 1.2.3.4 - - [17/Oct/2013:00:17:35 +0000] "GET / HTTP/1.1" 301 5 "-" "Amazon Route 53 Health Check Service" www.heroku.com
		...

    $ hk log -n 2 -s app -d web
		2013-10-17T00:17:34.288521+00:00 app[web.1]: Completed 200 OK in 10ms (Views: 10.0ms)
    2013-10-17T00:17:33.918946+00:00 heroku[web.5]: Started GET "/" for 1.2.3.4 at 2013-10-17 00:17:32 +0000
		2013-10-17T00:17:34.667654+00:00 heroku[router]: at=info method=GET path=/ host=www.heroku.com fwd="1.2.3.4" dyno=web.5 connect=3ms service=8ms status=301 bytes=0
    2013-10-17T00:17:35.079095+00:00 heroku[router]: at=info method=GET path=/ host=www.heroku.com fwd="1.2.3.4" dyno=web.1 connect=1ms service=6ms status=302 bytes=95
		...

    $ hk log -d web.5
    2013-10-17T00:17:33.918946+00:00 app[web.5]: Started GET "/" for 1.2.3.4 at 2013-10-17 00:17:32 +0000
    2013-10-17T00:17:33.918658+00:00 app[web.5]: Processing by PagesController#root as HTML
		...
`,
}

func init() {
	cmdLog.Flag.IntVar(&lines, "n", -1, "max number of log lines to request")
	cmdLog.Flag.StringVar(&source, "s", "", "only display logs from the given source")
	cmdLog.Flag.StringVar(&dyno, "d", "", "only display logs from the given dyno or process type")
}

func runLog(cmd *Command, args []string) {
	opts := heroku.LogSessionCreateOpts{}
	if dyno != "" {
		opts.Dyno = &dyno
	}
	if source != "" {
		opts.Source = &source
	}

	if lines != -1 {
		opts.Lines = &lines
	} else {
		tailopt := true
		lineopt := 10
		opts.Tail = &tailopt
		opts.Lines = &lineopt
	}

	session, err := client.LogSessionCreate(mustApp(), opts)
	if err != nil {
		log.Fatal(err)
	}
	resp, err := http.Get(session.LogplexURL)
	if err != nil {
		log.Fatal(err)
	}
	if resp.StatusCode/100 != 2 {
		if resp.StatusCode/100 == 4 {
			log.Fatal("Unauthorized")
		} else {
			log.Fatal("Unexpected error: " + resp.Status)
		}
	}

	writer := LineWriter(WriterAdapter{os.Stdout})

	scanner := bufio.NewScanner(resp.Body)
	scanner.Split(bufio.ScanLines)

	for scanner.Scan() {
		if _, err = writer.Writeln(scanner.Text()); err != nil {
			log.Fatal(err)
		}
	}

	resp.Body.Close()
}

type LineWriter interface {
	Writeln(p string) (int, error)
}

type WriterAdapter struct {
	io.Writer
}

func (w WriterAdapter) Writeln(p string) (n int, err error) {
	return fmt.Fprintln(w, p)
}

type colorizer struct {
	colors      map[string]string
	colorScheme []string
	filter      *regexp.Regexp
	writer      LineWriter
}

func newColorizer(writer LineWriter) *colorizer {
	return &colorizer{
		colors: make(map[string]string),
		colorScheme: []string{
			"36", //cyan
			"33", //yellow
			"32", //green
			"35", //magenta
			"31", //red
		},
		filter: regexp.MustCompile(`(?s)^(.*?\[([\w-]+)(?:[\d\.]+)?\]:)(.*)?$`),
		writer: writer,
	}
}

func (c *colorizer) resolve(p string) string {
	if color, ok := c.colors[p]; ok {
		return color
	}

	color := c.colorScheme[len(c.colors)%len(c.colorScheme)]
	c.colors[p] = color
	return color
}

func (c *colorizer) Writeln(p string) (n int, err error) {
	if c.filter.MatchString(p) {
		submatches := c.filter.FindStringSubmatch(p)
		return c.writer.Writeln(fmt.Sprintf("\033[%sm%s\033[0m%s", c.resolve(submatches[2]), submatches[1], submatches[3]))
	}

	return c.writer.Writeln(p)
}
<|endoftext|>


<commit_before>// Copyright 2015-2018 HenryLee. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tp

import (
	"log"
	"os"
	"sync"

	"github.com/henrylee2cn/go-logging"
	"github.com/henrylee2cn/go-logging/color"
	"github.com/henrylee2cn/goutil/graceful"
)

// Logger interface
type Logger interface {
	// Level returns the logger's level.
	Level() string
	// SetLevel sets the logger's level.
	SetLevel(level string)
	// Printf formats according to a format specifier and writes to standard output.
	// It returns the number of bytes written and any write error encountered.
	Printf(format string, args ...interface{})
	// Fatalf is equivalent to Criticalf followed by a call to os.Exit(1).
	Fatalf(format string, args ...interface{})
	// Panicf is equivalent to Criticalf followed by a call to panic().
	Panicf(format string, args ...interface{})
	// Criticalf logs a message using CRITICAL as log level.
	Criticalf(format string, args ...interface{})
	// Errorf logs a message using ERROR as log level.
	Errorf(format string, args ...interface{})
	// Warnf logs a message using WARNING as log level.
	Warnf(format string, args ...interface{})
	// Noticef logs a message using NOTICE as log level.
	Noticef(format string, args ...interface{})
	// Infof logs a message using INFO as log level.
	Infof(format string, args ...interface{})
	// Debugf logs a message using DEBUG as log level.
	Debugf(format string, args ...interface{})
	// Tracef logs a message using TRACE as log level.
	Tracef(format string, args ...interface{})
}

var (
	// global logger
	globalLogger = func() Logger {
		logger := newDefaultlogger("TRACE")
		graceful.SetLog(logger)
		return logger
	}()
)

func newDefaultlogger(level string) Logger {
	l := &defaultLogger{
		level: level,
	}
	l.newSet()
	return l
}

type defaultLogger struct {
	*logging.Logger
	level string
	mu    sync.RWMutex
}

func (l *defaultLogger) newSet() {
	var consoleLogBackend = &logging.LogBackend{
		Logger: log.New(color.NewColorableStdout(), "", 0),
		Color:  true,
	}
	consoleFormat := logging.MustStringFormatter("[%{time:2006/01/02 15:04:05.000}] [%{color:bold}%{level:.4s}%{color:reset}] %{message} <%{longfile}>")
	consoleBackendLevel := logging.AddModuleLevel(logging.NewBackendFormatter(consoleLogBackend, consoleFormat))
	level, err := logging.LogLevel(l.level)
	if err != nil {
		panic(err)
	}
	consoleBackendLevel.SetLevel(level, "")
	l.Logger = logging.NewLogger("teleport")
	l.Logger.SetBackend(consoleBackendLevel)
	l.Logger.ExtraCalldepth++
}

// Level returns the logger's level.
// Note: Concurrent is not safe!
func (l *defaultLogger) Level() string {
	l.mu.RLock()
	defer l.mu.RUnlock()
	return l.level
}

// SetLevel sets the logger's level.
// Note:
// Concurrent is not safe!
// the teleport default logger's level list: PRINT CRITICAL ERROR WARNING NOTICE INFO DEBUG TRACE
func (l *defaultLogger) SetLevel(level string) {
	l.mu.Lock()
	defer l.mu.Unlock()
	l.level = level
	l.newSet()
}

// GetLogger gets global logger.
func GetLogger() Logger {
	return globalLogger
}

// SetLogger sets global logger.
// Note: Concurrent is not safe!
func SetLogger(logger Logger) {
	if logger == nil {
		return
	}
	globalLogger = logger
	graceful.SetLog(logger)
}

// GetLoggerLevel gets the logger's level.
func GetLoggerLevel() string {
	return globalLogger.Level()
}

// SetLoggerLevel sets the logger's level.
func SetLoggerLevel(level string) {
	globalLogger.SetLevel(level)
}

// Printf formats according to a format specifier and writes to standard output.
// It returns the number of bytes written and any write error encountered.
func Printf(format string, args ...interface{}) {
	globalLogger.Printf(format, args...)
}

// Fatalf is equivalent to l.Criticalf followed by a call to os.Exit(1).
func Fatalf(format string, args ...interface{}) {
	globalLogger.Fatalf(format, args...)
	os.Exit(1)
}

// Panicf is equivalent to l.Criticalf followed by a call to panic().
func Panicf(format string, args ...interface{}) {
	globalLogger.Panicf(format, args...)
}

// Criticalf logs a message using CRITICAL as log level.
func Criticalf(format string, args ...interface{}) {
	globalLogger.Criticalf(format, args...)
}

// Errorf logs a message using ERROR as log level.
func Errorf(format string, args ...interface{}) {
	globalLogger.Errorf(format, args...)
}

// Warnf logs a message using WARNING as log level.
func Warnf(format string, args ...interface{}) {
	globalLogger.Warnf(format, args...)
}

// Noticef logs a message using NOTICE as log level.
func Noticef(format string, args ...interface{}) {
	globalLogger.Noticef(format, args...)
}

// Infof logs a message using INFO as log level.
func Infof(format string, args ...interface{}) {
	globalLogger.Infof(format, args...)
}

// Debugf logs a message using DEBUG as log level.
func Debugf(format string, args ...interface{}) {
	globalLogger.Debugf(format, args...)
}

// Tracef logs a message using TRACE as log level.
func Tracef(format string, args ...interface{}) {
	globalLogger.Tracef(format, args...)
}
<commit_msg>add OFF level to default logger<commit_after>// Copyright 2015-2018 HenryLee. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tp

import (
	"log"
	"os"
	"sync"

	"github.com/henrylee2cn/go-logging"
	"github.com/henrylee2cn/go-logging/color"
	"github.com/henrylee2cn/goutil/graceful"
)

// Logger interface
type Logger interface {
	// Level returns the logger's level.
	Level() string
	// SetLevel sets the logger's level.
	SetLevel(level string)
	// Printf formats according to a format specifier and writes to standard output.
	// It returns the number of bytes written and any write error encountered.
	Printf(format string, args ...interface{})
	// Fatalf is equivalent to Criticalf followed by a call to os.Exit(1).
	Fatalf(format string, args ...interface{})
	// Panicf is equivalent to Criticalf followed by a call to panic().
	Panicf(format string, args ...interface{})
	// Criticalf logs a message using CRITICAL as log level.
	Criticalf(format string, args ...interface{})
	// Errorf logs a message using ERROR as log level.
	Errorf(format string, args ...interface{})
	// Warnf logs a message using WARNING as log level.
	Warnf(format string, args ...interface{})
	// Noticef logs a message using NOTICE as log level.
	Noticef(format string, args ...interface{})
	// Infof logs a message using INFO as log level.
	Infof(format string, args ...interface{})
	// Debugf logs a message using DEBUG as log level.
	Debugf(format string, args ...interface{})
	// Tracef logs a message using TRACE as log level.
	Tracef(format string, args ...interface{})
}

var (
	// global logger
	globalLogger = func() Logger {
		logger := newDefaultlogger("TRACE")
		graceful.SetLog(logger)
		return logger
	}()
)

func newDefaultlogger(level string) Logger {
	l := &defaultLogger{
		level: level,
	}
	l.newSet()
	return l
}

type defaultLogger struct {
	*logging.Logger
	level string
	mu    sync.RWMutex
}

func (l *defaultLogger) newSet() {
	var consoleLogBackend = &logging.LogBackend{
		Logger: log.New(color.NewColorableStdout(), "", 0),
		Color:  true,
	}
	consoleFormat := logging.MustStringFormatter("[%{time:2006/01/02 15:04:05.000}] [%{color:bold}%{level:.4s}%{color:reset}] %{message} <%{longfile}>")
	consoleBackendLevel := logging.AddModuleLevel(logging.NewBackendFormatter(consoleLogBackend, consoleFormat))
	level, err := logging.LogLevel(l.level)
	if err != nil {
		panic(err)
	}
	consoleBackendLevel.SetLevel(level, "")
	l.Logger = logging.NewLogger("teleport")
	l.Logger.SetBackend(consoleBackendLevel)
	l.Logger.ExtraCalldepth++
}

// Level returns the logger's level.
// Note: Concurrent is not safe!
func (l *defaultLogger) Level() string {
	l.mu.RLock()
	defer l.mu.RUnlock()
	return l.level
}

// SetLevel sets the logger's level.
// Note:
// Concurrent is not safe!
// the teleport default logger's level list: OFF PRINT CRITICAL ERROR WARNING NOTICE INFO DEBUG TRACE
func (l *defaultLogger) SetLevel(level string) {
	l.mu.Lock()
	defer l.mu.Unlock()
	l.level = level
	l.newSet()
}

// GetLogger gets global logger.
func GetLogger() Logger {
	return globalLogger
}

// SetLogger sets global logger.
// Note: Concurrent is not safe!
func SetLogger(logger Logger) {
	if logger == nil {
		return
	}
	globalLogger = logger
	graceful.SetLog(logger)
}

// GetLoggerLevel gets the logger's level.
func GetLoggerLevel() string {
	return globalLogger.Level()
}

// SetLoggerLevel sets the logger's level.
func SetLoggerLevel(level string) {
	globalLogger.SetLevel(level)
}

// Printf formats according to a format specifier and writes to standard output.
// It returns the number of bytes written and any write error encountered.
func Printf(format string, args ...interface{}) {
	globalLogger.Printf(format, args...)
}

// Fatalf is equivalent to l.Criticalf followed by a call to os.Exit(1).
func Fatalf(format string, args ...interface{}) {
	globalLogger.Fatalf(format, args...)
	os.Exit(1)
}

// Panicf is equivalent to l.Criticalf followed by a call to panic().
func Panicf(format string, args ...interface{}) {
	globalLogger.Panicf(format, args...)
}

// Criticalf logs a message using CRITICAL as log level.
func Criticalf(format string, args ...interface{}) {
	globalLogger.Criticalf(format, args...)
}

// Errorf logs a message using ERROR as log level.
func Errorf(format string, args ...interface{}) {
	globalLogger.Errorf(format, args...)
}

// Warnf logs a message using WARNING as log level.
func Warnf(format string, args ...interface{}) {
	globalLogger.Warnf(format, args...)
}

// Noticef logs a message using NOTICE as log level.
func Noticef(format string, args ...interface{}) {
	globalLogger.Noticef(format, args...)
}

// Infof logs a message using INFO as log level.
func Infof(format string, args ...interface{}) {
	globalLogger.Infof(format, args...)
}

// Debugf logs a message using DEBUG as log level.
func Debugf(format string, args ...interface{}) {
	globalLogger.Debugf(format, args...)
}

// Tracef logs a message using TRACE as log level.
func Tracef(format string, args ...interface{}) {
	globalLogger.Tracef(format, args...)
}
<|endoftext|>


<commit_before>// Package log provides a log interface
package log

// Logger is a generic logging interface
type Logger interface {

	// Log inserts a log entry.  Arguments may be handled in the manner
	// of fmt.Print, but the underlying logger may also decide to handle
	// them differently.
	Log(v ...interface{})

	// Logf insets a log entry.  Arguments are handled in the manner of
	// fmt.Printf.
	Logf(format string, v ...interface{})
}

var (
	// The global default logger
	DefaultLogger Logger = &noOpLogger{}
)

// noOpLogger is used as a placeholder for the default logger
type noOpLogger struct{}

func (n *noOpLogger) Log(v ...interface{}) {}

func (n *noOpLogger) Logf(format string, v ...interface{}) {}

// Log logs using the default logger
func Log(v ...interface{}) {
	DefaultLogger.Log(v...)
}

// Logf logs formatted using the default logger
func Logf(format string, v ...interface{}) {
	DefaultLogger.Logf(format, v...)
}
<commit_msg>strip spaces<commit_after>// Package log provides a log interface
package log

// Logger is a generic logging interface
type Logger interface {
	// Log inserts a log entry.  Arguments may be handled in the manner
	// of fmt.Print, but the underlying logger may also decide to handle
	// them differently.
	Log(v ...interface{})
	// Logf insets a log entry.  Arguments are handled in the manner of
	// fmt.Printf.
	Logf(format string, v ...interface{})
}

var (
	// The global default logger
	DefaultLogger Logger = &noOpLogger{}
)

// noOpLogger is used as a placeholder for the default logger
type noOpLogger struct{}

func (n *noOpLogger) Log(v ...interface{}) {}

func (n *noOpLogger) Logf(format string, v ...interface{}) {}

// Log logs using the default logger
func Log(v ...interface{}) {
	DefaultLogger.Log(v...)
}

// Logf logs formatted using the default logger
func Logf(format string, v ...interface{}) {
	DefaultLogger.Logf(format, v...)
}
<|endoftext|>


<commit_before>package lru

import (
	"container/list"
	"encoding/base64"
	"hash/fnv"
	"io/ioutil"
	"os"
	"sync"
	"time"
)

type object struct {
	key        string
	size       uint64
	accessTime time.Time
}

type Cache struct {
	sync.Mutex

	list  *list.List
	table map[string]*list.Element
	size  uint64

	maxSize uint64
	path    string
}

func hashCacheKey(data string) string {
	hash := fnv.New64a()
	hash.Write([]byte(data))

	return base64.URLEncoding.EncodeToString(hash.Sum(nil))
}

func New(maxSize uint64, path string) *Cache {
	return &Cache{
		list:    list.New(),
		table:   make(map[string]*list.Element),
		maxSize: maxSize,
		path:    path,
	}
}

func (cache *Cache) FilePath(key string) string {
	return cache.path + "/" + hashCacheKey(key)
}

func (cache *Cache) Get(key string) ([]byte, bool) {
	cache.Lock()
	defer cache.Unlock()

	element := cache.table[key]

	if element == nil {
		return nil, false
	}

	cache.moveToFront(element)

	value, err := ioutil.ReadFile(cache.FilePath(element.Value.(*object).key))

	if err != nil {
		return nil, false
	}

	return value, true
}

func (cache *Cache) Set(key string, value []byte) {
	cache.Lock()
	defer cache.Unlock()

	if element := cache.table[key]; element != nil {
		cache.moveToFront(element)
	} else {
		cache.addNew(key, value)
	}
}

func (cache *Cache) Delete(key string) bool {
	cache.Lock()
	defer cache.Unlock()

	element := cache.table[key]

	if element == nil {
		return false
	}

	err := os.Remove(cache.FilePath(key))

	if err != nil {
		return false
	}

	cache.list.Remove(element)
	delete(cache.table, key)

	cache.size -= element.Value.(*object).size

	return true
}

func (cache *Cache) Clear() {
	cache.Lock()
	defer cache.Unlock()

	cache.clearFiles()
	cache.list.Init()
	cache.table = make(map[string]*list.Element)
	cache.size = 0
}

func (cache *Cache) Size() uint64 {
	cache.Lock()
	defer cache.Unlock()

	return cache.size
}

func (cache *Cache) MaxSize() uint64 {
	cache.Lock()
	defer cache.Unlock()

	return cache.maxSize
}

func (cache *Cache) Oldest() (oldest time.Time) {
	cache.Lock()
	defer cache.Unlock()

	if lastElem := cache.list.Back(); lastElem != nil {
		oldest = lastElem.Value.(*object).accessTime
	}

	return
}

func (cache *Cache) keys() []string {
	keys := make([]string, 0, cache.list.Len())

	for element := cache.list.Front(); element != nil; element = element.Next() {
		keys = append(keys, element.Value.(*object).key)
	}

	return keys
}

func (cache *Cache) moveToFront(element *list.Element) {
	cache.list.MoveToFront(element)
	element.Value.(*object).accessTime = time.Now()
}

func (cache *Cache) addNew(key string, value []byte) {
	newObject := &object{key, uint64(len(value)), time.Now()}
	futureSize := cache.size + newObject.size

	cache.trim(futureSize)

	if _, err := os.Stat(cache.FilePath(key)); os.IsNotExist(err) {
		err := ioutil.WriteFile(cache.FilePath(key), value, 0644)

		if err != nil {
			return
		}
	}

	element := cache.list.PushFront(newObject)
	cache.table[key] = element
	cache.size += newObject.size
}

func (cache *Cache) trim(futureSize uint64) {
	for futureSize > cache.maxSize {
		element := cache.list.Back()

		if element == nil {
			return
		}

		value := cache.list.Remove(element).(*object)

		os.RemoveAll(cache.FilePath(value.key))
		delete(cache.table, value.key)

		cache.size -= value.size
		futureSize -= value.size
	}
}

func (cache *Cache) clearFiles() {
	for _, key := range cache.keys() {
		os.RemoveAll(cache.FilePath(key))
	}
}
<commit_msg>Do not add to cache if it's larger than the cache itself<commit_after>package lru

import (
	"container/list"
	"encoding/base64"
	"hash/fnv"
	"io/ioutil"
	"os"
	"sync"
	"time"
)

type object struct {
	key        string
	size       uint64
	accessTime time.Time
}

type Cache struct {
	sync.Mutex

	list  *list.List
	table map[string]*list.Element
	size  uint64

	maxSize uint64
	path    string
}

func hashCacheKey(data string) string {
	hash := fnv.New64a()
	hash.Write([]byte(data))

	return base64.URLEncoding.EncodeToString(hash.Sum(nil))
}

func New(maxSize uint64, path string) *Cache {
	return &Cache{
		list:    list.New(),
		table:   make(map[string]*list.Element),
		maxSize: maxSize,
		path:    path,
	}
}

func (cache *Cache) FilePath(key string) string {
	return cache.path + "/" + hashCacheKey(key)
}

func (cache *Cache) Get(key string) ([]byte, bool) {
	cache.Lock()
	defer cache.Unlock()

	element := cache.table[key]

	if element == nil {
		return nil, false
	}

	cache.moveToFront(element)

	value, err := ioutil.ReadFile(cache.FilePath(element.Value.(*object).key))

	if err != nil {
		return nil, false
	}

	return value, true
}

func (cache *Cache) Set(key string, value []byte) {
	cache.Lock()
	defer cache.Unlock()

	if element := cache.table[key]; element != nil {
		cache.moveToFront(element)
	} else {
		cache.addNew(key, value)
	}
}

func (cache *Cache) Delete(key string) bool {
	cache.Lock()
	defer cache.Unlock()

	element := cache.table[key]

	if element == nil {
		return false
	}

	err := os.Remove(cache.FilePath(key))

	if err != nil {
		return false
	}

	cache.list.Remove(element)
	delete(cache.table, key)

	cache.size -= element.Value.(*object).size

	return true
}

func (cache *Cache) Clear() {
	cache.Lock()
	defer cache.Unlock()

	cache.clearFiles()
	cache.list.Init()
	cache.table = make(map[string]*list.Element)
	cache.size = 0
}

func (cache *Cache) Size() uint64 {
	cache.Lock()
	defer cache.Unlock()

	return cache.size
}

func (cache *Cache) MaxSize() uint64 {
	cache.Lock()
	defer cache.Unlock()

	return cache.maxSize
}

func (cache *Cache) Oldest() (oldest time.Time) {
	cache.Lock()
	defer cache.Unlock()

	if lastElem := cache.list.Back(); lastElem != nil {
		oldest = lastElem.Value.(*object).accessTime
	}

	return
}

func (cache *Cache) keys() []string {
	keys := make([]string, 0, cache.list.Len())

	for element := cache.list.Front(); element != nil; element = element.Next() {
		keys = append(keys, element.Value.(*object).key)
	}

	return keys
}

func (cache *Cache) moveToFront(element *list.Element) {
	cache.list.MoveToFront(element)
	element.Value.(*object).accessTime = time.Now()
}

func (cache *Cache) addNew(key string, value []byte) {
	size := uint64(len(value))

	if size > cache.maxSize {
		return
	}

	newObject := &object{key, size, time.Now()}
	futureSize := cache.size + newObject.size

	cache.trim(futureSize)

	if _, err := os.Stat(cache.FilePath(key)); os.IsNotExist(err) {
		err := ioutil.WriteFile(cache.FilePath(key), value, 0644)

		if err != nil {
			return
		}
	}

	element := cache.list.PushFront(newObject)
	cache.table[key] = element
	cache.size += newObject.size
}

func (cache *Cache) trim(futureSize uint64) {
	for futureSize > cache.maxSize {
		element := cache.list.Back()

		if element == nil {
			return
		}

		value := cache.list.Remove(element).(*object)

		os.RemoveAll(cache.FilePath(value.key))
		delete(cache.table, value.key)

		cache.size -= value.size
		futureSize -= value.size
	}
}

func (cache *Cache) clearFiles() {
	for _, key := range cache.keys() {
		os.RemoveAll(cache.FilePath(key))
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 Paul Bellamy. All rights reserved.
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package pipe

import (
	"fmt"
	"reflect"
)

func Map(input interface{}, fn interface{}) interface{} {
	checkMapFuncType(input, fn)

	inputValue := reflect.ValueOf(input)
	fnValue := reflect.ValueOf(fn)

	switch inputValue.Kind() {
	case reflect.Chan:
		return mapChan(inputValue, fnValue)
	case reflect.Array:
		return mapSlice(inputValue, fnValue)
	case reflect.Slice:
		return mapSlice(inputValue, fnValue)
	case reflect.Map:
		return mapMap(inputValue, fnValue)
	}
	panic("Map called on invalid type")
}

// Pass through the result of the map function for each item
func mapChan(input, fn reflect.Value) interface{} {
	outputType := reflect.ChanOf(reflect.BothDir, fn.Type().Out(0))
	output := reflect.MakeChan(outputType, 0)
	go func() {
		for {
			item, ok := input.Recv()
			if !ok {
				break
			}

			output.Send(fn.Call([]reflect.Value{item})[0])
		}
		output.Close()
	}()
	return output.Interface()
}

func mapSlice(input, fn reflect.Value) interface{} {
	outputType := reflect.SliceOf(fn.Type().Out(0))
	output := reflect.MakeSlice(outputType, 0, input.Len())
	for i := 0; i < input.Len(); i++ {
		output = reflect.Append(output, fn.Call([]reflect.Value{input.Index(i)})[0])
	}

	return output.Interface()
}

func mapMap(input, fn reflect.Value) interface{} {
	outputType := reflect.SliceOf(fn.Type().Out(0))
	output := reflect.MakeSlice(outputType, 0, input.Len())
	for _, key := range input.MapKeys() {
		output = reflect.Append(output, fn.Call([]reflect.Value{key, input.MapIndex(key)})[0])
	}

	return output.Interface()
}

func checkMapFuncType(input interface{}, fn interface{}) {
	inputType := reflect.TypeOf(input)
	fnType := reflect.TypeOf(fn)

	valid := (fnType.NumOut() == 1)
	switch inputType.Kind() {
	case reflect.Map:
		valid = fnType.NumIn() == 2 &&
			fnType.In(0) == inputType.Key() &&
			fnType.In(1) == inputType.Elem()
	default:
		valid = fnType.NumIn() == 1 &&
			fnType.In(0) == inputType.Elem()
	}

	if !valid {
		panic(fmt.Sprintf("Map fn must be of type func(%v) T, but was %v", inputType.Elem(), fnType))
	}
}
<commit_msg>using Seq for iteration in map<commit_after>// Copyright 2014 Paul Bellamy. All rights reserved.
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package pipe

import (
	"fmt"
	"reflect"
	"seq"
)

func Map(input interface{}, fn interface{}) interface{} {
	checkMapFuncType(input, fn)

	inputValue := reflect.ValueOf(input)
	fnValue := reflect.ValueOf(fn)

	switch inputValue.Kind() {
	case reflect.Chan:
		return mapChan(inputValue, fnValue)
	case reflect.Array:
		return mapSlice(inputValue, fnValue)
	case reflect.Slice:
		return mapSlice(inputValue, fnValue)
	case reflect.Map:
		return mapMap(inputValue, fnValue)
	}
	panic("Map called on invalid type")
}

// Pass through the result of the map function for each item
func mapChan(input, fn reflect.Value) interface{} {
	outputType := reflect.ChanOf(reflect.BothDir, fn.Type().Out(0))
	output := reflect.MakeChan(outputType, 0)
	go func() {
		for s := seq.New(input); !s.Empty(); s = s.Rest() {
			output.Send(fn.Call([]reflect.Value{reflect.ValueOf(s.First())})[0])
		}
		output.Close()
	}()
	return output.Interface()
}

func mapSlice(input, fn reflect.Value) interface{} {
	outputType := reflect.SliceOf(fn.Type().Out(0))
	output := reflect.MakeSlice(outputType, 0, input.Len())

	for s := seq.New(input); !s.Empty(); s = s.Rest() {
		output = reflect.Append(
			output,
			fn.Call([]reflect.Value{reflect.ValueOf(s.First())})[0],
		)
	}

	return output.Interface()
}

func mapMap(input, fn reflect.Value) interface{} {
	outputType := reflect.SliceOf(fn.Type().Out(0))
	output := reflect.MakeSlice(outputType, 0, input.Len())
	for _, key := range input.MapKeys() {
		output = reflect.Append(output, fn.Call([]reflect.Value{key, input.MapIndex(key)})[0])
	}

	return output.Interface()
}

func checkMapFuncType(input interface{}, fn interface{}) {
	inputType := reflect.TypeOf(input)
	fnType := reflect.TypeOf(fn)

	valid := (fnType.NumOut() == 1)
	switch inputType.Kind() {
	case reflect.Map:
		valid = fnType.NumIn() == 2 &&
			fnType.In(0) == inputType.Key() &&
			fnType.In(1) == inputType.Elem()
	default:
		valid = fnType.NumIn() == 1 &&
			fnType.In(0) == inputType.Elem()
	}

	if !valid {
		panic(fmt.Sprintf("Map fn must be of type func(%v) T, but was %v", inputType.Elem(), fnType))
	}
}
<|endoftext|>


<commit_before>package mds

import (
	"errors"
	"fmt"
	"github.com/mitchellh/mapstructure"
	"strings"
)

const (
	// mds version
	VERSION = "0.0.1"

	// type: MongoDB
	MONGODB = "MongoDB"

	// type: Redis
	//REDIS = "Redis"
)

type (
	// Global Mds
	Mds struct {
		DataStores map[string]interface{}
		Version    string
		Setuped    bool
	}
)

// mds to string
func String() string {
	return fmt.Sprintf("Version: %s, DataStores Count:%d", mds.Version, len(mds.DataStores))
}

// Debug flag
var DEBUG = false

// Debug output
func Debug(f string, msgs ...string) {
	if DEBUG {
		fmt.Printf(""+f+"\n", strings.Join(([]string)(msgs), " "))
	}
}

// Global Mds instance
var mds *Mds = &Mds{
	Version:    VERSION,
	DataStores: make(map[string]interface{}),
	Setuped:    false,
}

// Add a datastore
func AddDataStore(dn string, value interface{}) interface{} {
	mds.DataStores[dn] = value
	Debug("Add the datastore. dn=%s\n", dn)
	return value
}

// Get a datastore
func GetDataStore(dn string) (interface{}, error) {
	ds := mds.DataStores[dn]

	if ds == nil {
		return nil, errors.New("Datastore not found")
	}
	return ds, nil

}

// Get the Mongodb datastore
func GetDataStoreMongoDB(dn string) (*MongoDB, error) {
	ds, err := GetDataStore(dn)
	if err != nil {
		return nil, err
	}

	if ret, ok := ds.(*MongoDB); ok {
		return ret, nil
	}

	return nil, errors.New("Internal data store type is invalid")
}

// mds setup (Once)
// connected: セットアップと同時にコネクティングする
func Setup(dss []map[string]interface{}, autoconnected bool) error {

	if mds.Setuped {
		return errors.New("Already setup performed")
	}

	for _, ds := range dss {

		if !ds["Use"].(bool) {
			Debug("Skip the datastore. dn=%s\n", ds["Dn"].(string))
			continue
		}

		switch ds["Type"].(string) {
		case MONGODB:
			mongodb := &MongoDB{}

			err := mapstructure.Decode(ds, mongodb)

			if err != nil {
				return err
			}

			if autoconnected == true {
				Debug("auto-connecting dn=%s\n", ds["Dn"].(string))
				err := mongodb.Connect()
				if err != nil {
					return err
				}
			}

			AddDataStore(mongodb.Dn, mongodb)
		}
	}

	mds.Setuped = true

	return nil
}
<commit_msg>Fix: small fix<commit_after>package mds

import (
	"errors"
	"fmt"
	"github.com/mitchellh/mapstructure"
	"strings"
)

const (
	// mds version
	VERSION = "0.0.1"

	// type: MongoDB
	MONGODB = "MongoDB"

	// type: Redis
	//REDIS = "Redis"
)

type (
	// Global Mds
	Mds struct {
		DataStores map[string]interface{}
		Version    string
		Setuped    bool
	}
)

// mds to string
func String() string {
	return fmt.Sprintf("Version: %s, DataStores Count:%d", mds.Version, len(mds.DataStores))
}

// Debug flag
var DEBUG = false

// Debug output
func Debug(f string, msgs ...string) {
	if DEBUG {
		fmt.Printf(""+f+"\n", strings.Join(([]string)(msgs), " "))
	}
}

// Global Mds instance
var mds *Mds = &Mds{
	Version:    VERSION,
	DataStores: make(map[string]interface{}),
	Setuped:    false,
}

// Add a datastore
func AddDataStore(dn string, value interface{}) interface{} {
	mds.DataStores[dn] = value
	Debug("Add the datastore. dn=%s\n", dn)
	return value
}

// Get a datastore
func GetDataStore(dn string) (interface{}, error) {
	ds := mds.DataStores[dn]

	if ds == nil {
		return nil, errors.New("Datastore not found")
	}
	return ds, nil

}

// Get the Mongodb datastore
func GetDataStoreMongoDB(dn string) (*MongoDB, error) {
	ds, err := GetDataStore(dn)
	if err != nil {
		return nil, err
	}

	if ret, ok := ds.(*MongoDB); ok {
		return ret, nil
	}

	return nil, errors.New("Internal data store type is invalid")
}

// mds setup (Once)
// autoconnect: connect automatically
func Setup(dss []map[string]interface{}, autoconnect bool) error {

	if mds.Setuped {
		return errors.New("Already setup performed")
	}

	for _, ds := range dss {

		if !ds["Use"].(bool) {
			Debug("Skip the datastore. dn=%s\n", ds["Dn"].(string))
			continue
		}

		switch ds["Type"].(string) {
		case MONGODB:
			mongodb := &MongoDB{}

			err := mapstructure.Decode(ds, mongodb)

			if err != nil {
				return err
			}

			if autoconnect == true {
				Debug("auto-connecting dn=%s\n", ds["Dn"].(string))
				err := mongodb.Connect()
				if err != nil {
					return err
				}
			}

			AddDataStore(mongodb.Dn, mongodb)
		}
	}

	mds.Setuped = true

	return nil
}
<|endoftext|>


<commit_before>// +build ignore

package main

import (
	"bufio"
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
)

// ForbiddenImports are the packages from the stdlib that should not be used in
// our code.
var ForbiddenImports = map[string]bool{
	"errors": true,
}

var runCrossCompile = flag.Bool("cross-compile", true, "run cross compilation tests")

func init() {
	flag.Parse()
}

// CIEnvironment is implemented by environments where tests can be run.
type CIEnvironment interface {
	Prepare() error
	RunTests() error
	Teardown() error
}

// TravisEnvironment is the environment in which Travis tests run.
type TravisEnvironment struct {
	goxOSArch []string
	env       map[string]string
}

func (env *TravisEnvironment) getMinio() error {
	tempfile, err := os.Create(filepath.Join(os.Getenv("GOPATH"), "bin", "minio"))
	if err != nil {
		return fmt.Errorf("create tempfile for minio download failed: %v\n", err)
	}

	url := fmt.Sprintf("https://dl.minio.io/server/minio/release/%s-%s/minio",
		runtime.GOOS, runtime.GOARCH)
	msg("downloading %v\n", url)
	res, err := http.Get(url)
	if err != nil {
		return fmt.Errorf("error downloading minio server: %v\n", err)
	}

	_, err = io.Copy(tempfile, res.Body)
	if err != nil {
		return fmt.Errorf("error saving minio server to file: %v\n", err)
	}

	err = res.Body.Close()
	if err != nil {
		return fmt.Errorf("error closing HTTP download: %v\n", err)
	}

	err = tempfile.Close()
	if err != nil {
		msg("closing tempfile failed: %v\n", err)
		return fmt.Errorf("error closing minio server file: %v\n", err)
	}

	err = os.Chmod(tempfile.Name(), 0755)
	if err != nil {
		return fmt.Errorf("chmod(minio-server) failed: %v", err)
	}

	msg("downloaded minio server to %v\n", tempfile.Name())
	return nil
}

// Prepare installs dependencies and starts services in order to run the tests.
func (env *TravisEnvironment) Prepare() error {
	env.env = make(map[string]string)

	msg("preparing environment for Travis CI\n")

	pkgs := []string{
		"golang.org/x/tools/cmd/cover",
		"github.com/pierrre/gotestcover",
		"github.com/NebulousLabs/glyphcheck",
		"github.com/restic/rest-server",
	}

	for _, pkg := range pkgs {
		err := run("go", "get", pkg)
		if err != nil {
			return err
		}
	}

	if err := env.getMinio(); err != nil {
		return err
	}

	if *runCrossCompile {
		// only test cross compilation on linux with Travis
		if err := run("go", "get", "github.com/mitchellh/gox"); err != nil {
			return err
		}
		if runtime.GOOS == "linux" {
			env.goxOSArch = []string{
				"linux/386", "linux/amd64",
				"windows/386", "windows/amd64",
				"darwin/386", "darwin/amd64",
				"freebsd/386", "freebsd/amd64",
				"openbsd/386", "openbsd/amd64",
				"linux/arm", "freebsd/arm",
			}
		} else {
			env.goxOSArch = []string{runtime.GOOS + "/" + runtime.GOARCH}
		}

		msg("gox: OS/ARCH %v\n", env.goxOSArch)
	}

	return nil
}

// Teardown stops backend services and cleans the environment again.
func (env *TravisEnvironment) Teardown() error {
	msg("run travis teardown\n")
	return nil
}

// RunTests starts the tests for Travis.
func (env *TravisEnvironment) RunTests() error {
	// do not run fuse tests on darwin
	if runtime.GOOS == "darwin" {
		msg("skip fuse integration tests on %v\n", runtime.GOOS)
		_ = os.Setenv("RESTIC_TEST_FUSE", "0")
	}

	cwd, err := os.Getwd()
	if err != nil {
		return fmt.Errorf("Getwd() returned error: %v", err)
	}

	env.env["GOPATH"] = cwd + ":" + filepath.Join(cwd, "vendor")

	// ensure that the following tests cannot be silently skipped on Travis
	ensureTests := []string{
		"restic/backend/rest.TestBackendREST",
		"restic/backend/sftp.TestBackendSFTP",
		"restic/backend/s3.TestBackendMinio",
	}

	// if the test s3 repository is available, make sure that the test is not skipped
	if os.Getenv("RESTIC_TEST_S3_REPOSITORY") != "" {
		ensureTests = append(ensureTests, "restic/backend/s3.TestBackendS3")
	} else {
		msg("S3 repository not available\n")
	}

	// if the test swift service is available, make sure that the test is not skipped
	if os.Getenv("RESTIC_TEST_SWIFT") != "" {
		ensureTests = append(ensureTests, "restic/backend/swift.TestBackendSwift")
	} else {
		msg("Swift service not available\n")
	}

	// if the test b2 repository is available, make sure that the test is not skipped
	if os.Getenv("RESTIC_TEST_B2_REPOSITORY") != "" {
		ensureTests = append(ensureTests, "restic/backend/b2.TestBackendB2")
	} else {
		msg("B2 repository not available\n")
	}

	env.env["RESTIC_TEST_DISALLOW_SKIP"] = strings.Join(ensureTests, ",")

	if *runCrossCompile {
		// compile for all target architectures with tags
		for _, tags := range []string{"release", "debug"} {
			err := runWithEnv(env.env, "gox", "-verbose",
				"-osarch", strings.Join(env.goxOSArch, " "),
				"-tags", tags,
				"-output", "/tmp/{{.Dir}}_{{.OS}}_{{.Arch}}",
				"cmds/restic")
			if err != nil {
				return err
			}
		}
	}

	// run the build script
	if err := run("go", "run", "build.go"); err != nil {
		return err
	}

	// run the tests and gather coverage information
	err = runWithEnv(env.env, "gotestcover", "-coverprofile", "all.cov", "cmds/...", "restic/...")
	if err != nil {
		return err
	}

	if err = runGofmt(); err != nil {
		return err
	}

	if err = runGlyphcheck(); err != nil {
		return err
	}

	deps, err := findImports()
	if err != nil {
		return err
	}

	foundForbiddenImports := false
	for name, imports := range deps {
		for _, pkg := range imports {
			if _, ok := ForbiddenImports[pkg]; ok {
				fmt.Fprintf(os.Stderr, "========== package %v imports forbidden package %v\n", name, pkg)
				foundForbiddenImports = true
			}
		}
	}

	if foundForbiddenImports {
		return errors.New("CI: forbidden imports found")
	}

	return nil
}

// AppveyorEnvironment is the environment on Windows.
type AppveyorEnvironment struct{}

// Prepare installs dependencies and starts services in order to run the tests.
func (env *AppveyorEnvironment) Prepare() error {
	msg("preparing environment for Appveyor CI\n")
	return nil
}

// RunTests start the tests.
func (env *AppveyorEnvironment) RunTests() error {
	return run("go", "run", "build.go", "-v", "-T")
}

// Teardown is a noop.
func (env *AppveyorEnvironment) Teardown() error {
	return nil
}

// findGoFiles returns a list of go source code file names below dir.
func findGoFiles(dir string) (list []string, err error) {
	err = filepath.Walk(dir, func(name string, fi os.FileInfo, err error) error {
		relpath, err := filepath.Rel(dir, name)
		if err != nil {
			return err
		}

		if relpath == "vendor" || relpath == "pkg" {
			return filepath.SkipDir
		}

		if filepath.Ext(relpath) == ".go" {
			list = append(list, relpath)
		}

		return err
	})

	return list, err
}

func msg(format string, args ...interface{}) {
	fmt.Printf("CI: "+format, args...)
}

func updateEnv(env []string, override map[string]string) []string {
	var newEnv []string
	for _, s := range env {
		d := strings.SplitN(s, "=", 2)
		key := d[0]

		if _, ok := override[key]; ok {
			continue
		}

		newEnv = append(newEnv, s)
	}

	for k, v := range override {
		newEnv = append(newEnv, k+"="+v)
	}

	return newEnv
}

func findImports() (map[string][]string, error) {
	res := make(map[string][]string)

	cwd, err := os.Getwd()
	if err != nil {
		return nil, fmt.Errorf("Getwd() returned error: %v", err)
	}

	gopath := cwd + ":" + filepath.Join(cwd, "vendor")

	cmd := exec.Command("go", "list", "-f", `{{.ImportPath}} {{join .Imports " "}}`, "./src/...")
	cmd.Env = updateEnv(os.Environ(), map[string]string{"GOPATH": gopath})
	cmd.Stderr = os.Stderr

	output, err := cmd.Output()
	if err != nil {
		return nil, err
	}

	sc := bufio.NewScanner(bytes.NewReader(output))
	for sc.Scan() {
		wordScanner := bufio.NewScanner(strings.NewReader(sc.Text()))
		wordScanner.Split(bufio.ScanWords)

		if !wordScanner.Scan() {
			return nil, fmt.Errorf("package name not found in line: %s", output)
		}
		name := wordScanner.Text()
		var deps []string

		for wordScanner.Scan() {
			deps = append(deps, wordScanner.Text())
		}

		res[name] = deps
	}

	return res, nil
}

func runGofmt() error {
	dir, err := os.Getwd()
	if err != nil {
		return fmt.Errorf("Getwd(): %v\n", err)
	}

	files, err := findGoFiles(dir)
	if err != nil {
		return fmt.Errorf("error finding Go files: %v\n", err)
	}

	msg("runGofmt() with %d files\n", len(files))
	args := append([]string{"-l"}, files...)
	cmd := exec.Command("gofmt", args...)
	cmd.Stderr = os.Stderr

	buf, err := cmd.Output()
	if err != nil {
		return fmt.Errorf("error running gofmt: %v\noutput: %s\n", err, buf)
	}

	if len(buf) > 0 {
		return fmt.Errorf("not formatted with `gofmt`:\n%s\n", buf)
	}

	return nil
}

func runGlyphcheck() error {
	cmd := exec.Command("glyphcheck", "./...")
	cmd.Stderr = os.Stderr

	buf, err := cmd.Output()
	if err != nil {
		return fmt.Errorf("error running glyphcheck: %v\noutput: %s\n", err, buf)
	}

	return nil
}

func run(command string, args ...string) error {
	msg("run %v %v\n", command, strings.Join(args, " "))
	return runWithEnv(nil, command, args...)
}

// runWithEnv calls a command with the current environment, except the entries
// of the env map are set additionally.
func runWithEnv(env map[string]string, command string, args ...string) error {
	msg("runWithEnv %v %v\n", command, strings.Join(args, " "))
	cmd := exec.Command(command, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if env != nil {
		cmd.Env = updateEnv(os.Environ(), env)
	}
	err := cmd.Run()

	if err != nil {
		return fmt.Errorf("error running %v %v: %v",
			command, strings.Join(args, " "), err)
	}
	return nil
}

func isTravis() bool {
	return os.Getenv("TRAVIS_BUILD_DIR") != ""
}

func isAppveyor() bool {
	return runtime.GOOS == "windows"
}

func main() {
	var env CIEnvironment

	switch {
	case isTravis():
		env = &TravisEnvironment{}
	case isAppveyor():
		env = &AppveyorEnvironment{}
	default:
		fmt.Fprintln(os.Stderr, "unknown CI environment")
		os.Exit(1)
	}

	foundError := false
	for _, f := range []func() error{env.Prepare, env.RunTests, env.Teardown} {
		err := f()
		if err != nil {
			foundError = true
			fmt.Fprintf(os.Stderr, "error: %v\n", err)
		}
	}

	if foundError {
		os.Exit(1)
	}
}
<commit_msg>Correct path for rest-server<commit_after>// +build ignore

package main

import (
	"bufio"
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
)

// ForbiddenImports are the packages from the stdlib that should not be used in
// our code.
var ForbiddenImports = map[string]bool{
	"errors": true,
}

var runCrossCompile = flag.Bool("cross-compile", true, "run cross compilation tests")

func init() {
	flag.Parse()
}

// CIEnvironment is implemented by environments where tests can be run.
type CIEnvironment interface {
	Prepare() error
	RunTests() error
	Teardown() error
}

// TravisEnvironment is the environment in which Travis tests run.
type TravisEnvironment struct {
	goxOSArch []string
	env       map[string]string
}

func (env *TravisEnvironment) getMinio() error {
	tempfile, err := os.Create(filepath.Join(os.Getenv("GOPATH"), "bin", "minio"))
	if err != nil {
		return fmt.Errorf("create tempfile for minio download failed: %v\n", err)
	}

	url := fmt.Sprintf("https://dl.minio.io/server/minio/release/%s-%s/minio",
		runtime.GOOS, runtime.GOARCH)
	msg("downloading %v\n", url)
	res, err := http.Get(url)
	if err != nil {
		return fmt.Errorf("error downloading minio server: %v\n", err)
	}

	_, err = io.Copy(tempfile, res.Body)
	if err != nil {
		return fmt.Errorf("error saving minio server to file: %v\n", err)
	}

	err = res.Body.Close()
	if err != nil {
		return fmt.Errorf("error closing HTTP download: %v\n", err)
	}

	err = tempfile.Close()
	if err != nil {
		msg("closing tempfile failed: %v\n", err)
		return fmt.Errorf("error closing minio server file: %v\n", err)
	}

	err = os.Chmod(tempfile.Name(), 0755)
	if err != nil {
		return fmt.Errorf("chmod(minio-server) failed: %v", err)
	}

	msg("downloaded minio server to %v\n", tempfile.Name())
	return nil
}

// Prepare installs dependencies and starts services in order to run the tests.
func (env *TravisEnvironment) Prepare() error {
	env.env = make(map[string]string)

	msg("preparing environment for Travis CI\n")

	pkgs := []string{
		"golang.org/x/tools/cmd/cover",
		"github.com/pierrre/gotestcover",
		"github.com/NebulousLabs/glyphcheck",
		"github.com/restic/rest-server/cmd/rest-server",
	}

	for _, pkg := range pkgs {
		err := run("go", "get", pkg)
		if err != nil {
			return err
		}
	}

	if err := env.getMinio(); err != nil {
		return err
	}

	if *runCrossCompile {
		// only test cross compilation on linux with Travis
		if err := run("go", "get", "github.com/mitchellh/gox"); err != nil {
			return err
		}
		if runtime.GOOS == "linux" {
			env.goxOSArch = []string{
				"linux/386", "linux/amd64",
				"windows/386", "windows/amd64",
				"darwin/386", "darwin/amd64",
				"freebsd/386", "freebsd/amd64",
				"openbsd/386", "openbsd/amd64",
				"linux/arm", "freebsd/arm",
			}
		} else {
			env.goxOSArch = []string{runtime.GOOS + "/" + runtime.GOARCH}
		}

		msg("gox: OS/ARCH %v\n", env.goxOSArch)
	}

	return nil
}

// Teardown stops backend services and cleans the environment again.
func (env *TravisEnvironment) Teardown() error {
	msg("run travis teardown\n")
	return nil
}

// RunTests starts the tests for Travis.
func (env *TravisEnvironment) RunTests() error {
	// do not run fuse tests on darwin
	if runtime.GOOS == "darwin" {
		msg("skip fuse integration tests on %v\n", runtime.GOOS)
		_ = os.Setenv("RESTIC_TEST_FUSE", "0")
	}

	cwd, err := os.Getwd()
	if err != nil {
		return fmt.Errorf("Getwd() returned error: %v", err)
	}

	env.env["GOPATH"] = cwd + ":" + filepath.Join(cwd, "vendor")

	// ensure that the following tests cannot be silently skipped on Travis
	ensureTests := []string{
		"restic/backend/rest.TestBackendREST",
		"restic/backend/sftp.TestBackendSFTP",
		"restic/backend/s3.TestBackendMinio",
	}

	// if the test s3 repository is available, make sure that the test is not skipped
	if os.Getenv("RESTIC_TEST_S3_REPOSITORY") != "" {
		ensureTests = append(ensureTests, "restic/backend/s3.TestBackendS3")
	} else {
		msg("S3 repository not available\n")
	}

	// if the test swift service is available, make sure that the test is not skipped
	if os.Getenv("RESTIC_TEST_SWIFT") != "" {
		ensureTests = append(ensureTests, "restic/backend/swift.TestBackendSwift")
	} else {
		msg("Swift service not available\n")
	}

	// if the test b2 repository is available, make sure that the test is not skipped
	if os.Getenv("RESTIC_TEST_B2_REPOSITORY") != "" {
		ensureTests = append(ensureTests, "restic/backend/b2.TestBackendB2")
	} else {
		msg("B2 repository not available\n")
	}

	env.env["RESTIC_TEST_DISALLOW_SKIP"] = strings.Join(ensureTests, ",")

	if *runCrossCompile {
		// compile for all target architectures with tags
		for _, tags := range []string{"release", "debug"} {
			err := runWithEnv(env.env, "gox", "-verbose",
				"-osarch", strings.Join(env.goxOSArch, " "),
				"-tags", tags,
				"-output", "/tmp/{{.Dir}}_{{.OS}}_{{.Arch}}",
				"cmds/restic")
			if err != nil {
				return err
			}
		}
	}

	// run the build script
	if err := run("go", "run", "build.go"); err != nil {
		return err
	}

	// run the tests and gather coverage information
	err = runWithEnv(env.env, "gotestcover", "-coverprofile", "all.cov", "cmds/...", "restic/...")
	if err != nil {
		return err
	}

	if err = runGofmt(); err != nil {
		return err
	}

	if err = runGlyphcheck(); err != nil {
		return err
	}

	deps, err := findImports()
	if err != nil {
		return err
	}

	foundForbiddenImports := false
	for name, imports := range deps {
		for _, pkg := range imports {
			if _, ok := ForbiddenImports[pkg]; ok {
				fmt.Fprintf(os.Stderr, "========== package %v imports forbidden package %v\n", name, pkg)
				foundForbiddenImports = true
			}
		}
	}

	if foundForbiddenImports {
		return errors.New("CI: forbidden imports found")
	}

	return nil
}

// AppveyorEnvironment is the environment on Windows.
type AppveyorEnvironment struct{}

// Prepare installs dependencies and starts services in order to run the tests.
func (env *AppveyorEnvironment) Prepare() error {
	msg("preparing environment for Appveyor CI\n")
	return nil
}

// RunTests start the tests.
func (env *AppveyorEnvironment) RunTests() error {
	return run("go", "run", "build.go", "-v", "-T")
}

// Teardown is a noop.
func (env *AppveyorEnvironment) Teardown() error {
	return nil
}

// findGoFiles returns a list of go source code file names below dir.
func findGoFiles(dir string) (list []string, err error) {
	err = filepath.Walk(dir, func(name string, fi os.FileInfo, err error) error {
		relpath, err := filepath.Rel(dir, name)
		if err != nil {
			return err
		}

		if relpath == "vendor" || relpath == "pkg" {
			return filepath.SkipDir
		}

		if filepath.Ext(relpath) == ".go" {
			list = append(list, relpath)
		}

		return err
	})

	return list, err
}

func msg(format string, args ...interface{}) {
	fmt.Printf("CI: "+format, args...)
}

func updateEnv(env []string, override map[string]string) []string {
	var newEnv []string
	for _, s := range env {
		d := strings.SplitN(s, "=", 2)
		key := d[0]

		if _, ok := override[key]; ok {
			continue
		}

		newEnv = append(newEnv, s)
	}

	for k, v := range override {
		newEnv = append(newEnv, k+"="+v)
	}

	return newEnv
}

func findImports() (map[string][]string, error) {
	res := make(map[string][]string)

	cwd, err := os.Getwd()
	if err != nil {
		return nil, fmt.Errorf("Getwd() returned error: %v", err)
	}

	gopath := cwd + ":" + filepath.Join(cwd, "vendor")

	cmd := exec.Command("go", "list", "-f", `{{.ImportPath}} {{join .Imports " "}}`, "./src/...")
	cmd.Env = updateEnv(os.Environ(), map[string]string{"GOPATH": gopath})
	cmd.Stderr = os.Stderr

	output, err := cmd.Output()
	if err != nil {
		return nil, err
	}

	sc := bufio.NewScanner(bytes.NewReader(output))
	for sc.Scan() {
		wordScanner := bufio.NewScanner(strings.NewReader(sc.Text()))
		wordScanner.Split(bufio.ScanWords)

		if !wordScanner.Scan() {
			return nil, fmt.Errorf("package name not found in line: %s", output)
		}
		name := wordScanner.Text()
		var deps []string

		for wordScanner.Scan() {
			deps = append(deps, wordScanner.Text())
		}

		res[name] = deps
	}

	return res, nil
}

func runGofmt() error {
	dir, err := os.Getwd()
	if err != nil {
		return fmt.Errorf("Getwd(): %v\n", err)
	}

	files, err := findGoFiles(dir)
	if err != nil {
		return fmt.Errorf("error finding Go files: %v\n", err)
	}

	msg("runGofmt() with %d files\n", len(files))
	args := append([]string{"-l"}, files...)
	cmd := exec.Command("gofmt", args...)
	cmd.Stderr = os.Stderr

	buf, err := cmd.Output()
	if err != nil {
		return fmt.Errorf("error running gofmt: %v\noutput: %s\n", err, buf)
	}

	if len(buf) > 0 {
		return fmt.Errorf("not formatted with `gofmt`:\n%s\n", buf)
	}

	return nil
}

func runGlyphcheck() error {
	cmd := exec.Command("glyphcheck", "./...")
	cmd.Stderr = os.Stderr

	buf, err := cmd.Output()
	if err != nil {
		return fmt.Errorf("error running glyphcheck: %v\noutput: %s\n", err, buf)
	}

	return nil
}

func run(command string, args ...string) error {
	msg("run %v %v\n", command, strings.Join(args, " "))
	return runWithEnv(nil, command, args...)
}

// runWithEnv calls a command with the current environment, except the entries
// of the env map are set additionally.
func runWithEnv(env map[string]string, command string, args ...string) error {
	msg("runWithEnv %v %v\n", command, strings.Join(args, " "))
	cmd := exec.Command(command, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if env != nil {
		cmd.Env = updateEnv(os.Environ(), env)
	}
	err := cmd.Run()

	if err != nil {
		return fmt.Errorf("error running %v %v: %v",
			command, strings.Join(args, " "), err)
	}
	return nil
}

func isTravis() bool {
	return os.Getenv("TRAVIS_BUILD_DIR") != ""
}

func isAppveyor() bool {
	return runtime.GOOS == "windows"
}

func main() {
	var env CIEnvironment

	switch {
	case isTravis():
		env = &TravisEnvironment{}
	case isAppveyor():
		env = &AppveyorEnvironment{}
	default:
		fmt.Fprintln(os.Stderr, "unknown CI environment")
		os.Exit(1)
	}

	foundError := false
	for _, f := range []func() error{env.Prepare, env.RunTests, env.Teardown} {
		err := f()
		if err != nil {
			foundError = true
			fmt.Fprintf(os.Stderr, "error: %v\n", err)
		}
	}

	if foundError {
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package relay_test

import (
	"context"
	"net"
	"sync"
	"testing"
	"time"

	libp2p "github.com/libp2p/go-libp2p"
	relay "github.com/libp2p/go-libp2p/p2p/host/relay"

	ggio "github.com/gogo/protobuf/io"
	cid "github.com/ipfs/go-cid"
	autonat "github.com/libp2p/go-libp2p-autonat"
	autonatpb "github.com/libp2p/go-libp2p-autonat/pb"
	circuit "github.com/libp2p/go-libp2p-circuit"
	host "github.com/libp2p/go-libp2p-host"
	inet "github.com/libp2p/go-libp2p-net"
	peer "github.com/libp2p/go-libp2p-peer"
	pstore "github.com/libp2p/go-libp2p-peerstore"
	routing "github.com/libp2p/go-libp2p-routing"
	ma "github.com/multiformats/go-multiaddr"
	manet "github.com/multiformats/go-multiaddr-net"
)

// test specific parameters
func init() {
	autonat.AutoNATIdentifyDelay = 1 * time.Second
	autonat.AutoNATBootDelay = 2 * time.Second
	relay.BootDelay = 1 * time.Second
	relay.AdvertiseBootDelay = 100 * time.Millisecond
}

// mock routing
type mockRoutingTable struct {
	mx        sync.Mutex
	providers map[string]map[peer.ID]pstore.PeerInfo
	peers     map[peer.ID]pstore.PeerInfo
}

type mockRouting struct {
	h   host.Host
	tab *mockRoutingTable
}

func newMockRoutingTable() *mockRoutingTable {
	return &mockRoutingTable{providers: make(map[string]map[peer.ID]pstore.PeerInfo)}
}

func newMockRouting(h host.Host, tab *mockRoutingTable) *mockRouting {
	return &mockRouting{h: h, tab: tab}
}

func (m *mockRouting) FindPeer(ctx context.Context, p peer.ID) (pstore.PeerInfo, error) {
	m.tab.mx.Lock()
	defer m.tab.mx.Unlock()
	pi, ok := m.tab.peers[p]
	if !ok {
		return pstore.PeerInfo{}, routing.ErrNotFound
	}
	return pi, nil
}

func (m *mockRouting) Provide(ctx context.Context, cid cid.Cid, bcast bool) error {
	m.tab.mx.Lock()
	defer m.tab.mx.Unlock()

	pmap, ok := m.tab.providers[cid.String()]
	if !ok {
		pmap = make(map[peer.ID]pstore.PeerInfo)
		m.tab.providers[cid.String()] = pmap
	}

	pi := pstore.PeerInfo{ID: m.h.ID(), Addrs: m.h.Addrs()}
	pmap[m.h.ID()] = pi
	if m.tab.peers == nil {
		m.tab.peers = make(map[peer.ID]pstore.PeerInfo)
	}
	m.tab.peers[m.h.ID()] = pi

	return nil
}

func (m *mockRouting) FindProvidersAsync(ctx context.Context, cid cid.Cid, limit int) <-chan pstore.PeerInfo {
	ch := make(chan pstore.PeerInfo)
	go func() {
		defer close(ch)
		m.tab.mx.Lock()
		defer m.tab.mx.Unlock()

		pmap, ok := m.tab.providers[cid.String()]
		if !ok {
			return
		}

		for _, pi := range pmap {
			select {
			case ch <- pi:
			case <-ctx.Done():
				return
			}
		}
	}()

	return ch
}

// mock autonat
func makeAutoNATServicePrivate(ctx context.Context, t *testing.T) host.Host {
	h, err := libp2p.New(ctx)
	if err != nil {
		t.Fatal(err)
	}
	h.SetStreamHandler(autonat.AutoNATProto, sayAutoNATPrivate)
	return h
}

func sayAutoNATPrivate(s inet.Stream) {
	defer s.Close()
	w := ggio.NewDelimitedWriter(s)
	res := autonatpb.Message{
		Type:         autonatpb.Message_DIAL_RESPONSE.Enum(),
		DialResponse: newDialResponseError(autonatpb.Message_E_DIAL_ERROR, "no dialable addresses"),
	}
	w.WriteMsg(&res)
}

func newDialResponseError(status autonatpb.Message_ResponseStatus, text string) *autonatpb.Message_DialResponse {
	dr := new(autonatpb.Message_DialResponse)
	dr.Status = status.Enum()
	dr.StatusText = &text
	return dr
}

// connector
func connect(t *testing.T, a, b host.Host) {
	pinfo := pstore.PeerInfo{ID: a.ID(), Addrs: a.Addrs()}
	err := b.Connect(context.Background(), pinfo)
	if err != nil {
		t.Fatal(err)
	}
}

// and the actual test!
func TestAutoRelay(t *testing.T) {
	//t.Skip("fails 99% of the time")

	save := manet.Private4
	manet.Private4 = []*net.IPNet{}
	defer func() { manet.Private4 = save }()

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	mtab := newMockRoutingTable()
	makeRouting := func(h host.Host) (routing.PeerRouting, error) {
		mr := newMockRouting(h, mtab)
		return mr, nil
	}

	h1 := makeAutoNATServicePrivate(ctx, t)
	_, err := libp2p.New(ctx, libp2p.EnableRelay(circuit.OptHop), libp2p.EnableAutoRelay(), libp2p.Routing(makeRouting))
	if err != nil {
		t.Fatal(err)
	}
	h3, err := libp2p.New(ctx, libp2p.EnableRelay(), libp2p.EnableAutoRelay(), libp2p.Routing(makeRouting))
	if err != nil {
		t.Fatal(err)
	}
	h4, err := libp2p.New(ctx, libp2p.EnableRelay())

	// verify that we don't advertise relay addrs initially
	for _, addr := range h3.Addrs() {
		_, err := addr.ValueForProtocol(circuit.P_CIRCUIT)
		if err == nil {
			t.Fatal("relay addr advertised before auto detection")
		}
	}

	// connect to AutoNAT and let detection/discovery work its magic
	connect(t, h1, h3)
	time.Sleep(5 * time.Second)

	// verify that we now advertise relay addrs (but not unspecific relay addrs)
	unspecificRelay, err := ma.NewMultiaddr("/p2p-circuit")
	if err != nil {
		t.Fatal(err)
	}

	haveRelay := false
	for _, addr := range h3.Addrs() {
		if addr.Equal(unspecificRelay) {
			t.Fatal("unspecific relay addr advertised")
		}

		_, err := addr.ValueForProtocol(circuit.P_CIRCUIT)
		if err == nil {
			haveRelay = true
		}
	}

	if !haveRelay {
		t.Fatal("No relay addrs advertised")
	}

	// verify that we can connect through the relay
	var raddrs []ma.Multiaddr
	for _, addr := range h3.Addrs() {
		_, err := addr.ValueForProtocol(circuit.P_CIRCUIT)
		if err == nil {
			raddrs = append(raddrs, addr)
		}
	}

	err = h4.Connect(ctx, pstore.PeerInfo{ID: h3.ID(), Addrs: raddrs})
	if err != nil {
		t.Fatal(err)
	}

	// verify that we have pushed relay addrs to connected peers
	haveRelay = false
	for _, addr := range h1.Peerstore().Addrs(h3.ID()) {
		if addr.Equal(unspecificRelay) {
			t.Fatal("unspecific relay addr advertised")
		}

		_, err := addr.ValueForProtocol(circuit.P_CIRCUIT)
		if err == nil {
			haveRelay = true
		}
	}

	if !haveRelay {
		t.Fatal("No relay addrs pushed")
	}
}
<commit_msg>pacify the race detector<commit_after>package relay_test

import (
	"context"
	"net"
	"sync"
	"testing"
	"time"

	libp2p "github.com/libp2p/go-libp2p"
	relay "github.com/libp2p/go-libp2p/p2p/host/relay"

	ggio "github.com/gogo/protobuf/io"
	cid "github.com/ipfs/go-cid"
	autonat "github.com/libp2p/go-libp2p-autonat"
	autonatpb "github.com/libp2p/go-libp2p-autonat/pb"
	circuit "github.com/libp2p/go-libp2p-circuit"
	host "github.com/libp2p/go-libp2p-host"
	inet "github.com/libp2p/go-libp2p-net"
	peer "github.com/libp2p/go-libp2p-peer"
	pstore "github.com/libp2p/go-libp2p-peerstore"
	routing "github.com/libp2p/go-libp2p-routing"
	ma "github.com/multiformats/go-multiaddr"
	manet "github.com/multiformats/go-multiaddr-net"
)

// test specific parameters
func init() {
	autonat.AutoNATIdentifyDelay = 1 * time.Second
	autonat.AutoNATBootDelay = 2 * time.Second
	relay.BootDelay = 1 * time.Second
	relay.AdvertiseBootDelay = 100 * time.Millisecond
}

// mock routing
type mockRoutingTable struct {
	mx        sync.Mutex
	providers map[string]map[peer.ID]pstore.PeerInfo
	peers     map[peer.ID]pstore.PeerInfo
}

type mockRouting struct {
	h   host.Host
	tab *mockRoutingTable
}

func newMockRoutingTable() *mockRoutingTable {
	return &mockRoutingTable{providers: make(map[string]map[peer.ID]pstore.PeerInfo)}
}

func newMockRouting(h host.Host, tab *mockRoutingTable) *mockRouting {
	return &mockRouting{h: h, tab: tab}
}

func (m *mockRouting) FindPeer(ctx context.Context, p peer.ID) (pstore.PeerInfo, error) {
	m.tab.mx.Lock()
	defer m.tab.mx.Unlock()
	pi, ok := m.tab.peers[p]
	if !ok {
		return pstore.PeerInfo{}, routing.ErrNotFound
	}
	return pi, nil
}

func (m *mockRouting) Provide(ctx context.Context, cid cid.Cid, bcast bool) error {
	m.tab.mx.Lock()
	defer m.tab.mx.Unlock()

	pmap, ok := m.tab.providers[cid.String()]
	if !ok {
		pmap = make(map[peer.ID]pstore.PeerInfo)
		m.tab.providers[cid.String()] = pmap
	}

	pi := pstore.PeerInfo{ID: m.h.ID(), Addrs: m.h.Addrs()}
	pmap[m.h.ID()] = pi
	if m.tab.peers == nil {
		m.tab.peers = make(map[peer.ID]pstore.PeerInfo)
	}
	m.tab.peers[m.h.ID()] = pi

	return nil
}

func (m *mockRouting) FindProvidersAsync(ctx context.Context, cid cid.Cid, limit int) <-chan pstore.PeerInfo {
	ch := make(chan pstore.PeerInfo)
	go func() {
		defer close(ch)
		m.tab.mx.Lock()
		defer m.tab.mx.Unlock()

		pmap, ok := m.tab.providers[cid.String()]
		if !ok {
			return
		}

		for _, pi := range pmap {
			select {
			case ch <- pi:
			case <-ctx.Done():
				return
			}
		}
	}()

	return ch
}

// mock autonat
func makeAutoNATServicePrivate(ctx context.Context, t *testing.T) host.Host {
	h, err := libp2p.New(ctx)
	if err != nil {
		t.Fatal(err)
	}
	h.SetStreamHandler(autonat.AutoNATProto, sayAutoNATPrivate)
	return h
}

func sayAutoNATPrivate(s inet.Stream) {
	defer s.Close()
	w := ggio.NewDelimitedWriter(s)
	res := autonatpb.Message{
		Type:         autonatpb.Message_DIAL_RESPONSE.Enum(),
		DialResponse: newDialResponseError(autonatpb.Message_E_DIAL_ERROR, "no dialable addresses"),
	}
	w.WriteMsg(&res)
}

func newDialResponseError(status autonatpb.Message_ResponseStatus, text string) *autonatpb.Message_DialResponse {
	dr := new(autonatpb.Message_DialResponse)
	dr.Status = status.Enum()
	dr.StatusText = &text
	return dr
}

// connector
func connect(t *testing.T, a, b host.Host) {
	pinfo := pstore.PeerInfo{ID: a.ID(), Addrs: a.Addrs()}
	err := b.Connect(context.Background(), pinfo)
	if err != nil {
		t.Fatal(err)
	}
}

// and the actual test!
func TestAutoRelay(t *testing.T) {
	//t.Skip("fails 99% of the time")

	manet.Private4 = []*net.IPNet{}

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	mtab := newMockRoutingTable()
	makeRouting := func(h host.Host) (routing.PeerRouting, error) {
		mr := newMockRouting(h, mtab)
		return mr, nil
	}

	h1 := makeAutoNATServicePrivate(ctx, t)
	_, err := libp2p.New(ctx, libp2p.EnableRelay(circuit.OptHop), libp2p.EnableAutoRelay(), libp2p.Routing(makeRouting))
	if err != nil {
		t.Fatal(err)
	}
	h3, err := libp2p.New(ctx, libp2p.EnableRelay(), libp2p.EnableAutoRelay(), libp2p.Routing(makeRouting))
	if err != nil {
		t.Fatal(err)
	}
	h4, err := libp2p.New(ctx, libp2p.EnableRelay())

	// verify that we don't advertise relay addrs initially
	for _, addr := range h3.Addrs() {
		_, err := addr.ValueForProtocol(circuit.P_CIRCUIT)
		if err == nil {
			t.Fatal("relay addr advertised before auto detection")
		}
	}

	// connect to AutoNAT and let detection/discovery work its magic
	connect(t, h1, h3)
	time.Sleep(5 * time.Second)

	// verify that we now advertise relay addrs (but not unspecific relay addrs)
	unspecificRelay, err := ma.NewMultiaddr("/p2p-circuit")
	if err != nil {
		t.Fatal(err)
	}

	haveRelay := false
	for _, addr := range h3.Addrs() {
		if addr.Equal(unspecificRelay) {
			t.Fatal("unspecific relay addr advertised")
		}

		_, err := addr.ValueForProtocol(circuit.P_CIRCUIT)
		if err == nil {
			haveRelay = true
		}
	}

	if !haveRelay {
		t.Fatal("No relay addrs advertised")
	}

	// verify that we can connect through the relay
	var raddrs []ma.Multiaddr
	for _, addr := range h3.Addrs() {
		_, err := addr.ValueForProtocol(circuit.P_CIRCUIT)
		if err == nil {
			raddrs = append(raddrs, addr)
		}
	}

	err = h4.Connect(ctx, pstore.PeerInfo{ID: h3.ID(), Addrs: raddrs})
	if err != nil {
		t.Fatal(err)
	}

	// verify that we have pushed relay addrs to connected peers
	haveRelay = false
	for _, addr := range h1.Peerstore().Addrs(h3.ID()) {
		if addr.Equal(unspecificRelay) {
			t.Fatal("unspecific relay addr advertised")
		}

		_, err := addr.ValueForProtocol(circuit.P_CIRCUIT)
		if err == nil {
			haveRelay = true
		}
	}

	if !haveRelay {
		t.Fatal("No relay addrs pushed")
	}
}
<|endoftext|>


<commit_before>package ec2

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"launchpad.net/goamz/s3"
	"launchpad.net/goyaml"
)

const stateFile = "provider-state"

type bootstrapState struct {
	ZookeeperInstances []string `yaml:"zookeeper-instances"`
}

func (e *environ) saveState(state *bootstrapState) error {
	data, err := goyaml.Marshal(state)
	if err != nil {
		return err
	}
	return e.PutFile(stateFile, bytes.NewBuffer(data), int64(len(data)))
}

func (e *environ) loadState() (*bootstrapState, error) {
	r, err := e.GetFile(stateFile)
	if err != nil {
		return nil, fmt.Errorf("cannot read %q: %v", stateFile, err)
	}
	defer r.Close()
	data, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, fmt.Errorf("error reading %q: %v", stateFile, err)
	}
	var state bootstrapState
	err = goyaml.Unmarshal(data, &state)
	if err != nil {
		return nil, fmt.Errorf("error unmarshalling %q: %v", stateFile, err)
	}
	return &state, nil
}

func (e *environ) deleteState() error {
	// TODO delete the bucket contents and the bucket itself.
	err := e.RemoveFile(stateFile)
	if err != nil {
		return fmt.Errorf("cannot delete provider state: %v", err)
	}
	return nil
}

// makeBucket makes the environent's control bucket, the
// place where bootstrap information and deployed charms
// are stored. To avoid two round trips on every PUT operation,
// we do this only once for each environ.
func (e *environ) makeBucket() error {
	e.checkBucket.Do(func() {
		// try to make the bucket - PutBucket will succeed if the
		// bucket already exists.
		e.checkBucketError = e.bucket().PutBucket(s3.Private)
	})
	return e.checkBucketError
}

func (e *environ) PutFile(file string, r io.Reader, length int64) error {
	if err := e.makeBucket(); err != nil {
		return fmt.Errorf("cannot make S3 control bucket: %v", err)
	}
	err := e.bucket().PutReader(file, r, length, "binary/octet-stream", s3.Private)
	if err != nil {
		return fmt.Errorf("cannot write file %q to control bucket: %v", file, err)
	}
	return nil
}

func (e *environ) GetFile(file string) (r io.ReadCloser, err error) {
	for a := shortAttempt.start(); a.next(); {
		r, err = e.bucket().GetReader(file)
		if err, _ := err.(*s3.Error); err != nil && err.StatusCode == 404 {
			continue
		}
		return
	}
	return
}

func (e *environ) RemoveFile(file string) error {
	err := e.bucket().Del(file)
	// If we can't delete the object because the bucket doesn't
	// exist, then we don't care.
	if err, _ := err.(*s3.Error); err != nil && err.StatusCode == 404 {
		return nil
	}
	return err
}

func (e *environ) bucket() *s3.Bucket {
	return e.s3.Bucket(e.config.bucket)
}
<commit_msg>environs/ec2: add UploadTools stub<commit_after>package ec2

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"launchpad.net/goamz/s3"
	"launchpad.net/goyaml"
	"launchpad.net/juju/go/version"
)

const stateFile = "provider-state"

type bootstrapState struct {
	ZookeeperInstances []string `yaml:"zookeeper-instances"`
}

func (e *environ) saveState(state *bootstrapState) error {
	data, err := goyaml.Marshal(state)
	if err != nil {
		return err
	}
	return e.PutFile(stateFile, bytes.NewBuffer(data), int64(len(data)))
}

func (e *environ) loadState() (*bootstrapState, error) {
	r, err := e.GetFile(stateFile)
	if err != nil {
		return nil, fmt.Errorf("cannot read %q: %v", stateFile, err)
	}
	defer r.Close()
	data, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, fmt.Errorf("error reading %q: %v", stateFile, err)
	}
	var state bootstrapState
	err = goyaml.Unmarshal(data, &state)
	if err != nil {
		return nil, fmt.Errorf("error unmarshalling %q: %v", stateFile, err)
	}
	return &state, nil
}

func (e *environ) deleteState() error {
	// TODO delete the bucket contents and the bucket itself.
	err := e.RemoveFile(stateFile)
	if err != nil {
		return fmt.Errorf("cannot delete provider state: %v", err)
	}
	return nil
}

// makeBucket makes the environent's control bucket, the
// place where bootstrap information and deployed charms
// are stored. To avoid two round trips on every PUT operation,
// we do this only once for each environ.
func (e *environ) makeBucket() error {
	e.checkBucket.Do(func() {
		// try to make the bucket - PutBucket will succeed if the
		// bucket already exists.
		e.checkBucketError = e.bucket().PutBucket(s3.Private)
	})
	return e.checkBucketError
}

func (e *environ) PutFile(file string, r io.Reader, length int64) error {
	if err := e.makeBucket(); err != nil {
		return fmt.Errorf("cannot make S3 control bucket: %v", err)
	}
	err := e.bucket().PutReader(file, r, length, "binary/octet-stream", s3.Private)
	if err != nil {
		return fmt.Errorf("cannot write file %q to control bucket: %v", file, err)
	}
	return nil
}

func (e *environ) GetFile(file string) (r io.ReadCloser, err error) {
	for a := shortAttempt.start(); a.next(); {
		r, err = e.bucket().GetReader(file)
		if err, _ := err.(*s3.Error); err != nil && err.StatusCode == 404 {
			continue
		}
		return
	}
	return
}

func (e *environ) RemoveFile(file string) error {
	err := e.bucket().Del(file)
	// If we can't delete the object because the bucket doesn't
	// exist, then we don't care.
	if err, _ := err.(*s3.Error); err != nil && err.StatusCode == 404 {
		return nil
	}
	return err
}

func (e *environ) bucket() *s3.Bucket {
	return e.s3.Bucket(e.config.bucket)
}

func (*environ) UploadTools(r io.Reader, length int64, version version.Version) error {
	return fmt.Errorf("ec2 environ does not support executable upload")
}
<|endoftext|>