<commit_before>// Downloads torrents from the command-line.
package main

import (
	"expvar"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/anacrolix/dht"
	"github.com/anacrolix/envpprof"
	"github.com/anacrolix/tagflag"
	"github.com/dustin/go-humanize"
	"github.com/gosuri/uiprogress"
	"golang.org/x/time/rate"

	"github.com/anacrolix/torrent"
	"github.com/anacrolix/torrent/metainfo"
	"github.com/anacrolix/torrent/storage"
)

var progress = uiprogress.New()

func torrentBar(t *torrent.Torrent) {
	bar := progress.AddBar(1)
	bar.AppendCompleted()
	bar.AppendFunc(func(*uiprogress.Bar) (ret string) {
		select {
		case <-t.GotInfo():
		default:
			return "getting info"
		}
		if t.Seeding() {
			return "seeding"
		} else if t.BytesCompleted() == t.Info().TotalLength() {
			return "completed"
		} else {
			return fmt.Sprintf("downloading (%s/%s)", humanize.Bytes(uint64(t.BytesCompleted())), humanize.Bytes(uint64(t.Info().TotalLength())))
		}
	})
	bar.PrependFunc(func(*uiprogress.Bar) string {
		return t.Name()
	})
	go func() {
		<-t.GotInfo()
		tl := int(t.Info().TotalLength())
		if tl == 0 {
			bar.Set(1)
			return
		}
		bar.Total = tl
		for {
			bc := t.BytesCompleted()
			bar.Set(int(bc))
			time.Sleep(time.Second)
		}
	}()
}

func addTorrents(client *torrent.Client) {
	for _, arg := range flags.Torrent {
		t := func() *torrent.Torrent {
			if strings.HasPrefix(arg, "magnet:") {
				t, err := client.AddMagnet(arg)
				if err != nil {
					log.Fatalf("error adding magnet: %s", err)
				}
				return t
			} else if strings.HasPrefix(arg, "http://") || strings.HasPrefix(arg, "https://") {
				response, err := http.Get(arg)
				if err != nil {
					log.Fatalf("Error downloading torrent file: %s", err)
				}

				metaInfo, err := metainfo.Load(response.Body)
				defer response.Body.Close()
				if err != nil {
					fmt.Fprintf(os.Stderr, "error loading torrent file %q: %s\n", arg, err)
					os.Exit(1)
				}
				t, err := client.AddTorrent(metaInfo)
				if err != nil {
					log.Fatal(err)
				}
				return t
			} else if strings.HasPrefix(arg, "infohash:") {
				t, _ := client.AddTorrentInfoHash(metainfo.NewHashFromHex(strings.TrimPrefix(arg, "infohash:")))
				return t
			} else {
				metaInfo, err := metainfo.LoadFromFile(arg)
				if err != nil {
					fmt.Fprintf(os.Stderr, "error loading torrent file %q: %s\n", arg, err)
					os.Exit(1)
				}
				t, err := client.AddTorrent(metaInfo)
				if err != nil {
					log.Fatal(err)
				}
				return t
			}
		}()
		torrentBar(t)
		t.AddPeers(func() (ret []torrent.Peer) {
			for _, ta := range flags.TestPeer {
				ret = append(ret, torrent.Peer{
					IP:   ta.IP,
					Port: ta.Port,
				})
			}
			return
		}())
		go func() {
			<-t.GotInfo()
			t.DownloadAll()
		}()
	}
}

var flags = struct {
	Mmap         bool           `help:"memory-map torrent data"`
	TestPeer     []*net.TCPAddr `help:"addresses of some starting peers"`
	Seed         bool           `help:"seed after download is complete"`
	Addr         *net.TCPAddr   `help:"network listen addr"`
	UploadRate   tagflag.Bytes  `help:"max piece bytes to send per second"`
	DownloadRate tagflag.Bytes  `help:"max bytes per second down from peers"`
	Debug        bool
	tagflag.StartPos
	Torrent []string `arity:"+" help:"torrent file path or magnet uri"`
}{
	UploadRate:   -1,
	DownloadRate: -1,
}

func stdoutAndStderrAreSameFile() bool {
	fi1, _ := os.Stdout.Stat()
	fi2, _ := os.Stderr.Stat()
	return os.SameFile(fi1, fi2)
}

func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	tagflag.Parse(&flags)
	clientConfig := torrent.Config{
		DHTConfig: dht.ServerConfig{
			StartingNodes: dht.GlobalBootstrapAddrs,
		},
		Debug: flags.Debug,
		Seed:  flags.Seed,
	}
	if flags.Mmap {
		clientConfig.DefaultStorage = storage.NewMMap("")
	}
	if flags.Addr != nil {
		clientConfig.ListenAddr = flags.Addr.String()
	}
	if flags.UploadRate != -1 {
		clientConfig.UploadRateLimiter = rate.NewLimiter(rate.Limit(flags.UploadRate), 256<<10)
	}
	if flags.DownloadRate != -1 {
		clientConfig.DownloadRateLimiter = rate.NewLimiter(rate.Limit(flags.DownloadRate), 1<<20)
	}

	client, err := torrent.NewClient(&clientConfig)
	if err != nil {
		log.Fatalf("error creating client: %s", err)
	}
	defer client.Close()
	// Write status on the root path on the default HTTP muxer. This will be
	// bound to localhost somewhere if GOPPROF is set, thanks to the envpprof
	// import.
	http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		client.WriteStatus(w)
	})
	if stdoutAndStderrAreSameFile() {
		log.SetOutput(progress.Bypass())
	}
	progress.Start()
	addTorrents(client)
	if client.WaitAll() {
		log.Print("downloaded ALL the torrents")
	} else {
		log.Fatal("y u no complete torrents?!")
	}
	if flags.Seed {
		select {}
	}
	expvar.Do(func(kv expvar.KeyValue) {
		fmt.Printf("%s: %s\n", kv.Key, kv.Value)
	})
	envpprof.Stop()
}
<commit_msg>cmd/torrent: Add packed-blocklist flag<commit_after>// Downloads torrents from the command-line.
package main

import (
	"expvar"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/anacrolix/torrent/iplist"

	"github.com/anacrolix/dht"
	"github.com/anacrolix/envpprof"
	"github.com/anacrolix/tagflag"
	"github.com/dustin/go-humanize"
	"github.com/gosuri/uiprogress"
	"golang.org/x/time/rate"

	"github.com/anacrolix/torrent"
	"github.com/anacrolix/torrent/metainfo"
	"github.com/anacrolix/torrent/storage"
)

var progress = uiprogress.New()

func torrentBar(t *torrent.Torrent) {
	bar := progress.AddBar(1)
	bar.AppendCompleted()
	bar.AppendFunc(func(*uiprogress.Bar) (ret string) {
		select {
		case <-t.GotInfo():
		default:
			return "getting info"
		}
		if t.Seeding() {
			return "seeding"
		} else if t.BytesCompleted() == t.Info().TotalLength() {
			return "completed"
		} else {
			return fmt.Sprintf("downloading (%s/%s)", humanize.Bytes(uint64(t.BytesCompleted())), humanize.Bytes(uint64(t.Info().TotalLength())))
		}
	})
	bar.PrependFunc(func(*uiprogress.Bar) string {
		return t.Name()
	})
	go func() {
		<-t.GotInfo()
		tl := int(t.Info().TotalLength())
		if tl == 0 {
			bar.Set(1)
			return
		}
		bar.Total = tl
		for {
			bc := t.BytesCompleted()
			bar.Set(int(bc))
			time.Sleep(time.Second)
		}
	}()
}

func addTorrents(client *torrent.Client) {
	for _, arg := range flags.Torrent {
		t := func() *torrent.Torrent {
			if strings.HasPrefix(arg, "magnet:") {
				t, err := client.AddMagnet(arg)
				if err != nil {
					log.Fatalf("error adding magnet: %s", err)
				}
				return t
			} else if strings.HasPrefix(arg, "http://") || strings.HasPrefix(arg, "https://") {
				response, err := http.Get(arg)
				if err != nil {
					log.Fatalf("Error downloading torrent file: %s", err)
				}

				metaInfo, err := metainfo.Load(response.Body)
				defer response.Body.Close()
				if err != nil {
					fmt.Fprintf(os.Stderr, "error loading torrent file %q: %s\n", arg, err)
					os.Exit(1)
				}
				t, err := client.AddTorrent(metaInfo)
				if err != nil {
					log.Fatal(err)
				}
				return t
			} else if strings.HasPrefix(arg, "infohash:") {
				t, _ := client.AddTorrentInfoHash(metainfo.NewHashFromHex(strings.TrimPrefix(arg, "infohash:")))
				return t
			} else {
				metaInfo, err := metainfo.LoadFromFile(arg)
				if err != nil {
					fmt.Fprintf(os.Stderr, "error loading torrent file %q: %s\n", arg, err)
					os.Exit(1)
				}
				t, err := client.AddTorrent(metaInfo)
				if err != nil {
					log.Fatal(err)
				}
				return t
			}
		}()
		torrentBar(t)
		t.AddPeers(func() (ret []torrent.Peer) {
			for _, ta := range flags.TestPeer {
				ret = append(ret, torrent.Peer{
					IP:   ta.IP,
					Port: ta.Port,
				})
			}
			return
		}())
		go func() {
			<-t.GotInfo()
			t.DownloadAll()
		}()
	}
}

var flags = struct {
	Mmap            bool           `help:"memory-map torrent data"`
	TestPeer        []*net.TCPAddr `help:"addresses of some starting peers"`
	Seed            bool           `help:"seed after download is complete"`
	Addr            *net.TCPAddr   `help:"network listen addr"`
	UploadRate      tagflag.Bytes  `help:"max piece bytes to send per second"`
	DownloadRate    tagflag.Bytes  `help:"max bytes per second down from peers"`
	Debug           bool
	PackedBlocklist string
	tagflag.StartPos
	Torrent []string `arity:"+" help:"torrent file path or magnet uri"`
}{
	UploadRate:   -1,
	DownloadRate: -1,
}

func stdoutAndStderrAreSameFile() bool {
	fi1, _ := os.Stdout.Stat()
	fi2, _ := os.Stderr.Stat()
	return os.SameFile(fi1, fi2)
}

func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	tagflag.Parse(&flags)
	clientConfig := torrent.Config{
		DHTConfig: dht.ServerConfig{
			StartingNodes: dht.GlobalBootstrapAddrs,
		},
		Debug: flags.Debug,
		Seed:  flags.Seed,
	}
	if flags.PackedBlocklist != "" {
		blocklist, err := iplist.MMapPackedFile(flags.PackedBlocklist)
		if err != nil {
			log.Fatalf("error loading blocklist: %s", err)
		}
		defer blocklist.Close()
		clientConfig.IPBlocklist = blocklist
	}
	if flags.Mmap {
		clientConfig.DefaultStorage = storage.NewMMap("")
	}
	if flags.Addr != nil {
		clientConfig.ListenAddr = flags.Addr.String()
	}
	if flags.UploadRate != -1 {
		clientConfig.UploadRateLimiter = rate.NewLimiter(rate.Limit(flags.UploadRate), 256<<10)
	}
	if flags.DownloadRate != -1 {
		clientConfig.DownloadRateLimiter = rate.NewLimiter(rate.Limit(flags.DownloadRate), 1<<20)
	}

	client, err := torrent.NewClient(&clientConfig)
	if err != nil {
		log.Fatalf("error creating client: %s", err)
	}
	defer client.Close()
	// Write status on the root path on the default HTTP muxer. This will be
	// bound to localhost somewhere if GOPPROF is set, thanks to the envpprof
	// import.
	http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		client.WriteStatus(w)
	})
	if stdoutAndStderrAreSameFile() {
		log.SetOutput(progress.Bypass())
	}
	progress.Start()
	addTorrents(client)
	if client.WaitAll() {
		log.Print("downloaded ALL the torrents")
	} else {
		log.Fatal("y u no complete torrents?!")
	}
	if flags.Seed {
		select {}
	}
	expvar.Do(func(kv expvar.KeyValue) {
		fmt.Printf("%s: %s\n", kv.Key, kv.Value)
	})
	envpprof.Stop()
}
<|endoftext|>


<commit_before>package config

import (
	"errors"
	"sync"

	"github.com/nlopes/slack"
	"github.com/vivace-io/gonfig"
)

// ReadConfig accepts a file path and returns the parsed configuration, if
// found. If the file was not found, or could not be parsed, an error is
// returned and the configuration model is nil.
func ReadConfig(filepath string) (cfg *Configuration, err error) {
	cfg = new(Configuration)
	cfg.FilePath = filepath
	cfg.locker = new(sync.RWMutex)
	if err := cfg.Load(); err != nil {
		return nil, err
	}
	return
}

// Configuration holds the configuration for the application.
type Configuration struct {
	locker       *sync.RWMutex
	FilePath     string  `json:"-"`
	TemplateFile string  `json:"templateFile"`
	UserAgent    string  `json:"userAgent"`
	Teams        []*Team `json:"teams"`
}

// File returns the file name/path for gonfig interface
func (cfg *Configuration) File() string {
	return cfg.FilePath
}

// Save the configuration file on the file path set in Configuration.FilePath.
func (cfg *Configuration) Save() error {
	cfg.locker.Lock()
	defer cfg.locker.Unlock()
	if cfg.FilePath == "" {
		return errors.New("Configuration.FilePath was not set")
	}
	return gonfig.Save(cfg)
}

// Load the configuration from the FilePath. This overwrites any unsaved changes.
// Returns an error for an unset file path or any file system permission level
// errors.
func (cfg *Configuration) Load() error {
	cfg.locker.Lock()
	defer cfg.locker.Unlock()
	if cfg.FilePath == "" {
		return errors.New("Configuration.FilePath was not set")
	}
	return gonfig.Load(cfg)
}

// Team is the configuration object for a Slack team.
type Team struct {
	BotToken   string        `json:"botToken"`
	Channels   []Channel     `json:"channels"`
	Bot        *slack.Client `json:"-"` // Used only during run time.
	FailedAuth bool          `json:"-"` // Used only during run time. Denotes a failed token.
}

// Channel defines the configuration for a slack channel in a team, including its filters
type Channel struct {
	Name                string   `json:"channelName"`
	MinimumValue        int      `json:"minimumValue"`
	MaximumValue        int      `json:"maximumValue"`
	ExcludeLosses       bool     `json:"excludeLosses"`
	IncludeCharacters   []string `json:"includeCharacters"`
	IncludeCorporations []string `json:"includeCorporations"`
	IncludeAlliances    []string `json:"includeAlliances"`
	ExcludedShips       []string `json:"excludedShips"`
}
<commit_msg>fixes config methods for external package changes<commit_after>package config

import (
	"errors"
	"sync"

	"github.com/nlopes/slack"
	"github.com/vivace-io/gonfig"
)

// ReadConfig accepts a file path and returns the parsed configuration, if
// found. If the file was not found, or could not be parsed, an error is
// returned and the configuration model is nil.
func ReadConfig(filepath string) (cfg *Configuration, err error) {
	cfg = new(Configuration)
	cfg.FilePath = filepath
	cfg.locker = new(sync.RWMutex)
	if err := cfg.Load(); err != nil {
		return nil, err
	}
	return
}

// Configuration holds the configuration for the application.
type Configuration struct {
	locker       *sync.RWMutex
	FilePath     string  `json:"-"`
	TemplateFile string  `json:"templateFile"`
	UserAgent    string  `json:"userAgent"`
	Teams        []*Team `json:"teams"`
}

// File returns the file name/path for gonfig interface
func (cfg *Configuration) File() string {
	return cfg.FilePath
}

// Save the configuration file on the file path set in Configuration.FilePath.
func (cfg *Configuration) Save() error {
	cfg.locker.Lock()
	defer cfg.locker.Unlock()
	if cfg.FilePath == "" {
		return errors.New("Configuration.FilePath was not set")
	}
	return gonfig.Write(cfg, true)
}

// Load the configuration from the FilePath. This overwrites any unsaved changes.
// Returns an error for an unset file path or any file system permission level
// errors.
func (cfg *Configuration) Load() error {
	cfg.locker.Lock()
	defer cfg.locker.Unlock()
	if cfg.FilePath == "" {
		return errors.New("Configuration.FilePath was not set")
	}
	return gonfig.Read(cfg)
}

// Team is the configuration object for a Slack team.
type Team struct {
	BotToken   string        `json:"botToken"`
	Channels   []Channel     `json:"channels"`
	Bot        *slack.Client `json:"-"` // Used only during run time.
	FailedAuth bool          `json:"-"` // Used only during run time. Denotes a failed token.
}

// Channel defines the configuration for a slack channel in a team, including its filters
type Channel struct {
	Name                string   `json:"channelName"`
	MinimumValue        int      `json:"minimumValue"`
	MaximumValue        int      `json:"maximumValue"`
	ExcludeLosses       bool     `json:"excludeLosses"`
	IncludeCharacters   []string `json:"includeCharacters"`
	IncludeCorporations []string `json:"includeCorporations"`
	IncludeAlliances    []string `json:"includeAlliances"`
	ExcludedShips       []string `json:"excludedShips"`
}
<|endoftext|>


<commit_before>// Downloads torrents from the command-line.
package main

import (
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/anacrolix/dht"
	_ "github.com/anacrolix/envpprof"
	"github.com/anacrolix/tagflag"
	"github.com/dustin/go-humanize"
	"github.com/gosuri/uiprogress"
	"golang.org/x/time/rate"

	"github.com/anacrolix/torrent"
	"github.com/anacrolix/torrent/metainfo"
	"github.com/anacrolix/torrent/storage"
)

func resolvedPeerAddrs(ss []string) (ret []torrent.Peer, err error) {
	for _, s := range ss {
		var addr *net.TCPAddr
		addr, err = net.ResolveTCPAddr("tcp", s)
		if err != nil {
			return
		}
		ret = append(ret, torrent.Peer{
			IP:   addr.IP,
			Port: addr.Port,
		})
	}
	return
}

func torrentBar(t *torrent.Torrent) {
	bar := uiprogress.AddBar(1)
	bar.AppendCompleted()
	bar.AppendFunc(func(*uiprogress.Bar) (ret string) {
		select {
		case <-t.GotInfo():
		default:
			return "getting info"
		}
		if t.Seeding() {
			return "seeding"
		} else if t.BytesCompleted() == t.Info().TotalLength() {
			return "completed"
		} else {
			return fmt.Sprintf("downloading (%s/%s)", humanize.Bytes(uint64(t.BytesCompleted())), humanize.Bytes(uint64(t.Info().TotalLength())))
		}
	})
	bar.PrependFunc(func(*uiprogress.Bar) string {
		return t.Name()
	})
	go func() {
		<-t.GotInfo()
		tl := int(t.Info().TotalLength())
		if tl == 0 {
			bar.Set(1)
			return
		}
		bar.Total = tl
		for {
			bc := t.BytesCompleted()
			bar.Set(int(bc))
			time.Sleep(time.Second)
		}
	}()
}

func addTorrents(client *torrent.Client) {
	for _, arg := range flags.Torrent {
		t := func() *torrent.Torrent {
			if strings.HasPrefix(arg, "magnet:") {
				t, err := client.AddMagnet(arg)
				if err != nil {
					log.Fatalf("error adding magnet: %s", err)
				}
				return t
			} else if strings.HasPrefix(arg, "http://") || strings.HasPrefix(arg, "https://") {
				response, err := http.Get(arg)
				if err != nil {
					log.Fatalf("Error downloading torrent file: %s", err)
				}

				metaInfo, err := metainfo.Load(response.Body)
				defer response.Body.Close()
				if err != nil {
					fmt.Fprintf(os.Stderr, "error loading torrent file %q: %s\n", arg, err)
					os.Exit(1)
				}
				t, err := client.AddTorrent(metaInfo)
				if err != nil {
					log.Fatal(err)
				}
				return t
			} else if strings.HasPrefix(arg, "infohash:") {
				t, _ := client.AddTorrentInfoHash(metainfo.NewHashFromHex(strings.TrimPrefix(arg, "infohash:")))
				return t
			} else {
				metaInfo, err := metainfo.LoadFromFile(arg)
				if err != nil {
					fmt.Fprintf(os.Stderr, "error loading torrent file %q: %s\n", arg, err)
					os.Exit(1)
				}
				t, err := client.AddTorrent(metaInfo)
				if err != nil {
					log.Fatal(err)
				}
				return t
			}
		}()
		torrentBar(t)
		t.AddPeers(func() (ret []torrent.Peer) {
			for _, ta := range flags.TestPeer {
				ret = append(ret, torrent.Peer{
					IP:   ta.IP,
					Port: ta.Port,
				})
			}
			return
		}())
		go func() {
			<-t.GotInfo()
			t.DownloadAll()
		}()
	}
}

var flags = struct {
	Mmap         bool           `help:"memory-map torrent data"`
	TestPeer     []*net.TCPAddr `help:"addresses of some starting peers"`
	Seed         bool           `help:"seed after download is complete"`
	Addr         *net.TCPAddr   `help:"network listen addr"`
	UploadRate   tagflag.Bytes  `help:"max piece bytes to send per second"`
	DownloadRate tagflag.Bytes  `help:"max bytes per second down from peers"`
	tagflag.StartPos
	Torrent []string `arity:"+" help:"torrent file path or magnet uri"`
}{
	UploadRate:   -1,
	DownloadRate: -1,
}

func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	tagflag.Parse(&flags)
	clientConfig := torrent.Config{
		DHTConfig: dht.ServerConfig{
			StartingNodes: dht.GlobalBootstrapAddrs,
		},
	}
	if flags.Mmap {
		clientConfig.DefaultStorage = storage.NewMMap("")
	}
	if flags.Addr != nil {
		clientConfig.ListenAddr = flags.Addr.String()
	}
	if flags.Seed {
		clientConfig.Seed = true
	}
	if flags.UploadRate != -1 {
		clientConfig.UploadRateLimiter = rate.NewLimiter(rate.Limit(flags.UploadRate), 256<<10)
	}
	if flags.DownloadRate != -1 {
		clientConfig.DownloadRateLimiter = rate.NewLimiter(rate.Limit(flags.DownloadRate), 1<<20)
	}

	client, err := torrent.NewClient(&clientConfig)
	if err != nil {
		log.Fatalf("error creating client: %s", err)
	}
	defer client.Close()
	// Write status on the root path on the default HTTP muxer. This will be
	// bound to localhost somewhere if GOPPROF is set, thanks to the envpprof
	// import.
	http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		client.WriteStatus(w)
	})
	uiprogress.Start()
	addTorrents(client)
	if client.WaitAll() {
		log.Print("downloaded ALL the torrents")
	} else {
		log.Fatal("y u no complete torrents?!")
	}
	if flags.Seed {
		select {}
	}
}
<commit_msg>cmd/torrent: Enhance debugging<commit_after>// Downloads torrents from the command-line.
package main

import (
	"expvar"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/anacrolix/dht"
	"github.com/anacrolix/envpprof"
	"github.com/anacrolix/tagflag"
	"github.com/dustin/go-humanize"
	"github.com/gosuri/uiprogress"
	"golang.org/x/time/rate"

	"github.com/anacrolix/torrent"
	"github.com/anacrolix/torrent/metainfo"
	"github.com/anacrolix/torrent/storage"
)

func resolvedPeerAddrs(ss []string) (ret []torrent.Peer, err error) {
	for _, s := range ss {
		var addr *net.TCPAddr
		addr, err = net.ResolveTCPAddr("tcp", s)
		if err != nil {
			return
		}
		ret = append(ret, torrent.Peer{
			IP:   addr.IP,
			Port: addr.Port,
		})
	}
	return
}

func torrentBar(t *torrent.Torrent) {
	bar := uiprogress.AddBar(1)
	bar.AppendCompleted()
	bar.AppendFunc(func(*uiprogress.Bar) (ret string) {
		select {
		case <-t.GotInfo():
		default:
			return "getting info"
		}
		if t.Seeding() {
			return "seeding"
		} else if t.BytesCompleted() == t.Info().TotalLength() {
			return "completed"
		} else {
			return fmt.Sprintf("downloading (%s/%s)", humanize.Bytes(uint64(t.BytesCompleted())), humanize.Bytes(uint64(t.Info().TotalLength())))
		}
	})
	bar.PrependFunc(func(*uiprogress.Bar) string {
		return t.Name()
	})
	go func() {
		<-t.GotInfo()
		tl := int(t.Info().TotalLength())
		if tl == 0 {
			bar.Set(1)
			return
		}
		bar.Total = tl
		for {
			bc := t.BytesCompleted()
			bar.Set(int(bc))
			time.Sleep(time.Second)
		}
	}()
}

func addTorrents(client *torrent.Client) {
	for _, arg := range flags.Torrent {
		t := func() *torrent.Torrent {
			if strings.HasPrefix(arg, "magnet:") {
				t, err := client.AddMagnet(arg)
				if err != nil {
					log.Fatalf("error adding magnet: %s", err)
				}
				return t
			} else if strings.HasPrefix(arg, "http://") || strings.HasPrefix(arg, "https://") {
				response, err := http.Get(arg)
				if err != nil {
					log.Fatalf("Error downloading torrent file: %s", err)
				}

				metaInfo, err := metainfo.Load(response.Body)
				defer response.Body.Close()
				if err != nil {
					fmt.Fprintf(os.Stderr, "error loading torrent file %q: %s\n", arg, err)
					os.Exit(1)
				}
				t, err := client.AddTorrent(metaInfo)
				if err != nil {
					log.Fatal(err)
				}
				return t
			} else if strings.HasPrefix(arg, "infohash:") {
				t, _ := client.AddTorrentInfoHash(metainfo.NewHashFromHex(strings.TrimPrefix(arg, "infohash:")))
				return t
			} else {
				metaInfo, err := metainfo.LoadFromFile(arg)
				if err != nil {
					fmt.Fprintf(os.Stderr, "error loading torrent file %q: %s\n", arg, err)
					os.Exit(1)
				}
				t, err := client.AddTorrent(metaInfo)
				if err != nil {
					log.Fatal(err)
				}
				return t
			}
		}()
		torrentBar(t)
		t.AddPeers(func() (ret []torrent.Peer) {
			for _, ta := range flags.TestPeer {
				ret = append(ret, torrent.Peer{
					IP:   ta.IP,
					Port: ta.Port,
				})
			}
			return
		}())
		go func() {
			<-t.GotInfo()
			t.DownloadAll()
		}()
	}
}

var flags = struct {
	Mmap         bool           `help:"memory-map torrent data"`
	TestPeer     []*net.TCPAddr `help:"addresses of some starting peers"`
	Seed         bool           `help:"seed after download is complete"`
	Addr         *net.TCPAddr   `help:"network listen addr"`
	UploadRate   tagflag.Bytes  `help:"max piece bytes to send per second"`
	DownloadRate tagflag.Bytes  `help:"max bytes per second down from peers"`
	Debug        bool
	tagflag.StartPos
	Torrent []string `arity:"+" help:"torrent file path or magnet uri"`
}{
	UploadRate:   -1,
	DownloadRate: -1,
}

func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	tagflag.Parse(&flags)
	clientConfig := torrent.Config{
		DHTConfig: dht.ServerConfig{
			StartingNodes: dht.GlobalBootstrapAddrs,
		},
	}
	if flags.Mmap {
		clientConfig.DefaultStorage = storage.NewMMap("")
	}
	if flags.Addr != nil {
		clientConfig.ListenAddr = flags.Addr.String()
	}
	if flags.Seed {
		clientConfig.Seed = true
	}
	if flags.UploadRate != -1 {
		clientConfig.UploadRateLimiter = rate.NewLimiter(rate.Limit(flags.UploadRate), 256<<10)
	}
	if flags.DownloadRate != -1 {
		clientConfig.DownloadRateLimiter = rate.NewLimiter(rate.Limit(flags.DownloadRate), 1<<20)
	}

	client, err := torrent.NewClient(&clientConfig)
	if err != nil {
		log.Fatalf("error creating client: %s", err)
	}
	defer client.Close()
	// Write status on the root path on the default HTTP muxer. This will be
	// bound to localhost somewhere if GOPPROF is set, thanks to the envpprof
	// import.
	http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		client.WriteStatus(w)
	})
	uiprogress.Start()
	addTorrents(client)
	if client.WaitAll() {
		log.Print("downloaded ALL the torrents")
	} else {
		log.Fatal("y u no complete torrents?!")
	}
	if flags.Seed {
		select {}
	}
	expvar.Do(func(kv expvar.KeyValue) {
		fmt.Printf("%s: %s\n", kv.Key, kv.Value)
	})
	envpprof.Stop()
}
<|endoftext|>


<commit_before>// Downloads torrents from the command-line.
package main

import (
	"errors"
	"expvar"
	"fmt"
	"io"
	stdLog "log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/anacrolix/args"
	"github.com/anacrolix/envpprof"
	"github.com/anacrolix/log"
	"github.com/anacrolix/missinggo/v2"
	"github.com/anacrolix/tagflag"
	"github.com/anacrolix/torrent/bencode"
	"github.com/anacrolix/torrent/version"
	"github.com/davecgh/go-spew/spew"
	"github.com/dustin/go-humanize"
	"golang.org/x/time/rate"

	"github.com/anacrolix/torrent"
	"github.com/anacrolix/torrent/iplist"
	"github.com/anacrolix/torrent/metainfo"
	"github.com/anacrolix/torrent/storage"
)

func torrentBar(t *torrent.Torrent, pieceStates bool) {
	go func() {
		start := time.Now()
		if t.Info() == nil {
			fmt.Printf("%v: getting torrent info for %q\n", time.Since(start), t.Name())
			<-t.GotInfo()
		}
		lastStats := t.Stats()
		var lastLine string
		for range time.Tick(time.Second) {
			var completedPieces, partialPieces int
			psrs := t.PieceStateRuns()
			for _, r := range psrs {
				if r.Complete {
					completedPieces += r.Length
				}
				if r.Partial {
					partialPieces += r.Length
				}
			}
			stats := t.Stats()
			line := fmt.Sprintf(
				"%v: downloading %q: %s/%s, %d/%d pieces completed (%d partial): %v/s\n",
				time.Since(start),
				t.Name(),
				humanize.Bytes(uint64(t.BytesCompleted())),
				humanize.Bytes(uint64(t.Length())),
				completedPieces,
				t.NumPieces(),
				partialPieces,
				humanize.Bytes(uint64(stats.BytesReadUsefulData.Int64()-lastStats.BytesReadUsefulData.Int64())),
			)
			if line != lastLine {
				lastLine = line
				os.Stdout.WriteString(line)
			}
			if pieceStates {
				fmt.Println(psrs)
			}
			lastStats = stats
		}
	}()
}

type stringAddr string

func (stringAddr) Network() string   { return "" }
func (me stringAddr) String() string { return string(me) }

func resolveTestPeers(addrs []string) (ret []torrent.PeerInfo) {
	for _, ta := range addrs {
		ret = append(ret, torrent.PeerInfo{
			Addr: stringAddr(ta),
		})
	}
	return
}

func addTorrents(client *torrent.Client, flags downloadFlags) error {
	testPeers := resolveTestPeers(flags.TestPeer)
	for _, arg := range flags.Torrent {
		t, err := func() (*torrent.Torrent, error) {
			if strings.HasPrefix(arg, "magnet:") {
				t, err := client.AddMagnet(arg)
				if err != nil {
					return nil, fmt.Errorf("error adding magnet: %w", err)
				}
				return t, nil
			} else if strings.HasPrefix(arg, "http://") || strings.HasPrefix(arg, "https://") {
				response, err := http.Get(arg)
				if err != nil {
					return nil, fmt.Errorf("Error downloading torrent file: %s", err)
				}

				metaInfo, err := metainfo.Load(response.Body)
				defer response.Body.Close()
				if err != nil {
					return nil, fmt.Errorf("error loading torrent file %q: %s\n", arg, err)
				}
				t, err := client.AddTorrent(metaInfo)
				if err != nil {
					return nil, fmt.Errorf("adding torrent: %w", err)
				}
				return t, nil
			} else if strings.HasPrefix(arg, "infohash:") {
				t, _ := client.AddTorrentInfoHash(metainfo.NewHashFromHex(strings.TrimPrefix(arg, "infohash:")))
				return t, nil
			} else {
				metaInfo, err := metainfo.LoadFromFile(arg)
				if err != nil {
					return nil, fmt.Errorf("error loading torrent file %q: %s\n", arg, err)
				}
				t, err := client.AddTorrent(metaInfo)
				if err != nil {
					return nil, fmt.Errorf("adding torrent: %w", err)
				}
				return t, nil
			}
		}()
		if err != nil {
			return fmt.Errorf("adding torrent for %q: %w", arg, err)
		}
		if flags.Progress {
			torrentBar(t, flags.PieceStates)
		}
		t.AddPeers(testPeers)
		go func() {
			<-t.GotInfo()
			if len(flags.File) == 0 {
				t.DownloadAll()
			} else {
				for _, f := range t.Files() {
					for _, fileArg := range flags.File {
						if f.DisplayPath() == fileArg {
							f.Download()
						}
					}
				}
			}
		}()
	}
	return nil
}

type downloadFlags struct {
	Debug bool
	DownloadCmd
}

type DownloadCmd struct {
	Mmap               bool           `help:"memory-map torrent data"`
	TestPeer           []string       `help:"addresses of some starting peers"`
	Seed               bool           `help:"seed after download is complete"`
	Addr               string         `help:"network listen addr"`
	MaxUnverifiedBytes tagflag.Bytes  `help:"maximum number bytes to have pending verification"`
	UploadRate         *tagflag.Bytes `help:"max piece bytes to send per second"`
	DownloadRate       *tagflag.Bytes `help:"max bytes per second down from peers"`
	PackedBlocklist    string
	PublicIP           net.IP
	Progress           bool `default:"true"`
	PieceStates        bool
	Quiet              bool  `help:"discard client logging"`
	Stats              *bool `help:"print stats at termination"`
	Dht                bool  `default:"true"`

	TcpPeers        bool `default:"true"`
	UtpPeers        bool `default:"true"`
	Webtorrent      bool `default:"true"`
	DisableWebseeds bool

	Ipv4 bool `default:"true"`
	Ipv6 bool `default:"true"`
	Pex  bool `default:"true"`

	File    []string
	Torrent []string `arity:"+" help:"torrent file path or magnet uri" arg:"positional"`
}

func stdoutAndStderrAreSameFile() bool {
	fi1, _ := os.Stdout.Stat()
	fi2, _ := os.Stderr.Stat()
	return os.SameFile(fi1, fi2)
}

func statsEnabled(flags downloadFlags) bool {
	if flags.Stats == nil {
		return flags.Debug
	}
	return *flags.Stats
}

func exitSignalHandlers(notify *missinggo.SynchronizedEvent) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
	for {
		log.Printf("close signal received: %+v", <-c)
		notify.Set()
	}
}

func main() {
	defer envpprof.Stop()
	if err := mainErr(); err != nil {
		log.Printf("error in main: %v", err)
		os.Exit(1)
	}
}

func mainErr() error {
	stdLog.SetFlags(stdLog.Flags() | stdLog.Lshortfile)
	debug := args.Flag(args.FlagOpt{Long: "debug"})
	p := args.ParseMain(
		debug,
		args.Subcommand("metainfo", metainfoCmd),
		args.Subcommand("announce", func(p args.SubCmdCtx) error {
			var cmd AnnounceCmd
			err := p.NewParser().AddParams(
				args.Pos("tracker", &cmd.Tracker),
				args.Pos("infohash", &cmd.InfoHash)).Parse()
			if err != nil {
				return err
			}
			return announceErr(cmd)
		}),
		args.Subcommand("download", func(p args.SubCmdCtx) error {
			var dlf DownloadCmd
			err := p.NewParser().AddParams(
				append(args.FromStruct(&dlf), debug)...,
			).Parse()
			if err != nil {
				return err
			}
			return downloadErr(downloadFlags{
				Debug:       debug.Bool(),
				DownloadCmd: dlf,
			})
		}),
		args.Subcommand(
			"spew-bencoding",
			func(p args.SubCmdCtx) error {
				d := bencode.NewDecoder(os.Stdin)
				for i := 0; ; i++ {
					var v interface{}
					err := d.Decode(&v)
					if err == io.EOF {
						break
					}
					if err != nil {
						return fmt.Errorf("decoding message index %d: %w", i, err)
					}
					spew.Dump(v)
				}
				return nil
			},
			args.Help("reads bencoding from stdin into Go native types and spews the result"),
		),
		args.Subcommand("version", func(p args.SubCmdCtx) error {
			fmt.Printf("HTTP User-Agent: %q\n", version.DefaultHttpUserAgent)
			fmt.Printf("Torrent client version: %q\n", version.DefaultExtendedHandshakeClientVersion)
			fmt.Printf("Torrent version prefix: %q\n", version.DefaultBep20Prefix)
			return nil
		}),
	)
	if p.Err != nil {
		if errors.Is(p.Err, args.ErrHelped) {
			return nil
		}
		return p.Err
	}
	if !p.RanSubCmd {
		p.PrintChoices(os.Stderr)
		args.FatalUsage()
	}
	return nil
}

func downloadErr(flags downloadFlags) error {
	clientConfig := torrent.NewDefaultClientConfig()
	clientConfig.DisableWebseeds = flags.DisableWebseeds
	clientConfig.DisableTCP = !flags.TcpPeers
	clientConfig.DisableUTP = !flags.UtpPeers
	clientConfig.DisableIPv4 = !flags.Ipv4
	clientConfig.DisableIPv6 = !flags.Ipv6
	clientConfig.DisableAcceptRateLimiting = true
	clientConfig.NoDHT = !flags.Dht
	clientConfig.Debug = flags.Debug
	clientConfig.Seed = flags.Seed
	clientConfig.PublicIp4 = flags.PublicIP
	clientConfig.PublicIp6 = flags.PublicIP
	clientConfig.DisablePEX = !flags.Pex
	clientConfig.DisableWebtorrent = !flags.Webtorrent
	if flags.PackedBlocklist != "" {
		blocklist, err := iplist.MMapPackedFile(flags.PackedBlocklist)
		if err != nil {
			return fmt.Errorf("loading blocklist: %v", err)
		}
		defer blocklist.Close()
		clientConfig.IPBlocklist = blocklist
	}
	if flags.Mmap {
		clientConfig.DefaultStorage = storage.NewMMap("")
	}
	if flags.Addr != "" {
		clientConfig.SetListenAddr(flags.Addr)
	}
	if flags.UploadRate != nil {
		clientConfig.UploadRateLimiter = rate.NewLimiter(rate.Limit(*flags.UploadRate), 256<<10)
	}
	if flags.DownloadRate != nil {
		clientConfig.DownloadRateLimiter = rate.NewLimiter(rate.Limit(*flags.DownloadRate), 1<<20)
	}
	if flags.Quiet {
		clientConfig.Logger = log.Discard
	}
	clientConfig.MaxUnverifiedBytes = flags.MaxUnverifiedBytes.Int64()

	var stop missinggo.SynchronizedEvent
	defer func() {
		stop.Set()
	}()

	client, err := torrent.NewClient(clientConfig)
	if err != nil {
		return fmt.Errorf("creating client: %w", err)
	}
	var clientClose sync.Once //In certain situations, close was being called more than once.
	defer clientClose.Do(client.Close)
	go exitSignalHandlers(&stop)
	go func() {
		<-stop.C()
		clientClose.Do(client.Close)
	}()

	// Write status on the root path on the default HTTP muxer. This will be bound to localhost
	// somewhere if GOPPROF is set, thanks to the envpprof import.
	http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		client.WriteStatus(w)
	})
	err = addTorrents(client, flags)
	if err != nil {
		return fmt.Errorf("adding torrents: %w", err)
	}
	defer outputStats(client, flags)
	if client.WaitAll() {
		log.Print("downloaded ALL the torrents")
	} else {
		err = errors.New("y u no complete torrents?!")
	}
	if flags.Seed {
		if len(client.Torrents()) == 0 {
			log.Print("no torrents to seed")
		} else {
			outputStats(client, flags)
			<-stop.C()
		}
	}
	spew.Dump(expvar.Get("torrent").(*expvar.Map).Get("chunks received"))
	spew.Dump(client.ConnStats())
	clStats := client.ConnStats()
	sentOverhead := clStats.BytesWritten.Int64() - clStats.BytesWrittenData.Int64()
	log.Printf(
		"client read %v, %v was useful data. sent %v non-data bytes",
		humanize.Bytes(uint64(clStats.BytesRead.Int64())),
		100*float64(clStats.BytesReadUsefulData.Int64())/float64(clStats.BytesRead.Int64()),
		humanize.Bytes(uint64(sentOverhead)))
	return err
}

func outputStats(cl *torrent.Client, args downloadFlags) {
	if !statsEnabled(args) {
		return
	}
	expvar.Do(func(kv expvar.KeyValue) {
		fmt.Printf("%s: %s\n", kv.Key, kv.Value)
	})
	cl.WriteStatus(os.Stdout)
}
<commit_msg>cmd/torrent: Refactor and set progress interval to 3s<commit_after>// Downloads torrents from the command-line.
package main

import (
	"errors"
	"expvar"
	"fmt"
	"io"
	stdLog "log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/anacrolix/args"
	"github.com/anacrolix/envpprof"
	"github.com/anacrolix/log"
	"github.com/anacrolix/missinggo/v2"
	"github.com/anacrolix/tagflag"
	"github.com/anacrolix/torrent/bencode"
	"github.com/anacrolix/torrent/version"
	"github.com/davecgh/go-spew/spew"
	"github.com/dustin/go-humanize"
	"golang.org/x/time/rate"

	"github.com/anacrolix/torrent"
	"github.com/anacrolix/torrent/iplist"
	"github.com/anacrolix/torrent/metainfo"
	"github.com/anacrolix/torrent/storage"
)

func torrentBar(t *torrent.Torrent, pieceStates bool) {
	go func() {
		start := time.Now()
		if t.Info() == nil {
			fmt.Printf("%v: getting torrent info for %q\n", time.Since(start), t.Name())
			<-t.GotInfo()
		}
		lastStats := t.Stats()
		var lastLine string
		interval := 3 * time.Second
		for range time.Tick(interval) {
			var completedPieces, partialPieces int
			psrs := t.PieceStateRuns()
			for _, r := range psrs {
				if r.Complete {
					completedPieces += r.Length
				}
				if r.Partial {
					partialPieces += r.Length
				}
			}
			stats := t.Stats()
			byteRate := int64(time.Second)
			byteRate *= stats.BytesReadUsefulData.Int64() - lastStats.BytesReadUsefulData.Int64()
			byteRate /= int64(interval)
			line := fmt.Sprintf(
				"%v: downloading %q: %s/%s, %d/%d pieces completed (%d partial): %v/s\n",
				time.Since(start),
				t.Name(),
				humanize.Bytes(uint64(t.BytesCompleted())),
				humanize.Bytes(uint64(t.Length())),
				completedPieces,
				t.NumPieces(),
				partialPieces,
				humanize.Bytes(uint64(byteRate)),
			)
			if line != lastLine {
				lastLine = line
				os.Stdout.WriteString(line)
			}
			if pieceStates {
				fmt.Println(psrs)
			}
			lastStats = stats
		}
	}()
}

type stringAddr string

func (stringAddr) Network() string   { return "" }
func (me stringAddr) String() string { return string(me) }

func resolveTestPeers(addrs []string) (ret []torrent.PeerInfo) {
	for _, ta := range addrs {
		ret = append(ret, torrent.PeerInfo{
			Addr: stringAddr(ta),
		})
	}
	return
}

func addTorrents(client *torrent.Client, flags downloadFlags) error {
	testPeers := resolveTestPeers(flags.TestPeer)
	for _, arg := range flags.Torrent {
		t, err := func() (*torrent.Torrent, error) {
			if strings.HasPrefix(arg, "magnet:") {
				t, err := client.AddMagnet(arg)
				if err != nil {
					return nil, fmt.Errorf("error adding magnet: %w", err)
				}
				return t, nil
			} else if strings.HasPrefix(arg, "http://") || strings.HasPrefix(arg, "https://") {
				response, err := http.Get(arg)
				if err != nil {
					return nil, fmt.Errorf("Error downloading torrent file: %s", err)
				}

				metaInfo, err := metainfo.Load(response.Body)
				defer response.Body.Close()
				if err != nil {
					return nil, fmt.Errorf("error loading torrent file %q: %s\n", arg, err)
				}
				t, err := client.AddTorrent(metaInfo)
				if err != nil {
					return nil, fmt.Errorf("adding torrent: %w", err)
				}
				return t, nil
			} else if strings.HasPrefix(arg, "infohash:") {
				t, _ := client.AddTorrentInfoHash(metainfo.NewHashFromHex(strings.TrimPrefix(arg, "infohash:")))
				return t, nil
			} else {
				metaInfo, err := metainfo.LoadFromFile(arg)
				if err != nil {
					return nil, fmt.Errorf("error loading torrent file %q: %s\n", arg, err)
				}
				t, err := client.AddTorrent(metaInfo)
				if err != nil {
					return nil, fmt.Errorf("adding torrent: %w", err)
				}
				return t, nil
			}
		}()
		if err != nil {
			return fmt.Errorf("adding torrent for %q: %w", arg, err)
		}
		if flags.Progress {
			torrentBar(t, flags.PieceStates)
		}
		t.AddPeers(testPeers)
		go func() {
			<-t.GotInfo()
			if len(flags.File) == 0 {
				t.DownloadAll()
			} else {
				for _, f := range t.Files() {
					for _, fileArg := range flags.File {
						if f.DisplayPath() == fileArg {
							f.Download()
						}
					}
				}
			}
		}()
	}
	return nil
}

type downloadFlags struct {
	Debug bool
	DownloadCmd
}

type DownloadCmd struct {
	Mmap               bool           `help:"memory-map torrent data"`
	TestPeer           []string       `help:"addresses of some starting peers"`
	Seed               bool           `help:"seed after download is complete"`
	Addr               string         `help:"network listen addr"`
	MaxUnverifiedBytes tagflag.Bytes  `help:"maximum number bytes to have pending verification"`
	UploadRate         *tagflag.Bytes `help:"max piece bytes to send per second"`
	DownloadRate       *tagflag.Bytes `help:"max bytes per second down from peers"`
	PackedBlocklist    string
	PublicIP           net.IP
	Progress           bool `default:"true"`
	PieceStates        bool
	Quiet              bool  `help:"discard client logging"`
	Stats              *bool `help:"print stats at termination"`
	Dht                bool  `default:"true"`

	TcpPeers        bool `default:"true"`
	UtpPeers        bool `default:"true"`
	Webtorrent      bool `default:"true"`
	DisableWebseeds bool

	Ipv4 bool `default:"true"`
	Ipv6 bool `default:"true"`
	Pex  bool `default:"true"`

	File    []string
	Torrent []string `arity:"+" help:"torrent file path or magnet uri" arg:"positional"`
}

func stdoutAndStderrAreSameFile() bool {
	fi1, _ := os.Stdout.Stat()
	fi2, _ := os.Stderr.Stat()
	return os.SameFile(fi1, fi2)
}

func statsEnabled(flags downloadFlags) bool {
	if flags.Stats == nil {
		return flags.Debug
	}
	return *flags.Stats
}

func exitSignalHandlers(notify *missinggo.SynchronizedEvent) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
	for {
		log.Printf("close signal received: %+v", <-c)
		notify.Set()
	}
}

func main() {
	defer envpprof.Stop()
	if err := mainErr(); err != nil {
		log.Printf("error in main: %v", err)
		os.Exit(1)
	}
}

func mainErr() error {
	stdLog.SetFlags(stdLog.Flags() | stdLog.Lshortfile)
	debug := args.Flag(args.FlagOpt{Long: "debug"})
	p := args.ParseMain(
		debug,
		args.Subcommand("metainfo", metainfoCmd),
		args.Subcommand("announce", func(p args.SubCmdCtx) error {
			var cmd AnnounceCmd
			err := p.NewParser().AddParams(
				args.Pos("tracker", &cmd.Tracker),
				args.Pos("infohash", &cmd.InfoHash)).Parse()
			if err != nil {
				return err
			}
			return announceErr(cmd)
		}),
		args.Subcommand("download", func(p args.SubCmdCtx) error {
			var dlf DownloadCmd
			err := p.NewParser().AddParams(
				append(args.FromStruct(&dlf), debug)...,
			).Parse()
			if err != nil {
				return err
			}
			return downloadErr(downloadFlags{
				Debug:       debug.Bool(),
				DownloadCmd: dlf,
			})
		}),
		args.Subcommand(
			"spew-bencoding",
			func(p args.SubCmdCtx) error {
				d := bencode.NewDecoder(os.Stdin)
				for i := 0; ; i++ {
					var v interface{}
					err := d.Decode(&v)
					if err == io.EOF {
						break
					}
					if err != nil {
						return fmt.Errorf("decoding message index %d: %w", i, err)
					}
					spew.Dump(v)
				}
				return nil
			},
			args.Help("reads bencoding from stdin into Go native types and spews the result"),
		),
		args.Subcommand("version", func(p args.SubCmdCtx) error {
			fmt.Printf("HTTP User-Agent: %q\n", version.DefaultHttpUserAgent)
			fmt.Printf("Torrent client version: %q\n", version.DefaultExtendedHandshakeClientVersion)
			fmt.Printf("Torrent version prefix: %q\n", version.DefaultBep20Prefix)
			return nil
		}),
	)
	if p.Err != nil {
		if errors.Is(p.Err, args.ErrHelped) {
			return nil
		}
		return p.Err
	}
	if !p.RanSubCmd {
		p.PrintChoices(os.Stderr)
		args.FatalUsage()
	}
	return nil
}

func downloadErr(flags downloadFlags) error {
	clientConfig := torrent.NewDefaultClientConfig()
	clientConfig.DisableWebseeds = flags.DisableWebseeds
	clientConfig.DisableTCP = !flags.TcpPeers
	clientConfig.DisableUTP = !flags.UtpPeers
	clientConfig.DisableIPv4 = !flags.Ipv4
	clientConfig.DisableIPv6 = !flags.Ipv6
	clientConfig.DisableAcceptRateLimiting = true
	clientConfig.NoDHT = !flags.Dht
	clientConfig.Debug = flags.Debug
	clientConfig.Seed = flags.Seed
	clientConfig.PublicIp4 = flags.PublicIP
	clientConfig.PublicIp6 = flags.PublicIP
	clientConfig.DisablePEX = !flags.Pex
	clientConfig.DisableWebtorrent = !flags.Webtorrent
	if flags.PackedBlocklist != "" {
		blocklist, err := iplist.MMapPackedFile(flags.PackedBlocklist)
		if err != nil {
			return fmt.Errorf("loading blocklist: %v", err)
		}
		defer blocklist.Close()
		clientConfig.IPBlocklist = blocklist
	}
	if flags.Mmap {
		clientConfig.DefaultStorage = storage.NewMMap("")
	}
	if flags.Addr != "" {
		clientConfig.SetListenAddr(flags.Addr)
	}
	if flags.UploadRate != nil {
		clientConfig.UploadRateLimiter = rate.NewLimiter(rate.Limit(*flags.UploadRate), 256<<10)
	}
	if flags.DownloadRate != nil {
		clientConfig.DownloadRateLimiter = rate.NewLimiter(rate.Limit(*flags.DownloadRate), 1<<20)
	}
	if flags.Quiet {
		clientConfig.Logger = log.Discard
	}
	clientConfig.MaxUnverifiedBytes = flags.MaxUnverifiedBytes.Int64()

	var stop missinggo.SynchronizedEvent
	defer func() {
		stop.Set()
	}()

	client, err := torrent.NewClient(clientConfig)
	if err != nil {
		return fmt.Errorf("creating client: %w", err)
	}
	var clientClose sync.Once //In certain situations, close was being called more than once.
	defer clientClose.Do(client.Close)
	go exitSignalHandlers(&stop)
	go func() {
		<-stop.C()
		clientClose.Do(client.Close)
	}()

	// Write status on the root path on the default HTTP muxer. This will be bound to localhost
	// somewhere if GOPPROF is set, thanks to the envpprof import.
	http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		client.WriteStatus(w)
	})
	err = addTorrents(client, flags)
	if err != nil {
		return fmt.Errorf("adding torrents: %w", err)
	}
	defer outputStats(client, flags)
	if client.WaitAll() {
		log.Print("downloaded ALL the torrents")
	} else {
		err = errors.New("y u no complete torrents?!")
	}
	if flags.Seed {
		if len(client.Torrents()) == 0 {
			log.Print("no torrents to seed")
		} else {
			outputStats(client, flags)
			<-stop.C()
		}
	}
	spew.Dump(expvar.Get("torrent").(*expvar.Map).Get("chunks received"))
	spew.Dump(client.ConnStats())
	clStats := client.ConnStats()
	sentOverhead := clStats.BytesWritten.Int64() - clStats.BytesWrittenData.Int64()
	log.Printf(
		"client read %v, %v was useful data. sent %v non-data bytes",
		humanize.Bytes(uint64(clStats.BytesRead.Int64())),
		100*float64(clStats.BytesReadUsefulData.Int64())/float64(clStats.BytesRead.Int64()),
		humanize.Bytes(uint64(sentOverhead)))
	return err
}

func outputStats(cl *torrent.Client, args downloadFlags) {
	if !statsEnabled(args) {
		return
	}
	expvar.Do(func(kv expvar.KeyValue) {
		fmt.Printf("%s: %s\n", kv.Key, kv.Value)
	})
	cl.WriteStatus(os.Stdout)
}
<|endoftext|>


<commit_before>/*
   Copyright 2015 Daniel Gruber, Univa, My blog: http://www.gridengine.eu

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package main

// Run uc as proxy itself. Allows to stack clusters of cluster recursively.

import (
	"errors"
	"fmt"
	"github.com/dgruber/ubercluster"
	"log"
	"strings"
	"sync"
)

type inception struct {
	inceptionAddress string // address of uc itself
	config           Config // uc configuration object
}

// Implements the ProxyImplementer interface

// collects jobinfos from all clusters in parallel
type jiProtected struct {
	sync.Mutex
	sync.WaitGroup
	jobinfos []ubercluster.JobInfo
}

// requestJobInfos requests job infos of jobs in the
// given state from a cluster given by the address
func requestJobInfos(ji *jiProtected, state string, address string) {
	log.Println("Requesting from: ", address)
	jis := getJobs(address, state, "")
	log.Println("Got following jobinfos: ", jis)
	if jis != nil {
		ji.Lock()
		ji.jobinfos = append(ji.jobinfos, jis...)
		ji.Unlock()
	}
	ji.Done()
}

func (i *inception) GetJobInfosByFilter(filtered bool, filter ubercluster.JobInfo) []ubercluster.JobInfo {
	var jip jiProtected
	jip.jobinfos = make([]ubercluster.JobInfo, 0, 0)
	jip.Add(len(i.config.Cluster))
	// request clusters in parallel and wait for all of them
	for _, c := range i.config.Cluster {
		if addr := fmt.Sprintf("%s/", c.Address); addr == i.inceptionAddress {
			log.Println("Skipping own address ", c.Address)
			jip.Done()
			continue
		}
		go requestJobInfos(&jip, "all", fmt.Sprintf("%s/v1", c.Address))
	}
	// wait until we got all job infos from all cluster
	jip.Wait()

	return jip.jobinfos
}

func getJobFromCluster(i *inception, clustername string, jobid string) (*ubercluster.JobInfo, error) {
	// check if cluster name is known
	address := ""
	version := "v1"
	for _, c := range i.config.Cluster {
		if c.Name == clustername {
			address = c.Address
			version = c.ProtocolVersion
			break
		}
	}
	if address != "" {
		request := fmt.Sprintf("%s%s", address, version)
		log.Println("GetJobFromCluster request", request)
		if job, err := getJob(request, jobid); err == nil {
			return &job, nil
		} else {
			log.Println("error during requesting job: ", err)
			return nil, err
		}
	}
	return nil, errors.New("Couldn't find clustername in config: " + clustername)
}

func (i *inception) GetJobInfo(jobid string) *ubercluster.JobInfo {
	// search job id in all connected clusters
	// if it has a postfix - only in that cluster
	// 1301@mybiggridenginecluster search 1301 in the given cluster
	if strings.Contains(jobid, "@") {
		// get cluster name
		jobAtCluster := strings.Split(jobid, "@")
		if len(jobAtCluster) == 2 {
			job, _ := getJobFromCluster(i, jobAtCluster[1], jobAtCluster[0])
			return job
		} else {
			log.Println("Wrong job identifier (expected jobid@cluster or jobid) but is ", jobid)
		}
	} else {
		// request default cluster for the given job identifier
		job, _ := getJobFromCluster(i, "default", jobid)
		return job
	}
	return nil
}

func (i *inception) GetAllMachines(machines []string) ([]ubercluster.Machine, error) {
	allmachines := make([]ubercluster.Machine, 0, 0)
	for _, c := range i.config.Cluster {
		log.Println("Requesting from: ", c.Address)
		// we don't request our own address...
		if addr := fmt.Sprintf("%s/", c.Address); addr == i.inceptionAddress {
			continue
		}
		if ms, err := getMachines(getClusterAddress(c.Name), "all"); err == nil {
			allmachines = append(allmachines, ms...)
			log.Println("Appending: ", allmachines)
		} else {
			log.Println("Error while requesting machines from ", c.Name, err)
		}
		// TODO filter according request
		// TODO remove duplicates
	}
	return allmachines, nil
}

// GetAllQueues returns all queue names from all clusters which are
// connected to the uc tool.
func (i *inception) GetAllQueues(queues []string) ([]ubercluster.Queue, error) {
	allqueues := make([]ubercluster.Queue, 0, 0)
	// TODO go functions of course
	for _, c := range i.config.Cluster {
		log.Println("Requesting from: ", c.Address)
		// we don't request our own address...
		if addr := fmt.Sprintf("%s/", c.Address); addr == i.inceptionAddress {
			continue
		}
		if qs, err := getQueues(getClusterAddress(c.Name), "all"); err == nil {
			allqueues = append(allqueues, qs...)
			log.Println("Appending: ", allqueues)
		} else {
			log.Println("Error while requesting queues from ", c.Name, err)
		}
		// TODO filter according request
		// TODO remove duplicates
	}
	return allqueues, nil
}

func (i *inception) GetAllCategories() ([]string, error) {
	cat := make([]string, 0, 0)
	for _, c := range i.config.Cluster {
		log.Println("Requesting from: ", c.Address)
		if addr := fmt.Sprintf("%s/", c.Address); addr == i.inceptionAddress {
			log.Println("Skipping own address")
			continue
		}
		cat = append(cat, getJobCategories(getClusterAddress(c.Name), "ubercluster", "all")...)
	}
	return cat, nil
}

func (i *inception) DRMSVersion() string {
	return "0.1"
}

func (i *inception) DRMSName() string {
	return "ubercluster"
}

func (i *inception) RunJob(template ubercluster.JobTemplate) (string, error) {
	return "", nil
}

func (i *inception) JobOperation(jobsessionname, operation, jobid string) (string, error) {
	return "", nil
}

// start uc as proxy
func inceptionMode(address string) {
	var incept inception
	incept.config = config // configuration contains all connected clusters
	fmt.Println("Starting uc in inception mode as proxy listing at address: ", address)
	ubercluster.ProxyListenAndServe(address, "", "", "", &incept)
}
<commit_msg>DG: inception mode support now shared secret key<commit_after>/*
   Copyright 2015 Daniel Gruber, Univa, My blog: http://www.gridengine.eu

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package main

// Run uc as proxy itself. Allows to stack clusters of cluster recursively.

import (
	"errors"
	"fmt"
	"github.com/dgruber/ubercluster"
	"log"
	"strings"
	"sync"
)

type inception struct {
	inceptionAddress string // address of uc itself
	config           Config // uc configuration object
}

// Implements the ProxyImplementer interface

// collects jobinfos from all clusters in parallel
type jiProtected struct {
	sync.Mutex
	sync.WaitGroup
	jobinfos []ubercluster.JobInfo
}

// requestJobInfos requests job infos of jobs in the
// given state from a cluster given by the address
func requestJobInfos(ji *jiProtected, state string, address string) {
	log.Println("Requesting from: ", address)
	jis := getJobs(address, state, "")
	log.Println("Got following jobinfos: ", jis)
	if jis != nil {
		ji.Lock()
		ji.jobinfos = append(ji.jobinfos, jis...)
		ji.Unlock()
	}
	ji.Done()
}

func (i *inception) GetJobInfosByFilter(filtered bool, filter ubercluster.JobInfo) []ubercluster.JobInfo {
	var jip jiProtected
	jip.jobinfos = make([]ubercluster.JobInfo, 0, 0)
	jip.Add(len(i.config.Cluster))
	// request clusters in parallel and wait for all of them
	for _, c := range i.config.Cluster {
		if addr := fmt.Sprintf("%s/", c.Address); addr == i.inceptionAddress {
			log.Println("Skipping own address ", c.Address)
			jip.Done()
			continue
		}
		go requestJobInfos(&jip, "all", fmt.Sprintf("%s/v1", c.Address))
	}
	// wait until we got all job infos from all cluster
	jip.Wait()

	return jip.jobinfos
}

func getJobFromCluster(i *inception, clustername string, jobid string) (*ubercluster.JobInfo, error) {
	// check if cluster name is known
	address := ""
	version := "v1"
	for _, c := range i.config.Cluster {
		if c.Name == clustername {
			address = c.Address
			version = c.ProtocolVersion
			break
		}
	}
	if address != "" {
		request := fmt.Sprintf("%s%s", address, version)
		log.Println("GetJobFromCluster request", request)
		if job, err := getJob(request, jobid); err == nil {
			return &job, nil
		} else {
			log.Println("error during requesting job: ", err)
			return nil, err
		}
	}
	return nil, errors.New("Couldn't find clustername in config: " + clustername)
}

func (i *inception) GetJobInfo(jobid string) *ubercluster.JobInfo {
	// search job id in all connected clusters
	// if it has a postfix - only in that cluster
	// 1301@mybiggridenginecluster search 1301 in the given cluster
	if strings.Contains(jobid, "@") {
		// get cluster name
		jobAtCluster := strings.Split(jobid, "@")
		if len(jobAtCluster) == 2 {
			job, _ := getJobFromCluster(i, jobAtCluster[1], jobAtCluster[0])
			return job
		} else {
			log.Println("Wrong job identifier (expected jobid@cluster or jobid) but is ", jobid)
		}
	} else {
		// request default cluster for the given job identifier
		job, _ := getJobFromCluster(i, "default", jobid)
		return job
	}
	return nil
}

func (i *inception) GetAllMachines(machines []string) ([]ubercluster.Machine, error) {
	allmachines := make([]ubercluster.Machine, 0, 0)
	for _, c := range i.config.Cluster {
		log.Println("Requesting from: ", c.Address)
		// we don't request our own address...
		if addr := fmt.Sprintf("%s/", c.Address); addr == i.inceptionAddress {
			continue
		}
		if ms, err := getMachines(getClusterAddress(c.Name), "all"); err == nil {
			allmachines = append(allmachines, ms...)
			log.Println("Appending: ", allmachines)
		} else {
			log.Println("Error while requesting machines from ", c.Name, err)
		}
		// TODO filter according request
		// TODO remove duplicates
	}
	return allmachines, nil
}

// GetAllQueues returns all queue names from all clusters which are
// connected to the uc tool.
func (i *inception) GetAllQueues(queues []string) ([]ubercluster.Queue, error) {
	allqueues := make([]ubercluster.Queue, 0, 0)
	// TODO go functions of course
	for _, c := range i.config.Cluster {
		log.Println("Requesting from: ", c.Address)
		// we don't request our own address...
		if addr := fmt.Sprintf("%s/", c.Address); addr == i.inceptionAddress {
			continue
		}
		if qs, err := getQueues(getClusterAddress(c.Name), "all"); err == nil {
			allqueues = append(allqueues, qs...)
			log.Println("Appending: ", allqueues)
		} else {
			log.Println("Error while requesting queues from ", c.Name, err)
		}
		// TODO filter according request
		// TODO remove duplicates
	}
	return allqueues, nil
}

func (i *inception) GetAllCategories() ([]string, error) {
	cat := make([]string, 0, 0)
	for _, c := range i.config.Cluster {
		log.Println("Requesting from: ", c.Address)
		if addr := fmt.Sprintf("%s/", c.Address); addr == i.inceptionAddress {
			log.Println("Skipping own address")
			continue
		}
		cat = append(cat, getJobCategories(getClusterAddress(c.Name), "ubercluster", "all")...)
	}
	return cat, nil
}

func (i *inception) DRMSVersion() string {
	return "0.1"
}

func (i *inception) DRMSName() string {
	return "ubercluster"
}

func (i *inception) RunJob(template ubercluster.JobTemplate) (string, error) {
	return "", nil
}

func (i *inception) JobOperation(jobsessionname, operation, jobid string) (string, error) {
	return "", nil
}

// start uc as proxy
func inceptionMode(address string) {
	var incept inception
	incept.config = config // configuration contains all connected clusters
	fmt.Println("Starting uc in inception mode as proxy listing at address: ", address)
	ubercluster.ProxyListenAndServe(address, "", "", *otp, &incept)
}
<|endoftext|>


<commit_before>// Copyright 2012-2018 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// pox builds a portable executable as a squashfs image.
// It is intended to create files compatible with tinycore
// tcz files. One of more of the files can be programs
// but that is not required.
// This could have been a simple program but mksquashfs does not
// preserve path information.
// Yeah.
//
// Synopsis:
//     pox [-[-debug]|d] [-[-file]|f tcz-file] -[-create]|c FILE [...FILE]
//     pox [-[-debug]|d] [-[-file]|f tcz-file] -[-run|r] PROGRAM -- [...ARGS]
//
// Description:
//     pox makes portable executables in squashfs format compatible with
//     tcz format. We don't build in the execution code, rather, we set it
//     up so we can use the command itself. You can either create the TCZ image
//     or run a command within an image that was previously created.
//
// Options:
//     debug|d: verbose
//     file|f file: file name (default /tmp/pox.tcz)
//     run|r: Runs the first non-flag argument to pox.  Remaining arguments will
//            be passed to the program.  Use '--' before any flag-like arguments
//            to prevent pox from interpretting the flags.
//     create|c: create the TCZ file.
//     zip|z: Use zip and unzip instead of a loopback mounted squashfs.  Be sure
//            to use -z for both creation and running, or not at all.
//     Exactly one of -c and -r must be used on the same command.
//
// Example:
//	$ pox -c /bin/bash /bin/cat /bin/ls /etc/hosts
//	Will build a squashfs, which will be /tmp/pox.tcz
//
//	$ sudo pox -r /bin/bash
//	Will drop you into the /tmp/pox.tcz running bash
//	You can use ls and cat on /etc/hosts.
//
//	Simpler example, with arguments:
//	$ sudo pox -r /bin/ls -- -la
//	will run `ls -la` and exit.
//
//	$ sudo pox -r -- /bin/ls -la
//	Syntactically easier: the program name can come after '--'
//
// Notes:
// - When running a pox, you likely need sudo to chroot
//
// - Binaries run out of a chroot often need files you are unaware of.  For
// instance, if bash can't find terminfo files, it won't know to handle
// backspaces properly.  (They occur, but are not shown).  To fix this, pass pox
// all of the files you need.  For bash: `find /lib/terminfo -type f`.
//
// Other programs rely on help functions, such as '/bin/man'.  If your program
// has built-in help commands that trigger man pages, e.g. "git help foo",
// you'll want to include /bin/man too.  But you'll also need everything that
// man uses, such as /etc/manpath.config.  My advice: skip it.
//
// - When adding all files in a directory, the easiest thing to do is:
// `find $DIR -type f`  (Note the ticks: this is a bash command execution).
//
// - When creating a pox with an executable with shared libraries that are not
// installed on your system, such as for a project installed in your home
// directory, run pox from the installation prefix directory, such that the
// lib/ and bin/ are in pox's working directory.  Pox will strip its working
// directory from the paths of the files it builds.  Having bin/ in the root of
// the pox file helps with PATH lookups, and not having the full path from your
// machine in the pox file makes it easier to extract a pox file to /usr/local/.
//
// - Consider adding a --extract | -x option to install to the host.  One issue
// would be how to handle collisions, e.g. libc.  Your app may not like the libc
// on the system you run on.
package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"

	flag "github.com/spf13/pflag"
	"github.com/u-root/u-root/pkg/ldd"
	"github.com/u-root/u-root/pkg/loop"
	"github.com/u-root/u-root/pkg/mount"
)

const usage = "pox [-[-debug]|d] -[-run|r] | -[-create]|c  [-[-file]|f tcz-file] file [...file]"

var (
	debug  = flag.BoolP("debug", "d", false, "enable debug prints")
	run    = flag.BoolP("run", "r", false, "Run the first file argument")
	create = flag.BoolP("create", "c", false, "create it")
	zip    = flag.BoolP("zip", "z", false, "use zip instead of squashfs")
	file   = flag.StringP("output", "f", "/tmp/pox.tcz", "Output file")
	v      = func(string, ...interface{}) {}
)

// When chrooting, programs often want to access various system directories:
var chrootMounts = []struct {
	source string
	target string
	fstype string
	flags  uintptr
	data   string
	perm   os.FileMode // for target in the chroot
}{
	// mount --bind /sys /chroot/sys
	{"/sys", "/sys", "", mount.MS_BIND, "", 0555},
	// mount -t proc /proc /chroot/proc
	{"/proc", "/proc", "proc", 0, "", 0555},
	// mount --bind /dev /chroot/dev
	{"/dev", "/dev", "", mount.MS_BIND, "", 0755},
}

func poxCreate(names []string) error {
	if len(names) == 0 {
		return fmt.Errorf(usage)
	}
	l, err := ldd.Ldd(names)
	if err != nil {
		var stderr []byte
		if eerr, ok := err.(*exec.ExitError); ok {
			stderr = eerr.Stderr
		}
		return fmt.Errorf("Running ldd on %v: %v %s", names,
			err, stderr)
	}

	for _, dep := range l {
		v("%s", dep.FullName)
		names = append(names, dep.FullName)
	}
	// Now we need to make a template file hierarchy and put
	// the stuff we want in there.
	dir, err := ioutil.TempDir("", "pox")
	if err != nil {
		return err
	}
	if !*debug {
		defer os.RemoveAll(dir)
	}
	pwd, err := os.Getwd()
	if err != nil {
		return err
	}
	// We don't use defer() here to close files as
	// that can cause open failures with a large enough number.
	for _, f := range names {
		v("Process %v", f)
		fi, err := os.Stat(f)
		if err != nil {
			return err
		}
		in, err := os.Open(f)
		if err != nil {
			return err
		}
		f = strings.TrimPrefix(f, pwd)
		dfile := filepath.Join(dir, f)
		d := filepath.Dir(dfile)
		if err := os.MkdirAll(d, 0755); err != nil {
			in.Close()
			return err
		}
		out, err := os.OpenFile(dfile, os.O_WRONLY|os.O_CREATE,
			fi.Mode().Perm())
		if err != nil {
			in.Close()
			return err
		}
		_, err = io.Copy(out, in)
		in.Close()
		out.Close()
		if err != nil {
			return err
		}

	}
	for _, m := range chrootMounts {
		if err := os.MkdirAll(filepath.Join(dir, m.target), m.perm); err != nil {
			return err
		}
	}
	err = os.Remove(*file)
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	var c *exec.Cmd
	if *zip {
		fileAbs, err := filepath.Abs(*file)
		if err != nil {
			return err
		}
		c = exec.Command("zip", "-r", fileAbs, ".")
		c.Dir = dir
	} else {
		c = exec.Command("mksquashfs", dir, *file, "-noappend")
	}
	o, err := c.CombinedOutput()
	v("%v", string(o))
	if err != nil {
		return fmt.Errorf("%v: %v: %v", c.Args, string(o), err)
	}
	v("Done, your pox is in %v", *file)

	return nil
}

func poxRun(args []string) error {
	if len(args) == 0 {
		return fmt.Errorf(usage)
	}
	dir, err := ioutil.TempDir("", "pox")
	if err != nil {
		return err
	}
	if !*debug {
		defer os.RemoveAll(dir)
	}

	if *zip {
		c := exec.Command("unzip", *file, "-d", dir)
		o, err := c.CombinedOutput()
		v("%v", string(o))
		if err != nil {
			return fmt.Errorf("%v: %v: %v", c.Args, string(o), err)
		}
	} else {
		lo, err := loop.New(*file, "squashfs", "")
		if err != nil {
			return err
		}
		defer lo.Free() //nolint:errcheck

		mountPoint, err := lo.Mount(dir, 0)
		if err != nil {
			return err
		}
		defer mountPoint.Unmount(0) //nolint:errcheck
	}
	for _, m := range chrootMounts {
		mp, err := mount.Mount(m.source, filepath.Join(dir, m.target),
			m.fstype, m.data, m.flags)
		if err != nil {
			return err
		}
		defer mp.Unmount(0) //nolint:errcheck
	}

	// If you pass Command a path with no slashes, it'll use PATH from the
	// parent to resolve the path to exec.  Once we chroot, whatever path we
	// picked is undoubtably wrong.  Let's help them out: if they give us a
	// program with no /, let's look in /bin/.  If they want the root of the
	// chroot, they can use "./"
	if filepath.Base(args[0]) == args[0] {
		args[0] = filepath.Join(string(os.PathSeparator), "bin", args[0])
	}
	c := exec.Command(args[0], args[1:]...)
	c.Stdin, c.Stdout, c.Stderr = os.Stdin, os.Stdout, os.Stderr
	c.SysProcAttr = &syscall.SysProcAttr{
		Chroot: dir,
	}
	c.Env = append(os.Environ(), "PWD=.")

	if err = c.Run(); err != nil {
		v("pox command exited with: %v", err)
	}

	return nil
}

func pox() error {
	flag.Parse()
	if *debug {
		v = log.Printf
	}
	if (*create && *run) || (!*create && !*run) {
		return fmt.Errorf(usage)
	}
	if *create {
		return poxCreate(flag.Args())
	}
	if *run {
		return poxRun(flag.Args())
	}
	return fmt.Errorf(usage)
}

func main() {
	if err := pox(); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>pox: just use the list of names retured by ldd.Ldd()<commit_after>// Copyright 2012-2018 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// pox builds a portable executable as a squashfs image.
// It is intended to create files compatible with tinycore
// tcz files. One of more of the files can be programs
// but that is not required.
// This could have been a simple program but mksquashfs does not
// preserve path information.
// Yeah.
//
// Synopsis:
//     pox [-[-debug]|d] [-[-file]|f tcz-file] -[-create]|c FILE [...FILE]
//     pox [-[-debug]|d] [-[-file]|f tcz-file] -[-run|r] PROGRAM -- [...ARGS]
//
// Description:
//     pox makes portable executables in squashfs format compatible with
//     tcz format. We don't build in the execution code, rather, we set it
//     up so we can use the command itself. You can either create the TCZ image
//     or run a command within an image that was previously created.
//
// Options:
//     debug|d: verbose
//     file|f file: file name (default /tmp/pox.tcz)
//     run|r: Runs the first non-flag argument to pox.  Remaining arguments will
//            be passed to the program.  Use '--' before any flag-like arguments
//            to prevent pox from interpretting the flags.
//     create|c: create the TCZ file.
//     zip|z: Use zip and unzip instead of a loopback mounted squashfs.  Be sure
//            to use -z for both creation and running, or not at all.
//     Exactly one of -c and -r must be used on the same command.
//
// Example:
//	$ pox -c /bin/bash /bin/cat /bin/ls /etc/hosts
//	Will build a squashfs, which will be /tmp/pox.tcz
//
//	$ sudo pox -r /bin/bash
//	Will drop you into the /tmp/pox.tcz running bash
//	You can use ls and cat on /etc/hosts.
//
//	Simpler example, with arguments:
//	$ sudo pox -r /bin/ls -- -la
//	will run `ls -la` and exit.
//
//	$ sudo pox -r -- /bin/ls -la
//	Syntactically easier: the program name can come after '--'
//
// Notes:
// - When running a pox, you likely need sudo to chroot
//
// - Binaries run out of a chroot often need files you are unaware of.  For
// instance, if bash can't find terminfo files, it won't know to handle
// backspaces properly.  (They occur, but are not shown).  To fix this, pass pox
// all of the files you need.  For bash: `find /lib/terminfo -type f`.
//
// Other programs rely on help functions, such as '/bin/man'.  If your program
// has built-in help commands that trigger man pages, e.g. "git help foo",
// you'll want to include /bin/man too.  But you'll also need everything that
// man uses, such as /etc/manpath.config.  My advice: skip it.
//
// - When adding all files in a directory, the easiest thing to do is:
// `find $DIR -type f`  (Note the ticks: this is a bash command execution).
//
// - When creating a pox with an executable with shared libraries that are not
// installed on your system, such as for a project installed in your home
// directory, run pox from the installation prefix directory, such that the
// lib/ and bin/ are in pox's working directory.  Pox will strip its working
// directory from the paths of the files it builds.  Having bin/ in the root of
// the pox file helps with PATH lookups, and not having the full path from your
// machine in the pox file makes it easier to extract a pox file to /usr/local/.
//
// - Consider adding a --extract | -x option to install to the host.  One issue
// would be how to handle collisions, e.g. libc.  Your app may not like the libc
// on the system you run on.
package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"

	flag "github.com/spf13/pflag"
	"github.com/u-root/u-root/pkg/ldd"
	"github.com/u-root/u-root/pkg/loop"
	"github.com/u-root/u-root/pkg/mount"
)

const usage = "pox [-[-debug]|d] -[-run|r] | -[-create]|c  [-[-file]|f tcz-file] file [...file]"

var (
	debug  = flag.BoolP("debug", "d", false, "enable debug prints")
	run    = flag.BoolP("run", "r", false, "Run the first file argument")
	create = flag.BoolP("create", "c", false, "create it")
	zip    = flag.BoolP("zip", "z", false, "use zip instead of squashfs")
	file   = flag.StringP("output", "f", "/tmp/pox.tcz", "Output file")
	v      = func(string, ...interface{}) {}
)

// When chrooting, programs often want to access various system directories:
var chrootMounts = []struct {
	source string
	target string
	fstype string
	flags  uintptr
	data   string
	perm   os.FileMode // for target in the chroot
}{
	// mount --bind /sys /chroot/sys
	{"/sys", "/sys", "", mount.MS_BIND, "", 0555},
	// mount -t proc /proc /chroot/proc
	{"/proc", "/proc", "proc", 0, "", 0555},
	// mount --bind /dev /chroot/dev
	{"/dev", "/dev", "", mount.MS_BIND, "", 0755},
}

func poxCreate(bin []string) error {
	if len(bin) == 0 {
		return fmt.Errorf(usage)
	}
	l, err := ldd.Ldd(bin)
	if err != nil {
		var stderr []byte
		if eerr, ok := err.(*exec.ExitError); ok {
			stderr = eerr.Stderr
		}
		return fmt.Errorf("Running ldd on %v: %v %s", bin, err, stderr)
	}

	var names []string
	for _, dep := range l {
		v("%s", dep.FullName)
		names = append(names, dep.FullName)
	}
	// Now we need to make a template file hierarchy and put
	// the stuff we want in there.
	dir, err := ioutil.TempDir("", "pox")
	if err != nil {
		return err
	}
	if !*debug {
		defer os.RemoveAll(dir)
	}
	pwd, err := os.Getwd()
	if err != nil {
		return err
	}
	// We don't use defer() here to close files as
	// that can cause open failures with a large enough number.
	for _, f := range names {
		v("Process %v", f)
		fi, err := os.Stat(f)
		if err != nil {
			return err
		}
		in, err := os.Open(f)
		if err != nil {
			return err
		}
		f = strings.TrimPrefix(f, pwd)
		dfile := filepath.Join(dir, f)
		d := filepath.Dir(dfile)
		if err := os.MkdirAll(d, 0755); err != nil {
			in.Close()
			return err
		}
		out, err := os.OpenFile(dfile, os.O_WRONLY|os.O_CREATE,
			fi.Mode().Perm())
		if err != nil {
			in.Close()
			return err
		}
		_, err = io.Copy(out, in)
		in.Close()
		out.Close()
		if err != nil {
			return err
		}

	}
	for _, m := range chrootMounts {
		if err := os.MkdirAll(filepath.Join(dir, m.target), m.perm); err != nil {
			return err
		}
	}
	err = os.Remove(*file)
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	var c *exec.Cmd
	if *zip {
		fileAbs, err := filepath.Abs(*file)
		if err != nil {
			return err
		}
		c = exec.Command("zip", "-r", fileAbs, ".")
		c.Dir = dir
	} else {
		c = exec.Command("mksquashfs", dir, *file, "-noappend")
	}
	o, err := c.CombinedOutput()
	v("%v", string(o))
	if err != nil {
		return fmt.Errorf("%v: %v: %v", c.Args, string(o), err)
	}
	v("Done, your pox is in %v", *file)

	return nil
}

func poxRun(args []string) error {
	if len(args) == 0 {
		return fmt.Errorf(usage)
	}
	dir, err := ioutil.TempDir("", "pox")
	if err != nil {
		return err
	}
	if !*debug {
		defer os.RemoveAll(dir)
	}

	if *zip {
		c := exec.Command("unzip", *file, "-d", dir)
		o, err := c.CombinedOutput()
		v("%v", string(o))
		if err != nil {
			return fmt.Errorf("%v: %v: %v", c.Args, string(o), err)
		}
	} else {
		lo, err := loop.New(*file, "squashfs", "")
		if err != nil {
			return err
		}
		defer lo.Free() //nolint:errcheck

		mountPoint, err := lo.Mount(dir, 0)
		if err != nil {
			return err
		}
		defer mountPoint.Unmount(0) //nolint:errcheck
	}
	for _, m := range chrootMounts {
		mp, err := mount.Mount(m.source, filepath.Join(dir, m.target),
			m.fstype, m.data, m.flags)
		if err != nil {
			return err
		}
		defer mp.Unmount(0) //nolint:errcheck
	}

	// If you pass Command a path with no slashes, it'll use PATH from the
	// parent to resolve the path to exec.  Once we chroot, whatever path we
	// picked is undoubtably wrong.  Let's help them out: if they give us a
	// program with no /, let's look in /bin/.  If they want the root of the
	// chroot, they can use "./"
	if filepath.Base(args[0]) == args[0] {
		args[0] = filepath.Join(string(os.PathSeparator), "bin", args[0])
	}
	c := exec.Command(args[0], args[1:]...)
	c.Stdin, c.Stdout, c.Stderr = os.Stdin, os.Stdout, os.Stderr
	c.SysProcAttr = &syscall.SysProcAttr{
		Chroot: dir,
	}
	c.Env = append(os.Environ(), "PWD=.")

	if err = c.Run(); err != nil {
		v("pox command exited with: %v", err)
	}

	return nil
}

func pox() error {
	flag.Parse()
	if *debug {
		v = log.Printf
	}
	if (*create && *run) || (!*create && !*run) {
		return fmt.Errorf(usage)
	}
	if *create {
		return poxCreate(flag.Args())
	}
	if *run {
		return poxRun(flag.Args())
	}
	return fmt.Errorf(usage)
}

func main() {
	if err := pox(); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package task

import (
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/spf13/cobra"
	"github.com/taskcluster/slugid-go/slugid"
	"github.com/taskcluster/taskcluster-cli/config"
	tcclient "github.com/taskcluster/taskcluster-client-go"
	"github.com/taskcluster/taskcluster-client-go/queue"
)

var (
	createCmd = &cobra.Command{
		Use:     "create <image> <command>",
		Short:   "Creates and schedules a task, using the specified taskId if given.",
		PreRunE: checkCreateFlags,
		RunE:    runCreateTask,
	}

	now = time.Now().UTC()

	createPayload = &queue.TaskDefinitionRequest{
		SchedulerID: "taskcluster-cli",

		Created:  tcclient.Time(now),
		Deadline: tcclient.Time(now.Add(24 * time.Hour)),
		Expires:  tcclient.Time(now.Add(24*time.Hour).AddDate(1, 0, 0)),
	}
)

func init() {
	fs := createCmd.Flags()

	fs.String("task-id", "", "taskID of the task, if omitted, a new one will be assigned")
	fs.StringVar(&createPayload.ProvisionerID, "provisioner", "", "ID of the provisioner to use")
	fs.StringVar(&createPayload.WorkerType, "worker-type", "", "worker-type to use within the provisioner")
	fs.StringSliceP("env", "e", []string{}, "Environment variables to add to the task's environment (VARIABLE=VALUE)")
	fs.StringVar(&createPayload.Metadata.Name, "name", "", "Human readable name of the task")
	fs.StringVar(&createPayload.Metadata.Description, "description", "", "Human readable description of the task")
	fs.StringVar(&createPayload.Metadata.Owner, "owner", "", "Email of the task's owner")
	fs.StringVar(&createPayload.Metadata.Source, "source", "", "URL pointing to the source of the task")
	for _, f := range []string{"provisioner", "worker-type"} {
		createCmd.MarkFlagRequired(f)
	}

	Command.AddCommand(createCmd)
}

func checkCreateFlags(cmd *cobra.Command, args []string) error {
	for _, f := range []string{"provisioner", "worker-type"} {
		if !cmd.Flag(f).Changed {
			return fmt.Errorf("flag '%s' is required", f)
		}
	}
	return nil
}

func runCreateTask(cmd *cobra.Command, args []string) error {
	if len(args) < 2 {
		return errors.New("create requires at least 2 arguments: image and command")
	}

	var creds *tcclient.Credentials
	if config.Credentials != nil {
		creds = config.Credentials.ToClientCredentials()
	}

	var taskID string
	if s := stringFlagHelper(cmd.Flags(), "task-id"); s != "" {
		if slugid.Decode(s) != nil {
			taskID = s
		} else {
			return fmt.Errorf("task id '%s' is not a valid slugid", s)
		}
	} else {
		taskID = slugid.V4()
	}
	createPayload.TaskGroupID = taskID

	env := make(map[string]string)
	envs, err := cmd.Flags().GetStringSlice("env")
	if err != nil {
		return err
	}
	for _, e := range envs {
		p := strings.SplitN(e, "=", 2)
		switch len(p) {
		case 2:
			env[p[0]] = p[1]
		case 1:
			env[p[0]] = ""
		default:
			return fmt.Errorf("invalid environment option: %s", e)
		}
	}

	createPayload.Payload, err = json.Marshal(struct {
		Image       string            `json:"image"`
		Command     []string          `json:"command"`
		Environment map[string]string `json:"env"`
		MaxRunTime  int               `json:"maxRunTime"`
	}{
		Image:       args[0],
		Command:     args[1:],
		Environment: env,
		MaxRunTime:  7200, // 2 hours
	})
	if err != nil {
		return fmt.Errorf("could not marshal execution payload: %v", err)
	}

	if createPayload.Metadata.Source == "" {
		createPayload.Metadata.Source = "http://taskcluster-cli/task/create"
	}

	tags := make(map[string]string)
	tags["misc_info"] = "created by taskcluster-cli"
	createPayload.Tags, err = json.Marshal(tags)
	if err != nil {
		return fmt.Errorf("could not marshal tags: %v", err)
	}

	q := queue.New(creds)
	resp, err := q.CreateTask(taskID, createPayload)
	if err != nil {
		return fmt.Errorf("could not create task: %v", err)
	}

	// If we got no error, that means the task was successfully created
	fmt.Fprintf(cmd.OutOrStdout(), "Task %s created\n", resp.Status.TaskID)

	return nil
}
<commit_msg>Added Dependency flag and reworded help messages<commit_after>package task

import (
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/spf13/cobra"
	"github.com/taskcluster/slugid-go/slugid"
	"github.com/taskcluster/taskcluster-cli/config"
	tcclient "github.com/taskcluster/taskcluster-client-go"
	"github.com/taskcluster/taskcluster-client-go/queue"
)

var (
	createCmd = &cobra.Command{
		Use:     "create <image> <command>",
		Short:   "Creates and schedules a task, using the specified taskId if given.",
		PreRunE: checkCreateFlags,
		RunE:    runCreateTask,
	}

	now = time.Now().UTC()

	createPayload = &queue.TaskDefinitionRequest{
		SchedulerID: "taskcluster-cli",

		Created:  tcclient.Time(now),
		Deadline: tcclient.Time(now.Add(24 * time.Hour)),
		Expires:  tcclient.Time(now.Add(24*time.Hour).AddDate(1, 0, 0)),
	}
)

func init() {
	fs := createCmd.Flags()

	fs.String("task-id", "", "taskID of the task, if omitted, a new one will be assigned")
	fs.StringVar(&createPayload.ProvisionerID, "provisioner", "", "ID of the provisioner to use")
	fs.StringVar(&createPayload.WorkerType, "worker-type", "", "worker-type to use within the provisioner")
	fs.StringSliceP("env", "e", []string{}, "Environment variable to add to the task's environment (repeatable) (format: VARIABLE=VALUE)")
	fs.StringVar(&createPayload.Metadata.Name, "name", "", "Human readable name of the task")
	fs.StringVar(&createPayload.Metadata.Description, "description", "", "Human readable description of the task")
	fs.StringVar(&createPayload.Metadata.Owner, "owner", "", "Email of the task's owner")
	fs.StringVar(&createPayload.Metadata.Source, "source", "", "URL pointing to the source of the task")
	fs.StringSliceVar(&createPayload.Dependencies, "dependency", []string{}, "TaskID of a dependency (repeatable)")
	for _, f := range []string{"provisioner", "worker-type"} {
		createCmd.MarkFlagRequired(f)
	}

	Command.AddCommand(createCmd)
}

func checkCreateFlags(cmd *cobra.Command, args []string) error {
	for _, f := range []string{"provisioner", "worker-type"} {
		if !cmd.Flag(f).Changed {
			return fmt.Errorf("flag '%s' is required", f)
		}
	}
	return nil
}

func runCreateTask(cmd *cobra.Command, args []string) error {
	if len(args) < 2 {
		return errors.New("create requires at least 2 arguments: image and command")
	}

	var creds *tcclient.Credentials
	if config.Credentials != nil {
		creds = config.Credentials.ToClientCredentials()
	}

	var taskID string
	if s := stringFlagHelper(cmd.Flags(), "task-id"); s != "" {
		if slugid.Decode(s) != nil {
			taskID = s
		} else {
			return fmt.Errorf("task id '%s' is not a valid slugid", s)
		}
	} else {
		taskID = slugid.V4()
	}
	createPayload.TaskGroupID = taskID

	env := make(map[string]string)
	envs, err := cmd.Flags().GetStringSlice("env")
	if err != nil {
		return err
	}
	for _, e := range envs {
		p := strings.SplitN(e, "=", 2)
		switch len(p) {
		case 2:
			env[p[0]] = p[1]
		case 1:
			env[p[0]] = ""
		default:
			return fmt.Errorf("invalid environment option: %s", e)
		}
	}

	createPayload.Payload, err = json.Marshal(struct {
		Image       string            `json:"image"`
		Command     []string          `json:"command"`
		Environment map[string]string `json:"env"`
		MaxRunTime  int               `json:"maxRunTime"`
	}{
		Image:       args[0],
		Command:     args[1:],
		Environment: env,
		MaxRunTime:  7200, // 2 hours
	})
	if err != nil {
		return fmt.Errorf("could not marshal execution payload: %v", err)
	}

	if createPayload.Metadata.Source == "" {
		createPayload.Metadata.Source = "http://taskcluster-cli/task/create"
	}

	tags := make(map[string]string)
	tags["misc_info"] = "created by taskcluster-cli"
	createPayload.Tags, err = json.Marshal(tags)
	if err != nil {
		return fmt.Errorf("could not marshal tags: %v", err)
	}

	q := queue.New(creds)
	resp, err := q.CreateTask(taskID, createPayload)
	if err != nil {
		return fmt.Errorf("could not create task: %v", err)
	}

	// If we got no error, that means the task was successfully created
	fmt.Fprintf(cmd.OutOrStdout(), "Task %s created\n", resp.Status.TaskID)

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	irc "github.com/thoj/Go-IRC-Client-Library"
	"fmt"
	"os"
)

func main() {
	irccon := irc.IRC("testgo", "testgo")
	err := irccon.Connect("irc.efnet.net:6667")
	if err != nil {
		fmt.Printf("%s\n", err)
		fmt.Printf("%#v\n", irccon)
		os.Exit(1)
	}
	irccon.AddCallback("001", func(e *irc.IRCEvent) { irccon.Join("#testgo") })
	irccon.Loop();
}
<commit_msg>More than one callback in example<commit_after>package main

import (
//	irc "github.com/thoj/Go-IRC-Client-Library"
	"fmt"
	"os"
	"irc"
)

func main() {
	irccon := irc.IRC("testgo", "testgo")
	err := irccon.Connect("irc.efnet.net:6667")
	if err != nil {
		fmt.Printf("%s\n", err)
		fmt.Printf("%#v\n", irccon)
		os.Exit(1)
	}
	irccon.AddCallback("001", func(e *irc.IRCEvent) { irccon.Join("#testgo1") })
	irccon.AddCallback("001", func(e *irc.IRCEvent) { irccon.Join("#testgo2") })
	irccon.AddCallback("001", func(e *irc.IRCEvent) { irccon.Join("#testgo3") })
	irccon.AddCallback("001", func(e *irc.IRCEvent) { irccon.Join("#testgo4") })
	irccon.AddCallback("001", func(e *irc.IRCEvent) { irccon.Join("#testgo5") })
	irccon.AddCallback("001", func(e *irc.IRCEvent) { irccon.Join("#testgo6") })
	irccon.ReplaceCallback("001", 0, func(e *irc.IRCEvent) { irccon.Join("#testgo01") })
	irccon.ReplaceCallback("001", 1, func(e *irc.IRCEvent) { irccon.Join("#testgo02") })
	irccon.ReplaceCallback("001", 2, func(e *irc.IRCEvent) { irccon.Join("#testgo03") })
	irccon.ReplaceCallback("001", 3, func(e *irc.IRCEvent) { irccon.Join("#testgo04") })
	irccon.ReplaceCallback("001", 4, func(e *irc.IRCEvent) { irccon.Join("#testgo05") })
	irccon.ReplaceCallback("001", 6, func(e *irc.IRCEvent) { irccon.Join("#testgo06") })
	irccon.Loop();
}
<|endoftext|>