<commit_before>package clocky

import (
	"http"
	"io"
	"io/ioutil"
	"os"
	"strconv"
	"strings"
	"time"

	"appengine"
	"appengine/memcache"
	"appengine/urlfetch"
)

type Source struct {
	Key                 string
	URL                 string
	Refresh, Expiration int
}

var Sources = []*Source{
	&Source{
		Key: "nextbus",
		// http://www.sfmta.com/cms/asite/nextmunidata.htm
		URL: ("http://webservices.nextbus.com/service/publicXMLFeed?" +
			"command=predictionsForMultiStops&a=sf-muni" +
			"&stops=47|null|6825" +
			"&stops=49|null|6825" +
			"&stops=90|null|6825" +
			"&stops=10|null|5859" +
			"&stops=12|null|5859" +
			"&stops=1|null|4016" +
			"&stops=1|null|6297" +
			"&stops=27|null|5165"),
		Refresh:    10,
		Expiration: 300,
	},
	&Source{
		Key: "forecast",
		URL: ("http://forecast.weather.gov/MapClick.php?" +
			"lat=37.79570&lon=-122.42100&FcstType=dwml&unit=1"),
		// http://graphical.weather.gov/xml/mdl/XML/Design/WebServicesUseGuildlines.php
		Refresh:    3600,
		Expiration: 8 * 3600,
	},
	// A buoy near Crissy Field.
	&Source{
		Key: "conditions",
		URL: "http://www.weather.gov/xml/current_obs/display.php?stid=FTPC1",
		// http://graphical.weather.gov/xml/mdl/XML/Design/WebServicesUseGuildlines.php
		Refresh:    3600,
		Expiration: 4 * 3600,
	},
}

func (s Source) Fetch(c appengine.Context) os.Error {
	c.Debugf("fetching %s data", s.Key)

	transport := urlfetch.Transport{Context: c, DeadlineSeconds: 60}
	req, err := http.NewRequest("GET", s.URL, strings.NewReader(""))
	if err != nil {
		c.Errorf("%s", err)
		return err
	}
	resp, err := transport.RoundTrip(req)
	if err != nil {
		c.Errorf("%s", err)
		return err
	}

	contents, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		c.Errorf("%s", err)
		return err
	}
	resp.Body.Close()

	item := &memcache.Item{
		Key:        s.Key,
		Value:      contents,
		Expiration: int32(s.Expiration),
	}
	if err := memcache.Set(c, item); err != nil {
		c.Errorf("%s", err)
		return err
	}

	// We keep the last updated time in memcache.  It's not
	// updated atomically with the page, so it's only used to
	// limit the rate of fetches from the data servers.  Don't use
	// it for display; use the data creation times in the data
	// instead.  It doesn't matter to the user that we fetched a
	// weather forecast 3 minutes ago if the forecast is 48
	// minutes old.
	item = &memcache.Item{
		Key:   s.Key + "_fresh",
		Value: []byte(strconv.Itoa64(time.Seconds())),
	}
	if err := memcache.Set(c, item); err != nil {
		c.Errorf("%s", err)
		return err
	}

	c.Infof("fetched %d bytes of %s data", len(contents), s.Key)
	return nil
}

// Freshen calls Fetch iff the item is not known to be fresh.
func (s *Source) Freshen(c appengine.Context) os.Error {
	item, err := memcache.Get(c, s.Key+"_fresh")
	if err == memcache.ErrCacheMiss {
		return s.Fetch(c)
	} else if err != nil {
		c.Errorf("%s", err)
		// Something is wrong with memcache, so don't try to fetch.
		return err
	}

	fresh, err := strconv.Atoi64(string(item.Value))
	if err != nil {
		c.Errorf("%s", err)
		return s.Fetch(c)
	}

	stale := fresh + int64(s.Refresh)
	c.Debugf("%s stale = %s", s.Key, time.SecondsToUTC(stale).Format(time.RFC3339))
	if stale > time.Seconds() {
		return nil
	}
	return s.Fetch(c)
}

func warmup(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	ch := make(chan os.Error)
	for _, s := range Sources {
		go func(s *Source) { ch <- s.Freshen(c) }(s)
	}
	for _ = range Sources {
		if err := <-ch; err != nil {
			http.Error(w, err.String(), http.StatusInternalServerError)
			return
		}
	}
	io.WriteString(w, "ok\n")
}

func init() {
	http.HandleFunc("/_ah/warmup", warmup)
}
<commit_msg>Add debug tracing.<commit_after>package clocky

import (
	"fmt"
	"http"
	"io"
	"io/ioutil"
	"os"
	"strconv"
	"strings"
	"time"

	"appengine"
	"appengine/memcache"
	"appengine/urlfetch"
)

type Source struct {
	Key                 string
	URL                 string
	Refresh, Expiration int
}

var Sources = []*Source{
	&Source{
		Key: "nextbus",
		// http://www.sfmta.com/cms/asite/nextmunidata.htm
		URL: ("http://webservices.nextbus.com/service/publicXMLFeed?" +
			"command=predictionsForMultiStops&a=sf-muni" +
			"&stops=47|null|6825" +
			"&stops=49|null|6825" +
			"&stops=90|null|6825" +
			"&stops=10|null|5859" +
			"&stops=12|null|5859" +
			"&stops=1|null|4016" +
			"&stops=1|null|6297" +
			"&stops=27|null|5165"),
		Refresh:    10,
		Expiration: 300,
	},
	&Source{
		Key: "forecast",
		URL: ("http://forecast.weather.gov/MapClick.php?" +
			"lat=37.79570&lon=-122.42100&FcstType=dwml&unit=1"),
		// http://graphical.weather.gov/xml/mdl/XML/Design/WebServicesUseGuildlines.php
		Refresh:    3600,
		Expiration: 8 * 3600,
	},
	// A buoy near Crissy Field.
	&Source{
		Key: "conditions",
		URL: "http://www.weather.gov/xml/current_obs/display.php?stid=FTPC1",
		// http://graphical.weather.gov/xml/mdl/XML/Design/WebServicesUseGuildlines.php
		Refresh:    3600,
		Expiration: 4 * 3600,
	},
}

func (s Source) Fetch(c appengine.Context) os.Error {
	c.Debugf("fetching %s data", s.Key)

	transport := urlfetch.Transport{Context: c, DeadlineSeconds: 60}
	req, err := http.NewRequest("GET", s.URL, strings.NewReader(""))
	if err != nil {
		c.Errorf("%s", err)
		return err
	}
	c.Debugf("XXX created request for %s", s.Key)
	resp, err := transport.RoundTrip(req)
	if err != nil {
		c.Errorf("%s", err)
		return err
	}
	c.Debugf("XXX called transport for %s", s.Key)
	if resp.StatusCode != http.StatusOK {
		c.Errorf("fetch: bad status %d for %s", resp.StatusCode, s.Key)
		return fmt.Errorf("fetch: bad status %d for %s", resp.StatusCode, s.Key)
	}
	c.Debugf("XXX reading body for %s", s.Key)
	contents, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		c.Errorf("%s", err)
		return err
	}
	c.Debugf("XXX closing %s", s.Key)
	resp.Body.Close()

	item := &memcache.Item{
		Key:        s.Key,
		Value:      contents,
		Expiration: int32(s.Expiration),
	}
	c.Debugf("XXX setting memcache for %s", s.Key)
	if err := memcache.Set(c, item); err != nil {
		c.Errorf("%s", err)
		return err
	}

	// We keep the last updated time in memcache.  It's not
	// updated atomically with the page, so it's only used to
	// limit the rate of fetches from the data servers.  Don't use
	// it for display; use the data creation times in the data
	// instead.  It doesn't matter to the user that we fetched a
	// weather forecast 3 minutes ago if the forecast is 48
	// minutes old.
	item = &memcache.Item{
		Key:   s.Key + "_fresh",
		Value: []byte(strconv.Itoa64(time.Seconds())),
	}
	if err := memcache.Set(c, item); err != nil {
		c.Errorf("%s", err)
		return err
	}

	c.Infof("fetched %d bytes of %s data", len(contents), s.Key)
	return nil
}

// Freshen calls Fetch iff the item is not known to be fresh.
func (s *Source) Freshen(c appengine.Context) os.Error {
	item, err := memcache.Get(c, s.Key+"_fresh")
	if err == memcache.ErrCacheMiss {
		return s.Fetch(c)
	} else if err != nil {
		c.Errorf("%s", err)
		// Something is wrong with memcache, so don't try to fetch.
		return err
	}

	fresh, err := strconv.Atoi64(string(item.Value))
	if err != nil {
		c.Errorf("%s", err)
		return s.Fetch(c)
	}

	stale := fresh + int64(s.Refresh)
	c.Debugf("%s stale = %s", s.Key, time.SecondsToUTC(stale).Format(time.RFC3339))
	if stale > time.Seconds() {
		return nil
	}
	return s.Fetch(c)
}

func warmup(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	ch := make(chan os.Error)
	for _, s := range Sources {
		go func(s *Source) { ch <- s.Freshen(c) }(s)
	}
	for _ = range Sources {
		if err := <-ch; err != nil {
			http.Error(w, err.String(), http.StatusInternalServerError)
			return
		}
	}
	io.WriteString(w, "ok\n")
}

func init() {
	http.HandleFunc("/_ah/warmup", warmup)
}
<|endoftext|>


<commit_before>package couchdb

import (
	"reflect"
	"regexp"
	"testing"
)

var client = Client{"http://127.0.0.1:5984/"}

func TestInfo(t *testing.T) {
	info, err := client.Info()
	if err != nil {
		t.Fatal(err)
	}
	if info.Couchdb != "Welcome" {
		t.Error("Couchdb error")
	}
}

func TestLog(t *testing.T) {
	log, err := client.Log()
	if err != nil {
		t.Fatal(err)
	}
	valid := regexp.MustCompile("[info]")
	if valid.MatchString(log) == false {
		t.Error("invalid log")
	}
}

func TestActiveTasks(t *testing.T) {
	res, err := client.ActiveTasks()
	if err != nil {
		t.Fatal(err)
	}
	out := make([]Task, 0)
	if reflect.DeepEqual(out, res) == false {
		t.Error("active tasks should be an empty array")
	}
}

func TestAll(t *testing.T) {
	res, err := client.All()
	if err != nil {
		t.Fatal(err)
	}
	if res[0] != "_replicator" || res[1] != "_users" {
		t.Error("slice error")
	}
}

func TestGet(t *testing.T) {
	info, err := client.Get("_users")
	if err != nil {
		t.Fatal(err)
	}
	if info.DbName != "_users" {
		t.Error("DbName error")
	}
	if info.CompactRunning != false {
		t.Error("CompactRunning error")
	}
}

func TestCreate(t *testing.T) {
	status, err := client.Create("dummy")
	if err != nil {
		t.Fatal(err)
	}
	if status.Ok != true {
		t.Error("status error")
	}
	if status.Error != "" || status.Reason != "" {
		t.Error("error and reason error")
	}
}

func TestCreateFail(t *testing.T) {
	_, err := client.Create("dummy")
	if err == nil {
		t.Fatal("should not create duplicate database")
	}
}

func TestDelete(t *testing.T) {
	status, err := client.Delete("dummy")
	if err != nil {
		t.Fatal(err)
	}
	if status.Ok != true {
		t.Error("status error")
	}
	if status.Error != "" || status.Reason != "" {
		t.Error("error and reason error")
	}
}

func TestDeleteFail(t *testing.T) {
	_, err := client.Delete("dummy")
	if err == nil {
		t.Fatal("should not delete non existing database")
	}
}

func TestUse(t *testing.T) {
	out := Database{
		Url: "http://127.0.0.1:5984/_users/",
	}
	db := client.Use("_users")
	if reflect.DeepEqual(out, db) == false {
		t.Error("use error")
	}
}
<commit_msg>remove tests for error and reason fields<commit_after>package couchdb

import (
	"reflect"
	"regexp"
	"testing"
)

var client = Client{"http://127.0.0.1:5984/"}

func TestInfo(t *testing.T) {
	info, err := client.Info()
	if err != nil {
		t.Fatal(err)
	}
	if info.Couchdb != "Welcome" {
		t.Error("Couchdb error")
	}
}

func TestLog(t *testing.T) {
	log, err := client.Log()
	if err != nil {
		t.Fatal(err)
	}
	valid := regexp.MustCompile("[info]")
	if valid.MatchString(log) == false {
		t.Error("invalid log")
	}
}

func TestActiveTasks(t *testing.T) {
	res, err := client.ActiveTasks()
	if err != nil {
		t.Fatal(err)
	}
	out := make([]Task, 0)
	if reflect.DeepEqual(out, res) == false {
		t.Error("active tasks should be an empty array")
	}
}

func TestAll(t *testing.T) {
	res, err := client.All()
	if err != nil {
		t.Fatal(err)
	}
	if res[0] != "_replicator" || res[1] != "_users" {
		t.Error("slice error")
	}
}

func TestGet(t *testing.T) {
	info, err := client.Get("_users")
	if err != nil {
		t.Fatal(err)
	}
	if info.DbName != "_users" {
		t.Error("DbName error")
	}
	if info.CompactRunning != false {
		t.Error("CompactRunning error")
	}
}

func TestCreate(t *testing.T) {
	status, err := client.Create("dummy")
	if err != nil {
		t.Fatal(err)
	}
	if status.Ok != true {
		t.Error("status error")
	}
}

func TestCreateFail(t *testing.T) {
	_, err := client.Create("dummy")
	if err == nil {
		t.Fatal("should not create duplicate database")
	}
}

func TestDelete(t *testing.T) {
	status, err := client.Delete("dummy")
	if err != nil {
		t.Fatal(err)
	}
	if status.Ok != true {
		t.Error("status error")
	}
}

func TestDeleteFail(t *testing.T) {
	_, err := client.Delete("dummy")
	if err == nil {
		t.Fatal("should not delete non existing database")
	}
}

func TestUse(t *testing.T) {
	out := Database{
		Url: "http://127.0.0.1:5984/_users/",
	}
	db := client.Use("_users")
	if reflect.DeepEqual(out, db) == false {
		t.Error("use error")
	}
}
<|endoftext|>


<commit_before>package provisioner

import (
	"encoding/json"
	"fmt"
	"hash/crc32"

	"github.com/mitchellh/packer/packer"
)

var	shellProvisioner = map[string]interface{}{
	"type": "shell",
	"inline": []string{
		"sleep 30",
		// Refresh package entries
		"sudo apt-get update",
		// Install system software & CLI Tools
		"sudo apt-get install -y ubuntu-standard ubuntu-minimal htop git net-tools aptitude apache2 php5 libapache2-mod-php5 php5-cgi ruby screen fish sudo emacs mc iotop iftop software-properties-common libgd2-xpm",
		// Install NodeJS 0.10.26
		"wget -O - http://nodejs.org/dist/v0.10.26/node-v0.10.26-linux-x64.tar.gz | sudo tar -C /usr/local/ --strip-components=1 -zxv"
		// Install programming language runtimes/compilers
		"sudo apt-get install -y erlang ghc swi-prolog clisp ruby-dev ri rake python mercurial subversion cvs bzr default-jdk",
	},
}

// RawData is JSON abstraction mapped to Go struct. This can be
// Marshaled and injected directly into packer.ParseTemplate()
var RawData = []interface{}{
	shellProvisioner,
}

// PackerTemplate is ready to used and unmarshalled packer.Template. This is
// created only for future references where we might decide to not use
// packer.ParseTemplate()
var PackerTemplate = packer.Template{
	Provisioners: []packer.RawProvisionerConfig{
		{
			Type: "shell",
			RawConfig: shellProvisioner,
		},
	},
}

// Get the AMI name of the image associated to the current packer conf
func Ami() string {
	return "koding-"+Checksum()
}

// Get a hexadecimal checksum
// We use this to ensure idempotence
// and that an image is only crated once
func Checksum() string {
	// Get json representation of docker conf
	data, err := json.Marshal(RawData)
	if err != nil {
		// this should never happen
		panic("Failed to compute Checksum for packer config")
	}

	// integet checksum
	sum := crc32.ChecksumIEEE(data)

	return hex32(sum)
}

// Utility function to produce a hexadecimal string from an uint32
func hex32(x uint32) string {
	return fmt.Sprintf("%08x", x)
}

<commit_msg>AMI: Install golang-go package<commit_after>package provisioner

import (
	"encoding/json"
	"fmt"
	"hash/crc32"

	"github.com/mitchellh/packer/packer"
)

var	shellProvisioner = map[string]interface{}{
	"type": "shell",
	"inline": []string{
		"sleep 30",
		// Refresh package entries
		"sudo apt-get update",
		// Install system software & CLI Tools
		"sudo apt-get install -y ubuntu-standard ubuntu-minimal htop git net-tools aptitude apache2 php5 libapache2-mod-php5 php5-cgi ruby screen fish sudo emacs mc iotop iftop software-properties-common libgd2-xpm",
		// Install NodeJS 0.10.26
		"wget -O - http://nodejs.org/dist/v0.10.26/node-v0.10.26-linux-x64.tar.gz | sudo tar -C /usr/local/ --strip-components=1 -zxv"
		// Install programming language runtimes/compilers
		"sudo apt-get install -y erlang ghc swi-prolog clisp ruby-dev ri rake python mercurial subversion cvs bzr default-jdk golang-go",
	},
}

// RawData is JSON abstraction mapped to Go struct. This can be
// Marshaled and injected directly into packer.ParseTemplate()
var RawData = []interface{}{
	shellProvisioner,
}

// PackerTemplate is ready to used and unmarshalled packer.Template. This is
// created only for future references where we might decide to not use
// packer.ParseTemplate()
var PackerTemplate = packer.Template{
	Provisioners: []packer.RawProvisionerConfig{
		{
			Type: "shell",
			RawConfig: shellProvisioner,
		},
	},
}

// Get the AMI name of the image associated to the current packer conf
func Ami() string {
	return "koding-"+Checksum()
}

// Get a hexadecimal checksum
// We use this to ensure idempotence
// and that an image is only crated once
func Checksum() string {
	// Get json representation of docker conf
	data, err := json.Marshal(RawData)
	if err != nil {
		// this should never happen
		panic("Failed to compute Checksum for packer config")
	}

	// integet checksum
	sum := crc32.ChecksumIEEE(data)

	return hex32(sum)
}

// Utility function to produce a hexadecimal string from an uint32
func hex32(x uint32) string {
	return fmt.Sprintf("%08x", x)
}

<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"time"

	"koding/db/mongodb"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

func init() {
	registerAnalytic(numberOfTwoWeekEngagedUsers)
}

func numberOfTwoWeekEngagedUsers() (string, int) {
	var identifier string = "number_of_two_week_engaged_users"
	var year, month, _ = time.Now().Date()
	var startDateOfMonth = time.Date(year, month, 1, 0, 0, 0, 0, currentTimeLocation)

	// 14 isn't always the middle of the month, but it's easier to assume for now
	var middleOfMonth = time.Date(year, month, 15, 0, 0, 0, 0, currentTimeLocation)

	var iterQuery = func(c *mgo.Collection) *mgo.Query {
		var query = c.Find(bson.M{
			"createdAt": bson.M{"$gte": startDateOfMonth, "$lte": middleOfMonth},
		})

		return query
	}

	var possibleEngagedUsers = map[string]bool{}

	var iter = mongodb.Iter("jSessionHistories", iterQuery)
	var result map[string]interface{}

	for iter.Next(&result) {
		var username = result["username"].(string)
		possibleEngagedUsers[username] = true
	}

	var err = mongodb.IterClose(iter)
	if err != nil {
		fmt.Println(err)
	}

	fmt.Println("count of possibleEngagedUsers", len(possibleEngagedUsers))

	var possibleEngagedUsernames = []string{}
	for username, _ := range possibleEngagedUsers {
		possibleEngagedUsernames = append(possibleEngagedUsernames, username)
	}

	//----------------------------------------------------------
	// Second Query
	//----------------------------------------------------------

	var secondIterQuery = func(c *mgo.Collection) *mgo.Query {
		var query = c.Find(bson.M{
			"username":  bson.M{"$in": possibleEngagedUsernames},
			"createdAt": bson.M{"$gt": middleOfMonth},
		})

		return query
	}

	var engagedUsers = map[string]bool{}

	var secondIter = mongodb.Iter("jSessionHistories", secondIterQuery)
	var secondResult map[string]interface{}

	for secondIter.Next(&secondResult) {
		var username = result["username"].(string)
		engagedUsers[username] = true
	}

	err = mongodb.IterClose(iter)
	if err != nil {
		fmt.Println(err)
	}

	fmt.Println("count of engagedUsers", len(engagedUsers))

	var engagedUsernames = []string{}
	for username, _ := range engagedUsers {
		engagedUsernames = append(engagedUsernames, username)
	}

	var engagedUsernamesLength = len(engagedUsernames)

	return identifier, engagedUsernamesLength
}
<commit_msg>graphitefeeder: don't close already closed Iter<commit_after>package main

import (
	"fmt"
	"time"

	"koding/db/mongodb"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

func init() {
	registerAnalytic(numberOfTwoWeekEngagedUsers)
}

func numberOfTwoWeekEngagedUsers() (string, int) {
	var identifier string = "number_of_two_week_engaged_users"
	var year, month, _ = time.Now().Date()
	var startDateOfMonth = time.Date(year, month, 1, 0, 0, 0, 0, currentTimeLocation)

	// 14 isn't always the middle of the month, but it's easier to assume for now
	var middleOfMonth = time.Date(year, month, 15, 0, 0, 0, 0, currentTimeLocation)

	var iterQuery = func(c *mgo.Collection) *mgo.Query {
		var query = c.Find(bson.M{
			"createdAt": bson.M{"$gte": startDateOfMonth, "$lte": middleOfMonth},
		})

		return query
	}

	var possibleEngagedUsers = map[string]bool{}

	var iter = mongodb.Iter("jSessionHistories", iterQuery)
	var result map[string]interface{}

	for iter.Next(&result) {
		var username = result["username"].(string)
		possibleEngagedUsers[username] = true
	}

	var err = mongodb.IterClose(iter)
	if err != nil {
		fmt.Println(err)
	}

	fmt.Println("count of possibleEngagedUsers", len(possibleEngagedUsers))

	var possibleEngagedUsernames = []string{}
	for username, _ := range possibleEngagedUsers {
		possibleEngagedUsernames = append(possibleEngagedUsernames, username)
	}

	//----------------------------------------------------------
	// Second Query
	//----------------------------------------------------------

	var secondIterQuery = func(c *mgo.Collection) *mgo.Query {
		var query = c.Find(bson.M{
			"username":  bson.M{"$in": possibleEngagedUsernames},
			"createdAt": bson.M{"$gt": middleOfMonth},
		})

		return query
	}

	var engagedUsers = map[string]bool{}

	var secondIter = mongodb.Iter("jSessionHistories", secondIterQuery)
	var secondResult map[string]interface{}

	for secondIter.Next(&secondResult) {
		var username = result["username"].(string)
		engagedUsers[username] = true
	}

	err = mongodb.IterClose(secondIter)
	if err != nil {
		fmt.Println(err)
	}

	fmt.Println("count of engagedUsers", len(engagedUsers))

	var engagedUsernames = []string{}
	for username, _ := range engagedUsers {
		engagedUsernames = append(engagedUsernames, username)
	}

	var engagedUsernamesLength = len(engagedUsernames)

	return identifier, engagedUsernamesLength
}
<|endoftext|>


<commit_before>package torrent

import (
	"encoding/binary"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"testing"
	"time"

	"bitbucket.org/anacrolix/go.torrent/data/blob"

	"github.com/bradfitz/iter"

	"bitbucket.org/anacrolix/go.torrent/internal/testutil"
	"bitbucket.org/anacrolix/go.torrent/util"
	"bitbucket.org/anacrolix/utp"
	"github.com/anacrolix/libtorgo/bencode"
)

var TestingConfig = Config{
	ListenAddr:           ":0",
	NoDHT:                true,
	DisableTrackers:      true,
	NoDefaultBlocklist:   true,
	DisableMetainfoCache: true,
}

func TestClientDefault(t *testing.T) {
	cl, err := NewClient(&Config{
		NoDefaultBlocklist: true,
		ListenAddr:         ":0",
	})
	if err != nil {
		t.Fatal(err)
	}
	cl.Close()
}

func TestAddDropTorrent(t *testing.T) {
	cl, err := NewClient(&Config{
		ListenAddr: "localhost:0",
	})
	if err != nil {
		t.Fatal(err)
	}
	defer cl.Close()
	dir, mi := testutil.GreetingTestTorrent()
	defer os.RemoveAll(dir)
	tt, err := cl.AddTorrent(mi)
	if err != nil {
		t.Fatal(err)
	}
	tt.Drop()
}

func TestAddTorrentNoSupportedTrackerSchemes(t *testing.T) {
	t.SkipNow()
}

func TestAddTorrentNoUsableURLs(t *testing.T) {
	t.SkipNow()
}

func TestAddPeersToUnknownTorrent(t *testing.T) {
	t.SkipNow()
}

func TestPieceHashSize(t *testing.T) {
	if pieceHash.Size() != 20 {
		t.FailNow()
	}
}

func TestTorrentInitialState(t *testing.T) {
	dir, mi := testutil.GreetingTestTorrent()
	defer os.RemoveAll(dir)
	tor, err := newTorrent(func() (ih InfoHash) {
		util.CopyExact(ih[:], mi.Info.Hash)
		return
	}(), nil, 0)
	if err != nil {
		t.Fatal(err)
	}
	err = tor.setMetadata(mi.Info.Info, mi.Info.Bytes, nil)
	if err != nil {
		t.Fatal(err)
	}
	if len(tor.Pieces) != 3 {
		t.Fatal("wrong number of pieces")
	}
	p := tor.Pieces[0]
	tor.pendAllChunkSpecs(0)
	if len(p.PendingChunkSpecs) != 1 {
		t.Fatalf("should only be 1 chunk: %v", p.PendingChunkSpecs)
	}
	// TODO: Set chunkSize to 2, to test odd/even silliness.
	if false {
		if _, ok := p.PendingChunkSpecs[chunkSpec{
			Length: 13,
		}]; !ok {
			t.Fatal("pending chunk spec is incorrect")
		}
	}
}

func TestUnmarshalPEXMsg(t *testing.T) {
	var m peerExchangeMessage
	if err := bencode.Unmarshal([]byte("d5:added12:\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0ce"), &m); err != nil {
		t.Fatal(err)
	}
	if len(m.Added) != 2 {
		t.FailNow()
	}
	if m.Added[0].Port != 0x506 {
		t.FailNow()
	}
}

func TestReducedDialTimeout(t *testing.T) {
	for _, _case := range []struct {
		Max             time.Duration
		HalfOpenLimit   int
		PendingPeers    int
		ExpectedReduced time.Duration
	}{
		{nominalDialTimeout, 40, 0, nominalDialTimeout},
		{nominalDialTimeout, 40, 1, nominalDialTimeout},
		{nominalDialTimeout, 40, 39, nominalDialTimeout},
		{nominalDialTimeout, 40, 40, nominalDialTimeout / 2},
		{nominalDialTimeout, 40, 80, nominalDialTimeout / 3},
		{nominalDialTimeout, 40, 4000, nominalDialTimeout / 101},
	} {
		reduced := reducedDialTimeout(_case.Max, _case.HalfOpenLimit, _case.PendingPeers)
		expected := _case.ExpectedReduced
		if expected < minDialTimeout {
			expected = minDialTimeout
		}
		if reduced != expected {
			t.Fatalf("expected %s, got %s", _case.ExpectedReduced, reduced)
		}
	}
}

func TestUTPRawConn(t *testing.T) {
	l, err := utp.NewSocket("")
	if err != nil {
		t.Fatal(err)
	}
	defer l.Close()
	go func() {
		for {
			_, err := l.Accept()
			if err != nil {
				break
			}
		}
	}()
	// Connect a UTP peer to see if the RawConn will still work.
	utpPeer, err := func() *utp.Socket {
		s, _ := utp.NewSocket("")
		return s
	}().Dial(fmt.Sprintf("localhost:%d", util.AddrPort(l.Addr())))
	if err != nil {
		t.Fatalf("error dialing utp listener: %s", err)
	}
	defer utpPeer.Close()
	peer, err := net.ListenPacket("udp", ":0")
	if err != nil {
		t.Fatal(err)
	}
	defer peer.Close()

	msgsReceived := 0
	const N = 5000 // How many messages to send.
	readerStopped := make(chan struct{})
	// The reader goroutine.
	go func() {
		defer close(readerStopped)
		b := make([]byte, 500)
		for i := 0; i < N; i++ {
			n, _, err := l.PacketConn().ReadFrom(b)
			if err != nil {
				t.Fatalf("error reading from raw conn: %s", err)
			}
			msgsReceived++
			var d int
			fmt.Sscan(string(b[:n]), &d)
			if d != i {
				log.Printf("got wrong number: expected %d, got %d", i, d)
			}
		}
	}()
	udpAddr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("localhost:%d", util.AddrPort(l.Addr())))
	if err != nil {
		t.Fatal(err)
	}
	for i := 0; i < N; i++ {
		_, err := peer.WriteTo([]byte(fmt.Sprintf("%d", i)), udpAddr)
		if err != nil {
			t.Fatal(err)
		}
		time.Sleep(time.Microsecond)
	}
	select {
	case <-readerStopped:
	case <-time.After(time.Second):
		t.Fatal("reader timed out")
	}
	if msgsReceived != N {
		t.Fatalf("messages received: %d", msgsReceived)
	}
}

func TestTwoClientsArbitraryPorts(t *testing.T) {
	for i := 0; i < 2; i++ {
		cl, err := NewClient(&Config{
			ListenAddr: ":0",
		})
		if err != nil {
			t.Fatal(err)
		}
		defer cl.Close()
	}
}

func TestAddDropManyTorrents(t *testing.T) {
	cl, _ := NewClient(&TestingConfig)
	defer cl.Close()
	var m Magnet
	for i := range iter.N(1000) {
		binary.PutVarint(m.InfoHash[:], int64(i))
		cl.AddMagnet(m.String())
	}
}

func TestClientTransfer(t *testing.T) {
	greetingTempDir, mi := testutil.GreetingTestTorrent()
	defer os.RemoveAll(greetingTempDir)
	cfg := TestingConfig
	cfg.DataDir = greetingTempDir
	seeder, err := NewClient(&cfg)
	if err != nil {
		t.Fatal(err)
	}
	defer seeder.Close()
	seeder.AddTorrent(mi)
	leecherDataDir, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(leecherDataDir)
	// cfg.TorrentDataOpener = func(info *metainfo.Info) (data.Data, error) {
	// 	return blob.TorrentData(info, leecherDataDir), nil
	// }
	cfg.TorrentDataOpener = blob.NewStore(leecherDataDir).OpenTorrent
	leecher, _ := NewClient(&cfg)
	defer leecher.Close()
	leecherGreeting, _ := leecher.AddTorrent(mi)
	leecherGreeting.AddPeers([]Peer{
		Peer{
			IP:   util.AddrIP(seeder.ListenAddr()),
			Port: util.AddrPort(seeder.ListenAddr()),
		},
	})
	_greeting, err := ioutil.ReadAll(io.NewSectionReader(leecherGreeting, 0, leecherGreeting.Length()))
	if err != nil {
		t.Fatal(err)
	}
	greeting := string(_greeting)
	if greeting != testutil.GreetingFileContents {
		t.Fatal(":(")
	}
}

func TestReadaheadPieces(t *testing.T) {
	for _, case_ := range []struct {
		readaheadBytes, pieceLength int64
		readaheadPieces             int
	}{
		{5 * 1024 * 1024, 256 * 1024, 19},
		{5 * 1024 * 1024, 5 * 1024 * 1024, 0},
		{5*1024*1024 - 1, 5 * 1024 * 1024, 0},
		{5 * 1024 * 1024, 5*1024*1024 - 1, 1},
		{0, 5 * 1024 * 1024, -1},
		{5 * 1024 * 1024, 1048576, 4},
	} {
		if readaheadPieces(case_.readaheadBytes, case_.pieceLength) != case_.readaheadPieces {
			t.Fatalf("case failed: %s", case_)
		}
	}
}
<commit_msg>Use testing config<commit_after>package torrent

import (
	"encoding/binary"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"testing"
	"time"

	"bitbucket.org/anacrolix/go.torrent/data/blob"

	"github.com/bradfitz/iter"

	"bitbucket.org/anacrolix/go.torrent/internal/testutil"
	"bitbucket.org/anacrolix/go.torrent/util"
	"bitbucket.org/anacrolix/utp"
	"github.com/anacrolix/libtorgo/bencode"
)

var TestingConfig = Config{
	ListenAddr:           ":0",
	NoDHT:                true,
	DisableTrackers:      true,
	NoDefaultBlocklist:   true,
	DisableMetainfoCache: true,
}

func TestClientDefault(t *testing.T) {
	cl, err := NewClient(&Config{
		NoDefaultBlocklist: true,
		ListenAddr:         ":0",
	})
	if err != nil {
		t.Fatal(err)
	}
	cl.Close()
}

func TestAddDropTorrent(t *testing.T) {
	cl, err := NewClient(&TestingConfig)
	if err != nil {
		t.Fatal(err)
	}
	defer cl.Close()
	dir, mi := testutil.GreetingTestTorrent()
	defer os.RemoveAll(dir)
	tt, err := cl.AddTorrent(mi)
	if err != nil {
		t.Fatal(err)
	}
	tt.Drop()
}

func TestAddTorrentNoSupportedTrackerSchemes(t *testing.T) {
	t.SkipNow()
}

func TestAddTorrentNoUsableURLs(t *testing.T) {
	t.SkipNow()
}

func TestAddPeersToUnknownTorrent(t *testing.T) {
	t.SkipNow()
}

func TestPieceHashSize(t *testing.T) {
	if pieceHash.Size() != 20 {
		t.FailNow()
	}
}

func TestTorrentInitialState(t *testing.T) {
	dir, mi := testutil.GreetingTestTorrent()
	defer os.RemoveAll(dir)
	tor, err := newTorrent(func() (ih InfoHash) {
		util.CopyExact(ih[:], mi.Info.Hash)
		return
	}(), nil, 0)
	if err != nil {
		t.Fatal(err)
	}
	err = tor.setMetadata(mi.Info.Info, mi.Info.Bytes, nil)
	if err != nil {
		t.Fatal(err)
	}
	if len(tor.Pieces) != 3 {
		t.Fatal("wrong number of pieces")
	}
	p := tor.Pieces[0]
	tor.pendAllChunkSpecs(0)
	if len(p.PendingChunkSpecs) != 1 {
		t.Fatalf("should only be 1 chunk: %v", p.PendingChunkSpecs)
	}
	// TODO: Set chunkSize to 2, to test odd/even silliness.
	if false {
		if _, ok := p.PendingChunkSpecs[chunkSpec{
			Length: 13,
		}]; !ok {
			t.Fatal("pending chunk spec is incorrect")
		}
	}
}

func TestUnmarshalPEXMsg(t *testing.T) {
	var m peerExchangeMessage
	if err := bencode.Unmarshal([]byte("d5:added12:\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0ce"), &m); err != nil {
		t.Fatal(err)
	}
	if len(m.Added) != 2 {
		t.FailNow()
	}
	if m.Added[0].Port != 0x506 {
		t.FailNow()
	}
}

func TestReducedDialTimeout(t *testing.T) {
	for _, _case := range []struct {
		Max             time.Duration
		HalfOpenLimit   int
		PendingPeers    int
		ExpectedReduced time.Duration
	}{
		{nominalDialTimeout, 40, 0, nominalDialTimeout},
		{nominalDialTimeout, 40, 1, nominalDialTimeout},
		{nominalDialTimeout, 40, 39, nominalDialTimeout},
		{nominalDialTimeout, 40, 40, nominalDialTimeout / 2},
		{nominalDialTimeout, 40, 80, nominalDialTimeout / 3},
		{nominalDialTimeout, 40, 4000, nominalDialTimeout / 101},
	} {
		reduced := reducedDialTimeout(_case.Max, _case.HalfOpenLimit, _case.PendingPeers)
		expected := _case.ExpectedReduced
		if expected < minDialTimeout {
			expected = minDialTimeout
		}
		if reduced != expected {
			t.Fatalf("expected %s, got %s", _case.ExpectedReduced, reduced)
		}
	}
}

func TestUTPRawConn(t *testing.T) {
	l, err := utp.NewSocket("")
	if err != nil {
		t.Fatal(err)
	}
	defer l.Close()
	go func() {
		for {
			_, err := l.Accept()
			if err != nil {
				break
			}
		}
	}()
	// Connect a UTP peer to see if the RawConn will still work.
	utpPeer, err := func() *utp.Socket {
		s, _ := utp.NewSocket("")
		return s
	}().Dial(fmt.Sprintf("localhost:%d", util.AddrPort(l.Addr())))
	if err != nil {
		t.Fatalf("error dialing utp listener: %s", err)
	}
	defer utpPeer.Close()
	peer, err := net.ListenPacket("udp", ":0")
	if err != nil {
		t.Fatal(err)
	}
	defer peer.Close()

	msgsReceived := 0
	const N = 5000 // How many messages to send.
	readerStopped := make(chan struct{})
	// The reader goroutine.
	go func() {
		defer close(readerStopped)
		b := make([]byte, 500)
		for i := 0; i < N; i++ {
			n, _, err := l.PacketConn().ReadFrom(b)
			if err != nil {
				t.Fatalf("error reading from raw conn: %s", err)
			}
			msgsReceived++
			var d int
			fmt.Sscan(string(b[:n]), &d)
			if d != i {
				log.Printf("got wrong number: expected %d, got %d", i, d)
			}
		}
	}()
	udpAddr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("localhost:%d", util.AddrPort(l.Addr())))
	if err != nil {
		t.Fatal(err)
	}
	for i := 0; i < N; i++ {
		_, err := peer.WriteTo([]byte(fmt.Sprintf("%d", i)), udpAddr)
		if err != nil {
			t.Fatal(err)
		}
		time.Sleep(time.Microsecond)
	}
	select {
	case <-readerStopped:
	case <-time.After(time.Second):
		t.Fatal("reader timed out")
	}
	if msgsReceived != N {
		t.Fatalf("messages received: %d", msgsReceived)
	}
}

func TestTwoClientsArbitraryPorts(t *testing.T) {
	for i := 0; i < 2; i++ {
		cl, err := NewClient(&Config{
			ListenAddr: ":0",
		})
		if err != nil {
			t.Fatal(err)
		}
		defer cl.Close()
	}
}

func TestAddDropManyTorrents(t *testing.T) {
	cl, _ := NewClient(&TestingConfig)
	defer cl.Close()
	var m Magnet
	for i := range iter.N(1000) {
		binary.PutVarint(m.InfoHash[:], int64(i))
		cl.AddMagnet(m.String())
	}
}

func TestClientTransfer(t *testing.T) {
	greetingTempDir, mi := testutil.GreetingTestTorrent()
	defer os.RemoveAll(greetingTempDir)
	cfg := TestingConfig
	cfg.DataDir = greetingTempDir
	seeder, err := NewClient(&cfg)
	if err != nil {
		t.Fatal(err)
	}
	defer seeder.Close()
	seeder.AddTorrent(mi)
	leecherDataDir, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(leecherDataDir)
	// cfg.TorrentDataOpener = func(info *metainfo.Info) (data.Data, error) {
	// 	return blob.TorrentData(info, leecherDataDir), nil
	// }
	cfg.TorrentDataOpener = blob.NewStore(leecherDataDir).OpenTorrent
	leecher, _ := NewClient(&cfg)
	defer leecher.Close()
	leecherGreeting, _ := leecher.AddTorrent(mi)
	leecherGreeting.AddPeers([]Peer{
		Peer{
			IP:   util.AddrIP(seeder.ListenAddr()),
			Port: util.AddrPort(seeder.ListenAddr()),
		},
	})
	_greeting, err := ioutil.ReadAll(io.NewSectionReader(leecherGreeting, 0, leecherGreeting.Length()))
	if err != nil {
		t.Fatal(err)
	}
	greeting := string(_greeting)
	if greeting != testutil.GreetingFileContents {
		t.Fatal(":(")
	}
}

func TestReadaheadPieces(t *testing.T) {
	for _, case_ := range []struct {
		readaheadBytes, pieceLength int64
		readaheadPieces             int
	}{
		{5 * 1024 * 1024, 256 * 1024, 19},
		{5 * 1024 * 1024, 5 * 1024 * 1024, 0},
		{5*1024*1024 - 1, 5 * 1024 * 1024, 0},
		{5 * 1024 * 1024, 5*1024*1024 - 1, 1},
		{0, 5 * 1024 * 1024, -1},
		{5 * 1024 * 1024, 1048576, 4},
	} {
		if readaheadPieces(case_.readaheadBytes, case_.pieceLength) != case_.readaheadPieces {
			t.Fatalf("case failed: %s", case_)
		}
	}
}
<|endoftext|>


<commit_before>package containerd

import (
	"bytes"
	"context"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"syscall"
	"testing"
	"time"

	"github.com/containerd/containerd/namespaces"
	"github.com/containerd/containerd/testutil"
)

const (
	defaultRoot = "/var/lib/containerd-test"
	testImage   = "docker.io/library/alpine:latest"
)

var (
	address      string
	noDaemon     bool
	supportsCriu bool
)

func init() {
	flag.StringVar(&address, "address", "/run/containerd-test/containerd.sock", "The address to the containerd socket for use in the tests")
	flag.BoolVar(&noDaemon, "no-daemon", false, "Do not start a dedicated daemon for the tests")
	flag.Parse()
}

func testContext() (context.Context, context.CancelFunc) {
	ctx, cancel := context.WithCancel(context.Background())
	ctx = namespaces.WithNamespace(ctx, "testing")
	return ctx, cancel
}

func TestMain(m *testing.M) {
	if testing.Short() {
		os.Exit(m.Run())
	}
	testutil.RequiresRootM()
	// check if criu is installed on the system
	_, err := exec.LookPath("criu")
	supportsCriu = err == nil

	var (
		cmd         *exec.Cmd
		buf         = bytes.NewBuffer(nil)
		ctx, cancel = testContext()
	)
	defer cancel()

	if !noDaemon {
		// setup a new containerd daemon if !testing.Short
		cmd = exec.Command("containerd",
			"--root", defaultRoot,
			"--address", address,
		)
		cmd.Stderr = buf
		if err := cmd.Start(); err != nil {
			cmd.Wait()
			fmt.Fprintf(os.Stderr, "%s: %s", err, buf.String())
			os.Exit(1)
		}
	}

	client, err := waitForDaemonStart(ctx, address)
	if err != nil {
		if cmd.Process != nil {
			cmd.Process.Kill()
		}
		cmd.Wait()
		fmt.Fprintf(os.Stderr, "%s: %s", err, buf.String())
		os.Exit(1)
	}

	// pull a seed image
	if _, err = client.Pull(ctx, testImage, WithPullUnpack); err != nil {
		cmd.Process.Signal(syscall.SIGTERM)
		cmd.Wait()
		fmt.Fprintf(os.Stderr, "%s: %s", err, buf.String())
		os.Exit(1)

	}
	if err := client.Close(); err != nil {
		fmt.Fprintln(os.Stderr, err)
	}

	// run the test
	status := m.Run()

	if !noDaemon {
		// tear down the daemon and resources created
		if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {
			fmt.Fprintln(os.Stderr, err)
		}
		if _, err := cmd.Process.Wait(); err != nil {
			fmt.Fprintln(os.Stderr, err)
		}
		if err := os.RemoveAll(defaultRoot); err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		// only print containerd logs if the test failed
		if status != 0 {
			fmt.Fprintln(os.Stderr, buf.String())
		}
	}
	os.Exit(status)
}

func waitForDaemonStart(ctx context.Context, address string) (*Client, error) {
	var (
		client  *Client
		serving bool
		err     error
	)

	for i := 0; i < 20; i++ {
		if client == nil {
			client, err = New(address)
		}
		if err == nil {
			serving, err = client.IsServing(ctx)
			if serving {
				return client, nil
			}
		}
		time.Sleep(100 * time.Millisecond)
	}
	return nil, fmt.Errorf("containerd did not start within 2s: %v", err)
}

func newClient(t testing.TB, address string, opts ...ClientOpt) (*Client, error) {
	if testing.Short() {
		t.Skip()
	}
	// testutil.RequiresRoot(t) is not needed here (already called in TestMain)
	return New(address, opts...)
}

func TestNewClient(t *testing.T) {
	client, err := newClient(t, address)
	if err != nil {
		t.Fatal(err)
	}
	if client == nil {
		t.Fatal("New() returned nil client")
	}
	if err := client.Close(); err != nil {
		t.Errorf("client closed returned errror %v", err)
	}
}

func TestImagePull(t *testing.T) {
	client, err := newClient(t, address)
	if err != nil {
		t.Fatal(err)
	}
	defer client.Close()

	ctx, cancel := testContext()
	defer cancel()
	_, err = client.Pull(ctx, testImage)
	if err != nil {
		t.Error(err)
		return
	}
}
<commit_msg>Prevent a data race in client_test.go on test failure<commit_after>package containerd

import (
	"bytes"
	"context"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"syscall"
	"testing"
	"time"

	"github.com/containerd/containerd/namespaces"
	"github.com/containerd/containerd/testutil"
)

const (
	defaultRoot = "/var/lib/containerd-test"
	testImage   = "docker.io/library/alpine:latest"
)

var (
	address      string
	noDaemon     bool
	supportsCriu bool
)

func init() {
	flag.StringVar(&address, "address", "/run/containerd-test/containerd.sock", "The address to the containerd socket for use in the tests")
	flag.BoolVar(&noDaemon, "no-daemon", false, "Do not start a dedicated daemon for the tests")
	flag.Parse()
}

func testContext() (context.Context, context.CancelFunc) {
	ctx, cancel := context.WithCancel(context.Background())
	ctx = namespaces.WithNamespace(ctx, "testing")
	return ctx, cancel
}

func TestMain(m *testing.M) {
	if testing.Short() {
		os.Exit(m.Run())
	}
	testutil.RequiresRootM()
	// check if criu is installed on the system
	_, err := exec.LookPath("criu")
	supportsCriu = err == nil

	var (
		cmd         *exec.Cmd
		buf         = bytes.NewBuffer(nil)
		ctx, cancel = testContext()
	)
	defer cancel()

	if !noDaemon {
		// setup a new containerd daemon if !testing.Short
		cmd = exec.Command("containerd",
			"--root", defaultRoot,
			"--address", address,
		)
		cmd.Stderr = buf
		if err := cmd.Start(); err != nil {
			cmd.Wait()
			fmt.Fprintf(os.Stderr, "%s: %s", err, buf.String())
			os.Exit(1)
		}
	}

	client, err := waitForDaemonStart(ctx, address)
	if err != nil {
		if cmd.Process != nil {
			cmd.Process.Kill()
		}
		cmd.Wait()
		fmt.Fprintf(os.Stderr, "%s: %s", err, buf.String())
		os.Exit(1)
	}

	// pull a seed image
	if _, err = client.Pull(ctx, testImage, WithPullUnpack); err != nil {
		cmd.Process.Signal(syscall.SIGTERM)
		cmd.Wait()
		fmt.Fprintf(os.Stderr, "%s: %s", err, buf.String())
		os.Exit(1)

	}
	if err := client.Close(); err != nil {
		fmt.Fprintln(os.Stderr, err)
	}

	// run the test
	status := m.Run()

	if !noDaemon {
		// tear down the daemon and resources created
		if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {
			fmt.Fprintln(os.Stderr, err)
		}
		if err := cmd.Wait(); err != nil {
			fmt.Fprintln(os.Stderr, err)
		}
		if err := os.RemoveAll(defaultRoot); err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		// only print containerd logs if the test failed
		if status != 0 {
			fmt.Fprintln(os.Stderr, buf.String())
		}
	}
	os.Exit(status)
}

func waitForDaemonStart(ctx context.Context, address string) (*Client, error) {
	var (
		client  *Client
		serving bool
		err     error
	)

	for i := 0; i < 20; i++ {
		if client == nil {
			client, err = New(address)
		}
		if err == nil {
			serving, err = client.IsServing(ctx)
			if serving {
				return client, nil
			}
		}
		time.Sleep(100 * time.Millisecond)
	}
	return nil, fmt.Errorf("containerd did not start within 2s: %v", err)
}

func newClient(t testing.TB, address string, opts ...ClientOpt) (*Client, error) {
	if testing.Short() {
		t.Skip()
	}
	// testutil.RequiresRoot(t) is not needed here (already called in TestMain)
	return New(address, opts...)
}

func TestNewClient(t *testing.T) {
	client, err := newClient(t, address)
	if err != nil {
		t.Fatal(err)
	}
	if client == nil {
		t.Fatal("New() returned nil client")
	}
	if err := client.Close(); err != nil {
		t.Errorf("client closed returned errror %v", err)
	}
}

func TestImagePull(t *testing.T) {
	client, err := newClient(t, address)
	if err != nil {
		t.Fatal(err)
	}
	defer client.Close()

	ctx, cancel := testContext()
	defer cancel()
	_, err = client.Pull(ctx, testImage)
	if err != nil {
		t.Error(err)
		return
	}
}
<|endoftext|>


<commit_before>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package upload

import (
	"sort"
	"strings"

	"github.com/webx-top/echo"

	"github.com/admpub/nging/application/registry/upload/table"
)

type FieldInfo struct {
	Key     string
	Name    string
	checker Checker
}

func (info *FieldInfo) AddChecker(checker Checker) *FieldInfo {
	if checker == nil {
		return info
	}
	if info.checker == nil {
		info.checker = checker
		return info
	}
	oldChecker := info.checker
	info.checker = func(ctx echo.Context, tbl table.TableInfoStorer) (subdir string, name string, err error) {
		subdir, name, err = oldChecker(ctx, tbl)
		if err != nil {
			return
		}
		subdir2, name2, err2 := checker(ctx, tbl)
		if err2 != nil {
			err = err2
		}
		if len(subdir2) > 0 {
			subdir = subdir2
		}
		if len(name2) > 0 {
			name = name2
		}
		return
	}
	return info
}

// SubdirInfo 子目录信息
type SubdirInfo struct {
	Allowed     bool
	Key         string
	Name        string
	NameEN      string
	Description string

	tableName  string
	fieldInfos map[string]*FieldInfo
	checker    Checker
}

func NewSubdirInfo(key, name string, checkers ...Checker) *SubdirInfo {
	var checker Checker
	if len(checkers) > 0 {
		checker = checkers[0]
	}
	return &SubdirInfo{
		Allowed: true,
		Key:     key,
		Name:    name,
		checker: checker,
	}
}

func (i *SubdirInfo) CopyFrom(other *SubdirInfo) *SubdirInfo {
	i.Allowed = other.Allowed
	if len(other.Key) > 0 {
		i.Key = other.Key
	}
	if len(other.Name) > 0 {
		i.Name = other.Name
	}
	if len(other.NameEN) > 0 {
		i.NameEN = other.NameEN
	}
	if len(other.Description) > 0 {
		i.Description = other.Description
	}
	if len(other.tableName) > 0 {
		i.tableName = other.tableName
	}
	if other.checker != nil {
		if i.checker != nil {
			oldChecker := i.checker
			i.checker = func(ctx echo.Context, tbl table.TableInfoStorer) (subdir string, name string, err error) {
				subdir, name, err = oldChecker(ctx, tbl)
				if err != nil {
					return
				}
				subdir2, name2, err2 := other.checker(ctx, tbl)
				if err2 != nil {
					err = err2
				}
				if len(subdir2) > 0 {
					subdir = subdir2
				}
				if len(name2) > 0 {
					name = name2
				}
				return
			}
		} else {
			i.checker = other.checker
		}
	}
	if len(other.fieldInfos) > 0 {
		if i.fieldInfos == nil {
			i.fieldInfos = make(map[string]*FieldInfo)
		}
		for k, in := range other.fieldInfos {
			i.fieldInfos[k] = in
		}
	}
	return i
}

func (i *SubdirInfo) String() string {
	if len(i.Name) > 0 {
		return i.Name
	}
	return i.Key
}

func (i *SubdirInfo) TableName() string {
	if len(i.tableName) == 0 {
		r := strings.SplitN(i.Key, `-`, 2)
		switch len(r) {
		case 2:
			i.SetFieldName(r[1])
			fallthrough
		case 1:
			i.tableName = r[0]
		}
	}
	return i.tableName
}

func (i *SubdirInfo) ValidFieldName(fieldName string) bool {
	if i.fieldInfos == nil {
		return false
	}
	_, ok := i.fieldInfos[fieldName]
	return ok
}

func (i *SubdirInfo) FieldNames() []string {
	fieldNames := make([]string, len(i.fieldInfos))
	if i.fieldInfos == nil {
		return fieldNames
	}
	var index int
	for fieldName := range i.fieldInfos {
		fieldNames[index] = fieldName
		index++
	}
	sort.Strings(fieldNames)
	return fieldNames
}

func (i *SubdirInfo) FieldInfos() []echo.KV {
	fieldNames := i.FieldNames()
	r := make([]echo.KV, len(fieldNames))
	for index, fieldName := range fieldNames {
		info := i.fieldInfos[fieldName]
		r[index] = echo.KV{K: fieldName, V: info.Name}
	}
	return r
}

func (i *SubdirInfo) SetTableName(tableName string) *SubdirInfo {
	i.tableName = tableName
	return i
}

func (i *SubdirInfo) SetTable(tableName string, fieldNames ...string) *SubdirInfo {
	i.tableName = tableName
	if len(fieldNames) > 0 {
		i.SetFieldName(fieldNames...)
	}
	return i
}

func (i *SubdirInfo) SetFieldName(fieldNames ...string) *SubdirInfo {
	i.fieldInfos = map[string]*FieldInfo{}
	for _, fieldName := range fieldNames {
		i.AddFieldName(fieldName)
	}
	return i
}

func (i *SubdirInfo) parseFieldInfo(field string) (fieldName string, fieldText string) {
	r := strings.SplitN(field, ":", 2)
	switch len(r) {
	case 2:
		fieldText = r[1]
		fallthrough
	case 1:
		fieldName = r[0]
	}
	return
}

func (i *SubdirInfo) AddFieldName(fieldName string, checkers ...Checker) *SubdirInfo {
	var (
		fieldText string
		checker   Checker
	)
	if len(checkers) > 0 {
		checker = checkers[0]
	}
	fieldName, fieldText = i.parseFieldInfo(fieldName)
	info, ok := i.fieldInfos[fieldName]
	if !ok {
		i.fieldInfos[fieldName] = &FieldInfo{
			Key:     fieldName,
			Name:    fieldText,
			checker: checker,
		}
	} else {
		if len(fieldText) > 0 {
			info.Name = fieldText
		}
		info.AddChecker(checker)
	}
	return i
}

func (i *SubdirInfo) GetNameEN() string {
	if len(i.NameEN) > 0 {
		return i.NameEN
	}
	return i.Name
}

func (i *SubdirInfo) SetChecker(checker Checker, fieldNames ...string) *SubdirInfo {
	if len(fieldNames) > 0 {
		if i.fieldInfos == nil {
			i.fieldInfos = make(map[string]*FieldInfo)
		}
		info, ok := i.fieldInfos[fieldNames[0]]
		if !ok {
			panic(`not found:` + fieldNames[0])
		}
		info.AddChecker(checker)
		return i
	}
	i.checker = checker
	return i
}

func (i *SubdirInfo) Checker() Checker {
	return i.checker
}

func (i *SubdirInfo) MustChecker() Checker {
	if i.checker != nil {
		return i.checker
	}
	return func(ctx echo.Context, tab table.TableInfoStorer) (subdir string, name string, err error) {
		tab.SetTableName(i.TableName())
		if !i.ValidFieldName(tab.FieldName()) {
			err = table.ErrInvalidFieldName
		}
		checker := i.fieldInfos[tab.FieldName()].checker
		if checker == nil {
			checker = DefaultChecker
		}
		subdir, name, err = checker(ctx, tab)
		return
	}
}

func (i *SubdirInfo) SetAllowed(allowed bool) *SubdirInfo {
	i.Allowed = allowed
	return i
}

func (i *SubdirInfo) SetName(name string) *SubdirInfo {
	i.Name = name
	return i
}

func (i *SubdirInfo) SetNameEN(nameEN string) *SubdirInfo {
	i.NameEN = nameEN
	return i
}

func (i *SubdirInfo) SetDescription(description string) *SubdirInfo {
	i.Description = description
	return i
}
<commit_msg>update<commit_after>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package upload

import (
	"sort"
	"strings"

	"github.com/webx-top/echo"

	"github.com/admpub/nging/application/registry/upload/table"
)

type FieldInfo struct {
	Key     string
	Name    string
	checker Checker
}

func (info *FieldInfo) AddChecker(checker Checker) *FieldInfo {
	if checker == nil {
		return info
	}
	if info.checker == nil {
		info.checker = checker
		return info
	}
	oldChecker := info.checker
	info.checker = func(ctx echo.Context, tbl table.TableInfoStorer) (subdir string, name string, err error) {
		subdir, name, err = oldChecker(ctx, tbl)
		if err != nil {
			return
		}
		subdir2, name2, err2 := checker(ctx, tbl)
		if err2 != nil {
			err = err2
		}
		if len(subdir2) > 0 {
			subdir = subdir2
		}
		if len(name2) > 0 {
			name = name2
		}
		return
	}
	return info
}

// SubdirInfo 子目录信息
type SubdirInfo struct {
	Allowed     bool
	Key         string
	Name        string
	NameEN      string
	Description string

	tableName  string
	fieldInfos map[string]*FieldInfo
	checker    Checker
}

func NewSubdirInfo(key, name string, checkers ...Checker) *SubdirInfo {
	var checker Checker
	if len(checkers) > 0 {
		checker = checkers[0]
	}
	return &SubdirInfo{
		Allowed: true,
		Key:     key,
		Name:    name,
		checker: checker,
	}
}

func (i *SubdirInfo) CopyFrom(other *SubdirInfo) *SubdirInfo {
	i.Allowed = other.Allowed
	if len(other.Key) > 0 {
		i.Key = other.Key
	}
	if len(other.Name) > 0 {
		i.Name = other.Name
	}
	if len(other.NameEN) > 0 {
		i.NameEN = other.NameEN
	}
	if len(other.Description) > 0 {
		i.Description = other.Description
	}
	if len(other.tableName) > 0 {
		i.tableName = other.tableName
	}
	if other.checker != nil {
		if i.checker != nil {
			oldChecker := i.checker
			i.checker = func(ctx echo.Context, tbl table.TableInfoStorer) (subdir string, name string, err error) {
				subdir, name, err = oldChecker(ctx, tbl)
				if err != nil {
					return
				}
				subdir2, name2, err2 := other.checker(ctx, tbl)
				if err2 != nil {
					err = err2
				}
				if len(subdir2) > 0 {
					subdir = subdir2
				}
				if len(name2) > 0 {
					name = name2
				}
				return
			}
		} else {
			i.checker = other.checker
		}
	}
	if len(other.fieldInfos) > 0 {
		if i.fieldInfos == nil {
			i.fieldInfos = make(map[string]*FieldInfo)
		}
		for k, in := range other.fieldInfos {
			i.fieldInfos[k] = in
		}
	}
	return i
}

func (i *SubdirInfo) String() string {
	if len(i.Name) > 0 {
		return i.Name
	}
	return i.Key
}

func (i *SubdirInfo) TableName() string {
	if len(i.tableName) == 0 {
		r := strings.SplitN(i.Key, `-`, 2)
		switch len(r) {
		case 2:
			i.SetFieldName(r[1])
			fallthrough
		case 1:
			i.tableName = r[0]
		}
	}
	return i.tableName
}

func (i *SubdirInfo) ValidFieldName(fieldName string) bool {
	if i.fieldInfos == nil {
		return false
	}
	_, ok := i.fieldInfos[fieldName]
	return ok
}

func (i *SubdirInfo) FieldNames() []string {
	fieldNames := make([]string, len(i.fieldInfos))
	if i.fieldInfos == nil {
		return fieldNames
	}
	var index int
	for fieldName := range i.fieldInfos {
		fieldNames[index] = fieldName
		index++
	}
	sort.Strings(fieldNames)
	return fieldNames
}

func (i *SubdirInfo) FieldInfos() []echo.KV {
	fieldNames := i.FieldNames()
	r := make([]echo.KV, len(fieldNames))
	for index, fieldName := range fieldNames {
		info := i.fieldInfos[fieldName]
		r[index] = echo.KV{K: fieldName, V: info.Name}
	}
	return r
}

func (i *SubdirInfo) SetTableName(tableName string) *SubdirInfo {
	i.tableName = tableName
	return i
}

func (i *SubdirInfo) SetTable(tableName string, fieldNames ...string) *SubdirInfo {
	i.tableName = tableName
	if len(fieldNames) > 0 {
		i.SetFieldName(fieldNames...)
	}
	return i
}

func (i *SubdirInfo) SetFieldName(fieldNames ...string) *SubdirInfo {
	i.fieldInfos = map[string]*FieldInfo{}
	for _, fieldName := range fieldNames {
		i.AddFieldName(fieldName)
	}
	return i
}

func (i *SubdirInfo) parseFieldInfo(field string) (fieldName string, fieldText string) {
	r := strings.SplitN(field, ":", 2)
	switch len(r) {
	case 2:
		fieldText = r[1]
		fallthrough
	case 1:
		fieldName = r[0]
	}
	return
}

func (i *SubdirInfo) AddFieldName(fieldName string, checkers ...Checker) *SubdirInfo {
	var (
		fieldText string
		checker   Checker
	)
	if len(checkers) > 0 {
		checker = checkers[0]
	}
	fieldName, fieldText = i.parseFieldInfo(fieldName)
	info, ok := i.fieldInfos[fieldName]
	if !ok {
		i.fieldInfos[fieldName] = &FieldInfo{
			Key:     fieldName,
			Name:    fieldText,
			checker: checker,
		}
	} else {
		if len(fieldText) > 0 {
			info.Name = fieldText
		}
		info.AddChecker(checker)
	}
	return i
}

func (i *SubdirInfo) GetNameEN() string {
	if len(i.NameEN) > 0 {
		return i.NameEN
	}
	return i.Name
}

func (i *SubdirInfo) SetChecker(checker Checker, fieldNames ...string) *SubdirInfo {
	if len(fieldNames) > 0 {
		if i.fieldInfos == nil {
			i.fieldInfos = make(map[string]*FieldInfo)
		}
		info, ok := i.fieldInfos[fieldNames[0]]
		if !ok {
			panic(`not found: ` + i.Key + `.` + fieldNames[0])
		}
		info.AddChecker(checker)
		return i
	}
	i.checker = checker
	return i
}

func (i *SubdirInfo) Checker() Checker {
	return i.checker
}

func (i *SubdirInfo) MustChecker() Checker {
	if i.checker != nil {
		return i.checker
	}
	return func(ctx echo.Context, tab table.TableInfoStorer) (subdir string, name string, err error) {
		tab.SetTableName(i.TableName())
		if !i.ValidFieldName(tab.FieldName()) {
			err = table.ErrInvalidFieldName
		}
		checker := i.fieldInfos[tab.FieldName()].checker
		if checker == nil {
			checker = DefaultChecker
		}
		subdir, name, err = checker(ctx, tab)
		return
	}
}

func (i *SubdirInfo) SetAllowed(allowed bool) *SubdirInfo {
	i.Allowed = allowed
	return i
}

func (i *SubdirInfo) SetName(name string) *SubdirInfo {
	i.Name = name
	return i
}

func (i *SubdirInfo) SetNameEN(nameEN string) *SubdirInfo {
	i.NameEN = nameEN
	return i
}

func (i *SubdirInfo) SetDescription(description string) *SubdirInfo {
	i.Description = description
	return i
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/kr/env"
	"github.com/kr/secureheader"
	"golang.org/x/net/context"

	"chain/api"
	"chain/api/appdb"
	"chain/database/pg"
	"chain/metrics"
	"chain/metrics/librato"
	chainhttp "chain/net/http"
	"chain/net/http/gzip"
)

var (
	// config vars
	tlsCrt       = env.String("TLSCRT", "")
	tlsKey       = env.String("TLSKEY", "")
	listenAddr   = env.String("LISTEN", ":8080")
	dbURL        = env.String("DB_URL", "postgres:///api?sslmode=disable")
	maxDBConns   = 100
	maxIdleConns = 100
	samplePer    = env.Duration("SAMPLEPER", 10*time.Second)
	nouserSecret = env.String("NOUSER_SECRET", "")

	db       *sql.DB
	buildTag = "dev"
)

func main() {
	librato.URL = env.URL("LIBRATO_URL", "")
	librato.Prefix = "chain.api."

	sql.Register("schemadb", pg.SchemaDriver(buildTag))
	log.SetPrefix("api-" + buildTag + ": ")
	log.SetFlags(log.Lshortfile)
	env.Parse()

	if librato.URL.Host != "" {
		librato.Source, _ = os.Hostname()
		go librato.SampleMetrics(*samplePer)
	} else {
		log.Println("no metrics; set LIBRATO_URL for prod")
	}

	var err error
	db, err = sql.Open("schemadb", *dbURL)
	if err != nil {
		log.Fatal(err)
	}
	db.SetMaxOpenConns(maxDBConns)
	db.SetMaxIdleConns(maxIdleConns)
	appdb.Init(db)

	var h chainhttp.Handler
	h = api.Handler(*nouserSecret)
	h = metrics.Handler{Handler: h}
	h = gzip.Handler{Handler: h}

	bg := context.Background()
	bg = pg.NewContext(bg, db)
	http.Handle("/", chainhttp.ContextHandler{Context: bg, Handler: h})
	http.HandleFunc("/health", func(http.ResponseWriter, *http.Request) {})

	secureheader.DefaultConfig.PermitClearLoopback = true

	if *tlsCrt != "" {
		err = chainhttp.ListenAndServeTLS(*listenAddr, *tlsCrt, *tlsKey, secureheader.DefaultConfig)
	} else {
		err = http.ListenAndServe(*listenAddr, secureheader.DefaultConfig)
	}
	if err != nil {
		log.Fatalln("ListenAndServe:", err)
	}
}
<commit_msg>cmd/api: these aren't config vars<commit_after>package main

import (
	"database/sql"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/kr/env"
	"github.com/kr/secureheader"
	"golang.org/x/net/context"

	"chain/api"
	"chain/api/appdb"
	"chain/database/pg"
	"chain/metrics"
	"chain/metrics/librato"
	chainhttp "chain/net/http"
	"chain/net/http/gzip"
)

var (
	// config vars
	tlsCrt       = env.String("TLSCRT", "")
	tlsKey       = env.String("TLSKEY", "")
	listenAddr   = env.String("LISTEN", ":8080")
	dbURL        = env.String("DB_URL", "postgres:///api?sslmode=disable")
	samplePer    = env.Duration("SAMPLEPER", 10*time.Second)
	nouserSecret = env.String("NOUSER_SECRET", "")

	db       *sql.DB
	buildTag = "dev"
)

func main() {
	librato.URL = env.URL("LIBRATO_URL", "")
	librato.Prefix = "chain.api."

	sql.Register("schemadb", pg.SchemaDriver(buildTag))
	log.SetPrefix("api-" + buildTag + ": ")
	log.SetFlags(log.Lshortfile)
	env.Parse()

	if librato.URL.Host != "" {
		librato.Source, _ = os.Hostname()
		go librato.SampleMetrics(*samplePer)
	} else {
		log.Println("no metrics; set LIBRATO_URL for prod")
	}

	var err error
	db, err = sql.Open("schemadb", *dbURL)
	if err != nil {
		log.Fatal(err)
	}
	db.SetMaxOpenConns(100)
	db.SetMaxIdleConns(100)
	appdb.Init(db)

	var h chainhttp.Handler
	h = api.Handler(*nouserSecret)
	h = metrics.Handler{Handler: h}
	h = gzip.Handler{Handler: h}

	bg := context.Background()
	bg = pg.NewContext(bg, db)
	http.Handle("/", chainhttp.ContextHandler{Context: bg, Handler: h})
	http.HandleFunc("/health", func(http.ResponseWriter, *http.Request) {})

	secureheader.DefaultConfig.PermitClearLoopback = true

	if *tlsCrt != "" {
		err = chainhttp.ListenAndServeTLS(*listenAddr, *tlsCrt, *tlsKey, secureheader.DefaultConfig)
	} else {
		err = http.ListenAndServe(*listenAddr, secureheader.DefaultConfig)
	}
	if err != nil {
		log.Fatalln("ListenAndServe:", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"archive/zip"
	"bytes"
	"compress/flate"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/mholt/archiver"
	"github.com/nwaples/rardecode"
)

var (
	compressionLevel       int
	overwriteExisting      bool
	mkdirAll               bool
	selectiveCompression   bool
	implicitTopLevelFolder bool
	continueOnError        bool
)

func init() {
	flag.IntVar(&compressionLevel, "level", flate.DefaultCompression, "Compression level")
	flag.BoolVar(&overwriteExisting, "overwrite", false, "Overwrite existing files")
	flag.BoolVar(&mkdirAll, "mkdirs", false, "Make all necessary directories")
	flag.BoolVar(&selectiveCompression, "smart", true, "Only compress files which are not already compressed (zip only)")
	flag.BoolVar(&implicitTopLevelFolder, "folder-safe", true, "If an archive does not have a single top-level folder, create one implicitly")
	flag.BoolVar(&continueOnError, "allow-errors", true, "Log errors and continue processing")
}

func main() {
	if len(os.Args) >= 2 &&
		(os.Args[1] == "-h" || os.Args[1] == "--help" || os.Args[1] == "help") {
		fmt.Println(usageString())
		os.Exit(0)
	}
	if len(os.Args) < 3 {
		fatal(usageString())
	}
	flag.Parse()

	subcommand := flag.Arg(0)

	// get the format we're working with
	iface, err := getFormat(subcommand)
	if err != nil {
		fatal(err)
	}

	// run the desired command
	switch subcommand {
	case "archive":
		a, ok := iface.(archiver.Archiver)
		if !ok {
			fatalf("the archive command does not support the %s format", iface)
		}
		err = a.Archive(flag.Args()[2:], flag.Arg(1))

	case "unarchive":
		a, ok := iface.(archiver.Unarchiver)
		if !ok {
			fatalf("the unarchive command does not support the %s format", iface)
		}
		err = a.Unarchive(flag.Arg(1), flag.Arg(2))

	case "extract":
		e, ok := iface.(archiver.Extractor)
		if !ok {
			fatalf("the extract command does not support the %s format", iface)
		}
		err = e.Extract(flag.Arg(1), flag.Arg(2), flag.Arg(3))

	case "ls":
		w, ok := iface.(archiver.Walker)
		if !ok {
			fatalf("the ls command does not support the %s format", iface)
		}

		var count int
		err = w.Walk(flag.Arg(1), func(f archiver.File) error {
			count++
			switch h := f.Header.(type) {
			case *zip.FileHeader:
				fmt.Printf("%s\t%d\t%d\t%s\t%s\n",
					f.Mode(),
					h.Method,
					f.Size(),
					f.ModTime(),
					h.Name,
				)
			case *tar.Header:
				fmt.Printf("%s\t%s\t%s\t%d\t%s\t%s\n",
					f.Mode(),
					h.Uname,
					h.Gname,
					f.Size(),
					f.ModTime(),
					h.Name,
				)

			case *rardecode.FileHeader:
				fmt.Printf("%s\t%d\t%d\t%s\t%s\n",
					f.Mode(),
					int(h.HostOS),
					f.Size(),
					f.ModTime(),
					h.Name,
				)

			default:
				fmt.Printf("%s\t%d\t%s\t?/%s\n",
					f.Mode(),
					f.Size(),
					f.ModTime(),
					f.Name(), // we don't know full path from this
				)
			}
			return nil
		})

		fmt.Printf("total %d", count)

	case "compress":
		c, ok := iface.(archiver.Compressor)
		if !ok {
			fatalf("the compress command does not support the %s format", iface)
		}
		fc := archiver.FileCompressor{Compressor: c}

		in := flag.Arg(1)
		out := flag.Arg(2)

		var deleteWhenDone bool
		if cs, ok := c.(fmt.Stringer); ok && out == cs.String() {
			out = in + "." + out
			deleteWhenDone = true
		}

		err = fc.CompressFile(in, out)
		if err == nil && deleteWhenDone {
			err = os.Remove(in)
		}

	case "decompress":
		c, ok := iface.(archiver.Decompressor)
		if !ok {
			fatalf("the compress command does not support the %s format", iface)
		}
		fc := archiver.FileCompressor{Decompressor: c}

		in := flag.Arg(1)
		out := flag.Arg(2)

		var deleteWhenDone bool
		if cs, ok := c.(fmt.Stringer); ok && out == "" {
			out = strings.TrimSuffix(in, "."+cs.String())
			deleteWhenDone = true
		}

		err = fc.DecompressFile(in, out)
		if err == nil && deleteWhenDone {
			err = os.Remove(in)
		}

	default:
		fatalf("unrecognized command: %s", flag.Arg(0))
	}
	if err != nil {
		fatal(err)
	}
}

func getFormat(subcommand string) (interface{}, error) {
	formatPos := 1
	if subcommand == "compress" {
		formatPos = 2
	}

	// figure out which file format we're working with
	var ext string
	archiveName := flag.Arg(formatPos)
	for _, format := range supportedFormats {
		// match by extension, or, in the case of 'compress',
		// check the format without the leading dot; it allows
		// a shortcut to specify a format while replacing
		// the original file on disk
		if strings.HasSuffix(archiveName, format) ||
			(subcommand == "compress" &&
				archiveName == strings.TrimPrefix(format, ".")) {
			ext = format
			break
		}
	}

	// configure an archiver
	var iface interface{}
	mytar := &archiver.Tar{
		OverwriteExisting:      overwriteExisting,
		MkdirAll:               mkdirAll,
		ImplicitTopLevelFolder: implicitTopLevelFolder,
		ContinueOnError:        continueOnError,
	}

	switch ext {
	case ".rar":
		iface = &archiver.Rar{
			OverwriteExisting:      overwriteExisting,
			MkdirAll:               mkdirAll,
			ImplicitTopLevelFolder: implicitTopLevelFolder,
			ContinueOnError:        continueOnError,
			Password:               os.Getenv("ARCHIVE_PASSWORD"),
		}

	case ".tar":
		iface = mytar

	case ".tbz2":
		fallthrough
	case ".tar.bz2":
		iface = &archiver.TarBz2{
			Tar: mytar,
		}

	case ".tgz":
		fallthrough
	case ".tar.gz":
		iface = &archiver.TarGz{
			Tar:              mytar,
			CompressionLevel: compressionLevel,
		}

	case ".tlz4":
		fallthrough
	case ".tar.lz4":
		iface = &archiver.TarLz4{
			Tar:              mytar,
			CompressionLevel: compressionLevel,
		}

	case ".tsz":
		fallthrough
	case ".tar.sz":
		iface = &archiver.TarSz{
			Tar: mytar,
		}

	case ".txz":
		fallthrough
	case ".tar.xz":
		iface = &archiver.TarXz{
			Tar: mytar,
		}

	case ".zip":
		iface = &archiver.Zip{
			CompressionLevel:       compressionLevel,
			OverwriteExisting:      overwriteExisting,
			MkdirAll:               mkdirAll,
			SelectiveCompression:   selectiveCompression,
			ImplicitTopLevelFolder: implicitTopLevelFolder,
			ContinueOnError:        continueOnError,
		}

	case ".gz":
		iface = &archiver.Gz{
			CompressionLevel: compressionLevel,
		}

	case ".bz2":
		iface = &archiver.Bz2{
			CompressionLevel: compressionLevel,
		}

	case ".lz4":
		iface = &archiver.Lz4{
			CompressionLevel: compressionLevel,
		}

	case ".sz":
		iface = &archiver.Snappy{}

	case ".xz":
		iface = &archiver.Xz{}

	default:
		archiveExt := filepath.Ext(archiveName)
		if archiveExt == "" {
			return nil, fmt.Errorf("format missing (use file extension to specify archive/compression format)")
		}
		return nil, fmt.Errorf("unsupported format '%s'", archiveExt)
	}

	return iface, nil
}

func fatal(v ...interface{}) {
	fmt.Fprintln(os.Stderr, v...)
	os.Exit(1)
}

func fatalf(s string, v ...interface{}) {
	fmt.Fprintf(os.Stderr, s+"\n", v...)
	os.Exit(1)
}

func usageString() string {
	buf := new(bytes.Buffer)
	buf.WriteString(usage)
	flag.CommandLine.SetOutput(buf)
	flag.CommandLine.PrintDefaults()
	return buf.String()
}

// supportedFormats is the list of recognized
// file extensions. They are in an ordered slice
// because ordering is important, since some
// extensions can be substrings of others.
var supportedFormats = []string{
	".tar.bz2",
	".tar.gz",
	".tar.lz4",
	".tar.sz",
	".tar.xz",
	".rar",
	".tar",
	".zip",
	".gz",
	".bz2",
	".lz4",
	".sz",
	".xz",
}

const usage = `Usage: arc {archive|unarchive|extract|ls|compress|decompress|help} [arguments...]
  archive
    Create a new archive file. List the files/folders
    to include in the archive; at least one required.
  unarchive
    Extract an archive file. Provide the archive to
    open and the destination folder to extract into.
  extract
    Extract a single file or folder (recursively) from
    an archive. First argument is the source archive,
    second is the file to extract (exact path within the
    archive is required), and third is destination.
  ls
    List the contents of the archive.
  compress
    Compresses a file, destination optional.
  decompress
    Decompresses a file, destination optional.
  help
    Display this help text. Also -h or --help.

  SPECIFYING THE ARCHIVE FORMAT
    The format of the archive is determined by its
    file extension. Supported extensions:
      .zip
      .tar
      .tar.gz
      .tgz
      .tar.bz2
      .tbz2
      .tar.xz
      .txz
      .tar.lz4
      .tlz4
      .tar.sz
      .tsz
      .rar (open only)
      .bz2
      .gz
      .lz4
      .sz
      .xz

  (DE)COMPRESSING SINGLE FILES
    Some formats are compression-only, and can be used
    with the compress and decompress commands on a
    single file; they do not bundle multiple files.

    To replace a file when compressing, specify the
    source file name for the first argument, and the
    compression format (without leading dot) for the
    second argument. To replace a file when decompressing,
    specify only the source file and no destination.

  PASSWORD-PROTECTED RAR FILES
    Export the ARCHIVE_PASSWORD environment variable
    to be able to open password-protected rar archives.

  GLOBAL FLAG REFERENCE
    The following global flags may be used before the
    sub-command (some flags are format-specific):

`
<commit_msg>cmd/ls: Fix zip file listing<commit_after>package main

import (
	"archive/tar"
	"archive/zip"
	"bytes"
	"compress/flate"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/mholt/archiver"
	"github.com/nwaples/rardecode"
)

var (
	compressionLevel       int
	overwriteExisting      bool
	mkdirAll               bool
	selectiveCompression   bool
	implicitTopLevelFolder bool
	continueOnError        bool
)

func init() {
	flag.IntVar(&compressionLevel, "level", flate.DefaultCompression, "Compression level")
	flag.BoolVar(&overwriteExisting, "overwrite", false, "Overwrite existing files")
	flag.BoolVar(&mkdirAll, "mkdirs", false, "Make all necessary directories")
	flag.BoolVar(&selectiveCompression, "smart", true, "Only compress files which are not already compressed (zip only)")
	flag.BoolVar(&implicitTopLevelFolder, "folder-safe", true, "If an archive does not have a single top-level folder, create one implicitly")
	flag.BoolVar(&continueOnError, "allow-errors", true, "Log errors and continue processing")
}

func main() {
	if len(os.Args) >= 2 &&
		(os.Args[1] == "-h" || os.Args[1] == "--help" || os.Args[1] == "help") {
		fmt.Println(usageString())
		os.Exit(0)
	}
	if len(os.Args) < 3 {
		fatal(usageString())
	}
	flag.Parse()

	subcommand := flag.Arg(0)

	// get the format we're working with
	iface, err := getFormat(subcommand)
	if err != nil {
		fatal(err)
	}

	// run the desired command
	switch subcommand {
	case "archive":
		a, ok := iface.(archiver.Archiver)
		if !ok {
			fatalf("the archive command does not support the %s format", iface)
		}
		err = a.Archive(flag.Args()[2:], flag.Arg(1))

	case "unarchive":
		a, ok := iface.(archiver.Unarchiver)
		if !ok {
			fatalf("the unarchive command does not support the %s format", iface)
		}
		err = a.Unarchive(flag.Arg(1), flag.Arg(2))

	case "extract":
		e, ok := iface.(archiver.Extractor)
		if !ok {
			fatalf("the extract command does not support the %s format", iface)
		}
		err = e.Extract(flag.Arg(1), flag.Arg(2), flag.Arg(3))

	case "ls":
		w, ok := iface.(archiver.Walker)
		if !ok {
			fatalf("the ls command does not support the %s format", iface)
		}

		var count int
		err = w.Walk(flag.Arg(1), func(f archiver.File) error {
			count++
			switch h := f.Header.(type) {
			case zip.FileHeader:
				fmt.Printf("%s\t%d\t%d\t%s\t%s\n",
					f.Mode(),
					h.Method,
					f.Size(),
					f.ModTime(),
					h.Name,
				)
			case *tar.Header:
				fmt.Printf("%s\t%s\t%s\t%d\t%s\t%s\n",
					f.Mode(),
					h.Uname,
					h.Gname,
					f.Size(),
					f.ModTime(),
					h.Name,
				)

			case *rardecode.FileHeader:
				fmt.Printf("%s\t%d\t%d\t%s\t%s\n",
					f.Mode(),
					int(h.HostOS),
					f.Size(),
					f.ModTime(),
					h.Name,
				)

			default:
				fmt.Printf("%s\t%d\t%s\t?/%s\n",
					f.Mode(),
					f.Size(),
					f.ModTime(),
					f.Name(), // we don't know full path from this
				)
			}
			return nil
		})

		fmt.Printf("total %d", count)

	case "compress":
		c, ok := iface.(archiver.Compressor)
		if !ok {
			fatalf("the compress command does not support the %s format", iface)
		}
		fc := archiver.FileCompressor{Compressor: c}

		in := flag.Arg(1)
		out := flag.Arg(2)

		var deleteWhenDone bool
		if cs, ok := c.(fmt.Stringer); ok && out == cs.String() {
			out = in + "." + out
			deleteWhenDone = true
		}

		err = fc.CompressFile(in, out)
		if err == nil && deleteWhenDone {
			err = os.Remove(in)
		}

	case "decompress":
		c, ok := iface.(archiver.Decompressor)
		if !ok {
			fatalf("the compress command does not support the %s format", iface)
		}
		fc := archiver.FileCompressor{Decompressor: c}

		in := flag.Arg(1)
		out := flag.Arg(2)

		var deleteWhenDone bool
		if cs, ok := c.(fmt.Stringer); ok && out == "" {
			out = strings.TrimSuffix(in, "."+cs.String())
			deleteWhenDone = true
		}

		err = fc.DecompressFile(in, out)
		if err == nil && deleteWhenDone {
			err = os.Remove(in)
		}

	default:
		fatalf("unrecognized command: %s", flag.Arg(0))
	}
	if err != nil {
		fatal(err)
	}
}

func getFormat(subcommand string) (interface{}, error) {
	formatPos := 1
	if subcommand == "compress" {
		formatPos = 2
	}

	// figure out which file format we're working with
	var ext string
	archiveName := flag.Arg(formatPos)
	for _, format := range supportedFormats {
		// match by extension, or, in the case of 'compress',
		// check the format without the leading dot; it allows
		// a shortcut to specify a format while replacing
		// the original file on disk
		if strings.HasSuffix(archiveName, format) ||
			(subcommand == "compress" &&
				archiveName == strings.TrimPrefix(format, ".")) {
			ext = format
			break
		}
	}

	// configure an archiver
	var iface interface{}
	mytar := &archiver.Tar{
		OverwriteExisting:      overwriteExisting,
		MkdirAll:               mkdirAll,
		ImplicitTopLevelFolder: implicitTopLevelFolder,
		ContinueOnError:        continueOnError,
	}

	switch ext {
	case ".rar":
		iface = &archiver.Rar{
			OverwriteExisting:      overwriteExisting,
			MkdirAll:               mkdirAll,
			ImplicitTopLevelFolder: implicitTopLevelFolder,
			ContinueOnError:        continueOnError,
			Password:               os.Getenv("ARCHIVE_PASSWORD"),
		}

	case ".tar":
		iface = mytar

	case ".tbz2":
		fallthrough
	case ".tar.bz2":
		iface = &archiver.TarBz2{
			Tar: mytar,
		}

	case ".tgz":
		fallthrough
	case ".tar.gz":
		iface = &archiver.TarGz{
			Tar:              mytar,
			CompressionLevel: compressionLevel,
		}

	case ".tlz4":
		fallthrough
	case ".tar.lz4":
		iface = &archiver.TarLz4{
			Tar:              mytar,
			CompressionLevel: compressionLevel,
		}

	case ".tsz":
		fallthrough
	case ".tar.sz":
		iface = &archiver.TarSz{
			Tar: mytar,
		}

	case ".txz":
		fallthrough
	case ".tar.xz":
		iface = &archiver.TarXz{
			Tar: mytar,
		}

	case ".zip":
		iface = &archiver.Zip{
			CompressionLevel:       compressionLevel,
			OverwriteExisting:      overwriteExisting,
			MkdirAll:               mkdirAll,
			SelectiveCompression:   selectiveCompression,
			ImplicitTopLevelFolder: implicitTopLevelFolder,
			ContinueOnError:        continueOnError,
		}

	case ".gz":
		iface = &archiver.Gz{
			CompressionLevel: compressionLevel,
		}

	case ".bz2":
		iface = &archiver.Bz2{
			CompressionLevel: compressionLevel,
		}

	case ".lz4":
		iface = &archiver.Lz4{
			CompressionLevel: compressionLevel,
		}

	case ".sz":
		iface = &archiver.Snappy{}

	case ".xz":
		iface = &archiver.Xz{}

	default:
		archiveExt := filepath.Ext(archiveName)
		if archiveExt == "" {
			return nil, fmt.Errorf("format missing (use file extension to specify archive/compression format)")
		}
		return nil, fmt.Errorf("unsupported format '%s'", archiveExt)
	}

	return iface, nil
}

func fatal(v ...interface{}) {
	fmt.Fprintln(os.Stderr, v...)
	os.Exit(1)
}

func fatalf(s string, v ...interface{}) {
	fmt.Fprintf(os.Stderr, s+"\n", v...)
	os.Exit(1)
}

func usageString() string {
	buf := new(bytes.Buffer)
	buf.WriteString(usage)
	flag.CommandLine.SetOutput(buf)
	flag.CommandLine.PrintDefaults()
	return buf.String()
}

// supportedFormats is the list of recognized
// file extensions. They are in an ordered slice
// because ordering is important, since some
// extensions can be substrings of others.
var supportedFormats = []string{
	".tar.bz2",
	".tar.gz",
	".tar.lz4",
	".tar.sz",
	".tar.xz",
	".rar",
	".tar",
	".zip",
	".gz",
	".bz2",
	".lz4",
	".sz",
	".xz",
}

const usage = `Usage: arc {archive|unarchive|extract|ls|compress|decompress|help} [arguments...]
  archive
    Create a new archive file. List the files/folders
    to include in the archive; at least one required.
  unarchive
    Extract an archive file. Provide the archive to
    open and the destination folder to extract into.
  extract
    Extract a single file or folder (recursively) from
    an archive. First argument is the source archive,
    second is the file to extract (exact path within the
    archive is required), and third is destination.
  ls
    List the contents of the archive.
  compress
    Compresses a file, destination optional.
  decompress
    Decompresses a file, destination optional.
  help
    Display this help text. Also -h or --help.

  SPECIFYING THE ARCHIVE FORMAT
    The format of the archive is determined by its
    file extension. Supported extensions:
      .zip
      .tar
      .tar.gz
      .tgz
      .tar.bz2
      .tbz2
      .tar.xz
      .txz
      .tar.lz4
      .tlz4
      .tar.sz
      .tsz
      .rar (open only)
      .bz2
      .gz
      .lz4
      .sz
      .xz

  (DE)COMPRESSING SINGLE FILES
    Some formats are compression-only, and can be used
    with the compress and decompress commands on a
    single file; they do not bundle multiple files.

    To replace a file when compressing, specify the
    source file name for the first argument, and the
    compression format (without leading dot) for the
    second argument. To replace a file when decompressing,
    specify only the source file and no destination.

  PASSWORD-PROTECTED RAR FILES
    Export the ARCHIVE_PASSWORD environment variable
    to be able to open password-protected rar archives.

  GLOBAL FLAG REFERENCE
    The following global flags may be used before the
    sub-command (some flags are format-specific):

`
<|endoftext|>


<commit_before>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/google/gapid/core/os/file"
)

type gapicEnv struct {
	cfg      Config
	src      file.Path // Root of the GAPIC source tree
	javaExe  file.Path // java executable path
	javacExe file.Path // javac executable path
	jarExe   file.Path // jar executable path
	out      file.Path // build output
	platform string    // windows,linux,osx
}

func mkdirOrExit(p file.Path) file.Path {
	if err := file.Mkdir(p); err != nil {
		fmt.Fprintf(os.Stderr, "Could not make directory %v: %v", p.System(), err)
		os.Exit(1)
	}
	return p
}

func findOrExit(p file.Path) file.Path {
	if !p.Exists() {
		fmt.Fprintf(os.Stderr, "Could not find file %v", p.System())
		os.Exit(1)
	}
	return p
}

func gapic(ctx context.Context, cfg Config) *gapicEnv {
	if cfg.JavaHome.IsEmpty() {
		fmt.Println("Error building gapic: the JDK is required")
		os.Exit(1)
	}
	platform := runtime.GOOS
	switch runtime.GOOS {
	case "linux", "windows":
	case "darwin":
		platform = "osx"
	default:
		fmt.Println("Unsuported OS:", platform)
		os.Exit(1)
	}
	return &gapicEnv{
		cfg:      cfg,
		src:      srcRoot.Join("gapic"),
		javaExe:  findOrExit(cfg.JavaHome.Join("bin", "java"+hostExeExt)),
		javacExe: findOrExit(cfg.JavaHome.Join("bin", "javac"+hostExeExt)),
		jarExe:   findOrExit(cfg.JavaHome.Join("bin", "jar"+hostExeExt)),
		out:      mkdirOrExit(cfg.OutRoot.Join(string(cfg.Flavor), "java")),
		platform: platform,
	}
}

func (e *gapicEnv) build(ctx context.Context, options BuildOptions) {
	if options.DryRun {
		return
	}
	javaSrc := e.src.Join("src")
	javaOut := e.out.Join("gapic", e.platform)
	baseJar := e.out.Join("gapic-base-" + e.platform + ".jar")
	if e.needsBuilding(javaSrc.Join("main"), baseJar) ||
		e.needsBuilding(javaSrc.Join("rpclib"), baseJar) ||
		e.needsBuilding(javaSrc.Join("service"), baseJar) ||
		e.needsBuilding(javaSrc.Join("platform", e.platform), baseJar) {
		fmt.Println("Building gapic for", e.platform, "...")

		srcTxt := e.writeFile(e.out.Join("source-"+e.platform+".txt"), func(f *os.File) {
			nlWriter := func(p file.Path) { f.WriteString(p.System() + "\n") }
			e.findAllJavaFiles(javaSrc.Join("main"), nlWriter)
			e.findAllJavaFiles(javaSrc.Join("rpclib"), nlWriter)
			e.findAllJavaFiles(javaSrc.Join("service"), nlWriter)
			e.findAllJavaFiles(javaSrc.Join("platform", e.platform), nlWriter)
		})

		jarTxt := e.writeFile(e.out.Join("classpath-"+e.platform+".txt"), func(f *os.File) {
			listWriter := func(p file.Path) { f.WriteString(p.System() + string(os.PathListSeparator)) }
			e.findAllJars(e.src.Join("third_party"), listWriter)
		})

		run(ctx, e.out, e.javacExe, nil,
			"-d", e.mkdirCleanOrExit(javaOut).System(),
			"@"+srcTxt.System(),
			"-classpath", "@"+jarTxt.System(),
			"-encoding", "UTF-8",
			"-source", "1.8", "-target", "1.8")
		e.copyOrExit(e.src.Join("res"), javaOut)

		run(ctx, e.out, e.jarExe, nil,
			"-cf", baseJar.System(),
			"-C", javaOut.System(),
			".")
	}

	jarOut := e.out.Join("gapic-" + e.platform + ".jar")
	if e.needsBuilding(baseJar, jarOut) {
		fmt.Println("Building gapic JAR for", e.platform, "...")
		classOut := e.mkdirCleanOrExit(e.out.Join(e.platform))

		e.findAllJars(e.src.Join("third_party"), func(jarFile file.Path) {
			run(ctx, classOut, e.jarExe, nil, "-xf", jarFile.System())
		})
		run(ctx, classOut, e.jarExe, nil, "-xf", baseJar.System())

		// Kill the manifest and any signatures.
		os.RemoveAll(classOut.Join("META-INF", "MANIFEST.MF").System())
		for _, p := range classOut.Join("META-INF").Glob("*.RSA") {
			os.Remove(p.System())
		}
		for _, p := range classOut.Join("META-INF").Glob("*.SF") {
			os.Remove(p.System())
		}

		run(ctx, e.out, e.jarExe, nil,
			"-cef", "com.google.gapid.Main", jarOut.System(),
			"-C", classOut.System(),
			".")
	}
}

func (e *gapicEnv) run(ctx context.Context, options RunOptions, args ...string) {
	jar := e.out.Join("gapic-" + e.platform + ".jar")

	jargs := make([]string, 0)
	env := env(e.cfg)
	switch e.platform {
	case "linux":
		env.Set("SWT_GTK3", "0").
			Set("LIBOVERLAY_SCROLLBAR", "0")
	case "osx":
		jargs = append(args, "-XstartOnFirstThread")
	}
	jargs = append(jargs, "-jar", jar.System(), "-gapid", e.cfg.pkg().System())
	if options.WD.IsEmpty() {
		options.WD = file.Abs("")
	}
	run(ctx, options.WD, e.javaExe, env, append(jargs, args...)...)
}

func (e *gapicEnv) findAllJavaFiles(path file.Path, cb func(file.Path)) {
	err := path.Walk(func(p file.Path, err error) error {
		if err != nil {
			return nil
		}

		if !p.IsDir() && p.Ext() == ".java" {
			cb(p)
		}
		return nil
	})
	if err != nil {
		fmt.Println("Error looking for Java files:", err)
		os.Exit(1)
	}
}

func (e *gapicEnv) findAllJars(path file.Path, cb func(file.Path)) {
	err := path.Walk(func(p file.Path, err error) error {
		if err != nil {
			return err
		}
		if p.IsDir() {
			if p.Parent().Basename() == "platform" && p.Basename() != e.platform {
				return filepath.SkipDir
			}
		} else if p.Ext() == ".jar" && !strings.Contains(p.Basename(), "source") {
			cb(p)
		}
		return nil
	})
	if err != nil {
		fmt.Println("Error looking for JARs:", err)
		os.Exit(1)
	}
}

func (e *gapicEnv) mkdirCleanOrExit(p file.Path) file.Path {
	if p.Exists() {
		if err := os.RemoveAll(p.System()); err != nil {
			fmt.Println("Error building gapic:", err)
			os.Exit(1)
		}
	}
	return mkdirOrExit(p)
}

func (e *gapicEnv) writeFile(p file.Path, w func(*os.File)) file.Path {
	f, err := os.Create(p.System())
	if err != nil {
		fmt.Println("Failed to create file", p)
		os.Exit(1)
	}
	defer f.Close()
	w(f)
	return p
}

func (e *gapicEnv) copyOrExit(src, dest file.Path) {
	err := src.Walk(func(p file.Path, err error) error {
		if err != nil {
			return err
		}

		rel, err := p.RelativeTo(src)
		if err != nil {
			return err
		}

		if p.IsDir() {
			mkdirOrExit(dest.Join(rel))
		} else {
			s, err := os.Open(p.System())
			if err != nil {
				return err
			}
			defer s.Close()

			d, err := os.Create(dest.Join(rel).System())
			if err != nil {
				return err
			}
			defer d.Close()

			if _, err := io.Copy(d, s); err != nil {
				return err
			}
		}
		return err
	})
	if err != nil {
		fmt.Printf("Failed to copy %v to %v: %v\n", src, dest, err)
		os.Exit(1)
	}
}

var errNeedBuilding = errors.New("need building")

func (e *gapicEnv) needsBuilding(src, target file.Path) bool {
	targetInfo := target.Info()
	if targetInfo == nil {
		return true
	}

	err := src.Walk(func(p file.Path, err error) error {
		if err != nil {
			return err
		}

		if !p.IsDir() && p.Info().ModTime().After(targetInfo.ModTime()) {
			return errNeedBuilding
		}
		return nil
	})

	if err == errNeedBuilding {
		return true
	}

	if err != nil {
		fmt.Println("Failed to check for modified source files:", err)
		os.Exit(1)
	}
	return false
}
<commit_msg>do: Fix duplicated gapic args on macOS.<commit_after>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/google/gapid/core/os/file"
)

type gapicEnv struct {
	cfg      Config
	src      file.Path // Root of the GAPIC source tree
	javaExe  file.Path // java executable path
	javacExe file.Path // javac executable path
	jarExe   file.Path // jar executable path
	out      file.Path // build output
	platform string    // windows,linux,osx
}

func mkdirOrExit(p file.Path) file.Path {
	if err := file.Mkdir(p); err != nil {
		fmt.Fprintf(os.Stderr, "Could not make directory %v: %v", p.System(), err)
		os.Exit(1)
	}
	return p
}

func findOrExit(p file.Path) file.Path {
	if !p.Exists() {
		fmt.Fprintf(os.Stderr, "Could not find file %v", p.System())
		os.Exit(1)
	}
	return p
}

func gapic(ctx context.Context, cfg Config) *gapicEnv {
	if cfg.JavaHome.IsEmpty() {
		fmt.Println("Error building gapic: the JDK is required")
		os.Exit(1)
	}
	platform := runtime.GOOS
	switch runtime.GOOS {
	case "linux", "windows":
	case "darwin":
		platform = "osx"
	default:
		fmt.Println("Unsuported OS:", platform)
		os.Exit(1)
	}
	return &gapicEnv{
		cfg:      cfg,
		src:      srcRoot.Join("gapic"),
		javaExe:  findOrExit(cfg.JavaHome.Join("bin", "java"+hostExeExt)),
		javacExe: findOrExit(cfg.JavaHome.Join("bin", "javac"+hostExeExt)),
		jarExe:   findOrExit(cfg.JavaHome.Join("bin", "jar"+hostExeExt)),
		out:      mkdirOrExit(cfg.OutRoot.Join(string(cfg.Flavor), "java")),
		platform: platform,
	}
}

func (e *gapicEnv) build(ctx context.Context, options BuildOptions) {
	if options.DryRun {
		return
	}
	javaSrc := e.src.Join("src")
	javaOut := e.out.Join("gapic", e.platform)
	baseJar := e.out.Join("gapic-base-" + e.platform + ".jar")
	if e.needsBuilding(javaSrc.Join("main"), baseJar) ||
		e.needsBuilding(javaSrc.Join("rpclib"), baseJar) ||
		e.needsBuilding(javaSrc.Join("service"), baseJar) ||
		e.needsBuilding(javaSrc.Join("platform", e.platform), baseJar) {
		fmt.Println("Building gapic for", e.platform, "...")

		srcTxt := e.writeFile(e.out.Join("source-"+e.platform+".txt"), func(f *os.File) {
			nlWriter := func(p file.Path) { f.WriteString(p.System() + "\n") }
			e.findAllJavaFiles(javaSrc.Join("main"), nlWriter)
			e.findAllJavaFiles(javaSrc.Join("rpclib"), nlWriter)
			e.findAllJavaFiles(javaSrc.Join("service"), nlWriter)
			e.findAllJavaFiles(javaSrc.Join("platform", e.platform), nlWriter)
		})

		jarTxt := e.writeFile(e.out.Join("classpath-"+e.platform+".txt"), func(f *os.File) {
			listWriter := func(p file.Path) { f.WriteString(p.System() + string(os.PathListSeparator)) }
			e.findAllJars(e.src.Join("third_party"), listWriter)
		})

		run(ctx, e.out, e.javacExe, nil,
			"-d", e.mkdirCleanOrExit(javaOut).System(),
			"@"+srcTxt.System(),
			"-classpath", "@"+jarTxt.System(),
			"-encoding", "UTF-8",
			"-source", "1.8", "-target", "1.8")
		e.copyOrExit(e.src.Join("res"), javaOut)

		run(ctx, e.out, e.jarExe, nil,
			"-cf", baseJar.System(),
			"-C", javaOut.System(),
			".")
	}

	jarOut := e.out.Join("gapic-" + e.platform + ".jar")
	if e.needsBuilding(baseJar, jarOut) {
		fmt.Println("Building gapic JAR for", e.platform, "...")
		classOut := e.mkdirCleanOrExit(e.out.Join(e.platform))

		e.findAllJars(e.src.Join("third_party"), func(jarFile file.Path) {
			run(ctx, classOut, e.jarExe, nil, "-xf", jarFile.System())
		})
		run(ctx, classOut, e.jarExe, nil, "-xf", baseJar.System())

		// Kill the manifest and any signatures.
		os.RemoveAll(classOut.Join("META-INF", "MANIFEST.MF").System())
		for _, p := range classOut.Join("META-INF").Glob("*.RSA") {
			os.Remove(p.System())
		}
		for _, p := range classOut.Join("META-INF").Glob("*.SF") {
			os.Remove(p.System())
		}

		run(ctx, e.out, e.jarExe, nil,
			"-cef", "com.google.gapid.Main", jarOut.System(),
			"-C", classOut.System(),
			".")
	}
}

func (e *gapicEnv) run(ctx context.Context, options RunOptions, args ...string) {
	jar := e.out.Join("gapic-" + e.platform + ".jar")

	jargs := []string{}
	env := env(e.cfg)
	switch e.platform {
	case "linux":
		env.Set("SWT_GTK3", "0").
			Set("LIBOVERLAY_SCROLLBAR", "0")
	case "osx":
		jargs = []string{"-XstartOnFirstThread"}
	}
	jargs = append(jargs, "-jar", jar.System(), "-gapid", e.cfg.pkg().System())
	if options.WD.IsEmpty() {
		options.WD = file.Abs("")
	}
	run(ctx, options.WD, e.javaExe, env, append(jargs, args...)...)
}

func (e *gapicEnv) findAllJavaFiles(path file.Path, cb func(file.Path)) {
	err := path.Walk(func(p file.Path, err error) error {
		if err != nil {
			return nil
		}

		if !p.IsDir() && p.Ext() == ".java" {
			cb(p)
		}
		return nil
	})
	if err != nil {
		fmt.Println("Error looking for Java files:", err)
		os.Exit(1)
	}
}

func (e *gapicEnv) findAllJars(path file.Path, cb func(file.Path)) {
	err := path.Walk(func(p file.Path, err error) error {
		if err != nil {
			return err
		}
		if p.IsDir() {
			if p.Parent().Basename() == "platform" && p.Basename() != e.platform {
				return filepath.SkipDir
			}
		} else if p.Ext() == ".jar" && !strings.Contains(p.Basename(), "source") {
			cb(p)
		}
		return nil
	})
	if err != nil {
		fmt.Println("Error looking for JARs:", err)
		os.Exit(1)
	}
}

func (e *gapicEnv) mkdirCleanOrExit(p file.Path) file.Path {
	if p.Exists() {
		if err := os.RemoveAll(p.System()); err != nil {
			fmt.Println("Error building gapic:", err)
			os.Exit(1)
		}
	}
	return mkdirOrExit(p)
}

func (e *gapicEnv) writeFile(p file.Path, w func(*os.File)) file.Path {
	f, err := os.Create(p.System())
	if err != nil {
		fmt.Println("Failed to create file", p)
		os.Exit(1)
	}
	defer f.Close()
	w(f)
	return p
}

func (e *gapicEnv) copyOrExit(src, dest file.Path) {
	err := src.Walk(func(p file.Path, err error) error {
		if err != nil {
			return err
		}

		rel, err := p.RelativeTo(src)
		if err != nil {
			return err
		}

		if p.IsDir() {
			mkdirOrExit(dest.Join(rel))
		} else {
			s, err := os.Open(p.System())
			if err != nil {
				return err
			}
			defer s.Close()

			d, err := os.Create(dest.Join(rel).System())
			if err != nil {
				return err
			}
			defer d.Close()

			if _, err := io.Copy(d, s); err != nil {
				return err
			}
		}
		return err
	})
	if err != nil {
		fmt.Printf("Failed to copy %v to %v: %v\n", src, dest, err)
		os.Exit(1)
	}
}

var errNeedBuilding = errors.New("need building")

func (e *gapicEnv) needsBuilding(src, target file.Path) bool {
	targetInfo := target.Info()
	if targetInfo == nil {
		return true
	}

	err := src.Walk(func(p file.Path, err error) error {
		if err != nil {
			return err
		}

		if !p.IsDir() && p.Info().ModTime().After(targetInfo.ModTime()) {
			return errNeedBuilding
		}
		return nil
	})

	if err == errNeedBuilding {
		return true
	}

	if err != nil {
		fmt.Println("Failed to check for modified source files:", err)
		os.Exit(1)
	}
	return false
}
<|endoftext|>