<commit_before>// Run tests for all the remotes.  Run this with package names which
// need integration testing.
//
// See the `test` target in the Makefile.
package main

import (
	"flag"
	"go/build"
	"log"
	"os"
	"os/exec"
	"path"
	"regexp"
	"runtime"
	"strings"
	"time"

	_ "github.com/ncw/rclone/backend/all" // import all fs
	"github.com/ncw/rclone/fs"
	"github.com/ncw/rclone/fs/config"
	"github.com/ncw/rclone/fs/list"
	"github.com/ncw/rclone/fs/operations"
	"github.com/ncw/rclone/fstest"
)

type remoteConfig struct {
	Name     string
	SubDir   bool
	FastList bool
}

var (
	remotes = []remoteConfig{
		{
			Name:     "TestAmazonCloudDrive:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestB2:",
			SubDir:   true,
			FastList: true,
		},
		{
			Name:     "TestCryptDrive:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestCryptSwift:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestDrive:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestDropbox:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestGoogleCloudStorage:",
			SubDir:   true,
			FastList: true,
		},
		{
			Name:     "TestHubic:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestOneDrive:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestS3:",
			SubDir:   true,
			FastList: true,
		},
		{
			Name:     "TestSftp:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestSwift:",
			SubDir:   true,
			FastList: true,
		},
		{
			Name:     "TestYandex:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestFTP:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestBox:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestQingStor:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestAzureBlob:",
			SubDir:   true,
			FastList: true,
		},
		{
			Name:     "TestPcloud:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestWebdav:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestCache:",
			SubDir:   false,
			FastList: false,
		},
	}
	// Flags
	maxTries = flag.Int("maxtries", 5, "Number of times to try each test")
	runTests = flag.String("remotes", "", "Comma separated list of remotes to test, eg 'TestSwift:,TestS3'")
	clean    = flag.Bool("clean", false, "Instead of testing, clean all left over test directories")
	runOnly  = flag.String("run", "", "Run only those tests matching the regexp supplied")
	timeout  = flag.Duration("timeout", 30*time.Minute, "Maximum time to run each test for before giving up")
)

// test holds info about a running test
type test struct {
	pkg         string
	remote      string
	subdir      bool
	cmdLine     []string
	cmdString   string
	try         int
	err         error
	output      []byte
	failedTests []string
	runFlag     string
}

// newTest creates a new test
func newTest(pkg, remote string, subdir bool, fastlist bool) *test {
	binary := pkgBinary(pkg)
	t := &test{
		pkg:     pkg,
		remote:  remote,
		subdir:  subdir,
		cmdLine: []string{binary, "-test.timeout", (*timeout).String(), "-remote", remote},
		try:     1,
	}
	if *fstest.Verbose {
		t.cmdLine = append(t.cmdLine, "-test.v")
		fs.Config.LogLevel = fs.LogLevelDebug
	}
	if *runOnly != "" {
		t.cmdLine = append(t.cmdLine, "-test.run", *runOnly)
	}
	if subdir {
		t.cmdLine = append(t.cmdLine, "-subdir")
	}
	if fastlist {
		t.cmdLine = append(t.cmdLine, "-fast-list")
	}
	t.cmdString = toShell(t.cmdLine)
	return t
}

// dumpOutput prints the error output
func (t *test) dumpOutput() {
	log.Println("------------------------------------------------------------")
	log.Printf("---- %q ----", t.cmdString)
	log.Println(string(t.output))
	log.Println("------------------------------------------------------------")
}

var failRe = regexp.MustCompile(`(?m)^--- FAIL: (Test\w*) \(`)

// findFailures looks for all the tests which failed
func (t *test) findFailures() {
	oldFailedTests := t.failedTests
	t.failedTests = nil
	for _, matches := range failRe.FindAllSubmatch(t.output, -1) {
		t.failedTests = append(t.failedTests, string(matches[1]))
	}
	if len(t.failedTests) != 0 {
		t.runFlag = "^(" + strings.Join(t.failedTests, "|") + ")$"
	} else {
		t.runFlag = ""
	}
	if t.passed() && len(t.failedTests) != 0 {
		log.Printf("%q - Expecting no errors but got: %v", t.cmdString, t.failedTests)
		t.dumpOutput()
	} else if !t.passed() && len(t.failedTests) == 0 {
		log.Printf("%q - Expecting errors but got none: %v", t.cmdString, t.failedTests)
		t.dumpOutput()
		t.failedTests = oldFailedTests
	}
}

// nextCmdLine returns the next command line
func (t *test) nextCmdLine() []string {
	cmdLine := t.cmdLine[:]
	if t.runFlag != "" {
		cmdLine = append(cmdLine, "-test.run", t.runFlag)
	}
	return cmdLine
}

// if matches then is definitely OK in the shell
var shellOK = regexp.MustCompile("^[A-Za-z0-9./_:-]+$")

// converts a argv style input into a shell command
func toShell(args []string) (result string) {
	for _, arg := range args {
		if result != "" {
			result += " "
		}
		if shellOK.MatchString(arg) {
			result += arg
		} else {
			result += "'" + arg + "'"
		}
	}
	return result
}

// trial runs a single test
func (t *test) trial() {
	cmdLine := t.nextCmdLine()
	cmdString := toShell(cmdLine)
	log.Printf("%q - Starting (try %d/%d)", cmdString, t.try, *maxTries)
	cmd := exec.Command(cmdLine[0], cmdLine[1:]...)
	start := time.Now()
	t.output, t.err = cmd.CombinedOutput()
	duration := time.Since(start)
	t.findFailures()
	if t.passed() {
		log.Printf("%q - Finished OK in %v (try %d/%d)", cmdString, duration, t.try, *maxTries)
	} else {
		log.Printf("%q - Finished ERROR in %v (try %d/%d): %v: Failed %v", cmdString, duration, t.try, *maxTries, t.err, t.failedTests)
	}
}

// cleanFs runs a single clean fs for left over directories
func (t *test) cleanFs() error {
	f, err := fs.NewFs(t.remote)
	if err != nil {
		return err
	}
	entries, err := list.DirSorted(f, true, "")
	if err != nil {
		return err
	}
	return entries.ForDirError(func(dir fs.Directory) error {
		remote := dir.Remote()
		if fstest.MatchTestRemote.MatchString(remote) {
			log.Printf("Purging %s%s", t.remote, remote)
			dir, err := fs.NewFs(t.remote + remote)
			if err != nil {
				return err
			}
			return operations.Purge(dir, "")
		}
		return nil
	})
}

// clean runs a single clean on a fs for left over directories
func (t *test) clean() {
	log.Printf("%q - Starting clean (try %d/%d)", t.remote, t.try, *maxTries)
	start := time.Now()
	t.err = t.cleanFs()
	if t.err != nil {
		log.Printf("%q - Failed to purge %v", t.remote, t.err)
	}
	duration := time.Since(start)
	if t.passed() {
		log.Printf("%q - Finished OK in %v (try %d/%d)", t.cmdString, duration, t.try, *maxTries)
	} else {
		log.Printf("%q - Finished ERROR in %v (try %d/%d): %v", t.cmdString, duration, t.try, *maxTries, t.err)
	}
}

// passed returns true if the test passed
func (t *test) passed() bool {
	return t.err == nil
}

// run runs all the trials for this test
func (t *test) run(result chan<- *test) {
	for t.try = 1; t.try <= *maxTries; t.try++ {
		if *clean {
			if !t.subdir {
				t.clean()
			}
		} else {
			t.trial()
		}
		if t.passed() {
			break
		}
	}
	if !t.passed() {
		t.dumpOutput()
	}
	result <- t
}

// GOPATH returns the current GOPATH
func GOPATH() string {
	gopath := os.Getenv("GOPATH")
	if gopath == "" {
		gopath = build.Default.GOPATH
	}
	return gopath
}

// turn a package name into a binary name
func pkgBinaryName(pkg string) string {
	binary := path.Base(pkg) + ".test"
	if runtime.GOOS == "windows" {
		binary += ".exe"
	}
	return binary
}

// turn a package name into a binary path
func pkgBinary(pkg string) string {
	return path.Join(pkgPath(pkg), pkgBinaryName(pkg))
}

// returns the path to the package
func pkgPath(pkg string) string {
	return path.Join(GOPATH(), "src", pkg)
}

// cd into the package directory
func pkgChdir(pkg string) {
	err := os.Chdir(pkgPath(pkg))
	if err != nil {
		log.Fatalf("Failed to chdir to package %q: %v", pkg, err)
	}
}

// makeTestBinary makes the binary we will run
func makeTestBinary(pkg string) {
	binaryName := pkgBinaryName(pkg)
	log.Printf("%s: Making test binary %q", pkg, binaryName)
	pkgChdir(pkg)
	err := exec.Command("go", "test", "-c", "-o", binaryName).Run()
	if err != nil {
		log.Fatalf("Failed to make test binary: %v", err)
	}
	binary := pkgBinary(pkg)
	if _, err := os.Stat(binary); err != nil {
		log.Fatalf("Couldn't find test binary %q", binary)
	}
}

// removeTestBinary removes the binary made in makeTestBinary
func removeTestBinary(pkg string) {
	binary := pkgBinary(pkg)
	err := os.Remove(binary) // Delete the binary when finished
	if err != nil {
		log.Printf("Error removing test binary %q: %v", binary, err)
	}
}

func main() {
	flag.Parse()
	packages := flag.Args()
	log.Printf("Testing packages: %s", strings.Join(packages, ", "))
	if *runTests != "" {
		newRemotes := []remoteConfig{}
		for _, name := range strings.Split(*runTests, ",") {
			for i := range remotes {
				if remotes[i].Name == name {
					newRemotes = append(newRemotes, remotes[i])
					goto found
				}
			}
			log.Printf("Remote %q not found - inserting with default flags", name)
			newRemotes = append(newRemotes, remoteConfig{Name: name})
		found:
		}
		remotes = newRemotes
	}
	var names []string
	for _, remote := range remotes {
		names = append(names, remote.Name)
	}
	log.Printf("Testing remotes: %s", strings.Join(names, ", "))

	start := time.Now()
	if *clean {
		config.LoadConfig()
		packages = []string{"clean"}
	} else {
		for _, pkg := range packages {
			makeTestBinary(pkg)
			defer removeTestBinary(pkg)
		}
	}

	// start the tests
	results := make(chan *test, 8)
	awaiting := 0
	bools := []bool{false, true}
	if *clean {
		// Don't run -subdir and -fast-list if -clean
		bools = bools[:1]
	}
	for _, pkg := range packages {
		for _, remote := range remotes {
			for _, subdir := range bools {
				for _, fastlist := range bools {
					if (!subdir || subdir && remote.SubDir) && (!fastlist || fastlist && remote.FastList) {
						go newTest(pkg, remote.Name, subdir, fastlist).run(results)
						awaiting++
					}
				}
			}
		}
	}

	// Wait for the tests to finish
	var failed []*test
	for ; awaiting > 0; awaiting-- {
		t := <-results
		if !t.passed() {
			failed = append(failed, t)
		}
	}
	duration := time.Since(start)

	// Summarise results
	log.Printf("SUMMARY")
	if len(failed) == 0 {
		log.Printf("PASS: All tests finished OK in %v", duration)
	} else {
		log.Printf("FAIL: %d tests failed in %v", len(failed), duration)
		for _, t := range failed {
			log.Printf("  * %s", toShell(t.nextCmdLine()))
			log.Printf("    * Failed tests: %v", t.failedTests)
		}
		os.Exit(1)
	}
}
<commit_msg>fstest/test_all: set cache backend wait time to 30m to fix integration tests<commit_after>// Run tests for all the remotes.  Run this with package names which
// need integration testing.
//
// See the `test` target in the Makefile.
package main

import (
	"flag"
	"go/build"
	"log"
	"os"
	"os/exec"
	"path"
	"regexp"
	"runtime"
	"strings"
	"time"

	_ "github.com/ncw/rclone/backend/all" // import all fs
	"github.com/ncw/rclone/fs"
	"github.com/ncw/rclone/fs/config"
	"github.com/ncw/rclone/fs/list"
	"github.com/ncw/rclone/fs/operations"
	"github.com/ncw/rclone/fstest"
)

type remoteConfig struct {
	Name     string
	SubDir   bool
	FastList bool
}

var (
	remotes = []remoteConfig{
		{
			Name:     "TestAmazonCloudDrive:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestB2:",
			SubDir:   true,
			FastList: true,
		},
		{
			Name:     "TestCryptDrive:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestCryptSwift:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestDrive:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestDropbox:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestGoogleCloudStorage:",
			SubDir:   true,
			FastList: true,
		},
		{
			Name:     "TestHubic:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestOneDrive:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestS3:",
			SubDir:   true,
			FastList: true,
		},
		{
			Name:     "TestSftp:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestSwift:",
			SubDir:   true,
			FastList: true,
		},
		{
			Name:     "TestYandex:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestFTP:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestBox:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestQingStor:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestAzureBlob:",
			SubDir:   true,
			FastList: true,
		},
		{
			Name:     "TestPcloud:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestWebdav:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestCache:",
			SubDir:   false,
			FastList: false,
		},
	}
	// Flags
	maxTries = flag.Int("maxtries", 5, "Number of times to try each test")
	runTests = flag.String("remotes", "", "Comma separated list of remotes to test, eg 'TestSwift:,TestS3'")
	clean    = flag.Bool("clean", false, "Instead of testing, clean all left over test directories")
	runOnly  = flag.String("run", "", "Run only those tests matching the regexp supplied")
	timeout  = flag.Duration("timeout", 30*time.Minute, "Maximum time to run each test for before giving up")
)

// test holds info about a running test
type test struct {
	pkg         string
	remote      string
	subdir      bool
	cmdLine     []string
	cmdString   string
	try         int
	err         error
	output      []byte
	failedTests []string
	runFlag     string
}

// newTest creates a new test
func newTest(pkg, remote string, subdir bool, fastlist bool) *test {
	binary := pkgBinary(pkg)
	t := &test{
		pkg:     pkg,
		remote:  remote,
		subdir:  subdir,
		cmdLine: []string{binary, "-test.timeout", (*timeout).String(), "-remote", remote},
		try:     1,
	}
	if *fstest.Verbose {
		t.cmdLine = append(t.cmdLine, "-test.v")
		fs.Config.LogLevel = fs.LogLevelDebug
	}
	if *runOnly != "" {
		t.cmdLine = append(t.cmdLine, "-test.run", *runOnly)
	}
	if subdir {
		t.cmdLine = append(t.cmdLine, "-subdir")
	}
	if fastlist {
		t.cmdLine = append(t.cmdLine, "-fast-list")
	}
	t.cmdString = toShell(t.cmdLine)
	return t
}

// dumpOutput prints the error output
func (t *test) dumpOutput() {
	log.Println("------------------------------------------------------------")
	log.Printf("---- %q ----", t.cmdString)
	log.Println(string(t.output))
	log.Println("------------------------------------------------------------")
}

var failRe = regexp.MustCompile(`(?m)^--- FAIL: (Test\w*) \(`)

// findFailures looks for all the tests which failed
func (t *test) findFailures() {
	oldFailedTests := t.failedTests
	t.failedTests = nil
	for _, matches := range failRe.FindAllSubmatch(t.output, -1) {
		t.failedTests = append(t.failedTests, string(matches[1]))
	}
	if len(t.failedTests) != 0 {
		t.runFlag = "^(" + strings.Join(t.failedTests, "|") + ")$"
	} else {
		t.runFlag = ""
	}
	if t.passed() && len(t.failedTests) != 0 {
		log.Printf("%q - Expecting no errors but got: %v", t.cmdString, t.failedTests)
		t.dumpOutput()
	} else if !t.passed() && len(t.failedTests) == 0 {
		log.Printf("%q - Expecting errors but got none: %v", t.cmdString, t.failedTests)
		t.dumpOutput()
		t.failedTests = oldFailedTests
	}
}

// nextCmdLine returns the next command line
func (t *test) nextCmdLine() []string {
	cmdLine := t.cmdLine[:]
	if t.runFlag != "" {
		cmdLine = append(cmdLine, "-test.run", t.runFlag)
	}
	return cmdLine
}

// if matches then is definitely OK in the shell
var shellOK = regexp.MustCompile("^[A-Za-z0-9./_:-]+$")

// converts a argv style input into a shell command
func toShell(args []string) (result string) {
	for _, arg := range args {
		if result != "" {
			result += " "
		}
		if shellOK.MatchString(arg) {
			result += arg
		} else {
			result += "'" + arg + "'"
		}
	}
	return result
}

// trial runs a single test
func (t *test) trial() {
	cmdLine := t.nextCmdLine()
	cmdString := toShell(cmdLine)
	log.Printf("%q - Starting (try %d/%d)", cmdString, t.try, *maxTries)
	cmd := exec.Command(cmdLine[0], cmdLine[1:]...)
	start := time.Now()
	t.output, t.err = cmd.CombinedOutput()
	duration := time.Since(start)
	t.findFailures()
	if t.passed() {
		log.Printf("%q - Finished OK in %v (try %d/%d)", cmdString, duration, t.try, *maxTries)
	} else {
		log.Printf("%q - Finished ERROR in %v (try %d/%d): %v: Failed %v", cmdString, duration, t.try, *maxTries, t.err, t.failedTests)
	}
}

// cleanFs runs a single clean fs for left over directories
func (t *test) cleanFs() error {
	f, err := fs.NewFs(t.remote)
	if err != nil {
		return err
	}
	entries, err := list.DirSorted(f, true, "")
	if err != nil {
		return err
	}
	return entries.ForDirError(func(dir fs.Directory) error {
		remote := dir.Remote()
		if fstest.MatchTestRemote.MatchString(remote) {
			log.Printf("Purging %s%s", t.remote, remote)
			dir, err := fs.NewFs(t.remote + remote)
			if err != nil {
				return err
			}
			return operations.Purge(dir, "")
		}
		return nil
	})
}

// clean runs a single clean on a fs for left over directories
func (t *test) clean() {
	log.Printf("%q - Starting clean (try %d/%d)", t.remote, t.try, *maxTries)
	start := time.Now()
	t.err = t.cleanFs()
	if t.err != nil {
		log.Printf("%q - Failed to purge %v", t.remote, t.err)
	}
	duration := time.Since(start)
	if t.passed() {
		log.Printf("%q - Finished OK in %v (try %d/%d)", t.cmdString, duration, t.try, *maxTries)
	} else {
		log.Printf("%q - Finished ERROR in %v (try %d/%d): %v", t.cmdString, duration, t.try, *maxTries, t.err)
	}
}

// passed returns true if the test passed
func (t *test) passed() bool {
	return t.err == nil
}

// run runs all the trials for this test
func (t *test) run(result chan<- *test) {
	for t.try = 1; t.try <= *maxTries; t.try++ {
		if *clean {
			if !t.subdir {
				t.clean()
			}
		} else {
			t.trial()
		}
		if t.passed() {
			break
		}
	}
	if !t.passed() {
		t.dumpOutput()
	}
	result <- t
}

// GOPATH returns the current GOPATH
func GOPATH() string {
	gopath := os.Getenv("GOPATH")
	if gopath == "" {
		gopath = build.Default.GOPATH
	}
	return gopath
}

// turn a package name into a binary name
func pkgBinaryName(pkg string) string {
	binary := path.Base(pkg) + ".test"
	if runtime.GOOS == "windows" {
		binary += ".exe"
	}
	return binary
}

// turn a package name into a binary path
func pkgBinary(pkg string) string {
	return path.Join(pkgPath(pkg), pkgBinaryName(pkg))
}

// returns the path to the package
func pkgPath(pkg string) string {
	return path.Join(GOPATH(), "src", pkg)
}

// cd into the package directory
func pkgChdir(pkg string) {
	err := os.Chdir(pkgPath(pkg))
	if err != nil {
		log.Fatalf("Failed to chdir to package %q: %v", pkg, err)
	}
}

// makeTestBinary makes the binary we will run
func makeTestBinary(pkg string) {
	binaryName := pkgBinaryName(pkg)
	log.Printf("%s: Making test binary %q", pkg, binaryName)
	pkgChdir(pkg)
	err := exec.Command("go", "test", "-c", "-o", binaryName).Run()
	if err != nil {
		log.Fatalf("Failed to make test binary: %v", err)
	}
	binary := pkgBinary(pkg)
	if _, err := os.Stat(binary); err != nil {
		log.Fatalf("Couldn't find test binary %q", binary)
	}
}

// removeTestBinary removes the binary made in makeTestBinary
func removeTestBinary(pkg string) {
	binary := pkgBinary(pkg)
	err := os.Remove(binary) // Delete the binary when finished
	if err != nil {
		log.Printf("Error removing test binary %q: %v", binary, err)
	}
}

func main() {
	flag.Parse()
	packages := flag.Args()
	log.Printf("Testing packages: %s", strings.Join(packages, ", "))
	if *runTests != "" {
		newRemotes := []remoteConfig{}
		for _, name := range strings.Split(*runTests, ",") {
			for i := range remotes {
				if remotes[i].Name == name {
					newRemotes = append(newRemotes, remotes[i])
					goto found
				}
			}
			log.Printf("Remote %q not found - inserting with default flags", name)
			newRemotes = append(newRemotes, remoteConfig{Name: name})
		found:
		}
		remotes = newRemotes
	}
	var names []string
	for _, remote := range remotes {
		names = append(names, remote.Name)
	}
	log.Printf("Testing remotes: %s", strings.Join(names, ", "))

	start := time.Now()
	if *clean {
		config.LoadConfig()
		packages = []string{"clean"}
	} else {
		for _, pkg := range packages {
			makeTestBinary(pkg)
			defer removeTestBinary(pkg)
		}
	}

	// workaround for cache backend as we run simultaneous tests
	_ = os.Setenv("RCLONE_CACHE_DB_WAIT_TIME", "30m")

	// start the tests
	results := make(chan *test, 8)
	awaiting := 0
	bools := []bool{false, true}
	if *clean {
		// Don't run -subdir and -fast-list if -clean
		bools = bools[:1]
	}
	for _, pkg := range packages {
		for _, remote := range remotes {
			for _, subdir := range bools {
				for _, fastlist := range bools {
					if (!subdir || subdir && remote.SubDir) && (!fastlist || fastlist && remote.FastList) {
						go newTest(pkg, remote.Name, subdir, fastlist).run(results)
						awaiting++
					}
				}
			}
		}
	}

	// Wait for the tests to finish
	var failed []*test
	for ; awaiting > 0; awaiting-- {
		t := <-results
		if !t.passed() {
			failed = append(failed, t)
		}
	}
	duration := time.Since(start)

	// Summarise results
	log.Printf("SUMMARY")
	if len(failed) == 0 {
		log.Printf("PASS: All tests finished OK in %v", duration)
	} else {
		log.Printf("FAIL: %d tests failed in %v", len(failed), duration)
		for _, t := range failed {
			log.Printf("  * %s", toShell(t.nextCmdLine()))
			log.Printf("    * Failed tests: %v", t.failedTests)
		}
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"os"
	"time"

	"github.com/codegangsta/cli"
	mountpkg "github.com/googlecloudplatform/gcsfuse/internal/mount"
)

// Set up custom help text for gcsfuse; in particular the usage section.
func init() {
	cli.AppHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   {{.Name}} {{if .Flags}}[global options]{{end}} bucket mountpoint
   {{if .Version}}
VERSION:
   {{.Version}}
   {{end}}{{if len .Authors}}
AUTHOR(S):
   {{range .Authors}}{{ . }}{{end}}
   {{end}}{{if .Commands}}
COMMANDS:
   {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}}
   {{end}}{{end}}{{if .Flags}}
GLOBAL OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}{{end}}{{if .Copyright }}
COPYRIGHT:
   {{.Copyright}}
   {{end}}
`
}

func newApp() (app *cli.App) {
	app = &cli.App{
		Name:     "gcsfuse",
		Version:  gcsfuseVersion,
		Usage:    "Mount a GCS bucket locally",
		HideHelp: true,
		Writer:   os.Stderr,
		Flags: []cli.Flag{

			cli.BoolFlag{
				Name:  "help, h",
				Usage: "Print this help text and exit successfuly.",
			},

			/////////////////////////
			// File system
			/////////////////////////

			cli.StringSliceFlag{
				Name:  "o",
				Usage: "Additional system-specific mount options. Be careful!",
			},

			cli.IntFlag{
				Name:  "dir-mode",
				Value: 0755,
				Usage: "Permissions bits for directories. (default: 0755)",
			},

			cli.IntFlag{
				Name:  "file-mode",
				Value: 0644,
				Usage: "Permission bits for files (default: 0644)",
			},

			cli.IntFlag{
				Name:  "uid",
				Value: -1,
				Usage: "UID owner of all inodes.",
			},

			cli.IntFlag{
				Name:  "gid",
				Value: -1,
				Usage: "GID owner of all inodes.",
			},

			cli.BoolFlag{
				Name: "implicit-dirs",
				Usage: "Implicitly define directories based on content. See" +
					"docs/semantics.md",
			},

			/////////////////////////
			// GCS
			/////////////////////////

			cli.StringFlag{
				Name:  "key-file",
				Value: "",
				Usage: "Path to JSON key file for use with GCS. " +
					"(default: none, Google application default credentials used)",
			},

			cli.Float64Flag{
				Name:  "limit-bytes-per-sec",
				Value: -1,
				Usage: "Bandwidth limit for reading data, measured over a 30-second " +
					"window. (use -1 for no limit)",
			},

			cli.Float64Flag{
				Name:  "limit-ops-per-sec",
				Value: 5.0,
				Usage: "Operations per second limit, measured over a 30-second window " +
					"(use -1 for no limit)",
			},

			/////////////////////////
			// Tuning
			/////////////////////////

			cli.DurationFlag{
				Name:  "stat-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache StatObject results and inode attributes.",
			},

			cli.DurationFlag{
				Name:  "type-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache name -> file/dir mappings in directory " +
					"inodes.",
			},

			cli.StringFlag{
				Name:  "temp-dir",
				Value: "",
				Usage: "Temporary directory for local GCS object copies. " +
					"(default: system default, likely /tmp)",
			},

			/////////////////////////
			// Debugging
			/////////////////////////

			cli.BoolFlag{
				Name:  "debug_fuse",
				Usage: "Enable fuse-related debugging output.",
			},

			cli.BoolFlag{
				Name:  "debug_gcs",
				Usage: "Print GCS request and timing information.",
			},

			cli.BoolFlag{
				Name:  "debug_http",
				Usage: "Dump HTTP requests and responses to/from GCS.",
			},

			cli.BoolFlag{
				Name:  "debug_invariants",
				Usage: "Panic when internal invariants are violated.",
			},
		},
	}

	return
}

type flagStorage struct {
	// File system
	MountOptions map[string]string
	DirMode      os.FileMode
	FileMode     os.FileMode
	Uid          int64
	Gid          int64
	ImplicitDirs bool

	// GCS
	KeyFile                            string
	EgressBandwidthLimitBytesPerSecond float64
	OpRateLimitHz                      float64

	// Tuning
	StatCacheTTL time.Duration
	TypeCacheTTL time.Duration
	TempDir      string

	// Debugging
	DebugFuse       bool
	DebugGCS        bool
	DebugHTTP       bool
	DebugInvariants bool
}

// Add the flags accepted by run to the supplied flag set, returning the
// variables into which the flags will parse.
func populateFlags(c *cli.Context) (flags *flagStorage) {
	flags = &flagStorage{
		// File system
		MountOptions: make(map[string]string),
		DirMode:      os.FileMode(c.Int("dir-mode")),
		FileMode:     os.FileMode(c.Int("file-mode")),
		Uid:          int64(c.Int("uid")),
		Gid:          int64(c.Int("gid")),

		// GCS,
		KeyFile: c.String("key-file"),
		EgressBandwidthLimitBytesPerSecond: c.Float64("limit-bytes-per-sec"),
		OpRateLimitHz:                      c.Float64("limit-ops-per-sec"),

		// Tuning,
		StatCacheTTL: c.Duration("stat-cache-ttl"),
		TypeCacheTTL: c.Duration("type-cache-ttl"),
		TempDir:      c.String("temp-dir"),
		ImplicitDirs: c.Bool("implicit-dirs"),

		// Debugging,
		DebugFuse:       c.Bool("debug_fuse"),
		DebugGCS:        c.Bool("debug_gcs"),
		DebugHTTP:       c.Bool("debug_http"),
		DebugInvariants: c.Bool("debug_invariants"),
	}

	// Handle the repeated "-o" flag.
	for _, o := range c.StringSlice("o") {
		mountpkg.ParseOptions(flags.MountOptions, o)
	}

	return
}
<commit_msg>Include the Go version in the gcsfuse version string.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"os"
	"runtime"
	"time"

	"github.com/codegangsta/cli"
	mountpkg "github.com/googlecloudplatform/gcsfuse/internal/mount"
)

// Set up custom help text for gcsfuse; in particular the usage section.
func init() {
	cli.AppHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   {{.Name}} {{if .Flags}}[global options]{{end}} bucket mountpoint
   {{if .Version}}
VERSION:
   {{.Version}}
   {{end}}{{if len .Authors}}
AUTHOR(S):
   {{range .Authors}}{{ . }}{{end}}
   {{end}}{{if .Commands}}
COMMANDS:
   {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}}
   {{end}}{{end}}{{if .Flags}}
GLOBAL OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}{{end}}{{if .Copyright }}
COPYRIGHT:
   {{.Copyright}}
   {{end}}
`
}

func newApp() (app *cli.App) {
	version := fmt.Sprintf(
		"%s (built with Go version %s)",
		gcsfuseVersion,
		runtime.Version())

	app = &cli.App{
		Name:     "gcsfuse",
		Version:  version,
		Usage:    "Mount a GCS bucket locally",
		HideHelp: true,
		Writer:   os.Stderr,
		Flags: []cli.Flag{

			cli.BoolFlag{
				Name:  "help, h",
				Usage: "Print this help text and exit successfuly.",
			},

			/////////////////////////
			// File system
			/////////////////////////

			cli.StringSliceFlag{
				Name:  "o",
				Usage: "Additional system-specific mount options. Be careful!",
			},

			cli.IntFlag{
				Name:  "dir-mode",
				Value: 0755,
				Usage: "Permissions bits for directories. (default: 0755)",
			},

			cli.IntFlag{
				Name:  "file-mode",
				Value: 0644,
				Usage: "Permission bits for files (default: 0644)",
			},

			cli.IntFlag{
				Name:  "uid",
				Value: -1,
				Usage: "UID owner of all inodes.",
			},

			cli.IntFlag{
				Name:  "gid",
				Value: -1,
				Usage: "GID owner of all inodes.",
			},

			cli.BoolFlag{
				Name: "implicit-dirs",
				Usage: "Implicitly define directories based on content. See" +
					"docs/semantics.md",
			},

			/////////////////////////
			// GCS
			/////////////////////////

			cli.StringFlag{
				Name:  "key-file",
				Value: "",
				Usage: "Path to JSON key file for use with GCS. " +
					"(default: none, Google application default credentials used)",
			},

			cli.Float64Flag{
				Name:  "limit-bytes-per-sec",
				Value: -1,
				Usage: "Bandwidth limit for reading data, measured over a 30-second " +
					"window. (use -1 for no limit)",
			},

			cli.Float64Flag{
				Name:  "limit-ops-per-sec",
				Value: 5.0,
				Usage: "Operations per second limit, measured over a 30-second window " +
					"(use -1 for no limit)",
			},

			/////////////////////////
			// Tuning
			/////////////////////////

			cli.DurationFlag{
				Name:  "stat-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache StatObject results and inode attributes.",
			},

			cli.DurationFlag{
				Name:  "type-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache name -> file/dir mappings in directory " +
					"inodes.",
			},

			cli.StringFlag{
				Name:  "temp-dir",
				Value: "",
				Usage: "Temporary directory for local GCS object copies. " +
					"(default: system default, likely /tmp)",
			},

			/////////////////////////
			// Debugging
			/////////////////////////

			cli.BoolFlag{
				Name:  "debug_fuse",
				Usage: "Enable fuse-related debugging output.",
			},

			cli.BoolFlag{
				Name:  "debug_gcs",
				Usage: "Print GCS request and timing information.",
			},

			cli.BoolFlag{
				Name:  "debug_http",
				Usage: "Dump HTTP requests and responses to/from GCS.",
			},

			cli.BoolFlag{
				Name:  "debug_invariants",
				Usage: "Panic when internal invariants are violated.",
			},
		},
	}

	return
}

type flagStorage struct {
	// File system
	MountOptions map[string]string
	DirMode      os.FileMode
	FileMode     os.FileMode
	Uid          int64
	Gid          int64
	ImplicitDirs bool

	// GCS
	KeyFile                            string
	EgressBandwidthLimitBytesPerSecond float64
	OpRateLimitHz                      float64

	// Tuning
	StatCacheTTL time.Duration
	TypeCacheTTL time.Duration
	TempDir      string

	// Debugging
	DebugFuse       bool
	DebugGCS        bool
	DebugHTTP       bool
	DebugInvariants bool
}

// Add the flags accepted by run to the supplied flag set, returning the
// variables into which the flags will parse.
func populateFlags(c *cli.Context) (flags *flagStorage) {
	flags = &flagStorage{
		// File system
		MountOptions: make(map[string]string),
		DirMode:      os.FileMode(c.Int("dir-mode")),
		FileMode:     os.FileMode(c.Int("file-mode")),
		Uid:          int64(c.Int("uid")),
		Gid:          int64(c.Int("gid")),

		// GCS,
		KeyFile: c.String("key-file"),
		EgressBandwidthLimitBytesPerSecond: c.Float64("limit-bytes-per-sec"),
		OpRateLimitHz:                      c.Float64("limit-ops-per-sec"),

		// Tuning,
		StatCacheTTL: c.Duration("stat-cache-ttl"),
		TypeCacheTTL: c.Duration("type-cache-ttl"),
		TempDir:      c.String("temp-dir"),
		ImplicitDirs: c.Bool("implicit-dirs"),

		// Debugging,
		DebugFuse:       c.Bool("debug_fuse"),
		DebugGCS:        c.Bool("debug_gcs"),
		DebugHTTP:       c.Bool("debug_http"),
		DebugInvariants: c.Bool("debug_invariants"),
	}

	// Handle the repeated "-o" flag.
	for _, o := range c.StringSlice("o") {
		mountpkg.ParseOptions(flags.MountOptions, o)
	}

	return
}
<|endoftext|>


<commit_before>package cleanhttp

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"
)

func TestPrintablePathCheckHandler(t *testing.T) {
	getTestHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "Hello, client")
	})

	cases := map[string]struct {
		path       string
		expectCode int
		input      *HandlerInput
	}{
		"valid nil input": {
			path:       "/valid",
			expectCode: http.StatusOK,
			input:      nil,
		},

		"valid empty error status": {
			path:       "/valid",
			expectCode: http.StatusOK,
			input:      &HandlerInput{},
		},

		"invalid newline": {
			path:       "/invalid\n",
			expectCode: http.StatusBadRequest,
		},

		"invalid carriage return": {
			path:       "/invalid\r",
			expectCode: http.StatusBadRequest,
		},

		"invalid null": {
			path:       "/invalid\x00",
			expectCode: http.StatusBadRequest,
		},

		"invalid alternate status": {
			path:       "/invalid\n",
			expectCode: http.StatusInternalServerError,
			input: &HandlerInput{
				ErrStatus: http.StatusInternalServerError,
			},
		},
	}

	for name, tc := range cases {
		t.Run(name, func(t *testing.T) {
			// Create test HTTP server
			ts := httptest.NewServer(PrintablePathCheckHandler(getTestHandler, tc.input))
			defer ts.Close()

			res, err := http.Get(ts.URL + tc.path)
			if err != nil {
				t.Fatal(err)
			}

			if tc.expectCode != res.StatusCode {
				t.Fatalf("expected %d, got :%d", tc.expectCode, res.StatusCode)
			}
		})
	}
}
<commit_msg>Fix tests against newer Go<commit_after>package cleanhttp

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"
)

func TestPrintablePathCheckHandler(t *testing.T) {
	getTestHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "Hello, client")
	})

	cases := map[string]struct {
		path       string
		expectCode int
		input      *HandlerInput
	}{
		"valid nil input": {
			path:       "/valid",
			expectCode: http.StatusOK,
			input:      nil,
		},

		"valid empty error status": {
			path:       "/valid",
			expectCode: http.StatusOK,
			input:      &HandlerInput{},
		},

		"invalid newline": {
			path:       "/invalid%0A",
			expectCode: http.StatusBadRequest,
		},

		"invalid carriage return": {
			path:       "/invalid%0D",
			expectCode: http.StatusBadRequest,
		},

		"invalid null": {
			path:       "/invalid%00",
			expectCode: http.StatusBadRequest,
		},

		"invalid alternate status": {
			path:       "/invalid%0A",
			expectCode: http.StatusInternalServerError,
			input: &HandlerInput{
				ErrStatus: http.StatusInternalServerError,
			},
		},
	}

	for name, tc := range cases {
		t.Run(name, func(t *testing.T) {
			// Create test HTTP server
			ts := httptest.NewServer(PrintablePathCheckHandler(getTestHandler, tc.input))
			defer ts.Close()

			res, err := http.Get(ts.URL + tc.path)
			if err != nil {
				t.Fatal(err)
			}

			if tc.expectCode != res.StatusCode {
				t.Fatalf("expected %d, got :%d", tc.expectCode, res.StatusCode)
			}
		})
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 Claudemiro Alves Feitosa Neto. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package ipe

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"strconv"
	"strings"

	log "github.com/golang/glog"
	"github.com/gorilla/websocket"

	"github.com/dimiro1/ipe/utils"
)

var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
	CheckOrigin:     func(r *http.Request) bool { return true },
}

// Handle open Subscriber.
func onOpen(conn *websocket.Conn, w http.ResponseWriter, r *http.Request, sessionID string, app *app) websocketError {
	params := r.URL.Query()
	p := params.Get("protocol")

	protocol, err := strconv.Atoi(p)

	if err != nil {
		return newInvalidVersionStringFormatError()
	}

	switch {
	case strings.TrimSpace(p) == "":
		return newNoProtocolVersionSuppliedError()
	case protocol != supportedProtocolVersion:
		return newUnsupportedProtocolVersionError()
	case app.ApplicationDisabled:
		return newApplicationDisabledError()
	case app.OnlySSL:
		if r.TLS == nil {
			return newApplicationOnlyAccepsSSLError()
		}
	}

	// Create the new Subscriber
	connection := newConnection(sessionID, conn)
	app.Connect(connection)

	// Everything went fine. Huhu.
	if err := conn.WriteJSON(newConnectionEstablishedEvent(connection.SocketID)); err != nil {
		return newGenericReconnectImmediatelyError()
	}

	return nil
}

// Handle the close event
func onClose(sessionID string, app *app) {
	app.Disconnect(sessionID)
}

// Handle messages
//
// If there is an unrecoverable error then break the loop,
// otherwise just keep going.
func onMessage(conn *websocket.Conn, w http.ResponseWriter, r *http.Request, sessionID string, app *app) {
	var event struct {
		Event string `json:"event"`
	}

	for {
		_, message, err := conn.ReadMessage()

		if err != nil {
			log.Errorf("%+v", err)
			switch err {
			case io.EOF:
				onClose(sessionID, app)
			default:
				emitWSError(newGenericReconnectImmediatelyError(), conn)
			}
			break
		}

		if err := json.Unmarshal(message, &event); err != nil {
			emitWSError(newGenericReconnectImmediatelyError(), conn)
			break
		}

		log.Infof("websockets: Handling %s event", event.Event)

		switch event.Event {
		case "pusher:ping":
			if err := conn.WriteJSON(newPongEvent()); err != nil {
				emitWSError(newGenericReconnectImmediatelyError(), conn)
			}
		case "pusher:subscribe":
			subscribeEvent := subscribeEvent{}

			if err := json.Unmarshal(message, &subscribeEvent); err != nil {
				emitWSError(newGenericReconnectImmediatelyError(), conn)
				break
			}

			connection, err := app.FindConnection(sessionID)

			if err != nil {
				emitWSError(newGenericReconnectImmediatelyError(), conn)
				break
			}

			channelName := strings.TrimSpace(subscribeEvent.Data.Channel)

			if !utils.IsChannelNameValid(channelName) {
				emitWSError(newGenericError(fmt.Sprintf("This channel name is not valid")), conn)
				break
			}

			isPresence := utils.IsPresenceChannel(channelName)
			isPrivate := utils.IsPrivateChannel(channelName)

			if isPresence || isPrivate {
				toSign := []string{connection.SocketID, channelName}

				if isPresence || len(subscribeEvent.Data.ChannelData) > 0 {
					toSign = append(toSign, subscribeEvent.Data.ChannelData)
				}

				expectedAuthKey := fmt.Sprintf("%s:%s", app.Key, utils.HashMAC([]byte(strings.Join(toSign, ":")), []byte(app.Secret)))
				if subscribeEvent.Data.Auth != expectedAuthKey {
					emitWSError(newGenericError(fmt.Sprintf("Auth value for subscription to %s is invalid", channelName)), conn)
					continue
				}
			}

			channel := app.FindOrCreateChannelByChannelID(channelName)
			log.Info(subscribeEvent.Data.ChannelData)

			if err := app.Subscribe(channel, connection, subscribeEvent.Data.ChannelData); err != nil {
				emitWSError(newGenericReconnectImmediatelyError(), conn)
			}
		case "pusher:unsubscribe":
			unsubscribeEvent := unsubscribeEvent{}

			if err := json.Unmarshal(message, &unsubscribeEvent); err != nil {
				emitWSError(newGenericReconnectImmediatelyError(), conn)
			}

			connection, err := app.FindConnection(sessionID)

			if err != nil {
				emitWSError(newGenericError(fmt.Sprintf("Could not find a connection with the id %s", sessionID)), conn)
			}

			channel, err := app.FindChannelByChannelID(unsubscribeEvent.Data.Channel)

			if err != nil {
				emitWSError(newGenericError(fmt.Sprintf("Could not find a channel with the id %s", unsubscribeEvent.Data.Channel)), conn)
			}

			if err := app.Unsubscribe(channel, connection); err != nil {
				emitWSError(newGenericReconnectImmediatelyError(), conn)
				break
			}
		default: // CLient Events ??
			// see http://pusher.com/docs/client_api_guide/client_events#trigger-events
			if strings.HasPrefix(event.Event, "client-") {
				if !app.UserEvents {
					emitWSError(newGenericError("To send client events, you must enable this feature in the Settings."), conn)
				}

				clientEvent := rawEvent{}

				if err := json.Unmarshal(message, &clientEvent); err != nil {
					log.Error(err)
					emitWSError(newGenericReconnectImmediatelyError(), conn)
					break
				}

				channel, err := app.FindChannelByChannelID(clientEvent.Channel)

				if err != nil {
					emitWSError(newGenericError(fmt.Sprintf("Could not find a channel with the id %s", clientEvent.Channel)), conn)
				}

				if !channel.IsPresenceOrPrivate() {
					emitWSError(newGenericError("Client event rejected - only supported on private and presence channels"), conn)
					break
				}

				if err := app.Publish(channel, clientEvent, sessionID); err != nil {
					log.Error(err)
					emitWSError(newGenericReconnectImmediatelyError(), conn)
					break
				}
			}

		} // switch
	} // For
}

// Websocket GET /app/{key}
func wsHandler(ctx *applicationContext, p params, w http.ResponseWriter, r *http.Request) {
	conn, err := upgrader.Upgrade(w, r, nil)
	defer func() {
		if conn != nil {
			conn.Close()
		}
	}()

	if err != nil {
		log.Error(err)
		return
	}

	appKey := p.Get("key")

	app, err := ctx.DB.GetAppByKey(appKey)

	if err != nil {
		log.Error(err)
		emitWSError(newApplicationDoesNotExistsError(), conn)
		return
	}

	sessionID := utils.GenerateSessionID()

	if err := onOpen(conn, w, r, sessionID, app); err != nil {
		emitWSError(err, conn)
		return
	}

	onMessage(conn, w, r, sessionID, app)
}

// Emit an Websocket ErrorEvent
func emitWSError(err websocketError, conn *websocket.Conn) {

	event := newErrorEvent(err.GetCode(), err.GetMsg())

	if err := conn.WriteJSON(event); err != nil {
		log.Error(err)
	}
}
<commit_msg>Fixed a bug in ipe/websocket.go (TotalConnections were growing infinitely connection was not closed when a channel listener refreshed the browser tab or simply closed the browser tab)<commit_after>// Copyright 2014 Claudemiro Alves Feitosa Neto. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package ipe

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"strconv"
	"strings"

	log "github.com/golang/glog"
	"github.com/gorilla/websocket"

	"github.com/dimiro1/ipe/utils"
)

var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
	CheckOrigin:     func(r *http.Request) bool { return true },
}

// Handle open Subscriber.
func onOpen(conn *websocket.Conn, w http.ResponseWriter, r *http.Request, sessionID string, app *app) websocketError {
	params := r.URL.Query()
	p := params.Get("protocol")

	protocol, err := strconv.Atoi(p)

	if err != nil {
		return newInvalidVersionStringFormatError()
	}

	switch {
	case strings.TrimSpace(p) == "":
		return newNoProtocolVersionSuppliedError()
	case protocol != supportedProtocolVersion:
		return newUnsupportedProtocolVersionError()
	case app.ApplicationDisabled:
		return newApplicationDisabledError()
	case app.OnlySSL:
		if r.TLS == nil {
			return newApplicationOnlyAccepsSSLError()
		}
	}

	// Create the new Subscriber
	connection := newConnection(sessionID, conn)
	app.Connect(connection)

	// Everything went fine. Huhu.
	if err := conn.WriteJSON(newConnectionEstablishedEvent(connection.SocketID)); err != nil {
		return newGenericReconnectImmediatelyError()
	}

	return nil
}

// Handle the close event
func onClose(sessionID string, app *app) {
	app.Disconnect(sessionID)
}

// Handle messages
//
// If there is an unrecoverable error then break the loop,
// otherwise just keep going.
func onMessage(conn *websocket.Conn, w http.ResponseWriter, r *http.Request, sessionID string, app *app) {
	var event struct {
		Event string `json:"event"`
	}

	for {
		_, message, err := conn.ReadMessage()

		if err != nil {
			log.Errorf("%+v", err)
			if err == io.EOF {
				onClose(sessionID, app)
			} else if _, ok := err.(*websocket.CloseError); ok {
				onClose(sessionID, app)
			} else {
				emitWSError(newGenericReconnectImmediatelyError(), conn)
			}
			break
		}

		if err := json.Unmarshal(message, &event); err != nil {
			emitWSError(newGenericReconnectImmediatelyError(), conn)
			break
		}

		log.Infof("websockets: Handling %s event", event.Event)

		switch event.Event {
		case "pusher:ping":
			if err := conn.WriteJSON(newPongEvent()); err != nil {
				emitWSError(newGenericReconnectImmediatelyError(), conn)
			}
		case "pusher:subscribe":
			subscribeEvent := subscribeEvent{}

			if err := json.Unmarshal(message, &subscribeEvent); err != nil {
				emitWSError(newGenericReconnectImmediatelyError(), conn)
				break
			}

			connection, err := app.FindConnection(sessionID)

			if err != nil {
				emitWSError(newGenericReconnectImmediatelyError(), conn)
				break
			}

			channelName := strings.TrimSpace(subscribeEvent.Data.Channel)

			if !utils.IsChannelNameValid(channelName) {
				emitWSError(newGenericError(fmt.Sprintf("This channel name is not valid")), conn)
				break
			}

			isPresence := utils.IsPresenceChannel(channelName)
			isPrivate := utils.IsPrivateChannel(channelName)

			if isPresence || isPrivate {
				toSign := []string{connection.SocketID, channelName}

				if isPresence || len(subscribeEvent.Data.ChannelData) > 0 {
					toSign = append(toSign, subscribeEvent.Data.ChannelData)
				}

				expectedAuthKey := fmt.Sprintf("%s:%s", app.Key, utils.HashMAC([]byte(strings.Join(toSign, ":")), []byte(app.Secret)))
				if subscribeEvent.Data.Auth != expectedAuthKey {
					emitWSError(newGenericError(fmt.Sprintf("Auth value for subscription to %s is invalid", channelName)), conn)
					continue
				}
			}

			channel := app.FindOrCreateChannelByChannelID(channelName)
			log.Info(subscribeEvent.Data.ChannelData)

			if err := app.Subscribe(channel, connection, subscribeEvent.Data.ChannelData); err != nil {
				emitWSError(newGenericReconnectImmediatelyError(), conn)
			}
		case "pusher:unsubscribe":
			unsubscribeEvent := unsubscribeEvent{}

			if err := json.Unmarshal(message, &unsubscribeEvent); err != nil {
				emitWSError(newGenericReconnectImmediatelyError(), conn)
			}

			connection, err := app.FindConnection(sessionID)

			if err != nil {
				emitWSError(newGenericError(fmt.Sprintf("Could not find a connection with the id %s", sessionID)), conn)
			}

			channel, err := app.FindChannelByChannelID(unsubscribeEvent.Data.Channel)

			if err != nil {
				emitWSError(newGenericError(fmt.Sprintf("Could not find a channel with the id %s", unsubscribeEvent.Data.Channel)), conn)
			}

			if err := app.Unsubscribe(channel, connection); err != nil {
				emitWSError(newGenericReconnectImmediatelyError(), conn)
				break
			}
		default: // CLient Events ??
			// see http://pusher.com/docs/client_api_guide/client_events#trigger-events
			if strings.HasPrefix(event.Event, "client-") {
				if !app.UserEvents {
					emitWSError(newGenericError("To send client events, you must enable this feature in the Settings."), conn)
				}

				clientEvent := rawEvent{}

				if err := json.Unmarshal(message, &clientEvent); err != nil {
					log.Error(err)
					emitWSError(newGenericReconnectImmediatelyError(), conn)
					break
				}

				channel, err := app.FindChannelByChannelID(clientEvent.Channel)

				if err != nil {
					emitWSError(newGenericError(fmt.Sprintf("Could not find a channel with the id %s", clientEvent.Channel)), conn)
				}

				if !channel.IsPresenceOrPrivate() {
					emitWSError(newGenericError("Client event rejected - only supported on private and presence channels"), conn)
					break
				}

				if err := app.Publish(channel, clientEvent, sessionID); err != nil {
					log.Error(err)
					emitWSError(newGenericReconnectImmediatelyError(), conn)
					break
				}
			}

		} // switch
	} // For
}

// Websocket GET /app/{key}
func wsHandler(ctx *applicationContext, p params, w http.ResponseWriter, r *http.Request) {
	conn, err := upgrader.Upgrade(w, r, nil)
	defer func() {
		if conn != nil {
			conn.Close()
		}
	}()

	if err != nil {
		log.Error(err)
		return
	}

	appKey := p.Get("key")

	app, err := ctx.DB.GetAppByKey(appKey)

	if err != nil {
		log.Error(err)
		emitWSError(newApplicationDoesNotExistsError(), conn)
		return
	}

	sessionID := utils.GenerateSessionID()

	if err := onOpen(conn, w, r, sessionID, app); err != nil {
		emitWSError(err, conn)
		return
	}

	onMessage(conn, w, r, sessionID, app)
}

// Emit an Websocket ErrorEvent
func emitWSError(err websocketError, conn *websocket.Conn) {

	event := newErrorEvent(err.GetCode(), err.GetMsg())

	if err := conn.WriteJSON(event); err != nil {
		log.Error(err)
	}
}
<|endoftext|>


<commit_before>package boardgame

import (
	"math"
)

const emptyIndexSentinel = -1

//TODO: create a Stack interface that both GrowableStack and SizedStack
//implement.

//TODO: should deck, index, and cap all be private?

//Stack is one of the fundamental types in BoardGame. It represents an ordered
//stack of 0 or more Components, all from the same Deck. Each deck has 0 or
//more Stacks based off of it, and together they include all components in
//that deck, with no component residing in more than one stack. Stacks model
//things like a stack of cards, a collection of resource tokens, etc.
type GrowableStack struct {
	//Deck is the deck that we're a part of.
	deck *Deck
	//The indexes from the given deck that this stack contains, in order.
	indexes []int
	//size, if set, says the maxmimum number of items allowed in the Stack. 0
	//means that the Stack may grow without bound.
	maxLen int
}

//SizedStack is a Stack that has a fixed number of slots, any of which may be
//empty. Create a new one with NewSizedStack.
type SizedStack struct {
	//Deck is the deck we're a part of.
	deck *Deck
	//Indexes will always have a len of size. Slots that are "empty" will have
	//index of -1.
	indexes []int
	//Size is the number of slots.
	size int
}

//NewGrowableStack creates a new growable stack with the given Deck and Cap.
func NewGrowableStack(deck *Deck, maxLen int) *GrowableStack {

	if maxLen < 0 {
		maxLen = 0
	}

	return &GrowableStack{
		deck:    deck,
		indexes: nil,
		maxLen:  maxLen,
	}
}

//NewSizedStack creates a new SizedStack for the given deck, with the
//specified size.
func NewSizedStack(deck *Deck, size int) *SizedStack {
	if size < 0 {
		size = 0
	}

	indexes := make([]int, size)

	for i := 0; i < size; i++ {
		indexes[i] = emptyIndexSentinel
	}

	return &SizedStack{
		deck:    deck,
		indexes: indexes,
		size:    size,
	}
}

//Len returns the number of items in the stack.
func (s *GrowableStack) Len() int {
	return len(s.indexes)
}

//Len returns the number of slots in the stack. It will always equal Size.
func (s *SizedStack) Len() int {
	return len(s.indexes)
}

//ComponentAt fetches the component object representing the n-th object in
//this stack.
func (s *GrowableStack) ComponentAt(index int) *Component {

	//Substantially recreated in SizedStack.ComponentAt()
	if index >= s.Len() || index < 0 {
		return nil
	}

	if s.deck == nil {
		return nil
	}

	//We don't need to check that s.Indexes[index] is valid because it was
	//checked when it was set, and Decks are immutable.
	return s.deck.Components()[s.indexes[index]]
}

//ComponentAt fetches the component object representing the n-th object in
//this stack.
func (s *SizedStack) ComponentAt(index int) *Component {

	//Substantially recreated in GrowableStack.ComponentAt()

	if index >= s.Len() || index < 0 {
		return nil
	}

	if s.deck == nil {
		return nil
	}

	deckIndex := s.indexes[index]

	//Check if this is an empty slot
	if deckIndex == emptyIndexSentinel {
		return nil
	}

	//We don't need to check that s.Indexes[index] is valid because it was
	//checked when it was set, and Decks are immutable.
	return s.deck.Components()[deckIndex]
}

//SlotsRemaining returns the count of slots left in this stack. If Cap is 0
//(inifinite) this will be MaxInt64.
func (s *GrowableStack) SlotsRemaining() int {
	if s.maxLen <= 0 {
		return math.MaxInt64
	}
	return s.maxLen - s.Len()
}

//SlotsRemaining returns the count of unfilled slots in this stack.
func (s *SizedStack) SlotsRemaining() int {
	count := 0
	for _, index := range s.indexes {
		if index == emptyIndexSentinel {
			count++
		}
	}
	return count
}

//InsertFront puts the component at index 0 in this stack, moving all other
//items down by one. The Component you insert should not currently be a member
//of any other stacks, to maintain the deck invariant.
func (s *GrowableStack) InsertFront(c *Component) {

	//Based on how Decks and Chests are constructed, we know the components in
	//the chest hae the right gamename, so no need to check.

	if c.Deck.Name() != s.deck.Name() {
		//We can only add items that are in our deck.

		//TODO: communicate an error
		return
	}

	if s.SlotsRemaining() < 1 {
		return
	}

	s.indexes = append([]int{c.DeckIndex}, s.indexes...)
}

/*

//InsertBack puts the component at the last index in this stack. The
//Component you insert should not currently be a member of any other stacks,
//to maintain the deck invariant.
func (s *Stack) InsertBack(c Component) {

}

//RemoveFront removes the component from the first slot in this stack,
//shifting all later components down by 1. You should then insert the
//component in another stack to maintain the deck invariant.
func (s *Stack) RemoveFront() Component {

}

//RemoveBack removes the component from the last slot in this stack. You
//should then insert the component in another stack to maintain the deck
//invariant.
func (s *Stack) RemoveBack() Component {

}

*/
<commit_msg>Create Stack interface, which both GrowableStack and SizedStack conform to. Part of #12.<commit_after>package boardgame

import (
	"math"
)

const emptyIndexSentinel = -1

//Stack is one of the fundamental types in BoardGame. It represents an ordered
//stack of 0 or more Components, all from the same Deck. Each deck has 0 or
//more Stacks based off of it, and together they include all components in
//that deck, with no component residing in more than one stack. Stacks model
//things like a stack of cards, a collection of resource tokens, etc. There
//are two concrete types of Stacks: GrowableStack's and SizedStack's.
type Stack interface {
	//Len returns the number of slots in the Stack. For a GrowableStack this
	//is the number of items in the stack. For SizedStacks, this is the number
	//of slots--even if some are unfilled.
	Len() int
	//ComponentAt retrieves the component at the given index in the stack.
	ComponentAt(index int) *Component
	//SlotsRemaining returns how many slots there are left in this stack to
	//add items.
	SlotsRemaining() int
}

type GrowableStack struct {
	//Deck is the deck that we're a part of.
	deck *Deck
	//The indexes from the given deck that this stack contains, in order.
	indexes []int
	//size, if set, says the maxmimum number of items allowed in the Stack. 0
	//means that the Stack may grow without bound.
	maxLen int
}

//SizedStack is a Stack that has a fixed number of slots, any of which may be
//empty. Create a new one with NewSizedStack.
type SizedStack struct {
	//Deck is the deck we're a part of.
	deck *Deck
	//Indexes will always have a len of size. Slots that are "empty" will have
	//index of -1.
	indexes []int
	//Size is the number of slots.
	size int
}

//NewGrowableStack creates a new growable stack with the given Deck and Cap.
func NewGrowableStack(deck *Deck, maxLen int) *GrowableStack {

	if maxLen < 0 {
		maxLen = 0
	}

	return &GrowableStack{
		deck:    deck,
		indexes: nil,
		maxLen:  maxLen,
	}
}

//NewSizedStack creates a new SizedStack for the given deck, with the
//specified size.
func NewSizedStack(deck *Deck, size int) *SizedStack {
	if size < 0 {
		size = 0
	}

	indexes := make([]int, size)

	for i := 0; i < size; i++ {
		indexes[i] = emptyIndexSentinel
	}

	return &SizedStack{
		deck:    deck,
		indexes: indexes,
		size:    size,
	}
}

//Len returns the number of items in the stack.
func (s *GrowableStack) Len() int {
	return len(s.indexes)
}

//Len returns the number of slots in the stack. It will always equal Size.
func (s *SizedStack) Len() int {
	return len(s.indexes)
}

//ComponentAt fetches the component object representing the n-th object in
//this stack.
func (s *GrowableStack) ComponentAt(index int) *Component {

	//Substantially recreated in SizedStack.ComponentAt()
	if index >= s.Len() || index < 0 {
		return nil
	}

	if s.deck == nil {
		return nil
	}

	//We don't need to check that s.Indexes[index] is valid because it was
	//checked when it was set, and Decks are immutable.
	return s.deck.Components()[s.indexes[index]]
}

//ComponentAt fetches the component object representing the n-th object in
//this stack.
func (s *SizedStack) ComponentAt(index int) *Component {

	//Substantially recreated in GrowableStack.ComponentAt()

	if index >= s.Len() || index < 0 {
		return nil
	}

	if s.deck == nil {
		return nil
	}

	deckIndex := s.indexes[index]

	//Check if this is an empty slot
	if deckIndex == emptyIndexSentinel {
		return nil
	}

	//We don't need to check that s.Indexes[index] is valid because it was
	//checked when it was set, and Decks are immutable.
	return s.deck.Components()[deckIndex]
}

//SlotsRemaining returns the count of slots left in this stack. If Cap is 0
//(inifinite) this will be MaxInt64.
func (s *GrowableStack) SlotsRemaining() int {
	if s.maxLen <= 0 {
		return math.MaxInt64
	}
	return s.maxLen - s.Len()
}

//SlotsRemaining returns the count of unfilled slots in this stack.
func (s *SizedStack) SlotsRemaining() int {
	count := 0
	for _, index := range s.indexes {
		if index == emptyIndexSentinel {
			count++
		}
	}
	return count
}

//InsertFront puts the component at index 0 in this stack, moving all other
//items down by one. The Component you insert should not currently be a member
//of any other stacks, to maintain the deck invariant.
func (s *GrowableStack) InsertFront(c *Component) {

	//Based on how Decks and Chests are constructed, we know the components in
	//the chest hae the right gamename, so no need to check.

	if c.Deck.Name() != s.deck.Name() {
		//We can only add items that are in our deck.

		//TODO: communicate an error
		return
	}

	if s.SlotsRemaining() < 1 {
		return
	}

	s.indexes = append([]int{c.DeckIndex}, s.indexes...)
}

/*

//InsertBack puts the component at the last index in this stack. The
//Component you insert should not currently be a member of any other stacks,
//to maintain the deck invariant.
func (s *Stack) InsertBack(c Component) {

}

//RemoveFront removes the component from the first slot in this stack,
//shifting all later components down by 1. You should then insert the
//component in another stack to maintain the deck invariant.
func (s *Stack) RemoveFront() Component {

}

//RemoveBack removes the component from the last slot in this stack. You
//should then insert the component in another stack to maintain the deck
//invariant.
func (s *Stack) RemoveBack() Component {

}

*/
<|endoftext|>


<commit_before>package net

import (
	"fmt"
	"net"

	"github.com/vishvananda/netlink"
)

// A network is considered free if it does not overlap any existing
// routes on this host. This is the same approach taken by Docker.
func CheckNetworkFree(subnet *net.IPNet, ignoreIfaceNames map[string]struct{}) error {
	return forEachRoute(ignoreIfaceNames, func(name string, route netlink.Route) error {
		if route.Dst != nil && overlaps(route.Dst, subnet) {
			return fmt.Errorf("Network %s overlaps with existing route %s on host.", subnet, route.Dst)
		}
		return nil
	})
}

// Two networks overlap if the start-point of one is inside the other.
func overlaps(n1, n2 *net.IPNet) bool {
	return n1.Contains(n2.IP) || n2.Contains(n1.IP)
}

// For a specific address, we only care if it is actually *inside* an
// existing route, because weave-local traffic never hits IP routing.
func CheckAddressOverlap(addr net.IP, ignoreIfaceNames map[string]struct{}) error {
	return forEachRoute(ignoreIfaceNames, func(name string, route netlink.Route) error {
		if route.Dst != nil && route.Dst.Contains(addr) {
			return fmt.Errorf("Address %s overlaps with existing route %s on host.", addr, route.Dst)
		}
		return nil
	})
}

func forEachRoute(ignoreIfaceNames map[string]struct{}, check func(name string, r netlink.Route) error) error {
	routes, err := netlink.RouteList(nil, netlink.FAMILY_V4)
	if err != nil {
		return err
	}
	for _, route := range routes {
		link, err := netlink.LinkByIndex(route.LinkIndex)
		if err != nil {
			continue
		}
		ifaceName := link.Attrs().Name
		if _, found := ignoreIfaceNames[ifaceName]; found {
			continue
		}
		if err := check(ifaceName, route); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>bypass netlink.LinkByIndex<commit_after>package net

import (
	"fmt"
	"net"

	"github.com/vishvananda/netlink"
)

// A network is considered free if it does not overlap any existing
// routes on this host. This is the same approach taken by Docker.
func CheckNetworkFree(subnet *net.IPNet, ignoreIfaceNames map[string]struct{}) error {
	return forEachRoute(ignoreIfaceNames, func(route netlink.Route) error {
		if route.Dst != nil && overlaps(route.Dst, subnet) {
			return fmt.Errorf("Network %s overlaps with existing route %s on host.", subnet, route.Dst)
		}
		return nil
	})
}

// Two networks overlap if the start-point of one is inside the other.
func overlaps(n1, n2 *net.IPNet) bool {
	return n1.Contains(n2.IP) || n2.Contains(n1.IP)
}

// For a specific address, we only care if it is actually *inside* an
// existing route, because weave-local traffic never hits IP routing.
func CheckAddressOverlap(addr net.IP, ignoreIfaceNames map[string]struct{}) error {
	return forEachRoute(ignoreIfaceNames, func(route netlink.Route) error {
		if route.Dst != nil && route.Dst.Contains(addr) {
			return fmt.Errorf("Address %s overlaps with existing route %s on host.", addr, route.Dst)
		}
		return nil
	})
}

func forEachRoute(ignoreIfaceNames map[string]struct{}, check func(r netlink.Route) error) error {
	ignoreIfaceIndices := make(map[int]struct{})
	for ifaceName := range ignoreIfaceNames {
		if iface, err := net.InterfaceByName(ifaceName); err == nil {
			ignoreIfaceIndices[iface.Index] = struct{}{}
		}
	}
	routes, err := netlink.RouteList(nil, netlink.FAMILY_V4)
	if err != nil {
		return err
	}
	for _, route := range routes {
		if _, found := ignoreIfaceIndices[route.LinkIndex]; found {
			continue
		}
		if err := check(route); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2012, Bryan Matsuo. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*  Filename:    lexer.go
 *  Author:      Bryan Matsuo <bryan.matsuo [at] gmail.com>
 *  Created:     2012-11-02 22:10:59.782356 -0700 PDT
 *  Description: Main source file in go-lexer
 */

// Package lexer does ....
package lexer

import (
)
<commit_msg>basic code for a synchronous lexer<commit_after>// Copyright 2012, Bryan Matsuo. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*  Filename:    lexer.go
 *  Author:      Bryan Matsuo <bryan.matsuo [at] gmail.com>
 *  Created:     2012-11-02 22:10:59.782356 -0700 PDT
 *  Description: Main source file in go-lexer
 */

// Package lexer provides a simple scanner and types for handrolling lexers.
// The implementation is based on Rob Pikes talk.
//	  	http://www.youtube.com/watch?v=HxaD_trXwRE
package lexer

import (
	"container/ring"
	"fmt"
	"math"
	"strings"
	"unicode"
	"unicode/utf8"
)

const EOF rune = 0x04

// A state function that scans runes from the lexer's input and emits items.
// The state functions are responsible for emitting ItemEOF.
type StateFn func(*Lexer) StateFn

type Lexer struct {
	input string     // string being scanned
	start int        // start position for the current lexeme
	pos   int        // current position
	width int        // length of the last rune read
	last  rune       // the last rune read
	state StateFn    // the current state
	items *ring.Ring // buffer of lexed items
}

// Create a new lexer. Use the lexer by instantiating it and repeatedly calling Next.
func New(start StateFn, input string) *Lexer {
	return &Lexer{input: input}
}

// The starting position of the current item.
func (l *Lexer) Start() int {
	return l.start
}

// The position following the last read rune.
func (l *Lexer) Pos() int {
	return l.pos
}

// The contents of the item currently being lexed.
func (l *Lexer) Current() string {
	return l.input[l.start:l.pos]
}

// The last rune read from the input stream.
func (l *Lexer) Last() (r rune, width int) {
	return l.last, l.width
}

// Add one rune of the input stream to the current lexeme.
func (l *Lexer) Advance() rune {
	if l.pos >= len(l.input) {
		l.width = 0
		return EOF
	}
	l.last, l.width = utf8.DecodeRuneInString(l.input[l.pos:])
	l.pos += l.width
	return l.last
}

// Remove the last rune from the current lexeme and place back in the stream.
func (l *Lexer) Backup() {
	l.pos -= l.width
}

// Returns the next rune in the input stream without adding it to the current lexeme.
func (l *Lexer) Peek() (r rune, width int) {
	r := l.Advance()
	width = l.width
	l.Backup()
	return
}

// Throw away the current lexeme (do not call Emit).
func (l *Lexer) Ignore() {
	l.start = l.pos
}

// Advance the lexer only if the next rune is in the valid string.
func (l *Lexer) Accept(valid string) (ok bool) {
	if ok = strings.IndexRune(valid, l.Advance()) >= 0; !ok {
		l.Backup()
	}
	return
}

// Advance the lexer as long the next rune is in the valid string.
func (l *Lexer) AcceptRun(valid string) (n int) {
	for l.Accept(valid) {
		n++
	}
	return
}

// Like Advance but uses a range table for efficiency.
func (l *Lexer) AcceptRangeTable(rangeTab *unicode.RangeTable) (ok bool) {
	r := l.Advance()
	if ok = unicode.Is(rangeTab, r); !ok {
		l.Backup()
	}
	return
}

// Like AdvanceRun but uses a range table for efficiency.
func (l *Lexer) AcceptRangeTableRun(rangeTab *unicode.RangeTable) (n int) {
	for l.AcceptRangeTable(rangeTab) {
		n++
	}
	return
}

// Emit an error from the Lexer.
func (l *Lexer) Errorf(format string, args ...interface{}) StateFn {
	l.enqueue(&Item{
		ItemError,
		l.start,
		fmt.Sprintf(format, args...),
	})
	return nil
}

// Emit the current value as an Item with the specified type.
func (l *Lexer) Emit(t ItemType) {
	l.enqueue(&Item{
		t,
		l.start,
		l.input[l.start:l.pos],
	})
	l.start = l.pos
}

// The method by which items are extracted from the input.
// Can result in infinite loops if state functions fail to emit ItemEOF.
func (l *Lexer) Next() *Item {
	for l.items == nil {
		l.state = l.state(l)
	}
	return l.dequeue()
}

func (l *Lexer) enqueue(i *Item) {
	r := ring.New(1)
	r.Value = i
	if l.items == nil {
		l.items = r
	} else {
		l.items.Prev().Link(r)
	}
}

func (l *Lexer) dequeue() *Item {
	if l.items == nil {
		return nil
	}
	r := l.items.Unlink(1)
	if r == l.items {
		l.items = nil
	}
	return r.Value.(*Item)
}

// A type for all the types of items in the language being lexed.
type ItemType uint16

// Special item types.
const (
	ItemEOF ItemType = math.MaxUint16 - iota
	ItemError
)

// An individual scanned item (a lexeme).
type Item struct {
	Type  ItemType
	Pos   int
	Value string
}

func (i *Item) String() string {
	switch i.Type {
	case ItemError:
		return i.Value
	case ItemEOF:
		return "EOF"
	}
	if len(i.Value) > 10 {
		return fmt.Sprintf("%.10q...", i.Value)
	}
	return i.Value
}
<|endoftext|>


<commit_before>/*
 * tracker.go
 *
 * The tracker code pulls double duty as both our tracker (helps peers find each other) but
 * is also the endpoint where people download torrent files.
 *
 * Copyright (c) 2014 by authors and contributors. Please see the included LICENSE file for
 * licensing information.
 *
 */

package torrent

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"sync"
	"time"

	bencode "github.com/jackpal/bencode-go"
)

type Peer struct {
	Id   string `peer id`
	Ip   string `ip`
	Port uint16 `port`
}

type PeerResponse struct {
	Interval int    `interval`
	Peers    []Peer `peers`
}

type Tracker struct {
	// We keep a separate set of peers for each info_hash. We don't actually verify that these
	// hashes are valid; so there's a pretty easy DoS here. This system is designed to be used
	// in a production environment with good actors. TODO: harden.
	// TODO: We need a way of droppign peers that have not reported in a while.
	PeerSeen     map[string]map[string]time.Time
	PeerList     map[string]map[string]Peer
	peerListLock sync.Mutex

	// Lock used by all methods that affect the seed process.
	seedStartLock sync.Mutex

	// The key in the watchers map is how these watchers can be queried for the latest data
	// see handleServeLastUpdated()
	//
	// Careful: there is no locking here. It's assumed that the only time this is
	// written is from the very initial setup of the app and never during runtime. If that
	// changes we'll need locking. (This may actually be technically a little racy right
	// now if there's a ton of requests during power-on, since we start listening
	// before the watchers are created.)
	watchers map[string]*Watcher // List of watchers who might have files.
	ctorrent string              // path to the ctorrent executable.
}

// findFile searches all of our watchers for a given filename (FQFN). If found, it returns
// the pointer to the File structure representing this file.
func (self *Tracker) findFile(name string) *File {
	for _, watcher := range self.watchers {
		if file := watcher.GetFile(name); file != nil {
			return file
		}
	}
	return nil
}

// findLastUpdatedFile goes through all the watchers and returns the file with the latest
// modification time or nil if no such file could be found; only considers files that have
// non-nil metadata, as there are cases where we don't generate metadata for files
// that exist (e.g. 0-length)
func (self *Tracker) findLastUpdatedFile(watchers []*Watcher) *File {
	var last_updated *File = nil
	for _, watcher := range watchers {
		for _, file := range watcher.GetFiles() {
			file.Lock.Lock()
			if file.MetadataInfo == nil {
				file.Lock.Unlock()
				continue
			}
			file.Lock.Unlock()
			if last_updated == nil || file.ModTime.After(last_updated.ModTime) {
				last_updated = file
			}
		}
	}
	return last_updated
}

// startSeed attempts to start up a seeding process for a given torrent file.
func (self *Tracker) startSeed(file *File, metadata *Metadata) {
	self.seedStartLock.Lock()

	if file.SeedCommand != nil {
		self.seedStartLock.Unlock()
		return
	}

	tmp, err := ioutil.TempFile("", "distributor.")
	if err != nil {
		LogFatal("TempFile failed: %s", err)
	}
	LogDebug("Temporary file for %s: %s", file.Name, tmp.Name())

	err = bencode.Marshal(tmp, *metadata)
	if err != nil {
		self.seedStartLock.Unlock()
		LogError("Failed to bencode %s: %s", file.Name, err)
		return
	}

	err = tmp.Sync()
	if err != nil {
		self.seedStartLock.Unlock()
		LogError("Failed to fsync: %s", err)
		return
	}

	file.SeedCommand = exec.Command(
		self.ctorrent,
		"-s",
		file.FQFN,
		"-e",
		"4",
		"-p",
		"8999",
		tmp.Name())
	self.seedStartLock.Unlock()

	// TODO: Read from output pipes, because they could fill up?

	go func() {
		LogDebug("Seed starting: %s", file.Name)
		file.SeedCommand.Run()
		LogDebug("Seed exited: %s", file.Name)

		// Try to clean up temporary file.
		tmp.Close()
		os.Remove(tmp.Name())

		// Seeds exit after 4 hours. Then they get restarted if someone requests them.
		self.seedStartLock.Lock()
		file.SeedCommand = nil
		self.seedStartLock.Unlock()
	}()
}

// handleServe is the endpoint that is responsible for generating torrent files and giving them
// out to the requestors.
// TODO: how to return 404 etc from here?
func (self *Tracker) handleServe(w http.ResponseWriter, r *http.Request) {
	LogDebug("Request: %s", r.URL.RequestURI())
	pieces := strings.SplitN(r.URL.RequestURI(), "?", 2)
	if len(pieces) != 2 {
		io.WriteString(w, "invalid request")
		return
	}

	file := self.findFile(pieces[1])
	self.serveFile(w, r, file)
}

// handleServeLatest is the endpoint that is responsible for serving the latest file that was updated
func (self *Tracker) handleServeLastUpdated(w http.ResponseWriter, r *http.Request) {
	LogDebug("Request: %s", r.URL.RequestURI())

	var query_watchers []*Watcher

	pieces := strings.SplitN(r.URL.RequestURI(), "?", 2)
	if len(pieces) > 2 {
		io.WriteString(w, "invalid request")
		return
	} else if len(pieces) == 2 {
		// query the specified watcher
		watcher := self.watchers[pieces[1]]
		if watcher == nil {
			io.WriteString(w, "invalid watcher name")
			return
		}
		query_watchers = append(query_watchers, watcher)
	} else {
		// query all watchers
		for _, watcher := range self.watchers {
			query_watchers = append(query_watchers, watcher)
		}
	}

	file := self.findLastUpdatedFile(query_watchers)
	self.serveFile(w, r, file)
}

func (self *Tracker) serveFile(w http.ResponseWriter, r *http.Request, file *File) {
	if file == nil {
		http.Error(w, "File not found", 404)
		return
	}

	for {
		// TODO: This could run infinitely in a case where the file is requested and deleted or
		// replaced, so we keep checking a structure that never will get filled in since it's no
		// longer active.
		file.Lock.Lock()
		if file.MetadataInfo == nil {
			file.Lock.Unlock()
			LogDebug("Request for missing metadata on %v. Sleeping.", file.Name)
			time.Sleep(1 * time.Second)
			continue
		}
		file.Lock.Unlock()
		break
	}

	file.Lock.Lock()
	md := Metadata{
		// Using Host like this is probably safe, but is potentially a hack.
		Announce: fmt.Sprintf("http://%s/announce", r.Host),
		Info:     *file.MetadataInfo,
	}
	file.Lock.Unlock()

	if file.SeedCommand == nil {
		self.startSeed(file, &md)
	}

	err := bencode.Marshal(w, md)
	if err != nil {
		LogError("Failed to bencode %s: %s", file.Name, err)
	}
}

// parsePeer extracts a Peer structure from a query string.
func parsePeer(r *http.Request, values url.Values) (*Peer, error) {
	var peer_id, ip, strport []string
	ok := true

	// I don't know how to make this cleaner in Go. Halp. :-(
	peer_id, ok = values["peer_id"]
	if ok && len(peer_id) == 1 {
		strport, ok = values["port"]
	}
	if !ok {
		return nil, errors.New("missing required argument")
	}

	ip, ok = values["ip"]
	if !ok {
		// TODO: This seems fragile.
		addr := strings.Split(r.RemoteAddr, ":")
		if len(addr) != 2 {
			LogFatal("Got weird address: %s", r.RemoteAddr)
		}
		ip = []string{addr[0]}
	}

	port, err := strconv.ParseUint(strport[0], 10, 16)
	if err != nil {
		return nil, errors.New("port invalid")
	}

	return &Peer{
		Id:   peer_id[0],
		Ip:   ip[0],
		Port: uint16(port),
	}, nil
}

// handleAnnounce is the endpoint for torrent clients to announce themselves and request
// other peers.
func (self *Tracker) handleAnnounce(w http.ResponseWriter, r *http.Request) {
	values := r.URL.Query()

	peer, err := parsePeer(r, values)
	if err != nil {
		io.WriteString(w, err.Error())
		return
	}
	LogDebug("Request from peer at %s:%d.", peer.Ip, peer.Port)

	// Get other arguments and validate them.
	var info_hash string
	if info_hash_list, ok := values["info_hash"]; ok && len(info_hash_list) == 1 {
		info_hash = info_hash_list[0]
	}

	var event string
	if event_list, ok := values["event"]; ok && len(event_list) == 1 {
		event = event_list[0]
	}

	var numwant uint64
	if numwant_list, ok := values["numwant"]; ok && len(numwant_list) == 1 {
		numwant, err = strconv.ParseUint(numwant_list[0], 10, 8)
		if err != nil || numwant > 100 {
			numwant = 100
		}
	} else {
		numwant = 50
	}

	// Lock this now since we're validated our inputs.
	self.peerListLock.Lock()
	defer self.peerListLock.Unlock()

	peers, ok := self.PeerList[info_hash]
	if !ok {
		peers = make(map[string]Peer)
		self.PeerList[info_hash] = peers
	}

	peerseen, ok := self.PeerSeen[info_hash]
	if !ok {
		peerseen = make(map[string]time.Time)
		self.PeerSeen[info_hash] = peerseen
	}

	// Add this peer to the set if they don't exist, plus possibly purge other peers on this IP and port.
	if _, ok := peers[peer.Id]; !ok {
		// Remove any other peers on this IP address and port. This is kind of a hack since we don't have
		// "last reported time" at the moment. If a new peer starts up on a host, then we remove
		// the other one.
		toRemove := make([]string, 0, 10)
		for id, tmpPeer := range peers {
			if tmpPeer.Ip == peer.Ip && tmpPeer.Port == peer.Port {
				toRemove = append(toRemove, id)
			}
		}
		for _, id := range toRemove {
			delete(peers, id)
			delete(peerseen, id)
		}

		// Finally insert this new peer.
		peers[peer.Id] = *peer
	}

	// Always update the timestamp so we know when people report.
	peerseen[peer.Id] = time.Now()

	// Remove old peers.
	for id := range peers {
		peerlastseen, pok := peerseen[id]
		if pok && time.Since(peerlastseen) > 300*time.Second {
			delete(peers, id)
			delete(peerseen, id)
		}
	}

	// If they're stopping, then remove this peer from the valid list.
	if event == "stopped" {
		LogInfo("Peer %s:%d is leaving the swarm.", peer.Ip, peer.Port)
		delete(peers, peer.Id)
		delete(peerseen, peer.Id)
	}

	// We give the user back N random peers by just picking a window into our peer list.
	ct := 0
	outPeers := make([]Peer, 0, numwant)
	for _, tmpPeer := range peers {
		if ct++; ct > cap(outPeers) {
			break
		}

		if tmpPeer.Ip == peer.Ip && tmpPeer.Port == peer.Port {
			// This helps avoid giving peers connections to their own machine, which seems
			// to confuse ctorrent. It seems to mostly affect small clusters.
			continue
		}
		outPeers = append(outPeers, tmpPeer)
		LogDebug("[%s:%d] peer %s:%d", peer.Ip, peer.Port, tmpPeer.Ip, tmpPeer.Port)
	}
	LogInfo("Giving peer %s:%d a list of %d peers (out of %d).",
		peer.Ip, peer.Port, len(outPeers), len(peers))

	// Build the output dictionary and return it.
	err = bencode.Marshal(w, PeerResponse{Interval: rand.Intn(120) + 120, Peers: outPeers})
	if err != nil {
		LogError("Failed to bencode: %s", err)
	}
}

// starTracker spins up a tracker on a given ip:port for the given set of watchers.
func StartTracker(ip string, port int,
	ctorrentPath string,
	watchers map[string]*Watcher) *Tracker {
	tracker := &Tracker{
		PeerList: make(map[string]map[string]Peer),
		PeerSeen: make(map[string]map[string]time.Time),
		watchers: watchers,
		ctorrent: ctorrentPath,
	}

	http.HandleFunc("/serve", tracker.handleServe)
	http.HandleFunc("/serve_last_updated", tracker.handleServeLastUpdated)
	http.HandleFunc("/announce", tracker.handleAnnounce)

	go func() {
		err := http.ListenAndServe(fmt.Sprintf("%s:%d", ip, port), nil)
		LogFatal("HTTP server exited: %s", err)
	}()

	return tracker
}
<commit_msg>More efficient cleanup<commit_after>/*
 * tracker.go
 *
 * The tracker code pulls double duty as both our tracker (helps peers find each other) but
 * is also the endpoint where people download torrent files.
 *
 * Copyright (c) 2014 by authors and contributors. Please see the included LICENSE file for
 * licensing information.
 *
 */

package torrent

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"sync"
	"time"

	bencode "github.com/jackpal/bencode-go"
)

type Peer struct {
	Id   string `peer id`
	Ip   string `ip`
	Port uint16 `port`
}

type PeerResponse struct {
	Interval int    `interval`
	Peers    []Peer `peers`
}

type Tracker struct {
	// We keep a separate set of peers for each info_hash. We don't actually verify that these
	// hashes are valid; so there's a pretty easy DoS here. This system is designed to be used
	// in a production environment with good actors. TODO: harden.
	// TODO: We need a way of droppign peers that have not reported in a while.
	PeerSeen     map[string]map[string]time.Time
	PeerList     map[string]map[string]Peer
	peerListLock sync.Mutex

	// Lock used by all methods that affect the seed process.
	seedStartLock sync.Mutex

	// The key in the watchers map is how these watchers can be queried for the latest data
	// see handleServeLastUpdated()
	//
	// Careful: there is no locking here. It's assumed that the only time this is
	// written is from the very initial setup of the app and never during runtime. If that
	// changes we'll need locking. (This may actually be technically a little racy right
	// now if there's a ton of requests during power-on, since we start listening
	// before the watchers are created.)
	watchers map[string]*Watcher // List of watchers who might have files.
	ctorrent string              // path to the ctorrent executable.
}

// findFile searches all of our watchers for a given filename (FQFN). If found, it returns
// the pointer to the File structure representing this file.
func (self *Tracker) findFile(name string) *File {
	for _, watcher := range self.watchers {
		if file := watcher.GetFile(name); file != nil {
			return file
		}
	}
	return nil
}

// findLastUpdatedFile goes through all the watchers and returns the file with the latest
// modification time or nil if no such file could be found; only considers files that have
// non-nil metadata, as there are cases where we don't generate metadata for files
// that exist (e.g. 0-length)
func (self *Tracker) findLastUpdatedFile(watchers []*Watcher) *File {
	var last_updated *File = nil
	for _, watcher := range watchers {
		for _, file := range watcher.GetFiles() {
			file.Lock.Lock()
			if file.MetadataInfo == nil {
				file.Lock.Unlock()
				continue
			}
			file.Lock.Unlock()
			if last_updated == nil || file.ModTime.After(last_updated.ModTime) {
				last_updated = file
			}
		}
	}
	return last_updated
}

// startSeed attempts to start up a seeding process for a given torrent file.
func (self *Tracker) startSeed(file *File, metadata *Metadata) {
	self.seedStartLock.Lock()

	if file.SeedCommand != nil {
		self.seedStartLock.Unlock()
		return
	}

	tmp, err := ioutil.TempFile("", "distributor.")
	if err != nil {
		LogFatal("TempFile failed: %s", err)
	}
	LogDebug("Temporary file for %s: %s", file.Name, tmp.Name())

	err = bencode.Marshal(tmp, *metadata)
	if err != nil {
		self.seedStartLock.Unlock()
		LogError("Failed to bencode %s: %s", file.Name, err)
		return
	}

	err = tmp.Sync()
	if err != nil {
		self.seedStartLock.Unlock()
		LogError("Failed to fsync: %s", err)
		return
	}

	file.SeedCommand = exec.Command(
		self.ctorrent,
		"-s",
		file.FQFN,
		"-e",
		"4",
		"-p",
		"8999",
		tmp.Name())
	self.seedStartLock.Unlock()

	// TODO: Read from output pipes, because they could fill up?

	go func() {
		LogDebug("Seed starting: %s", file.Name)
		file.SeedCommand.Run()
		LogDebug("Seed exited: %s", file.Name)

		// Try to clean up temporary file.
		tmp.Close()
		os.Remove(tmp.Name())

		// Seeds exit after 4 hours. Then they get restarted if someone requests them.
		self.seedStartLock.Lock()
		file.SeedCommand = nil
		self.seedStartLock.Unlock()
	}()
}

// handleServe is the endpoint that is responsible for generating torrent files and giving them
// out to the requestors.
// TODO: how to return 404 etc from here?
func (self *Tracker) handleServe(w http.ResponseWriter, r *http.Request) {
	LogDebug("Request: %s", r.URL.RequestURI())
	pieces := strings.SplitN(r.URL.RequestURI(), "?", 2)
	if len(pieces) != 2 {
		io.WriteString(w, "invalid request")
		return
	}

	file := self.findFile(pieces[1])
	self.serveFile(w, r, file)
}

// handleServeLatest is the endpoint that is responsible for serving the latest file that was updated
func (self *Tracker) handleServeLastUpdated(w http.ResponseWriter, r *http.Request) {
	LogDebug("Request: %s", r.URL.RequestURI())

	var query_watchers []*Watcher

	pieces := strings.SplitN(r.URL.RequestURI(), "?", 2)
	if len(pieces) > 2 {
		io.WriteString(w, "invalid request")
		return
	} else if len(pieces) == 2 {
		// query the specified watcher
		watcher := self.watchers[pieces[1]]
		if watcher == nil {
			io.WriteString(w, "invalid watcher name")
			return
		}
		query_watchers = append(query_watchers, watcher)
	} else {
		// query all watchers
		for _, watcher := range self.watchers {
			query_watchers = append(query_watchers, watcher)
		}
	}

	file := self.findLastUpdatedFile(query_watchers)
	self.serveFile(w, r, file)
}

func (self *Tracker) serveFile(w http.ResponseWriter, r *http.Request, file *File) {
	if file == nil {
		http.Error(w, "File not found", 404)
		return
	}

	for {
		// TODO: This could run infinitely in a case where the file is requested and deleted or
		// replaced, so we keep checking a structure that never will get filled in since it's no
		// longer active.
		file.Lock.Lock()
		if file.MetadataInfo == nil {
			file.Lock.Unlock()
			LogDebug("Request for missing metadata on %v. Sleeping.", file.Name)
			time.Sleep(1 * time.Second)
			continue
		}
		file.Lock.Unlock()
		break
	}

	file.Lock.Lock()
	md := Metadata{
		// Using Host like this is probably safe, but is potentially a hack.
		Announce: fmt.Sprintf("http://%s/announce", r.Host),
		Info:     *file.MetadataInfo,
	}
	file.Lock.Unlock()

	if file.SeedCommand == nil {
		self.startSeed(file, &md)
	}

	err := bencode.Marshal(w, md)
	if err != nil {
		LogError("Failed to bencode %s: %s", file.Name, err)
	}
}

// parsePeer extracts a Peer structure from a query string.
func parsePeer(r *http.Request, values url.Values) (*Peer, error) {
	var peer_id, ip, strport []string
	ok := true

	// I don't know how to make this cleaner in Go. Halp. :-(
	peer_id, ok = values["peer_id"]
	if ok && len(peer_id) == 1 {
		strport, ok = values["port"]
	}
	if !ok {
		return nil, errors.New("missing required argument")
	}

	ip, ok = values["ip"]
	if !ok {
		// TODO: This seems fragile.
		addr := strings.Split(r.RemoteAddr, ":")
		if len(addr) != 2 {
			LogFatal("Got weird address: %s", r.RemoteAddr)
		}
		ip = []string{addr[0]}
	}

	port, err := strconv.ParseUint(strport[0], 10, 16)
	if err != nil {
		return nil, errors.New("port invalid")
	}

	return &Peer{
		Id:   peer_id[0],
		Ip:   ip[0],
		Port: uint16(port),
	}, nil
}

// handleAnnounce is the endpoint for torrent clients to announce themselves and request
// other peers.
func (self *Tracker) handleAnnounce(w http.ResponseWriter, r *http.Request) {
	values := r.URL.Query()

	peer, err := parsePeer(r, values)
	if err != nil {
		io.WriteString(w, err.Error())
		return
	}
	LogDebug("Request from peer at %s:%d.", peer.Ip, peer.Port)

	// Get other arguments and validate them.
	var info_hash string
	if info_hash_list, ok := values["info_hash"]; ok && len(info_hash_list) == 1 {
		info_hash = info_hash_list[0]
	}

	var event string
	if event_list, ok := values["event"]; ok && len(event_list) == 1 {
		event = event_list[0]
	}

	var numwant uint64
	if numwant_list, ok := values["numwant"]; ok && len(numwant_list) == 1 {
		numwant, err = strconv.ParseUint(numwant_list[0], 10, 8)
		if err != nil || numwant > 100 {
			numwant = 100
		}
	} else {
		numwant = 50
	}

	// Lock this now since we're validated our inputs.
	self.peerListLock.Lock()
	defer self.peerListLock.Unlock()

	peers, ok := self.PeerList[info_hash]
	if !ok {
		peers = make(map[string]Peer)
		self.PeerList[info_hash] = peers
	}

	peerseen, ok := self.PeerSeen[info_hash]
	if !ok {
		peerseen = make(map[string]time.Time)
		self.PeerSeen[info_hash] = peerseen
	}

	// Add this peer to the set if they don't exist, plus possibly purge other peers on this IP and port.
	if _, ok := peers[peer.Id]; !ok {
		// Remove any other peers on this IP address and port. This is kind of a hack since we don't have
		// "last reported time" at the moment. If a new peer starts up on a host, then we remove
		// the other one.
		toRemove := make([]string, 0, 10)
		for id, tmpPeer := range peers {
			if tmpPeer.Ip == peer.Ip && tmpPeer.Port == peer.Port {
				toRemove = append(toRemove, id)
			}
		}
		for _, id := range toRemove {
			delete(peers, id)
			delete(peerseen, id)
		}

		// Finally insert this new peer.
		peers[peer.Id] = *peer
	}

	// Always update the timestamp so we know when people report.
	peerseen[peer.Id] = time.Now()

	// If they're stopping, then remove this peer from the valid list.
	if event == "stopped" {
		LogInfo("Peer %s:%d is leaving the swarm.", peer.Ip, peer.Port)
		delete(peers, peer.Id)
		delete(peerseen, peer.Id)
	}

	// We give the user back N random peers by just picking a window into our peer list.
	ct := 0
	outPeers := make([]Peer, 0, numwant)
	for id, tmpPeer := range peers {
		// Don't hand out timed-out peers.
		if time.Since(peerseen[id]) > 600*time.Second {
			delete(peers, id)
			delete(peerseen, id)
			continue
		}

		if ct++; ct > cap(outPeers) {
			break
		}

		if tmpPeer.Ip == peer.Ip && tmpPeer.Port == peer.Port {
			// This helps avoid giving peers connections to their own machine, which seems
			// to confuse ctorrent. It seems to mostly affect small clusters.
			continue
		}
		outPeers = append(outPeers, tmpPeer)
		LogDebug("[%s:%d] peer %s:%d", peer.Ip, peer.Port, tmpPeer.Ip, tmpPeer.Port)
	}
	LogInfo("Giving peer %s:%d a list of %d peers (out of %d).",
		peer.Ip, peer.Port, len(outPeers), len(peers))

	// Build the output dictionary and return it.
	err = bencode.Marshal(w, PeerResponse{Interval: rand.Intn(120) + 300, Peers: outPeers})
	if err != nil {
		LogError("Failed to bencode: %s", err)
	}
}

// starTracker spins up a tracker on a given ip:port for the given set of watchers.
func StartTracker(ip string, port int,
	ctorrentPath string,
	watchers map[string]*Watcher) *Tracker {
	tracker := &Tracker{
		PeerList: make(map[string]map[string]Peer),
		PeerSeen: make(map[string]map[string]time.Time),
		watchers: watchers,
		ctorrent: ctorrentPath,
	}

	http.HandleFunc("/serve", tracker.handleServe)
	http.HandleFunc("/serve_last_updated", tracker.handleServeLastUpdated)
	http.HandleFunc("/announce", tracker.handleAnnounce)

	go func() {
		err := http.ListenAndServe(fmt.Sprintf("%s:%d", ip, port), nil)
		LogFatal("HTTP server exited: %s", err)
	}()

	return tracker
}
<|endoftext|>


<commit_before>package torrent

import (
	"github.com/cenkalti/rain/internal/piecewriter"
	"github.com/cenkalti/rain/internal/urldownloader"
)

func (t *torrent) handleWebseedPieceResult(msg *urldownloader.PieceResult) {
	if msg.Error != nil {
		t.log.Errorln("webseed download error:", msg.Error)
		// TODO handle error
		return
	}

	t.log.Debugln("piece download completed. index:", msg.Index)

	piece := &t.pieces[msg.Index]

	t.resumerStats.BytesDownloaded += int64(len(msg.Buffer.Data))

	// TODO update download speed
	// t.downloadSpeed.Update(int64(len(msg.Buffer.Data)))

	if piece.Writing {
		panic("piece is already writing")
	}
	piece.Writing = true

	// Prevent receiving piece messages to avoid more than 1 write per torrent.
	t.pieceMessagesC.Suspend()
	t.webseedPieceResultC.Suspend()

	pw := piecewriter.New(piece, msg.Downloader, msg.Buffer)
	go pw.Run(t.pieceWriterResultC, t.doneC)
}
<commit_msg>update download speed on webseed piece downloaded<commit_after>package torrent

import (
	"github.com/cenkalti/rain/internal/piecewriter"
	"github.com/cenkalti/rain/internal/urldownloader"
)

func (t *torrent) handleWebseedPieceResult(msg *urldownloader.PieceResult) {
	if msg.Error != nil {
		t.log.Errorln("webseed download error:", msg.Error)
		// TODO handle error
		return
	}

	t.log.Debugln("piece download completed. index:", msg.Index)

	piece := &t.pieces[msg.Index]

	t.resumerStats.BytesDownloaded += int64(len(msg.Buffer.Data))
	t.downloadSpeed.Update(int64(len(msg.Buffer.Data)))

	if piece.Writing {
		panic("piece is already writing")
	}
	piece.Writing = true

	// Prevent receiving piece messages to avoid more than 1 write per torrent.
	t.pieceMessagesC.Suspend()
	t.webseedPieceResultC.Suspend()

	pw := piecewriter.New(piece, msg.Downloader, msg.Buffer)
	go pw.Run(t.pieceWriterResultC, t.doneC)
}
<|endoftext|>