<commit_before>// Copyright 2014 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// TODO(adg): accept -a flag on 'create' and 'commit' (like git commit -a)
// TODO(adg): accept -r flag on 'upload' to nominate reviewer
// TODO(adg): support 'create' from non-master branches
// TODO(adg): check style of commit message
// TODO(adg): write doc comment
// TOOD(adg): print gerrit votes on 'pending'

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
)

var (
	hookFile = filepath.FromSlash(".git/hooks/commit-msg")
	verbose  = flag.Bool("v", false, "verbose output")
)

const usage = `Usage: %s [-v] <command>
Type "%s help" for more information.
`

const help = `Usage: %s [-v] <command>

The review command is a wrapper for the git command that provides a simple
interface to the "single-commit feature branch" development model.

Available comands:

	create <name>
		Create a local branch with the provided name
		and commit the staged changes to it.

	commit
		Amend local branch HEAD commit with the staged changes.

	diff
		View differences between remote branch HEAD and
		the local branch HEAD.
		(The differences introduced by this change.)

	upload
		Upload HEAD commit to the code review server.

	sync
		Fetch changes from the remote repository and merge them to the
		current branch, rebasing the HEAD commit (if any) on top of
		them. If the HEAD commit has been submitted, switch back to the
		master branch and delete the feature branch.

	pending
		Show local branches and their head commits.

`

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, usage, os.Args[0], os.Args[0])
		os.Exit(2)
	}
	flag.Parse()

	goToRepoRoot()
	installHook()

	switch flag.Arg(0) {
	case "help":
		fmt.Fprintf(os.Stdout, help, os.Args[0])
	case "create", "cr":
		name := flag.Arg(1)
		if name == "" {
			flag.Usage()
		}
		create(name)
	case "commit", "co":
		commit()
	case "diff", "d":
		diff()
	case "upload", "u":
		upload()
	case "sync", "s":
		doSync()
	case "pending", "p":
		pending()
	default:
		flag.Usage()
	}
}

func create(name string) {
	if !hasStagedChanges() {
		dief("no staged changes.\nDid you forget to 'git add'?")
	}
	if currentBranch() != "master" {
		dief("must run 'create' from the master branch.\n" +
			"(Try 'review sync' or 'git checkout master' first.)")
	}
	run("git", "checkout", "-q", "-b", name)
	if err := runErr("git", "commit", "-q"); err != nil {
		verbosef("Commit failed: %v\nSwitching back to master.\n", err)
		run("git", "checkout", "-q", "master")
		run("git", "branch", "-q", "-d", name)
	}
}

func commit() {
	if !hasStagedChanges() {
		dief("no staged changes. Did you forget to 'git add'?")
	}
	if currentBranch() == "master" {
		dief("can't commit to master branch.")
	}
	run("git", "commit", "-q", "--amend", "-C", "HEAD")
}

func diff() {
	run("git", "diff", "HEAD^", "HEAD")
}

func upload() {
	if currentBranch() == "master" {
		dief("can't upload from master branch.")
	}
	run("git", "push", "-q", "origin", "HEAD:refs/for/master")
}

func doSync() {
	run("git", "fetch", "-q")

	// If we're on master, just fast-forward.
	branch := currentBranch()
	if branch == "master" {
		run("git", "merge", "-q", "--ff-only", "origin/master")
		return
	}

	// Check that exactly this commit was submitted to master. If so,
	// switch back to master, fast-forward, delete the feature branch.
	if branchContains("origin/master", "HEAD") {
		run("git", "checkout", "-q", "master")
		run("git", "merge", "-q", "--ff-only", "origin/master")
		run("git", "branch", "-q", "-d", branch)
		return
	}

	// Check whether a rebased version of this commit was submitted to
	// master. If so, switch back to master, fast-forward, and
	// provide instructions for deleting the feature branch.
	// (We're not 100% sure that the feature branch HEAD was submitted,
	// so be cautious.)
	if headSubmitted(branch) {
		run("git", "checkout", "-q", "master")
		run("git", "merge", "-q", "--ff-only", "origin/master")
		fmt.Fprintf(os.Stderr, "Switched back to master from %q, "+
			"which I think has been submitted.\n"+
			"If you agree, and no longer need branch %q, run:\n"+
			"\tgit branch -D %v\n",
			branch, branch, branch)
		return
	}

	// Bump master HEAD to that of origin/master, just in case the user
	// switches back to master with "git checkout master" later.
	// TODO(adg): maybe we shouldn't do this at all?
	if !branchContains("origin/master", "master") {
		run("git", "branch", "-f", "master", "origin/master")
	}

	// We have un-submitted changes on this feature branch; rebase.
	run("git", "rebase", "-q", "origin/master")
}

func pending() {
	var (
		wg      sync.WaitGroup
		origin  = originURL()
		current = currentBranch()
	)
	if current == "master" {
		fmt.Println("On master branch.")
	}
	for _, branch := range localBranches() {
		if branch == "master" {
			continue
		}
		wg.Add(1)
		go func(branch string) {
			defer wg.Done()
			p := ""
			if branch == current {
				p = "* "
			}
			id := headChangeId(branch)
			c, err := getChange(origin, id)
			switch err {
			case notFound:
				// TODO(adg): read local commit msg
				var msg string
				fmt.Printf("%v%v:\n\t%v\n\t(not uploaded)\n",
					p, branch, msg)
			case nil:
				status := ""
				switch c.Status {
				case "MERGED":
					status += " [submitted]"
				case "ABANDONED":
					status += " [abandoned]"
				}
				fmt.Printf("%v%v%v:\n\t%v\n\t%v\n",
					p, branch, status, c.Subject, c.URL)
			default:
				fmt.Fprintf(os.Stderr, "fetching change for %q: %v\n", branch, err)
			}
		}(branch)
	}
	wg.Wait()
}

func originURL() string {
	out, err := exec.Command("git", "config", "remote.origin.url").CombinedOutput()
	if err != nil {
		dief("could not find URL for 'origin' remote.\n"+
			"Did you check out from the right place?\n"+
			"git config remote.origin.url: %v\n"+
			"%s", err, out)
	}
	return string(out)
}

func localBranches() (branches []string) {
	for _, s := range runLines("git", "branch", "-l", "-q") {
		branches = append(branches, strings.TrimPrefix(s, "* "))
	}
	return branches
}

func branchContains(branch, rev string) bool {
	for _, s := range runLines("git", "branch", "-r", "--contains", rev) {
		if s == branch {
			return true
		}
	}
	return false
}

var stagedRe = regexp.MustCompile(`^[ACDMR]  `)

func hasStagedChanges() bool {
	for _, s := range gitStatus() {
		if stagedRe.MatchString(s) {
			return true
		}
	}
	return false
}

func currentBranch() string {
	const p = "## "
	for _, s := range gitStatus() {
		if strings.HasPrefix(s, p) {
			return strings.TrimPrefix(s, p)
		}
	}
	dief("could not find current branch with 'git status'.")
	panic("unreachable")
}

func gitStatus() []string {
	return runLines("git", "status", "-b", "--porcelain")
}

func headSubmitted(branch string) bool {
	s := "Change-Id: " + headChangeId(branch)
	return len(runOutput("git", "log", "--grep", s, "origin/master")) > 0
}

func headChangeId(branch string) string {
	const (
		p = "Change-Id: "
		f = "--format=format:%b"
	)
	for _, s := range runLines("git", "log", "-n", "1", f, branch, "--") {
		if strings.HasPrefix(s, p) {
			return strings.TrimSpace(strings.TrimPrefix(s, p))
		}
	}
	dief("no Change-Id line found in HEAD commit on branch %s.", branch)
	panic("unreachable")
}

func goToRepoRoot() {
	prevDir, err := os.Getwd()
	if err != nil {
		dief("could not get current directory: %v", err)
	}
	for {
		if _, err := os.Stat(".git"); err == nil {
			return
		}
		if err := os.Chdir(".."); err != nil {
			dief("could not chdir: %v", err)
		}
		currentDir, err := os.Getwd()
		if err != nil {
			dief("could not get current directory: %v", err)
		}
		if currentDir == prevDir {
			dief("git root not found.\n" +
				"Run from within the Git tree please.")
		}
		prevDir = currentDir
	}
}

func installHook() {
	_, err := os.Stat(hookFile)
	if err == nil {
		return
	}
	if !os.IsNotExist(err) {
		dief("error checking for hook file: %v", err)
	}
	verbosef("Presubmit hook to add Change-Id to commit messages is missing.\n"+
		"Automatically creating it at %v.\n", hookFile)
	hookContent := []byte(commitMsgHook)
	if err := ioutil.WriteFile(hookFile, hookContent, 0700); err != nil {
		dief("error writing hook file: %v", err)
	}
}

func dief(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "review: "+format+"\n", args...)
	os.Exit(1)
}

func run(command string, args ...string) {
	if err := runErr(command, args...); err != nil {
		if !*verbose {
			// If we're not in verbose mode, print the command
			// before dying to give context to the failure.
			fmt.Fprintln(os.Stderr, commandString(command, args))
		}
		dief("%v", err)
	}
}

func runErr(command string, args ...string) error {
	if *verbose {
		fmt.Fprintln(os.Stderr, commandString(command, args))
	}
	cmd := exec.Command(command, args...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func verbosef(format string, args ...interface{}) {
	if *verbose {
		fmt.Fprintf(os.Stderr, format, args...)
	}
}

func runOutput(command string, args ...string) string {
	b, err := exec.Command(command, args...).CombinedOutput()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n%s\n", commandString(command, args), b)
		dief("%v", err)
	}
	return string(b)
}

func runLines(command string, args ...string) []string {
	var s []string
	for _, l := range strings.Split(runOutput(command, args...), "\n") {
		l = strings.TrimSpace(l)
		if l != "" {
			s = append(s, l)
		}
	}
	return s
}

func commandString(command string, args []string) string {
	return strings.Join(append([]string{command}, args...), " ")
}
<commit_msg>Parse output of 'git status' more conservatively.<commit_after>// Copyright 2014 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// TODO(adg): accept -a flag on 'create' and 'commit' (like git commit -a)
// TODO(adg): accept -r flag on 'upload' to nominate reviewer
// TODO(adg): support 'create' from non-master branches
// TODO(adg): check style of commit message
// TODO(adg): write doc comment
// TOOD(adg): print gerrit votes on 'pending'

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
)

var (
	hookFile = filepath.FromSlash(".git/hooks/commit-msg")
	verbose  = flag.Bool("v", false, "verbose output")
)

const usage = `Usage: %s [-v] <command>
Type "%s help" for more information.
`

const help = `Usage: %s [-v] <command>

The review command is a wrapper for the git command that provides a simple
interface to the "single-commit feature branch" development model.

Available comands:

	create <name>
		Create a local branch with the provided name
		and commit the staged changes to it.

	commit
		Amend local branch HEAD commit with the staged changes.

	diff
		View differences between remote branch HEAD and
		the local branch HEAD.
		(The differences introduced by this change.)

	upload
		Upload HEAD commit to the code review server.

	sync
		Fetch changes from the remote repository and merge them to the
		current branch, rebasing the HEAD commit (if any) on top of
		them. If the HEAD commit has been submitted, switch back to the
		master branch and delete the feature branch.

	pending
		Show local branches and their head commits.

`

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, usage, os.Args[0], os.Args[0])
		os.Exit(2)
	}
	flag.Parse()

	goToRepoRoot()
	installHook()

	switch flag.Arg(0) {
	case "help":
		fmt.Fprintf(os.Stdout, help, os.Args[0])
	case "create", "cr":
		name := flag.Arg(1)
		if name == "" {
			flag.Usage()
		}
		create(name)
	case "commit", "co":
		commit()
	case "diff", "d":
		diff()
	case "upload", "u":
		upload()
	case "sync", "s":
		doSync()
	case "pending", "p":
		pending()
	default:
		flag.Usage()
	}
}

func create(name string) {
	if !hasStagedChanges() {
		dief("no staged changes.\nDid you forget to 'git add'?")
	}
	if b := currentBranch(); b != "master" {
		dief("must run 'create' from the master branch (now on %q).\n"+
			"(Try 'review sync' or 'git checkout master' first.)",
			b)
	}
	run("git", "checkout", "-q", "-b", name)
	if err := runErr("git", "commit", "-q"); err != nil {
		verbosef("Commit failed: %v\nSwitching back to master.\n", err)
		run("git", "checkout", "-q", "master")
		run("git", "branch", "-q", "-d", name)
	}
}

func commit() {
	if !hasStagedChanges() {
		dief("no staged changes. Did you forget to 'git add'?")
	}
	if currentBranch() == "master" {
		dief("can't commit to master branch.")
	}
	run("git", "commit", "-q", "--amend", "-C", "HEAD")
}

func diff() {
	run("git", "diff", "HEAD^", "HEAD")
}

func upload() {
	if currentBranch() == "master" {
		dief("can't upload from master branch.")
	}
	run("git", "push", "-q", "origin", "HEAD:refs/for/master")
}

func doSync() {
	run("git", "fetch", "-q")

	// If we're on master, just fast-forward.
	branch := currentBranch()
	if branch == "master" {
		run("git", "merge", "-q", "--ff-only", "origin/master")
		return
	}

	// Check that exactly this commit was submitted to master. If so,
	// switch back to master, fast-forward, delete the feature branch.
	if branchContains("origin/master", "HEAD") {
		run("git", "checkout", "-q", "master")
		run("git", "merge", "-q", "--ff-only", "origin/master")
		run("git", "branch", "-q", "-d", branch)
		fmt.Printf("Change on %q submitted; branch deleted.\n"+
			"On master branch.\n", branch)
		return
	}

	// Check whether a rebased version of this commit was submitted to
	// master. If so, switch back to master, fast-forward, and
	// provide instructions for deleting the feature branch.
	// (We're not 100% sure that the feature branch HEAD was submitted,
	// so be cautious.)
	if headSubmitted(branch) {
		run("git", "checkout", "-q", "master")
		run("git", "merge", "-q", "--ff-only", "origin/master")
		fmt.Fprintf(os.Stderr,
			"I think the change on %q has been submitted.\n"+
				"If you agree, and no longer need branch %q, "+
				"run:\n\tgit branch -D %v\n"+
				"On master branch.\n",
			branch, branch, branch)
		return
	}

	// Bump master HEAD to that of origin/master, just in case the user
	// switches back to master with "git checkout master" later.
	// TODO(adg): maybe we shouldn't do this at all?
	if !branchContains("origin/master", "master") {
		run("git", "branch", "-f", "master", "origin/master")
	}

	// We have un-submitted changes on this feature branch; rebase.
	run("git", "rebase", "-q", "origin/master")
}

func pending() {
	var (
		wg      sync.WaitGroup
		origin  = originURL()
		current = currentBranch()
	)
	if current == "master" {
		fmt.Println("On master branch.")
	}
	for _, branch := range localBranches() {
		if branch == "master" {
			continue
		}
		wg.Add(1)
		go func(branch string) {
			defer wg.Done()
			p := ""
			if branch == current {
				p = "* "
			}
			id := headChangeId(branch)
			c, err := getChange(origin, id)
			switch err {
			case notFound:
				// TODO(adg): read local commit msg
				var msg string
				fmt.Printf("%v%v:\n\t%v\n\t(not uploaded)\n",
					p, branch, msg)
			case nil:
				status := ""
				switch c.Status {
				case "MERGED":
					status += " [submitted]"
				case "ABANDONED":
					status += " [abandoned]"
				}
				fmt.Printf("%v%v%v:\n\t%v\n\t%v\n",
					p, branch, status, c.Subject, c.URL)
			default:
				fmt.Fprintf(os.Stderr, "fetching change for %q: %v\n", branch, err)
			}
		}(branch)
	}
	wg.Wait()
}

func originURL() string {
	out, err := exec.Command("git", "config", "remote.origin.url").CombinedOutput()
	if err != nil {
		dief("could not find URL for 'origin' remote.\n"+
			"Did you check out from the right place?\n"+
			"git config remote.origin.url: %v\n"+
			"%s", err, out)
	}
	return string(out)
}

func localBranches() (branches []string) {
	for _, s := range runLines("git", "branch", "-l", "-q") {
		branches = append(branches, strings.TrimPrefix(s, "* "))
	}
	return branches
}

func branchContains(branch, rev string) bool {
	for _, s := range runLines("git", "branch", "-r", "--contains", rev) {
		if s == branch {
			return true
		}
	}
	return false
}

var stagedRe = regexp.MustCompile(`^[ACDMR]  `)

func hasStagedChanges() bool {
	for _, s := range gitStatus() {
		if stagedRe.MatchString(s) {
			return true
		}
	}
	return false
}

func currentBranch() string {
	return strings.TrimSpace(runOutput("git", "rev-parse", "--abbrev-ref", "HEAD"))
}

func gitStatus() []string {
	return runLines("git", "status", "-b", "--porcelain")
}

func headSubmitted(branch string) bool {
	s := "Change-Id: " + headChangeId(branch)
	return len(runOutput("git", "log", "--grep", s, "origin/master")) > 0
}

func headChangeId(branch string) string {
	const (
		p = "Change-Id: "
		f = "--format=format:%b"
	)
	for _, s := range runLines("git", "log", "-n", "1", f, branch, "--") {
		if strings.HasPrefix(s, p) {
			return strings.TrimSpace(strings.TrimPrefix(s, p))
		}
	}
	dief("no Change-Id line found in HEAD commit on branch %s.", branch)
	panic("unreachable")
}

func goToRepoRoot() {
	prevDir, err := os.Getwd()
	if err != nil {
		dief("could not get current directory: %v", err)
	}
	for {
		if _, err := os.Stat(".git"); err == nil {
			return
		}
		if err := os.Chdir(".."); err != nil {
			dief("could not chdir: %v", err)
		}
		currentDir, err := os.Getwd()
		if err != nil {
			dief("could not get current directory: %v", err)
		}
		if currentDir == prevDir {
			dief("git root not found.\n" +
				"Run from within the Git tree please.")
		}
		prevDir = currentDir
	}
}

func installHook() {
	_, err := os.Stat(hookFile)
	if err == nil {
		return
	}
	if !os.IsNotExist(err) {
		dief("error checking for hook file: %v", err)
	}
	verbosef("Presubmit hook to add Change-Id to commit messages is missing.\n"+
		"Automatically creating it at %v.\n", hookFile)
	hookContent := []byte(commitMsgHook)
	if err := ioutil.WriteFile(hookFile, hookContent, 0700); err != nil {
		dief("error writing hook file: %v", err)
	}
}

func dief(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "review: "+format+"\n", args...)
	os.Exit(1)
}

func run(command string, args ...string) {
	if err := runErr(command, args...); err != nil {
		if !*verbose {
			// If we're not in verbose mode, print the command
			// before dying to give context to the failure.
			fmt.Fprintln(os.Stderr, commandString(command, args))
		}
		dief("%v", err)
	}
}

func runErr(command string, args ...string) error {
	if *verbose {
		fmt.Fprintln(os.Stderr, commandString(command, args))
	}
	cmd := exec.Command(command, args...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func verbosef(format string, args ...interface{}) {
	if *verbose {
		fmt.Fprintf(os.Stderr, format, args...)
	}
}

func runOutput(command string, args ...string) string {
	b, err := exec.Command(command, args...).CombinedOutput()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n%s\n", commandString(command, args), b)
		dief("%v", err)
	}
	return string(b)
}

func runLines(command string, args ...string) []string {
	var s []string
	for _, l := range strings.Split(runOutput(command, args...), "\n") {
		l = strings.TrimSpace(l)
		if l != "" {
			s = append(s, l)
		}
	}
	return s
}

func commandString(command string, args []string) string {
	return strings.Join(append([]string{command}, args...), " ")
}
<|endoftext|>


<commit_before>package client

import (
	"context"
	"fmt"

	"github.com/mcilloni/go-openbaton/catalogue"
	pop "github.com/mcilloni/openbaton-docker/pop/proto"
)

// Flavour returns the flavour having the given filter as an OpenBaton DeploymentFlavour.
func (cln *Client) Flavour(ctx context.Context, f Filter) (*catalogue.DeploymentFlavour, error) {
	flavs, err := cln.fetchFlavours(ctx, f)
	if err != nil {
		return nil, err
	}

	switch len(flavs) {
	case 0:
		return nil, nil

	case 1:
		return flavs[0], nil

	default:
		return nil, fmt.Errorf("too many flavours returned from query")
	}
}

// Flavours returns all the available flavours as OpenBaton DeploymentFlavour.
func (cln *Client) Flavours(ctx context.Context) ([]*catalogue.DeploymentFlavour, error) {
	return cln.fetchFlavours(ctx, nil)
}

// Image returns the image on the server having the given filter as an OpenBaton NFVImage struct.
func (cln *Client) Image(ctx context.Context, f Filter) (*catalogue.NFVImage, error) {
	imgs, err := cln.fetchImages(ctx, f)
	if err != nil {
		return nil, err
	}

	switch len(imgs) {
	case 0:
		return nil, nil

	case 1:
		return imgs[0], nil

	default:
		return nil, fmt.Errorf("too many images returned from query")
	}
}

// Images returns the images on the server as OpenBaton NFVImage structs.
func (cln *Client) Images(ctx context.Context) ([]*catalogue.NFVImage, error) {
	return cln.fetchImages(ctx, nil)
}

// Network returns the network on the server having the given filter as an OpenBaton Network struct.
func (cln *Client) Network(ctx context.Context, f Filter) (*catalogue.Network, error) {
	nets, err := cln.fetchNetworks(ctx, f)
	if err != nil {
		return nil, err
	}

	switch len(nets) {
	case 0:
		return nil, nil

	case 1:
		return nets[0], nil

	default:
		return nil, fmt.Errorf("too many networks returned from query")
	}
}

// Networks returns the networks on the server as OpenBaton Network structs.
func (cln *Client) Networks(ctx context.Context) ([]*catalogue.Network, error) {
	return cln.fetchNetworks(ctx, nil)
}

// Server returns the container on the server having the given id as an OpenBaton Server struct.
func (cln *Client) Server(ctx context.Context, f Filter) (*catalogue.Server, error) {
	srvs, err := cln.fetchServers(ctx, f)
	if err != nil {
		return nil, err
	}

	switch len(srvs) {
	case 0:
		return nil, nil

	case 1:
		return srvs[0], nil

	default:
		return nil, fmt.Errorf("too many servers returned from query")
	}
}

// Servers returns the containers on the server as OpenBaton Server structs.
func (cln *Client) Servers(ctx context.Context) ([]*catalogue.Server, error) {
	return cln.fetchServers(ctx, nil)
}

// fetchFlavours fetches and converts pop Flavours into DeploymentFlavours.
func (cln *Client) fetchFlavours(ctx context.Context, f Filter) ([]*catalogue.DeploymentFlavour, error) {
	var rflavs []*pop.Flavour

	op := func(stub pop.PopClient) error {
		flist, err := stub.Flavours(ctx, filter(f))
		if err != nil {
			return err
		}

		if flist == nil {
			rflavs = []*pop.Flavour{}
		} else {
			rflavs = flist.List
		}

		return nil
	}

	if err := cln.doRetry(op); err != nil {
		return nil, err
	}

	return cln.makeFlavours(rflavs), nil
}

// fetchImages fetches and converts pop Images into NFVImages.
func (cln *Client) fetchImages(ctx context.Context, f Filter) ([]*catalogue.NFVImage, error) {
	var imgs []*pop.Image

	op := func(stub pop.PopClient) error {
		ilist, err := stub.Images(ctx, filter(f))
		if err != nil {
			return err
		}

		if ilist == nil {
			imgs = []*pop.Image{}
		} else {
			imgs = ilist.List
		}

		return nil
	}

	if err := cln.doRetry(op); err != nil {
		return nil, err
	}

	return cln.makeImages(imgs), nil
}

// fetchNetworks fetches and converts pop Networks into catalogue.Network instances.
func (cln *Client) fetchNetworks(ctx context.Context, f Filter) ([]*catalogue.Network, error) {
	var rnets []*pop.Network

	op := func(stub pop.PopClient) error {
		nlist, err := stub.Networks(ctx, filter(f))
		if err != nil {
			return err
		}

		if nlist == nil {
			rnets = []*pop.Network{}
		} else {
			rnets = nlist.List
		}

		return nil
	}

	if err := cln.doRetry(op); err != nil {
		return nil, err
	}

	return cln.makeNetworks(rnets), nil
}

// fetchServers gets and creates catalogue.Server instances from pop containers.
func (cln *Client) fetchServers(ctx context.Context, f Filter) ([]*catalogue.Server, error) {
	var conts []*pop.Container

	op := func(stub pop.PopClient) error {
		clist, err := stub.Containers(ctx, filter(f))
		if err != nil {
			return err
		}

		if clist == nil {
			conts = []*pop.Container{}
		} else {
			conts = clist.List
		}

		return nil
	}

	if err := cln.doRetry(op); err != nil {
		return nil, err
	}

	return cln.makeServers(ctx, conts)
}

// makeFlavour converts a pop Flavour into a DeploymentFlavour.
func (cln *Client) makeFlavour(flav *pop.Flavour) *catalogue.DeploymentFlavour {
	return &catalogue.DeploymentFlavour{
		ExtID:      flav.Id,
		FlavourKey: flav.Name,
	}
}

// makeFlavours converts a list of pop Flavour into a list of DeploymentFlavour.
func (cln *Client) makeFlavours(flavs []*pop.Flavour) []*catalogue.DeploymentFlavour {
	depFlavs := make([]*catalogue.DeploymentFlavour, len(flavs))

	for i, flav := range flavs {
		depFlavs[i] = cln.makeFlavour(flav)
	}

	return depFlavs
}

// makeImage converts a pop Image into one (or more) NFVImage.
func (cln *Client) makeImage(img *pop.Image) []*catalogue.NFVImage {
	base := catalogue.NFVImage{
		ExtID:   img.Id,
		Created: catalogue.UnixDate(img.Created),
	}

	if img.Names == nil || len(img.Names) == 0 {
		return []*catalogue.NFVImage{&base}
	}

	ret := make([]*catalogue.NFVImage, 0, len(img.Names))

	// create an image for each tag.
	for _, name := range img.Names {
		img := new(catalogue.NFVImage)
		*img = base

		img.Name = name

		ret = append(ret, img)
	}

	return ret
}

// makeImages converts a list of pop Image into a list of NFVImage.
func (cln *Client) makeImages(imgs []*pop.Image) []*catalogue.NFVImage {
	nfvImgs := make([]*catalogue.NFVImage, 0, len(imgs)) // pre allocate at least len(imgs)

	for _, img := range imgs {
		nfvImgs = append(nfvImgs, cln.makeImage(img)...)
	}

	return nfvImgs
}

// makeNetwork converts a pop Network into a catalogue Network.
func (cln *Client) makeNetwork(net *pop.Network) *catalogue.Network {
	subs := make([]*catalogue.Subnet, len(net.Subnets))

	for i, rsub := range net.Subnets {
		subs[i] = &catalogue.Subnet{
			ExtID:     net.Id,
			CIDR:      rsub.Cidr,
			GatewayIP: rsub.Gateway,
		}
	}

	return &catalogue.Network{
		ExtID:    net.Id,
		Name:     net.Name,
		External: net.External,
		Subnets:  subs,
	}
}

// makeNetwork converts a list of pop Network into a list of catalogue Network.
func (cln *Client) makeNetworks(rnets []*pop.Network) []*catalogue.Network {
	nets := make([]*catalogue.Network, len(rnets))

	for i, rnet := range rnets {
		nets[i] = cln.makeNetwork(rnet)
	}

	return nets
}

// makeServer converts a pop Container into a catalogue Server.
func (cln *Client) makeServer(ctx context.Context, cont *pop.Container) (srv *catalogue.Server, err error) {
	// also fetch the image
	var nfvImage *catalogue.NFVImage
	if cont.ImageId != "" {
		nfvImage, err = cln.Image(ctx, IDFilter(cont.ImageId))
		if err != nil {
			return nil, err
		}
	}

	var deploymentFlavour *catalogue.DeploymentFlavour
	if cont.FlavourId != "" {
		deploymentFlavour, err = cln.Flavour(ctx, IDFilter(cont.FlavourId))
		if err != nil {
			return nil, err
		}
	}

	name := ""
	if cont.Names != nil && len(cont.Names) > 0 {
		name = cont.Names[0]
	}

	ipMap := make(map[string][]string)

	if cont.Endpoints != nil {
		for netname, ep := range cont.Endpoints {
			if ep.Ipv4 != nil && ep.Ipv4.Address != "" {
				// no ipv6 for now
				ipMap[netname] = []string{ep.Ipv4.Address}
			}
		}
	}

	return &catalogue.Server{
		ExtID:          cont.Id,
		Name:           name,
		Status:         cont.Status.String(),
		ExtendedStatus: cont.ExtendedStatus,
		Image:          nfvImage,
		Flavour:        deploymentFlavour,
		IPs:            ipMap,
		FloatingIPs:    map[string]string{},
		Created:        catalogue.UnixDate(cont.Created),
	}, nil
}

// makeServers converts a list of pop Containers into a list of catalogue Server.
func (cln *Client) makeServers(ctx context.Context, conts []*pop.Container) ([]*catalogue.Server, error) {
	servs := make([]*catalogue.Server, len(conts))

	for i, cont := range conts {
		serv, err := cln.makeServer(ctx, cont)
		if err != nil {
			return nil, err
		}

		servs[i] = serv
	}

	return servs, nil
}
<commit_msg>Saner filters<commit_after>package client

import (
	"context"
	"fmt"

	"github.com/mcilloni/go-openbaton/catalogue"
	pop "github.com/mcilloni/openbaton-docker/pop/proto"
)

// Flavour returns the flavour having the given filter as an OpenBaton DeploymentFlavour.
func (cln *Client) Flavour(ctx context.Context, f Filter) (*catalogue.DeploymentFlavour, error) {
	flavs, err := cln.fetchFlavours(ctx, f)
	if err != nil {
		return nil, err
	}

	switch len(flavs) {
	case 0:
		return nil, nil

	case 1:
		return flavs[0], nil

	default:
		return nil, fmt.Errorf("too many flavours returned from query")
	}
}

// Flavours returns all the available flavours as OpenBaton DeploymentFlavour.
func (cln *Client) Flavours(ctx context.Context) ([]*catalogue.DeploymentFlavour, error) {
	return cln.fetchFlavours(ctx, nil)
}

// Image returns the image on the server having the given filter as an OpenBaton NFVImage struct.
func (cln *Client) Image(ctx context.Context, f Filter) (*catalogue.NFVImage, error) {
	imgs, err := cln.fetchImages(ctx, f)
	if err != nil {
		return nil, err
	}

	switch len(imgs) {
	case 0:
		return nil, nil

	case 1:
		return imgs[0], nil

	default:
		return nil, fmt.Errorf("too many images returned from query")
	}
}

// Images returns the images on the server as OpenBaton NFVImage structs.
func (cln *Client) Images(ctx context.Context) ([]*catalogue.NFVImage, error) {
	return cln.fetchImages(ctx, nil)
}

// Network returns the network on the server having the given filter as an OpenBaton Network struct.
func (cln *Client) Network(ctx context.Context, f Filter) (*catalogue.Network, error) {
	nets, err := cln.fetchNetworks(ctx, f)
	if err != nil {
		return nil, err
	}

	switch len(nets) {
	case 0:
		return nil, nil

	case 1:
		return nets[0], nil

	default:
		return nil, fmt.Errorf("too many networks returned from query")
	}
}

// Networks returns the networks on the server as OpenBaton Network structs.
func (cln *Client) Networks(ctx context.Context) ([]*catalogue.Network, error) {
	return cln.fetchNetworks(ctx, nil)
}

// Server returns the container on the server having the given id as an OpenBaton Server struct.
func (cln *Client) Server(ctx context.Context, f Filter) (*catalogue.Server, error) {
	srvs, err := cln.fetchServers(ctx, f)
	if err != nil {
		return nil, err
	}

	switch len(srvs) {
	case 0:
		return nil, nil

	case 1:
		return srvs[0], nil

	default:
		return nil, fmt.Errorf("too many servers returned from query")
	}
}

// Servers returns the containers on the server as OpenBaton Server structs.
func (cln *Client) Servers(ctx context.Context) ([]*catalogue.Server, error) {
	return cln.fetchServers(ctx, nil)
}

// fetchFlavours fetches and converts pop Flavours into DeploymentFlavours.
func (cln *Client) fetchFlavours(ctx context.Context, f Filter) ([]*catalogue.DeploymentFlavour, error) {
	var rflavs []*pop.Flavour

	op := func(stub pop.PopClient) error {
		flist, err := stub.Flavours(ctx, filter(f))
		if err != nil {
			return err
		}

		if flist == nil {
			rflavs = []*pop.Flavour{}
		} else {
			rflavs = flist.List
		}

		return nil
	}

	if err := cln.doRetry(op); err != nil {
		return nil, err
	}

	return cln.makeFlavours(rflavs), nil
}

// fetchImages fetches and converts pop Images into NFVImages.
func (cln *Client) fetchImages(ctx context.Context, f Filter) ([]*catalogue.NFVImage, error) {
	var imgs []*pop.Image

	op := func(stub pop.PopClient) error {
		ilist, err := stub.Images(ctx, filter(f))
		if err != nil {
			return err
		}

		if ilist == nil {
			imgs = []*pop.Image{}
		} else {
			imgs = ilist.List
		}

		return nil
	}

	if err := cln.doRetry(op); err != nil {
		return nil, err
	}

	// if there is no such image, end here
	if len(imgs) == 0 {
		return []*catalogue.NFVImage{}, nil
	}

	nfvImgs := cln.makeImages(imgs)

	// if we are filtering for a name, than
	// we need to get just a single image and set the right name to it.
	// if we are filtering for an ID, then just return one of them, they just have
	// different names.
	switch rf := f.(type) {
	case NameFilter:
		nfvImgs[0].Name = string(rf)
		return nfvImgs[:1], nil

	case IDFilter:
		return nfvImgs[:1], nil

	default:
		return nfvImgs, nil
	}
}

// fetchNetworks fetches and converts pop Networks into catalogue.Network instances.
func (cln *Client) fetchNetworks(ctx context.Context, f Filter) ([]*catalogue.Network, error) {
	var rnets []*pop.Network

	op := func(stub pop.PopClient) error {
		nlist, err := stub.Networks(ctx, filter(f))
		if err != nil {
			return err
		}

		if nlist == nil {
			rnets = []*pop.Network{}
		} else {
			rnets = nlist.List
		}

		return nil
	}

	if err := cln.doRetry(op); err != nil {
		return nil, err
	}

	return cln.makeNetworks(rnets), nil
}

// fetchServers gets and creates catalogue.Server instances from pop containers.
func (cln *Client) fetchServers(ctx context.Context, f Filter) ([]*catalogue.Server, error) {
	var conts []*pop.Container

	op := func(stub pop.PopClient) error {
		clist, err := stub.Containers(ctx, filter(f))
		if err != nil {
			return err
		}

		if clist == nil {
			conts = []*pop.Container{}
		} else {
			conts = clist.List
		}

		return nil
	}

	if err := cln.doRetry(op); err != nil {
		return nil, err
	}

	return cln.makeServers(ctx, conts)
}

// makeFlavour converts a pop Flavour into a DeploymentFlavour.
func (cln *Client) makeFlavour(flav *pop.Flavour) *catalogue.DeploymentFlavour {
	return &catalogue.DeploymentFlavour{
		ExtID:      flav.Id,
		FlavourKey: flav.Name,
	}
}

// makeFlavours converts a list of pop Flavour into a list of DeploymentFlavour.
func (cln *Client) makeFlavours(flavs []*pop.Flavour) []*catalogue.DeploymentFlavour {
	depFlavs := make([]*catalogue.DeploymentFlavour, len(flavs))

	for i, flav := range flavs {
		depFlavs[i] = cln.makeFlavour(flav)
	}

	return depFlavs
}

// makeImage converts a pop Image into one (or more) NFVImage.
func (cln *Client) makeImage(img *pop.Image) []*catalogue.NFVImage {
	base := catalogue.NFVImage{
		ExtID:   img.Id,
		Created: catalogue.UnixDate(img.Created),
	}

	if img.Names == nil || len(img.Names) == 0 {
		return []*catalogue.NFVImage{&base}
	}

	ret := make([]*catalogue.NFVImage, 0, len(img.Names))

	// create an image for each tag.
	for _, name := range img.Names {
		img := new(catalogue.NFVImage)
		*img = base

		img.Name = name

		ret = append(ret, img)
	}

	return ret
}

// makeImages converts a list of pop Image into a list of NFVImage.
func (cln *Client) makeImages(imgs []*pop.Image) []*catalogue.NFVImage {
	nfvImgs := make([]*catalogue.NFVImage, 0, len(imgs)) // pre allocate at least len(imgs)

	for _, img := range imgs {
		nfvImgs = append(nfvImgs, cln.makeImage(img)...)
	}

	return nfvImgs
}

// makeNetwork converts a pop Network into a catalogue Network.
func (cln *Client) makeNetwork(net *pop.Network) *catalogue.Network {
	subs := make([]*catalogue.Subnet, len(net.Subnets))

	for i, rsub := range net.Subnets {
		subs[i] = &catalogue.Subnet{
			ExtID:     net.Id,
			CIDR:      rsub.Cidr,
			GatewayIP: rsub.Gateway,
		}
	}

	return &catalogue.Network{
		ExtID:    net.Id,
		Name:     net.Name,
		External: net.External,
		Subnets:  subs,
	}
}

// makeNetwork converts a list of pop Network into a list of catalogue Network.
func (cln *Client) makeNetworks(rnets []*pop.Network) []*catalogue.Network {
	nets := make([]*catalogue.Network, len(rnets))

	for i, rnet := range rnets {
		nets[i] = cln.makeNetwork(rnet)
	}

	return nets
}

// makeServer converts a pop Container into a catalogue Server.
func (cln *Client) makeServer(ctx context.Context, cont *pop.Container) (srv *catalogue.Server, err error) {
	// also fetch the image
	var nfvImage *catalogue.NFVImage
	if cont.ImageId != "" {
		nfvImage, err = cln.Image(ctx, IDFilter(cont.ImageId))
		if err != nil {
			return nil, err
		}
	}

	var deploymentFlavour *catalogue.DeploymentFlavour
	if cont.FlavourId != "" {
		deploymentFlavour, err = cln.Flavour(ctx, IDFilter(cont.FlavourId))
		if err != nil {
			return nil, err
		}
	}

	name := ""
	if cont.Names != nil && len(cont.Names) > 0 {
		name = cont.Names[0]
	}

	ipMap := make(map[string][]string)

	if cont.Endpoints != nil {
		for netname, ep := range cont.Endpoints {
			if ep.Ipv4 != nil && ep.Ipv4.Address != "" {
				// no ipv6 for now
				ipMap[netname] = []string{ep.Ipv4.Address}
			}
		}
	}

	return &catalogue.Server{
		ExtID:          cont.Id,
		Name:           name,
		Status:         cont.Status.String(),
		ExtendedStatus: cont.ExtendedStatus,
		Image:          nfvImage,
		Flavour:        deploymentFlavour,
		IPs:            ipMap,
		FloatingIPs:    map[string]string{},
		Created:        catalogue.UnixDate(cont.Created),
	}, nil
}

// makeServers converts a list of pop Containers into a list of catalogue Server.
func (cln *Client) makeServers(ctx context.Context, conts []*pop.Container) ([]*catalogue.Server, error) {
	servs := make([]*catalogue.Server, len(conts))

	for i, cont := range conts {
		serv, err := cln.makeServer(ctx, cont)
		if err != nil {
			return nil, err
		}

		servs[i] = serv
	}

	return servs, nil
}

<|endoftext|>


<commit_before>package context

// #include <stdlib.h>
// #include "sass2scss.h"
import "C"

import (
	"bytes"
	"io"
	"io/ioutil"
	"os"
	"testing"
	"unsafe"
)

// ToScss converts Sass to Scss with libsass sass2scss.h
func ToScss(r io.Reader, w io.Writer) error {
	bs, _ := ioutil.ReadAll(r)
	in := C.CString(string(bs))
	defer C.free(unsafe.Pointer(in))

	chars := C.sass2scss(
		// FIXME: readers would be much more efficient
		in,
		// SASS2SCSS_PRETTIFY_1 Egyptian brackets
		C.int(1),
	)
	_, err := io.WriteString(w, C.GoString(chars))
	return err
}

func testToScss(t *testing.T) {
	file, err := os.Open("../test/whitespace/one.sass")
	if err != nil {
		t.Fatal(err)
	}
	e := `$font-stack:    Helvetica, sans-serif;
$primary-color: #333;

body {
  font: 100% $font-stack;
  color: $primary-color; }
`

	var bytes bytes.Buffer
	ToScss(file, &bytes)

	if bytes.String() != e {
		t.Errorf("got:\n%s\nwanted:\n%s", bytes.String(), e)
	}
}
<commit_msg>update test<commit_after>package context

// #include <stdlib.h>
// #include "sass2scss.h"
import "C"

import (
	"bytes"
	"io"
	"io/ioutil"
	"os"
	"testing"
	"unsafe"
)

// ToScss converts Sass to Scss with libsass sass2scss.h
func ToScss(r io.Reader, w io.Writer) error {
	bs, _ := ioutil.ReadAll(r)
	in := C.CString(string(bs))
	defer C.free(unsafe.Pointer(in))

	chars := C.sass2scss(
		// FIXME: readers would be much more efficient
		in,
		// SASS2SCSS_PRETTIFY_1 Egyptian brackets
		C.int(1),
	)
	_, err := io.WriteString(w, C.GoString(chars))
	return err
}

func testToScss(t *testing.T) {
	file, err := os.Open("../test/whitespace/one.sass")
	if err != nil {
		t.Fatal(err)
	}
	e := `$font-stack:    Helvetica, sans-serif;
$primary-color: #333;
`

	var bytes bytes.Buffer
	ToScss(file, &bytes)

	if bytes.String() != e {
		t.Errorf("got:\n%s\nwanted:\n%s", bytes.String(), e)
	}
}
<|endoftext|>


<commit_before>package torrentfs

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"testing"
	"time"

	"bazil.org/fuse"
	fusefs "bazil.org/fuse/fs"
	_ "github.com/anacrolix/envpprof"
	"github.com/anacrolix/missinggo"
	"github.com/stretchr/testify/assert"
	netContext "golang.org/x/net/context"

	"github.com/anacrolix/torrent"
	"github.com/anacrolix/torrent/data/mmap"
	"github.com/anacrolix/torrent/internal/testutil"
	"github.com/anacrolix/torrent/metainfo"
)

func init() {
	log.SetFlags(log.Flags() | log.Lshortfile)
}

func TestTCPAddrString(t *testing.T) {
	l, err := net.Listen("tcp4", "localhost:0")
	if err != nil {
		t.Fatal(err)
	}
	defer l.Close()
	c, err := net.Dial("tcp", l.Addr().String())
	if err != nil {
		t.Fatal(err)
	}
	defer c.Close()
	ras := c.RemoteAddr().String()
	ta := &net.TCPAddr{
		IP:   net.IPv4(127, 0, 0, 1),
		Port: missinggo.AddrPort(l.Addr()),
	}
	s := ta.String()
	if ras != s {
		t.FailNow()
	}
}

type testLayout struct {
	BaseDir   string
	MountDir  string
	Completed string
	Metainfo  *metainfo.MetaInfo
}

func (me *testLayout) Destroy() error {
	return os.RemoveAll(me.BaseDir)
}

func newGreetingLayout() (tl testLayout, err error) {
	tl.BaseDir, err = ioutil.TempDir("", "torrentfs")
	if err != nil {
		return
	}
	tl.Completed = filepath.Join(tl.BaseDir, "completed")
	os.Mkdir(tl.Completed, 0777)
	tl.MountDir = filepath.Join(tl.BaseDir, "mnt")
	os.Mkdir(tl.MountDir, 0777)
	name := testutil.CreateDummyTorrentData(tl.Completed)
	metaInfoBuf := &bytes.Buffer{}
	testutil.CreateMetaInfo(name, metaInfoBuf)
	tl.Metainfo, err = metainfo.Load(metaInfoBuf)
	return
}

// Unmount without first killing the FUSE connection while there are FUSE
// operations blocked inside the filesystem code.
func TestUnmountWedged(t *testing.T) {
	layout, err := newGreetingLayout()
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err := layout.Destroy()
		if err != nil {
			t.Log(err)
		}
	}()
	client, err := torrent.NewClient(&torrent.Config{
		DataDir:         filepath.Join(layout.BaseDir, "incomplete"),
		DisableTrackers: true,
		NoDHT:           true,
		ListenAddr:      "redonk",
		DisableTCP:      true,
		DisableUTP:      true,

		NoDefaultBlocklist: true,
	})
	if err != nil {
		t.Fatal(err)
	}
	defer client.Close()
	client.AddTorrent(layout.Metainfo)
	fs := New(client)
	fuseConn, err := fuse.Mount(layout.MountDir)
	if err != nil {
		msg := fmt.Sprintf("error mounting: %s", err)
		if strings.Contains(err.Error(), "fuse") || err.Error() == "exit status 71" {
			t.Skip(msg)
		}
		t.Fatal(msg)
	}
	go func() {
		server := fusefs.New(fuseConn, &fusefs.Config{
			Debug: func(msg interface{}) {
				t.Log(msg)
			},
		})
		server.Serve(fs)
	}()
	<-fuseConn.Ready
	if err := fuseConn.MountError; err != nil {
		t.Fatalf("mount error: %s", err)
	}
	// Read the greeting file, though it will never be available. This should
	// "wedge" FUSE, requiring the fs object to be forcibly destroyed. The
	// read call will return with a FS error.
	go func() {
		_, err := ioutil.ReadFile(filepath.Join(layout.MountDir, layout.Metainfo.Info.Name))
		if err == nil {
			t.Fatal("expected error reading greeting")
		}
	}()

	// Wait until the read has blocked inside the filesystem code.
	fs.mu.Lock()
	for fs.blockedReads != 1 {
		fs.event.Wait()
	}
	fs.mu.Unlock()

	fs.Destroy()

	for {
		err = fuse.Unmount(layout.MountDir)
		if err != nil {
			t.Logf("error unmounting: %s", err)
			time.Sleep(time.Millisecond)
		} else {
			break
		}
	}

	err = fuseConn.Close()
	if err != nil {
		t.Fatalf("error closing fuse conn: %s", err)
	}
}

func TestDownloadOnDemand(t *testing.T) {
	layout, err := newGreetingLayout()
	if err != nil {
		t.Fatal(err)
	}
	defer layout.Destroy()
	seeder, err := torrent.NewClient(&torrent.Config{
		DataDir:         layout.Completed,
		DisableTrackers: true,
		NoDHT:           true,
		ListenAddr:      ":0",
		Seed:            true,

		NoDefaultBlocklist: true,
		// Ensure that the metainfo is obtained over the wire, since we added
		// the torrent to the seeder by magnet.
		DisableMetainfoCache: true,
	})
	if err != nil {
		t.Fatalf("error creating seeder client: %s", err)
	}
	seeder.SetIPBlockList(nil)
	defer seeder.Close()
	http.HandleFunc("/seeder", func(w http.ResponseWriter, req *http.Request) {
		seeder.WriteStatus(w)
	})
	_, err = seeder.AddMagnet(fmt.Sprintf("magnet:?xt=urn:btih:%x", layout.Metainfo.Info.Hash))
	if err != nil {
		t.Fatal(err)
	}
	leecher, err := torrent.NewClient(&torrent.Config{
		DisableTrackers: true,
		NoDHT:           true,
		ListenAddr:      ":0",
		DisableTCP:      true,

		NoDefaultBlocklist: true,

		TorrentDataOpener: func(info *metainfo.Info) torrent.Data {
			ret, _ := mmap.TorrentData(info, filepath.Join(layout.BaseDir, "download"))
			return ret
		},

		// This can be used to check if clients can connect to other clients
		// with the same ID.

		// PeerID: seeder.PeerID(),
	})
	leecher.SetIPBlockList(nil)
	http.HandleFunc("/leecher", func(w http.ResponseWriter, req *http.Request) {
		leecher.WriteStatus(w)
	})
	defer leecher.Close()
	leecherTorrent, _ := leecher.AddTorrent(layout.Metainfo)
	leecherTorrent.AddPeers([]torrent.Peer{func() torrent.Peer {
		_, port, err := net.SplitHostPort(seeder.ListenAddr().String())
		if err != nil {
			panic(err)
		}
		portInt64, err := strconv.ParseInt(port, 0, 0)
		if err != nil {
			panic(err)
		}
		return torrent.Peer{
			IP: func() net.IP {
				ret, _ := net.ResolveIPAddr("ip", "localhost")
				return ret.IP
			}(),
			Port: int(portInt64),
		}
	}()})
	fs := New(leecher)
	defer fs.Destroy()
	root, _ := fs.Root()
	node, _ := root.(fusefs.NodeStringLookuper).Lookup(netContext.Background(), "greeting")
	var attr fuse.Attr
	node.Attr(netContext.Background(), &attr)
	size := attr.Size
	resp := &fuse.ReadResponse{
		Data: make([]byte, size),
	}
	node.(fusefs.HandleReader).Read(netContext.Background(), &fuse.ReadRequest{
		Size: int(size),
	}, resp)
	content := resp.Data
	if string(content) != testutil.GreetingFileContents {
		t.Fatalf("%q != %q", string(content), testutil.GreetingFileContents)
	}
}

func TestIsSubPath(t *testing.T) {
	for _, case_ := range []struct {
		parent, child string
		is            bool
	}{
		{"", "", false},
		{"", "/", true},
		{"a/b", "a/bc", false},
		{"a/b", "a/b", false},
		{"a/b", "a/b/c", true},
		{"a/b", "a//b", false},
	} {
		assert.Equal(t, case_.is, isSubPath(case_.parent, case_.child))
	}
}
<commit_msg>fs: Tidy up tests<commit_after>package torrentfs

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	_ "net/http/pprof"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"bazil.org/fuse"
	fusefs "bazil.org/fuse/fs"
	_ "github.com/anacrolix/envpprof"
	"github.com/anacrolix/missinggo"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	netContext "golang.org/x/net/context"

	"github.com/anacrolix/torrent"
	"github.com/anacrolix/torrent/data/mmap"
	"github.com/anacrolix/torrent/internal/testutil"
	"github.com/anacrolix/torrent/metainfo"
)

func init() {
	log.SetFlags(log.Flags() | log.Lshortfile)
}

func TestTCPAddrString(t *testing.T) {
	l, err := net.Listen("tcp4", "localhost:0")
	if err != nil {
		t.Fatal(err)
	}
	defer l.Close()
	c, err := net.Dial("tcp", l.Addr().String())
	if err != nil {
		t.Fatal(err)
	}
	defer c.Close()
	ras := c.RemoteAddr().String()
	ta := &net.TCPAddr{
		IP:   net.IPv4(127, 0, 0, 1),
		Port: missinggo.AddrPort(l.Addr()),
	}
	s := ta.String()
	if ras != s {
		t.FailNow()
	}
}

type testLayout struct {
	BaseDir   string
	MountDir  string
	Completed string
	Metainfo  *metainfo.MetaInfo
}

func (me *testLayout) Destroy() error {
	return os.RemoveAll(me.BaseDir)
}

func newGreetingLayout() (tl testLayout, err error) {
	tl.BaseDir, err = ioutil.TempDir("", "torrentfs")
	if err != nil {
		return
	}
	tl.Completed = filepath.Join(tl.BaseDir, "completed")
	os.Mkdir(tl.Completed, 0777)
	tl.MountDir = filepath.Join(tl.BaseDir, "mnt")
	os.Mkdir(tl.MountDir, 0777)
	name := testutil.CreateDummyTorrentData(tl.Completed)
	metaInfoBuf := &bytes.Buffer{}
	testutil.CreateMetaInfo(name, metaInfoBuf)
	tl.Metainfo, err = metainfo.Load(metaInfoBuf)
	return
}

// Unmount without first killing the FUSE connection while there are FUSE
// operations blocked inside the filesystem code.
func TestUnmountWedged(t *testing.T) {
	layout, err := newGreetingLayout()
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err := layout.Destroy()
		if err != nil {
			t.Log(err)
		}
	}()
	client, err := torrent.NewClient(&torrent.Config{
		DataDir:         filepath.Join(layout.BaseDir, "incomplete"),
		DisableTrackers: true,
		NoDHT:           true,
		ListenAddr:      "redonk",
		DisableTCP:      true,
		DisableUTP:      true,

		NoDefaultBlocklist: true,
	})
	if err != nil {
		t.Fatal(err)
	}
	defer client.Close()
	client.AddTorrent(layout.Metainfo)
	fs := New(client)
	fuseConn, err := fuse.Mount(layout.MountDir)
	if err != nil {
		msg := fmt.Sprintf("error mounting: %s", err)
		if strings.Contains(err.Error(), "fuse") || err.Error() == "exit status 71" {
			t.Skip(msg)
		}
		t.Fatal(msg)
	}
	go func() {
		server := fusefs.New(fuseConn, &fusefs.Config{
			Debug: func(msg interface{}) {
				t.Log(msg)
			},
		})
		server.Serve(fs)
	}()
	<-fuseConn.Ready
	if err := fuseConn.MountError; err != nil {
		t.Fatalf("mount error: %s", err)
	}
	// Read the greeting file, though it will never be available. This should
	// "wedge" FUSE, requiring the fs object to be forcibly destroyed. The
	// read call will return with a FS error.
	go func() {
		_, err := ioutil.ReadFile(filepath.Join(layout.MountDir, layout.Metainfo.Info.Name))
		if err == nil {
			t.Fatal("expected error reading greeting")
		}
	}()

	// Wait until the read has blocked inside the filesystem code.
	fs.mu.Lock()
	for fs.blockedReads != 1 {
		fs.event.Wait()
	}
	fs.mu.Unlock()

	fs.Destroy()

	for {
		err = fuse.Unmount(layout.MountDir)
		if err != nil {
			t.Logf("error unmounting: %s", err)
			time.Sleep(time.Millisecond)
		} else {
			break
		}
	}

	err = fuseConn.Close()
	if err != nil {
		t.Fatalf("error closing fuse conn: %s", err)
	}
}

func TestDownloadOnDemand(t *testing.T) {
	layout, err := newGreetingLayout()
	require.NoError(t, err)
	defer layout.Destroy()
	seeder, err := torrent.NewClient(&torrent.Config{
		DataDir:         layout.Completed,
		DisableTrackers: true,
		NoDHT:           true,
		ListenAddr:      ":0",
		Seed:            true,

		NoDefaultBlocklist: true,
		// Ensure that the metainfo is obtained over the wire, since we added
		// the torrent to the seeder by magnet.
		DisableMetainfoCache: true,
	})
	seeder.SetIPBlockList(nil)
	require.NoError(t, err)
	defer seeder.Close()
	testutil.ExportStatusWriter(seeder, "s")
	_, err = seeder.AddMagnet(fmt.Sprintf("magnet:?xt=urn:btih:%x", layout.Metainfo.Info.Hash))
	require.NoError(t, err)
	leecher, err := torrent.NewClient(&torrent.Config{
		DisableTrackers: true,
		NoDHT:           true,
		ListenAddr:      ":0",
		DisableTCP:      true,

		NoDefaultBlocklist: true,

		TorrentDataOpener: func(info *metainfo.Info) torrent.Data {
			ret, _ := mmap.TorrentData(info, filepath.Join(layout.BaseDir, "download"))
			return ret
		},

		// This can be used to check if clients can connect to other clients
		// with the same ID.

		// PeerID: seeder.PeerID(),
	})
	leecher.SetIPBlockList(nil)
	testutil.ExportStatusWriter(leecher, "l")
	defer leecher.Close()
	leecherTorrent, _ := leecher.AddTorrent(layout.Metainfo)
	leecherTorrent.AddPeers([]torrent.Peer{
		torrent.Peer{
			IP:   missinggo.AddrIP(seeder.ListenAddr()),
			Port: missinggo.AddrPort(seeder.ListenAddr()),
		},
	})
	fs := New(leecher)
	defer fs.Destroy()
	root, _ := fs.Root()
	node, _ := root.(fusefs.NodeStringLookuper).Lookup(netContext.Background(), "greeting")
	var attr fuse.Attr
	node.Attr(netContext.Background(), &attr)
	size := attr.Size
	resp := &fuse.ReadResponse{
		Data: make([]byte, size),
	}
	node.(fusefs.HandleReader).Read(netContext.Background(), &fuse.ReadRequest{
		Size: int(size),
	}, resp)
	assert.EqualValues(t, testutil.GreetingFileContents, resp.Data)
}

func TestIsSubPath(t *testing.T) {
	for _, case_ := range []struct {
		parent, child string
		is            bool
	}{
		{"", "", false},
		{"", "/", true},
		{"a/b", "a/bc", false},
		{"a/b", "a/b", false},
		{"a/b", "a/b/c", true},
		{"a/b", "a//b", false},
	} {
		assert.Equal(t, case_.is, isSubPath(case_.parent, case_.child))
	}
}
<|endoftext|>


<commit_before>package operation

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"path/filepath"
	"runtime/debug"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/util"
)

type UploadResult struct {
	Name       string `json:"name,omitempty"`
	Size       uint32 `json:"size,omitempty"`
	Error      string `json:"error,omitempty"`
	ETag       string `json:"eTag,omitempty"`
	CipherKey  []byte `json:"cipherKey,omitempty"`
	Mime       string `json:"mime,omitempty"`
	Gzip       uint32 `json:"gzip,omitempty"`
	ContentMd5 string `json:"contentMd5,omitempty"`
	RetryCount int    `json:"-"`
}

func (uploadResult *UploadResult) ToPbFileChunk(fileId string, offset int64) *filer_pb.FileChunk {
	fid, _ := filer_pb.ToFileIdObject(fileId)
	return &filer_pb.FileChunk{
		FileId:       fileId,
		Offset:       offset,
		Size:         uint64(uploadResult.Size),
		Mtime:        time.Now().UnixNano(),
		ETag:         uploadResult.ETag,
		CipherKey:    uploadResult.CipherKey,
		IsCompressed: uploadResult.Gzip > 0,
		Fid:          fid,
	}
}

// HTTPClient interface for testing
type HTTPClient interface {
	Do(req *http.Request) (*http.Response, error)
}

var (
	HttpClient HTTPClient
)

func init() {
	HttpClient = &http.Client{Transport: &http.Transport{
		MaxIdleConns:        1024,
		MaxIdleConnsPerHost: 1024,
	}}
}

var fileNameEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`)

// Upload sends a POST request to a volume server to upload the content with adjustable compression level
func UploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
	uploadResult, err = retriedUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt)
	return
}

// Upload sends a POST request to a volume server to upload the content with fast compression
func Upload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error, data []byte) {
	uploadResult, err, data = doUpload(uploadUrl, filename, cipher, reader, isInputCompressed, mtype, pairMap, jwt)
	return
}

func doUpload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error, data []byte) {
	bytesReader, ok := reader.(*util.BytesReader)
	if ok {
		data = bytesReader.Bytes
	} else {
		data, err = ioutil.ReadAll(reader)
		if err != nil {
			err = fmt.Errorf("read input: %v", err)
			return
		}
	}
	uploadResult, uploadErr := retriedUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt)
	return uploadResult, uploadErr, data
}

func retriedUploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
	for i := 0; i < 3; i++ {
		uploadResult, err = doUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt)
		if err == nil {
			uploadResult.RetryCount = i
			return
		} else {
			glog.Warningf("uploading to %s: %v", uploadUrl, err)
		}
		time.Sleep(time.Millisecond * time.Duration(237*(i+1)))
	}
	return
}

func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
	contentIsGzipped := isInputCompressed
	shouldGzipNow := false
	if !isInputCompressed {
		if mtype == "" {
			mtype = http.DetectContentType(data)
			// println("detect1 mimetype to", mtype)
			if mtype == "application/octet-stream" {
				mtype = ""
			}
		}
		if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Base(filename), mtype); iAmSure && shouldBeCompressed {
			shouldGzipNow = true
		} else if !iAmSure && mtype == "" && len(data) > 16*1024 {
			var compressed []byte
			compressed, err = util.GzipData(data[0:128])
			shouldGzipNow = len(compressed)*10 < 128*9 // can not compress to less than 90%
		}
	}

	var clearDataLen int

	// gzip if possible
	// this could be double copying
	clearDataLen = len(data)
	clearData := data
	if shouldGzipNow && !cipher {
		compressed, compressErr := util.GzipData(data)
		// fmt.Printf("data is compressed from %d ==> %d\n", len(data), len(compressed))
		if compressErr == nil {
			data = compressed
			contentIsGzipped = true
		}
	} else if isInputCompressed {
		// just to get the clear data length
		clearData, err = util.DecompressData(data)
		if err == nil {
			clearDataLen = len(clearData)
		}
	}

	if cipher {
		// encrypt(gzip(data))

		// encrypt
		cipherKey := util.GenCipherKey()
		encryptedData, encryptionErr := util.Encrypt(clearData, cipherKey)
		if encryptionErr != nil {
			err = fmt.Errorf("encrypt input: %v", encryptionErr)
			return
		}

		// upload data
		uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) {
			_, err = w.Write(encryptedData)
			return
		}, "", false, len(encryptedData), "", nil, jwt)
		if uploadResult == nil {
			return
		}
		uploadResult.Name = filename
		uploadResult.Mime = mtype
		uploadResult.CipherKey = cipherKey
		uploadResult.Size = uint32(clearDataLen)
	} else {
		// upload data
		uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) {
			_, err = w.Write(data)
			return
		}, filename, contentIsGzipped, len(data), mtype, pairMap, jwt)
		if uploadResult == nil {
			return
		}
		uploadResult.Size = uint32(clearDataLen)
		if contentIsGzipped {
			uploadResult.Gzip = 1
		}
	}

	return uploadResult, err
}

func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, originalDataSize int, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
	buf := GetBuffer()
	defer PutBuffer(buf)
	body_writer := multipart.NewWriter(buf)
	h := make(textproto.MIMEHeader)
	h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, fileNameEscaper.Replace(filename)))
	h.Set("Idempotency-Key", uploadUrl)
	if mtype == "" {
		mtype = mime.TypeByExtension(strings.ToLower(filepath.Ext(filename)))
	}
	if mtype != "" {
		h.Set("Content-Type", mtype)
	}
	if isGzipped {
		h.Set("Content-Encoding", "gzip")
	}

	file_writer, cp_err := body_writer.CreatePart(h)
	if cp_err != nil {
		glog.V(0).Infoln("error creating form file", cp_err.Error())
		return nil, cp_err
	}
	if err := fillBufferFunction(file_writer); err != nil {
		glog.V(0).Infoln("error copying data", err)
		return nil, err
	}
	content_type := body_writer.FormDataContentType()
	if err := body_writer.Close(); err != nil {
		glog.V(0).Infoln("error closing body", err)
		return nil, err
	}

	req, postErr := http.NewRequest("POST", uploadUrl, bytes.NewReader(buf.Bytes()))
	if postErr != nil {
		glog.V(1).Infof("create upload request %s: %v", uploadUrl, postErr)
		return nil, fmt.Errorf("create upload request %s: %v", uploadUrl, postErr)
	}
	req.Header.Set("Content-Type", content_type)
	for k, v := range pairMap {
		req.Header.Set(k, v)
	}
	if jwt != "" {
		req.Header.Set("Authorization", "BEARER "+string(jwt))
	}
	// print("+")
	resp, post_err := HttpClient.Do(req)
	if post_err != nil {
		glog.Errorf("upload %s %d bytes to %v: %v", filename, originalDataSize, uploadUrl, post_err)
		debug.PrintStack()
		return nil, fmt.Errorf("upload %s %d bytes to %v: %v", filename, originalDataSize, uploadUrl, post_err)
	}
	// print("-")
	defer util.CloseResponse(resp)

	var ret UploadResult
	etag := getEtag(resp)
	if resp.StatusCode == http.StatusNoContent {
		ret.ETag = etag
		return &ret, nil
	}

	resp_body, ra_err := ioutil.ReadAll(resp.Body)
	if ra_err != nil {
		return nil, fmt.Errorf("read response body %v: %v", uploadUrl, ra_err)
	}

	unmarshal_err := json.Unmarshal(resp_body, &ret)
	if unmarshal_err != nil {
		glog.Errorf("unmarshal %s: %v", uploadUrl, string(resp_body))
		return nil, fmt.Errorf("unmarshal %v: %v", uploadUrl, unmarshal_err)
	}
	if ret.Error != "" {
		return nil, fmt.Errorf("unmarshalled error %v: %v", uploadUrl, ret.Error)
	}
	ret.ETag = etag
	ret.ContentMd5 = resp.Header.Get("Content-MD5")
	return &ret, nil
}

func getEtag(r *http.Response) (etag string) {
	etag = r.Header.Get("ETag")
	if strings.HasPrefix(etag, "\"") && strings.HasSuffix(etag, "\"") {
		etag = etag[1 : len(etag)-1]
	}
	return
}
<commit_msg>skip connection reset error<commit_after>package operation

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"path/filepath"
	"runtime/debug"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/util"
)

type UploadResult struct {
	Name       string `json:"name,omitempty"`
	Size       uint32 `json:"size,omitempty"`
	Error      string `json:"error,omitempty"`
	ETag       string `json:"eTag,omitempty"`
	CipherKey  []byte `json:"cipherKey,omitempty"`
	Mime       string `json:"mime,omitempty"`
	Gzip       uint32 `json:"gzip,omitempty"`
	ContentMd5 string `json:"contentMd5,omitempty"`
	RetryCount int    `json:"-"`
}

func (uploadResult *UploadResult) ToPbFileChunk(fileId string, offset int64) *filer_pb.FileChunk {
	fid, _ := filer_pb.ToFileIdObject(fileId)
	return &filer_pb.FileChunk{
		FileId:       fileId,
		Offset:       offset,
		Size:         uint64(uploadResult.Size),
		Mtime:        time.Now().UnixNano(),
		ETag:         uploadResult.ETag,
		CipherKey:    uploadResult.CipherKey,
		IsCompressed: uploadResult.Gzip > 0,
		Fid:          fid,
	}
}

// HTTPClient interface for testing
type HTTPClient interface {
	Do(req *http.Request) (*http.Response, error)
}

var (
	HttpClient HTTPClient
)

func init() {
	HttpClient = &http.Client{Transport: &http.Transport{
		MaxIdleConns:        1024,
		MaxIdleConnsPerHost: 1024,
	}}
}

var fileNameEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`)

// Upload sends a POST request to a volume server to upload the content with adjustable compression level
func UploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
	uploadResult, err = retriedUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt)
	return
}

// Upload sends a POST request to a volume server to upload the content with fast compression
func Upload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error, data []byte) {
	uploadResult, err, data = doUpload(uploadUrl, filename, cipher, reader, isInputCompressed, mtype, pairMap, jwt)
	return
}

func doUpload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error, data []byte) {
	bytesReader, ok := reader.(*util.BytesReader)
	if ok {
		data = bytesReader.Bytes
	} else {
		data, err = ioutil.ReadAll(reader)
		if err != nil {
			err = fmt.Errorf("read input: %v", err)
			return
		}
	}
	uploadResult, uploadErr := retriedUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt)
	return uploadResult, uploadErr, data
}

func retriedUploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
	for i := 0; i < 3; i++ {
		uploadResult, err = doUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt)
		if err == nil {
			uploadResult.RetryCount = i
			return
		} else {
			glog.Warningf("uploading to %s: %v", uploadUrl, err)
		}
		time.Sleep(time.Millisecond * time.Duration(237*(i+1)))
	}
	return
}

func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
	contentIsGzipped := isInputCompressed
	shouldGzipNow := false
	if !isInputCompressed {
		if mtype == "" {
			mtype = http.DetectContentType(data)
			// println("detect1 mimetype to", mtype)
			if mtype == "application/octet-stream" {
				mtype = ""
			}
		}
		if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Base(filename), mtype); iAmSure && shouldBeCompressed {
			shouldGzipNow = true
		} else if !iAmSure && mtype == "" && len(data) > 16*1024 {
			var compressed []byte
			compressed, err = util.GzipData(data[0:128])
			shouldGzipNow = len(compressed)*10 < 128*9 // can not compress to less than 90%
		}
	}

	var clearDataLen int

	// gzip if possible
	// this could be double copying
	clearDataLen = len(data)
	clearData := data
	if shouldGzipNow && !cipher {
		compressed, compressErr := util.GzipData(data)
		// fmt.Printf("data is compressed from %d ==> %d\n", len(data), len(compressed))
		if compressErr == nil {
			data = compressed
			contentIsGzipped = true
		}
	} else if isInputCompressed {
		// just to get the clear data length
		clearData, err = util.DecompressData(data)
		if err == nil {
			clearDataLen = len(clearData)
		}
	}

	if cipher {
		// encrypt(gzip(data))

		// encrypt
		cipherKey := util.GenCipherKey()
		encryptedData, encryptionErr := util.Encrypt(clearData, cipherKey)
		if encryptionErr != nil {
			err = fmt.Errorf("encrypt input: %v", encryptionErr)
			return
		}

		// upload data
		uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) {
			_, err = w.Write(encryptedData)
			return
		}, "", false, len(encryptedData), "", nil, jwt)
		if uploadResult == nil {
			return
		}
		uploadResult.Name = filename
		uploadResult.Mime = mtype
		uploadResult.CipherKey = cipherKey
		uploadResult.Size = uint32(clearDataLen)
	} else {
		// upload data
		uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) {
			_, err = w.Write(data)
			return
		}, filename, contentIsGzipped, len(data), mtype, pairMap, jwt)
		if uploadResult == nil {
			return
		}
		uploadResult.Size = uint32(clearDataLen)
		if contentIsGzipped {
			uploadResult.Gzip = 1
		}
	}

	return uploadResult, err
}

func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, originalDataSize int, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
	buf := GetBuffer()
	defer PutBuffer(buf)
	body_writer := multipart.NewWriter(buf)
	h := make(textproto.MIMEHeader)
	h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, fileNameEscaper.Replace(filename)))
	h.Set("Idempotency-Key", uploadUrl)
	if mtype == "" {
		mtype = mime.TypeByExtension(strings.ToLower(filepath.Ext(filename)))
	}
	if mtype != "" {
		h.Set("Content-Type", mtype)
	}
	if isGzipped {
		h.Set("Content-Encoding", "gzip")
	}

	file_writer, cp_err := body_writer.CreatePart(h)
	if cp_err != nil {
		glog.V(0).Infoln("error creating form file", cp_err.Error())
		return nil, cp_err
	}
	if err := fillBufferFunction(file_writer); err != nil {
		glog.V(0).Infoln("error copying data", err)
		return nil, err
	}
	content_type := body_writer.FormDataContentType()
	if err := body_writer.Close(); err != nil {
		glog.V(0).Infoln("error closing body", err)
		return nil, err
	}

	req, postErr := http.NewRequest("POST", uploadUrl, bytes.NewReader(buf.Bytes()))
	if postErr != nil {
		glog.V(1).Infof("create upload request %s: %v", uploadUrl, postErr)
		return nil, fmt.Errorf("create upload request %s: %v", uploadUrl, postErr)
	}
	req.Header.Set("Content-Type", content_type)
	for k, v := range pairMap {
		req.Header.Set(k, v)
	}
	if jwt != "" {
		req.Header.Set("Authorization", "BEARER "+string(jwt))
	}
	// print("+")
	resp, post_err := HttpClient.Do(req)
	if post_err != nil {
		if !strings.Contains(post_err.Error(), "connection reset by peer"){
			glog.Errorf("upload %s %d bytes to %v: %v", filename, originalDataSize, uploadUrl, post_err)
			debug.PrintStack()
		}
		return nil, fmt.Errorf("upload %s %d bytes to %v: %v", filename, originalDataSize, uploadUrl, post_err)
	}
	// print("-")
	defer util.CloseResponse(resp)

	var ret UploadResult
	etag := getEtag(resp)
	if resp.StatusCode == http.StatusNoContent {
		ret.ETag = etag
		return &ret, nil
	}

	resp_body, ra_err := ioutil.ReadAll(resp.Body)
	if ra_err != nil {
		return nil, fmt.Errorf("read response body %v: %v", uploadUrl, ra_err)
	}

	unmarshal_err := json.Unmarshal(resp_body, &ret)
	if unmarshal_err != nil {
		glog.Errorf("unmarshal %s: %v", uploadUrl, string(resp_body))
		return nil, fmt.Errorf("unmarshal %v: %v", uploadUrl, unmarshal_err)
	}
	if ret.Error != "" {
		return nil, fmt.Errorf("unmarshalled error %v: %v", uploadUrl, ret.Error)
	}
	ret.ETag = etag
	ret.ContentMd5 = resp.Header.Get("Content-MD5")
	return &ret, nil
}

func getEtag(r *http.Response) (etag string) {
	etag = r.Header.Get("ETag")
	if strings.HasPrefix(etag, "\"") && strings.HasSuffix(etag, "\"") {
		etag = etag[1 : len(etag)-1]
	}
	return
}
<|endoftext|>


<commit_before>package main

import "ole"
import "ole/oleutil"
import "syscall"

func main() {
	ole.CoInitialize(0)
	xmlhttp, _ := oleutil.CreateDispatch("Microsoft.XMLHTTP")
	oleutil.CallMethod(xmlhttp, "open", "GET", "http://rss.slashdot.org/Slashdot/slashdot", false)
	oleutil.CallMethod(xmlhttp, "send", nil)
	state := -1
	for state != 4 {
		result, _ := oleutil.GetProperty(xmlhttp, "readyState")
		state = int(result.Val)
		syscall.Sleep(10000000)
	}
	result, _ := oleutil.GetProperty(xmlhttp, "responseXml")
	responseXml := result.ToIDispatch()
	result, _ = oleutil.CallMethod(responseXml, "selectNodes", "rdf:RDF/item")
	items := result.ToIDispatch()
	result, _ = oleutil.GetProperty(items, "length")

	for n := 0; n < int(result.Val); n++ {
		result, _ := oleutil.GetProperty(items, "item", n)
		item := result.ToIDispatch()

		result, _ = oleutil.CallMethod(item, "selectSingleNode", "title")
		title := result.ToIDispatch()
		result, _ = oleutil.GetProperty(title, "text")
		println(result.ToString())

		result, _ = oleutil.CallMethod(item, "selectSingleNode", "link")
		link := result.ToIDispatch()
		result, _ = oleutil.GetProperty(link, "text")
		println("  " + result.ToString())

		title.Release()
		link.Release()
		item.Release()
	}
	items.Release()
	xmlhttp.Release()
}
<commit_msg>rss reader<commit_after>package main

import "ole"
import "ole/oleutil"
import "syscall"

func main() {
	ole.CoInitialize(0)
	xmlhttp, _ := oleutil.CreateDispatch("Microsoft.XMLHTTP")
	oleutil.CallMethod(xmlhttp, "open", "GET", "http://rss.slashdot.org/Slashdot/slashdot", false)
	oleutil.CallMethod(xmlhttp, "send", nil)
	state := -1
	for state != 4 {
		result, _ := oleutil.GetProperty(xmlhttp, "readyState")
		state = int(result.Val)
		syscall.Sleep(10000000)
	}
	result, _ := oleutil.GetProperty(xmlhttp, "responseXml")
	responseXml := result.ToIDispatch()
	result, _ = oleutil.CallMethod(responseXml, "selectNodes", "rdf:RDF/item")
	items := result.ToIDispatch()
	result, _ = oleutil.GetProperty(items, "length")

	for n := 0; n < int(result.Val); n++ {
		result, _ := oleutil.GetProperty(items, "item", n)
		item := result.ToIDispatch()

		result, _ = oleutil.CallMethod(item, "selectSingleNode", "title")
		title := result.ToIDispatch()
		result, _ = oleutil.GetProperty(title, "text")
		println(result.ToString())

		result, _ = oleutil.CallMethod(item, "selectSingleNode", "link")
		link := result.ToIDispatch()
		result, _ = oleutil.GetProperty(link, "text")
		println("  " + result.ToString())

		title.Release()
		link.Release()
		item.Release()
	}
	items.Release()
	xmlhttp.Release()
}
<|endoftext|>


<commit_before>package crawl

import (
	"errors"
	"fmt"
	"log"
	"strconv"
	"time"

	"github.com/nzai/stockrecorder/config"
)

//	启动
func Start() {

	markets := config.GetArray("market", "markets")
	if len(markets) == 0 {
		//	未定义市场就直接返回
		return
	}

	for _, market := range markets {
		//	抓取市场数据
		err := marketJob(market)
		if err != nil {
			log.Fatalf("启动[%s]数据抓取任务失败:%v", market, err)
		}
	}
}

//	抓取市场数据任务
func marketJob(market string) error {
	//	任务启动时间(hour)
	starthour := config.GetString(market, "starthour", "")
	if starthour == "" {
		return errors.New(fmt.Sprintf("市场[%s]的starthour配置有误", market))
	}

	hour, err := strconv.Atoi(starthour)
	if err != nil {
		return err
	}

	now := time.Now()
	startTime := time.Date(now.Year(), now.Month(), now.Day(), hour, 0, 0, 0, now.Location())
	//	现在距离开始的时间间隔
	duration := startTime.Sub(now)
	if now.After(startTime) {
		//	今天的开始时间已经过了
		duration = duration + time.Hour*24
	}

	log.Printf("%s后开始抓取%s的数据", duration.String(), market)

	time.AfterFunc(duration, func() {
		//	到点后立即运行第一次
		go func(m string) {
			//	抓取雅虎今日数据
			err := yahooToday(market)
			if err != nil {
				log.Fatal(err)
			}
		}(market)

		//	之后每天运行一次
		ticker := time.NewTicker(time.Hour * 24)
		for _ = range ticker.C {
			//	抓取雅虎今日数据
			err := yahooToday(market)
			if err != nil {
				log.Fatal(err)
			}
		}
	})

	return nil
}
<commit_msg>启动抓取任务时传入交易结束时间，方便推算所属日期<commit_after>package crawl

import (
	"errors"
	"fmt"
	"log"
	"strconv"
	"time"

	"github.com/nzai/stockrecorder/config"
)

//	启动
func Start() {

	markets := config.GetArray("market", "markets")
	if len(markets) == 0 {
		//	未定义市场就直接返回
		return
	}

	for _, market := range markets {
		//	抓取市场数据
		err := marketJob(market)
		if err != nil {
			log.Fatalf("启动[%s]数据抓取任务失败:%v", market, err)
		}
	}
}

//	抓取市场数据任务
func marketJob(market string) error {
	//	任务启动时间(hour)
	endhour := config.GetString(market, "endhour", "")
	if endhour == "" {
		return errors.New(fmt.Sprintf("市场[%s]的starthour配置有误", market))
	}

	hour, err := strconv.Atoi(endhour)
	if err != nil {
		return err
	}

	now := time.Now()
	startTime := time.Date(now.Year(), now.Month(), now.Day(), hour, 0, 0, 0, now.Location())
	//	现在距离开始的时间间隔
	duration := startTime.Sub(now)
	if now.After(startTime) {
		//	今天的开始时间已经过了
		duration = duration + time.Hour*24
	}

	log.Printf("%s后开始抓取%s的数据", duration.String(), market)

	time.AfterFunc(duration, func() {
		//	到点后立即运行第一次
		go func(m string, h int) {
			//	抓取雅虎今日数据
			err := yahooToday(market, h)
			if err != nil {
				log.Fatal(err)
			}
		}(market, hour)

		//	之后每天运行一次
		ticker := time.NewTicker(time.Hour * 24)
		for _ = range ticker.C {
			//	抓取雅虎今日数据
			err := yahooToday(market, hour)
			if err != nil {
				log.Fatal(err)
			}
		}
	})

	return nil
}
<|endoftext|>


<commit_before>package middleware

import (
	"bufio"
	"io"
	"net"
	"net/http"
)

func wrapWriter(w http.ResponseWriter) writerProxy {
	_, cn := w.(http.CloseNotifier)
	_, fl := w.(http.Flusher)
	_, hj := w.(http.Hijacker)
	_, rf := w.(io.ReaderFrom)

	bw := basicWriter{ResponseWriter: w}
	if cn && fl && hj && rf {
		return &fancyWriter{bw}
	}
	return &bw
}

type writerProxy interface {
	http.ResponseWriter
	maybeWriteHeader()
	status() int
}

type basicWriter struct {
	http.ResponseWriter
	wroteHeader bool
	code        int
}

func (b *basicWriter) WriteHeader(code int) {
	b.code = code
	b.wroteHeader = true
	b.ResponseWriter.WriteHeader(code)
}
func (b *basicWriter) Write(buf []byte) (int, error) {
	b.maybeWriteHeader()
	return b.ResponseWriter.Write(buf)
}
func (b *basicWriter) maybeWriteHeader() {
	if !b.wroteHeader {
		b.WriteHeader(http.StatusOK)
	}
}
func (b *basicWriter) status() int {
	return b.code
}
func (b *basicWriter) Unwrap() http.ResponseWriter {
	return b.ResponseWriter
}

type fancyWriter struct {
	basicWriter
}

func (f *fancyWriter) CloseNotify() <-chan bool {
	cn := f.basicWriter.ResponseWriter.(http.CloseNotifier)
	return cn.CloseNotify()
}
func (f *fancyWriter) Flush() {
	fl := f.basicWriter.ResponseWriter.(http.Flusher)
	fl.Flush()
}
func (f *fancyWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	hj := f.basicWriter.ResponseWriter.(http.Hijacker)
	return hj.Hijack()
}
func (f *fancyWriter) ReadFrom(r io.Reader) (int64, error) {
	rf := f.basicWriter.ResponseWriter.(io.ReaderFrom)
	f.basicWriter.maybeWriteHeader()
	return rf.ReadFrom(r)
}

var _ http.CloseNotifier = &fancyWriter{}
var _ http.Flusher = &fancyWriter{}
var _ http.Hijacker = &fancyWriter{}
var _ io.ReaderFrom = &fancyWriter{}
<commit_msg>Record the first written status, not the last<commit_after>package middleware

import (
	"bufio"
	"io"
	"net"
	"net/http"
)

func wrapWriter(w http.ResponseWriter) writerProxy {
	_, cn := w.(http.CloseNotifier)
	_, fl := w.(http.Flusher)
	_, hj := w.(http.Hijacker)
	_, rf := w.(io.ReaderFrom)

	bw := basicWriter{ResponseWriter: w}
	if cn && fl && hj && rf {
		return &fancyWriter{bw}
	}
	return &bw
}

type writerProxy interface {
	http.ResponseWriter
	maybeWriteHeader()
	status() int
}

type basicWriter struct {
	http.ResponseWriter
	wroteHeader bool
	code        int
}

func (b *basicWriter) WriteHeader(code int) {
	if !b.wroteHeader {
		b.code = code
		b.wroteHeader = true
		b.ResponseWriter.WriteHeader(code)
	}
}
func (b *basicWriter) Write(buf []byte) (int, error) {
	b.maybeWriteHeader()
	return b.ResponseWriter.Write(buf)
}
func (b *basicWriter) maybeWriteHeader() {
	if !b.wroteHeader {
		b.WriteHeader(http.StatusOK)
	}
}
func (b *basicWriter) status() int {
	return b.code
}
func (b *basicWriter) Unwrap() http.ResponseWriter {
	return b.ResponseWriter
}

type fancyWriter struct {
	basicWriter
}

func (f *fancyWriter) CloseNotify() <-chan bool {
	cn := f.basicWriter.ResponseWriter.(http.CloseNotifier)
	return cn.CloseNotify()
}
func (f *fancyWriter) Flush() {
	fl := f.basicWriter.ResponseWriter.(http.Flusher)
	fl.Flush()
}
func (f *fancyWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	hj := f.basicWriter.ResponseWriter.(http.Hijacker)
	return hj.Hijack()
}
func (f *fancyWriter) ReadFrom(r io.Reader) (int64, error) {
	rf := f.basicWriter.ResponseWriter.(io.ReaderFrom)
	f.basicWriter.maybeWriteHeader()
	return rf.ReadFrom(r)
}

var _ http.CloseNotifier = &fancyWriter{}
var _ http.Flusher = &fancyWriter{}
var _ http.Hijacker = &fancyWriter{}
var _ io.ReaderFrom = &fancyWriter{}
<|endoftext|>