<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/machine/drivers"
	"github.com/docker/machine/utils"
)

var (
	validHostNameChars   = `[a-zA-Z0-9\-\.]`
	validHostNamePattern = regexp.MustCompile(`^` + validHostNameChars + `+$`)
	ErrInvalidHostname   = errors.New("Invalid hostname specified")
)

const (
	swarmDockerImage = "ehazlett/swarm:latest"
)

type Host struct {
	Name           string `json:"-"`
	DriverName     string
	Driver         drivers.Driver
	CaCertPath     string
	ServerCertPath string
	ServerKeyPath  string
	PrivateKeyPath string
	ClientCertPath string
	storePath      string
}

type DockerConfig struct {
	EngineConfig     string
	EngineConfigPath string
}

type hostConfig struct {
	DriverName string
}

func NewHost(name, driverName, storePath, caCert, privateKey string) (*Host, error) {
	driver, err := drivers.NewDriver(driverName, name, storePath, caCert, privateKey)
	if err != nil {
		return nil, err
	}
	return &Host{
		Name:           name,
		DriverName:     driverName,
		Driver:         driver,
		CaCertPath:     caCert,
		PrivateKeyPath: privateKey,
		storePath:      storePath,
	}, nil
}

func LoadHost(name string, storePath string) (*Host, error) {
	if _, err := os.Stat(storePath); os.IsNotExist(err) {
		return nil, fmt.Errorf("Host %q does not exist", name)
	}

	host := &Host{Name: name, storePath: storePath}
	if err := host.LoadConfig(); err != nil {
		return nil, err
	}
	return host, nil
}

func ValidateHostName(name string) (string, error) {
	if !validHostNamePattern.MatchString(name) {
		return name, ErrInvalidHostname
	}
	return name, nil
}

func GenerateClientCertificate(caCertPath, privateKeyPath string) error {
	var (
		org  = "docker-machine"
		bits = 2048
	)

	clientCertPath := filepath.Join(utils.GetMachineDir(), "cert.pem")
	clientKeyPath := filepath.Join(utils.GetMachineDir(), "key.pem")

	if err := os.MkdirAll(utils.GetMachineDir(), 0700); err != nil {
		return err
	}

	log.Debugf("generating client cert: %s", clientCertPath)
	if err := utils.GenerateCert([]string{""}, clientCertPath, clientKeyPath, caCertPath, privateKeyPath, org, bits); err != nil {
		return fmt.Errorf("error generating client cert: %s", err)
	}

	return nil
}

func (h *Host) ConfigureSwarm(discovery string, master bool, host string, addr string) error {
	d := h.Driver

	if d.DriverName() == "none" {
		return nil
	}

	if addr == "" {
		ip, err := d.GetIP()
		if err != nil {
			return err
		}
		// TODO: remove hardcoded port
		addr = fmt.Sprintf("%s:2376", ip)
	}

	var (
		role string
		args string
	)

	if master {
		role = "manage"
		// TODO: remove hardcoded paths
		args = fmt.Sprintf("--tlsverify --tlscacert=/etc/docker/ca.pem --tlscert=/etc/docker/server.pem --tlskey=/etc/docker/server-key.pem -H %s",
			host)
	} else {
		role = "join"
		args = fmt.Sprintf("--addr %s", addr)
	}

	args = fmt.Sprintf("%s %s", args, discovery)

	u, err := url.Parse(host)
	if err != nil {
		return err
	}

	parts := strings.Split(u.Host, ":")
	port := parts[1]

	cmd, err := d.GetSSHCommand(fmt.Sprintf("sudo docker pull %s", swarmDockerImage))
	if err != nil {
		return err
	}
	if err := cmd.Run(); err != nil {
		return err
	}

	cmd, err = d.GetSSHCommand(fmt.Sprintf("sudo docker run -d -p %s:%s --restart=always --name swarm-agent -v /etc/docker:/etc/docker %s %s %s",
		port, port, swarmDockerImage, role, args))
	if err != nil {
		return err
	}
	if err := cmd.Run(); err != nil {
		return err
	}

	return nil
}

func (h *Host) ConfigureAuth() error {
	d := h.Driver

	if d.DriverName() == "none" {
		return nil
	}

	ip, err := h.Driver.GetIP()
	if err != nil {
		return err
	}

	serverCertPath := filepath.Join(h.storePath, "server.pem")
	serverKeyPath := filepath.Join(h.storePath, "server-key.pem")

	org := h.Name
	bits := 2048

	log.Debugf("generating server cert: %s", serverCertPath)

	if err := utils.GenerateCert([]string{ip}, serverCertPath, serverKeyPath, h.CaCertPath, h.PrivateKeyPath, org, bits); err != nil {
		return fmt.Errorf("error generating server cert: %s", err)
	}

	if err := d.StopDocker(); err != nil {
		return err
	}

	cmd, err := d.GetSSHCommand(fmt.Sprintf("sudo mkdir -p %s", d.GetDockerConfigDir()))
	if err != nil {
		return err
	}
	if err := cmd.Run(); err != nil {
		return err
	}

	// upload certs and configure TLS auth
	caCert, err := ioutil.ReadFile(h.CaCertPath)
	if err != nil {
		return err
	}

	// due to windows clients, we cannot use filepath.Join as the paths
	// will be mucked on the linux hosts
	machineCaCertPath := path.Join(d.GetDockerConfigDir(), "ca.pem")

	serverCert, err := ioutil.ReadFile(serverCertPath)
	if err != nil {
		return err
	}
	machineServerCertPath := path.Join(d.GetDockerConfigDir(), "server.pem")

	serverKey, err := ioutil.ReadFile(serverKeyPath)
	if err != nil {
		return err
	}
	machineServerKeyPath := path.Join(d.GetDockerConfigDir(), "server-key.pem")

	cmd, err = d.GetSSHCommand(fmt.Sprintf("echo \"%s\" | sudo tee -a %s", string(caCert), machineCaCertPath))
	if err != nil {
		return err
	}
	if err := cmd.Run(); err != nil {
		return err
	}

	cmd, err = d.GetSSHCommand(fmt.Sprintf("echo \"%s\" | sudo tee -a %s", string(serverKey), machineServerKeyPath))
	if err != nil {
		return err
	}
	if err := cmd.Run(); err != nil {
		return err
	}

	cmd, err = d.GetSSHCommand(fmt.Sprintf("echo \"%s\" | sudo tee -a %s", string(serverCert), machineServerCertPath))
	if err != nil {
		return err
	}
	if err := cmd.Run(); err != nil {
		return err
	}

	dockerUrl, err := h.Driver.GetURL()
	if err != nil {
		return err
	}
	u, err := url.Parse(dockerUrl)
	if err != nil {
		return err
	}
	dockerPort := 2376
	parts := strings.Split(u.Host, ":")
	if len(parts) == 2 {
		dPort, err := strconv.Atoi(parts[1])
		if err != nil {
			return err
		}
		dockerPort = dPort
	}

	cfg := h.generateDockerConfig(dockerPort, machineCaCertPath, machineServerKeyPath, machineServerCertPath)

	cmd, err = d.GetSSHCommand(fmt.Sprintf("echo \"%s\" | sudo tee -a %s", cfg.EngineConfig, cfg.EngineConfigPath))
	if err != nil {
		return err
	}
	if err := cmd.Run(); err != nil {
		return err
	}

	if err := d.StartDocker(); err != nil {
		return err
	}

	return nil
}

func (h *Host) generateDockerConfig(dockerPort int, caCertPath string, serverKeyPath string, serverCertPath string) *DockerConfig {
	d := h.Driver
	var (
		daemonOpts    string
		daemonOptsCfg string
		daemonCfg     string
	)

	// TODO @ehazlett: template?
	defaultDaemonOpts := fmt.Sprintf(`--tlsverify \
--tlscacert=%s \
--tlskey=%s \
--tlscert=%s`, caCertPath, serverKeyPath, serverCertPath)

	switch d.DriverName() {

	case "virtualbox", "vmwarefusion", "vmwarevsphere", "hyper-v":
		daemonOpts = fmt.Sprintf("-H tcp://0.0.0.0:%d", dockerPort)
		daemonOptsCfg = path.Join(d.GetDockerConfigDir(), "profile")
		opts := fmt.Sprintf("%s %s", defaultDaemonOpts, daemonOpts)
		daemonCfg = fmt.Sprintf(`EXTRA_ARGS='%s'
CACERT=%s
SERVERCERT=%s
SERVERKEY=%s
DOCKER_TLS=no`, opts, caCertPath, serverKeyPath, serverCertPath)
	default:
		daemonOpts = fmt.Sprintf("--host=unix:///var/run/docker.sock --host=tcp://0.0.0.0:%d", dockerPort)
		daemonOptsCfg = "/etc/default/docker"
		opts := fmt.Sprintf("%s %s", defaultDaemonOpts, daemonOpts)
		daemonCfg = fmt.Sprintf("export DOCKER_OPTS=\\\"%s\\\"", opts)
	}

	return &DockerConfig{
		EngineConfig:     daemonCfg,
		EngineConfigPath: daemonOptsCfg,
	}
}

func (h *Host) Create(name string) error {
	name, err := ValidateHostName(name)
	if err != nil {
		return err
	}

	if err := h.Driver.Create(); err != nil {
		return err
	}

	if err := h.SaveConfig(); err != nil {
		return err
	}

	return nil
}

func (h *Host) Start() error {
	return h.Driver.Start()
}

func (h *Host) Stop() error {
	return h.Driver.Stop()
}

func (h *Host) Upgrade() error {
	return h.Driver.Upgrade()
}

func (h *Host) Remove(force bool) error {
	if err := h.Driver.Remove(); err != nil {
		if !force {
			return err
		}
	}
	return h.removeStorePath()
}

func (h *Host) removeStorePath() error {
	file, err := os.Stat(h.storePath)
	if err != nil {
		return err
	}
	if !file.IsDir() {
		return fmt.Errorf("%q is not a directory", h.storePath)
	}
	return os.RemoveAll(h.storePath)
}

func (h *Host) GetURL() (string, error) {
	return h.Driver.GetURL()
}

func (h *Host) LoadConfig() error {
	data, err := ioutil.ReadFile(filepath.Join(h.storePath, "config.json"))
	if err != nil {
		return err
	}

	// First pass: find the driver name and load the driver
	var config hostConfig
	if err := json.Unmarshal(data, &config); err != nil {
		return err
	}

	driver, err := drivers.NewDriver(config.DriverName, h.Name, h.storePath, h.CaCertPath, h.PrivateKeyPath)
	if err != nil {
		return err
	}
	h.Driver = driver

	// Second pass: unmarshal driver config into correct driver
	if err := json.Unmarshal(data, &h); err != nil {
		return err
	}

	return nil
}

func (h *Host) SaveConfig() error {
	data, err := json.Marshal(h)
	if err != nil {
		return err
	}
	if err := ioutil.WriteFile(filepath.Join(h.storePath, "config.json"), data, 0600); err != nil {
		return err
	}
	return nil
}
<commit_msg>virtualbox swarm support<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/machine/drivers"
	"github.com/docker/machine/utils"
)

var (
	validHostNameChars   = `[a-zA-Z0-9\-\.]`
	validHostNamePattern = regexp.MustCompile(`^` + validHostNameChars + `+$`)
	ErrInvalidHostname   = errors.New("Invalid hostname specified")
)

const (
	swarmDockerImage = "ehazlett/swarm:latest"
)

type Host struct {
	Name           string `json:"-"`
	DriverName     string
	Driver         drivers.Driver
	CaCertPath     string
	ServerCertPath string
	ServerKeyPath  string
	PrivateKeyPath string
	ClientCertPath string
	storePath      string
}

type DockerConfig struct {
	EngineConfig     string
	EngineConfigPath string
}

type hostConfig struct {
	DriverName string
}

func NewHost(name, driverName, storePath, caCert, privateKey string) (*Host, error) {
	driver, err := drivers.NewDriver(driverName, name, storePath, caCert, privateKey)
	if err != nil {
		return nil, err
	}
	return &Host{
		Name:           name,
		DriverName:     driverName,
		Driver:         driver,
		CaCertPath:     caCert,
		PrivateKeyPath: privateKey,
		storePath:      storePath,
	}, nil
}

func LoadHost(name string, storePath string) (*Host, error) {
	if _, err := os.Stat(storePath); os.IsNotExist(err) {
		return nil, fmt.Errorf("Host %q does not exist", name)
	}

	host := &Host{Name: name, storePath: storePath}
	if err := host.LoadConfig(); err != nil {
		return nil, err
	}
	return host, nil
}

func ValidateHostName(name string) (string, error) {
	if !validHostNamePattern.MatchString(name) {
		return name, ErrInvalidHostname
	}
	return name, nil
}

func GenerateClientCertificate(caCertPath, privateKeyPath string) error {
	var (
		org  = "docker-machine"
		bits = 2048
	)

	clientCertPath := filepath.Join(utils.GetMachineDir(), "cert.pem")
	clientKeyPath := filepath.Join(utils.GetMachineDir(), "key.pem")

	if err := os.MkdirAll(utils.GetMachineDir(), 0700); err != nil {
		return err
	}

	log.Debugf("generating client cert: %s", clientCertPath)
	if err := utils.GenerateCert([]string{""}, clientCertPath, clientKeyPath, caCertPath, privateKeyPath, org, bits); err != nil {
		return fmt.Errorf("error generating client cert: %s", err)
	}

	return nil
}

func (h *Host) ConfigureSwarm(discovery string, master bool, host string, addr string) error {
	d := h.Driver

	if d.DriverName() == "none" {
		return nil
	}

	if addr == "" {
		ip, err := d.GetIP()
		if err != nil {
			return err
		}
		// TODO: remove hardcoded port
		addr = fmt.Sprintf("%s:2376", ip)
	}

	var (
		role string
		args string
	)

	if master {
		role = "manage"
		basePath := d.GetDockerConfigDir()
		tlsCaCert := path.Join(basePath, "ca.pem")
		tlsCert := path.Join(basePath, "server.pem")
		tlsKey := path.Join(basePath, "server-key.pem")
		args = fmt.Sprintf("--tlsverify --tlscacert=%s --tlscert=%s --tlskey=%s -H %s",
			tlsCaCert, tlsCert, tlsKey, host)
	} else {
		role = "join"
		args = fmt.Sprintf("--addr %s", addr)
	}

	args = fmt.Sprintf("%s %s", args, discovery)

	u, err := url.Parse(host)
	if err != nil {
		return err
	}

	parts := strings.Split(u.Host, ":")
	port := parts[1]

	cmd, err := d.GetSSHCommand(fmt.Sprintf("sudo docker pull %s", swarmDockerImage))
	if err != nil {
		return err
	}
	if err := cmd.Run(); err != nil {
		return err
	}

	cmd, err = d.GetSSHCommand(fmt.Sprintf("sudo docker run -d -p %s:%s --restart=always --name swarm-agent -v %s:%s %s %s %s",
		port, port, d.GetDockerConfigDir(), d.GetDockerConfigDir(), swarmDockerImage, role, args))
	if err != nil {
		return err
	}
	if err := cmd.Run(); err != nil {
		return err
	}

	return nil
}

func (h *Host) ConfigureAuth() error {
	d := h.Driver

	if d.DriverName() == "none" {
		return nil
	}

	ip, err := h.Driver.GetIP()
	if err != nil {
		return err
	}

	serverCertPath := filepath.Join(h.storePath, "server.pem")
	serverKeyPath := filepath.Join(h.storePath, "server-key.pem")

	org := h.Name
	bits := 2048

	log.Debugf("generating server cert: %s", serverCertPath)

	if err := utils.GenerateCert([]string{ip}, serverCertPath, serverKeyPath, h.CaCertPath, h.PrivateKeyPath, org, bits); err != nil {
		return fmt.Errorf("error generating server cert: %s", err)
	}

	if err := d.StopDocker(); err != nil {
		return err
	}

	cmd, err := d.GetSSHCommand(fmt.Sprintf("sudo mkdir -p %s", d.GetDockerConfigDir()))
	if err != nil {
		return err
	}
	if err := cmd.Run(); err != nil {
		return err
	}

	// upload certs and configure TLS auth
	caCert, err := ioutil.ReadFile(h.CaCertPath)
	if err != nil {
		return err
	}

	// due to windows clients, we cannot use filepath.Join as the paths
	// will be mucked on the linux hosts
	machineCaCertPath := path.Join(d.GetDockerConfigDir(), "ca.pem")

	serverCert, err := ioutil.ReadFile(serverCertPath)
	if err != nil {
		return err
	}
	machineServerCertPath := path.Join(d.GetDockerConfigDir(), "server.pem")

	serverKey, err := ioutil.ReadFile(serverKeyPath)
	if err != nil {
		return err
	}
	machineServerKeyPath := path.Join(d.GetDockerConfigDir(), "server-key.pem")

	cmd, err = d.GetSSHCommand(fmt.Sprintf("echo \"%s\" | sudo tee -a %s", string(caCert), machineCaCertPath))
	if err != nil {
		return err
	}
	if err := cmd.Run(); err != nil {
		return err
	}

	cmd, err = d.GetSSHCommand(fmt.Sprintf("echo \"%s\" | sudo tee -a %s", string(serverKey), machineServerKeyPath))
	if err != nil {
		return err
	}
	if err := cmd.Run(); err != nil {
		return err
	}

	cmd, err = d.GetSSHCommand(fmt.Sprintf("echo \"%s\" | sudo tee -a %s", string(serverCert), machineServerCertPath))
	if err != nil {
		return err
	}
	if err := cmd.Run(); err != nil {
		return err
	}

	dockerUrl, err := h.Driver.GetURL()
	if err != nil {
		return err
	}
	u, err := url.Parse(dockerUrl)
	if err != nil {
		return err
	}
	dockerPort := 2376
	parts := strings.Split(u.Host, ":")
	if len(parts) == 2 {
		dPort, err := strconv.Atoi(parts[1])
		if err != nil {
			return err
		}
		dockerPort = dPort
	}

	cfg := h.generateDockerConfig(dockerPort, machineCaCertPath, machineServerKeyPath, machineServerCertPath)

	cmd, err = d.GetSSHCommand(fmt.Sprintf("echo \"%s\" | sudo tee -a %s", cfg.EngineConfig, cfg.EngineConfigPath))
	if err != nil {
		return err
	}
	if err := cmd.Run(); err != nil {
		return err
	}

	if err := d.StartDocker(); err != nil {
		return err
	}

	return nil
}

func (h *Host) generateDockerConfig(dockerPort int, caCertPath string, serverKeyPath string, serverCertPath string) *DockerConfig {
	d := h.Driver
	var (
		daemonOpts    string
		daemonOptsCfg string
		daemonCfg     string
	)

	// TODO @ehazlett: template?
	defaultDaemonOpts := fmt.Sprintf(`--tlsverify \
--tlscacert=%s \
--tlskey=%s \
--tlscert=%s`, caCertPath, serverKeyPath, serverCertPath)

	switch d.DriverName() {

	case "virtualbox", "vmwarefusion", "vmwarevsphere", "hyper-v":
		daemonOpts = fmt.Sprintf("-H tcp://0.0.0.0:%d", dockerPort)
		daemonOptsCfg = path.Join(d.GetDockerConfigDir(), "profile")
		opts := fmt.Sprintf("%s %s", defaultDaemonOpts, daemonOpts)
		daemonCfg = fmt.Sprintf(`EXTRA_ARGS='%s'
CACERT=%s
SERVERCERT=%s
SERVERKEY=%s
DOCKER_TLS=no`, opts, caCertPath, serverKeyPath, serverCertPath)
	default:
		daemonOpts = fmt.Sprintf("--host=unix:///var/run/docker.sock --host=tcp://0.0.0.0:%d", dockerPort)
		daemonOptsCfg = "/etc/default/docker"
		opts := fmt.Sprintf("%s %s", defaultDaemonOpts, daemonOpts)
		daemonCfg = fmt.Sprintf("export DOCKER_OPTS=\\\"%s\\\"", opts)
	}

	return &DockerConfig{
		EngineConfig:     daemonCfg,
		EngineConfigPath: daemonOptsCfg,
	}
}

func (h *Host) Create(name string) error {
	name, err := ValidateHostName(name)
	if err != nil {
		return err
	}

	if err := h.Driver.Create(); err != nil {
		return err
	}

	if err := h.SaveConfig(); err != nil {
		return err
	}

	return nil
}

func (h *Host) Start() error {
	return h.Driver.Start()
}

func (h *Host) Stop() error {
	return h.Driver.Stop()
}

func (h *Host) Upgrade() error {
	return h.Driver.Upgrade()
}

func (h *Host) Remove(force bool) error {
	if err := h.Driver.Remove(); err != nil {
		if !force {
			return err
		}
	}
	return h.removeStorePath()
}

func (h *Host) removeStorePath() error {
	file, err := os.Stat(h.storePath)
	if err != nil {
		return err
	}
	if !file.IsDir() {
		return fmt.Errorf("%q is not a directory", h.storePath)
	}
	return os.RemoveAll(h.storePath)
}

func (h *Host) GetURL() (string, error) {
	return h.Driver.GetURL()
}

func (h *Host) LoadConfig() error {
	data, err := ioutil.ReadFile(filepath.Join(h.storePath, "config.json"))
	if err != nil {
		return err
	}

	// First pass: find the driver name and load the driver
	var config hostConfig
	if err := json.Unmarshal(data, &config); err != nil {
		return err
	}

	driver, err := drivers.NewDriver(config.DriverName, h.Name, h.storePath, h.CaCertPath, h.PrivateKeyPath)
	if err != nil {
		return err
	}
	h.Driver = driver

	// Second pass: unmarshal driver config into correct driver
	if err := json.Unmarshal(data, &h); err != nil {
		return err
	}

	return nil
}

func (h *Host) SaveConfig() error {
	data, err := json.Marshal(h)
	if err != nil {
		return err
	}
	if err := ioutil.WriteFile(filepath.Join(h.storePath, "config.json"), data, 0600); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package inspector provides helper functions for traversal over the
// syntax trees of a package, including node filtering by type, and
// materialization of the traversal stack.
//
// During construction, the inspector does a complete traversal and
// builds a list of push/pop events and their node type. Subsequent
// method calls that request a traversal scan this list, rather than walk
// the AST, and perform type filtering using efficient bit sets.
//
// Experiments suggest the inspector's traversals are about 2.5x faster
// than ast.Inspect, but it may take around 5 traversals for this
// benefit to amortize the inspector's construction cost.
// If efficiency is the primary concern, do not use use Inspector for
// one-off traversals.
package inspector

// There are four orthogonal features in a traversal:
//  1 type filtering
//  2 pruning
//  3 postorder calls to f
//  4 stack
// Rather than offer all of them in the API,
// only a few combinations are exposed:
// - Preorder is the fastest and has fewest features,
//   but is the most commonly needed traversal.
// - Nodes and WithStack both provide pruning and postorder calls,
//   even though few clients need it, because supporting two versions
//   is not justified.
// More combinations could be supported by expressing them as
// wrappers around a more generic traversal, but this was measured
// and found to degrade performance significantly (30%).

import (
	"go/ast"
)

// An Inspector provides methods for inspecting
// (traversing) the syntax trees of a package.
type Inspector struct {
	events []event
}

// New returns an Inspector for the specified syntax trees.
func New(files []*ast.File) *Inspector {
	return &Inspector{traverse(files)}
}

// An event represents a push or a pop
// of an ast.Node during a traversal.
type event struct {
	node  ast.Node
	typ   uint64 // typeOf(node)
	index int    // 1 + index of corresponding pop event, or 0 if this is a pop
}

// Preorder visits all the nodes of the files supplied to New in
// depth-first order. It calls f(n) for each node n before it visits
// n's children.
//
// The types argument, if non-empty, enables type-based filtering of
// events. The function f if is called only for nodes whose type
// matches an element of the types slice.
func (in *Inspector) Preorder(types []ast.Node, f func(ast.Node)) {
	// Because it avoids postorder calls to f, and the pruning
	// check, Preorder is almost twice as fast as Nodes. The two
	// features seem to contribute similar slowdowns (~1.4x each).

	mask := maskOf(types)
	for i := 0; i < len(in.events); {
		ev := in.events[i]
		if ev.typ&mask != 0 {
			if ev.index > 0 {
				f(ev.node)
			}
		}
		i++
	}
}

// Nodes visits the nodes of the files supplied to New in depth-first
// order. It calls f(n, true) for each node n before it visits n's
// children. If f returns true, Nodes invokes f recursively for each
// of the non-nil children of the node, followed by a call of
// f(n, false).
//
// The types argument, if non-empty, enables type-based filtering of
// events. The function f if is called only for nodes whose type
// matches an element of the types slice.
func (in *Inspector) Nodes(types []ast.Node, f func(n ast.Node, push bool) (prune bool)) {
	mask := maskOf(types)
	for i := 0; i < len(in.events); {
		ev := in.events[i]
		if ev.typ&mask != 0 {
			if ev.index > 0 {
				// push
				if !f(ev.node, true) {
					i = ev.index // jump to corresponding pop + 1
					continue
				}
			} else {
				// pop
				f(ev.node, false)
			}
		}
		i++
	}
}

// WithStack visits nodes in a similar manner to Nodes, but it
// supplies each call to f an additional argument, the current
// traversal stack. The stack's first element is the outermost node,
// an *ast.File; its last is the innermost, n.
func (in *Inspector) WithStack(types []ast.Node, f func(n ast.Node, push bool, stack []ast.Node) (prune bool)) {
	mask := maskOf(types)
	var stack []ast.Node
	for i := 0; i < len(in.events); {
		ev := in.events[i]
		if ev.index > 0 {
			// push
			stack = append(stack, ev.node)
			if ev.typ&mask != 0 {
				if !f(ev.node, true, stack) {
					i = ev.index
					stack = stack[:len(stack)-1]
					continue
				}
			}
		} else {
			// pop
			if ev.typ&mask != 0 {
				f(ev.node, false, stack)
			}
			stack = stack[:len(stack)-1]
		}
		i++
	}
}

// traverse builds the table of events representing a traversal.
func traverse(files []*ast.File) []event {
	// Preallocate approximate number of events
	// based on source file extent.
	// This makes traverse faster by 4x (!).
	var extent int
	for _, f := range files {
		extent += int(f.End() - f.Pos())
	}
	// This estimate is based on the net/http package.
	events := make([]event, 0, extent*33/100)

	var stack []event
	for _, f := range files {
		ast.Inspect(f, func(n ast.Node) bool {
			if n != nil {
				// push
				ev := event{
					node:  n,
					typ:   typeOf(n),
					index: len(events), // push event temporarily holds own index
				}
				stack = append(stack, ev)
				events = append(events, ev)
			} else {
				// pop
				ev := stack[len(stack)-1]
				stack = stack[:len(stack)-1]

				events[ev.index].index = len(events) + 1 // make push refer to pop

				ev.index = 0 // turn ev into a pop event
				events = append(events, ev)
			}
			return true
		})
	}

	return events
}
<commit_msg>go/ast/inspector: fix a small typo<commit_after>// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package inspector provides helper functions for traversal over the
// syntax trees of a package, including node filtering by type, and
// materialization of the traversal stack.
//
// During construction, the inspector does a complete traversal and
// builds a list of push/pop events and their node type. Subsequent
// method calls that request a traversal scan this list, rather than walk
// the AST, and perform type filtering using efficient bit sets.
//
// Experiments suggest the inspector's traversals are about 2.5x faster
// than ast.Inspect, but it may take around 5 traversals for this
// benefit to amortize the inspector's construction cost.
// If efficiency is the primary concern, do not use Inspector for
// one-off traversals.
package inspector

// There are four orthogonal features in a traversal:
//  1 type filtering
//  2 pruning
//  3 postorder calls to f
//  4 stack
// Rather than offer all of them in the API,
// only a few combinations are exposed:
// - Preorder is the fastest and has fewest features,
//   but is the most commonly needed traversal.
// - Nodes and WithStack both provide pruning and postorder calls,
//   even though few clients need it, because supporting two versions
//   is not justified.
// More combinations could be supported by expressing them as
// wrappers around a more generic traversal, but this was measured
// and found to degrade performance significantly (30%).

import (
	"go/ast"
)

// An Inspector provides methods for inspecting
// (traversing) the syntax trees of a package.
type Inspector struct {
	events []event
}

// New returns an Inspector for the specified syntax trees.
func New(files []*ast.File) *Inspector {
	return &Inspector{traverse(files)}
}

// An event represents a push or a pop
// of an ast.Node during a traversal.
type event struct {
	node  ast.Node
	typ   uint64 // typeOf(node)
	index int    // 1 + index of corresponding pop event, or 0 if this is a pop
}

// Preorder visits all the nodes of the files supplied to New in
// depth-first order. It calls f(n) for each node n before it visits
// n's children.
//
// The types argument, if non-empty, enables type-based filtering of
// events. The function f if is called only for nodes whose type
// matches an element of the types slice.
func (in *Inspector) Preorder(types []ast.Node, f func(ast.Node)) {
	// Because it avoids postorder calls to f, and the pruning
	// check, Preorder is almost twice as fast as Nodes. The two
	// features seem to contribute similar slowdowns (~1.4x each).

	mask := maskOf(types)
	for i := 0; i < len(in.events); {
		ev := in.events[i]
		if ev.typ&mask != 0 {
			if ev.index > 0 {
				f(ev.node)
			}
		}
		i++
	}
}

// Nodes visits the nodes of the files supplied to New in depth-first
// order. It calls f(n, true) for each node n before it visits n's
// children. If f returns true, Nodes invokes f recursively for each
// of the non-nil children of the node, followed by a call of
// f(n, false).
//
// The types argument, if non-empty, enables type-based filtering of
// events. The function f if is called only for nodes whose type
// matches an element of the types slice.
func (in *Inspector) Nodes(types []ast.Node, f func(n ast.Node, push bool) (prune bool)) {
	mask := maskOf(types)
	for i := 0; i < len(in.events); {
		ev := in.events[i]
		if ev.typ&mask != 0 {
			if ev.index > 0 {
				// push
				if !f(ev.node, true) {
					i = ev.index // jump to corresponding pop + 1
					continue
				}
			} else {
				// pop
				f(ev.node, false)
			}
		}
		i++
	}
}

// WithStack visits nodes in a similar manner to Nodes, but it
// supplies each call to f an additional argument, the current
// traversal stack. The stack's first element is the outermost node,
// an *ast.File; its last is the innermost, n.
func (in *Inspector) WithStack(types []ast.Node, f func(n ast.Node, push bool, stack []ast.Node) (prune bool)) {
	mask := maskOf(types)
	var stack []ast.Node
	for i := 0; i < len(in.events); {
		ev := in.events[i]
		if ev.index > 0 {
			// push
			stack = append(stack, ev.node)
			if ev.typ&mask != 0 {
				if !f(ev.node, true, stack) {
					i = ev.index
					stack = stack[:len(stack)-1]
					continue
				}
			}
		} else {
			// pop
			if ev.typ&mask != 0 {
				f(ev.node, false, stack)
			}
			stack = stack[:len(stack)-1]
		}
		i++
	}
}

// traverse builds the table of events representing a traversal.
func traverse(files []*ast.File) []event {
	// Preallocate approximate number of events
	// based on source file extent.
	// This makes traverse faster by 4x (!).
	var extent int
	for _, f := range files {
		extent += int(f.End() - f.Pos())
	}
	// This estimate is based on the net/http package.
	events := make([]event, 0, extent*33/100)

	var stack []event
	for _, f := range files {
		ast.Inspect(f, func(n ast.Node) bool {
			if n != nil {
				// push
				ev := event{
					node:  n,
					typ:   typeOf(n),
					index: len(events), // push event temporarily holds own index
				}
				stack = append(stack, ev)
				events = append(events, ev)
			} else {
				// pop
				ev := stack[len(stack)-1]
				stack = stack[:len(stack)-1]

				events[ev.index].index = len(events) + 1 // make push refer to pop

				ev.index = 0 // turn ev into a pop event
				events = append(events, ev)
			}
			return true
		})
	}

	return events
}
<|endoftext|>


<commit_before>package plaintext

import (
	"bytes"
	"golang.org/x/net/html"
)

// HTMLText extracts plain text from HTML markup
type HTMLText struct {
	InspectImageAlt bool
}

// InspectImageAlt is a sample for options  WIP
func InspectImageAlt(opt *HTMLText) error {
	opt.InspectImageAlt = true
	return nil
}

// NewHTMLText creates a new HTMLText extractor, using options.
func NewHTMLText(options ...func(*HTMLText) error) (*HTMLText, error) {
	extractor := HTMLText{}
	for _, option := range options {
		err := option(&extractor)
		if err != nil {
			return nil, err
		}
	}
	return &extractor, nil
}

// Text satifies the plaintext.Extractor interface
func (p *HTMLText) Text(raw []byte) []byte {
	isCodeTag := false
	isStyleTag := false
	isScriptTag := false

	out := bytes.Buffer{}

	z := html.NewTokenizer(bytes.NewReader(raw))
	for {
		tt := z.Next()
		switch tt {
		case html.ErrorToken:
			return bytes.TrimSpace(out.Bytes())
		case html.StartTagToken:
			tn, hasAttr := z.TagName()
			if bytes.Equal(tn, []byte("code")) {
				isCodeTag = true
				continue
			}
			if bytes.Equal(tn, []byte("style")) {
				isStyleTag = true
				continue
			}
			if bytes.Equal(tn, []byte("script")) {
				isScriptTag = true
				continue
			}
			if bytes.Equal(tn, []byte("img")) {
				var key, val []byte
				for hasAttr {
					key, val, hasAttr = z.TagAttr()
					if len(val) > 0 && bytes.Equal(key, []byte("alt")) {
						out.Write(val)
						out.Write([]byte(" "))
					}
				}
			}
		case html.EndTagToken:
			tn, _ := z.TagName()
			if bytes.Equal(tn, []byte("code")) {
				isCodeTag = false
				continue
			}
			if bytes.Equal(tn, []byte("style")) {
				isStyleTag = false
				continue
			}
			if bytes.Equal(tn, []byte("script")) {
				isScriptTag = false
				continue
			}
		case html.TextToken:
			if isCodeTag || isStyleTag || isScriptTag {
				continue
			}
			if plaintext := bytes.TrimSpace(z.Text()); len(plaintext) > 0 {
				out.Write(plaintext)
				out.Write([]byte(" "))
			}
		}
	}
}
<commit_msg>make output even prettier<commit_after>package plaintext

import (
	"bytes"
	"golang.org/x/net/html"
)

var blockTag = map[string]struct{}{
	"br":         {},
	"h1":         {},
	"h2":         {},
	"h3":         {},
	"h4":         {},
	"h5":         {},
	"pre":        {},
	"li":         {},
	"p":          {},
	"div":        {},
	"blockquote": {},
}

func isBlock(tag []byte) bool {
	_, ok := blockTag[string(tag)]
	return ok
}

// HTMLText extracts plain text from HTML markup
type HTMLText struct {
	InspectImageAlt bool
}

// InspectImageAlt is a sample for options  WIP
func InspectImageAlt(opt *HTMLText) error {
	opt.InspectImageAlt = true
	return nil
}

// NewHTMLText creates a new HTMLText extractor, using options.
func NewHTMLText(options ...func(*HTMLText) error) (*HTMLText, error) {
	extractor := HTMLText{}
	for _, option := range options {
		err := option(&extractor)
		if err != nil {
			return nil, err
		}
	}
	return &extractor, nil
}

// Text satifies the plaintext.Extractor interface
func (p *HTMLText) Text(raw []byte) []byte {
	isCodeTag := false
	isStyleTag := false
	isScriptTag := false

	out := bytes.Buffer{}

	z := html.NewTokenizer(bytes.NewReader(raw))
	for {
		tt := z.Next()
		switch tt {
		case html.ErrorToken:
			return bytes.TrimSpace(out.Bytes())
		case html.StartTagToken:
			tn, hasAttr := z.TagName()
			if bytes.Equal(tn, []byte("code")) {
				isCodeTag = true
				continue
			}
			if bytes.Equal(tn, []byte("style")) {
				isStyleTag = true
				continue
			}
			if bytes.Equal(tn, []byte("script")) {
				isScriptTag = true
				continue
			}
			if bytes.Equal(tn, []byte("img")) {
				var key, val []byte
				for hasAttr {
					key, val, hasAttr = z.TagAttr()
					if len(val) > 0 && bytes.Equal(key, []byte("alt")) {
						out.Write(val)
						out.Write([]byte(" "))
					}
				}
			}
		case html.EndTagToken:
			tn, _ := z.TagName()
			if bytes.Equal(tn, []byte("code")) {
				isCodeTag = false
				continue
			}
			if bytes.Equal(tn, []byte("style")) {
				isStyleTag = false
				continue
			}
			if bytes.Equal(tn, []byte("script")) {
				isScriptTag = false
				continue
			}
			if isBlock(tn) {
				out.Write([]byte("\n"))
			}
		case html.TextToken:
			if isCodeTag || isStyleTag || isScriptTag {
				continue
			}
			if plaintext := bytes.TrimSpace(z.Text()); len(plaintext) > 0 {
				out.Write(plaintext)
				out.Write([]byte(" "))
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/dustin/gomemcached/client"
)

type BlobOwnership struct {
	OID   string               `json:"oid"`
	Nodes map[string]time.Time `json:"nodes"`
	Type  string               `json:"type"`
}

func recordBlobOwnership(h string) error {
	sid := serverIdentifier()

	k := "/" + h
	return couchbase.Do(k, func(mc *memcached.Client, vb uint16) error {
		_, err := mc.CAS(vb, k, func(in []byte) []byte {
			ownership := BlobOwnership{}
			err := json.Unmarshal(in, &ownership)
			if err == nil {
				ownership.Nodes[sid] = time.Now().UTC()
			} else {
				ownership.Nodes = map[string]time.Time{
					sid: time.Now().UTC(),
				}
				ownership.OID = h
			}
			ownership.Type = "blobowner"

			rv, err := json.Marshal(&ownership)
			if err != nil {
				log.Fatalf("Error marshaling blob ownership: %v", err)
			}
			return rv
		}, 0)
		return err
	})
}

func recordBlobAccess(h string) {
	_, err := couchbase.Incr("/"+h+"/r", 1, 1, 0)
	if err != nil {
		log.Printf("Error incrementing counter for %v: %v", h, err)
	}

	_, err = couchbase.Incr("/"+serverIdentifier()+"/r", 1, 1, 0)
	if err != nil {
		log.Printf("Error incrementing node identifier: %v", h, err)
	}
}

func putUserFile(w http.ResponseWriter, req *http.Request) {
	sh := getHash()

	tmpf, err := ioutil.TempFile(*root, "tmp")
	if err != nil {
		log.Printf("Error writing tmp file: %v", err)
		w.WriteHeader(500)
		return
	}

	length, err := io.Copy(io.MultiWriter(tmpf, sh), req.Body)
	if err != nil {
		log.Printf("Error writing data from client: %v", err)
		w.WriteHeader(500)
		return
	}

	h := hex.EncodeToString(sh.Sum([]byte{}))
	fn := hashFilename(h)

	err = os.Rename(tmpf.Name(), fn)
	if err != nil {
		os.MkdirAll(filepath.Dir(fn), 0777)
		err = os.Rename(tmpf.Name(), fn)
		if err != nil {
			log.Printf("Error renaming %v to %v: %v", tmpf.Name(), fn, err)
			w.WriteHeader(500)
			os.Remove(tmpf.Name())
			return
		}
	}

	log.Printf("Wrote %v -> %v (%#v)", req.URL.Path, h, req.Header)

	fm := fileMeta{
		req.Header,
		h,
		length,
	}

	err = storeMeta(req.URL.Path[1:], fm)
	if err != nil {
		log.Printf("Error storing file meta: %v", err)
		w.WriteHeader(500)
		fmt.Fprintf(w, "Error recording blob ownership: %v", err)
		return
	}

	err = recordBlobOwnership(h)
	if err != nil {
		log.Printf("Error storing blob ownership: %v", err)
		w.WriteHeader(500)
		fmt.Fprintf(w, "Error recording blob ownership: %v", err)
		return
	}

	w.WriteHeader(204)
}

func putRawHash(w http.ResponseWriter, req *http.Request) {
	inputhash := ""
	form, err := url.ParseQuery(req.URL.RawQuery)
	if err == nil {
		inputhash = form.Get("oid")
	}

	if inputhash == "" {
		w.WriteHeader(400)
		w.Write([]byte("No oid specified"))
		return
	}

	tmpf, err := ioutil.TempFile(*root, "tmp")
	if err != nil {
		log.Printf("Error writing tmp file: %v", err)
		w.WriteHeader(500)
		os.Remove(tmpf.Name())
		return
	}

	sh := getHash()
	_, err = io.Copy(io.MultiWriter(tmpf, sh), req.Body)
	if err != nil {
		log.Printf("Error writing data from client: %v", err)
		w.WriteHeader(500)
		os.Remove(tmpf.Name())
		return
	}

	h := hex.EncodeToString(sh.Sum([]byte{}))
	if h != inputhash {
		w.WriteHeader(400)
		fmt.Fprintf(w, "Content hashed to %v, expected %v", h, inputhash)
		os.Remove(tmpf.Name())
		return

	}
	fn := hashFilename(h)

	err = os.Rename(tmpf.Name(), fn)
	if err != nil {
		os.MkdirAll(filepath.Dir(fn), 0777)
		err = os.Rename(tmpf.Name(), fn)
		if err != nil {
			log.Printf("Error renaming %v to %v: %v", tmpf.Name(), fn, err)
			w.WriteHeader(500)
			os.Remove(tmpf.Name())
			return
		}
	}

	err = recordBlobOwnership(h)
	if err != nil {
		log.Printf("Error recording blob ownership: %v", err)
		w.WriteHeader(500)
		fmt.Fprintf(w, "Error recording blob ownership: %v", err)
		return
	}

	w.WriteHeader(204)
}

func doPut(w http.ResponseWriter, req *http.Request) {
	if req.URL.Path == "/" {
		putRawHash(w, req)
	} else {
		putUserFile(w, req)
	}
}

func isResponseHeader(s string) bool {
	switch s {
	case "Content-Type", "Content-Length":
		return true
	}
	return false
}

func doGetUserDoc(w http.ResponseWriter, req *http.Request) {
	path := req.URL.Path
	if path == "/" {
		path = *defaultPath
	}

	if len(path) > 0 && path[0] == '/' {
		path = path[1:]
	}

	got := fileMeta{}
	err := couchbase.Get(path, &got)
	if err != nil {
		log.Printf("Error getting file %#v: %v", path, err)
		w.WriteHeader(404)
		return
	}

	log.Printf("Need to find blob %v", got.OID)

	f, err := os.Open(hashFilename(got.OID))
	if err != nil {
		getBlobFromRemote(w, got.OID)
		return;
	}
	defer f.Close()

	for k, v := range got.Headers {
		if isResponseHeader(k) {
			w.Header()[k] = v
		}
	}
	w.WriteHeader(200)

	_, err = io.Copy(w, f)
	if err != nil {
		log.Printf("Failed to write file: %v", err)
	}
	go recordBlobAccess(got.OID)
}

func getBlobFromRemote(w http.ResponseWriter, oid string) {

	// Find the owners of this blob
	ownership := BlobOwnership{}
	oidkey := "/" + oid
	err := couchbase.Get(oidkey, &ownership)
	if err != nil {
		log.Printf("Missing ownership record for OID: %v", oid)
		// Not sure 404 is the right response here
		w.WriteHeader(404)
		return
	}

	// Loop through the nodes that claim to own this blob
	// If we encounter any errors along the way, try the next node
	for sid, _ := range ownership.Nodes {
	   log.Printf("Trying to get %s from %s", oid, sid)
		sidaddr, err := getNodeAddress(sid)
		if err != nil {
			log.Printf("Missing node record for %s", sid)
			continue
		}

		remoteOidURL := fmt.Sprintf("http://%s/?oid=%s", sidaddr, oid)
		resp, err := http.Get(remoteOidURL)
		if err != nil {
			log.Printf("Error reading oid %s from node %s", oid, sid)
			continue
		}

		if resp.StatusCode != 200 {
			log.Printf("Error response code %d from node %s", resp.StatusCode, sid)
			continue
		}

		_, err2 := io.Copy(w, resp.Body)
		if err2 != nil {
			log.Printf("Failed to write from remote stream %v", err2)
		}
		return
	}

	//if we got to this point, no node in the list actually had it
	log.Printf("Don't have hash file: %v and no remote nodes could help", oid)
	w.WriteHeader(500)
	return
}

func doList(w http.ResponseWriter, req *http.Request) {
	w.WriteHeader(200)
	explen := getHash().Size() * 2
	filepath.Walk(*root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.IsDir() && !strings.HasPrefix(info.Name(), "tmp") &&
			len(info.Name()) == explen {
			_, e := w.Write([]byte(info.Name() + "\n"))
			return e
		}
		return nil
	})
}

func doGet(w http.ResponseWriter, req *http.Request) {
	switch {
	case req.URL.Path == "/" && req.FormValue("oid") != "":
		http.ServeFile(w, req, hashFilename(req.FormValue("oid")))
	case req.URL.Path == "/" && req.FormValue("list") != "":
		doList(w, req)
	default:
		doGetUserDoc(w, req)
	}
}

func httpHandler(w http.ResponseWriter, req *http.Request) {
	defer req.Body.Close()
	switch req.Method {
	case "PUT":
		doPut(w, req)
	case "GET":
		doGet(w, req)
	default:
		w.WriteHeader(http.StatusMethodNotAllowed)
	}
}
<commit_msg>Record blob length.<commit_after>package main

import (
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/dustin/gomemcached/client"
)

type BlobOwnership struct {
	OID    string               `json:"oid"`
	Length int64                `json:"length"`
	Nodes  map[string]time.Time `json:"nodes"`
	Type   string               `json:"type"`
}

func recordBlobOwnership(h string, l int64) error {
	sid := serverIdentifier()

	k := "/" + h
	return couchbase.Do(k, func(mc *memcached.Client, vb uint16) error {
		_, err := mc.CAS(vb, k, func(in []byte) []byte {
			ownership := BlobOwnership{}
			err := json.Unmarshal(in, &ownership)
			if err == nil {
				ownership.Nodes[sid] = time.Now().UTC()
			} else {
				ownership.Nodes = map[string]time.Time{
					sid: time.Now().UTC(),
				}
			}
			ownership.OID = h
			ownership.Length = l
			ownership.Type = "blobowner"

			rv, err := json.Marshal(&ownership)
			if err != nil {
				log.Fatalf("Error marshaling blob ownership: %v", err)
			}
			return rv
		}, 0)
		return err
	})
}

func recordBlobAccess(h string) {
	_, err := couchbase.Incr("/"+h+"/r", 1, 1, 0)
	if err != nil {
		log.Printf("Error incrementing counter for %v: %v", h, err)
	}

	_, err = couchbase.Incr("/"+serverIdentifier()+"/r", 1, 1, 0)
	if err != nil {
		log.Printf("Error incrementing node identifier: %v", h, err)
	}
}

func putUserFile(w http.ResponseWriter, req *http.Request) {
	sh := getHash()

	tmpf, err := ioutil.TempFile(*root, "tmp")
	if err != nil {
		log.Printf("Error writing tmp file: %v", err)
		w.WriteHeader(500)
		return
	}

	length, err := io.Copy(io.MultiWriter(tmpf, sh), req.Body)
	if err != nil {
		log.Printf("Error writing data from client: %v", err)
		w.WriteHeader(500)
		return
	}

	h := hex.EncodeToString(sh.Sum([]byte{}))
	fn := hashFilename(h)

	err = os.Rename(tmpf.Name(), fn)
	if err != nil {
		os.MkdirAll(filepath.Dir(fn), 0777)
		err = os.Rename(tmpf.Name(), fn)
		if err != nil {
			log.Printf("Error renaming %v to %v: %v", tmpf.Name(), fn, err)
			w.WriteHeader(500)
			os.Remove(tmpf.Name())
			return
		}
	}

	log.Printf("Wrote %v -> %v (%#v)", req.URL.Path, h, req.Header)

	fm := fileMeta{
		req.Header,
		h,
		length,
	}

	err = storeMeta(req.URL.Path[1:], fm)
	if err != nil {
		log.Printf("Error storing file meta: %v", err)
		w.WriteHeader(500)
		fmt.Fprintf(w, "Error recording blob ownership: %v", err)
		return
	}

	err = recordBlobOwnership(h, length)
	if err != nil {
		log.Printf("Error storing blob ownership: %v", err)
		w.WriteHeader(500)
		fmt.Fprintf(w, "Error recording blob ownership: %v", err)
		return
	}

	w.WriteHeader(204)
}

func putRawHash(w http.ResponseWriter, req *http.Request) {
	inputhash := ""
	form, err := url.ParseQuery(req.URL.RawQuery)
	if err == nil {
		inputhash = form.Get("oid")
	}

	if inputhash == "" {
		w.WriteHeader(400)
		w.Write([]byte("No oid specified"))
		return
	}

	tmpf, err := ioutil.TempFile(*root, "tmp")
	if err != nil {
		log.Printf("Error writing tmp file: %v", err)
		w.WriteHeader(500)
		os.Remove(tmpf.Name())
		return
	}

	sh := getHash()
	length, err := io.Copy(io.MultiWriter(tmpf, sh), req.Body)
	if err != nil {
		log.Printf("Error writing data from client: %v", err)
		w.WriteHeader(500)
		os.Remove(tmpf.Name())
		return
	}

	h := hex.EncodeToString(sh.Sum([]byte{}))
	if h != inputhash {
		w.WriteHeader(400)
		fmt.Fprintf(w, "Content hashed to %v, expected %v", h, inputhash)
		os.Remove(tmpf.Name())
		return

	}
	fn := hashFilename(h)

	err = os.Rename(tmpf.Name(), fn)
	if err != nil {
		os.MkdirAll(filepath.Dir(fn), 0777)
		err = os.Rename(tmpf.Name(), fn)
		if err != nil {
			log.Printf("Error renaming %v to %v: %v", tmpf.Name(), fn, err)
			w.WriteHeader(500)
			os.Remove(tmpf.Name())
			return
		}
	}

	err = recordBlobOwnership(h, length)
	if err != nil {
		log.Printf("Error recording blob ownership: %v", err)
		w.WriteHeader(500)
		fmt.Fprintf(w, "Error recording blob ownership: %v", err)
		return
	}

	w.WriteHeader(204)
}

func doPut(w http.ResponseWriter, req *http.Request) {
	if req.URL.Path == "/" {
		putRawHash(w, req)
	} else {
		putUserFile(w, req)
	}
}

func isResponseHeader(s string) bool {
	switch s {
	case "Content-Type", "Content-Length":
		return true
	}
	return false
}

func doGetUserDoc(w http.ResponseWriter, req *http.Request) {
	path := req.URL.Path
	if path == "/" {
		path = *defaultPath
	}

	if len(path) > 0 && path[0] == '/' {
		path = path[1:]
	}

	got := fileMeta{}
	err := couchbase.Get(path, &got)
	if err != nil {
		log.Printf("Error getting file %#v: %v", path, err)
		w.WriteHeader(404)
		return
	}

	log.Printf("Need to find blob %v", got.OID)

	f, err := os.Open(hashFilename(got.OID))
	if err != nil {
		getBlobFromRemote(w, got.OID)
		return;
	}
	defer f.Close()

	for k, v := range got.Headers {
		if isResponseHeader(k) {
			w.Header()[k] = v
		}
	}
	w.WriteHeader(200)

	_, err = io.Copy(w, f)
	if err != nil {
		log.Printf("Failed to write file: %v", err)
	}
	go recordBlobAccess(got.OID)
}

func getBlobFromRemote(w http.ResponseWriter, oid string) {

	// Find the owners of this blob
	ownership := BlobOwnership{}
	oidkey := "/" + oid
	err := couchbase.Get(oidkey, &ownership)
	if err != nil {
		log.Printf("Missing ownership record for OID: %v", oid)
		// Not sure 404 is the right response here
		w.WriteHeader(404)
		return
	}

	// Loop through the nodes that claim to own this blob
	// If we encounter any errors along the way, try the next node
	for sid, _ := range ownership.Nodes {
	   log.Printf("Trying to get %s from %s", oid, sid)
		sidaddr, err := getNodeAddress(sid)
		if err != nil {
			log.Printf("Missing node record for %s", sid)
			continue
		}

		remoteOidURL := fmt.Sprintf("http://%s/?oid=%s", sidaddr, oid)
		resp, err := http.Get(remoteOidURL)
		if err != nil {
			log.Printf("Error reading oid %s from node %s", oid, sid)
			continue
		}

		if resp.StatusCode != 200 {
			log.Printf("Error response code %d from node %s", resp.StatusCode, sid)
			continue
		}

		_, err2 := io.Copy(w, resp.Body)
		if err2 != nil {
			log.Printf("Failed to write from remote stream %v", err2)
		}
		return
	}

	//if we got to this point, no node in the list actually had it
	log.Printf("Don't have hash file: %v and no remote nodes could help", oid)
	w.WriteHeader(500)
	return
}

func doList(w http.ResponseWriter, req *http.Request) {
	w.WriteHeader(200)
	explen := getHash().Size() * 2
	filepath.Walk(*root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.IsDir() && !strings.HasPrefix(info.Name(), "tmp") &&
			len(info.Name()) == explen {
			_, e := w.Write([]byte(info.Name() + "\n"))
			return e
		}
		return nil
	})
}

func doGet(w http.ResponseWriter, req *http.Request) {
	switch {
	case req.URL.Path == "/" && req.FormValue("oid") != "":
		http.ServeFile(w, req, hashFilename(req.FormValue("oid")))
	case req.URL.Path == "/" && req.FormValue("list") != "":
		doList(w, req)
	default:
		doGetUserDoc(w, req)
	}
}

func httpHandler(w http.ResponseWriter, req *http.Request) {
	defer req.Body.Close()
	switch req.Method {
	case "PUT":
		doPut(w, req)
	case "GET":
		doGet(w, req)
	default:
		w.WriteHeader(http.StatusMethodNotAllowed)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package sqlparser

import (
	"fmt"
	"testing"

	"github.com/stretchr/testify/require"
)

func TestLiteralID(t *testing.T) {
	testcases := []struct {
		in  string
		id  int
		out string
	}{{
		in:  "`aa`",
		id:  ID,
		out: "aa",
	}, {
		in:  "```a```",
		id:  ID,
		out: "`a`",
	}, {
		in:  "`a``b`",
		id:  ID,
		out: "a`b",
	}, {
		in:  "`a``b`c",
		id:  ID,
		out: "a`b",
	}, {
		in:  "`a``b",
		id:  LEX_ERROR,
		out: "a`b",
	}, {
		in:  "`a``b``",
		id:  LEX_ERROR,
		out: "a`b`",
	}, {
		in:  "``",
		id:  LEX_ERROR,
		out: "",
	}, {
		in:  "@x",
		id:  AT_ID,
		out: "x",
	}, {
		in:  "@@x",
		id:  AT_AT_ID,
		out: "x",
	}, {
		in:  "@@`x y`",
		id:  AT_AT_ID,
		out: "x y",
	}, {
		in:  "@@`@x @y`",
		id:  AT_AT_ID,
		out: "@x @y",
	}}

	for _, tcase := range testcases {
		tkn := NewStringTokenizer(tcase.in)
		id, out := tkn.Scan()
		if tcase.id != id || string(out) != tcase.out {
			t.Errorf("Scan(%s): %d, %s, want %d, %s", tcase.in, id, out, tcase.id, tcase.out)
		}
	}
}

func tokenName(id int) string {
	if id == STRING {
		return "STRING"
	} else if id == LEX_ERROR {
		return "LEX_ERROR"
	}
	return fmt.Sprintf("%d", id)
}

func TestString(t *testing.T) {
	testcases := []struct {
		in   string
		id   int
		want string
	}{{
		in:   "''",
		id:   STRING,
		want: "",
	}, {
		in:   "''''",
		id:   STRING,
		want: "'",
	}, {
		in:   "'hello'",
		id:   STRING,
		want: "hello",
	}, {
		in:   "'\\n'",
		id:   STRING,
		want: "\n",
	}, {
		in:   "'\\nhello\\n'",
		id:   STRING,
		want: "\nhello\n",
	}, {
		in:   "'a''b'",
		id:   STRING,
		want: "a'b",
	}, {
		in:   "'a\\'b'",
		id:   STRING,
		want: "a'b",
	}, {
		in:   "'\\'",
		id:   LEX_ERROR,
		want: "'",
	}, {
		in:   "'",
		id:   LEX_ERROR,
		want: "",
	}, {
		in:   "'hello\\'",
		id:   LEX_ERROR,
		want: "hello'",
	}, {
		in:   "'hello",
		id:   LEX_ERROR,
		want: "hello",
	}, {
		in:   "'hello\\",
		id:   LEX_ERROR,
		want: "hello",
	}}

	for _, tcase := range testcases {
		t.Run(tcase.in, func(t *testing.T) {
			id, got := NewStringTokenizer(tcase.in).Scan()
			if tcase.id != id || string(got) != tcase.want {
				t.Errorf("Scan(%q) = (%s, %q), want (%s, %q)", tcase.in, tokenName(id), got, tokenName(tcase.id), tcase.want)
			}
		})
	}
}

func TestSplitStatement(t *testing.T) {
	testcases := []struct {
		in  string
		sql string
		rem string
	}{{
		in:  "select * from table",
		sql: "select * from table",
	}, {
		in:  "select * from table; ",
		sql: "select * from table",
		rem: " ",
	}, {
		in:  "select * from table; select * from table2;",
		sql: "select * from table",
		rem: " select * from table2;",
	}, {
		in:  "select * from /* comment */ table;",
		sql: "select * from /* comment */ table",
	}, {
		in:  "select * from /* comment ; */ table;",
		sql: "select * from /* comment ; */ table",
	}, {
		in:  "select * from table where semi = ';';",
		sql: "select * from table where semi = ';'",
	}, {
		in:  "-- select * from table",
		sql: "-- select * from table",
	}, {
		in:  " ",
		sql: " ",
	}, {
		in:  "",
		sql: "",
	}}

	for _, tcase := range testcases {
		sql, rem, err := SplitStatement(tcase.in)
		if err != nil {
			t.Errorf("EndOfStatementPosition(%s): ERROR: %v", tcase.in, err)
			continue
		}

		if tcase.sql != sql {
			t.Errorf("EndOfStatementPosition(%s) got sql \"%s\" want \"%s\"", tcase.in, sql, tcase.sql)
		}

		if tcase.rem != rem {
			t.Errorf("EndOfStatementPosition(%s) got remainder \"%s\" want \"%s\"", tcase.in, rem, tcase.rem)
		}
	}
}

func TestVersion(t *testing.T) {
	testcases := []struct {
		version string
		in      string
		id      []int
	}{{
		version: "5.7.9",
		in:      "/*!80102 SELECT*/ FROM IN EXISTS",
		id:      []int{FROM, IN, EXISTS, eofChar},
	}, {
		version: "8.1.1",
		in:      "/*!80102 SELECT*/ FROM IN EXISTS",
		id:      []int{FROM, IN, EXISTS, eofChar},
	}, {
		version: "8.2.1",
		in:      "/*!80102 SELECT*/ FROM IN EXISTS",
		id:      []int{SELECT, FROM, IN, EXISTS, eofChar},
	}, {
		version: "8.1.2",
		in:      "/*!80102 SELECT*/ FROM IN EXISTS",
		id:      []int{SELECT, FROM, IN, EXISTS, eofChar},
	}}

	for _, tcase := range testcases {
		t.Run(tcase.in, func(t *testing.T) {
			MySQLVersion = tcase.version
			tok := NewStringTokenizer(tcase.in)
			for _, expectedID := range tcase.id {
				id, _ := tok.Scan()
				require.Equal(t, expectedID, id)
			}
		})
	}
}

func TestConvertMySQLVersion(t *testing.T) {
	testcases := []struct {
		version          string
		convertedVersion []int
		error            string
	}{{
		version:          "5.7.9",
		convertedVersion: []int{5, 7, 9},
	}, {
		version:          "0008.08.9",
		convertedVersion: []int{8, 8, 9},
	}, {
		version:          "501.7.9, Vitess - 10.0.1",
		convertedVersion: []int{501, 7, 9},
	}, {
		version:          "8.1 Vitess - 10.0.1",
		convertedVersion: []int{8, 1, 0},
	}, {
		version: "Vitess - 10.0.1",
		error:   "MySQL version not correctly setup - Vitess - 10.0.1.",
	}, {
		version:          "5.7.9.22",
		convertedVersion: []int{5, 7, 9},
	}}

	for _, tcase := range testcases {
		t.Run(tcase.version, func(t *testing.T) {
			output, err := convertMySQLVersion(tcase.version)
			if tcase.error != "" {
				require.EqualError(t, err, tcase.error)
			} else {
				require.NoError(t, err)
				require.Equal(t, tcase.convertedVersion, output)
			}
		})
	}
}

func TestConvertCommentVersion(t *testing.T) {
	testcases := []struct {
		version          string
		convertedVersion []int
		error            string
	}{{
		version:          "50709",
		convertedVersion: []int{5, 7, 9},
	}, {
		version:          "50110",
		convertedVersion: []int{5, 1, 10},
	}, {
		version:          "32312",
		convertedVersion: []int{3, 23, 12},
	}, {
		version:          "40100",
		convertedVersion: []int{4, 1, 0},
	}, {
		version:          "80016",
		convertedVersion: []int{8, 0, 16},
	}, {
		version:          "",
		convertedVersion: []int{0, 0, 0},
	}}

	for _, tcase := range testcases {
		t.Run(tcase.version, func(t *testing.T) {
			output, err := convertCommentVersion(tcase.version)
			if tcase.error != "" {
				require.EqualError(t, err, tcase.error)
			} else {
				require.NoError(t, err)
				require.Equal(t, tcase.convertedVersion, output)
			}
		})
	}
}

func TestExtractMySQLComment(t *testing.T) {
	testcases := []struct {
		comment string
		version string
	}{{
		comment: "/*!50108 SELECT * FROM */",
		version: "50108",
	}, {
		comment: "/*!5018 SELECT * FROM */",
		version: "",
	}, {
		comment: "/*!SELECT * FROM */",
		version: "",
	}}

	for _, tcase := range testcases {
		t.Run(tcase.version, func(t *testing.T) {
			output, _ := ExtractMysqlComment(tcase.comment)
			require.Equal(t, tcase.version, output)
		})
	}
}
<commit_msg>fixed test<commit_after>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package sqlparser

import (
	"fmt"
	"testing"

	"github.com/stretchr/testify/require"
)

func TestLiteralID(t *testing.T) {
	testcases := []struct {
		in  string
		id  int
		out string
	}{{
		in:  "`aa`",
		id:  ID,
		out: "aa",
	}, {
		in:  "```a```",
		id:  ID,
		out: "`a`",
	}, {
		in:  "`a``b`",
		id:  ID,
		out: "a`b",
	}, {
		in:  "`a``b`c",
		id:  ID,
		out: "a`b",
	}, {
		in:  "`a``b",
		id:  LEX_ERROR,
		out: "a`b",
	}, {
		in:  "`a``b``",
		id:  LEX_ERROR,
		out: "a`b`",
	}, {
		in:  "``",
		id:  LEX_ERROR,
		out: "",
	}, {
		in:  "@x",
		id:  AT_ID,
		out: "x",
	}, {
		in:  "@@x",
		id:  AT_AT_ID,
		out: "x",
	}, {
		in:  "@@`x y`",
		id:  AT_AT_ID,
		out: "x y",
	}, {
		in:  "@@`@x @y`",
		id:  AT_AT_ID,
		out: "@x @y",
	}}

	for _, tcase := range testcases {
		tkn := NewStringTokenizer(tcase.in)
		id, out := tkn.Scan()
		if tcase.id != id || string(out) != tcase.out {
			t.Errorf("Scan(%s): %d, %s, want %d, %s", tcase.in, id, out, tcase.id, tcase.out)
		}
	}
}

func tokenName(id int) string {
	if id == STRING {
		return "STRING"
	} else if id == LEX_ERROR {
		return "LEX_ERROR"
	}
	return fmt.Sprintf("%d", id)
}

func TestString(t *testing.T) {
	testcases := []struct {
		in   string
		id   int
		want string
	}{{
		in:   "''",
		id:   STRING,
		want: "",
	}, {
		in:   "''''",
		id:   STRING,
		want: "'",
	}, {
		in:   "'hello'",
		id:   STRING,
		want: "hello",
	}, {
		in:   "'\\n'",
		id:   STRING,
		want: "\n",
	}, {
		in:   "'\\nhello\\n'",
		id:   STRING,
		want: "\nhello\n",
	}, {
		in:   "'a''b'",
		id:   STRING,
		want: "a'b",
	}, {
		in:   "'a\\'b'",
		id:   STRING,
		want: "a'b",
	}, {
		in:   "'\\'",
		id:   LEX_ERROR,
		want: "'",
	}, {
		in:   "'",
		id:   LEX_ERROR,
		want: "",
	}, {
		in:   "'hello\\'",
		id:   LEX_ERROR,
		want: "hello'",
	}, {
		in:   "'hello",
		id:   LEX_ERROR,
		want: "hello",
	}, {
		in:   "'hello\\",
		id:   LEX_ERROR,
		want: "hello",
	}}

	for _, tcase := range testcases {
		t.Run(tcase.in, func(t *testing.T) {
			id, got := NewStringTokenizer(tcase.in).Scan()
			if tcase.id != id || string(got) != tcase.want {
				t.Errorf("Scan(%q) = (%s, %q), want (%s, %q)", tcase.in, tokenName(id), got, tokenName(tcase.id), tcase.want)
			}
		})
	}
}

func TestSplitStatement(t *testing.T) {
	testcases := []struct {
		in  string
		sql string
		rem string
	}{{
		in:  "select * from table",
		sql: "select * from table",
	}, {
		in:  "select * from table; ",
		sql: "select * from table",
		rem: " ",
	}, {
		in:  "select * from table; select * from table2;",
		sql: "select * from table",
		rem: " select * from table2;",
	}, {
		in:  "select * from /* comment */ table;",
		sql: "select * from /* comment */ table",
	}, {
		in:  "select * from /* comment ; */ table;",
		sql: "select * from /* comment ; */ table",
	}, {
		in:  "select * from table where semi = ';';",
		sql: "select * from table where semi = ';'",
	}, {
		in:  "-- select * from table",
		sql: "-- select * from table",
	}, {
		in:  " ",
		sql: " ",
	}, {
		in:  "",
		sql: "",
	}}

	for _, tcase := range testcases {
		sql, rem, err := SplitStatement(tcase.in)
		if err != nil {
			t.Errorf("EndOfStatementPosition(%s): ERROR: %v", tcase.in, err)
			continue
		}

		if tcase.sql != sql {
			t.Errorf("EndOfStatementPosition(%s) got sql \"%s\" want \"%s\"", tcase.in, sql, tcase.sql)
		}

		if tcase.rem != rem {
			t.Errorf("EndOfStatementPosition(%s) got remainder \"%s\" want \"%s\"", tcase.in, rem, tcase.rem)
		}
	}
}

func TestVersion(t *testing.T) {
	testcases := []struct {
		version string
		in      string
		id      []int
	}{{
		version: "5.7.9",
		in:      "/*!80102 SELECT*/ FROM IN EXISTS",
		id:      []int{FROM, IN, EXISTS, 0},
	}, {
		version: "8.1.1",
		in:      "/*!80102 SELECT*/ FROM IN EXISTS",
		id:      []int{FROM, IN, EXISTS, 0},
	}, {
		version: "8.2.1",
		in:      "/*!80102 SELECT*/ FROM IN EXISTS",
		id:      []int{SELECT, FROM, IN, EXISTS, 0},
	}, {
		version: "8.1.2",
		in:      "/*!80102 SELECT*/ FROM IN EXISTS",
		id:      []int{SELECT, FROM, IN, EXISTS, 0},
	}}

	for _, tcase := range testcases {
		t.Run(tcase.version+"_"+tcase.in, func(t *testing.T) {
			MySQLVersion = tcase.version
			tok := NewStringTokenizer(tcase.in)
			for _, expectedID := range tcase.id {
				id, _ := tok.Scan()
				require.Equal(t, expectedID, id)
			}
		})
	}
}

func TestConvertMySQLVersion(t *testing.T) {
	testcases := []struct {
		version          string
		convertedVersion []int
		error            string
	}{{
		version:          "5.7.9",
		convertedVersion: []int{5, 7, 9},
	}, {
		version:          "0008.08.9",
		convertedVersion: []int{8, 8, 9},
	}, {
		version:          "501.7.9, Vitess - 10.0.1",
		convertedVersion: []int{501, 7, 9},
	}, {
		version:          "8.1 Vitess - 10.0.1",
		convertedVersion: []int{8, 1, 0},
	}, {
		version: "Vitess - 10.0.1",
		error:   "MySQL version not correctly setup - Vitess - 10.0.1.",
	}, {
		version:          "5.7.9.22",
		convertedVersion: []int{5, 7, 9},
	}}

	for _, tcase := range testcases {
		t.Run(tcase.version, func(t *testing.T) {
			output, err := convertMySQLVersion(tcase.version)
			if tcase.error != "" {
				require.EqualError(t, err, tcase.error)
			} else {
				require.NoError(t, err)
				require.Equal(t, tcase.convertedVersion, output)
			}
		})
	}
}

func TestConvertCommentVersion(t *testing.T) {
	testcases := []struct {
		version          string
		convertedVersion []int
		error            string
	}{{
		version:          "50709",
		convertedVersion: []int{5, 7, 9},
	}, {
		version:          "50110",
		convertedVersion: []int{5, 1, 10},
	}, {
		version:          "32312",
		convertedVersion: []int{3, 23, 12},
	}, {
		version:          "40100",
		convertedVersion: []int{4, 1, 0},
	}, {
		version:          "80016",
		convertedVersion: []int{8, 0, 16},
	}, {
		version:          "",
		convertedVersion: []int{0, 0, 0},
	}}

	for _, tcase := range testcases {
		t.Run(tcase.version, func(t *testing.T) {
			output, err := convertCommentVersion(tcase.version)
			if tcase.error != "" {
				require.EqualError(t, err, tcase.error)
			} else {
				require.NoError(t, err)
				require.Equal(t, tcase.convertedVersion, output)
			}
		})
	}
}

func TestExtractMySQLComment(t *testing.T) {
	testcases := []struct {
		comment string
		version string
	}{{
		comment: "/*!50108 SELECT * FROM */",
		version: "50108",
	}, {
		comment: "/*!5018 SELECT * FROM */",
		version: "",
	}, {
		comment: "/*!SELECT * FROM */",
		version: "",
	}}

	for _, tcase := range testcases {
		t.Run(tcase.version, func(t *testing.T) {
			output, _ := ExtractMysqlComment(tcase.comment)
			require.Equal(t, tcase.version, output)
		})
	}
}
<|endoftext|>


<commit_before>package rwvfs

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	pathpkg "path"
	"strconv"
	"strings"
	"syscall"

	"golang.org/x/tools/godoc/vfs"
)

// HTTP creates a new VFS that accesses paths on an HTTP server.
func HTTP(base *url.URL, httpClient *http.Client) FileSystem {
	if httpClient == nil {
		httpClient = http.DefaultClient
	}
	return &httpFS{
		baseURL:    base,
		httpClient: httpClient,
	}
}

type httpFS struct {
	baseURL    *url.URL
	httpClient *http.Client
}

func (c *httpFS) String() string { return "http(" + c.baseURL.String() + ")" }

func (c *httpFS) Lstat(path string) (os.FileInfo, error) {
	fi, err := c.stat(c.httpClient, path)
	if err != nil {
		err = &os.PathError{"lstat", path, err}
	}
	return fi, err
}

func (c *httpFS) Stat(path string) (os.FileInfo, error) {
	fi, err := c.stat(c.httpClient, path)
	if err != nil {
		err = &os.PathError{"stat", path, err}
	}
	return fi, err
}

func (c *httpFS) stat(httpClient *http.Client, path string) (os.FileInfo, error) {
	fi := fileInfo{name: pathpkg.Base(path)}
	req, err := c.newRequest("HEAD", path, nil)
	if err != nil {
		return nil, err
	}
	resp, err := c.send(httpClient, req)
	if resp != nil {
		defer resp.Body.Close()
		// Don't check for errors, so that this VFS can be used
		// against HTTP endpoints other than just those created by
		// HTTPHandler.
		setHTTPResponseFileInfo(resp, &fi)
	}
	if err != nil && !isIgnoredRedirectErr(err) {
		return nil, err
	}
	return fi, nil
}

func setHTTPResponseFileInfo(resp *http.Response, fi *fileInfo) error {
	if lastMod := resp.Header.Get("last-modified"); lastMod != "" {
		var err error
		fi.modTime, err = http.ParseTime(lastMod)
		if err != nil {
			return err
		}
	}
	if resp.StatusCode >= 300 && resp.StatusCode <= 399 {
		fi.symlink = true
	}
	switch resp.Header.Get("content-type") {
	case httpFileContentType: // default, nothing to do
	case httpDirContentType:
		fi.dir = true
	case httpSymlinkContentType:
		fi.symlink = true
	}
	fi.size = resp.ContentLength
	return nil
}

const (
	// MIME types for the HTTP response Content-Type header to
	// indicate which type of resource exists at a path.
	httpFileContentType    = "application/vnd.rwvfs.file"
	httpDirContentType     = "application/vnd.rwvfs.dir"
	httpSymlinkContentType = "application/vnd.rwvfs.symlink"
)

func (c *httpFS) ReadDir(path string) ([]os.FileInfo, error) {
	req, err := c.newRequest("GET", path, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("accept", httpDirContentType)

	resp, err := c.send(nil, req)
	if err != nil {
		return nil, &os.PathError{"readdir", path, err}
	}
	defer resp.Body.Close()

	if contentType := resp.Header.Get("content-type"); contentType != httpDirContentType {
		return nil, &os.PathError{"readdir", path, syscall.ENOTDIR}
	}

	var fis []*fileInfoJSON
	if err := json.NewDecoder(resp.Body).Decode(&fis); err != nil {
		return nil, err
	}
	fis2 := make([]os.FileInfo, len(fis))
	for i, fi := range fis {
		fis2[i] = fi
	}
	return fis2, nil
}

func (c *httpFS) Open(name string) (vfs.ReadSeekCloser, error) {
	req, err := c.newRequest("GET", name, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("accept", httpFileContentType)

	resp, err := c.send(nil, req)
	if err != nil {
		return nil, &os.PathError{"open", name, err}
	}
	return failSeeker{resp.Body}, nil
}

type failSeeker struct{ io.ReadCloser }

func (failSeeker) Seek(offset int64, whence int) (int64, error) {
	// TODO(sqs): is Seek used by any clients of rwvfs? if so,
	// consider buffering the HTTP response so it can actually
	// implement Seek.
	return 0, errors.New("rwvfs.HTTP VFS does not support seeking")
}

func (c *httpFS) Create(path string) (io.WriteCloser, error) {
	return &httpFilePost{c: c, path: path}, nil
}

type httpFilePost struct {
	bytes.Buffer
	c    *httpFS
	path string

	closed bool
}

func (f *httpFilePost) Close() error {
	if f.closed {
		return nil
	}
	f.closed = true
	req, err := f.c.newRequest("PUT", f.path, ioutil.NopCloser(&f.Buffer))
	if err != nil {
		return err
	}
	req.Header.Set("content-type", httpFileContentType)
	req.ContentLength = int64(f.Buffer.Len())
	resp, err := f.c.send(nil, req)
	if err != nil {
		return &os.PathError{"create", f.path, err}
	}
	return resp.Body.Close()
}

func (c *httpFS) Mkdir(name string) error {
	req, err := c.newRequest("PUT", name, nil)
	if err != nil {
		return err
	}
	req.Header.Set("content-type", httpDirContentType)

	resp, err := c.send(nil, req)
	if err != nil {
		return &os.PathError{"mkdir", name, err}
	}
	return resp.Body.Close()
}

func (c *httpFS) Remove(name string) error {
	req, err := c.newRequest("DELETE", name, nil)
	if err != nil {
		return err
	}
	resp, err := c.send(nil, req)
	if err != nil {
		return &os.PathError{"remove", name, err}
	}
	return resp.Body.Close()
}

// newRequest creates a new (unsent) HTTP request.
func (c *httpFS) newRequest(method, path string, body io.Reader) (*http.Request, error) {
	urlPath := pathpkg.Join(c.baseURL.Path, strings.TrimPrefix(path, "/"))
	url := c.baseURL.ResolveReference(&url.URL{Path: urlPath})
	return http.NewRequest(method, url.String(), body)
}

// send issues a request using the provided HTTP client (or
// c.httpClient if nil). If the response has a non-20x (200-299) HTTP
// status code, a non-nil error is returned. Callers are responsible
// for closing the HTTP response body unless a non-nil error is
// returned.
func (c *httpFS) send(httpClient *http.Client, req *http.Request) (*http.Response, error) {
	if httpClient == nil {
		httpClient = c.httpClient
	}
	resp, err := httpClient.Do(req)
	isHTTP20x := resp != nil && (resp.StatusCode >= 200 && resp.StatusCode <= 299)
	if resp != nil && (err != nil || !isHTTP20x) {
		body, _ := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		resp.Body = ioutil.NopCloser(bytes.NewReader(body))
	}
	if err != nil {
		return resp, err
	}
	if resp != nil && !isHTTP20x {
		resp.Body.Close()
		switch resp.StatusCode {
		case http.StatusNotFound:
			err = os.ErrNotExist
		case http.StatusConflict:
			err = os.ErrExist
		default:
			err = fmt.Errorf("http status %d", resp.StatusCode)
		}
	}
	return resp, err
}

// HTTPHandler creates an http.Handler that allows HTTP clients to
// access fs. It should be accessed by clients created using this
// package's HTTP func.
func HTTPHandler(fs FileSystem, logTo io.Writer) http.Handler {
	if logTo == nil {
		logTo = ioutil.Discard
	}
	return &httpFSHandler{fs, log.New(logTo, "rwvfs HTTP: ", log.Flags())}
}

type httpFSHandler struct {
	fs  FileSystem
	log *log.Logger
}

func (h *httpFSHandler) ServeHTTPAndReturnError(w http.ResponseWriter, r *http.Request) error {
	var err error
	switch r.Method {
	case "GET":
		err = h.get(w, r)
	case "HEAD":
		err = h.stat(w, r)
	case "PUT":
		err = h.put(w, r)
	case "DELETE":
		err = h.remove(w, r)
	}
	return err
}

func (h *httpFSHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	err := h.ServeHTTPAndReturnError(w, r)
	var status int
	if os.IsNotExist(err) {
		status = http.StatusNotFound
	} else if os.IsExist(err) {
		status = http.StatusConflict
	} else {
		status = http.StatusInternalServerError
	}
	if err != nil {
		h.log.Printf("rwvfs %s %s: %s", r.Method, r.URL, err)
		http.Error(w, "", status)
	}
}

func (h *httpFSHandler) get(w http.ResponseWriter, r *http.Request) error {
	switch r.Header.Get("accept") {
	case httpFileContentType:
		return h.open(w, r)
	case httpDirContentType:
		return h.readDir(w, r)
	}

	fi, err := h.fs.Stat(r.URL.Path)
	if err != nil {
		return err
	}
	if fi.IsDir() {
		return h.readDir(w, r)
	}
	return h.open(w, r)
}

func (h *httpFSHandler) open(w http.ResponseWriter, r *http.Request) error {
	fi, err := h.fs.Stat(r.URL.Path)
	if err != nil {
		return err
	}

	notMod := false
	if ifModSince, err := http.ParseTime(r.Header.Get("if-modified-since")); err == nil {
		if !fi.ModTime().IsZero() && fi.ModTime().Before(ifModSince) {
			w.WriteHeader(http.StatusNotModified)
			notMod = true
		}
	}

	writeFileInfoHeaders(w, fi, true)

	if notMod {
		return nil
	}

	f, err := h.fs.Open(r.URL.Path)
	if err != nil {
		return err
	}
	defer f.Close()
	_, err = io.Copy(w, f)
	return err
}

func (h *httpFSHandler) readDir(w http.ResponseWriter, r *http.Request) error {
	fis, err := h.fs.ReadDir(r.URL.Path)
	if err != nil {
		return err
	}

	fi, err := h.fs.Stat(r.URL.Path)
	if err != nil {
		return err
	}
	writeFileInfoHeaders(w, fi, false)
	w.WriteHeader(http.StatusOK)

	jsonFIs := make([]*fileInfoJSON, len(fis))
	for i, fi := range fis {
		jsonFIs[i] = &fileInfoJSON{fi}
	}

	return json.NewEncoder(w).Encode(jsonFIs)
}

func (h *httpFSHandler) stat(w http.ResponseWriter, r *http.Request) error {
	fi, err := h.fs.Stat(r.URL.Path)
	if err != nil {
		return err
	}
	writeFileInfoHeaders(w, fi, true)
	w.WriteHeader(http.StatusOK)
	return nil
}

func writeFileInfoHeaders(w http.ResponseWriter, fi os.FileInfo, writeContentLength bool) {
	if writeContentLength {
		w.Header().Set("content-length", strconv.FormatInt(fi.Size(), 10))
	}
	if !fi.ModTime().IsZero() {
		w.Header().Set("last-modified", fi.ModTime().Format(http.TimeFormat))
	}
	if fi.IsDir() {
		w.Header().Set("content-type", httpDirContentType)
	} else if fi.Mode()&os.ModeSymlink > 0 {
		// TODO(sqs): get link dest (requires adding Readlink to VFS
		// interface?
		w.Header().Set("content-type", httpSymlinkContentType)
	} else {
		w.Header().Set("content-type", httpFileContentType)
	}
}

func (h *httpFSHandler) put(w http.ResponseWriter, r *http.Request) error {
	switch r.Header.Get("content-type") {
	case httpFileContentType:
		return h.create(w, r)
	case httpDirContentType:
		return h.mkdir(w, r)
	}
	http.Error(w, "", http.StatusBadRequest)
	return nil
}

func (h *httpFSHandler) create(w http.ResponseWriter, r *http.Request) error {
	if r.ContentLength == -1 {
		http.Error(w, "", http.StatusLengthRequired)
		return nil
	}
	f, err := h.fs.Create(slash(r.URL.Path))
	if err != nil {
		return err
	}
	defer f.Close()
	if _, err := io.Copy(f, r.Body); err != nil {
		return err
	}
	w.WriteHeader(http.StatusCreated)
	return nil
}

func (h *httpFSHandler) mkdir(w http.ResponseWriter, r *http.Request) error {
	return h.fs.Mkdir(slash(r.URL.Path))
}

func (h *httpFSHandler) remove(w http.ResponseWriter, r *http.Request) error {
	return h.fs.Remove(r.URL.Path)
}

// ignoreRedirectsHTTPClient returns an HTTP client identical to c
// (using the same Transport, etc.)  except that when it encounters a
// redirect, it returns errIgnoredRedirect.
func ignoreRedirectsHTTPClient(c *http.Client) *http.Client {
	c2 := *c
	c2.CheckRedirect = func(r *http.Request, via []*http.Request) error { return errIgnoredRedirect }
	return &c2
}

var errIgnoredRedirect = errors.New("not following redirect")

func isIgnoredRedirectErr(err error) bool {
	if err, ok := err.(*url.Error); ok && err.Err == errIgnoredRedirect {
		return true
	}
	return false
}

func asJSON(v interface{}) string {
	b, _ := json.Marshal(v)
	return string(b)
}
<commit_msg>pass along error from f.Close in create<commit_after>package rwvfs

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	pathpkg "path"
	"strconv"
	"strings"
	"syscall"

	"golang.org/x/tools/godoc/vfs"
)

// HTTP creates a new VFS that accesses paths on an HTTP server.
func HTTP(base *url.URL, httpClient *http.Client) FileSystem {
	if httpClient == nil {
		httpClient = http.DefaultClient
	}
	return &httpFS{
		baseURL:    base,
		httpClient: httpClient,
	}
}

type httpFS struct {
	baseURL    *url.URL
	httpClient *http.Client
}

func (c *httpFS) String() string { return "http(" + c.baseURL.String() + ")" }

func (c *httpFS) Lstat(path string) (os.FileInfo, error) {
	fi, err := c.stat(c.httpClient, path)
	if err != nil {
		err = &os.PathError{"lstat", path, err}
	}
	return fi, err
}

func (c *httpFS) Stat(path string) (os.FileInfo, error) {
	fi, err := c.stat(c.httpClient, path)
	if err != nil {
		err = &os.PathError{"stat", path, err}
	}
	return fi, err
}

func (c *httpFS) stat(httpClient *http.Client, path string) (os.FileInfo, error) {
	fi := fileInfo{name: pathpkg.Base(path)}
	req, err := c.newRequest("HEAD", path, nil)
	if err != nil {
		return nil, err
	}
	resp, err := c.send(httpClient, req)
	if resp != nil {
		defer resp.Body.Close()
		// Don't check for errors, so that this VFS can be used
		// against HTTP endpoints other than just those created by
		// HTTPHandler.
		setHTTPResponseFileInfo(resp, &fi)
	}
	if err != nil && !isIgnoredRedirectErr(err) {
		return nil, err
	}
	return fi, nil
}

func setHTTPResponseFileInfo(resp *http.Response, fi *fileInfo) error {
	if lastMod := resp.Header.Get("last-modified"); lastMod != "" {
		var err error
		fi.modTime, err = http.ParseTime(lastMod)
		if err != nil {
			return err
		}
	}
	if resp.StatusCode >= 300 && resp.StatusCode <= 399 {
		fi.symlink = true
	}
	switch resp.Header.Get("content-type") {
	case httpFileContentType: // default, nothing to do
	case httpDirContentType:
		fi.dir = true
	case httpSymlinkContentType:
		fi.symlink = true
	}
	fi.size = resp.ContentLength
	return nil
}

const (
	// MIME types for the HTTP response Content-Type header to
	// indicate which type of resource exists at a path.
	httpFileContentType    = "application/vnd.rwvfs.file"
	httpDirContentType     = "application/vnd.rwvfs.dir"
	httpSymlinkContentType = "application/vnd.rwvfs.symlink"
)

func (c *httpFS) ReadDir(path string) ([]os.FileInfo, error) {
	req, err := c.newRequest("GET", path, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("accept", httpDirContentType)

	resp, err := c.send(nil, req)
	if err != nil {
		return nil, &os.PathError{"readdir", path, err}
	}
	defer resp.Body.Close()

	if contentType := resp.Header.Get("content-type"); contentType != httpDirContentType {
		return nil, &os.PathError{"readdir", path, syscall.ENOTDIR}
	}

	var fis []*fileInfoJSON
	if err := json.NewDecoder(resp.Body).Decode(&fis); err != nil {
		return nil, err
	}
	fis2 := make([]os.FileInfo, len(fis))
	for i, fi := range fis {
		fis2[i] = fi
	}
	return fis2, nil
}

func (c *httpFS) Open(name string) (vfs.ReadSeekCloser, error) {
	req, err := c.newRequest("GET", name, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("accept", httpFileContentType)

	resp, err := c.send(nil, req)
	if err != nil {
		return nil, &os.PathError{"open", name, err}
	}
	return failSeeker{resp.Body}, nil
}

type failSeeker struct{ io.ReadCloser }

func (failSeeker) Seek(offset int64, whence int) (int64, error) {
	// TODO(sqs): is Seek used by any clients of rwvfs? if so,
	// consider buffering the HTTP response so it can actually
	// implement Seek.
	return 0, errors.New("rwvfs.HTTP VFS does not support seeking")
}

func (c *httpFS) Create(path string) (io.WriteCloser, error) {
	return &httpFilePost{c: c, path: path}, nil
}

type httpFilePost struct {
	bytes.Buffer
	c    *httpFS
	path string

	closed bool
}

func (f *httpFilePost) Close() error {
	if f.closed {
		return nil
	}
	f.closed = true
	req, err := f.c.newRequest("PUT", f.path, ioutil.NopCloser(&f.Buffer))
	if err != nil {
		return err
	}
	req.Header.Set("content-type", httpFileContentType)
	req.ContentLength = int64(f.Buffer.Len())
	resp, err := f.c.send(nil, req)
	if err != nil {
		return &os.PathError{"create", f.path, err}
	}
	return resp.Body.Close()
}

func (c *httpFS) Mkdir(name string) error {
	req, err := c.newRequest("PUT", name, nil)
	if err != nil {
		return err
	}
	req.Header.Set("content-type", httpDirContentType)

	resp, err := c.send(nil, req)
	if err != nil {
		return &os.PathError{"mkdir", name, err}
	}
	return resp.Body.Close()
}

func (c *httpFS) Remove(name string) error {
	req, err := c.newRequest("DELETE", name, nil)
	if err != nil {
		return err
	}
	resp, err := c.send(nil, req)
	if err != nil {
		return &os.PathError{"remove", name, err}
	}
	return resp.Body.Close()
}

// newRequest creates a new (unsent) HTTP request.
func (c *httpFS) newRequest(method, path string, body io.Reader) (*http.Request, error) {
	urlPath := pathpkg.Join(c.baseURL.Path, strings.TrimPrefix(path, "/"))
	url := c.baseURL.ResolveReference(&url.URL{Path: urlPath})
	return http.NewRequest(method, url.String(), body)
}

// send issues a request using the provided HTTP client (or
// c.httpClient if nil). If the response has a non-20x (200-299) HTTP
// status code, a non-nil error is returned. Callers are responsible
// for closing the HTTP response body unless a non-nil error is
// returned.
func (c *httpFS) send(httpClient *http.Client, req *http.Request) (*http.Response, error) {
	if httpClient == nil {
		httpClient = c.httpClient
	}
	resp, err := httpClient.Do(req)
	isHTTP20x := resp != nil && (resp.StatusCode >= 200 && resp.StatusCode <= 299)
	if resp != nil && (err != nil || !isHTTP20x) {
		body, _ := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		resp.Body = ioutil.NopCloser(bytes.NewReader(body))
	}
	if err != nil {
		return resp, err
	}
	if resp != nil && !isHTTP20x {
		resp.Body.Close()
		switch resp.StatusCode {
		case http.StatusNotFound:
			err = os.ErrNotExist
		case http.StatusConflict:
			err = os.ErrExist
		default:
			err = fmt.Errorf("http status %d", resp.StatusCode)
		}
	}
	return resp, err
}

// HTTPHandler creates an http.Handler that allows HTTP clients to
// access fs. It should be accessed by clients created using this
// package's HTTP func.
func HTTPHandler(fs FileSystem, logTo io.Writer) http.Handler {
	if logTo == nil {
		logTo = ioutil.Discard
	}
	return &httpFSHandler{fs, log.New(logTo, "rwvfs HTTP: ", log.Flags())}
}

type httpFSHandler struct {
	fs  FileSystem
	log *log.Logger
}

func (h *httpFSHandler) ServeHTTPAndReturnError(w http.ResponseWriter, r *http.Request) error {
	var err error
	switch r.Method {
	case "GET":
		err = h.get(w, r)
	case "HEAD":
		err = h.stat(w, r)
	case "PUT":
		err = h.put(w, r)
	case "DELETE":
		err = h.remove(w, r)
	}
	return err
}

func (h *httpFSHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	err := h.ServeHTTPAndReturnError(w, r)
	var status int
	if os.IsNotExist(err) {
		status = http.StatusNotFound
	} else if os.IsExist(err) {
		status = http.StatusConflict
	} else {
		status = http.StatusInternalServerError
	}
	if err != nil {
		h.log.Printf("rwvfs %s %s: %s", r.Method, r.URL, err)
		http.Error(w, "", status)
	}
}

func (h *httpFSHandler) get(w http.ResponseWriter, r *http.Request) error {
	switch r.Header.Get("accept") {
	case httpFileContentType:
		return h.open(w, r)
	case httpDirContentType:
		return h.readDir(w, r)
	}

	fi, err := h.fs.Stat(r.URL.Path)
	if err != nil {
		return err
	}
	if fi.IsDir() {
		return h.readDir(w, r)
	}
	return h.open(w, r)
}

func (h *httpFSHandler) open(w http.ResponseWriter, r *http.Request) error {
	fi, err := h.fs.Stat(r.URL.Path)
	if err != nil {
		return err
	}

	notMod := false
	if ifModSince, err := http.ParseTime(r.Header.Get("if-modified-since")); err == nil {
		if !fi.ModTime().IsZero() && fi.ModTime().Before(ifModSince) {
			w.WriteHeader(http.StatusNotModified)
			notMod = true
		}
	}

	writeFileInfoHeaders(w, fi, true)

	if notMod {
		return nil
	}

	f, err := h.fs.Open(r.URL.Path)
	if err != nil {
		return err
	}
	defer f.Close()
	_, err = io.Copy(w, f)
	return err
}

func (h *httpFSHandler) readDir(w http.ResponseWriter, r *http.Request) error {
	fis, err := h.fs.ReadDir(r.URL.Path)
	if err != nil {
		return err
	}

	fi, err := h.fs.Stat(r.URL.Path)
	if err != nil {
		return err
	}
	writeFileInfoHeaders(w, fi, false)
	w.WriteHeader(http.StatusOK)

	jsonFIs := make([]*fileInfoJSON, len(fis))
	for i, fi := range fis {
		jsonFIs[i] = &fileInfoJSON{fi}
	}

	return json.NewEncoder(w).Encode(jsonFIs)
}

func (h *httpFSHandler) stat(w http.ResponseWriter, r *http.Request) error {
	fi, err := h.fs.Stat(r.URL.Path)
	if err != nil {
		return err
	}
	writeFileInfoHeaders(w, fi, true)
	w.WriteHeader(http.StatusOK)
	return nil
}

func writeFileInfoHeaders(w http.ResponseWriter, fi os.FileInfo, writeContentLength bool) {
	if writeContentLength {
		w.Header().Set("content-length", strconv.FormatInt(fi.Size(), 10))
	}
	if !fi.ModTime().IsZero() {
		w.Header().Set("last-modified", fi.ModTime().Format(http.TimeFormat))
	}
	if fi.IsDir() {
		w.Header().Set("content-type", httpDirContentType)
	} else if fi.Mode()&os.ModeSymlink > 0 {
		// TODO(sqs): get link dest (requires adding Readlink to VFS
		// interface?
		w.Header().Set("content-type", httpSymlinkContentType)
	} else {
		w.Header().Set("content-type", httpFileContentType)
	}
}

func (h *httpFSHandler) put(w http.ResponseWriter, r *http.Request) error {
	switch r.Header.Get("content-type") {
	case httpFileContentType:
		return h.create(w, r)
	case httpDirContentType:
		return h.mkdir(w, r)
	}
	http.Error(w, "", http.StatusBadRequest)
	return nil
}

func (h *httpFSHandler) create(w http.ResponseWriter, r *http.Request) error {
	if r.ContentLength == -1 {
		http.Error(w, "", http.StatusLengthRequired)
		return nil
	}
	f, err := h.fs.Create(slash(r.URL.Path))
	if err != nil {
		return err
	}
	defer f.Close()
	if _, err := io.Copy(f, r.Body); err != nil {
		return err
	}
	if err := f.Close(); err != nil {
		return err
	}
	w.WriteHeader(http.StatusCreated)
	return nil
}

func (h *httpFSHandler) mkdir(w http.ResponseWriter, r *http.Request) error {
	return h.fs.Mkdir(slash(r.URL.Path))
}

func (h *httpFSHandler) remove(w http.ResponseWriter, r *http.Request) error {
	return h.fs.Remove(r.URL.Path)
}

// ignoreRedirectsHTTPClient returns an HTTP client identical to c
// (using the same Transport, etc.)  except that when it encounters a
// redirect, it returns errIgnoredRedirect.
func ignoreRedirectsHTTPClient(c *http.Client) *http.Client {
	c2 := *c
	c2.CheckRedirect = func(r *http.Request, via []*http.Request) error { return errIgnoredRedirect }
	return &c2
}

var errIgnoredRedirect = errors.New("not following redirect")

func isIgnoredRedirectErr(err error) bool {
	if err, ok := err.(*url.Error); ok && err.Err == errIgnoredRedirect {
		return true
	}
	return false
}

func asJSON(v interface{}) string {
	b, _ := json.Marshal(v)
	return string(b)
}
<|endoftext|>