<commit_before>// Package igc implements an IGC parser.
package igc

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"regexp"
	"strings"
	"time"

	"github.com/twpayne/go-geom"
)

var (
	// ErrInvalidCharacter is returned when an invalid character is encountered.
	ErrInvalidCharacter = errors.New("invalid character")
	// ErrInvalidCharactersBeforeARecord is returned when invalid characters are encountered before the A record.
	ErrInvalidCharactersBeforeARecord = errors.New("invalid characters before A record")
	// ErrInvalidBRecord is returned when an invalid B record is encountered.
	ErrInvalidBRecord = errors.New("invalid B record")
	// ErrInvalidHRecord is returned when an invalid H record is encountered.
	ErrInvalidHRecord = errors.New("invalid H record")
	// ErrInvalidIRecord is returned when an invalid I record is encountered.
	ErrInvalidIRecord = errors.New("invalid I record")
	// ErrEmptyLine is returned when an empty line is encountered.
	ErrEmptyLine = errors.New("empty line")
	// ErrMissingARecord is returned when no A record is found.
	ErrMissingARecord = errors.New("missing A record")
	// ErrOutOfRange is returned when a value is out of range.
	ErrOutOfRange = errors.New("out of range")

	hRegexp = regexp.MustCompile(`H([FP])([A-Z]{3})(.*?):(.*?)\s*\z`)
)

// An Errors is a map of errors encountered at each line.
type Errors map[int]error

// A Header is an IGC header.
type Header struct {
	Source   string
	Key      string
	KeyExtra string
	Value    string
}

// A T represents a parsed IGC file.
type T struct {
	Headers    []Header
	LineString *geom.LineString
}

func (es Errors) Error() string {
	var ss []string
	for lineno, e := range es {
		ss = append(ss, fmt.Sprintf("%d: %s", lineno, e.Error()))
	}
	return strings.Join(ss, "\n")
}

// parseDec parses a decimal value in s[start:stop].
func parseDec(s string, start, stop int) (int, error) {
	result := 0
	for i := start; i < stop; i++ {
		if c := s[i]; '0' <= c && c <= '9' {
			result = 10*result + int(c) - '0'
		} else {
			return 0, ErrInvalidCharacter
		}
	}
	return result, nil
}

// parseDecInRange parsers a decimal value in s[start:stop], and returns an
// error if it is outside the range [min, max).
func parseDecInRange(s string, start, stop, min, max int) (int, error) {
	if result, err := parseDec(s, start, stop); err != nil {
		return result, err
	} else if result < min || max <= result {
		return result, ErrOutOfRange
	} else {
		return result, nil
	}
}

// parser contains the state of a parser.
type parser struct {
	headers           []Header
	coords            []float64
	year, month, day  int
	startAt           time.Time
	lastDate          time.Time
	ladStart, ladStop int
	lodStart, lodStop int
	tdsStart, tdsStop int
	bRecordLen        int
}

// newParser creates a new parser.
func newParser() *parser {
	return &parser{bRecordLen: 35}
}

// parseB parses a B record from line and updates the state of p.
func (p *parser) parseB(line string) error {

	if len(line) != p.bRecordLen {
		return ErrInvalidBRecord
	}

	var err error

	var hour, minute, second, nsec int
	if hour, err = parseDecInRange(line, 1, 3, 0, 24); err != nil {
		return err
	}
	if minute, err = parseDecInRange(line, 3, 5, 0, 60); err != nil {
		return err
	}
	if second, err = parseDecInRange(line, 5, 7, 0, 60); err != nil {
		return err
	}
	if p.tdsStart != 0 {
		decisecond, err := parseDecInRange(line, p.tdsStart, p.tdsStop, 0, 10)
		if err != nil {
			return err
		}
		nsec = decisecond * 1e8
	}
	date := time.Date(p.year, time.Month(p.month), p.day, hour, minute, second, nsec, time.UTC)
	if date.Before(p.lastDate) {
		p.day++
		date = time.Date(p.year, time.Month(p.month), p.day, hour, minute, second, nsec, time.UTC)
	}

	if p.startAt.IsZero() {
		p.startAt = date
	}

	var latDeg, latMilliMin int
	if latDeg, err = parseDecInRange(line, 7, 9, 0, 90); err != nil {
		return err
	}
	// special case: latMilliMin should be in the range [0, 60000) but a number of flight recorders generate latMilliMins of 60000
	// FIXME check what happens in negative (S, W) hemispheres
	if latMilliMin, err = parseDecInRange(line, 9, 14, 0, 60000+1); err != nil {
		return err
	}
	lat := float64(60000*latDeg+latMilliMin) / 60000.
	if p.ladStart != 0 {
		if lad, err := parseDec(line, p.ladStart, p.ladStop); err == nil {
			lat += float64(lad) / 6000000.
		} else {
			return err
		}
	}
	switch c := line[14]; c {
	case 'N':
	case 'S':
		lat = -lat
	default:
		return ErrInvalidCharacter
	}

	var lngDeg, lngMilliMin int
	if lngDeg, err = parseDecInRange(line, 15, 18, 0, 180); err != nil {
		return err
	}
	if lngMilliMin, err = parseDecInRange(line, 18, 23, 0, 60000+1); err != nil {
		return err
	}
	lng := float64(60000*lngDeg+lngMilliMin) / 60000.
	if p.lodStart != 0 {
		if lod, err := parseDec(line, p.lodStart, p.lodStop); err == nil {
			lng += float64(lod) / 6000000.
		} else {
			return err
		}
	}
	switch c := line[23]; c {
	case 'E':
	case 'W':
		lng = -lng
	default:
		return ErrInvalidCharacter
	}

	var pressureAlt, ellipsoidAlt int
	if pressureAlt, err = parseDec(line, 25, 30); err != nil {
		return err
	}
	if ellipsoidAlt, err = parseDec(line, 30, 35); err != nil {
		return err
	}

	p.coords = append(p.coords, lng, lat, float64(ellipsoidAlt), float64(date.UnixNano())/1e9, float64(pressureAlt))
	p.lastDate = date

	return nil

}

// parseB parses an H record from line and updates the state of p.
func (p *parser) parseH(line string) error {
	if m := hRegexp.FindStringSubmatch(line); m != nil {
		p.headers = append(p.headers, Header{
			Source:   m[1],
			Key:      m[2],
			KeyExtra: m[3],
			Value:    m[4],
		})
	}
	switch {
	case strings.HasPrefix(line, "HFDTE"):
		return p.parseHFDTE(line)
	default:
		return nil
	}
}

// parseB parses an HFDTE record from line and updates the state of p.
func (p *parser) parseHFDTE(line string) error {
	var err error
	var day, month, year int
	if len(line) != 11 {
		return ErrInvalidHRecord
	}
	if day, err = parseDecInRange(line, 5, 7, 1, 31+1); err != nil {
		return err
	}
	if month, err = parseDecInRange(line, 7, 9, 1, 12+1); err != nil {
		return err
	}
	if year, err = parseDec(line, 9, 11); err != nil {
		return err
	}
	// FIXME check for invalid dates
	p.day = day
	p.month = month
	if year < 70 {
		p.year = 2000 + year
	} else {
		p.year = 1970 + year
	}
	return nil
}

// parseB parses an I record from line and updates the state of p.
func (p *parser) parseI(line string) error {
	var err error
	var n int
	if len(line) < 3 {
		return ErrInvalidIRecord
	}
	if n, err = parseDec(line, 1, 3); err != nil {
		return err
	}
	if len(line) < 7*n+3 {
		return ErrInvalidIRecord
	}
	for i := 0; i < n; i++ {
		var start, stop int
		if start, err = parseDec(line, 7*i+3, 7*i+5); err != nil {
			return err
		}
		if stop, err = parseDec(line, 7*i+5, 7*i+7); err != nil {
			return err
		}
		if start != p.bRecordLen+1 || stop < start {
			return ErrInvalidIRecord
		}
		p.bRecordLen = stop
		switch line[7*i+7 : 7*i+10] {
		case "LAD":
			p.ladStart, p.ladStop = start-1, stop
		case "LOD":
			p.lodStart, p.lodStop = start-1, stop
		case "TDS":
			p.tdsStart, p.tdsStop = start-1, stop
		}
	}
	return nil
}

// parseLine parses a single record from line and updates the state of p.
func (p *parser) parseLine(line string) error {
	switch line[0] {
	case 'B':
		return p.parseB(line)
	case 'H':
		return p.parseH(line)
	case 'I':
		return p.parseI(line)
	default:
		return nil
	}
}

// doParse reads r, parsers all the records it finds, updating the state of p.
func doParse(r io.Reader) (*parser, Errors) {
	errors := make(Errors)
	p := newParser()
	s := bufio.NewScanner(r)
	foundA := false
	leadingNoise := false
	for lineno := 1; s.Scan(); lineno++ {
		line := s.Text()
		if len(line) == 0 {
			// errors[lineno] = ErrEmptyLine
		} else if foundA {
			if err := p.parseLine(line); err != nil {
				errors[lineno] = err
			}
		} else {
			if c := line[0]; c == 'A' {
				foundA = true
			} else if 'A' <= c && c <= 'Z' {
				// All records that start with an uppercase character must be valid.
				leadingNoise = true
				continue
			} else if i := strings.IndexRune(line, 'A'); i != -1 {
				// Strip any leading noise.
				// The noise must include at least one unprintable character (like XOFF or a fragment of a Unicode BOM).
				for _, c := range line[:i] {
					if !(c == ' ' || ('A' <= c && c <= 'Z')) {
						foundA = true
						leadingNoise = true
						line = line[i:]
						break
					}
				}
			}
		}
	}
	if !foundA {
		errors[1] = ErrMissingARecord
	} else if leadingNoise {
		errors[1] = ErrInvalidCharactersBeforeARecord
	}
	return p, errors
}

// Read reads a igc.T from r, which should contain IGC records.
func Read(r io.Reader) (*T, error) {
	p, errors := doParse(r)
	if len(errors) != 0 {
		return nil, errors
	}
	return &T{
		Headers:    p.headers,
		LineString: geom.NewLineStringFlat(geom.Layout(5), p.coords),
	}, nil
}
<commit_msg>Avoid variable shadowing<commit_after>// Package igc implements an IGC parser.
package igc

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"regexp"
	"strings"
	"time"

	"github.com/twpayne/go-geom"
)

var (
	// ErrInvalidCharacter is returned when an invalid character is encountered.
	ErrInvalidCharacter = errors.New("invalid character")
	// ErrInvalidCharactersBeforeARecord is returned when invalid characters are encountered before the A record.
	ErrInvalidCharactersBeforeARecord = errors.New("invalid characters before A record")
	// ErrInvalidBRecord is returned when an invalid B record is encountered.
	ErrInvalidBRecord = errors.New("invalid B record")
	// ErrInvalidHRecord is returned when an invalid H record is encountered.
	ErrInvalidHRecord = errors.New("invalid H record")
	// ErrInvalidIRecord is returned when an invalid I record is encountered.
	ErrInvalidIRecord = errors.New("invalid I record")
	// ErrEmptyLine is returned when an empty line is encountered.
	ErrEmptyLine = errors.New("empty line")
	// ErrMissingARecord is returned when no A record is found.
	ErrMissingARecord = errors.New("missing A record")
	// ErrOutOfRange is returned when a value is out of range.
	ErrOutOfRange = errors.New("out of range")

	hRegexp = regexp.MustCompile(`H([FP])([A-Z]{3})(.*?):(.*?)\s*\z`)
)

// An Errors is a map of errors encountered at each line.
type Errors map[int]error

// A Header is an IGC header.
type Header struct {
	Source   string
	Key      string
	KeyExtra string
	Value    string
}

// A T represents a parsed IGC file.
type T struct {
	Headers    []Header
	LineString *geom.LineString
}

func (es Errors) Error() string {
	var ss []string
	for lineno, e := range es {
		ss = append(ss, fmt.Sprintf("%d: %s", lineno, e.Error()))
	}
	return strings.Join(ss, "\n")
}

// parseDec parses a decimal value in s[start:stop].
func parseDec(s string, start, stop int) (int, error) {
	result := 0
	for i := start; i < stop; i++ {
		if c := s[i]; '0' <= c && c <= '9' {
			result = 10*result + int(c) - '0'
		} else {
			return 0, ErrInvalidCharacter
		}
	}
	return result, nil
}

// parseDecInRange parsers a decimal value in s[start:stop], and returns an
// error if it is outside the range [min, max).
func parseDecInRange(s string, start, stop, min, max int) (int, error) {
	if result, err := parseDec(s, start, stop); err != nil {
		return result, err
	} else if result < min || max <= result {
		return result, ErrOutOfRange
	} else {
		return result, nil
	}
}

// parser contains the state of a parser.
type parser struct {
	headers           []Header
	coords            []float64
	year, month, day  int
	startAt           time.Time
	lastDate          time.Time
	ladStart, ladStop int
	lodStart, lodStop int
	tdsStart, tdsStop int
	bRecordLen        int
}

// newParser creates a new parser.
func newParser() *parser {
	return &parser{bRecordLen: 35}
}

// parseB parses a B record from line and updates the state of p.
func (p *parser) parseB(line string) error {

	if len(line) != p.bRecordLen {
		return ErrInvalidBRecord
	}

	var err error

	var hour, minute, second, nsec int
	if hour, err = parseDecInRange(line, 1, 3, 0, 24); err != nil {
		return err
	}
	if minute, err = parseDecInRange(line, 3, 5, 0, 60); err != nil {
		return err
	}
	if second, err = parseDecInRange(line, 5, 7, 0, 60); err != nil {
		return err
	}
	if p.tdsStart != 0 {
		var decisecond int
		decisecond, err = parseDecInRange(line, p.tdsStart, p.tdsStop, 0, 10)
		if err != nil {
			return err
		}
		nsec = decisecond * 1e8
	}
	date := time.Date(p.year, time.Month(p.month), p.day, hour, minute, second, nsec, time.UTC)
	if date.Before(p.lastDate) {
		p.day++
		date = time.Date(p.year, time.Month(p.month), p.day, hour, minute, second, nsec, time.UTC)
	}

	if p.startAt.IsZero() {
		p.startAt = date
	}

	var latDeg, latMilliMin int
	if latDeg, err = parseDecInRange(line, 7, 9, 0, 90); err != nil {
		return err
	}
	// special case: latMilliMin should be in the range [0, 60000) but a number of flight recorders generate latMilliMins of 60000
	// FIXME check what happens in negative (S, W) hemispheres
	if latMilliMin, err = parseDecInRange(line, 9, 14, 0, 60000+1); err != nil {
		return err
	}
	lat := float64(60000*latDeg+latMilliMin) / 60000.
	if p.ladStart != 0 {
		var lad int
		if lad, err = parseDec(line, p.ladStart, p.ladStop); err == nil {
			lat += float64(lad) / 6000000.
		} else {
			return err
		}
	}
	switch c := line[14]; c {
	case 'N':
	case 'S':
		lat = -lat
	default:
		return ErrInvalidCharacter
	}

	var lngDeg, lngMilliMin int
	if lngDeg, err = parseDecInRange(line, 15, 18, 0, 180); err != nil {
		return err
	}
	if lngMilliMin, err = parseDecInRange(line, 18, 23, 0, 60000+1); err != nil {
		return err
	}
	lng := float64(60000*lngDeg+lngMilliMin) / 60000.
	if p.lodStart != 0 {
		var lod int
		if lod, err = parseDec(line, p.lodStart, p.lodStop); err == nil {
			lng += float64(lod) / 6000000.
		} else {
			return err
		}
	}
	switch c := line[23]; c {
	case 'E':
	case 'W':
		lng = -lng
	default:
		return ErrInvalidCharacter
	}

	var pressureAlt, ellipsoidAlt int
	if pressureAlt, err = parseDec(line, 25, 30); err != nil {
		return err
	}
	if ellipsoidAlt, err = parseDec(line, 30, 35); err != nil {
		return err
	}

	p.coords = append(p.coords, lng, lat, float64(ellipsoidAlt), float64(date.UnixNano())/1e9, float64(pressureAlt))
	p.lastDate = date

	return nil

}

// parseB parses an H record from line and updates the state of p.
func (p *parser) parseH(line string) error {
	if m := hRegexp.FindStringSubmatch(line); m != nil {
		p.headers = append(p.headers, Header{
			Source:   m[1],
			Key:      m[2],
			KeyExtra: m[3],
			Value:    m[4],
		})
	}
	switch {
	case strings.HasPrefix(line, "HFDTE"):
		return p.parseHFDTE(line)
	default:
		return nil
	}
}

// parseB parses an HFDTE record from line and updates the state of p.
func (p *parser) parseHFDTE(line string) error {
	var err error
	var day, month, year int
	if len(line) != 11 {
		return ErrInvalidHRecord
	}
	if day, err = parseDecInRange(line, 5, 7, 1, 31+1); err != nil {
		return err
	}
	if month, err = parseDecInRange(line, 7, 9, 1, 12+1); err != nil {
		return err
	}
	if year, err = parseDec(line, 9, 11); err != nil {
		return err
	}
	// FIXME check for invalid dates
	p.day = day
	p.month = month
	if year < 70 {
		p.year = 2000 + year
	} else {
		p.year = 1970 + year
	}
	return nil
}

// parseB parses an I record from line and updates the state of p.
func (p *parser) parseI(line string) error {
	var err error
	var n int
	if len(line) < 3 {
		return ErrInvalidIRecord
	}
	if n, err = parseDec(line, 1, 3); err != nil {
		return err
	}
	if len(line) < 7*n+3 {
		return ErrInvalidIRecord
	}
	for i := 0; i < n; i++ {
		var start, stop int
		if start, err = parseDec(line, 7*i+3, 7*i+5); err != nil {
			return err
		}
		if stop, err = parseDec(line, 7*i+5, 7*i+7); err != nil {
			return err
		}
		if start != p.bRecordLen+1 || stop < start {
			return ErrInvalidIRecord
		}
		p.bRecordLen = stop
		switch line[7*i+7 : 7*i+10] {
		case "LAD":
			p.ladStart, p.ladStop = start-1, stop
		case "LOD":
			p.lodStart, p.lodStop = start-1, stop
		case "TDS":
			p.tdsStart, p.tdsStop = start-1, stop
		}
	}
	return nil
}

// parseLine parses a single record from line and updates the state of p.
func (p *parser) parseLine(line string) error {
	switch line[0] {
	case 'B':
		return p.parseB(line)
	case 'H':
		return p.parseH(line)
	case 'I':
		return p.parseI(line)
	default:
		return nil
	}
}

// doParse reads r, parsers all the records it finds, updating the state of p.
func doParse(r io.Reader) (*parser, Errors) {
	errors := make(Errors)
	p := newParser()
	s := bufio.NewScanner(r)
	foundA := false
	leadingNoise := false
	for lineno := 1; s.Scan(); lineno++ {
		line := s.Text()
		if len(line) == 0 {
			// errors[lineno] = ErrEmptyLine
		} else if foundA {
			if err := p.parseLine(line); err != nil {
				errors[lineno] = err
			}
		} else {
			if c := line[0]; c == 'A' {
				foundA = true
			} else if 'A' <= c && c <= 'Z' {
				// All records that start with an uppercase character must be valid.
				leadingNoise = true
				continue
			} else if i := strings.IndexRune(line, 'A'); i != -1 {
				// Strip any leading noise.
				// The noise must include at least one unprintable character (like XOFF or a fragment of a Unicode BOM).
				for _, c := range line[:i] {
					if !(c == ' ' || ('A' <= c && c <= 'Z')) {
						foundA = true
						leadingNoise = true
						line = line[i:]
						break
					}
				}
			}
		}
	}
	if !foundA {
		errors[1] = ErrMissingARecord
	} else if leadingNoise {
		errors[1] = ErrInvalidCharactersBeforeARecord
	}
	return p, errors
}

// Read reads a igc.T from r, which should contain IGC records.
func Read(r io.Reader) (*T, error) {
	p, errors := doParse(r)
	if len(errors) != 0 {
		return nil, errors
	}
	return &T{
		Headers:    p.headers,
		LineString: geom.NewLineStringFlat(geom.Layout(5), p.coords),
	}, nil
}
<|endoftext|>


<commit_before>package environs_test

import (
	"fmt"
	"io/ioutil"
	. "launchpad.net/gocheck"
	"launchpad.net/juju/go/environs"
	"launchpad.net/juju/go/environs/dummy"
	"launchpad.net/juju/go/version"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

type ToolsSuite struct{
	env environs.Environ
}

func (t *ToolsSuite) SetUpTest(c *C) {
	env, err := environs.NewEnviron("dummy", map[string]interface{}{
		"type": "dummy",
		"zookeeper": false,
	})
	c.Assert(err, IsNil)
	t.env = env
}

func (t *ToolsSuite) TearDownTest(c *C) {
	dummy.Reset()
}

var envs *environs.Environs

var currentToolsPath = toolsPath(version.Current.String(), environs.CurrentSeries, environs.CurrentArch)

func mkVersion(vers string) version.Version {
	v, err := version.Parse(vers)
	if err != nil {
		panic(err)
	}
	return v
}

func toolsPathForVersion(v version.Version, series, arch string) string {
	return fmt.Sprintf("tools/juju-%v-%s-%s.tgz", v, series, arch)
}

func toolsPath(vers, os, arch string) string {
	return toolsPathForVersion(mkVersion(vers), os, arch)
}

var _ = Suite(&ToolsSuite{})

func (t *ToolsSuite) TestPutTools(c *C) {
	err := environs.PutTools(t.env.Storage())
	c.Assert(err, IsNil)

	for i, getTools := range []func(environs.Environ, string) error{
		environs.GetTools,
		getToolsWithTar,
	} {
		c.Logf("test %d", i)
		// Unarchive the tool executables into a temp directory.
		dir := c.MkDir()
		err = getTools(t.env, dir)
		c.Assert(err, IsNil)

		// Verify that each tool executes and produces some
		// characteristic output.
		for _, tool := range []string{"juju", "jujud"} {
			out, err := exec.Command(filepath.Join(dir, tool), "arble").CombinedOutput()
			if err != nil {
				c.Assert(err, FitsTypeOf, (*exec.ExitError)(nil))
			}
			c.Check(string(out), Matches, fmt.Sprintf(`usage: %s (.|\n)*`, tool))
		}
	}
}

// getToolsWithTar is the same as GetTools but uses tar
// itself so we're not just testing the Go tar package against
// itself.
func getToolsWithTar(env environs.Environ, dir string) error {
	// TODO search the store for the right tools.
	r, err := env.Storage().Get(currentToolsPath)
	if err != nil {
		return err
	}
	defer r.Close()

	// unarchive using actual tar command so we're
	// not just verifying the Go tar package against itself.
	cmd := exec.Command("tar", "xz")
	cmd.Dir = dir
	cmd.Stdin = r
	out, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("tar extract failed: %s", out)
	}
	return nil
}

// Test that the upload procedure fails correctly
// when the build process fails (because of a bad Go source
// file in this case).
func (t *ToolsSuite) TestUploadBadBuild(c *C) {
	gopath := c.MkDir()
	pkgdir := filepath.Join(gopath, "src", "launchpad.net", "juju", "go", "cmd", "broken")
	err := os.MkdirAll(pkgdir, 0777)
	c.Assert(err, IsNil)

	err = ioutil.WriteFile(filepath.Join(pkgdir, "broken.go"), []byte("nope"), 0666)
	c.Assert(err, IsNil)

	defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
	os.Setenv("GOPATH", gopath)

	err = environs.PutTools(t.env.Storage())
	c.Assert(err, ErrorMatches, `build failed: exit status 1; can't load package:(.|\n)*`)
}

type toolsSpec struct {
	version string
	os      string
	arch    string
}

var findToolsTests = []struct {
	version    version.Version
	contents []string
	expect   string
	err      string
}{{
	version: version.Current,
	contents: []string{currentToolsPath},
	expect: currentToolsPath,
}, {
	version: mkVersion("1.0.0"),
	contents: []string{
		toolsPath("0.0.9", environs.CurrentSeries, environs.CurrentArch),
	},
	err: "no compatible tools found",
},{
	version: mkVersion("1.0.0"),
	contents: []string{
		toolsPath("2.0.9", environs.CurrentSeries, environs.CurrentArch),
	},
	err: "no compatible tools found",
}, {
	version: mkVersion("1.99.99"),
	contents: []string{
		toolsPath("1.0.0", environs.CurrentSeries, environs.CurrentArch),
	},
	expect: toolsPath("1.0.0", environs.CurrentSeries, environs.CurrentArch),
}, {
	version: mkVersion("1.0.0"),
	contents: []string{
		toolsPath("1.0.9", environs.CurrentSeries, environs.CurrentArch),
		toolsPath("1.0.10", environs.CurrentSeries, environs.CurrentArch),
		toolsPath("1.0.11", environs.CurrentSeries, environs.CurrentArch),
	},
	expect: toolsPath("1.0.11", environs.CurrentSeries, environs.CurrentArch),
}, {
	version: mkVersion("1.0.0"),
	contents: []string{
		toolsPath("1.9.11", environs.CurrentSeries, environs.CurrentArch),
		toolsPath("1.10.10", environs.CurrentSeries, environs.CurrentArch),
		toolsPath("1.11.9", environs.CurrentSeries, environs.CurrentArch),
	},
	expect: toolsPath("1.11.9", environs.CurrentSeries, environs.CurrentArch),
}, {
	version: mkVersion("1.0.0"),
	contents: []string{
		toolsPath("1.9.9", "foo", environs.CurrentArch),
		toolsPath("1.9.9", environs.CurrentSeries, "foo"),
		toolsPath("1.0.0", environs.CurrentSeries, environs.CurrentArch),
	},
	expect: toolsPath("1.0.0", environs.CurrentSeries, environs.CurrentArch),
},
}

func (t *ToolsSuite) TestFindTools(c *C) {
	originalVersion := version.Current
	defer func() {
		version.Current = originalVersion
	}()

	for i, tt := range findToolsTests {
		c.Logf("test %d", i)
		version.Current = tt.version
		for _, name := range tt.contents {
			err := t.env.Storage().Put(name, strings.NewReader(name), int64(len(name)))
			c.Assert(err, IsNil)
		}
		url, err := environs.FindTools(t.env)
		if tt.err != "" {
			c.Assert(err, ErrorMatches, tt.err)
		} else {
			c.Assert(err, IsNil)
			resp, err := http.Get(url)
			c.Assert(err, IsNil)
			data, err := ioutil.ReadAll(resp.Body)
			c.Assert(err, IsNil)
			c.Assert(string(data), Equals, tt.expect, Commentf("url %s", url))
		}
		t.env.Destroy(nil)
	}
}
<commit_msg>environs: test public storage heuristics<commit_after>package environs_test

import (
	"fmt"
	"io/ioutil"
	. "launchpad.net/gocheck"
	"launchpad.net/juju/go/environs"
	"launchpad.net/juju/go/environs/dummy"
	"launchpad.net/juju/go/version"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

type ToolsSuite struct{
	env environs.Environ
}

func (t *ToolsSuite) SetUpTest(c *C) {
	env, err := environs.NewEnviron(map[string]interface{}{
		"name": "test",
		"type": "dummy",
		"zookeeper": false,
	})
	c.Assert(err, IsNil)
	t.env = env
}

func (t *ToolsSuite) TearDownTest(c *C) {
	dummy.Reset()
}

var envs *environs.Environs

var currentToolsPath = toolsPath(version.Current.String(), environs.CurrentSeries, environs.CurrentArch)

func mkVersion(vers string) version.Version {
	v, err := version.Parse(vers)
	if err != nil {
		panic(err)
	}
	return v
}

func toolsPathForVersion(v version.Version, series, arch string) string {
	return fmt.Sprintf("tools/juju-%v-%s-%s.tgz", v, series, arch)
}

func toolsPath(vers, os, arch string) string {
	return toolsPathForVersion(mkVersion(vers), os, arch)
}

var _ = Suite(&ToolsSuite{})

func (t *ToolsSuite) TestPutTools(c *C) {
	err := environs.PutTools(t.env.Storage())
	c.Assert(err, IsNil)

	for i, getTools := range []func(environs.Environ, string) error{
		environs.GetTools,
		getToolsWithTar,
	} {
		c.Logf("test %d", i)
		// Unarchive the tool executables into a temp directory.
		dir := c.MkDir()
		err = getTools(t.env, dir)
		c.Assert(err, IsNil)

		// Verify that each tool executes and produces some
		// characteristic output.
		for _, tool := range []string{"juju", "jujud"} {
			out, err := exec.Command(filepath.Join(dir, tool), "arble").CombinedOutput()
			if err != nil {
				c.Assert(err, FitsTypeOf, (*exec.ExitError)(nil))
			}
			c.Check(string(out), Matches, fmt.Sprintf(`usage: %s (.|\n)*`, tool))
		}
	}
}

// getToolsWithTar is the same as GetTools but uses tar
// itself so we're not just testing the Go tar package against
// itself.
func getToolsWithTar(env environs.Environ, dir string) error {
	// TODO search the store for the right tools.
	r, err := env.Storage().Get(currentToolsPath)
	if err != nil {
		return err
	}
	defer r.Close()

	// unarchive using actual tar command so we're
	// not just verifying the Go tar package against itself.
	cmd := exec.Command("tar", "xz")
	cmd.Dir = dir
	cmd.Stdin = r
	out, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("tar extract failed: %s", out)
	}
	return nil
}

// Test that the upload procedure fails correctly
// when the build process fails (because of a bad Go source
// file in this case).
func (t *ToolsSuite) TestUploadBadBuild(c *C) {
	gopath := c.MkDir()
	pkgdir := filepath.Join(gopath, "src", "launchpad.net", "juju", "go", "cmd", "broken")
	err := os.MkdirAll(pkgdir, 0777)
	c.Assert(err, IsNil)

	err = ioutil.WriteFile(filepath.Join(pkgdir, "broken.go"), []byte("nope"), 0666)
	c.Assert(err, IsNil)

	defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
	os.Setenv("GOPATH", gopath)

	err = environs.PutTools(t.env.Storage())
	c.Assert(err, ErrorMatches, `build failed: exit status 1; can't load package:(.|\n)*`)
}

type toolsSpec struct {
	version string
	os      string
	arch    string
}

var findToolsTests = []struct {
	version    version.Version
	contents []string
	publicContents []string
	expect   string
	expectPublic bool
	err      string
}{{
	// current version should be satisfied by current tools path.
	version: version.Current,
	contents: []string{currentToolsPath},
	expect: currentToolsPath,
}, {
	// major versions don't match.
	version: mkVersion("1.0.0"),
	contents: []string{
		toolsPath("0.0.9", environs.CurrentSeries, environs.CurrentArch),
	},
	err: "no compatible tools found",
}, {
	// major versions don't match.
	version: mkVersion("1.0.0"),
	contents: []string{
		toolsPath("2.0.9", environs.CurrentSeries, environs.CurrentArch),
	},
	err: "no compatible tools found",
}, {
	// fall back to public storage when nothing found in private.
	version: mkVersion("1.0.0"),
	contents: []string{
		toolsPath("0.0.9", environs.CurrentSeries, environs.CurrentArch),
	},
	publicContents: []string{
		toolsPath("1.0.0", environs.CurrentSeries, environs.CurrentArch),
	},
	expect: "public-" + toolsPath("1.0.0", environs.CurrentSeries, environs.CurrentArch),
}, {
	// always use private storage in preference to public storage.
	version: mkVersion("1.0.0"),
	contents: []string{
		toolsPath("1.0.2", environs.CurrentSeries, environs.CurrentArch),
	},
	publicContents: []string{
		toolsPath("1.0.9", environs.CurrentSeries, environs.CurrentArch),
	},
	expect: toolsPath("1.0.2", environs.CurrentSeries, environs.CurrentArch),
}, {
	// we'll use an earlier version if the major version number matches.
	version: mkVersion("1.99.99"),
	contents: []string{
		toolsPath("1.0.0", environs.CurrentSeries, environs.CurrentArch),
	},
	expect: toolsPath("1.0.0", environs.CurrentSeries, environs.CurrentArch),
}, {
	// check that version comparing is numeric, not alphabetical.
	version: mkVersion("1.0.0"),
	contents: []string{
		toolsPath("1.0.9", environs.CurrentSeries, environs.CurrentArch),
		toolsPath("1.0.10", environs.CurrentSeries, environs.CurrentArch),
		toolsPath("1.0.11", environs.CurrentSeries, environs.CurrentArch),
	},
	expect: toolsPath("1.0.11", environs.CurrentSeries, environs.CurrentArch),
}, {
	// minor version wins over patch version.
	version: mkVersion("1.0.0"),
	contents: []string{
		toolsPath("1.9.11", environs.CurrentSeries, environs.CurrentArch),
		toolsPath("1.10.10", environs.CurrentSeries, environs.CurrentArch),
		toolsPath("1.11.9", environs.CurrentSeries, environs.CurrentArch),
	},
	expect: toolsPath("1.11.9", environs.CurrentSeries, environs.CurrentArch),
}, {
	// mismatching series or architecture is ignored.
	version: mkVersion("1.0.0"),
	contents: []string{
		toolsPath("1.9.9", "foo", environs.CurrentArch),
		toolsPath("1.9.9", environs.CurrentSeries, "foo"),
		toolsPath("1.0.0", environs.CurrentSeries, environs.CurrentArch),
	},
	expect: toolsPath("1.0.0", environs.CurrentSeries, environs.CurrentArch),
},
}

func (t *ToolsSuite) TestFindTools(c *C) {
	originalVersion := version.Current
	defer func() {
		version.Current = originalVersion
	}()

	for i, tt := range findToolsTests {
		c.Logf("test %d", i)
		version.Current = tt.version
		for _, name := range tt.contents {
			err := t.env.Storage().Put(name, strings.NewReader(name), int64(len(name)))
			c.Assert(err, IsNil)
		}
		// The contents of all files in the public storage is prefixed with "public-" so
		// that we can easily tell if we've got the right thing.
		for _, name := range tt.publicContents {
			data := "public-" + name
			err := t.env.PublicStorage().(environs.Storage).Put(name, strings.NewReader(data), int64(len(data)))
			c.Assert(err, IsNil)
		}
		url, err := environs.FindTools(t.env)
		if tt.err != "" {
			c.Assert(err, ErrorMatches, tt.err)
		} else {
			c.Assert(err, IsNil)
			resp, err := http.Get(url)
			c.Assert(err, IsNil)
			data, err := ioutil.ReadAll(resp.Body)
			c.Assert(err, IsNil)
			c.Assert(string(data), Equals, tt.expect, Commentf("url %s", url))
		}
		t.env.Destroy(nil)
	}
}
<|endoftext|>


<commit_before>/*
This is the local node discovery protocol. In principle we might be better
served by something more standardized, such as mDNS / DNS-SD. In practice, this
was much easier and quicker to get up and running.

The mode of operation is to periodically (currently once every 30 seconds)
broadcast an Announcement packet to UDP port 21025. The packet has the
following format:

     0                   1                   2                   3
     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
    |                   Magic Number (0x20121025)                   |
    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
    |          Port Number          |           Reserved            |
    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
    |                        Length of NodeID                       |
    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
    /                                                               /
    \                   NodeID (variable length)                    \
    /                                                               /
    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
    |                          Length of IP                         |
    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
    /                                                               /
    \                     IP (variable length)                      \
    /                                                               /
    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

This is the XDR encoding of:

struct Announcement {
	unsigned int Magic;
	unsigned short Port;
	string NodeID<>;
}

(Hence NodeID is padded to a multiple of 32 bits)

The sending node's address is not encoded in local announcement -- the Length
of IP field is set to zero and the address is taken to be the source address of
the announcement. In announcement packets sent by a discovery server in
response to a query, the IP is present and the length is either 4 (IPv4) or 16
(IPv6).

Every time such a packet is received, a local table that maps NodeID to Address
is updated. When the local node wants to connect to another node with the
address specification 'dynamic', this table is consulted.

For external discovery, an identical packet is sent every 30 minutes to the
external discovery server. The server keeps information for up to 60 minutes.
To query the server, and UDP packet with the format below is sent.

     0                   1                   2                   3
     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
    |                   Magic Number (0x19760309)                   |
    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
    |                        Length of NodeID                       |
    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
    /                                                               /
    \                   NodeID (variable length)                    \
    /                                                               /
    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

This is the XDR encoding of:

struct Announcement {
	unsigned int Magic;
	string NodeID<>;
}

(Hence NodeID is padded to a multiple of 32 bits)

It is answered with an announcement packet for the queried node ID if the
information is available. There is no answer for queries about unknown nodes. A
reasonable timeout is recommended instead. (This, combined with server side
rate limits for packets per source IP and queries per node ID, prevents the
server from being used as an amplifier in a DDoS attack.)
*/
package discover

import (
	"fmt"
	"log"
	"net"
	"sync"
	"time"

	"github.com/calmh/syncthing/buffers"
)

const (
	AnnouncementPort  = 21025
	AnnouncementMagic = 0x20121025
	QueryMagic        = 0x19760309
)

type Discoverer struct {
	MyID             string
	ListenPort       int
	BroadcastIntv    time.Duration
	ExtListenPort    int
	ExtBroadcastIntv time.Duration

	conn         *net.UDPConn
	registry     map[string]string
	registryLock sync.RWMutex
	extServer    string
}

// We tolerate a certain amount of errors because we might be running on
// laptops that sleep and wake, have intermittent network connectivity, etc.
// When we hit this many errors in succession, we stop.
const maxErrors = 30

func NewDiscoverer(id string, port int, extPort int, extServer string) (*Discoverer, error) {
	local4 := &net.UDPAddr{IP: net.IP{0, 0, 0, 0}, Port: AnnouncementPort}
	conn, err := net.ListenUDP("udp4", local4)
	if err != nil {
		return nil, err
	}

	disc := &Discoverer{
		MyID:             id,
		ListenPort:       port,
		BroadcastIntv:    30 * time.Second,
		ExtListenPort:    extPort,
		ExtBroadcastIntv: 1800 * time.Second,

		conn:      conn,
		registry:  make(map[string]string),
		extServer: extServer,
	}

	go disc.recvAnnouncements()

	if disc.ListenPort > 0 {
		disc.sendAnnouncements()
	}
	if len(disc.extServer) > 0 && disc.ExtListenPort > 0 {
		disc.sendExtAnnouncements()
	}

	return disc, nil
}

func (d *Discoverer) sendAnnouncements() {
	remote4 := &net.UDPAddr{IP: net.IP{255, 255, 255, 255}, Port: AnnouncementPort}

	buf := EncodePacket(Packet{AnnouncementMagic, uint16(d.ListenPort), d.MyID, nil})
	go d.writeAnnouncements(buf, remote4, d.BroadcastIntv)
}

func (d *Discoverer) sendExtAnnouncements() {
	extIP, err := net.ResolveUDPAddr("udp", d.extServer+":22025")
	if err != nil {
		log.Printf("discover/external: %v; no external announcements", err)
		return
	}

	buf := EncodePacket(Packet{AnnouncementMagic, uint16(d.ExtListenPort), d.MyID, nil})
	go d.writeAnnouncements(buf, extIP, d.ExtBroadcastIntv)
}

func (d *Discoverer) writeAnnouncements(buf []byte, remote *net.UDPAddr, intv time.Duration) {
	var errCounter = 0
	var err error
	for errCounter < maxErrors {
		_, _, err = d.conn.WriteMsgUDP(buf, nil, remote)
		if err != nil {
			log.Println("discover/write: warning:", err)
			errCounter++
		} else {
			errCounter = 0
		}
		time.Sleep(intv)
	}
	log.Println("discover/write: %v: stopping due to too many errors:", remote, err)
}

func (d *Discoverer) recvAnnouncements() {
	var buf = make([]byte, 1024)
	var errCounter = 0
	var err error
	for errCounter < maxErrors {
		n, addr, err := d.conn.ReadFromUDP(buf)
		if err != nil {
			errCounter++
			time.Sleep(time.Second)
			continue
		}

		pkt, err := DecodePacket(buf[:n])
		if err != nil || pkt.Magic != AnnouncementMagic {
			errCounter++
			time.Sleep(time.Second)
			continue
		}

		errCounter = 0

		if pkt.ID != d.MyID {
			nodeAddr := fmt.Sprintf("%s:%d", addr.IP.String(), pkt.Port)
			d.registryLock.Lock()
			if d.registry[pkt.ID] != nodeAddr {
				d.registry[pkt.ID] = nodeAddr
			}
			d.registryLock.Unlock()
		}
	}
	log.Println("discover/read: stopping due to too many errors:", err)
}

func (d *Discoverer) externalLookup(node string) (string, bool) {
	extIP, err := net.ResolveUDPAddr("udp", d.extServer+":22025")
	if err != nil {
		log.Printf("discover/external: %v; no external lookup", err)
		return "", false
	}

	conn, err := net.DialUDP("udp", nil, extIP)
	if err != nil {
		log.Printf("discover/external: %v; no external lookup", err)
		return "", false
	}
	defer conn.Close()

	err = conn.SetDeadline(time.Now().Add(5 * time.Second))
	if err != nil {
		log.Printf("discover/external: %v; no external lookup", err)
		return "", false
	}

	_, err = conn.Write(EncodePacket(Packet{QueryMagic, 0, node, nil}))
	if err != nil {
		log.Printf("discover/external: %v; no external lookup", err)
		return "", false
	}

	var buf = buffers.Get(1024)
	defer buffers.Put(buf)

	n, err := conn.Read(buf)
	if err != nil {
		log.Printf("discover/external/read: %v; no external lookup", err)
		return "", false
	}

	pkt, err := DecodePacket(buf[:n])
	if err != nil {
		log.Printf("discover/external/read: %v; no external lookup", err)
		return "", false
	}

	if pkt.Magic != AnnouncementMagic {
		log.Printf("discover/external/read: bad magic; no external lookup", err)
		return "", false
	}

	return fmt.Sprintf("%s:%d", ipStr(pkt.IP), pkt.Port), true
}

func (d *Discoverer) Lookup(node string) (string, bool) {
	d.registryLock.Lock()
	addr, ok := d.registry[node]
	d.registryLock.Unlock()

	if ok {
		return addr, true
	} else if len(d.extServer) != 0 {
		// We might want to cache this, but not permanently so it needs some intelligence
		return d.externalLookup(node)
	}
	return "", false
}
<commit_msg>Don't complain about expected timeout<commit_after>/*
This is the local node discovery protocol. In principle we might be better
served by something more standardized, such as mDNS / DNS-SD. In practice, this
was much easier and quicker to get up and running.

The mode of operation is to periodically (currently once every 30 seconds)
broadcast an Announcement packet to UDP port 21025. The packet has the
following format:

     0                   1                   2                   3
     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
    |                   Magic Number (0x20121025)                   |
    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
    |          Port Number          |           Reserved            |
    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
    |                        Length of NodeID                       |
    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
    /                                                               /
    \                   NodeID (variable length)                    \
    /                                                               /
    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
    |                          Length of IP                         |
    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
    /                                                               /
    \                     IP (variable length)                      \
    /                                                               /
    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

This is the XDR encoding of:

struct Announcement {
	unsigned int Magic;
	unsigned short Port;
	string NodeID<>;
}

(Hence NodeID is padded to a multiple of 32 bits)

The sending node's address is not encoded in local announcement -- the Length
of IP field is set to zero and the address is taken to be the source address of
the announcement. In announcement packets sent by a discovery server in
response to a query, the IP is present and the length is either 4 (IPv4) or 16
(IPv6).

Every time such a packet is received, a local table that maps NodeID to Address
is updated. When the local node wants to connect to another node with the
address specification 'dynamic', this table is consulted.

For external discovery, an identical packet is sent every 30 minutes to the
external discovery server. The server keeps information for up to 60 minutes.
To query the server, and UDP packet with the format below is sent.

     0                   1                   2                   3
     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
    |                   Magic Number (0x19760309)                   |
    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
    |                        Length of NodeID                       |
    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
    /                                                               /
    \                   NodeID (variable length)                    \
    /                                                               /
    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

This is the XDR encoding of:

struct Announcement {
	unsigned int Magic;
	string NodeID<>;
}

(Hence NodeID is padded to a multiple of 32 bits)

It is answered with an announcement packet for the queried node ID if the
information is available. There is no answer for queries about unknown nodes. A
reasonable timeout is recommended instead. (This, combined with server side
rate limits for packets per source IP and queries per node ID, prevents the
server from being used as an amplifier in a DDoS attack.)
*/
package discover

import (
	"fmt"
	"log"
	"net"
	"sync"
	"time"

	"github.com/calmh/syncthing/buffers"
)

const (
	AnnouncementPort  = 21025
	AnnouncementMagic = 0x20121025
	QueryMagic        = 0x19760309
)

type Discoverer struct {
	MyID             string
	ListenPort       int
	BroadcastIntv    time.Duration
	ExtListenPort    int
	ExtBroadcastIntv time.Duration

	conn         *net.UDPConn
	registry     map[string]string
	registryLock sync.RWMutex
	extServer    string
}

// We tolerate a certain amount of errors because we might be running on
// laptops that sleep and wake, have intermittent network connectivity, etc.
// When we hit this many errors in succession, we stop.
const maxErrors = 30

func NewDiscoverer(id string, port int, extPort int, extServer string) (*Discoverer, error) {
	local4 := &net.UDPAddr{IP: net.IP{0, 0, 0, 0}, Port: AnnouncementPort}
	conn, err := net.ListenUDP("udp4", local4)
	if err != nil {
		return nil, err
	}

	disc := &Discoverer{
		MyID:             id,
		ListenPort:       port,
		BroadcastIntv:    30 * time.Second,
		ExtListenPort:    extPort,
		ExtBroadcastIntv: 1800 * time.Second,

		conn:      conn,
		registry:  make(map[string]string),
		extServer: extServer,
	}

	go disc.recvAnnouncements()

	if disc.ListenPort > 0 {
		disc.sendAnnouncements()
	}
	if len(disc.extServer) > 0 && disc.ExtListenPort > 0 {
		disc.sendExtAnnouncements()
	}

	return disc, nil
}

func (d *Discoverer) sendAnnouncements() {
	remote4 := &net.UDPAddr{IP: net.IP{255, 255, 255, 255}, Port: AnnouncementPort}

	buf := EncodePacket(Packet{AnnouncementMagic, uint16(d.ListenPort), d.MyID, nil})
	go d.writeAnnouncements(buf, remote4, d.BroadcastIntv)
}

func (d *Discoverer) sendExtAnnouncements() {
	extIP, err := net.ResolveUDPAddr("udp", d.extServer+":22025")
	if err != nil {
		log.Printf("discover/external: %v; no external announcements", err)
		return
	}

	buf := EncodePacket(Packet{AnnouncementMagic, uint16(d.ExtListenPort), d.MyID, nil})
	go d.writeAnnouncements(buf, extIP, d.ExtBroadcastIntv)
}

func (d *Discoverer) writeAnnouncements(buf []byte, remote *net.UDPAddr, intv time.Duration) {
	var errCounter = 0
	var err error
	for errCounter < maxErrors {
		_, _, err = d.conn.WriteMsgUDP(buf, nil, remote)
		if err != nil {
			log.Println("discover/write: warning:", err)
			errCounter++
		} else {
			errCounter = 0
		}
		time.Sleep(intv)
	}
	log.Println("discover/write: %v: stopping due to too many errors:", remote, err)
}

func (d *Discoverer) recvAnnouncements() {
	var buf = make([]byte, 1024)
	var errCounter = 0
	var err error
	for errCounter < maxErrors {
		n, addr, err := d.conn.ReadFromUDP(buf)
		if err != nil {
			errCounter++
			time.Sleep(time.Second)
			continue
		}

		pkt, err := DecodePacket(buf[:n])
		if err != nil || pkt.Magic != AnnouncementMagic {
			errCounter++
			time.Sleep(time.Second)
			continue
		}

		errCounter = 0

		if pkt.ID != d.MyID {
			nodeAddr := fmt.Sprintf("%s:%d", addr.IP.String(), pkt.Port)
			d.registryLock.Lock()
			if d.registry[pkt.ID] != nodeAddr {
				d.registry[pkt.ID] = nodeAddr
			}
			d.registryLock.Unlock()
		}
	}
	log.Println("discover/read: stopping due to too many errors:", err)
}

func (d *Discoverer) externalLookup(node string) (string, bool) {
	extIP, err := net.ResolveUDPAddr("udp", d.extServer+":22025")
	if err != nil {
		log.Printf("discover/external: %v; no external lookup", err)
		return "", false
	}

	conn, err := net.DialUDP("udp", nil, extIP)
	if err != nil {
		log.Printf("discover/external: %v; no external lookup", err)
		return "", false
	}
	defer conn.Close()

	err = conn.SetDeadline(time.Now().Add(5 * time.Second))
	if err != nil {
		log.Printf("discover/external: %v; no external lookup", err)
		return "", false
	}

	_, err = conn.Write(EncodePacket(Packet{QueryMagic, 0, node, nil}))
	if err != nil {
		log.Printf("discover/external: %v; no external lookup", err)
		return "", false
	}

	var buf = buffers.Get(1024)
	defer buffers.Put(buf)

	n, err := conn.Read(buf)
	if err != nil {
		if err, ok := err.(net.Error); ok && err.Timeout() {
			// Expected if the server doesn't know about requested node ID
			return "", false
		}
		log.Printf("discover/external/read: %v; no external lookup", err)
		return "", false
	}

	pkt, err := DecodePacket(buf[:n])
	if err != nil {
		log.Printf("discover/external/read: %v; no external lookup", err)
		return "", false
	}

	if pkt.Magic != AnnouncementMagic {
		log.Printf("discover/external/read: bad magic; no external lookup", err)
		return "", false
	}

	return fmt.Sprintf("%s:%d", ipStr(pkt.IP), pkt.Port), true
}

func (d *Discoverer) Lookup(node string) (string, bool) {
	d.registryLock.Lock()
	addr, ok := d.registry[node]
	d.registryLock.Unlock()

	if ok {
		return addr, true
	} else if len(d.extServer) != 0 {
		// We might want to cache this, but not permanently so it needs some intelligence
		return d.externalLookup(node)
	}
	return "", false
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package build

import (
	"crypto/hmac"
	"fmt"
	"http"
	"json"
	"os"

	"appengine"
	"appengine/datastore"
	"cache"
)

const commitsPerPage = 30

// defaultPackages specifies the Package records to be created by initHandler.
var defaultPackages = []*Package{
	&Package{Name: "Go"},
}

// commitHandler retrieves commit data or records a new commit.
//
// For GET requests it returns a Commit value for the specified
// packagePath and hash.
//
// For POST requests it reads a JSON-encoded Commit value from the request
// body and creates a new Commit entity. It also updates the "tip" Tag for
// each new commit at tip.
//
// This handler is used by a gobuilder process in -commit mode.
func commitHandler(r *http.Request) (interface{}, os.Error) {
	c := appengine.NewContext(r)
	com := new(Commit)

	if r.Method == "GET" {
		com.PackagePath = r.FormValue("packagePath")
		com.Hash = r.FormValue("hash")
		if err := datastore.Get(c, com.Key(c), com); err != nil {
			return nil, fmt.Errorf("getting Commit: %v", err)
		}
		return com, nil
	}
	if r.Method != "POST" {
		return nil, errBadMethod(r.Method)
	}

	// POST request
	defer r.Body.Close()
	if err := json.NewDecoder(r.Body).Decode(com); err != nil {
		return nil, fmt.Errorf("decoding Body: %v", err)
	}
	if len(com.Desc) > maxDatastoreStringLen {
		com.Desc = com.Desc[:maxDatastoreStringLen]
	}
	if err := com.Valid(); err != nil {
		return nil, fmt.Errorf("validating Commit: %v", err)
	}
	defer cache.Tick(c)
	tx := func(c appengine.Context) os.Error {
		return addCommit(c, com)
	}
	return nil, datastore.RunInTransaction(c, tx, nil)
}

// addCommit adds the Commit entity to the datastore and updates the tip Tag.
// It must be run inside a datastore transaction.
func addCommit(c appengine.Context, com *Commit) os.Error {
	var tc Commit // temp value so we don't clobber com
	err := datastore.Get(c, com.Key(c), &tc)
	if err != datastore.ErrNoSuchEntity {
		// if this commit is already in the datastore, do nothing
		if err == nil {
			return nil
		}
		return fmt.Errorf("getting Commit: %v", err)
	}
	// get the next commit number
	p, err := GetPackage(c, com.PackagePath)
	if err != nil {
		return fmt.Errorf("GetPackage: %v", err)
	}
	com.Num = p.NextNum
	p.NextNum++
	if _, err := datastore.Put(c, p.Key(c), p); err != nil {
		return fmt.Errorf("putting Package: %v", err)
	}
	// if this isn't the first Commit test the parent commit exists
	if com.Num > 0 {
		n, err := datastore.NewQuery("Commit").
			Filter("Hash =", com.ParentHash).
			Ancestor(p.Key(c)).
			Count(c)
		if err != nil {
			return fmt.Errorf("testing for parent Commit: %v", err)
		}
		if n == 0 {
			return os.NewError("parent commit not found")
		}
	}
	// update the tip Tag if this is the Go repo
	if p.Path == "" {
		t := &Tag{Kind: "tip", Hash: com.Hash}
		if _, err = datastore.Put(c, t.Key(c), t); err != nil {
			return fmt.Errorf("putting Tag: %v", err)
		}
	}
	// put the Commit
	if _, err = datastore.Put(c, com.Key(c), com); err != nil {
		return fmt.Errorf("putting Commit: %v", err)
	}
	return nil
}

// tagHandler records a new tag. It reads a JSON-encoded Tag value from the
// request body and updates the Tag entity for the Kind of tag provided.
//
// This handler is used by a gobuilder process in -commit mode.
func tagHandler(r *http.Request) (interface{}, os.Error) {
	if r.Method != "POST" {
		return nil, errBadMethod(r.Method)
	}

	t := new(Tag)
	defer r.Body.Close()
	if err := json.NewDecoder(r.Body).Decode(t); err != nil {
		return nil, err
	}
	if err := t.Valid(); err != nil {
		return nil, err
	}
	c := appengine.NewContext(r)
	defer cache.Tick(c)
	_, err := datastore.Put(c, t.Key(c), t)
	return nil, err
}

// Todo is a todoHandler response.
type Todo struct {
	Kind string // "build-go-commit" or "build-package"
	Data interface{}
}

// todoHandler returns the next action to be performed by a builder.
// It expects "builder" and "kind" query parameters and returns a *Todo value.
// Multiple "kind" parameters may be specified.
func todoHandler(r *http.Request) (interface{}, os.Error) {
	c := appengine.NewContext(r)
	now := cache.Now(c)
	key := "build-todo-" + r.Form.Encode()
	cachedTodo := new(Todo)
	if cache.Get(r, now, key, cachedTodo) {
		return cachedTodo, nil
	}
	var todo *Todo
	var err os.Error
	builder := r.FormValue("builder")
	for _, kind := range r.Form["kind"] {
		var data interface{}
		switch kind {
		case "build-go-commit":
			data, err = buildTodo(c, builder, "", "")
		case "build-package":
			packagePath := r.FormValue("packagePath")
			goHash := r.FormValue("goHash")
			data, err = buildTodo(c, builder, packagePath, goHash)
		}
		if data != nil || err != nil {
			todo = &Todo{Kind: kind, Data: data}
			break
		}
	}
	if err == nil {
		cache.Set(r, now, key, todo)
	}
	return todo, err
}

// buildTodo returns the next Commit to be built (or nil if none available).
//
// If packagePath and goHash are empty, it scans the first 20 Go Commits in
// Num-descending order and returns the first one it finds that doesn't have a
// Result for this builder.
//
// If provided with non-empty packagePath and goHash args, it scans the first
// 20 Commits in Num-descending order for the specified packagePath and
// returns the first that doesn't have a Result for this builder and goHash.
func buildTodo(c appengine.Context, builder, packagePath, goHash string) (interface{}, os.Error) {
	p, err := GetPackage(c, packagePath)
	if err != nil {
		return nil, err
	}

	t := datastore.NewQuery("Commit").
		Ancestor(p.Key(c)).
		Limit(commitsPerPage).
		Order("-Num").
		Run(c)
	for {
		com := new(Commit)
		if _, err := t.Next(com); err != nil {
			if err == datastore.Done {
				err = nil
			}
			return nil, err
		}
		if com.Result(builder, goHash) == nil {
			return com, nil
		}
	}
	panic("unreachable")
}

// packagesHandler returns a list of the non-Go Packages monitored
// by the dashboard.
func packagesHandler(r *http.Request) (interface{}, os.Error) {
	c := appengine.NewContext(r)
	now := cache.Now(c)
	const key = "build-packages"
	var p []*Package
	if cache.Get(r, now, key, &p) {
		return p, nil
	}
	p, err := Packages(c)
	if err != nil {
		return nil, err
	}
	cache.Set(r, now, key, p)
	return p, nil
}

// resultHandler records a build result.
// It reads a JSON-encoded Result value from the request body,
// creates a new Result entity, and updates the relevant Commit entity.
// If the Log field is not empty, resultHandler creates a new Log entity
// and updates the LogHash field before putting the Commit entity.
func resultHandler(r *http.Request) (interface{}, os.Error) {
	if r.Method != "POST" {
		return nil, errBadMethod(r.Method)
	}

	c := appengine.NewContext(r)
	res := new(Result)
	defer r.Body.Close()
	if err := json.NewDecoder(r.Body).Decode(res); err != nil {
		return nil, fmt.Errorf("decoding Body: %v", err)
	}
	if err := res.Valid(); err != nil {
		return nil, fmt.Errorf("validating Result: %v", err)
	}
	defer cache.Tick(c)
	// store the Log text if supplied
	if len(res.Log) > 0 {
		hash, err := PutLog(c, res.Log)
		if err != nil {
			return nil, fmt.Errorf("putting Log: %v", err)
		}
		res.LogHash = hash
	}
	tx := func(c appengine.Context) os.Error {
		// check Package exists
		if _, err := GetPackage(c, res.PackagePath); err != nil {
			return fmt.Errorf("GetPackage: %v", err)
		}
		// put Result
		if _, err := datastore.Put(c, res.Key(c), res); err != nil {
			return fmt.Errorf("putting Result: %v", err)
		}
		// add Result to Commit
		com := &Commit{PackagePath: res.PackagePath, Hash: res.Hash}
		if err := com.AddResult(c, res); err != nil {
			return fmt.Errorf("AddResult: %v", err)
		}
		// Send build failure notifications, if necessary.
		// Note this must run after the call AddResult, which
		// populates the Commit's ResultData field.
		return notifyOnFailure(c, com, res.Builder)
	}
	return nil, datastore.RunInTransaction(c, tx, nil)
}

// logHandler displays log text for a given hash.
// It handles paths like "/log/hash".
func logHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-type", "text/plain")
	c := appengine.NewContext(r)
	hash := r.URL.Path[len("/log/"):]
	key := datastore.NewKey(c, "Log", hash, 0, nil)
	l := new(Log)
	if err := datastore.Get(c, key, l); err != nil {
		logErr(w, r, err)
		return
	}
	b, err := l.Text()
	if err != nil {
		logErr(w, r, err)
		return
	}
	w.Write(b)
}

type dashHandler func(*http.Request) (interface{}, os.Error)

type dashResponse struct {
	Response interface{}
	Error    string
}

// errBadMethod is returned by a dashHandler when
// the request has an unsuitable method.
type errBadMethod string

func (e errBadMethod) String() string {
	return "bad method: " + string(e)
}

// AuthHandler wraps a http.HandlerFunc with a handler that validates the
// supplied key and builder query parameters.
func AuthHandler(h dashHandler) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		c := appengine.NewContext(r)

		// Put the URL Query values into r.Form to avoid parsing the
		// request body when calling r.FormValue.
		r.Form = r.URL.Query()

		var err os.Error
		var resp interface{}

		// Validate key query parameter for POST requests only.
		key := r.FormValue("key")
		builder := r.FormValue("builder")
		if r.Method == "POST" && !validKey(c, key, builder) {
			err = os.NewError("invalid key: " + key)
		}

		// Call the original HandlerFunc and return the response.
		if err == nil {
			resp, err = h(r)
		}

		// Write JSON response.
		dashResp := &dashResponse{Response: resp}
		if err != nil {
			c.Errorf("%v", err)
			dashResp.Error = err.String()
		}
		w.Header().Set("Content-Type", "application/json")
		if err = json.NewEncoder(w).Encode(dashResp); err != nil {
			c.Criticalf("encoding response: %v", err)
		}
	}
}

func initHandler(w http.ResponseWriter, r *http.Request) {
	// TODO(adg): devise a better way of bootstrapping new packages
	c := appengine.NewContext(r)
	defer cache.Tick(c)
	for _, p := range defaultPackages {
		if err := datastore.Get(c, p.Key(c), new(Package)); err == nil {
			continue
		} else if err != datastore.ErrNoSuchEntity {
			logErr(w, r, err)
			return
		}
		if _, err := datastore.Put(c, p.Key(c), p); err != nil {
			logErr(w, r, err)
			return
		}
	}
	fmt.Fprint(w, "OK")
}

func keyHandler(w http.ResponseWriter, r *http.Request) {
	builder := r.FormValue("builder")
	if builder == "" {
		logErr(w, r, os.NewError("must supply builder in query string"))
		return
	}
	c := appengine.NewContext(r)
	fmt.Fprint(w, builderKey(c, builder))
}

func init() {
	// admin handlers
	http.HandleFunc("/init", initHandler)
	http.HandleFunc("/key", keyHandler)

	// authenticated handlers
	http.HandleFunc("/commit", AuthHandler(commitHandler))
	http.HandleFunc("/packages", AuthHandler(packagesHandler))
	http.HandleFunc("/result", AuthHandler(resultHandler))
	http.HandleFunc("/tag", AuthHandler(tagHandler))
	http.HandleFunc("/todo", AuthHandler(todoHandler))

	// public handlers
	http.HandleFunc("/log/", logHandler)
}

func validHash(hash string) bool {
	// TODO(adg): correctly validate a hash
	return hash != ""
}

func validKey(c appengine.Context, key, builder string) bool {
	if appengine.IsDevAppServer() {
		return true
	}
	if key == secretKey(c) {
		return true
	}
	return key == builderKey(c, builder)
}

func builderKey(c appengine.Context, builder string) string {
	h := hmac.NewMD5([]byte(secretKey(c)))
	h.Write([]byte(builder))
	return fmt.Sprintf("%x", h.Sum())
}

func logErr(w http.ResponseWriter, r *http.Request, err os.Error) {
	appengine.NewContext(r).Errorf("Error: %v", err)
	w.WriteHeader(http.StatusInternalServerError)
	fmt.Fprint(w, "Error: ", err)
}
<commit_msg>dashboard: fix todo caching nil<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package build

import (
	"crypto/hmac"
	"fmt"
	"http"
	"json"
	"os"

	"appengine"
	"appengine/datastore"
	"cache"
)

const commitsPerPage = 30

// defaultPackages specifies the Package records to be created by initHandler.
var defaultPackages = []*Package{
	&Package{Name: "Go"},
}

// commitHandler retrieves commit data or records a new commit.
//
// For GET requests it returns a Commit value for the specified
// packagePath and hash.
//
// For POST requests it reads a JSON-encoded Commit value from the request
// body and creates a new Commit entity. It also updates the "tip" Tag for
// each new commit at tip.
//
// This handler is used by a gobuilder process in -commit mode.
func commitHandler(r *http.Request) (interface{}, os.Error) {
	c := appengine.NewContext(r)
	com := new(Commit)

	if r.Method == "GET" {
		com.PackagePath = r.FormValue("packagePath")
		com.Hash = r.FormValue("hash")
		if err := datastore.Get(c, com.Key(c), com); err != nil {
			return nil, fmt.Errorf("getting Commit: %v", err)
		}
		return com, nil
	}
	if r.Method != "POST" {
		return nil, errBadMethod(r.Method)
	}

	// POST request
	defer r.Body.Close()
	if err := json.NewDecoder(r.Body).Decode(com); err != nil {
		return nil, fmt.Errorf("decoding Body: %v", err)
	}
	if len(com.Desc) > maxDatastoreStringLen {
		com.Desc = com.Desc[:maxDatastoreStringLen]
	}
	if err := com.Valid(); err != nil {
		return nil, fmt.Errorf("validating Commit: %v", err)
	}
	defer cache.Tick(c)
	tx := func(c appengine.Context) os.Error {
		return addCommit(c, com)
	}
	return nil, datastore.RunInTransaction(c, tx, nil)
}

// addCommit adds the Commit entity to the datastore and updates the tip Tag.
// It must be run inside a datastore transaction.
func addCommit(c appengine.Context, com *Commit) os.Error {
	var tc Commit // temp value so we don't clobber com
	err := datastore.Get(c, com.Key(c), &tc)
	if err != datastore.ErrNoSuchEntity {
		// if this commit is already in the datastore, do nothing
		if err == nil {
			return nil
		}
		return fmt.Errorf("getting Commit: %v", err)
	}
	// get the next commit number
	p, err := GetPackage(c, com.PackagePath)
	if err != nil {
		return fmt.Errorf("GetPackage: %v", err)
	}
	com.Num = p.NextNum
	p.NextNum++
	if _, err := datastore.Put(c, p.Key(c), p); err != nil {
		return fmt.Errorf("putting Package: %v", err)
	}
	// if this isn't the first Commit test the parent commit exists
	if com.Num > 0 {
		n, err := datastore.NewQuery("Commit").
			Filter("Hash =", com.ParentHash).
			Ancestor(p.Key(c)).
			Count(c)
		if err != nil {
			return fmt.Errorf("testing for parent Commit: %v", err)
		}
		if n == 0 {
			return os.NewError("parent commit not found")
		}
	}
	// update the tip Tag if this is the Go repo
	if p.Path == "" {
		t := &Tag{Kind: "tip", Hash: com.Hash}
		if _, err = datastore.Put(c, t.Key(c), t); err != nil {
			return fmt.Errorf("putting Tag: %v", err)
		}
	}
	// put the Commit
	if _, err = datastore.Put(c, com.Key(c), com); err != nil {
		return fmt.Errorf("putting Commit: %v", err)
	}
	return nil
}

// tagHandler records a new tag. It reads a JSON-encoded Tag value from the
// request body and updates the Tag entity for the Kind of tag provided.
//
// This handler is used by a gobuilder process in -commit mode.
func tagHandler(r *http.Request) (interface{}, os.Error) {
	if r.Method != "POST" {
		return nil, errBadMethod(r.Method)
	}

	t := new(Tag)
	defer r.Body.Close()
	if err := json.NewDecoder(r.Body).Decode(t); err != nil {
		return nil, err
	}
	if err := t.Valid(); err != nil {
		return nil, err
	}
	c := appengine.NewContext(r)
	defer cache.Tick(c)
	_, err := datastore.Put(c, t.Key(c), t)
	return nil, err
}

// Todo is a todoHandler response.
type Todo struct {
	Kind string // "build-go-commit" or "build-package"
	Data interface{}
}

// todoHandler returns the next action to be performed by a builder.
// It expects "builder" and "kind" query parameters and returns a *Todo value.
// Multiple "kind" parameters may be specified.
func todoHandler(r *http.Request) (interface{}, os.Error) {
	c := appengine.NewContext(r)
	now := cache.Now(c)
	key := "build-todo-" + r.Form.Encode()
	var todo *Todo
	if cache.Get(r, now, key, &todo) {
		return todo, nil
	}
	var err os.Error
	builder := r.FormValue("builder")
	for _, kind := range r.Form["kind"] {
		var data interface{}
		switch kind {
		case "build-go-commit":
			data, err = buildTodo(c, builder, "", "")
		case "build-package":
			packagePath := r.FormValue("packagePath")
			goHash := r.FormValue("goHash")
			data, err = buildTodo(c, builder, packagePath, goHash)
		}
		if data != nil || err != nil {
			todo = &Todo{Kind: kind, Data: data}
			break
		}
	}
	if err == nil {
		cache.Set(r, now, key, todo)
	}
	return todo, err
}

// buildTodo returns the next Commit to be built (or nil if none available).
//
// If packagePath and goHash are empty, it scans the first 20 Go Commits in
// Num-descending order and returns the first one it finds that doesn't have a
// Result for this builder.
//
// If provided with non-empty packagePath and goHash args, it scans the first
// 20 Commits in Num-descending order for the specified packagePath and
// returns the first that doesn't have a Result for this builder and goHash.
func buildTodo(c appengine.Context, builder, packagePath, goHash string) (interface{}, os.Error) {
	p, err := GetPackage(c, packagePath)
	if err != nil {
		return nil, err
	}

	t := datastore.NewQuery("Commit").
		Ancestor(p.Key(c)).
		Limit(commitsPerPage).
		Order("-Num").
		Run(c)
	for {
		com := new(Commit)
		if _, err := t.Next(com); err != nil {
			if err == datastore.Done {
				err = nil
			}
			return nil, err
		}
		if com.Result(builder, goHash) == nil {
			return com, nil
		}
	}
	panic("unreachable")
}

// packagesHandler returns a list of the non-Go Packages monitored
// by the dashboard.
func packagesHandler(r *http.Request) (interface{}, os.Error) {
	c := appengine.NewContext(r)
	now := cache.Now(c)
	const key = "build-packages"
	var p []*Package
	if cache.Get(r, now, key, &p) {
		return p, nil
	}
	p, err := Packages(c)
	if err != nil {
		return nil, err
	}
	cache.Set(r, now, key, p)
	return p, nil
}

// resultHandler records a build result.
// It reads a JSON-encoded Result value from the request body,
// creates a new Result entity, and updates the relevant Commit entity.
// If the Log field is not empty, resultHandler creates a new Log entity
// and updates the LogHash field before putting the Commit entity.
func resultHandler(r *http.Request) (interface{}, os.Error) {
	if r.Method != "POST" {
		return nil, errBadMethod(r.Method)
	}

	c := appengine.NewContext(r)
	res := new(Result)
	defer r.Body.Close()
	if err := json.NewDecoder(r.Body).Decode(res); err != nil {
		return nil, fmt.Errorf("decoding Body: %v", err)
	}
	if err := res.Valid(); err != nil {
		return nil, fmt.Errorf("validating Result: %v", err)
	}
	defer cache.Tick(c)
	// store the Log text if supplied
	if len(res.Log) > 0 {
		hash, err := PutLog(c, res.Log)
		if err != nil {
			return nil, fmt.Errorf("putting Log: %v", err)
		}
		res.LogHash = hash
	}
	tx := func(c appengine.Context) os.Error {
		// check Package exists
		if _, err := GetPackage(c, res.PackagePath); err != nil {
			return fmt.Errorf("GetPackage: %v", err)
		}
		// put Result
		if _, err := datastore.Put(c, res.Key(c), res); err != nil {
			return fmt.Errorf("putting Result: %v", err)
		}
		// add Result to Commit
		com := &Commit{PackagePath: res.PackagePath, Hash: res.Hash}
		if err := com.AddResult(c, res); err != nil {
			return fmt.Errorf("AddResult: %v", err)
		}
		// Send build failure notifications, if necessary.
		// Note this must run after the call AddResult, which
		// populates the Commit's ResultData field.
		return notifyOnFailure(c, com, res.Builder)
	}
	return nil, datastore.RunInTransaction(c, tx, nil)
}

// logHandler displays log text for a given hash.
// It handles paths like "/log/hash".
func logHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-type", "text/plain")
	c := appengine.NewContext(r)
	hash := r.URL.Path[len("/log/"):]
	key := datastore.NewKey(c, "Log", hash, 0, nil)
	l := new(Log)
	if err := datastore.Get(c, key, l); err != nil {
		logErr(w, r, err)
		return
	}
	b, err := l.Text()
	if err != nil {
		logErr(w, r, err)
		return
	}
	w.Write(b)
}

type dashHandler func(*http.Request) (interface{}, os.Error)

type dashResponse struct {
	Response interface{}
	Error    string
}

// errBadMethod is returned by a dashHandler when
// the request has an unsuitable method.
type errBadMethod string

func (e errBadMethod) String() string {
	return "bad method: " + string(e)
}

// AuthHandler wraps a http.HandlerFunc with a handler that validates the
// supplied key and builder query parameters.
func AuthHandler(h dashHandler) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		c := appengine.NewContext(r)

		// Put the URL Query values into r.Form to avoid parsing the
		// request body when calling r.FormValue.
		r.Form = r.URL.Query()

		var err os.Error
		var resp interface{}

		// Validate key query parameter for POST requests only.
		key := r.FormValue("key")
		builder := r.FormValue("builder")
		if r.Method == "POST" && !validKey(c, key, builder) {
			err = os.NewError("invalid key: " + key)
		}

		// Call the original HandlerFunc and return the response.
		if err == nil {
			resp, err = h(r)
		}

		// Write JSON response.
		dashResp := &dashResponse{Response: resp}
		if err != nil {
			c.Errorf("%v", err)
			dashResp.Error = err.String()
		}
		w.Header().Set("Content-Type", "application/json")
		if err = json.NewEncoder(w).Encode(dashResp); err != nil {
			c.Criticalf("encoding response: %v", err)
		}
	}
}

func initHandler(w http.ResponseWriter, r *http.Request) {
	// TODO(adg): devise a better way of bootstrapping new packages
	c := appengine.NewContext(r)
	defer cache.Tick(c)
	for _, p := range defaultPackages {
		if err := datastore.Get(c, p.Key(c), new(Package)); err == nil {
			continue
		} else if err != datastore.ErrNoSuchEntity {
			logErr(w, r, err)
			return
		}
		if _, err := datastore.Put(c, p.Key(c), p); err != nil {
			logErr(w, r, err)
			return
		}
	}
	fmt.Fprint(w, "OK")
}

func keyHandler(w http.ResponseWriter, r *http.Request) {
	builder := r.FormValue("builder")
	if builder == "" {
		logErr(w, r, os.NewError("must supply builder in query string"))
		return
	}
	c := appengine.NewContext(r)
	fmt.Fprint(w, builderKey(c, builder))
}

func init() {
	// admin handlers
	http.HandleFunc("/init", initHandler)
	http.HandleFunc("/key", keyHandler)

	// authenticated handlers
	http.HandleFunc("/commit", AuthHandler(commitHandler))
	http.HandleFunc("/packages", AuthHandler(packagesHandler))
	http.HandleFunc("/result", AuthHandler(resultHandler))
	http.HandleFunc("/tag", AuthHandler(tagHandler))
	http.HandleFunc("/todo", AuthHandler(todoHandler))

	// public handlers
	http.HandleFunc("/log/", logHandler)
}

func validHash(hash string) bool {
	// TODO(adg): correctly validate a hash
	return hash != ""
}

func validKey(c appengine.Context, key, builder string) bool {
	if appengine.IsDevAppServer() {
		return true
	}
	if key == secretKey(c) {
		return true
	}
	return key == builderKey(c, builder)
}

func builderKey(c appengine.Context, builder string) string {
	h := hmac.NewMD5([]byte(secretKey(c)))
	h.Write([]byte(builder))
	return fmt.Sprintf("%x", h.Sum())
}

func logErr(w http.ResponseWriter, r *http.Request, err os.Error) {
	appengine.NewContext(r).Errorf("Error: %v", err)
	w.WriteHeader(http.StatusInternalServerError)
	fmt.Fprint(w, "Error: ", err)
}
<|endoftext|>


<commit_before><commit_msg>Close #8.<commit_after><|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"syscall"
	"time"

	"github.com/rfjakob/gocryptfs/cryptfs"
	"github.com/rfjakob/gocryptfs/pathfs_frontend"

	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/go-fuse/fuse/nodefs"
	"github.com/hanwen/go-fuse/fuse/pathfs"
)

const (
	PROGRAM_NAME = "gocryptfs"

	// Exit codes
	ERREXIT_USAGE      = 1
	ERREXIT_MOUNT      = 3
	ERREXIT_CIPHERDIR  = 6
	ERREXIT_INIT       = 7
	ERREXIT_LOADCONF   = 8
	ERREXIT_PASSWORD   = 9
	ERREXIT_MOUNTPOINT = 10
)

// GitVersion will be set by the build script "build.bash"
var GitVersion = "[version not set - please compile using ./build.bash]"

func initDir(args *argContainer) {
	err := checkDirEmpty(args.cipherdir)
	if err != nil {
		fmt.Printf("Invalid CIPHERDIR: %v\n", err)
		os.Exit(ERREXIT_INIT)
	}

	// Create gocryptfs.diriv in the root dir
	diriv := cryptfs.RandBytes(cryptfs.DIRIV_LEN)
	dirivPath := filepath.Join(args.cipherdir, cryptfs.DIRIV_FILENAME)
	cryptfs.Debug.Printf("Creating %s\n", dirivPath)
	// 0444 permissions: the file is not secret but should not be written to
	err = ioutil.WriteFile(dirivPath, diriv, 0444)
	if err != nil {
		fmt.Println(err)
		os.Exit(ERREXIT_INIT)
	}

	// Create gocryptfs.conf
	cryptfs.Info.Printf("Choose a password for protecting your files.\n")
	password := readPasswordTwice(args.extpass)
	err = cryptfs.CreateConfFile(args.config, password, args.plaintextnames)
	if err != nil {
		fmt.Println(err)
		os.Exit(ERREXIT_INIT)
	}
	cryptfs.Info.Printf("The filesystem is now ready for mounting.\n")
	os.Exit(0)
}

func usageText() {
	printVersion()
	fmt.Printf("\n")
	fmt.Printf("Usage: %s -init|-passwd [OPTIONS] CIPHERDIR\n", PROGRAM_NAME)
	fmt.Printf("  or   %s [OPTIONS] CIPHERDIR MOUNTPOINT\n", PROGRAM_NAME)
	fmt.Printf("\nOptions:\n")
	flagSet.PrintDefaults()
}

type argContainer struct {
	debug, init, zerokey, fusedebug, openssl, passwd, foreground, version,
	plaintextnames, quiet bool
	masterkey, mountpoint, cipherdir, cpuprofile, config, extpass string
	notifypid                                                     int
}

var flagSet *flag.FlagSet

// loadConfig - load the config file "filename", prompting the user for the password
func loadConfig(args *argContainer) (masterkey []byte, confFile *cryptfs.ConfFile) {
	// Check if the file exists at all before prompting for a password
	_, err := os.Stat(args.config)
	if err != nil {
		fmt.Println(err)
		os.Exit(ERREXIT_LOADCONF)
	}
	fmt.Printf("Password: ")
	pw := readPassword(args.extpass)
	cryptfs.Info.Printf("Decrypting master key... ")
	cryptfs.Warn.Disable() // Silence DecryptBlock() error messages on incorrect password
	masterkey, confFile, err = cryptfs.LoadConfFile(args.config, pw)
	cryptfs.Warn.Enable()
	if err != nil {
		fmt.Println(err)
		fmt.Println("Wrong password.")
		os.Exit(ERREXIT_LOADCONF)
	}
	cryptfs.Info.Printf("done.\n")

	return masterkey, confFile
}

// changePassword - change the password of config file "filename"
func changePassword(args *argContainer) {
	masterkey, confFile := loadConfig(args)
	fmt.Printf("Please enter your new password.\n")
	newPw := readPasswordTwice(args.extpass)
	confFile.EncryptKey(masterkey, newPw)
	err := confFile.WriteFile()
	if err != nil {
		fmt.Println(err)
		os.Exit(ERREXIT_INIT)
	}
	cryptfs.Info.Printf("Password changed.\n")
	os.Exit(0)
}

// printVersion - print a version string like
// "gocryptfs v0.3.1-31-g6736212-dirty; on-disk format 2"
func printVersion() {
	fmt.Printf("%s %s; on-disk format %d\n", PROGRAM_NAME, GitVersion, cryptfs.HEADER_CURRENT_VERSION)
}

func main() {
	runtime.GOMAXPROCS(4)
	var err error

	// Parse command line arguments
	var args argContainer
	flagSet = flag.NewFlagSet(PROGRAM_NAME, flag.ExitOnError)
	flagSet.Usage = usageText
	flagSet.BoolVar(&args.debug, "debug", false, "Enable debug output")
	flagSet.BoolVar(&args.fusedebug, "fusedebug", false, "Enable fuse library debug output")
	flagSet.BoolVar(&args.init, "init", false, "Initialize encrypted directory")
	flagSet.BoolVar(&args.zerokey, "zerokey", false, "Use all-zero dummy master key")
	flagSet.BoolVar(&args.openssl, "openssl", true, "Use OpenSSL instead of built-in Go crypto")
	flagSet.BoolVar(&args.passwd, "passwd", false, "Change password")
	flagSet.BoolVar(&args.foreground, "f", false, "Stay in the foreground")
	flagSet.BoolVar(&args.version, "version", false, "Print version and exit")
	flagSet.BoolVar(&args.plaintextnames, "plaintextnames", false, "Do not encrypt "+
		"file names")
	flagSet.BoolVar(&args.quiet, "q", false, "Quiet - silence informational messages")
	flagSet.StringVar(&args.masterkey, "masterkey", "", "Mount with explicit master key")
	flagSet.StringVar(&args.cpuprofile, "cpuprofile", "", "Write cpu profile to specified file")
	flagSet.StringVar(&args.config, "config", "", "Use specified config file instead of CIPHERDIR/gocryptfs.conf")
	flagSet.StringVar(&args.extpass, "extpass", "", "Use external program for the password prompt")
	flagSet.IntVar(&args.notifypid, "notifypid", 0, "Send USR1 to the specified process after "+
		"successful mount - used internally for daemonization")
	flagSet.Parse(os.Args[1:])

	// By default, let the child handle everything.
	// The parent *could* handle operations that do not require backgrounding by
	// itself, but that would make the code paths more complicated.
	if !args.foreground {
		forkChild() // does not return
	}
	// Getting here means we *are* the child
	// "-v"
	if args.version {
		printVersion()
		os.Exit(0)
	}
	if args.debug {
		cryptfs.Debug.Enable()
		cryptfs.Debug.Printf("Debug output enabled\n")
	}
	// Every operation below requires CIPHERDIR. Check that we have it.
	if flagSet.NArg() >= 1 {
		args.cipherdir, _ = filepath.Abs(flagSet.Arg(0))
		err := checkDir(args.cipherdir)
		if err != nil {
			fmt.Printf("Invalid CIPHERDIR: %v\n", err)
			os.Exit(ERREXIT_CIPHERDIR)
		}
	} else {
		usageText()
		os.Exit(ERREXIT_USAGE)
	}
	// "-q"
	if args.quiet {
		cryptfs.Info.Disable()
	}
	// "-config"
	if args.config != "" {
		args.config, err = filepath.Abs(args.config)
		if err != nil {
			fmt.Printf("Invalid \"-config\" setting: %v\n", err)
		}
		cryptfs.Info.Printf("Using config file at custom location %s\n", args.config)
	} else {
		args.config = filepath.Join(args.cipherdir, cryptfs.ConfDefaultName)
	}
	// "-cpuprofile"
	if args.cpuprofile != "" {
		f, err := os.Create(args.cpuprofile)
		if err != nil {
			fmt.Println(err)
			os.Exit(ERREXIT_INIT)
		}
		cryptfs.Info.Printf("Writing CPU profile to %s\n", args.cpuprofile)
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}
	// "-openssl"
	if args.openssl == false {
		cryptfs.Info.Printf("Openssl disabled\n")
	}
	// Operation flags: init, passwd or mount
	// "-init"
	if args.init {
		if flagSet.NArg() > 1 {
			fmt.Printf("Usage: %s -init [OPTIONS] CIPHERDIR\n", PROGRAM_NAME)
			os.Exit(ERREXIT_USAGE)
		}
		initDir(&args) // does not return
	}
	// "-passwd"
	if args.passwd {
		if flagSet.NArg() > 1 {
			fmt.Printf("Usage: %s -passwd [OPTIONS] CIPHERDIR\n", PROGRAM_NAME)
			os.Exit(ERREXIT_USAGE)
		}
		changePassword(&args) // does not return
	}
	// Mount
	// Check mountpoint
	if flagSet.NArg() < 2 {
		usageText()
		os.Exit(ERREXIT_USAGE)
	}
	args.mountpoint, err = filepath.Abs(flagSet.Arg(1))
	if err != nil {
		fmt.Printf("Invalid MOUNTPOINT: %v\n", err)
		os.Exit(ERREXIT_MOUNTPOINT)
	}
	err = checkDirEmpty(args.mountpoint)
	if err != nil {
		fmt.Printf("Invalid MOUNTPOINT: %v\n", err)
		os.Exit(ERREXIT_MOUNTPOINT)
	}
	// Get master key
	var masterkey []byte
	if args.masterkey != "" {
		// "-masterkey"
		cryptfs.Info.Printf("Using explicit master key.\n")
		masterkey = parseMasterKey(args.masterkey)
		cryptfs.Info.Printf("THE MASTER KEY IS VISIBLE VIA \"ps -auxwww\", ONLY USE THIS MODE FOR EMERGENCIES.\n")
	} else if args.zerokey {
		// "-zerokey"
		cryptfs.Info.Printf("Using all-zero dummy master key.\n")
		cryptfs.Info.Printf("ZEROKEY MODE PROVIDES NO SECURITY AT ALL AND SHOULD ONLY BE USED FOR TESTING.\n")
		masterkey = make([]byte, cryptfs.KEY_LEN)
	} else {
		// Load master key from config file
		var confFile *cryptfs.ConfFile
		masterkey, confFile = loadConfig(&args)
		printMasterKey(masterkey)
		args.plaintextnames = confFile.IsFeatureFlagSet(cryptfs.FlagPlaintextNames)
	}
	// Initialize FUSE server
	srv := pathfsFrontend(masterkey, args.cipherdir, args.mountpoint, args.fusedebug, args.openssl, args.plaintextnames)
	cryptfs.Info.Println("Filesystem ready.")
	// We are ready - send USR1 signal to our parent
	if args.notifypid > 0 {
		sendUsr1(args.notifypid)
	}
	// Wait for SIGINT in the background and unmount ourselves if we get it.
	// This prevents a dangling "Transport endpoint is not connected" mountpoint.
	handleSigint(srv, args.mountpoint)
	// Jump into server loop. Returns when it gets an umount request from the kernel.
	srv.Serve()
	// main exits with code 0
}

// pathfsFrontend - initialize FUSE server based on go-fuse's PathFS
// Calls os.Exit on errors
func pathfsFrontend(key []byte, cipherdir string, mountpoint string,
	debug bool, openssl bool, plaintextNames bool) *fuse.Server {

	finalFs := pathfs_frontend.NewFS(key, cipherdir, openssl, plaintextNames)
	pathFsOpts := &pathfs.PathNodeFsOptions{ClientInodes: true}
	pathFs := pathfs.NewPathNodeFs(finalFs, pathFsOpts)
	fuseOpts := &nodefs.Options{
		// These options are to be compatible with libfuse defaults,
		// making benchmarking easier.
		NegativeTimeout: time.Second,
		AttrTimeout:     time.Second,
		EntryTimeout:    time.Second,
	}
	conn := nodefs.NewFileSystemConnector(pathFs.Root(), fuseOpts)
	var mOpts fuse.MountOptions
	mOpts.AllowOther = false
	// Set values shown in "df -T" and friends
	// First column, "Filesystem"
	mOpts.Options = append(mOpts.Options, "fsname="+cipherdir)
	// Second column, "Type", will be shown as "fuse." + Name
	mOpts.Name = "gocryptfs"

	srv, err := fuse.NewServer(conn.RawFS(), mountpoint, &mOpts)
	if err != nil {
		fmt.Printf("Mount failed: %v", err)
		os.Exit(ERREXIT_MOUNT)
	}
	srv.SetDebug(debug)

	return srv
}

func handleSigint(srv *fuse.Server, mountpoint string) {
	ch := make(chan os.Signal, 1)
	signal.Notify(ch, os.Interrupt)
	signal.Notify(ch, syscall.SIGTERM)
	go func() {
		<-ch
		err := srv.Unmount()
		if err != nil {
			fmt.Print(err)
			cryptfs.Info.Printf("Trying lazy unmount\n")
			cmd := exec.Command("fusermount", "-u", "-z", mountpoint)
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr
			cmd.Run()
		}
		os.Exit(1)
	}()
}
<commit_msg>main: pass args struct instead of having a huge function call<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"syscall"
	"time"

	"github.com/rfjakob/gocryptfs/cryptfs"
	"github.com/rfjakob/gocryptfs/pathfs_frontend"

	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/go-fuse/fuse/nodefs"
	"github.com/hanwen/go-fuse/fuse/pathfs"
)

const (
	PROGRAM_NAME = "gocryptfs"

	// Exit codes
	ERREXIT_USAGE      = 1
	ERREXIT_MOUNT      = 3
	ERREXIT_CIPHERDIR  = 6
	ERREXIT_INIT       = 7
	ERREXIT_LOADCONF   = 8
	ERREXIT_PASSWORD   = 9
	ERREXIT_MOUNTPOINT = 10
)

// GitVersion will be set by the build script "build.bash"
var GitVersion = "[version not set - please compile using ./build.bash]"

func initDir(args *argContainer) {
	err := checkDirEmpty(args.cipherdir)
	if err != nil {
		fmt.Printf("Invalid CIPHERDIR: %v\n", err)
		os.Exit(ERREXIT_INIT)
	}

	// Create gocryptfs.diriv in the root dir
	diriv := cryptfs.RandBytes(cryptfs.DIRIV_LEN)
	dirivPath := filepath.Join(args.cipherdir, cryptfs.DIRIV_FILENAME)
	cryptfs.Debug.Printf("Creating %s\n", dirivPath)
	// 0444 permissions: the file is not secret but should not be written to
	err = ioutil.WriteFile(dirivPath, diriv, 0444)
	if err != nil {
		fmt.Println(err)
		os.Exit(ERREXIT_INIT)
	}

	// Create gocryptfs.conf
	cryptfs.Info.Printf("Choose a password for protecting your files.\n")
	password := readPasswordTwice(args.extpass)
	err = cryptfs.CreateConfFile(args.config, password, args.plaintextnames)
	if err != nil {
		fmt.Println(err)
		os.Exit(ERREXIT_INIT)
	}
	cryptfs.Info.Printf("The filesystem is now ready for mounting.\n")
	os.Exit(0)
}

func usageText() {
	printVersion()
	fmt.Printf("\n")
	fmt.Printf("Usage: %s -init|-passwd [OPTIONS] CIPHERDIR\n", PROGRAM_NAME)
	fmt.Printf("  or   %s [OPTIONS] CIPHERDIR MOUNTPOINT\n", PROGRAM_NAME)
	fmt.Printf("\nOptions:\n")
	flagSet.PrintDefaults()
}

type argContainer struct {
	debug, init, zerokey, fusedebug, openssl, passwd, foreground, version,
	plaintextnames, quiet bool
	masterkey, mountpoint, cipherdir, cpuprofile, config, extpass string
	notifypid                                                     int
}

var flagSet *flag.FlagSet

// loadConfig - load the config file "filename", prompting the user for the password
func loadConfig(args *argContainer) (masterkey []byte, confFile *cryptfs.ConfFile) {
	// Check if the file exists at all before prompting for a password
	_, err := os.Stat(args.config)
	if err != nil {
		fmt.Println(err)
		os.Exit(ERREXIT_LOADCONF)
	}
	fmt.Printf("Password: ")
	pw := readPassword(args.extpass)
	cryptfs.Info.Printf("Decrypting master key... ")
	cryptfs.Warn.Disable() // Silence DecryptBlock() error messages on incorrect password
	masterkey, confFile, err = cryptfs.LoadConfFile(args.config, pw)
	cryptfs.Warn.Enable()
	if err != nil {
		fmt.Println(err)
		fmt.Println("Wrong password.")
		os.Exit(ERREXIT_LOADCONF)
	}
	cryptfs.Info.Printf("done.\n")

	return masterkey, confFile
}

// changePassword - change the password of config file "filename"
func changePassword(args *argContainer) {
	masterkey, confFile := loadConfig(args)
	fmt.Printf("Please enter your new password.\n")
	newPw := readPasswordTwice(args.extpass)
	confFile.EncryptKey(masterkey, newPw)
	err := confFile.WriteFile()
	if err != nil {
		fmt.Println(err)
		os.Exit(ERREXIT_INIT)
	}
	cryptfs.Info.Printf("Password changed.\n")
	os.Exit(0)
}

// printVersion - print a version string like
// "gocryptfs v0.3.1-31-g6736212-dirty; on-disk format 2"
func printVersion() {
	fmt.Printf("%s %s; on-disk format %d\n", PROGRAM_NAME, GitVersion, cryptfs.HEADER_CURRENT_VERSION)
}

func main() {
	runtime.GOMAXPROCS(4)
	var err error

	// Parse command line arguments
	var args argContainer
	flagSet = flag.NewFlagSet(PROGRAM_NAME, flag.ExitOnError)
	flagSet.Usage = usageText
	flagSet.BoolVar(&args.debug, "debug", false, "Enable debug output")
	flagSet.BoolVar(&args.fusedebug, "fusedebug", false, "Enable fuse library debug output")
	flagSet.BoolVar(&args.init, "init", false, "Initialize encrypted directory")
	flagSet.BoolVar(&args.zerokey, "zerokey", false, "Use all-zero dummy master key")
	flagSet.BoolVar(&args.openssl, "openssl", true, "Use OpenSSL instead of built-in Go crypto")
	flagSet.BoolVar(&args.passwd, "passwd", false, "Change password")
	flagSet.BoolVar(&args.foreground, "f", false, "Stay in the foreground")
	flagSet.BoolVar(&args.version, "version", false, "Print version and exit")
	flagSet.BoolVar(&args.plaintextnames, "plaintextnames", false, "Do not encrypt "+
		"file names")
	flagSet.BoolVar(&args.quiet, "q", false, "Quiet - silence informational messages")
	flagSet.StringVar(&args.masterkey, "masterkey", "", "Mount with explicit master key")
	flagSet.StringVar(&args.cpuprofile, "cpuprofile", "", "Write cpu profile to specified file")
	flagSet.StringVar(&args.config, "config", "", "Use specified config file instead of CIPHERDIR/gocryptfs.conf")
	flagSet.StringVar(&args.extpass, "extpass", "", "Use external program for the password prompt")
	flagSet.IntVar(&args.notifypid, "notifypid", 0, "Send USR1 to the specified process after "+
		"successful mount - used internally for daemonization")
	flagSet.Parse(os.Args[1:])

	// By default, let the child handle everything.
	// The parent *could* handle operations that do not require backgrounding by
	// itself, but that would make the code paths more complicated.
	if !args.foreground {
		forkChild() // does not return
	}
	// Getting here means we *are* the child
	// "-v"
	if args.version {
		printVersion()
		os.Exit(0)
	}
	if args.debug {
		cryptfs.Debug.Enable()
		cryptfs.Debug.Printf("Debug output enabled\n")
	}
	// Every operation below requires CIPHERDIR. Check that we have it.
	if flagSet.NArg() >= 1 {
		args.cipherdir, _ = filepath.Abs(flagSet.Arg(0))
		err := checkDir(args.cipherdir)
		if err != nil {
			fmt.Printf("Invalid CIPHERDIR: %v\n", err)
			os.Exit(ERREXIT_CIPHERDIR)
		}
	} else {
		usageText()
		os.Exit(ERREXIT_USAGE)
	}
	// "-q"
	if args.quiet {
		cryptfs.Info.Disable()
	}
	// "-config"
	if args.config != "" {
		args.config, err = filepath.Abs(args.config)
		if err != nil {
			fmt.Printf("Invalid \"-config\" setting: %v\n", err)
		}
		cryptfs.Info.Printf("Using config file at custom location %s\n", args.config)
	} else {
		args.config = filepath.Join(args.cipherdir, cryptfs.ConfDefaultName)
	}
	// "-cpuprofile"
	if args.cpuprofile != "" {
		f, err := os.Create(args.cpuprofile)
		if err != nil {
			fmt.Println(err)
			os.Exit(ERREXIT_INIT)
		}
		cryptfs.Info.Printf("Writing CPU profile to %s\n", args.cpuprofile)
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}
	// "-openssl"
	if args.openssl == false {
		cryptfs.Info.Printf("Openssl disabled\n")
	}
	// Operation flags: init, passwd or mount
	// "-init"
	if args.init {
		if flagSet.NArg() > 1 {
			fmt.Printf("Usage: %s -init [OPTIONS] CIPHERDIR\n", PROGRAM_NAME)
			os.Exit(ERREXIT_USAGE)
		}
		initDir(&args) // does not return
	}
	// "-passwd"
	if args.passwd {
		if flagSet.NArg() > 1 {
			fmt.Printf("Usage: %s -passwd [OPTIONS] CIPHERDIR\n", PROGRAM_NAME)
			os.Exit(ERREXIT_USAGE)
		}
		changePassword(&args) // does not return
	}
	// Mount
	// Check mountpoint
	if flagSet.NArg() < 2 {
		usageText()
		os.Exit(ERREXIT_USAGE)
	}
	args.mountpoint, err = filepath.Abs(flagSet.Arg(1))
	if err != nil {
		fmt.Printf("Invalid MOUNTPOINT: %v\n", err)
		os.Exit(ERREXIT_MOUNTPOINT)
	}
	err = checkDirEmpty(args.mountpoint)
	if err != nil {
		fmt.Printf("Invalid MOUNTPOINT: %v\n", err)
		os.Exit(ERREXIT_MOUNTPOINT)
	}
	// Get master key
	var masterkey []byte
	if args.masterkey != "" {
		// "-masterkey"
		cryptfs.Info.Printf("Using explicit master key.\n")
		masterkey = parseMasterKey(args.masterkey)
		cryptfs.Info.Printf("THE MASTER KEY IS VISIBLE VIA \"ps -auxwww\", ONLY USE THIS MODE FOR EMERGENCIES.\n")
	} else if args.zerokey {
		// "-zerokey"
		cryptfs.Info.Printf("Using all-zero dummy master key.\n")
		cryptfs.Info.Printf("ZEROKEY MODE PROVIDES NO SECURITY AT ALL AND SHOULD ONLY BE USED FOR TESTING.\n")
		masterkey = make([]byte, cryptfs.KEY_LEN)
	} else {
		// Load master key from config file
		var confFile *cryptfs.ConfFile
		masterkey, confFile = loadConfig(&args)
		printMasterKey(masterkey)
		args.plaintextnames = confFile.IsFeatureFlagSet(cryptfs.FlagPlaintextNames)
	}
	// Initialize FUSE server
	srv := pathfsFrontend(masterkey, args)
	cryptfs.Info.Println("Filesystem ready.")
	// We are ready - send USR1 signal to our parent
	if args.notifypid > 0 {
		sendUsr1(args.notifypid)
	}
	// Wait for SIGINT in the background and unmount ourselves if we get it.
	// This prevents a dangling "Transport endpoint is not connected" mountpoint.
	handleSigint(srv, args.mountpoint)
	// Jump into server loop. Returns when it gets an umount request from the kernel.
	srv.Serve()
	// main exits with code 0
}

// pathfsFrontend - initialize FUSE server based on go-fuse's PathFS.
// Calls os.Exit on errors
func pathfsFrontend(key []byte, args argContainer) *fuse.Server {

	finalFs := pathfs_frontend.NewFS(key, args.cipherdir, args.openssl, args.plaintextnames)
	pathFsOpts := &pathfs.PathNodeFsOptions{ClientInodes: true}
	pathFs := pathfs.NewPathNodeFs(finalFs, pathFsOpts)
	fuseOpts := &nodefs.Options{
		// These options are to be compatible with libfuse defaults,
		// making benchmarking easier.
		NegativeTimeout: time.Second,
		AttrTimeout:     time.Second,
		EntryTimeout:    time.Second,
	}
	conn := nodefs.NewFileSystemConnector(pathFs.Root(), fuseOpts)
	var mOpts fuse.MountOptions
	mOpts.AllowOther = false
	// Set values shown in "df -T" and friends
	// First column, "Filesystem"
	mOpts.Options = append(mOpts.Options, "fsname="+args.cipherdir)
	// Second column, "Type", will be shown as "fuse." + Name
	mOpts.Name = "gocryptfs"

	srv, err := fuse.NewServer(conn.RawFS(), args.mountpoint, &mOpts)
	if err != nil {
		fmt.Printf("Mount failed: %v", err)
		os.Exit(ERREXIT_MOUNT)
	}
	srv.SetDebug(args.debug)

	return srv
}

func handleSigint(srv *fuse.Server, mountpoint string) {
	ch := make(chan os.Signal, 1)
	signal.Notify(ch, os.Interrupt)
	signal.Notify(ch, syscall.SIGTERM)
	go func() {
		<-ch
		err := srv.Unmount()
		if err != nil {
			fmt.Print(err)
			cryptfs.Info.Printf("Trying lazy unmount\n")
			cmd := exec.Command("fusermount", "-u", "-z", mountpoint)
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr
			cmd.Run()
		}
		os.Exit(1)
	}()
}
<|endoftext|>


<commit_before>/*
hwaf manages hep-waf based applications and libraries.

 Subcommands:

 init [directory]

 self-update

 configure

 build/make

 install

 bdist
 bdist-rpm
 bdist-dmg
 bdist-deb

 run

 shell

 setup

 add-pkg <pkg-uri> [<pkg-version>]
 checkout|co == alias(add-pkg)
*/
package main

import (
	"fmt"
	"os"

	"github.com/sbinet/go-commander"
	"github.com/sbinet/go-flag"
)

var g_cmd *commander.Commander

func init() {
	g_cmd = &commander.Commander{
		Name: os.Args[0],
		Commands: []*commander.Command{
			hwaf_make_cmd_init(),
			hwaf_make_cmd_setup(),
			hwaf_make_cmd_version(),

			hwaf_make_cmd_waf(),
			hwaf_make_cmd_waf_configure(),
			hwaf_make_cmd_waf_build(),
			hwaf_make_cmd_waf_install(),
			hwaf_make_cmd_waf_clean(),
			hwaf_make_cmd_waf_distclean(),
			hwaf_make_cmd_waf_shell(),
			hwaf_make_cmd_waf_run(),
		},
		Flag: flag.NewFlagSet("hwaf", flag.ExitOnError),
	Commanders: []*commander.Commander{
			hwaf_make_cmd_pkg(),
			hwaf_make_cmd_waf_show(),
			hwaf_make_cmd_pmgr(),
			hwaf_make_cmd_self(),
	 	},
	}
}

func main() {

	if len(os.Args) == 1 && path_exists("wscript") {
		os.Args = append(os.Args, "build", "install")
	}

	// fiddle with environment to allow locate hep-waftools
	hwaf_setup_waf_env()

	err := g_cmd.Flag.Parse(os.Args[1:])
	if err != nil {
		fmt.Printf("**error** %v\n", err)
		os.Exit(1)
	}

	args := g_cmd.Flag.Args()
	err = g_cmd.Run(args)
	if err != nil {
		fmt.Printf("**error** %v\n", err)
		os.Exit(1)
	}

	return
}
<commit_msg>bdist,bdist-rpm: first stab<commit_after>/*
hwaf manages hep-waf based applications and libraries.

 Subcommands:

 init [directory]

 self-update

 configure

 build/make

 install

 bdist
 bdist-rpm
 bdist-dmg
 bdist-deb

 run

 shell

 setup

 add-pkg <pkg-uri> [<pkg-version>]
 checkout|co == alias(add-pkg)
*/
package main

import (
	"fmt"
	"os"

	"github.com/sbinet/go-commander"
	"github.com/sbinet/go-flag"
)

var g_cmd *commander.Commander

func init() {
	g_cmd = &commander.Commander{
		Name: os.Args[0],
		Commands: []*commander.Command{
			hwaf_make_cmd_init(),
			hwaf_make_cmd_setup(),
			hwaf_make_cmd_version(),

			hwaf_make_cmd_waf(),
			hwaf_make_cmd_waf_configure(),
			hwaf_make_cmd_waf_build(),
			hwaf_make_cmd_waf_install(),
			hwaf_make_cmd_waf_clean(),
			hwaf_make_cmd_waf_distclean(),
			hwaf_make_cmd_waf_shell(),
			hwaf_make_cmd_waf_run(),

			hwaf_make_cmd_waf_bdist(),
			hwaf_make_cmd_waf_bdist_rpm(),
		},
		Flag: flag.NewFlagSet("hwaf", flag.ExitOnError),
	Commanders: []*commander.Commander{
			hwaf_make_cmd_pkg(),
			hwaf_make_cmd_waf_show(),
			hwaf_make_cmd_pmgr(),
			hwaf_make_cmd_self(),
	 	},
	}
}

func main() {

	if len(os.Args) == 1 && path_exists("wscript") {
		os.Args = append(os.Args, "build", "install")
	}

	// fiddle with environment to allow locate hep-waftools
	hwaf_setup_waf_env()

	err := g_cmd.Flag.Parse(os.Args[1:])
	if err != nil {
		fmt.Printf("**error** %v\n", err)
		os.Exit(1)
	}

	args := g_cmd.Flag.Args()
	err = g_cmd.Run(args)
	if err != nil {
		fmt.Printf("**error** %v\n", err)
		os.Exit(1)
	}

	return
}
<|endoftext|>