<commit_before>// Package smaz is a pure Go implementation of the smaz algorithm
// (https://github.com/antirez/smaz) for compressing small strings.
package smaz

import (
	"errors"

	"github.com/kjk/smaz/trie"
)

var codeStrings = []string{" ",
	"the", "e", "t", "a", "of", "o", "and", "i", "n", "s", "e ", "r", " th",
	" t", "in", "he", "th", "h", "he ", "to", "\r\n", "l", "s ", "d", " a", "an",
	"er", "c", " o", "d ", "on", " of", "re", "of ", "t ", ", ", "is", "u", "at",
	"   ", "n ", "or", "which", "f", "m", "as", "it", "that", "\n", "was", "en",
	"  ", " w", "es", " an", " i", "\r", "f ", "g", "p", "nd", " s", "nd ", "ed ",
	"w", "ed", "http://", "for", "te", "ing", "y ", "The", " c", "ti", "r ", "his",
	"st", " in", "ar", "nt", ",", " to", "y", "ng", " h", "with", "le", "al", "to ",
	"b", "ou", "be", "were", " b", "se", "o ", "ent", "ha", "ng ", "their", "\"",
	"hi", "from", " f", "in ", "de", "ion", "me", "v", ".", "ve", "all", "re ",
	"ri", "ro", "is ", "co", "f t", "are", "ea", ". ", "her", " m", "er ", " p",
	"es ", "by", "they", "di", "ra", "ic", "not", "s, ", "d t", "at ", "ce", "la",
	"h ", "ne", "as ", "tio", "on ", "n t", "io", "we", " a ", "om", ", a", "s o",
	"ur", "li", "ll", "ch", "had", "this", "e t", "g ", "e\r\n", " wh", "ere",
	" co", "e o", "a ", "us", " d", "ss", "\n\r\n", "\r\n\r", "=\"", " be", " e",
	"s a", "ma", "one", "t t", "or ", "but", "el", "so", "l ", "e s", "s,", "no",
	"ter", " wa", "iv", "ho", "e a", " r", "hat", "s t", "ns", "ch ", "wh", "tr",
	"ut", "/", "have", "ly ", "ta", " ha", " on", "tha", "-", " l", "ati", "en ",
	"pe", " re", "there", "ass", "si", " fo", "wa", "ec", "our", "who", "its", "z",
	"fo", "rs", ">", "ot", "un", "<", "im", "th ", "nc", "ate", "><", "ver", "ad",
	" we", "ly", "ee", " n", "id", " cl", "ac", "il", "</", "rt", " wi", "div",
	"e, ", " it", "whi", " ma", "ge", "x", "e c", "men", ".com",
}

var codes = make([][]byte, len(codeStrings))
var codeTrie = trie.New()

func init() {
	for i, code := range codeStrings {
		codes[i] = []byte(code)
		codeTrie.Put([]byte(code), i)
	}
}

func next(d []byte, n int) ([]byte, []byte) {
	if len(d) < n {
		return d, nil
	}
	return d[:n], d[n:]
}

func appendSrc(dst, src []byte) []byte {
	// We can write a max of 255 continuous verbatim characters, because the
	// length of the continous verbatim section is represented by a single byte.
	for len(src) > 0 {
		chunk, src = next(src, 255)
		if len(chunk) == 1 {
			// 254 is code for a single verbatim byte
			dst = append(dst, byte(254))
			dst = append(dst, chunk[0])
		} else {
			// 255 is code for a verbatim string. It is followed by a byte
			// containing the length of the string.
			dst = append(dst, byte(255))
			dst = append(dst, byte(len(chunk)))
			dst = append(dst, chunk...)
		}
	}
	return dst
}

// Encode returns the encoded form of src. The returned slice may be a sub-slice
// of dst if dst was large enough to hold the entire encoded block. Otherwise,
// a newly allocated slice will be returned. It is valid to pass a nil dst.
func Encode(dst, src []byte) []byte {
	dst = dst[:0]
	root := codeTrie.Root()
	currPos := 0
	nLeft := len(src)
	tmp := src
	code := 0
	prefixLen := 0
	for currPos < nLeft {
		node := root
		for i, c := range tmp {
			next := node.Walk(c)
			if next == nil {
				break
			}
			node = next
			if node.Terminal() {
				prefixLen = i + 1
				code = node.Val()
			}
		}
		if prefixLen == 0 {
			currPos++
			tmp = tmp[1:]
			continue
		}
		dst = appendSrc(dst, src[:currPos])
		dst = append(dst, byte(code))
		src = src[currPos+prefixLen:]
		currPos = 0
		nLeft = len(src)
		tmp = src
		prefixLen = 0
	}
	dst = appendSrc(dst, src)
	return dst
}

// ErrCorrupt reports that the input is invalid.
var ErrCorrupt = errors.New("smaz: corrupt input")

// Decode returns the decoded form of src. The returned slice may be a sub-slice
// of dst if dst was large enough to hold the entire decoded block. Otherwise,
// a newly allocated slice will be returned. It is valid to pass a nil dst.
func Decode(dst, src []byte) ([]byte, error) {
	if cap(dst) < len(src) {
		dst = make([]byte, 0, len(src))
	}
	for len(src) > 0 {
		n := int(src[0])
		switch n {
		case 254: // Verbatim byte
			if len(src) < 2 {
				return nil, ErrCorrupt
			}
			dst = append(dst, src[1])
			src = src[2:]
		case 255: // Verbatim string
			if len(src) < 2 {
				return nil, ErrCorrupt
			}
			n = int(src[1])
			if len(src) < n+2 {
				return nil, ErrCorrupt
			}
			dst = append(dst, src[2:n+2]...)
			src = src[n+2:]
		default: // Look up encoded value
			d := codes[n]
			dst = append(dst, d...)
			src = src[1:]
		}
	}

	return dst, nil
}
<commit_msg>slight compression improvement by simplifying appendSrc()<commit_after>// Package smaz is a pure Go implementation of the smaz algorithm
// (https://github.com/antirez/smaz) for compressing small strings.
package smaz

import (
	"errors"

	"github.com/kjk/smaz/trie"
)

var codeStrings = []string{" ",
	"the", "e", "t", "a", "of", "o", "and", "i", "n", "s", "e ", "r", " th",
	" t", "in", "he", "th", "h", "he ", "to", "\r\n", "l", "s ", "d", " a", "an",
	"er", "c", " o", "d ", "on", " of", "re", "of ", "t ", ", ", "is", "u", "at",
	"   ", "n ", "or", "which", "f", "m", "as", "it", "that", "\n", "was", "en",
	"  ", " w", "es", " an", " i", "\r", "f ", "g", "p", "nd", " s", "nd ", "ed ",
	"w", "ed", "http://", "for", "te", "ing", "y ", "The", " c", "ti", "r ", "his",
	"st", " in", "ar", "nt", ",", " to", "y", "ng", " h", "with", "le", "al", "to ",
	"b", "ou", "be", "were", " b", "se", "o ", "ent", "ha", "ng ", "their", "\"",
	"hi", "from", " f", "in ", "de", "ion", "me", "v", ".", "ve", "all", "re ",
	"ri", "ro", "is ", "co", "f t", "are", "ea", ". ", "her", " m", "er ", " p",
	"es ", "by", "they", "di", "ra", "ic", "not", "s, ", "d t", "at ", "ce", "la",
	"h ", "ne", "as ", "tio", "on ", "n t", "io", "we", " a ", "om", ", a", "s o",
	"ur", "li", "ll", "ch", "had", "this", "e t", "g ", "e\r\n", " wh", "ere",
	" co", "e o", "a ", "us", " d", "ss", "\n\r\n", "\r\n\r", "=\"", " be", " e",
	"s a", "ma", "one", "t t", "or ", "but", "el", "so", "l ", "e s", "s,", "no",
	"ter", " wa", "iv", "ho", "e a", " r", "hat", "s t", "ns", "ch ", "wh", "tr",
	"ut", "/", "have", "ly ", "ta", " ha", " on", "tha", "-", " l", "ati", "en ",
	"pe", " re", "there", "ass", "si", " fo", "wa", "ec", "our", "who", "its", "z",
	"fo", "rs", ">", "ot", "un", "<", "im", "th ", "nc", "ate", "><", "ver", "ad",
	" we", "ly", "ee", " n", "id", " cl", "ac", "il", "</", "rt", " wi", "div",
	"e, ", " it", "whi", " ma", "ge", "x", "e c", "men", ".com",
}

var codes = make([][]byte, len(codeStrings))
var codeTrie = trie.New()

func init() {
	for i, code := range codeStrings {
		codes[i] = []byte(code)
		codeTrie.Put([]byte(code), i)
	}
}

func appendSrc(dst, src []byte) []byte {
	// We can write a max of 255 continuous verbatim characters, because the
	// length of the continous verbatim section is represented by a single byte.
	for len(src) > 0 {
		left := len(src)
		if left == 1 {
			// 254 is code for a single verbatim byte
			dst = append(dst, byte(254))
			return append(dst, src[0])
		}
		toCopy := left
		if left > 255 {
			toCopy = 255
		}
		// 255 is code for a verbatim string. It is followed by a byte
		// containing the length of the string.
		dst = append(dst, byte(255))
		dst = append(dst, byte(toCopy))
		dst = append(dst, src[:toCopy]...)
		src = src[toCopy:]
	}
	return dst
}

// Encode returns the encoded form of src. The returned slice may be a sub-slice
// of dst if dst was large enough to hold the entire encoded block. Otherwise,
// a newly allocated slice will be returned. It is valid to pass a nil dst.
func Encode(dst, src []byte) []byte {
	dst = dst[:0]
	root := codeTrie.Root()
	currPos := 0
	nLeft := len(src)
	tmp := src
	code := 0
	prefixLen := 0
	for currPos < nLeft {
		node := root
		for i, c := range tmp {
			next := node.Walk(c)
			if next == nil {
				break
			}
			node = next
			if node.Terminal() {
				prefixLen = i + 1
				code = node.Val()
			}
		}
		if prefixLen == 0 {
			currPos++
			tmp = tmp[1:]
			continue
		}
		dst = appendSrc(dst, src[:currPos])
		dst = append(dst, byte(code))
		src = src[currPos+prefixLen:]
		currPos = 0
		nLeft = len(src)
		tmp = src
		prefixLen = 0
	}
	dst = appendSrc(dst, src)
	return dst
}

// ErrCorrupt reports that the input is invalid.
var ErrCorrupt = errors.New("smaz: corrupt input")

// Decode returns the decoded form of src. The returned slice may be a sub-slice
// of dst if dst was large enough to hold the entire decoded block. Otherwise,
// a newly allocated slice will be returned. It is valid to pass a nil dst.
func Decode(dst, src []byte) ([]byte, error) {
	if cap(dst) < len(src) {
		dst = make([]byte, 0, len(src))
	}
	for len(src) > 0 {
		n := int(src[0])
		switch n {
		case 254: // Verbatim byte
			if len(src) < 2 {
				return nil, ErrCorrupt
			}
			dst = append(dst, src[1])
			src = src[2:]
		case 255: // Verbatim string
			if len(src) < 2 {
				return nil, ErrCorrupt
			}
			n = int(src[1])
			if len(src) < n+2 {
				return nil, ErrCorrupt
			}
			dst = append(dst, src[2:n+2]...)
			src = src[n+2:]
		default: // Look up encoded value
			d := codes[n]
			dst = append(dst, d...)
			src = src[1:]
		}
	}

	return dst, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"go/build"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"sourcegraph.com/sourcegraph/srclib"
	"sourcegraph.com/sourcegraph/srclib/unit"
)

func init() {
	_, err := parser.AddCommand("scan",
		"scan for Go packages",
		"Scan the directory tree rooted at the current directory for Go packages.",
		&scanCmd,
	)
	if err != nil {
		log.Fatal(err)
	}
}

type ScanCmd struct {
	Repo   string `long:"repo" description:"repository URI" value-name:"URI"`
	Subdir string `long:"subdir" description:"subdirectory in repository" value-name:"DIR"`
}

var scanCmd ScanCmd

func (c *ScanCmd) Execute(args []string) error {
	if c.Repo == "" && os.Getenv("IN_DOCKER_CONTAINER") != "" {
		log.Println("Warning: no --repo specified, and tool is running in a Docker container (i.e., without awareness of host's GOPATH). Go import paths in source units produced by the scanner may be inaccurate. To fix this, ensure that the --repo URI is specified. Report this issue if you are seeing it unexpectedly.")
	}

	if err := json.NewDecoder(os.Stdin).Decode(&config); err != nil {
		return err
	}
	if err := os.Stdin.Close(); err != nil {
		return err
	}

	// Automatically detect vendored dirs (check for vendor/src and
	// Godeps/_workspace/src) and set up GOPATH pointing to them if
	// they exist.
	var setAutoGOPATH bool
	if config.GOPATH == "" {
		vendorDirs := []string{"vendor", "Godeps/_workspace"}
		var foundGOPATHs []string
		for _, vdir := range vendorDirs {
			if fi, err := os.Stat(filepath.Join(cwd, vdir, "src")); err == nil && fi.Mode().IsDir() {
				foundGOPATHs = append(foundGOPATHs, vdir)
				setAutoGOPATH = true
				log.Printf("Adding %s to GOPATH (auto-detected Go vendored dependencies source dir %s). If you don't want this, make a Srcfile with a GOPATH property set to something other than the empty string.", vdir, filepath.Join(vdir, "src"))
			}
		}
		config.GOPATH = strings.Join(foundGOPATHs, string(filepath.ListSeparator))
	}

	if err := config.apply(); err != nil {
		return err
	}

	cwd, err := filepath.EvalSymlinks(getCWD())
	if err != nil {
		return err
	}
	scanDir := cwd
	if !isInGopath(scanDir) {
		scanDir = filepath.Join(cwd, srclibGopath, "src", filepath.FromSlash(c.Repo))
		buildContext.GOPATH = filepath.Join(cwd, srclibGopath) + string(os.PathListSeparator) + buildContext.GOPATH

		os.RemoveAll(srclibGopath) // ignore error
		if err := os.MkdirAll(filepath.Dir(scanDir), 0777); err != nil {
			return err
		}
		rel, err := filepath.Rel(filepath.Dir(scanDir), cwd)
		if err != nil {
			return err
		}
		if err := os.Symlink(rel, scanDir); err != nil {
			return err
		}
	}

	units, err := scan(scanDir)
	if err != nil {
		return err
	}

	if len(config.PkgPatterns) != 0 {
		matchers := make([]func(name string) bool, len(config.PkgPatterns))
		for i, pattern := range config.PkgPatterns {
			matchers[i] = matchPattern(pattern)
		}

		var filteredUnits []*unit.SourceUnit
		for _, unit := range units {
			for _, m := range matchers {
				if m(unit.Name) {
					filteredUnits = append(filteredUnits, unit)
					break
				}
			}
		}
		units = filteredUnits
	}

	// Fix up import paths to be consistent when running as a program and as
	// a Docker container. But if a GOROOT is set, then we probably want import
	// paths to not contain the repo, so only do this if there's no GOROOT set
	// in the Srcfile.
	if os.Getenv("IN_DOCKER_CONTAINER") != "" && config.GOROOT == "" {
		for _, u := range units {
			pkg := u.Data.(*build.Package)
			pkg.ImportPath = filepath.Join(c.Repo, c.Subdir, pkg.Dir)
			u.Name = pkg.ImportPath
		}
	}

	// Make vendored dep unit names (package import paths) relative to
	// vendored src dir, not to top-level dir.
	if config.GOPATH != "" {
		dirs := filepath.SplitList(config.GOPATH)
		for _, dir := range dirs {
			relDir, err := filepath.Rel(cwd, dir)
			if err != nil {
				return err
			}
			srcDir := filepath.Join(relDir, "src")
			for _, u := range units {
				pkg := u.Data.(*build.Package)
				if strings.HasPrefix(pkg.Dir, srcDir) {
					relImport, err := filepath.Rel(srcDir, pkg.Dir)
					if err != nil {
						return err
					}
					pkg.ImportPath = relImport
					u.Name = pkg.ImportPath
				}
			}
		}
	}

	// make files relative to repository root
	for _, u := range units {
		pkgSubdir := filepath.Join(c.Subdir, u.Data.(*build.Package).Dir)
		for i, f := range u.Files {
			u.Files[i] = filepath.Join(pkgSubdir, f)
		}
	}

	// If we automatically set the GOPATH based on the presence of
	// vendor dirs, then we need to pass the GOPATH to the units
	// because it is not persisted in the Srcfile. Otherwise the other
	// tools would never see the auto-set GOPATH.
	if setAutoGOPATH {
		for _, u := range units {
			if u.Config == nil {
				u.Config = map[string]interface{}{}
			}

			dirs := filepath.SplitList(config.GOPATH)
			for i, dir := range dirs {
				relDir, err := filepath.Rel(cwd, dir)
				if err != nil {
					return err
				}
				dirs[i] = relDir
			}
			u.Config["GOPATH"] = strings.Join(dirs, string(filepath.ListSeparator))
		}
	}

	b, err := json.MarshalIndent(units, "", "  ")
	if err != nil {
		return err
	}
	if _, err := os.Stdout.Write(b); err != nil {
		return err
	}
	return nil
}

func isInGopath(path string) bool {
	for _, gopath := range filepath.SplitList(buildContext.GOPATH) {
		if strings.HasPrefix(evalSymlinks(path), filepath.Join(evalSymlinks(gopath), "src")) {
			return true
		}
	}
	return false
}

func scan(scanDir string) ([]*unit.SourceUnit, error) {
	// TODO(sqs): include xtest, but we'll have to make them have a distinctly
	// namespaced def path from the non-xtest pkg.

	pkgs, err := scanForPackages(scanDir)
	if err != nil {
		return nil, err
	}

	var units []*unit.SourceUnit
	for _, pkg := range pkgs {
		// Collect all files
		var files []string
		files = append(files, pkg.GoFiles...)
		files = append(files, pkg.CgoFiles...)
		files = append(files, pkg.IgnoredGoFiles...)
		files = append(files, pkg.CFiles...)
		files = append(files, pkg.CXXFiles...)
		files = append(files, pkg.MFiles...)
		files = append(files, pkg.HFiles...)
		files = append(files, pkg.SFiles...)
		files = append(files, pkg.SwigFiles...)
		files = append(files, pkg.SwigCXXFiles...)
		files = append(files, pkg.SysoFiles...)
		files = append(files, pkg.TestGoFiles...)
		files = append(files, pkg.XTestGoFiles...)

		// Collect all imports. We use a map to remove duplicates.
		var imports []string
		imports = append(imports, pkg.Imports...)
		imports = append(imports, pkg.TestImports...)
		imports = append(imports, pkg.XTestImports...)
		imports = uniq(imports)
		sort.Strings(imports)

		// Create appropriate type for (unit).SourceUnit
		deps := make([]interface{}, len(imports))
		for i, imp := range imports {
			deps[i] = imp
		}

		// Make all dirs relative to the current one.
		makeRel := func(dirs ...*string) (err error) {
			for _, d := range dirs {
				if *d == "" {
					continue
				}
				*d, err = filepath.Rel(scanDir, *d)
				if err != nil {
					return err
				}
			}
			return nil
		}
		if err := makeRel(&pkg.Dir, &pkg.BinDir, &pkg.ConflictDir); err != nil {
			return nil, err
		}

		// Root differs depending on the system, so it's hard to compare results
		// across environments (when running as a program). Clear it so we can
		// compare results in tests more easily.
		pkg.Root = ""

		units = append(units, &unit.SourceUnit{
			Name:         pkg.ImportPath,
			Type:         "GoPackage",
			Dir:          pkg.Dir,
			Files:        files,
			Data:         pkg,
			Dependencies: deps,
			Ops:          map[string]*srclib.ToolRef{"depresolve": nil, "graph": nil},
		})
	}

	return units, nil
}

func scanForPackages(dir string) ([]*build.Package, error) {
	var pkgs []*build.Package

	pkg, err := buildContext.ImportDir(dir, 0)
	if _, isNoGoError := err.(*build.NoGoError); err != nil && !isNoGoError {
		return nil, err
	}
	if err == nil {
		pkgs = append(pkgs, pkg)
	}

	infos, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, err
	}
	for _, info := range infos {
		name := info.Name()
		fullPath := filepath.Join(dir, name)
		if info.IsDir() && ((name[0] != '.' && name[0] != '_' && name != "testdata") || (strings.HasSuffix(fullPath, "Godeps/_workspace") && !config.SkipGodeps)) {
			subPkgs, err := scanForPackages(fullPath)
			if err != nil {
				return nil, err
			}
			pkgs = append(pkgs, subPkgs...)
		}
	}

	return pkgs, nil
}

// matchPattern(pattern)(name) reports whether
// name matches pattern.  Pattern is a limited glob
// pattern in which '...' means 'any string' and there
// is no other special syntax.
func matchPattern(pattern string) func(name string) bool {
	re := regexp.QuoteMeta(pattern)
	re = strings.Replace(re, `\.\.\.`, `.*`, -1)
	// Special case: foo/... matches foo too.
	if strings.HasSuffix(re, `/.*`) {
		re = re[:len(re)-len(`/.*`)] + `(/.*)?`
	}
	reg := regexp.MustCompile(`^` + re + `$`)
	return func(name string) bool {
		return reg.MatchString(name)
	}
}
<commit_msg>don't store pkg.BinDir and pkg.ConflictDir<commit_after>package main

import (
	"encoding/json"
	"go/build"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"sourcegraph.com/sourcegraph/srclib"
	"sourcegraph.com/sourcegraph/srclib/unit"
)

func init() {
	_, err := parser.AddCommand("scan",
		"scan for Go packages",
		"Scan the directory tree rooted at the current directory for Go packages.",
		&scanCmd,
	)
	if err != nil {
		log.Fatal(err)
	}
}

type ScanCmd struct {
	Repo   string `long:"repo" description:"repository URI" value-name:"URI"`
	Subdir string `long:"subdir" description:"subdirectory in repository" value-name:"DIR"`
}

var scanCmd ScanCmd

func (c *ScanCmd) Execute(args []string) error {
	if c.Repo == "" && os.Getenv("IN_DOCKER_CONTAINER") != "" {
		log.Println("Warning: no --repo specified, and tool is running in a Docker container (i.e., without awareness of host's GOPATH). Go import paths in source units produced by the scanner may be inaccurate. To fix this, ensure that the --repo URI is specified. Report this issue if you are seeing it unexpectedly.")
	}

	if err := json.NewDecoder(os.Stdin).Decode(&config); err != nil {
		return err
	}
	if err := os.Stdin.Close(); err != nil {
		return err
	}

	// Automatically detect vendored dirs (check for vendor/src and
	// Godeps/_workspace/src) and set up GOPATH pointing to them if
	// they exist.
	var setAutoGOPATH bool
	if config.GOPATH == "" {
		vendorDirs := []string{"vendor", "Godeps/_workspace"}
		var foundGOPATHs []string
		for _, vdir := range vendorDirs {
			if fi, err := os.Stat(filepath.Join(cwd, vdir, "src")); err == nil && fi.Mode().IsDir() {
				foundGOPATHs = append(foundGOPATHs, vdir)
				setAutoGOPATH = true
				log.Printf("Adding %s to GOPATH (auto-detected Go vendored dependencies source dir %s). If you don't want this, make a Srcfile with a GOPATH property set to something other than the empty string.", vdir, filepath.Join(vdir, "src"))
			}
		}
		config.GOPATH = strings.Join(foundGOPATHs, string(filepath.ListSeparator))
	}

	if err := config.apply(); err != nil {
		return err
	}

	cwd, err := filepath.EvalSymlinks(getCWD())
	if err != nil {
		return err
	}
	scanDir := cwd
	if !isInGopath(scanDir) {
		scanDir = filepath.Join(cwd, srclibGopath, "src", filepath.FromSlash(c.Repo))
		buildContext.GOPATH = filepath.Join(cwd, srclibGopath) + string(os.PathListSeparator) + buildContext.GOPATH

		os.RemoveAll(srclibGopath) // ignore error
		if err := os.MkdirAll(filepath.Dir(scanDir), 0777); err != nil {
			return err
		}
		rel, err := filepath.Rel(filepath.Dir(scanDir), cwd)
		if err != nil {
			return err
		}
		if err := os.Symlink(rel, scanDir); err != nil {
			return err
		}
	}

	units, err := scan(scanDir)
	if err != nil {
		return err
	}

	if len(config.PkgPatterns) != 0 {
		matchers := make([]func(name string) bool, len(config.PkgPatterns))
		for i, pattern := range config.PkgPatterns {
			matchers[i] = matchPattern(pattern)
		}

		var filteredUnits []*unit.SourceUnit
		for _, unit := range units {
			for _, m := range matchers {
				if m(unit.Name) {
					filteredUnits = append(filteredUnits, unit)
					break
				}
			}
		}
		units = filteredUnits
	}

	// Fix up import paths to be consistent when running as a program and as
	// a Docker container. But if a GOROOT is set, then we probably want import
	// paths to not contain the repo, so only do this if there's no GOROOT set
	// in the Srcfile.
	if os.Getenv("IN_DOCKER_CONTAINER") != "" && config.GOROOT == "" {
		for _, u := range units {
			pkg := u.Data.(*build.Package)
			pkg.ImportPath = filepath.Join(c.Repo, c.Subdir, pkg.Dir)
			u.Name = pkg.ImportPath
		}
	}

	// Make vendored dep unit names (package import paths) relative to
	// vendored src dir, not to top-level dir.
	if config.GOPATH != "" {
		dirs := filepath.SplitList(config.GOPATH)
		for _, dir := range dirs {
			relDir, err := filepath.Rel(cwd, dir)
			if err != nil {
				return err
			}
			srcDir := filepath.Join(relDir, "src")
			for _, u := range units {
				pkg := u.Data.(*build.Package)
				if strings.HasPrefix(pkg.Dir, srcDir) {
					relImport, err := filepath.Rel(srcDir, pkg.Dir)
					if err != nil {
						return err
					}
					pkg.ImportPath = relImport
					u.Name = pkg.ImportPath
				}
			}
		}
	}

	// make files relative to repository root
	for _, u := range units {
		pkgSubdir := filepath.Join(c.Subdir, u.Data.(*build.Package).Dir)
		for i, f := range u.Files {
			u.Files[i] = filepath.Join(pkgSubdir, f)
		}
	}

	// If we automatically set the GOPATH based on the presence of
	// vendor dirs, then we need to pass the GOPATH to the units
	// because it is not persisted in the Srcfile. Otherwise the other
	// tools would never see the auto-set GOPATH.
	if setAutoGOPATH {
		for _, u := range units {
			if u.Config == nil {
				u.Config = map[string]interface{}{}
			}

			dirs := filepath.SplitList(config.GOPATH)
			for i, dir := range dirs {
				relDir, err := filepath.Rel(cwd, dir)
				if err != nil {
					return err
				}
				dirs[i] = relDir
			}
			u.Config["GOPATH"] = strings.Join(dirs, string(filepath.ListSeparator))
		}
	}

	b, err := json.MarshalIndent(units, "", "  ")
	if err != nil {
		return err
	}
	if _, err := os.Stdout.Write(b); err != nil {
		return err
	}
	return nil
}

func isInGopath(path string) bool {
	for _, gopath := range filepath.SplitList(buildContext.GOPATH) {
		if strings.HasPrefix(evalSymlinks(path), filepath.Join(evalSymlinks(gopath), "src")) {
			return true
		}
	}
	return false
}

func scan(scanDir string) ([]*unit.SourceUnit, error) {
	// TODO(sqs): include xtest, but we'll have to make them have a distinctly
	// namespaced def path from the non-xtest pkg.

	pkgs, err := scanForPackages(scanDir)
	if err != nil {
		return nil, err
	}

	var units []*unit.SourceUnit
	for _, pkg := range pkgs {
		// Collect all files
		var files []string
		files = append(files, pkg.GoFiles...)
		files = append(files, pkg.CgoFiles...)
		files = append(files, pkg.IgnoredGoFiles...)
		files = append(files, pkg.CFiles...)
		files = append(files, pkg.CXXFiles...)
		files = append(files, pkg.MFiles...)
		files = append(files, pkg.HFiles...)
		files = append(files, pkg.SFiles...)
		files = append(files, pkg.SwigFiles...)
		files = append(files, pkg.SwigCXXFiles...)
		files = append(files, pkg.SysoFiles...)
		files = append(files, pkg.TestGoFiles...)
		files = append(files, pkg.XTestGoFiles...)

		// Collect all imports. We use a map to remove duplicates.
		var imports []string
		imports = append(imports, pkg.Imports...)
		imports = append(imports, pkg.TestImports...)
		imports = append(imports, pkg.XTestImports...)
		imports = uniq(imports)
		sort.Strings(imports)

		// Create appropriate type for (unit).SourceUnit
		deps := make([]interface{}, len(imports))
		for i, imp := range imports {
			deps[i] = imp
		}

		pkg.Dir, err = filepath.Rel(scanDir, pkg.Dir)
		if err != nil {
			return nil, err
		}
		pkg.BinDir = ""
		pkg.ConflictDir = ""

		// Root differs depending on the system, so it's hard to compare results
		// across environments (when running as a program). Clear it so we can
		// compare results in tests more easily.
		pkg.Root = ""

		units = append(units, &unit.SourceUnit{
			Name:         pkg.ImportPath,
			Type:         "GoPackage",
			Dir:          pkg.Dir,
			Files:        files,
			Data:         pkg,
			Dependencies: deps,
			Ops:          map[string]*srclib.ToolRef{"depresolve": nil, "graph": nil},
		})
	}

	return units, nil
}

func scanForPackages(dir string) ([]*build.Package, error) {
	var pkgs []*build.Package

	pkg, err := buildContext.ImportDir(dir, 0)
	if _, isNoGoError := err.(*build.NoGoError); err != nil && !isNoGoError {
		return nil, err
	}
	if err == nil {
		pkgs = append(pkgs, pkg)
	}

	infos, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, err
	}
	for _, info := range infos {
		name := info.Name()
		fullPath := filepath.Join(dir, name)
		if info.IsDir() && ((name[0] != '.' && name[0] != '_' && name != "testdata") || (strings.HasSuffix(fullPath, "Godeps/_workspace") && !config.SkipGodeps)) {
			subPkgs, err := scanForPackages(fullPath)
			if err != nil {
				return nil, err
			}
			pkgs = append(pkgs, subPkgs...)
		}
	}

	return pkgs, nil
}

// matchPattern(pattern)(name) reports whether
// name matches pattern.  Pattern is a limited glob
// pattern in which '...' means 'any string' and there
// is no other special syntax.
func matchPattern(pattern string) func(name string) bool {
	re := regexp.QuoteMeta(pattern)
	re = strings.Replace(re, `\.\.\.`, `.*`, -1)
	// Special case: foo/... matches foo too.
	if strings.HasSuffix(re, `/.*`) {
		re = re[:len(re)-len(`/.*`)] + `(/.*)?`
	}
	reg := regexp.MustCompile(`^` + re + `$`)
	return func(name string) bool {
		return reg.MatchString(name)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/hmac"
	"crypto/sha256"
	"fmt"
	"net"
	"sort"
	"time"

	"github.com/briandowns/spinner"
	"github.com/miekg/dns"
)

var DSP = []struct {
	Qtype   uint16
	Entries []string
}{
	{dns.TypeSOA, []string{""}},
	{dns.TypeNS, []string{""}},
	{dns.TypeDS, []string{""}},
	{dns.TypeDNSKEY, []string{""}},
	{dns.TypeMX, []string{""}},
	{dns.TypeTXT, []string{"", "_amazonses.", "_dmarc.", "api.", "api._domainkey.", "cm._domainkey.", "default.", "default._domainkey.", "dk._domainkey.", "googleapps._domainkey.", "mail._domainkey.", "mailjet.", "mesmtp._domainkey."}},
	{dns.TypeA, []string{"", "_dmarc.", "admin.", "administration.", "ads.", "adserver.", "alerts.", "alpha.", "analytics.", "ap.", "apache.", "app.", "apps.", "appserver.", "auth.", "autodiscover.", "backup.", "beta.", "blog.", "calendar.", "cdn.", "cdn.", "chat.", "citrix.", "clients.", "cms.", "confluence.", "corp.", "corp.", "crs.", "cvs.", "database.", "db.", "demo.", "dev.", "devel.", "development.", "devsql.", "dhcp.", "direct.", "dmz.", "dns.", "dns0.", "dns00.", "dns01.", "dns010.", "dns02.", "dns03.", "dns04.", "dns05.", "dns06.", "dns07.", "dns08.", "dns09.", "dns1.", "dns10.", "dns2.", "dns3.", "dns4.", "dns5.", "dns6.", "dns7.", "dns8.", "dns9.", "download.", "emergency.", "en.", "enterpriseenrollment.", "enterpriseregistration.", "erp.", "eshop.", "exchange.", "f5.", "fileserver.", "firewall.", "forum.", "ftp.", "gateway.", "gc.", "git.", "gw.", "help.", "home.", "host.", "http.", "id.", "images.", "imap.", "imap4.", "info.", "internal.", "internet.", "intranet.", "ipv6.", "jenkins.", "jira.", "lab.", "lb0.", "lb00.", "lb01.", "lb010.", "lb02.", "lb03.", "lb04.", "lb05.", "lb06.", "lb07.", "lb08.", "lb09.", "lb1.", "lb10.", "lb2.", "lb3.", "lb4.", "lb5.", "lb6.", "lb7.", "lb8.", "lb9.", "ldap.", "linux.", "local.", "log.", "log.", "lyncdiscover.", "mail.", "mail0.", "mail00.", "mail01.", "mail010.", "mail02.", "mail03.", "mail04.", "mail05.", "mail06.", "mail07.", "mail08.", "mail09.", "mail1.", "mail10.", "mail2.", "mail3.", "mail4.", "mail5.", "mail6.", "mail7.", "mail8.", "mail9.", "mailgate.", "mailing.", "main.", "manage.", "mgmt.", "mirror.", "mobile.", "monitor.", "msoid.", "mssql.", "mta.", "mx.", "mx0.", "mx00.", "mx01.", "mx010.", "mx02.", "mx03.", "mx04.", "mx05.", "mx06.", "mx07.", "mx08.", "mx09.", "mx1.", "mx10.", "mx2.", "mx3.", "mx4.", "mx5.", "mx6.", "mx7.", "mx8.", "mx9.", "mysql-master.", "mysql-slave.", "mysql.", "new.", "news.", "noc.", "ns.", "ns0.", "ns00.", "ns01.", "ns010.", "ns02.", "ns03.", "ns04.", "ns05.", "ns06.", "ns07.", "ns08.", "ns09.", "ns1.", "ns10.", "ns2.", "ns3.", "ns4.", "ns5.", "ns6.", "ns7.", "ns8.", "ns9.", "ntp.", "office.", "ops.", "oracle.", "owa.", "pbx.", "piwik.", "pop.", "pop3.", "preprod.", "prod.", "production.", "projects.", "rdp.", "remote.", "robot.", "safe.", "secure.", "server.", "shop.", "sip.", "smtp.", "sql.", "sql0.", "sql00.", "sql01.", "sql010.", "sql02.", "sql03.", "sql04.", "sql05.", "sql06.", "sql07.", "sql08.", "sql09.", "sql1.", "sql10.", "sql2.", "sql3.", "sql4.", "sql5.", "sql6.", "sql7.", "sql8.", "sql9.", "squid.", "ssh.", "ssl.", "stage.", "staging.", "stats.", "support.", "svn.", "syslog.", "test.", "testing.", "upload.", "val.", "vm.", "vnc.", "voip.", "vpn.", "web0.", "web00.", "web01.", "web010", "web02.", "web03.", "web04.", "web05.", "web06.", "web07.", "web08.", "web09.", "web1.", "web10.", "web2.", "web3.", "web4.", "web5.", "web6.", "web7.", "web8.", "web9.", "webmail.", "webshop.", "whois.", "wiki.", "www.", "xml."}},
	{dns.TypeSRV, []string{"_afpovertcp._tcp.", "_autodiscover._tcp.", "_caldav._tcp.", "_client._smtp.", "_gc._tcp.", "_h323cs._tcp.", "_h323cs._udp.", "_h323ls._tcp.", "_h323ls._udp.", "_h323rs._tcp.", "_h323rs._tcp.", "_http._tcp.", "_iax.udp.", "_imap._tcp.", "_imaps._tcp.", "_jabber-client._tcp.", "_jabber._tcp.", "_kerberos-adm._tcp.", "_kerberos._tcp.", "_kerberos._tcp.dc._msdcs.", "_kerberos._udp.", "_kpasswd._tcp.", "_kpasswd._udp.", "_ldap._tcp.", "_ldap._tcp.dc._msdcs.", "_ldap._tcp.gc._msdcs.", "_ldap._tcp.pdc._msdcs.", "_msdcs.", "_mysqlsrv._tcp.", "_ntp._udp.", "_pop3._tcp.", "_pop3s._tcp.", "_sip._tcp.", "_sip._tls.", "_sip._udp.", "_sipfederationtls._tcp.", "_sipinternaltls._tcp.", "_sips._tcp.", "_smtp._tcp.", "_ssh._tcp.", "_stun._tcp.", "_stun._udp.", "_tcp.", "_tls.", "_udp.", "_vlmcs._tcp.", "_vlmcs._udp.", "_wpad._tcp.", "_xmpp-client._tcp.", "_xmpp-server._tcp.", "_zip._tls"}},
}

type ScanResponse struct {
	RR  []dns.RR
	NS  string
	Rtt time.Duration
}

func zoneTransfer(domain, server string) []string {
	var records []string
	t := new(dns.Transfer)
	req := prepMsg()
	req.Question[0] = dns.Question{dns.Fqdn(domain), dns.TypeAXFR, dns.ClassINET}
	q, err := t.In(req, net.JoinHostPort(server, "53"))
	if err != nil {
		return records
		//	fmt.Println("error", err)
	}
	for res := range q {
		if res.Error != nil {
			break
		}
		for _, rr := range res.RR {
			records = append(records, rr.String())
		}
	}
	sort.Strings(records)
	return records
}

func domainscan(domain string) {
	var ips []net.IP
	respc := make(chan ScanResponse, 100)

	servers, _ := findNS(dns.Fqdn(domain))
	for _, server := range servers {
		ips = append(ips, server.IP...)
	}
	s := spinner.New(spinner.CharSets[14], 100*time.Millisecond)

	scanEntries := 0
	for _, src := range DSP {
		scanEntries = scanEntries + len(src.Entries)
	}

	*flagQPS = *flagQPS * len(servers)

	for _, ip := range ips {
		res := zoneTransfer(domain, ip.String())
		if len(res) > 0 {
			zt := ""
			for _, rr := range res {
				zt = zt + fmt.Sprintln(rr)
			}
			fmt.Println(zt)
			// only print one. Further scanning not needed
			return
			// TODO compare hashes
			//	fmt.Printf("%x\n", Hash("key", []byte(zt)))
		} else {
			fmt.Printf("%s ", ip.String())
		}
	}
	fmt.Println(": AXFR denied")

	s.Suffix = " Scanning... will take approx " + fmt.Sprintf("%#v seconds", scanEntries/(*flagQPS))
	s.Start()

	res, _, _ := queryRRset(dns.Fqdn("*."+domain), dns.TypeA, ips[0].String(), true)
	// TODO handle * record correctly
	if len(res) != 0 {
		s.Stop()
		fmt.Println()
		for _, rr := range res {
			fmt.Println(rr.String())
		}
		return
	}

	go func() {
		i := -1
		limiter := time.Tick(time.Millisecond * time.Duration(1000/(*flagQPS)))
		for _, src := range DSP {
			qtype := src.Qtype
			for _, entry := range src.Entries {
				i++
				<-limiter
				var rrs []dns.RR
				// roundrobin
				ns := ips[i%len(ips)]
				if qtype == dns.TypeA {
					res, _, err := query(dns.Fqdn(entry+domain), dns.TypeA, ns.String(), true)
					if err != nil {
						//fmt.Println(err)
					} else {
						a := extractRR(res.Answer, dns.TypeA)
						cname := extractRR(res.Answer, dns.TypeCNAME)
						rrs = append(rrs, a...)
						rrs = append(rrs, cname...)
					}
					res2, rtt, err := queryRRset(dns.Fqdn(entry+domain), dns.TypeAAAA, ns.String(), true)
					if err != nil && len(res2) != 0 {
						//fmt.Println(err)
					}
					rrs = append(rrs, res2...)
					respc <- ScanResponse{RR: rrs, NS: ns.String(), Rtt: rtt}
					continue
				}
				res, rtt, err := queryRRset(dns.Fqdn(entry+domain), qtype, ns.String(), true)
				if err != nil && len(res) != 0 {
					//fmt.Println(err)
				}
				respc <- ScanResponse{RR: res, NS: ns.String(), Rtt: rtt}
			}
		}
	}()

	var responses []string
	i := 0
	for resp := range respc {
		if len(resp.RR) > 0 {
			for _, rr := range resp.RR {
				responses = append(responses, rr.String())
			}
		}
		if i == scanEntries-1 {
			break
		}
		i++
	}

	s.Stop()

	sort.Strings(responses)
	for _, response := range responses {
		fmt.Println(response)
	}
}

func Hash(tag string, data []byte) []byte {
	h := hmac.New(sha256.New, []byte(tag))
	h.Write(data)
	return h.Sum(nil)
}
<commit_msg>Add concurrency for scanning. Query every NS concurrently<commit_after>package main

import (
	"crypto/hmac"
	"crypto/sha256"
	"fmt"
	"net"
	"sort"
	"time"

	"github.com/briandowns/spinner"
	"github.com/miekg/dns"
)

var DSP = []struct {
	Qtype   uint16
	Entries []string
}{
	{dns.TypeSOA, []string{""}},
	{dns.TypeNS, []string{""}},
	{dns.TypeDS, []string{""}},
	{dns.TypeDNSKEY, []string{""}},
	{dns.TypeMX, []string{""}},
	{dns.TypeTXT, []string{"", "_amazonses.", "_dmarc.", "api.", "api._domainkey.", "cm._domainkey.", "default.", "default._domainkey.", "dk._domainkey.", "googleapps._domainkey.", "mail._domainkey.", "mailjet.", "mesmtp._domainkey."}},
	{dns.TypeA, []string{"", "_dmarc.", "admin.", "administration.", "ads.", "adserver.", "alerts.", "alpha.", "analytics.", "ap.", "apache.", "app.", "apps.", "appserver.", "auth.", "autodiscover.", "backup.", "beta.", "blog.", "calendar.", "cdn.", "cdn.", "chat.", "citrix.", "clients.", "cms.", "confluence.", "corp.", "corp.", "crs.", "cvs.", "database.", "db.", "demo.", "dev.", "devel.", "development.", "devsql.", "dhcp.", "direct.", "dmz.", "dns.", "dns0.", "dns00.", "dns01.", "dns010.", "dns02.", "dns03.", "dns04.", "dns05.", "dns06.", "dns07.", "dns08.", "dns09.", "dns1.", "dns10.", "dns2.", "dns3.", "dns4.", "dns5.", "dns6.", "dns7.", "dns8.", "dns9.", "download.", "emergency.", "en.", "enterpriseenrollment.", "enterpriseregistration.", "erp.", "eshop.", "exchange.", "f5.", "fileserver.", "firewall.", "forum.", "ftp.", "gateway.", "gc.", "git.", "gw.", "help.", "home.", "host.", "http.", "id.", "images.", "imap.", "imap4.", "info.", "internal.", "internet.", "intranet.", "ipv6.", "jenkins.", "jira.", "lab.", "lb0.", "lb00.", "lb01.", "lb010.", "lb02.", "lb03.", "lb04.", "lb05.", "lb06.", "lb07.", "lb08.", "lb09.", "lb1.", "lb10.", "lb2.", "lb3.", "lb4.", "lb5.", "lb6.", "lb7.", "lb8.", "lb9.", "ldap.", "linux.", "local.", "log.", "log.", "lyncdiscover.", "mail.", "mail0.", "mail00.", "mail01.", "mail010.", "mail02.", "mail03.", "mail04.", "mail05.", "mail06.", "mail07.", "mail08.", "mail09.", "mail1.", "mail10.", "mail2.", "mail3.", "mail4.", "mail5.", "mail6.", "mail7.", "mail8.", "mail9.", "mailgate.", "mailing.", "main.", "manage.", "mgmt.", "mirror.", "mobile.", "monitor.", "msoid.", "mssql.", "mta.", "mx.", "mx0.", "mx00.", "mx01.", "mx010.", "mx02.", "mx03.", "mx04.", "mx05.", "mx06.", "mx07.", "mx08.", "mx09.", "mx1.", "mx10.", "mx2.", "mx3.", "mx4.", "mx5.", "mx6.", "mx7.", "mx8.", "mx9.", "mysql-master.", "mysql-slave.", "mysql.", "new.", "news.", "noc.", "ns.", "ns0.", "ns00.", "ns01.", "ns010.", "ns02.", "ns03.", "ns04.", "ns05.", "ns06.", "ns07.", "ns08.", "ns09.", "ns1.", "ns10.", "ns2.", "ns3.", "ns4.", "ns5.", "ns6.", "ns7.", "ns8.", "ns9.", "ntp.", "office.", "ops.", "oracle.", "owa.", "pbx.", "piwik.", "pop.", "pop3.", "preprod.", "prod.", "production.", "projects.", "rdp.", "remote.", "robot.", "safe.", "secure.", "server.", "shop.", "sip.", "smtp.", "sql.", "sql0.", "sql00.", "sql01.", "sql010.", "sql02.", "sql03.", "sql04.", "sql05.", "sql06.", "sql07.", "sql08.", "sql09.", "sql1.", "sql10.", "sql2.", "sql3.", "sql4.", "sql5.", "sql6.", "sql7.", "sql8.", "sql9.", "squid.", "ssh.", "ssl.", "stage.", "staging.", "stats.", "support.", "svn.", "syslog.", "test.", "testing.", "upload.", "val.", "vm.", "vnc.", "voip.", "vpn.", "web0.", "web00.", "web01.", "web010", "web02.", "web03.", "web04.", "web05.", "web06.", "web07.", "web08.", "web09.", "web1.", "web10.", "web2.", "web3.", "web4.", "web5.", "web6.", "web7.", "web8.", "web9.", "webmail.", "webshop.", "whois.", "wiki.", "www.", "xml."}},
	{dns.TypeSRV, []string{"_afpovertcp._tcp.", "_autodiscover._tcp.", "_caldav._tcp.", "_client._smtp.", "_gc._tcp.", "_h323cs._tcp.", "_h323cs._udp.", "_h323ls._tcp.", "_h323ls._udp.", "_h323rs._tcp.", "_h323rs._tcp.", "_http._tcp.", "_iax.udp.", "_imap._tcp.", "_imaps._tcp.", "_jabber-client._tcp.", "_jabber._tcp.", "_kerberos-adm._tcp.", "_kerberos._tcp.", "_kerberos._tcp.dc._msdcs.", "_kerberos._udp.", "_kpasswd._tcp.", "_kpasswd._udp.", "_ldap._tcp.", "_ldap._tcp.dc._msdcs.", "_ldap._tcp.gc._msdcs.", "_ldap._tcp.pdc._msdcs.", "_msdcs.", "_mysqlsrv._tcp.", "_ntp._udp.", "_pop3._tcp.", "_pop3s._tcp.", "_sip._tcp.", "_sip._tls.", "_sip._udp.", "_sipfederationtls._tcp.", "_sipinternaltls._tcp.", "_sips._tcp.", "_smtp._tcp.", "_ssh._tcp.", "_stun._tcp.", "_stun._udp.", "_tcp.", "_tls.", "_udp.", "_vlmcs._tcp.", "_vlmcs._udp.", "_wpad._tcp.", "_xmpp-client._tcp.", "_xmpp-server._tcp.", "_zip._tls"}},
}

type ScanResponse struct {
	RR  []dns.RR
	NS  string
	Rtt time.Duration
}

type ScanRequest struct {
	Qtype  uint16
	Query  string
	Domain string
}

func zoneTransfer(domain, server string) []string {
	var records []string
	t := new(dns.Transfer)
	req := prepMsg()
	req.Question[0] = dns.Question{dns.Fqdn(domain), dns.TypeAXFR, dns.ClassINET}
	q, err := t.In(req, net.JoinHostPort(server, "53"))
	if err != nil {
		return records
		//	fmt.Println("error", err)
	}
	for res := range q {
		if res.Error != nil {
			break
		}
		for _, rr := range res.RR {
			records = append(records, rr.String())
		}
	}
	sort.Strings(records)
	return records
}

func domainscan(domain string) {
	var ips []net.IP
	respc := make(chan ScanResponse, 100)

	servers, _ := findNS(dns.Fqdn(domain))
	for _, server := range servers {
		ips = append(ips, server.IP...)
	}
	s := spinner.New(spinner.CharSets[14], 100*time.Millisecond)

	scanEntries := 0
	for _, src := range DSP {
		scanEntries = scanEntries + len(src.Entries)
	}

	*flagQPS = *flagQPS * len(servers)

	for _, ip := range ips {
		res := zoneTransfer(domain, ip.String())
		if len(res) > 0 {
			zt := ""
			for _, rr := range res {
				zt = zt + fmt.Sprintln(rr)
			}
			fmt.Println(zt)
			// only print one. Further scanning not needed
			return
			// TODO compare hashes
			//	fmt.Printf("%x\n", Hash("key", []byte(zt)))
		} else {
			fmt.Printf("%s ", ip.String())
		}
	}
	fmt.Println(": AXFR denied")

	s.Suffix = " Scanning... will take approx " + fmt.Sprintf("%#v seconds", scanEntries/(len(servers)*(*flagQPS)))
	s.Start()

	res, _, _ := queryRRset(dns.Fqdn("*."+domain), dns.TypeA, ips[0].String(), true)
	// TODO handle * record correctly
	if len(res) != 0 {
		s.Stop()
		fmt.Println()
		for _, rr := range res {
			fmt.Println(rr.String())
		}
		return
	}

	var nsc []chan ScanRequest
	for _, server := range servers {
		c := make(chan ScanRequest, 100)
		nsc = append(nsc, c)
		//fmt.Printf("scanning %s\n", server.IP[0])
		go func(c chan ScanRequest, ns net.IP) {
			limiter := time.Tick(time.Millisecond * time.Duration(1000/(*flagQPS)))
			for request := range c {
				var rrs []dns.RR
				<-limiter
				qtype := request.Qtype
				entry := request.Query
				domain := request.Domain
				if qtype == dns.TypeA {
					res, _, err := query(dns.Fqdn(entry+domain), dns.TypeA, ns.String(), true)
					if err != nil {
						//fmt.Println(err)
					} else {
						a := extractRR(res.Answer, dns.TypeA)
						cname := extractRR(res.Answer, dns.TypeCNAME)
						rrs = append(rrs, a...)
						rrs = append(rrs, cname...)
					}
					res2, rtt, err := queryRRset(dns.Fqdn(entry+domain), dns.TypeAAAA, ns.String(), true)
					if err != nil && len(res2) != 0 {
						//fmt.Println(err)
					}
					rrs = append(rrs, res2...)
					respc <- ScanResponse{RR: rrs, NS: ns.String(), Rtt: rtt}
					continue
				}
				res, rtt, err := queryRRset(dns.Fqdn(entry+domain), qtype, ns.String(), true)
				if err != nil && len(res) != 0 {
					//fmt.Println(err)
				}
				respc <- ScanResponse{RR: res, NS: ns.String(), Rtt: rtt}
			}
		}(c, server.IP[0])
	}

	i := -1
	for _, src := range DSP {
		for _, entry := range src.Entries {
			i++
			q := ScanRequest{src.Qtype, entry, domain}
			nsc[i%len(servers)] <- q
		}
	}

	var responses []string
	i = 0
	for resp := range respc {
		if len(resp.RR) > 0 {
			for _, rr := range resp.RR {
				responses = append(responses, rr.String())
			}
		}
		if i == scanEntries-1 {
			break
		}
		i++
	}

	s.Stop()

	sort.Strings(responses)
	for _, response := range responses {
		fmt.Println(response)
	}
}

func Hash(tag string, data []byte) []byte {
	h := hmac.New(sha256.New, []byte(tag))
	h.Write(data)
	return h.Sum(nil)
}
<|endoftext|>


<commit_before>// for each target domain in the given prioritized list:
//     for front candidate in the given prioritized list:
//        check whether we can reach the target domain by attempting domain
//        fronting through the front candidate.  Log successfully proxied pairs
//        to stdout.  Log detected errors to stderr.

// XXX: record where we left off, to enable resuming an interrupted scan.

package main

import (
	"bufio"
	"crypto/sha1"
	"crypto/tls"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"runtime"
	"sync"
	"time"

	"gopkg.in/getlantern/tlsdialer.v1"
)

var workers = runtime.NumCPU()

var headRe *regexp.Regexp = regexp.MustCompile("(?s)(.*)")

var titleRe *regexp.Regexp = regexp.MustCompile("(?s)<title>(.*)</title>")

const reportFmt = "%-24s %-24s %-12s %-12s %-12s %s\n"

var dialTimeout = 10 * time.Second

var fronts = flag.String("fronts", "fronts.txt", "Path to the file containing front candidates, one domain per line.")

var targets = flag.String("targets", "targets.txt", "Path to the file containing fronting targets, one domain per line.")

// ResponseFeatures contains the characteristics we'll use when comparing
// direct vs domain fronted Responses, to estimate the likelihood that we are
// successfully fronting to a target.
//
// We are also checking whether we get a 200 OK response from the site at all,
// but we don't include this in ResponseFeatures becouse we are not reporting
// sites for which we don't get a 200 OK through domain fronting.
type ResponseFeatures struct {
	// Hash of the whole body, or "" if the body is the empty string
	BodyHash string
	// Hash of the response body's  element, or "" if it has none
	HeadHash string
	// Hash of the <title> element, or "" if the response body has none.
	TitleHash string
}

type Pairing struct {
	// The domain we're trying to reach.
	Target string
	// A URL where we can hopefully reach Target without redirects.
	TargetURL url.URL
	// The domain we're trying to domain front through.
	Front string
	// A URL where we can hopefully reach Front without redirects.
	FrontURL url.URL

	Features ResponseFeatures
}

func main() {
	flag.Parse()
	runtime.GOMAXPROCS(runtime.NumCPU() * 10)
	start := time.Now()
	fmt.Println()
	fmt.Printf(reportFmt, "TARGET", "FRONT", "BODIES", "HEADS", "TITLES", "REDIRECTED URL")
	fmt.Printf(reportFmt, "======", "=====", "======", "=====", "======", "==============")
	fmt.Println()
	main_()
	fmt.Println("**************** COMPLETE ******************")
	fmt.Printf("Scan took %.2fs\n", time.Since(start).Seconds())
}

func main_() {

	tasksChan := make(chan Pairing, workers)

	workersWg := sync.WaitGroup{}

	// Size of work queue is O(n^2) on the size of the input lists, so we
	// generate it lazily.
	go feedTasks(tasksChan)

	workersWg.Add(workers)
	for i := 0; i < workers; i++ {
		go work(tasksChan, &workersWg)
	}

	// Task completion handling:
	//
	// The feedTasks goroutine will close tasksChan when it has processed all
	// input.  Each worker reports workersWg.Done() when it learns that
	// tasksChan is closed.  By that time, the worker has sent all its results
	// to the resultsChan.  Therefore, by the time workersWg.Wait() returns,
	// all `Pairing`s have been checked and all the successful ones have been
	// printed out.
	workersWg.Wait()
}

func work(tasksChan <-chan Pairing, workersWg *sync.WaitGroup) {
	for task := range tasksChan {
		features, err := proxiedResponseFeatures(task)
		if err != nil {
			logDebug("proxiedResponseFeatures error:", err)
			continue
		}
		bodiesMatch := matchReport(task.Features.BodyHash, features.BodyHash)
		headsMatch := matchReport(task.Features.HeadHash, features.HeadHash)
		titlesMatch := matchReport(task.Features.TitleHash, features.TitleHash)
		if bodiesMatch != "match" && headsMatch != "match" && titlesMatch != "match" {
			logDebug("Got an OK response fronting", task.Target, "through", task.Front, "but nothing matches.")
			continue
		}
		fmt.Printf(
			reportFmt,
			task.Target,
			task.Front,
			bodiesMatch,
			headsMatch,
			titlesMatch,
			task.TargetURL.String(),
		)
	}
	logDebug("One worker done")
	workersWg.Done()
}

func proxiedResponseFeatures(p Pairing) (ResponseFeatures, error) {
	logDebug("canProxy", p)
	client := &http.Client{
		Transport: &http.Transport{
			Dial: func(network, addr string) (conn net.Conn, err error) {
				return tlsdialer.DialWithDialer(
					&net.Dialer{Timeout: dialTimeout},
					"tcp",
					p.FrontURL.Host+":443",
					// We can't use a domain front that requires a properly
					// populated SNI, so let's make those fail.
					false,
					&tls.Config{InsecureSkipVerify: true},
				)
			},
		},
		CheckRedirect: noRedirect,
	}
	req, err := http.NewRequest("GET", p.TargetURL.String(), nil)
	if err != nil {
		logErr("building GET request", err)
		return ResponseFeatures{}, err
	}
	resp, err := client.Do(req)
	if err != nil {
		logErr("performing GET request", err)
		return ResponseFeatures{}, err
	}
	return responseFeatures(resp)
}

// feedTasks reads input files, enqueues testing tasks in taskChan, and closes
// taskChan when done.
func feedTasks(taskChan chan<- Pairing) {
	fronts, err := readLines(*fronts)
	if err != nil {
		panic(err)
	}
	targets, err := readLines(*targets)
	if err != nil {
		panic(err)
	}

	// Domains for which we can't get a hash, even without domain fronting.
	badDomains := make(map[string]bool)

	// No-redirect URLs
	nru := make(map[string]url.URL)

	// No-redirect response features
	nrf := make(map[string]ResponseFeatures)

	nilURL := url.URL{}
	nilRF := ResponseFeatures{}

	// URLnRF just provides caching and some logging on top of followRedirects.
	URLnRF := func(domain string) (url.URL, ResponseFeatures) {
		if _, ok := badDomains[domain]; ok {
			logDebug("Known bad domain", domain)
			return nilURL, nilRF
		}
		u, ok := nru[domain]
		if ok {
			logDebug("Cached domain", domain, u.String())
			return u, nrf[domain]
		} else {
			u, rf, err := followRedirects(domain)
			if err != nil {
				logErr("can't reach "+domain+" *without* a proxy", err)
				badDomains[domain] = true
				return nilURL, nilRF
			} else {
				logDebug("Got new url", u.String(), "features", rf)
				nru[domain] = u
				nrf[domain] = rf
				return u, rf
			}
		}
	}

	for _, target := range targets {
		logDebug("Trying", target)
		targetURL, features := URLnRF(target)
		if targetURL == nilURL {
			logDebug("Got null URL for", target)
			continue
		}
		for _, front := range fronts {
			logDebug("Enqueuing pairing:", front, target)
			frontURL, _ := URLnRF(front)
			if frontURL == nilURL {
				// Some CDNs like akamai or Fastly don't like to have their /s
				// requested, but will still domain front.
				frontURL = url.URL{Scheme: "http", Host: front}
			}
			taskChan <- Pairing{
				Target:    target,
				TargetURL: targetURL,
				Front:     front,
				FrontURL:  frontURL,
				Features:  features,
			}
		}
	}
	logDebug("Enqueued all input")
	close(taskChan)
}

// followRedirects makes a request for the given domain without domain fronting,
// returning a URL that will hopefully not redirect, and the relevant features
// of the response.
func followRedirects(domain string) (u url.URL, rf ResponseFeatures, err error) {
	retriesLeft := 10
	// use http scheme to avoid getting double-TLSed
	u = url.URL{Scheme: "http", Host: domain}
	client := &http.Client{
		Transport: &http.Transport{
			Dial: func(network, addr string) (conn net.Conn, err error) {
				logDebug("** trying to get", addr)
				return tls.DialWithDialer(
					&net.Dialer{Timeout: dialTimeout},
					"tcp",
					domain+":443",
					&tls.Config{InsecureSkipVerify: true},
				)
			},
		},
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			retriesLeft--
			if retriesLeft == 0 {
				return errors.New("Too many redirects")
			}
			// use http scheme to avoid getting double-TLSed
			req.URL.Scheme = "http"
			u = *req.URL
			logDebug("(re)setting URL to", u.String())
			return nil
		},
	}
	// use http scheme to avoid getting double-TLSed
	req, err := http.NewRequest("GET", "http://"+domain, nil)
	if err != nil {
		logErr("building GET request", err)
		return url.URL{}, rf, err
	}
	resp, err := client.Do(req)
	if err != nil {
		logErr("performing GET request", err)
		return url.URL{}, rf, err
	}
	if resp.StatusCode != 200 {
		return url.URL{}, rf, errors.New("Non-200 response:" + resp.Status)
	}
	rf, err = responseFeatures(resp)
	if err != nil {
		logErr("getting response features", err)
		return url.URL{}, rf, err
	}
	return
}

func responseFeatures(resp *http.Response) (ret ResponseFeatures, err error) {
	body, err := ioutil.ReadAll(resp.Body)
	defer resp.Body.Close()
	if err != nil {
		return ret, err
	}
	// In what follows, we take advantage of the fact that all ret fields start
	// at their zero value (in particular, "" for strings).
	if body == nil {
		return ret, nil
	}
	ret.BodyHash = hash(body)
	if headMatches := headRe.FindSubmatch(body); headMatches != nil {
		ret.HeadHash = hash(headMatches[1])
	}
	if titleMatches := titleRe.FindSubmatch(body); titleMatches != nil {
		logDebug("Title:", string(titleMatches[1]))
		ret.TitleHash = hash(titleMatches[1])
	}
	return
}

func hash(s []byte) string {
	h := sha1.New()
	h.Write(s)
	bs := h.Sum(nil)
	return fmt.Sprintf("%x", bs)
}

func matchReport(expected, actual string) string {
	if actual == expected {
		if actual == "" {
			return "both empty"
		} else {
			return "match"
		}
	} else {
		if actual == "" {
			return "actual empty"
		} else {
			return "differ"
		}
	}
}

func readLines(filename string) ([]string, error) {
	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	scanner := bufio.NewScanner(file)
	ret := make([]string, 0)
	for scanner.Scan() {
		ret = append(ret, scanner.Text())
	}
	return ret, scanner.Err()
}

func noRedirect(req *http.Request, via []*http.Request) error {
	return errors.New("Don't redirect!: " + req.URL.String())
}

func logErr(msg string, err error) {
	if err != nil {
		log.Printf("ERROR %s: %v\n", msg, err)
	}
}

// Poor man's debug level switch.  Comment out the body to suppress noise.
// XXX: learn about the idiomatic way to do this in Go
func logDebug(vs ...interface{}) {
	//log.Println(vs...)
}
<commit_msg>try www.example.com where example.com will fail without redirect<commit_after>// for each target domain in the given prioritized list:
//     for front candidate in the given prioritized list:
//        check whether we can reach the target domain by attempting domain
//        fronting through the front candidate.  Log successfully proxied pairs
//        to stdout.  Log detected errors to stderr.

// XXX: record where we left off, to enable resuming an interrupted scan.

package main

import (
	"bufio"
	"crypto/sha1"
	"crypto/tls"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"runtime"
	"sync"
	"time"

	"gopkg.in/getlantern/tlsdialer.v1"
)

var workers = runtime.NumCPU()

var headRe *regexp.Regexp = regexp.MustCompile("(?s)(.*)")

var titleRe *regexp.Regexp = regexp.MustCompile("(?s)<title>(.*)</title>")

const reportFmt = "%-24s %-24s %-12s %-12s %-12s %s\n"

var dialTimeout = 10 * time.Second

var fronts = flag.String("fronts", "fronts.txt", "Path to the file containing front candidates, one domain per line.")

var targets = flag.String("targets", "targets.txt", "Path to the file containing fronting targets, one domain per line.")

// ResponseFeatures contains the characteristics we'll use when comparing
// direct vs domain fronted Responses, to estimate the likelihood that we are
// successfully fronting to a target.
//
// We are also checking whether we get a 200 OK response from the site at all,
// but we don't include this in ResponseFeatures becouse we are not reporting
// sites for which we don't get a 200 OK through domain fronting.
type ResponseFeatures struct {
	// Hash of the whole body, or "" if the body is the empty string
	BodyHash string
	// Hash of the response body's  element, or "" if it has none
	HeadHash string
	// Hash of the <title> element, or "" if the response body has none.
	TitleHash string
}

type Pairing struct {
	// The domain we're trying to reach.
	Target string
	// A URL where we can hopefully reach Target without redirects.
	TargetURL url.URL
	// The domain we're trying to domain front through.
	Front string
	// A URL where we can hopefully reach Front without redirects.
	FrontURL url.URL

	Features ResponseFeatures
}

func main() {
	flag.Parse()
	runtime.GOMAXPROCS(runtime.NumCPU() * 10)
	start := time.Now()
	fmt.Println()
	fmt.Printf(reportFmt, "TARGET", "FRONT", "BODIES", "HEADS", "TITLES", "REDIRECTED URL")
	fmt.Printf(reportFmt, "======", "=====", "======", "=====", "======", "==============")
	fmt.Println()
	main_()
	fmt.Println("**************** COMPLETE ******************")
	fmt.Printf("Scan took %.2fs\n", time.Since(start).Seconds())
}

func main_() {

	tasksChan := make(chan Pairing, workers)

	workersWg := sync.WaitGroup{}

	// Size of work queue is O(n^2) on the size of the input lists, so we
	// generate it lazily.
	go feedTasks(tasksChan)

	workersWg.Add(workers)
	for i := 0; i < workers; i++ {
		go work(tasksChan, &workersWg)
	}

	// Task completion handling:
	//
	// The feedTasks goroutine will close tasksChan when it has processed all
	// input.  Each worker reports workersWg.Done() when it learns that
	// tasksChan is closed.  By that time, the worker has sent all its results
	// to the resultsChan.  Therefore, by the time workersWg.Wait() returns,
	// all `Pairing`s have been checked and all the successful ones have been
	// printed out.
	workersWg.Wait()
}

func work(tasksChan <-chan Pairing, workersWg *sync.WaitGroup) {
	for task := range tasksChan {
		features, err := proxiedResponseFeatures(task)
		if err != nil {
			logDebug("proxiedResponseFeatures error:", err)
			continue
		}
		bodiesMatch := matchReport(task.Features.BodyHash, features.BodyHash)
		headsMatch := matchReport(task.Features.HeadHash, features.HeadHash)
		titlesMatch := matchReport(task.Features.TitleHash, features.TitleHash)
		if bodiesMatch != "match" && headsMatch != "match" && titlesMatch != "match" {
			logDebug("Got an OK response fronting", task.Target, "through", task.Front, "but nothing matches.")
			continue
		}
		fmt.Printf(
			reportFmt,
			task.Target,
			task.Front,
			bodiesMatch,
			headsMatch,
			titlesMatch,
			task.TargetURL.String(),
		)
	}
	logDebug("One worker done")
	workersWg.Done()
}

func proxiedResponseFeatures(p Pairing) (ResponseFeatures, error) {
	logDebug("canProxy", p)
	client := &http.Client{
		Transport: &http.Transport{
			Dial: func(network, addr string) (conn net.Conn, err error) {
				return tlsdialer.DialWithDialer(
					&net.Dialer{Timeout: dialTimeout},
					"tcp",
					p.FrontURL.Host+":443",
					// We can't use a domain front that requires a properly
					// populated SNI, so let's make those fail.
					false,
					&tls.Config{InsecureSkipVerify: true},
				)
			},
		},
		CheckRedirect: noRedirect,
	}
	req, err := http.NewRequest("GET", p.TargetURL.String(), nil)
	if err != nil {
		logErr("building GET request", err)
		return ResponseFeatures{}, err
	}
	resp, err := client.Do(req)
	if err != nil {
		logErr("performing GET request", err)
		return ResponseFeatures{}, err
	}
	return responseFeatures(resp)
}

// feedTasks reads input files, enqueues testing tasks in taskChan, and closes
// taskChan when done.
func feedTasks(taskChan chan<- Pairing) {
	fronts, err := readLines(*fronts)
	if err != nil {
		panic(err)
	}
	targets, err := readLines(*targets)
	if err != nil {
		panic(err)
	}

	// Domains for which we can't get a hash, even without domain fronting.
	badDomains := make(map[string]bool)

	// No-redirect URLs
	nru := make(map[string]url.URL)

	// No-redirect response features
	nrf := make(map[string]ResponseFeatures)

	nilURL := url.URL{}
	nilRF := ResponseFeatures{}

	// URLnRF just provides caching and some logging on top of followRedirects.
	URLnRF := func(domain string) (url.URL, ResponseFeatures) {
		if _, ok := badDomains[domain]; ok {
			logDebug("Known bad domain", domain)
			return nilURL, nilRF
		}
		u, ok := nru[domain]
		if ok {
			logDebug("Cached domain", domain, u.String())
			return u, nrf[domain]
		} else {
			u, rf, err := followRedirects(domain)
			if err != nil {
				logErr("can't reach "+domain+" *without* a proxy", err)
				badDomains[domain] = true
				return nilURL, nilRF
			} else {
				logDebug("Got new url", u.String(), "features", rf)
				nru[domain] = u
				nrf[domain] = rf
				return u, rf
			}
		}
	}

	for _, target := range targets {
		logDebug("Trying", target)
		targetURL, features := URLnRF(target)
		if targetURL == nilURL {
			logDebug("Got null URL for", target)
			// This works for Baidu at least.  Worth trying, should there be
			// more sites for which www. works, the naked domain doesn't, and
			// which won't automatically redirect.
			targetURL, features = URLnRF("www." + target)
			if targetURL == nilURL {
				continue
			}
		}
		for _, front := range fronts {
			logDebug("Enqueuing pairing:", front, target)
			frontURL, _ := URLnRF(front)
			if frontURL == nilURL {
				// Some CDNs like akamai or Fastly don't like to have their /s
				// requested, but will still domain front.
				frontURL = url.URL{Scheme: "http", Host: front}
			}
			taskChan <- Pairing{
				Target:    target,
				TargetURL: targetURL,
				Front:     front,
				FrontURL:  frontURL,
				Features:  features,
			}
		}
	}
	logDebug("Enqueued all input")
	close(taskChan)
}

// followRedirects makes a request for the given domain without domain fronting,
// returning a URL that will hopefully not redirect, and the relevant features
// of the response.
func followRedirects(domain string) (u url.URL, rf ResponseFeatures, err error) {
	retriesLeft := 10
	// use http scheme to avoid getting double-TLSed
	u = url.URL{Scheme: "http", Host: domain}
	client := &http.Client{
		Transport: &http.Transport{
			Dial: func(network, addr string) (conn net.Conn, err error) {
				logDebug("** trying to get", addr)
				return tls.DialWithDialer(
					&net.Dialer{Timeout: dialTimeout},
					"tcp",
					domain+":443",
					&tls.Config{InsecureSkipVerify: true},
				)
			},
		},
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			retriesLeft--
			if retriesLeft == 0 {
				return errors.New("Too many redirects")
			}
			// use http scheme to avoid getting double-TLSed
			req.URL.Scheme = "http"
			u = *req.URL
			logDebug("(re)setting URL to", u.String())
			return nil
		},
	}
	// use http scheme to avoid getting double-TLSed
	req, err := http.NewRequest("GET", "http://"+domain, nil)
	if err != nil {
		logErr("building GET request", err)
		return url.URL{}, rf, err
	}
	resp, err := client.Do(req)
	if err != nil {
		logErr("performing GET request", err)
		return url.URL{}, rf, err
	}
	if resp.StatusCode != 200 {
		return url.URL{}, rf, errors.New("Non-200 response:" + resp.Status)
	}
	rf, err = responseFeatures(resp)
	if err != nil {
		logErr("getting response features", err)
		return url.URL{}, rf, err
	}
	return
}

func responseFeatures(resp *http.Response) (ret ResponseFeatures, err error) {
	body, err := ioutil.ReadAll(resp.Body)
	defer resp.Body.Close()
	if err != nil {
		return ret, err
	}
	// In what follows, we take advantage of the fact that all ret fields start
	// at their zero value (in particular, "" for strings).
	if body == nil {
		return ret, nil
	}
	ret.BodyHash = hash(body)
	if headMatches := headRe.FindSubmatch(body); headMatches != nil {
		ret.HeadHash = hash(headMatches[1])
	}
	if titleMatches := titleRe.FindSubmatch(body); titleMatches != nil {
		logDebug("Title:", string(titleMatches[1]))
		ret.TitleHash = hash(titleMatches[1])
	}
	return
}

func hash(s []byte) string {
	h := sha1.New()
	h.Write(s)
	bs := h.Sum(nil)
	return fmt.Sprintf("%x", bs)
}

func matchReport(expected, actual string) string {
	if actual == expected {
		if actual == "" {
			return "both empty"
		} else {
			return "match"
		}
	} else {
		if actual == "" {
			return "actual empty"
		} else {
			return "differ"
		}
	}
}

func readLines(filename string) ([]string, error) {
	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	scanner := bufio.NewScanner(file)
	ret := make([]string, 0)
	for scanner.Scan() {
		ret = append(ret, scanner.Text())
	}
	return ret, scanner.Err()
}

func noRedirect(req *http.Request, via []*http.Request) error {
	return errors.New("Don't redirect!: " + req.URL.String())
}

func logErr(msg string, err error) {
	if err != nil {
		log.Printf("ERROR %s: %v\n", msg, err)
	}
}

// Poor man's debug level switch.  Comment out the body to suppress noise.
// XXX: learn about the idiomatic way to do this in Go
func logDebug(vs ...interface{}) {
	//log.Println(vs...)
}
<|endoftext|>


<commit_before>package service

import (
    "bytes"
    "encoding/base64"
    "fmt"
    "net/mail"
    "net/smtp"
    "sync"
    "time"
)

const (
    //authTypes
    Plain       = "plain"
    MD5         = "cram-md5"
    Unencrypted = "unencrypted"

    //tags
    SMTP = "smtp"

    //conf
    Limit = 60 * 60
    Step  = 21 * 60
)

type SmtpServer struct {
    lock    sync.Mutex
    rate    time.Time
    client  *smtp.Client
    auth    smtp.Auth
    running bool
    conf    *SmtpConf
}

//smtp.PlainAuth refuses to send your password over an unencrypted connection.
//this auth will use plain authentication anyways.
type unencryptedAuth struct {
    smtp.Auth
}

func (a unencryptedAuth) Start(server *smtp.ServerInfo) (string, []byte, error) {
    s := *server
    s.TLS = true
    return a.Auth.Start(&s)
}

type OptionConf struct {
}

type SmtpConf struct {
    Host     string
    Port     int
    Username string
    Password string
    AuthType string
    //options
    TimeOut   uint32
    SickLimit uint32
    SickStep  uint32
}

func (conf *SmtpConf) smtpAuth() (auth smtp.Auth) {
    switch conf.AuthType {
    case Plain:
        auth = smtp.PlainAuth("", conf.Username, conf.Password, conf.Host)
    case MD5:
        auth = smtp.CRAMMD5Auth(conf.Username, conf.Password)
    case Unencrypted:
        auth = unencryptedAuth{
            smtp.PlainAuth("", conf.Username, conf.Password, conf.Host),
        }
    default:
    }
    return
}

func NewSmtpServer(conf *SmtpConf) (server Server) {
    server = &SmtpServer{
        auth: conf.smtpAuth(),
        conf: conf,
    }
    return
}

func (s *SmtpServer) Init(conf ...interface{}) (err error) {
    s.rate = time.Now()
    if s.conf.TimeOut == 0 {
        s.conf.TimeOut = Step / 5
    }
    if s.conf.SickLimit == 0 {
        s.conf.SickLimit = Limit
    }
    if s.conf.SickStep == 0 {
        s.conf.SickStep = Step
    }
    return s.connect()
}

func (s *SmtpServer) Send(m *Message) error {
    return s.send(m)
}

func (s *SmtpServer) Close() (err error) {
    s.lock.Lock()
    defer s.lock.Unlock()
    if s.Timeout() {
        err = s.closeConn()
    }
    return
}

func (s *SmtpServer) Rate() time.Time {
    return s.rate
}

func (s *SmtpServer) Tag() string {
    return SMTP
}

func (s *SmtpServer) Running() bool {
    return s.running
}

func (s *SmtpServer) Timeout() bool {
    return time.Now().Sub(s.rate) > time.Duration(s.conf.TimeOut)*time.Second
}

func (s *SmtpServer) Sick() bool {
    return s.rate.Sub(time.Now()) > time.Duration(s.conf.SickLimit)*time.Second
}

func (s *SmtpServer) closeConn() (err error) {
    if s.running {
        err = s.client.Quit()
        s.running = false
    }
    return
}

func (s *SmtpServer) connect() (err error) {
    s.client, err = smtp.Dial(fmt.Sprintf("%s:%d", s.conf.Host, s.conf.Port))
    if err != nil {
        return
    }
    if s.auth != nil {
        if ok, _ := s.client.Extension("StartTLS"); ok {
            if err = s.client.StartTLS(nil); err != nil {
                return
            }
        }
        err = s.client.Auth(s.auth)
        if err != nil {
            return
        }
    }
    s.running = true
    return
}

func (s *SmtpServer) upgrade() {
    s.rate = s.rate.Add(time.Duration(s.conf.SickStep) * time.Second)
}

func (s *SmtpServer) send(m *Message) (err error) {
    s.lock.Lock()
    defer s.lock.Unlock()

    if !s.running {
        err = s.connect()
        if err != nil {
            s.upgrade()
            return
        }
    }

    if m.Mass {
        err = s.massSend(s.client, m, s.getFromAddr(m))
    } else {
        err = s.singleSend(s.client, m, s.getFromAddr(m))
    }
    if err != nil {
        return
    }
    s.rate = time.Now()
    return
}

func (s *SmtpServer) getFromAddr(m *Message) string {
    if s.auth == nil {
        return m.From
    }
    return s.conf.Username
}

func (s *SmtpServer) formatBody(m *Message, to string) []byte {
    b64 := base64.NewEncoding("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/")
    header := make(map[string]string, 6)
    from := mail.Address{m.SenderName, s.getFromAddr(m)}
    header["From"] = from.String()
    header["To"] = to
    header["Subject"] = fmt.Sprintf("=?UTF-8?B?%s?=", b64.EncodeToString([]byte(m.Subject)))
    header["MIME-Version"] = "1.0"
    header["Content-Type"] = "text/html; charset=UTF-8"
    header["Content-Transfer-Encoding"] = "base64"

    var body bytes.Buffer
    for k, v := range header {
        body.WriteString(fmt.Sprintf("%s: %s\r\n", k, v))
    }
    body.WriteString("\r\n")
    body.WriteString(b64.EncodeToString([]byte(m.Body)))

    return body.Bytes()
}

func (s *SmtpServer) massSend(client *smtp.Client, m *Message, sendAddr string) (err error) {
    err = s.client.Mail(sendAddr)
    if err != nil {
        if s.auth != nil {
            s.upgrade()
        }
        return
    }
    var toStr bytes.Buffer
    for name, addr := range m.To {
        to := mail.Address{name, addr}
        toStr.WriteString(to.String())
        toStr.WriteString(", ")
        if err = client.Rcpt(addr); err != nil {
            return
        }
    }
    w, err := client.Data()
    if err != nil {
        return
    }
    if _, err = w.Write(s.formatBody(m, toStr.String())); err != nil {
        return
    }
    err = w.Close()
    return
}

func (s *SmtpServer) singleSend(client *smtp.Client, m *Message, sendAddr string) (err error) {
    for name, addr := range m.To {
        err = s.client.Mail(sendAddr)
        if err != nil {
            if s.auth != nil {
                s.upgrade()
            }
            break
        }
        if err = client.Rcpt(addr); err != nil {
            break
        }
        w, err := client.Data()
        if err != nil {
            break
        }
        to := mail.Address{name, addr}
        if _, err = w.Write(s.formatBody(m, to.String())); err != nil {
            break
        }
        if err = w.Close(); err != nil {
            break
        }
    }
    return
}
<commit_msg>changed default value of timeout<commit_after>package service

import (
    "bytes"
    "encoding/base64"
    "fmt"
    "net/mail"
    "net/smtp"
    "sync"
    "time"
)

const (
    //authTypes
    Plain       = "plain"
    MD5         = "cram-md5"
    Unencrypted = "unencrypted"

    //tags
    SMTP = "smtp"

    //conf
    Limit = 60 * 60
    Step  = 21 * 60
)

type SmtpServer struct {
    lock    sync.Mutex
    rate    time.Time
    client  *smtp.Client
    auth    smtp.Auth
    running bool
    conf    *SmtpConf
}

//smtp.PlainAuth refuses to send your password over an unencrypted connection.
//this auth will use plain authentication anyways.
type unencryptedAuth struct {
    smtp.Auth
}

func (a unencryptedAuth) Start(server *smtp.ServerInfo) (string, []byte, error) {
    s := *server
    s.TLS = true
    return a.Auth.Start(&s)
}

type SmtpConf struct {
    Host     string
    Port     int
    Username string
    Password string
    AuthType string
    //options
    TimeOut   uint32
    SickLimit uint32
    SickStep  uint32
}

func (conf *SmtpConf) smtpAuth() (auth smtp.Auth) {
    switch conf.AuthType {
    case Plain:
        auth = smtp.PlainAuth("", conf.Username, conf.Password, conf.Host)
    case MD5:
        auth = smtp.CRAMMD5Auth(conf.Username, conf.Password)
    case Unencrypted:
        auth = unencryptedAuth{
            smtp.PlainAuth("", conf.Username, conf.Password, conf.Host),
        }
    default:
    }
    return
}

func NewSmtpServer(conf *SmtpConf) (server Server) {
    server = &SmtpServer{
        auth: conf.smtpAuth(),
        conf: conf,
    }
    return
}

func (s *SmtpServer) Init(conf ...interface{}) (err error) {
    s.rate = time.Now()
    if s.conf.TimeOut == 0 {
        s.conf.TimeOut = Step / 8
    }
    if s.conf.SickLimit == 0 {
        s.conf.SickLimit = Limit
    }
    if s.conf.SickStep == 0 {
        s.conf.SickStep = Step
    }
    return s.connect()
}

func (s *SmtpServer) Send(m *Message) error {
    return s.send(m)
}

func (s *SmtpServer) Close() (err error) {
    s.lock.Lock()
    defer s.lock.Unlock()
    if s.Timeout() {
        err = s.closeConn()
    }
    return
}

func (s *SmtpServer) Rate() time.Time {
    return s.rate
}

func (s *SmtpServer) Tag() string {
    return SMTP
}

func (s *SmtpServer) Running() bool {
    return s.running
}

func (s *SmtpServer) Timeout() bool {
    return time.Now().Sub(s.rate) > time.Duration(s.conf.TimeOut)*time.Second
}

func (s *SmtpServer) Sick() bool {
    return s.rate.Sub(time.Now()) > time.Duration(s.conf.SickLimit)*time.Second
}

func (s *SmtpServer) closeConn() (err error) {
    if s.running {
        err = s.client.Quit()
        s.running = false
    }
    return
}

func (s *SmtpServer) connect() (err error) {
    s.client, err = smtp.Dial(fmt.Sprintf("%s:%d", s.conf.Host, s.conf.Port))
    if err != nil {
        return
    }
    if s.auth != nil {
        if ok, _ := s.client.Extension("StartTLS"); ok {
            if err = s.client.StartTLS(nil); err != nil {
                return
            }
        }
        err = s.client.Auth(s.auth)
        if err != nil {
            return
        }
    }
    s.running = true
    return
}

func (s *SmtpServer) upgrade() {
    s.rate = s.rate.Add(time.Duration(s.conf.SickStep) * time.Second)
}

func (s *SmtpServer) send(m *Message) (err error) {
    s.lock.Lock()
    defer s.lock.Unlock()

    if !s.running {
        err = s.connect()
        if err != nil {
            s.upgrade()
            return
        }
    }

    if m.Mass {
        err = s.massSend(s.client, m, s.getFromAddr(m))
    } else {
        err = s.singleSend(s.client, m, s.getFromAddr(m))
    }
    if err != nil {
        return
    }
    s.rate = time.Now()
    return
}

func (s *SmtpServer) getFromAddr(m *Message) string {
    if s.auth == nil {
        return m.From
    }
    return s.conf.Username
}

func (s *SmtpServer) formatBody(m *Message, to string) []byte {
    b64 := base64.NewEncoding("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/")
    header := make(map[string]string, 6)
    from := mail.Address{m.SenderName, s.getFromAddr(m)}
    header["From"] = from.String()
    header["To"] = to
    header["Subject"] = fmt.Sprintf("=?UTF-8?B?%s?=", b64.EncodeToString([]byte(m.Subject)))
    header["MIME-Version"] = "1.0"
    header["Content-Type"] = "text/html; charset=UTF-8"
    header["Content-Transfer-Encoding"] = "base64"

    var body bytes.Buffer
    for k, v := range header {
        body.WriteString(fmt.Sprintf("%s: %s\r\n", k, v))
    }
    body.WriteString("\r\n")
    body.WriteString(b64.EncodeToString([]byte(m.Body)))

    return body.Bytes()
}

func (s *SmtpServer) massSend(client *smtp.Client, m *Message, sendAddr string) (err error) {
    err = s.client.Mail(sendAddr)
    if err != nil {
        if s.auth != nil {
            s.upgrade()
        }
        return
    }
    var toStr bytes.Buffer
    for name, addr := range m.To {
        to := mail.Address{name, addr}
        toStr.WriteString(to.String())
        toStr.WriteString(", ")
        if err = client.Rcpt(addr); err != nil {
            return
        }
    }
    w, err := client.Data()
    if err != nil {
        return
    }
    if _, err = w.Write(s.formatBody(m, toStr.String())); err != nil {
        return
    }
    err = w.Close()
    return
}

func (s *SmtpServer) singleSend(client *smtp.Client, m *Message, sendAddr string) (err error) {
    for name, addr := range m.To {
        err = s.client.Mail(sendAddr)
        if err != nil {
            if s.auth != nil {
                s.upgrade()
            }
            break
        }
        if err = client.Rcpt(addr); err != nil {
            break
        }
        w, err := client.Data()
        if err != nil {
            break
        }
        to := mail.Address{name, addr}
        if _, err = w.Write(s.formatBody(m, to.String())); err != nil {
            break
        }
        if err = w.Close(); err != nil {
            break
        }
    }
    return
}
<|endoftext|>


<commit_before>// Package timeutils provides some convenience functions around time.
package timeutils // import "hkjn.me/timeutils"

import (
	"fmt"
	"log"
	"strconv"
	"strings"
	"time"
)

var (
	StdLayout     = "2006-01-02 15:04" // a simple time layout
	EmptyTimeZone = TimeZone("")       // the empty timezone
)

// TimeZone is a string that can be given to time.LoadLocation.
type TimeZone string

// Day describes a specific day and can be used as map keys.
type Day struct {
	Year, Day int
	Month     time.Month
	TimeZone  TimeZone
}

// DayFromTime returns the equivalent day object from time.
func DayFromTime(t time.Time) Day {
	year, month, day := t.Date()
	return Day{year, day, month, TimeZone(t.Location().String())}
}

// ToTime returns the time object representing the start of the day.
func (d Day) ToTime() time.Time {
	return time.Date(d.Year, d.Month, d.Day, 0, 0, 0, 0, MustLoadLoc(d.TimeZone))
}

// String returns a description of the day object.
func (d Day) String() string {
	return d.ToTime().Format("2006-01-02 MST")
}

// String returns a description of the day object without the TZ.
func (d Day) ShortString() string {
	return d.ToTime().Format("2006-01-02")
}

// MustLoadLoc loads the time.Location specified by the string, or panics.
func MustLoadLoc(tz TimeZone) *time.Location {
	loc, err := time.LoadLocation(string(tz))
	if err != nil {
		log.Fatalf("bad location %q: %v\n", tz, err)
	}
	return loc
}

// MustParseDuration returns the time.Duration specified by the string, or panics.
func MustParseDuration(s string) time.Duration {
	d, err := time.ParseDuration(s)
	if err != nil {
		log.Fatalf("bad duration %q: %v\n", s, err)
	}
	return d
}

// Must panics if error is non-nil.
//
// Intended use is to wrap function calls that must succeed, e.g:
//   t := Must(ParseStd("2013-07-31 07:30"))
func Must(t time.Time, err error) time.Time {
	if err != nil {
		log.Fatalf("got err: %v\n", err)
	}
	return t
}

// AsMillis returns the number of milliseconds (in offset by sec) for the time.
func AsMillis(t time.Time, offset int) int {
	return int(t.UTC().UnixNano()/1000000) + offset*1000
}

// ParseStd returns the string value parsed using StdLayout.
func ParseStd(value string) (time.Time, error) {
	return time.Parse(StdLayout, value)
}

// daysIn returns the number of days in a month for a given year.
func daysIn(m time.Month, year int) int {
	// This is equivalent to the unexported time.daysIn(m, year).
	return time.Date(year, m+1, 0, 0, 0, 0, 0, time.UTC).Day()
}

// Weekday gets the Monday-indexed number for the time.Weekday.
func Weekday(d time.Weekday) int {
	day := (d - 1) % 7
	if day < 0 {
		day += 7
	}
	return int(day)
}

// StartOfWeek returns the start of the current week for the time.
func StartOfWeek(t time.Time) time.Time {
	// Figure out number of days to back up until Mon:
	// Sun is 0 -> 6, Sat is 6 -> 5, etc.
	toMon := Weekday(t.Weekday())
	y, m, d := t.AddDate(0, 0, -int(toMon)).Date()
	// Result is 00:00:00 on that year, month, day.
	return time.Date(y, m, d, 0, 0, 0, 0, t.Location())
}

// Parse extracts time from string-based info, with some constraints.
//
// The described time cannot be in the future, or more than 1000 years in the past.
//
// Note that month is 0-indexed, unlike time.Month.
func Parse(year, month, day, hourMinute string, loc *time.Location) (time.Time, error) {
	now := time.Now().In(loc)

	y64, err := strconv.ParseInt(year, 10, 0)
	y := int(y64)
	if err != nil {
		return time.Time{}, err
	}
	if y < now.Year()-1000 {
		return time.Time{}, fmt.Errorf("bad year; %d is too far in the past", y)
	}
	m, err := strconv.ParseInt(month, 10, 0)
	if err != nil {
		return time.Time{}, err
	}
	if m < 0 || m > 11 {
		return time.Time{}, fmt.Errorf("bad month: %d is not within [0, 11]", m)
	}
	// Month +1 since time.Month is [1, 12].
	m = m + 1
	d64, err := strconv.ParseInt(day, 10, 0)
	d := int(d64)
	if err != nil {
		return time.Time{}, err
	}
	if d < 1 {
		return time.Time{}, fmt.Errorf("bad day: %d; can't be negative", d)
	} else if d > daysIn(time.Month(m), y) {
		return time.Time{}, fmt.Errorf("bad day: %d; only %d days in %v, %d", d, daysIn(time.Month(m), y), time.Month(m), y)
	}
	parts := strings.Split(hourMinute, ":")
	if len(parts) != 2 {
		return time.Time{}, fmt.Errorf("bad hour/minute: %s", hourMinute)
	}
	h, err := strconv.ParseInt(parts[0], 10, 0)
	if err != nil {
		return time.Time{}, err
	}
	if h < 0 || h > 60 {
		return time.Time{}, fmt.Errorf("bad hour: %d", h)
	}
	min, err := strconv.ParseInt(parts[1], 10, 0)
	if err != nil {
		return time.Time{}, err
	}
	if min < 0 || min > 60 {
		return time.Time{}, fmt.Errorf("bad minute: %d", min)
	}

	t := time.Time(time.Date(int(y), time.Month(m), int(d), int(h), int(min), 0, 0, loc))
	if t.After(now) {
		return time.Time{}, fmt.Errorf("bad time; %v is in the future", time.Time(t))
	}
	return t, nil
}

// Selector holds info useful to make time selections.
type Selector struct {
	SelectedDay   int
	SelectedMonth time.Month
	SelectedYear  int
	SelectedTime  string
	Months        []time.Month
	Years         []int
	DaysInMonth   []int
}

// Create populates a Selector from given starting point.
func (s *Selector) Create(from time.Time) {
	days := make([]int, 31)
	for d := 0; d < 31; d++ { // TODO: Actual number of days / month (change dynamically on selection?).
		days[d] = d + 1
	}
	numYears := 5
	years := make([]int, numYears)
	for i := 0; i < numYears; i++ {
		years[i] = from.Year() - i
	}
	*s = Selector{
		SelectedYear:  from.Year(),
		SelectedMonth: from.Month() - 1, // -1 to give [0, 11]
		SelectedDay:   from.Day(),
		SelectedTime:  from.Format("15:04"),
		DaysInMonth:   days,
		Months:        make([]time.Month, 12),
		Years:         years,
	}
	for i := 1; i <= 12; i++ {
		s.Months[i-1] = time.Month(i)
	}
}

// DescDuration returns a human-readable string describing the duration.
func DescDuration(d time.Duration) string {
	if d < time.Minute {
		return fmt.Sprintf("%0.1f sec ago", d.Seconds())
	} else if d < time.Hour {
		return fmt.Sprintf("%0.1f min ago", d.Minutes())
	} else if d < time.Hour*24 {
		return fmt.Sprintf("%0.1f hrs ago", d.Hours())
	} else {
		return fmt.Sprintf("%0.1f days ago", d.Hours()/24.0)
	}
}
<commit_msg>Fix typo in comment<commit_after>// Package timeutils provides some convenience functions around time.
package timeutils // import "hkjn.me/timeutils"

import (
	"fmt"
	"log"
	"strconv"
	"strings"
	"time"
)

var (
	StdLayout     = "2006-01-02 15:04" // a simple time layout
	EmptyTimeZone = TimeZone("")       // the empty timezone
)

// TimeZone is a string that can be given to time.LoadLocation.
type TimeZone string

// Day describes a specific day and can be used as map keys.
type Day struct {
	Year, Day int
	Month     time.Month
	TimeZone  TimeZone
}

// DayFromTime returns the equivalent day object from time.
func DayFromTime(t time.Time) Day {
	year, month, day := t.Date()
	return Day{year, day, month, TimeZone(t.Location().String())}
}

// ToTime returns the time object representing the start of the day.
func (d Day) ToTime() time.Time {
	return time.Date(d.Year, d.Month, d.Day, 0, 0, 0, 0, MustLoadLoc(d.TimeZone))
}

// String returns a description of the day object.
func (d Day) String() string {
	return d.ToTime().Format("2006-01-02 MST")
}

// ShortString returns a description of the day object without the TZ.
func (d Day) ShortString() string {
	return d.ToTime().Format("2006-01-02")
}

// MustLoadLoc loads the time.Location specified by the string, or panics.
func MustLoadLoc(tz TimeZone) *time.Location {
	loc, err := time.LoadLocation(string(tz))
	if err != nil {
		log.Fatalf("bad location %q: %v\n", tz, err)
	}
	return loc
}

// MustParseDuration returns the time.Duration specified by the string, or panics.
func MustParseDuration(s string) time.Duration {
	d, err := time.ParseDuration(s)
	if err != nil {
		log.Fatalf("bad duration %q: %v\n", s, err)
	}
	return d
}

// Must panics if error is non-nil.
//
// Intended use is to wrap function calls that must succeed, e.g:
//   t := Must(ParseStd("2013-07-31 07:30"))
func Must(t time.Time, err error) time.Time {
	if err != nil {
		log.Fatalf("got err: %v\n", err)
	}
	return t
}

// AsMillis returns the number of milliseconds (in offset by sec) for the time.
func AsMillis(t time.Time, offset int) int {
	return int(t.UTC().UnixNano()/1000000) + offset*1000
}

// ParseStd returns the string value parsed using StdLayout.
func ParseStd(value string) (time.Time, error) {
	return time.Parse(StdLayout, value)
}

// daysIn returns the number of days in a month for a given year.
func daysIn(m time.Month, year int) int {
	// This is equivalent to the unexported time.daysIn(m, year).
	return time.Date(year, m+1, 0, 0, 0, 0, 0, time.UTC).Day()
}

// Weekday gets the Monday-indexed number for the time.Weekday.
func Weekday(d time.Weekday) int {
	day := (d - 1) % 7
	if day < 0 {
		day += 7
	}
	return int(day)
}

// StartOfWeek returns the start of the current week for the time.
func StartOfWeek(t time.Time) time.Time {
	// Figure out number of days to back up until Mon:
	// Sun is 0 -> 6, Sat is 6 -> 5, etc.
	toMon := Weekday(t.Weekday())
	y, m, d := t.AddDate(0, 0, -int(toMon)).Date()
	// Result is 00:00:00 on that year, month, day.
	return time.Date(y, m, d, 0, 0, 0, 0, t.Location())
}

// Parse extracts time from string-based info, with some constraints.
//
// The described time cannot be in the future, or more than 1000 years in the past.
//
// Note that month is 0-indexed, unlike time.Month.
func Parse(year, month, day, hourMinute string, loc *time.Location) (time.Time, error) {
	now := time.Now().In(loc)

	y64, err := strconv.ParseInt(year, 10, 0)
	y := int(y64)
	if err != nil {
		return time.Time{}, err
	}
	if y < now.Year()-1000 {
		return time.Time{}, fmt.Errorf("bad year; %d is too far in the past", y)
	}
	m, err := strconv.ParseInt(month, 10, 0)
	if err != nil {
		return time.Time{}, err
	}
	if m < 0 || m > 11 {
		return time.Time{}, fmt.Errorf("bad month: %d is not within [0, 11]", m)
	}
	// Month +1 since time.Month is [1, 12].
	m = m + 1
	d64, err := strconv.ParseInt(day, 10, 0)
	d := int(d64)
	if err != nil {
		return time.Time{}, err
	}
	if d < 1 {
		return time.Time{}, fmt.Errorf("bad day: %d; can't be negative", d)
	} else if d > daysIn(time.Month(m), y) {
		return time.Time{}, fmt.Errorf("bad day: %d; only %d days in %v, %d", d, daysIn(time.Month(m), y), time.Month(m), y)
	}
	parts := strings.Split(hourMinute, ":")
	if len(parts) != 2 {
		return time.Time{}, fmt.Errorf("bad hour/minute: %s", hourMinute)
	}
	h, err := strconv.ParseInt(parts[0], 10, 0)
	if err != nil {
		return time.Time{}, err
	}
	if h < 0 || h > 60 {
		return time.Time{}, fmt.Errorf("bad hour: %d", h)
	}
	min, err := strconv.ParseInt(parts[1], 10, 0)
	if err != nil {
		return time.Time{}, err
	}
	if min < 0 || min > 60 {
		return time.Time{}, fmt.Errorf("bad minute: %d", min)
	}

	t := time.Time(time.Date(int(y), time.Month(m), int(d), int(h), int(min), 0, 0, loc))
	if t.After(now) {
		return time.Time{}, fmt.Errorf("bad time; %v is in the future", time.Time(t))
	}
	return t, nil
}

// Selector holds info useful to make time selections.
type Selector struct {
	SelectedDay   int
	SelectedMonth time.Month
	SelectedYear  int
	SelectedTime  string
	Months        []time.Month
	Years         []int
	DaysInMonth   []int
}

// Create populates a Selector from given starting point.
func (s *Selector) Create(from time.Time) {
	days := make([]int, 31)
	for d := 0; d < 31; d++ { // TODO: Actual number of days / month (change dynamically on selection?).
		days[d] = d + 1
	}
	numYears := 5
	years := make([]int, numYears)
	for i := 0; i < numYears; i++ {
		years[i] = from.Year() - i
	}
	*s = Selector{
		SelectedYear:  from.Year(),
		SelectedMonth: from.Month() - 1, // -1 to give [0, 11]
		SelectedDay:   from.Day(),
		SelectedTime:  from.Format("15:04"),
		DaysInMonth:   days,
		Months:        make([]time.Month, 12),
		Years:         years,
	}
	for i := 1; i <= 12; i++ {
		s.Months[i-1] = time.Month(i)
	}
}

// DescDuration returns a human-readable string describing the duration.
func DescDuration(d time.Duration) string {
	if d < time.Minute {
		return fmt.Sprintf("%0.1f sec ago", d.Seconds())
	} else if d < time.Hour {
		return fmt.Sprintf("%0.1f min ago", d.Minutes())
	} else if d < time.Hour*24 {
		return fmt.Sprintf("%0.1f hrs ago", d.Hours())
	} else {
		return fmt.Sprintf("%0.1f days ago", d.Hours()/24.0)
	}
}
<|endoftext|>