<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"io"
	"os"
	"os/exec"
	"regexp"
	"strconv"
	"strings"

	"golang.org/x/oauth2"

	"github.com/google/go-github/github"
	"github.com/haya14busa/errorformat"
	"github.com/haya14busa/watchdogs"
	"github.com/mattn/go-shellwords"
)

const usageMessage = "" +
	`Usage:	watchdogs [flags]
	watchdogs accepts any compiler or linter results from stdin and filters
	them by diff for review. watchdogs also can posts the results as a comment to
	GitHub if you use watchdogs in CI service.
`

// flags
var (
	diffCmd    string
	diffCmdDoc = `diff command (e.g. "git diff"). diff flag is ignored if you pass "ci" flag`

	diffStrip int
	efms      strslice

	ci    string
	ciDoc = `CI service (supported travis, circle-ci, droneio(OSS 0.4), common)
	If you use "ci" flag, you need to set WATCHDOGS_GITHUB_API_TOKEN environment
	variable.  Go to https://github.com/settings/tokens and create new Personal
	access token with repo scope.

	"common" requires following environment variables
		CI_PULL_REQUEST	Pull Request number (e.g. 14)
		CI_COMMIT	SHA1 for the current build
		CI_REPO_OWNER	repository owner (e.g. "haya14busa" for https://github.com/haya14busa/watchdogs)
		CI_REPO_NAME	repository name (e.g. "watchdogs" for https://github.com/haya14busa/watchdogs)
`
)

func init() {
	flag.StringVar(&diffCmd, "diff", "", diffCmdDoc)
	flag.IntVar(&diffStrip, "strip", 1, "strip NUM leading components from diff file names (equivalent to `patch -p`) (default is 1 for git diff)")
	flag.Var(&efms, "efm", "list of errorformat (https://github.com/haya14busa/errorformat)")
	flag.StringVar(&ci, "ci", "", ciDoc)
}

func usage() {
	fmt.Fprintln(os.Stderr, usageMessage)
	fmt.Fprintln(os.Stderr, "Flags:")
	flag.PrintDefaults()
	os.Exit(2)
}

func main() {
	flag.Usage = usage
	flag.Parse()
	if err := run(os.Stdin, os.Stdout, diffCmd, diffStrip, efms, ci); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func run(r io.Reader, w io.Writer, diffCmd string, diffStrip int, efms []string, ci string) error {
	p, err := efmParser(efms)
	if err != nil {
		return err
	}

	var cs watchdogs.CommentService
	var ds watchdogs.DiffService

	if ci != "" {
		if os.Getenv("WATCHDOGS_GITHUB_API_TOKEN") != "" {
			gs, isPR, err := githubService(ci)
			if err != nil {
				return err
			}
			if !isPR {
				fmt.Fprintf(os.Stderr, "this is not PullRequest build. CI: %v\n", ci)
				return nil
			}
			cs = gs
			ds = gs
		} else {
			fmt.Fprintf(os.Stderr, "WATCHDOGS_GITHUB_API_TOKEN is not set\n")
			return nil
		}
	} else {
		// local
		cs = watchdogs.NewCommentWriter(w)
		d, err := diffService(diffCmd, diffStrip)
		if err != nil {
			return err
		}
		ds = d
	}

	app := watchdogs.NewWatchdogs(p, cs, ds)
	if err := app.Run(r); err != nil {
		return err
	}
	if fcs, ok := cs.(FlashCommentService); ok {
		// Output log to writer
		for _, c := range fcs.ListPostComments() {
			fmt.Fprintln(w, strings.Join(c.Lines, "\n"))
		}
		return fcs.Flash()
	}
	return nil
}

// FlashCommentService is CommentService which uses Flash method to post comment.
type FlashCommentService interface {
	watchdogs.CommentService
	ListPostComments() []*watchdogs.Comment
	Flash() error
}

func efmParser(efms []string) (watchdogs.Parser, error) {
	efm, err := errorformat.NewErrorformat(efms)
	if err != nil {
		return nil, err
	}
	return watchdogs.NewErrorformatParser(efm), nil
}

func diffService(s string, strip int) (watchdogs.DiffService, error) {
	cmds, err := shellwords.Parse(s)
	if err != nil {
		return nil, err
	}
	if len(cmds) < 1 {
		return nil, errors.New("diff command is empty")
	}
	cmd := exec.Command(cmds[0], cmds[1:]...)
	d := watchdogs.NewDiffCmd(cmd, strip)
	return d, nil
}

func githubService(ci string) (githubservice *watchdogs.GitHubPullRequest, isPR bool, err error) {
	token, err := nonEmptyEnv("WATCHDOGS_GITHUB_API_TOKEN")
	if err != nil {
		return nil, false, err
	}
	var g *GitHubPR
	switch ci {
	case "travis":
		g, isPR, err = travis()
	case "circle-ci":
		g, isPR, err = circleci()
	case "droneio":
		g, isPR, err = droneio()
	case "common":
		g, isPR, err = commonci()
	default:
		return nil, false, fmt.Errorf("unsupported CI: %v", ci)
	}
	if err != nil {
		return nil, false, err
	}
	// TODO: support commit build
	if !isPR {
		return nil, false, nil
	}
	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: token},
	)
	tc := oauth2.NewClient(oauth2.NoContext, ts)
	client := github.NewClient(tc)
	githubservice = watchdogs.NewGitHubPullReqest(client, g.owner, g.repo, g.pr, g.sha)
	return githubservice, true, nil
}

func travis() (g *GitHubPR, isPR bool, err error) {
	prs := os.Getenv("TRAVIS_PULL_REQUEST")
	if prs == "false" {
		return nil, false, nil
	}
	pr, err := strconv.Atoi(prs)
	if err != nil {
		return nil, true, fmt.Errorf("unexpected env variable. TRAVIS_PULL_REQUEST=%v", prs)
	}
	reposlug, err := nonEmptyEnv("TRAVIS_REPO_SLUG")
	if err != nil {
		return nil, true, err
	}
	rss := strings.SplitN(reposlug, "/", 2)
	if len(rss) < 2 {
		return nil, true, fmt.Errorf("unexpected env variable. TRAVIS_REPO_SLUG=%v", reposlug)
	}
	owner, repo := rss[0], rss[1]

	sha, err := nonEmptyEnv("TRAVIS_PULL_REQUEST_SHA")
	if err != nil {
		return nil, true, err
	}

	g = &GitHubPR{
		owner: owner,
		repo:  repo,
		pr:    pr,
		sha:   sha,
	}
	return g, true, nil
}

// https://circleci.com/docs/environment-variables/
func circleci() (g *GitHubPR, isPR bool, err error) {
	var prs string // pull request number in string
	// For Pull Request from a same repository
	// e.g. https: //github.com/haya14busa/watchdogs/pull/6
	// it might be better to support CI_PULL_REQUESTS instead.
	prs = os.Getenv("CI_PULL_REQUEST")
	if prs == "" {
		// For Pull Request by a fork repository
		// e.g. 6
		prs = os.Getenv("CIRCLE_PR_NUMBER")
	}
	if prs == "" {
		// not a pull-request build
		return nil, false, nil
	}
	// regexp.MustCompile() in func intentionally because this func is called
	// once for one run.
	re := regexp.MustCompile(`[1-9]\d*$`)
	prm := re.FindString(prs)
	pr, err := strconv.Atoi(prm)
	if err != nil {
		return nil, true, fmt.Errorf("unexpected env variable (CI_PULL_REQUEST or CIRCLE_PR_NUMBER): %v", prs)
	}
	owner, err := nonEmptyEnv("CIRCLE_PROJECT_USERNAME")
	if err != nil {
		return nil, true, err
	}
	repo, err := nonEmptyEnv("CIRCLE_PROJECT_REPONAME")
	if err != nil {
		return nil, true, err
	}
	sha, err := nonEmptyEnv("CIRCLE_SHA1")
	if err != nil {
		return nil, true, err
	}
	g = &GitHubPR{
		owner: owner,
		repo:  repo,
		pr:    pr,
		sha:   sha,
	}
	return g, true, nil
}

// http://readme.drone.io/usage/variables/
func droneio() (g *GitHubPR, isPR bool, err error) {
	var prs string // pull request number in string
	prs = os.Getenv("DRONE_PULL_REQUEST")
	if prs == "" {
		// not a pull-request build
		return nil, false, nil
	}
	pr, err := strconv.Atoi(prs)
	if err != nil {
		return nil, true, fmt.Errorf("unexpected env variable (DRONE_PULL_REQUEST): %v", prs)
	}
	reposlug, err := nonEmptyEnv("DRONE_REPO") // e.g. haya14busa/watchdogs
	if err != nil {
		return nil, true, err
	}
	rss := strings.SplitN(reposlug, "/", 2)
	if len(rss) < 2 {
		return nil, true, fmt.Errorf("unexpected env variable. DRONE_REPO=%v", reposlug)
	}
	owner, repo := rss[0], rss[1]
	sha, err := nonEmptyEnv("DRONE_COMMIT")
	if err != nil {
		return nil, true, err
	}
	g = &GitHubPR{
		owner: owner,
		repo:  repo,
		pr:    pr,
		sha:   sha,
	}
	return g, true, nil
}

func commonci() (g *GitHubPR, isPR bool, err error) {
	var prs string // pull request number in string
	prs = os.Getenv("CI_PULL_REQUEST")
	if prs == "" {
		// not a pull-request build
		return nil, false, nil
	}
	pr, err := strconv.Atoi(prs)
	if err != nil {
		return nil, true, fmt.Errorf("unexpected env variable (CI_PULL_REQUEST): %v", prs)
	}
	owner, err := nonEmptyEnv("CI_REPO_OWNER")
	if err != nil {
		return nil, true, err
	}
	repo, err := nonEmptyEnv("CI_REPO_NAME")
	if err != nil {
		return nil, true, err
	}
	sha, err := nonEmptyEnv("CI_COMMIT")
	if err != nil {
		return nil, true, err
	}
	g = &GitHubPR{
		owner: owner,
		repo:  repo,
		pr:    pr,
		sha:   sha,
	}
	return g, true, nil
}

// GitHubPR represents required information about GitHub PullRequest.
type GitHubPR struct {
	owner string
	repo  string
	pr    int
	sha   string
}

func nonEmptyEnv(env string) (string, error) {
	v := os.Getenv(env)
	if v == "" {
		return "", fmt.Errorf("environment variable $%v is not set", env)
	}
	return v, nil
}

type strslice []string

func (ss *strslice) String() string {
	return fmt.Sprintf("%v", *ss)
}

func (ss *strslice) Set(value string) error {
	*ss = append(*ss, value)
	return nil
}
<commit_msg>revert: test<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"io"
	"os"
	"os/exec"
	"regexp"
	"strconv"
	"strings"

	"golang.org/x/oauth2"

	"github.com/google/go-github/github"
	"github.com/haya14busa/errorformat"
	"github.com/haya14busa/watchdogs"
	"github.com/mattn/go-shellwords"
)

// TODO(#14): remove
func F() {
}

const usageMessage = "" +
	`Usage:	watchdogs [flags]
	watchdogs accepts any compiler or linter results from stdin and filters
	them by diff for review. watchdogs also can posts the results as a comment to
	GitHub if you use watchdogs in CI service.
`

// flags
var (
	diffCmd    string
	diffCmdDoc = `diff command (e.g. "git diff"). diff flag is ignored if you pass "ci" flag`

	diffStrip int
	efms      strslice

	ci    string
	ciDoc = `CI service (supported travis, circle-ci, droneio(OSS 0.4), common)
	If you use "ci" flag, you need to set WATCHDOGS_GITHUB_API_TOKEN environment
	variable.  Go to https://github.com/settings/tokens and create new Personal
	access token with repo scope.

	"common" requires following environment variables
		CI_PULL_REQUEST	Pull Request number (e.g. 14)
		CI_COMMIT	SHA1 for the current build
		CI_REPO_OWNER	repository owner (e.g. "haya14busa" for https://github.com/haya14busa/watchdogs)
		CI_REPO_NAME	repository name (e.g. "watchdogs" for https://github.com/haya14busa/watchdogs)
`
)

func init() {
	flag.StringVar(&diffCmd, "diff", "", diffCmdDoc)
	flag.IntVar(&diffStrip, "strip", 1, "strip NUM leading components from diff file names (equivalent to `patch -p`) (default is 1 for git diff)")
	flag.Var(&efms, "efm", "list of errorformat (https://github.com/haya14busa/errorformat)")
	flag.StringVar(&ci, "ci", "", ciDoc)
}

func usage() {
	fmt.Fprintln(os.Stderr, usageMessage)
	fmt.Fprintln(os.Stderr, "Flags:")
	flag.PrintDefaults()
	os.Exit(2)
}

func main() {
	flag.Usage = usage
	flag.Parse()
	if err := run(os.Stdin, os.Stdout, diffCmd, diffStrip, efms, ci); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func run(r io.Reader, w io.Writer, diffCmd string, diffStrip int, efms []string, ci string) error {
	p, err := efmParser(efms)
	if err != nil {
		return err
	}

	var cs watchdogs.CommentService
	var ds watchdogs.DiffService

	if ci != "" {
		if os.Getenv("WATCHDOGS_GITHUB_API_TOKEN") != "" {
			gs, isPR, err := githubService(ci)
			if err != nil {
				return err
			}
			if !isPR {
				fmt.Fprintf(os.Stderr, "this is not PullRequest build. CI: %v\n", ci)
				return nil
			}
			cs = gs
			ds = gs
		} else {
			fmt.Fprintf(os.Stderr, "WATCHDOGS_GITHUB_API_TOKEN is not set\n")
			return nil
		}
	} else {
		// local
		cs = watchdogs.NewCommentWriter(w)
		d, err := diffService(diffCmd, diffStrip)
		if err != nil {
			return err
		}
		ds = d
	}

	app := watchdogs.NewWatchdogs(p, cs, ds)
	if err := app.Run(r); err != nil {
		return err
	}
	if fcs, ok := cs.(FlashCommentService); ok {
		// Output log to writer
		for _, c := range fcs.ListPostComments() {
			fmt.Fprintln(w, strings.Join(c.Lines, "\n"))
		}
		return fcs.Flash()
	}
	return nil
}

// FlashCommentService is CommentService which uses Flash method to post comment.
type FlashCommentService interface {
	watchdogs.CommentService
	ListPostComments() []*watchdogs.Comment
	Flash() error
}

func efmParser(efms []string) (watchdogs.Parser, error) {
	efm, err := errorformat.NewErrorformat(efms)
	if err != nil {
		return nil, err
	}
	return watchdogs.NewErrorformatParser(efm), nil
}

func diffService(s string, strip int) (watchdogs.DiffService, error) {
	cmds, err := shellwords.Parse(s)
	if err != nil {
		return nil, err
	}
	if len(cmds) < 1 {
		return nil, errors.New("diff command is empty")
	}
	cmd := exec.Command(cmds[0], cmds[1:]...)
	d := watchdogs.NewDiffCmd(cmd, strip)
	return d, nil
}

func githubService(ci string) (githubservice *watchdogs.GitHubPullRequest, isPR bool, err error) {
	token, err := nonEmptyEnv("WATCHDOGS_GITHUB_API_TOKEN")
	if err != nil {
		return nil, false, err
	}
	var g *GitHubPR
	switch ci {
	case "travis":
		g, isPR, err = travis()
	case "circle-ci":
		g, isPR, err = circleci()
	case "droneio":
		g, isPR, err = droneio()
	case "common":
		g, isPR, err = commonci()
	default:
		return nil, false, fmt.Errorf("unsupported CI: %v", ci)
	}
	if err != nil {
		return nil, false, err
	}
	// TODO: support commit build
	if !isPR {
		return nil, false, nil
	}
	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: token},
	)
	tc := oauth2.NewClient(oauth2.NoContext, ts)
	client := github.NewClient(tc)
	githubservice = watchdogs.NewGitHubPullReqest(client, g.owner, g.repo, g.pr, g.sha)
	return githubservice, true, nil
}

func travis() (g *GitHubPR, isPR bool, err error) {
	prs := os.Getenv("TRAVIS_PULL_REQUEST")
	if prs == "false" {
		return nil, false, nil
	}
	pr, err := strconv.Atoi(prs)
	if err != nil {
		return nil, true, fmt.Errorf("unexpected env variable. TRAVIS_PULL_REQUEST=%v", prs)
	}
	reposlug, err := nonEmptyEnv("TRAVIS_REPO_SLUG")
	if err != nil {
		return nil, true, err
	}
	rss := strings.SplitN(reposlug, "/", 2)
	if len(rss) < 2 {
		return nil, true, fmt.Errorf("unexpected env variable. TRAVIS_REPO_SLUG=%v", reposlug)
	}
	owner, repo := rss[0], rss[1]

	sha, err := nonEmptyEnv("TRAVIS_PULL_REQUEST_SHA")
	if err != nil {
		return nil, true, err
	}

	g = &GitHubPR{
		owner: owner,
		repo:  repo,
		pr:    pr,
		sha:   sha,
	}
	return g, true, nil
}

// https://circleci.com/docs/environment-variables/
func circleci() (g *GitHubPR, isPR bool, err error) {
	var prs string // pull request number in string
	// For Pull Request from a same repository
	// e.g. https: //github.com/haya14busa/watchdogs/pull/6
	// it might be better to support CI_PULL_REQUESTS instead.
	prs = os.Getenv("CI_PULL_REQUEST")
	if prs == "" {
		// For Pull Request by a fork repository
		// e.g. 6
		prs = os.Getenv("CIRCLE_PR_NUMBER")
	}
	if prs == "" {
		// not a pull-request build
		return nil, false, nil
	}
	// regexp.MustCompile() in func intentionally because this func is called
	// once for one run.
	re := regexp.MustCompile(`[1-9]\d*$`)
	prm := re.FindString(prs)
	pr, err := strconv.Atoi(prm)
	if err != nil {
		return nil, true, fmt.Errorf("unexpected env variable (CI_PULL_REQUEST or CIRCLE_PR_NUMBER): %v", prs)
	}
	owner, err := nonEmptyEnv("CIRCLE_PROJECT_USERNAME")
	if err != nil {
		return nil, true, err
	}
	repo, err := nonEmptyEnv("CIRCLE_PROJECT_REPONAME")
	if err != nil {
		return nil, true, err
	}
	sha, err := nonEmptyEnv("CIRCLE_SHA1")
	if err != nil {
		return nil, true, err
	}
	g = &GitHubPR{
		owner: owner,
		repo:  repo,
		pr:    pr,
		sha:   sha,
	}
	return g, true, nil
}

// http://readme.drone.io/usage/variables/
func droneio() (g *GitHubPR, isPR bool, err error) {
	var prs string // pull request number in string
	prs = os.Getenv("DRONE_PULL_REQUEST")
	if prs == "" {
		// not a pull-request build
		return nil, false, nil
	}
	pr, err := strconv.Atoi(prs)
	if err != nil {
		return nil, true, fmt.Errorf("unexpected env variable (DRONE_PULL_REQUEST): %v", prs)
	}
	reposlug, err := nonEmptyEnv("DRONE_REPO") // e.g. haya14busa/watchdogs
	if err != nil {
		return nil, true, err
	}
	rss := strings.SplitN(reposlug, "/", 2)
	if len(rss) < 2 {
		return nil, true, fmt.Errorf("unexpected env variable. DRONE_REPO=%v", reposlug)
	}
	owner, repo := rss[0], rss[1]
	sha, err := nonEmptyEnv("DRONE_COMMIT")
	if err != nil {
		return nil, true, err
	}
	g = &GitHubPR{
		owner: owner,
		repo:  repo,
		pr:    pr,
		sha:   sha,
	}
	return g, true, nil
}

func commonci() (g *GitHubPR, isPR bool, err error) {
	var prs string // pull request number in string
	prs = os.Getenv("CI_PULL_REQUEST")
	if prs == "" {
		// not a pull-request build
		return nil, false, nil
	}
	pr, err := strconv.Atoi(prs)
	if err != nil {
		return nil, true, fmt.Errorf("unexpected env variable (CI_PULL_REQUEST): %v", prs)
	}
	owner, err := nonEmptyEnv("CI_REPO_OWNER")
	if err != nil {
		return nil, true, err
	}
	repo, err := nonEmptyEnv("CI_REPO_NAME")
	if err != nil {
		return nil, true, err
	}
	sha, err := nonEmptyEnv("CI_COMMIT")
	if err != nil {
		return nil, true, err
	}
	g = &GitHubPR{
		owner: owner,
		repo:  repo,
		pr:    pr,
		sha:   sha,
	}
	return g, true, nil
}

// GitHubPR represents required information about GitHub PullRequest.
type GitHubPR struct {
	owner string
	repo  string
	pr    int
	sha   string
}

func nonEmptyEnv(env string) (string, error) {
	v := os.Getenv(env)
	if v == "" {
		return "", fmt.Errorf("environment variable $%v is not set", env)
	}
	return v, nil
}

type strslice []string

func (ss *strslice) String() string {
	return fmt.Sprintf("%v", *ss)
}

func (ss *strslice) Set(value string) error {
	*ss = append(*ss, value)
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"github.com/tidwall/gjson"
	"github.com/whosonfirst/go-whosonfirst-crawl"
	"github.com/whosonfirst/go-whosonfirst-csv"
	"github.com/whosonfirst/globe"			// for to make DrawPreparedPaths public
	"github.com/whosonfirst/go-whosonfirst-uri"
	"image/color"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"strings"
	"time"
)

func DrawFeature(feature []byte, gl *globe.Globe) error {

	geom_type := gjson.GetBytes(feature, "geometry.type")

	if !geom_type.Exists() {
		return errors.New("Geometry is missing a type property")
	}

	coords := gjson.GetBytes(feature, "geometry.coordinates")

	if !coords.Exists() {
		return errors.New("Geometry is missing a coordinates property")
	}

	switch geom_type.String() {

	case "Point":

	/*
		lonlat := coords.Array()
		lat := lonlat[1].Float()
		lon := lonlat[0].Float()

		gl.DrawDot(lat, lon, 0.01, globe.Color(green))
		*/

	case "Polygon":

		paths := make([][]*globe.Point, 0)

		for _, ring := range coords.Array() {

			path := make([]*globe.Point, 0)

			for _, r := range ring.Array() {

				lonlat := r.Array()
				lat := lonlat[1].Float()
				lon := lonlat[0].Float()

				pt := globe.NewPoint(lat, lon)
				path = append(path, &pt)
			}

			paths = append(paths, path)
		}

		gl.DrawPaths(paths)

	case "MultiPolygon":
		// log.Println("Can't process MultiPolygon")

	default:
		return errors.New("Unsupported geometry type")
	}

	return nil
}

func main() {

	outfile := flag.String("out", "", "Where to write globe")
	size := flag.Int("size", 1600, "The size of the globe (in pixels)")
	mode := flag.String("mode", "meta", "... (default is 'meta' for one or more meta files)")

	center := flag.String("center", "", "")
	center_lat := flag.Float64("latitude", 37.755244, "")
	center_lon := flag.Float64("longitude", -122.447777, "")

	flag.Parse()

	if *center != "" {

		latlon := strings.Split(*center, ",")

		lat, err := strconv.ParseFloat(latlon[0], 64)

		if err != nil {
			log.Fatal(err)
		}

		lon, err := strconv.ParseFloat(latlon[1], 64)

		if err != nil {
			log.Fatal(err)
		}

		*center_lat = lat
		*center_lon = lon
	}

	green := color.NRGBA{0x00, 0x64, 0x3c, 192}
	g := globe.New()
	g.DrawGraticule(10.0)

	t1 := time.Now()

	if *mode == "meta" {

		for _, path := range flag.Args() {

			reader, err := csv.NewDictReaderFromPath(path)

			if err != nil {
				log.Fatal(err)
			}

			for {
				row, err := reader.Read()

				if err == io.EOF {
					break
				}

				if err != nil {
					log.Println(err, path)
					break
				}

				str_lat, ok := row["geom_latitude"]

				if !ok {
					continue
				}

				str_lon, ok := row["geom_longitude"]

				if !ok {
					continue
				}

				lat, err := strconv.ParseFloat(str_lat, 64)

				if err != nil {
					log.Println(err, str_lat)
					continue
				}

				lon, err := strconv.ParseFloat(str_lon, 64)

				if err != nil {
					log.Println(err, str_lon)
					continue
				}

				g.DrawDot(lat, lon, 0.01, globe.Color(green))
			}
		}

	} else if *mode == "repo" {

		for _, path := range flag.Args() {

			var cb = func(path string, info os.FileInfo) error {

				if info.IsDir() {
					return nil
				}

				is_wof, err := uri.IsWOFFile(path)

				if err != nil {
					log.Printf("unable to determine whether %s is a WOF file, because %s\n", path, err)
					return err
				}

				if !is_wof {
					return nil
				}

				is_alt, err := uri.IsAltFile(path)

				if err != nil {
					log.Printf("unable to determine whether %s is an alt (WOF) file, because %s\n", path, err)
					return err
				}

				if is_alt {
					return nil
				}

				fh, err := os.Open(path)

				if err != nil {
					log.Printf("failed to open %s, because %s\n", path, err)
					return err
				}

				defer fh.Close()

				feature, err := ioutil.ReadAll(fh)

				if err != nil {
					log.Printf("failed to read %s, because %s\n", path, err)
					return err
				}

				return DrawFeature(feature, g)
			}

			cr := crawl.NewCrawler(path)
			cr.Crawl(cb)
		}

	} else {

		log.Fatal("Invalid mode")
	}

	t2 := time.Since(t1)

	log.Printf("time to read all the things %v\n", t2)

	t3 := time.Now()

	g.CenterOn(*center_lat, *center_lon)
	err := g.SavePNG(*outfile, *size)

	t4 := time.Since(t3)

	log.Printf("time to draw all the things %v\n", t4)

	if err != nil {
		log.Fatal(err)
	}

}
<commit_msg>add DrawRow<commit_after>package main

import (
	"errors"
	"flag"
	"github.com/tidwall/gjson"
	"github.com/whosonfirst/globe" // for to make DrawPreparedPaths public
	"github.com/whosonfirst/go-whosonfirst-crawl"
	"github.com/whosonfirst/go-whosonfirst-csv"
	"github.com/whosonfirst/go-whosonfirst-uri"
	"image/color"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

func DrawFeature(feature []byte, gl *globe.Globe) error {

	geom_type := gjson.GetBytes(feature, "geometry.type")

	if !geom_type.Exists() {
		return errors.New("Geometry is missing a type property")
	}

	coords := gjson.GetBytes(feature, "geometry.coordinates")

	if !coords.Exists() {
		return errors.New("Geometry is missing a coordinates property")
	}

	switch geom_type.String() {

	case "Point":

	/*
		lonlat := coords.Array()
		lat := lonlat[1].Float()
		lon := lonlat[0].Float()

		gl.DrawDot(lat, lon, 0.01, globe.Color(green))
	*/

	case "Polygon":

		paths := make([][]*globe.Point, 0)

		for _, ring := range coords.Array() {

			path := make([]*globe.Point, 0)

			for _, r := range ring.Array() {

				lonlat := r.Array()
				lat := lonlat[1].Float()
				lon := lonlat[0].Float()

				pt := globe.NewPoint(lat, lon)
				path = append(path, &pt)
			}

			paths = append(paths, path)
		}

		gl.DrawPaths(paths)

	case "MultiPolygon":
		// log.Println("Can't process MultiPolygon")

	default:
		return errors.New("Unsupported geometry type")
	}

	return nil
}

func DrawRow(path string, row map[string]string, g *globe.Globe, throttle chan bool) error {

	<-throttle

	defer func() {
		throttle <- true
	}()

	rel_path, ok := row["path"]

	if !ok {
		log.Println("Missing path")
		return nil
	}

	meta := filepath.Dir(path)
	root := filepath.Dir(meta)
	data := filepath.Join(root, "data")

	abs_path := filepath.Join(data, rel_path)

	fh, err := os.Open(abs_path)

	if err != nil {
		log.Fatal("failed to open %s, because %s\n", abs_path, err)
	}

	defer fh.Close()

	feature, err := ioutil.ReadAll(fh)

	if err != nil {
		log.Fatal("failed to read %s, because %s\n", abs_path, err)
	}

	return DrawFeature(feature, g)
}

func main() {

	outfile := flag.String("out", "", "Where to write globe")
	size := flag.Int("size", 1600, "The size of the globe (in pixels)")
	mode := flag.String("mode", "meta", "... (default is 'meta' for one or more meta files)")

	feature := flag.Bool("feature", false, "...")

	center := flag.String("center", "", "")
	center_lat := flag.Float64("latitude", 37.755244, "")
	center_lon := flag.Float64("longitude", -122.447777, "")

	flag.Parse()

	if *center != "" {

		latlon := strings.Split(*center, ",")

		lat, err := strconv.ParseFloat(latlon[0], 64)

		if err != nil {
			log.Fatal(err)
		}

		lon, err := strconv.ParseFloat(latlon[1], 64)

		if err != nil {
			log.Fatal(err)
		}

		*center_lat = lat
		*center_lon = lon
	}

	green := color.NRGBA{0x00, 0x64, 0x3c, 192}
	g := globe.New()
	g.DrawGraticule(10.0)

	t1 := time.Now()

	if *mode == "meta" {

		max_fh := 10
		throttle := make(chan bool, max_fh)

		for i := 0; i < max_fh; i++ {
			throttle <- true
		}

		for _, path := range flag.Args() {

			reader, err := csv.NewDictReaderFromPath(path)

			if err != nil {
				log.Fatal(err)
			}

			for {
				row, err := reader.Read()

				if err == io.EOF {
					break
				}

				if err != nil {
					log.Println(err, path)
					break
				}

				if *feature {
					DrawRow(path, row, g, throttle)
					continue
				}

				str_lat, ok := row["geom_latitude"]

				if !ok {
					continue
				}

				str_lon, ok := row["geom_longitude"]

				if !ok {
					continue
				}

				lat, err := strconv.ParseFloat(str_lat, 64)

				if err != nil {
					log.Println(err, str_lat)
					continue
				}

				lon, err := strconv.ParseFloat(str_lon, 64)

				if err != nil {
					log.Println(err, str_lon)
					continue
				}

				g.DrawDot(lat, lon, 0.01, globe.Color(green))
			}
		}

	} else if *mode == "repo" {

		for _, path := range flag.Args() {

			var cb = func(path string, info os.FileInfo) error {

				if info.IsDir() {
					return nil
				}

				is_wof, err := uri.IsWOFFile(path)

				if err != nil {
					log.Printf("unable to determine whether %s is a WOF file, because %s\n", path, err)
					return err
				}

				if !is_wof {
					return nil
				}

				is_alt, err := uri.IsAltFile(path)

				if err != nil {
					log.Printf("unable to determine whether %s is an alt (WOF) file, because %s\n", path, err)
					return err
				}

				if is_alt {
					return nil
				}

				fh, err := os.Open(path)

				if err != nil {
					log.Printf("failed to open %s, because %s\n", path, err)
					return err
				}

				defer fh.Close()

				feature, err := ioutil.ReadAll(fh)

				if err != nil {
					log.Printf("failed to read %s, because %s\n", path, err)
					return err
				}

				return DrawFeature(feature, g)
			}

			cr := crawl.NewCrawler(path)
			cr.Crawl(cb)
		}

	} else {

		log.Fatal("Invalid mode")
	}

	t2 := time.Since(t1)

	log.Printf("time to read all the things %v\n", t2)

	t3 := time.Now()

	g.CenterOn(*center_lat, *center_lon)
	err := g.SavePNG(*outfile, *size)

	t4 := time.Since(t3)

	log.Printf("time to draw all the things %v\n", t4)

	if err != nil {
		log.Fatal(err)
	}

}
<|endoftext|>


<commit_before>// Implementation of MinHash with random permutation functions.

// The original MinHash paper:
// http://cs.brown.edu/courses/cs253/papers/nearduplicate.pdf

// The b-Bit MinWise Hashing:
// http://research.microsoft.com/pubs/120078/wfc0398-lips.pdf
// paper explains the idea behind the one-bit MinHash and storage space saving.
package minhash

import (
	"errors"
	"math"
	"math/big"
	"math/rand"
)

const (
	// The maximum size (in bit) for the 1-bit minhash
	BIT_ARRAY_SIZE = 128
	onebitMask     = uint32(0x1)
	mersennePrime  = (1 << 61) - 1
)

func popCount(bits uint32) uint32 {
	bits = (bits & 0x55555555) + (bits >> 1 & 0x55555555)
	bits = (bits & 0x33333333) + (bits >> 2 & 0x33333333)
	bits = (bits & 0x0f0f0f0f) + (bits >> 4 & 0x0f0f0f0f)
	bits = (bits & 0x00ff00ff) + (bits >> 8 & 0x00ff00ff)
	return (bits & 0x0000ffff) + (bits >> 16 & 0x0000ffff)
}

func popCountBig(bits *big.Int) int {
	result := 0
	for _, v := range bits.Bytes() {
		result += int(popCount(uint32(v)))
	}
	return result
}

// http://en.wikipedia.org/wiki/Universal_hashing
type permutation func(uint32) uint32

func createPermutation(a, b uint32, p uint64, m int) permutation {
	return func(x uint32) uint32 {
		return uint32((uint(a*x+b) % uint(p)) % uint(m))
	}
}

// The MinHash signagure
type MinHash struct {
	Permutations []permutation
	HashValues   []uint32
	Seed         int64
}

// Create a new MinHash signature.
// `seed` is used to generate random permutation functions.
// `numHashFunctions` number of permuation functions will
// be generated.
// Higher number of permutations results in better estimation,
// but reduces performance. 128 is a good number to start.
func New(seed int64, numHashFunctions int) (*MinHash, error) {
	if numHashFunctions <= 0 {
		return nil, errors.New("Cannot have non-positive number of permutations")
	}
	s := new(MinHash)
	s.HashValues = make([]uint32, numHashFunctions)
	s.Permutations = make([]permutation, numHashFunctions)
	s.Seed = seed
	rand.Seed(s.Seed)
	var a uint32
	for i := 0; i < numHashFunctions; i++ {
		s.HashValues[i] = math.MaxUint32
		for {
			a = rand.Uint32()
			if a != 0 {
				break
			}
		}
		s.Permutations[i] = createPermutation(a,
			rand.Uint32(), mersennePrime, (1 << 32))
	}
	return s, nil
}

// Consumes a 32-bit hash and then computes all permutations and retains
// the minimum value for each permutations.
// Using a good hash function is decisive in estimation accuracy. See
// http://programmers.stackexchange.com/a/145633.
func (sig *MinHash) Digest(hv uint32) {
	var phv uint32
	for i := range sig.Permutations {
		phv = (sig.Permutations[i])(hv)
		if phv < sig.HashValues[i] {
			sig.HashValues[i] = phv
		}
	}
}

// Compute the estimation of Jaccard Similarity among MinHash signatures.
func EstimateJaccard(sigs ...*MinHash) (float64, error) {
	if sigs == nil || len(sigs) == 0 {
		return 0.0, errors.New("Less than 2 MinHash signatures were given")
	}
	numPerm := len(sigs[0].Permutations)
	for _, sig := range sigs[1:] {
		if sigs[0].Seed != sig.Seed {
			return 0.0, errors.New("Cannot compare MinHash signatures with different seed")
		}
		if numPerm != len(sig.Permutations) {
			return 0.0, errors.New("Cannot compare MinHash signatures with different numbers of permutations")
		}
	}
	intersection := 0
	var currRowAgree int
	for i := 0; i < numPerm; i++ {
		currRowAgree = 1
		for _, sig := range sigs[1:] {
			if sigs[0].HashValues[i] != sig.HashValues[i] {
				currRowAgree = 0
				break
			}
		}
		intersection += currRowAgree
	}
	return float64(intersection) / float64(numPerm), nil
}

// For b-Bit MinHash see:
// http://research.microsoft.com/pubs/120078/wfc0398-lips.pdf
// This is the 1-bit signature that can be used to compute
// similarity with other 1-bit signatures.
// Using signature incurs some loss of precision, but reduced storage cost.
// Usually a good idea to use this when the actual Jaccard is > 0.5.
type OneBitMinHash struct {
	Size     int
	BitArray *big.Int
	Seed     int64
}

// Export the full MinHash signature to OneBitMinHash.
// Keeping only the lowest bit of every hash value.
// If the number of hash functions exceeds the maximum size
// of the bit array `BIT_ARRAY_SIZE`, only the first
// `BIT_ARRAY_SIZE` number of hash values will be exported.
func (sig *MinHash) ExportOneBit() *OneBitMinHash {
	var numExportedHashValues int
	if len(sig.Permutations) > BIT_ARRAY_SIZE {
		numExportedHashValues = BIT_ARRAY_SIZE
	} else {
		numExportedHashValues = len(sig.Permutations)
	}
	sigOneBit := OneBitMinHash{
		BitArray: big.NewInt(0),
		Seed:     sig.Seed,
		Size:     numExportedHashValues,
	}
	for i := 0; i < numExportedHashValues; i++ {
		sigOneBit.BitArray.SetBit(sigOneBit.BitArray, i, uint(sig.HashValues[i]&onebitMask))
	}
	return &sigOneBit
}

// Estimate Jaccard similarity of OneBitMinHash signatures
func EstimateJaccardOnBit(sigs ...*OneBitMinHash) (float64, error) {
	if sigs == nil || len(sigs) == 0 {
		return 0.0, errors.New("Less than 2 OneBitMinHash signatures were given")
	}
	for _, sig := range sigs[1:] {
		if sigs[0].Seed != sig.Seed {
			return 0.0, errors.New("Cannot compare OneBitMinHash signatures with different seed")
		}
		if sigs[0].Size != sig.Size {
			return 0.0, errors.New("Cannot compare OneBitMinHash signatures with different numbers of permutations")
		}
	}
	commonBits := big.NewInt(0)
	for _, sig := range sigs {
		commonBits.Xor(commonBits, sig.BitArray)
	}
	return 2.0 * (float64((sigs[0].Size-popCountBig(commonBits)))/float64(sigs[0].Size) - 0.5), nil
}
<commit_msg>fix doc<commit_after>// Implementation of MinHash with random permutation functions.
//
// The original MinHash paper:
// http://cs.brown.edu/courses/cs253/papers/nearduplicate.pdf
//
// The b-Bit MinWise Hashing:
// http://research.microsoft.com/pubs/120078/wfc0398-lips.pdf
// paper explains the idea behind the one-bit MinHash and storage space saving.
package minhash

import (
	"errors"
	"math"
	"math/big"
	"math/rand"
)

const (
	// The maximum size (in bit) for the 1-bit minhash
	BIT_ARRAY_SIZE = 128
	onebitMask     = uint32(0x1)
	mersennePrime  = (1 << 61) - 1
)

func popCount(bits uint32) uint32 {
	bits = (bits & 0x55555555) + (bits >> 1 & 0x55555555)
	bits = (bits & 0x33333333) + (bits >> 2 & 0x33333333)
	bits = (bits & 0x0f0f0f0f) + (bits >> 4 & 0x0f0f0f0f)
	bits = (bits & 0x00ff00ff) + (bits >> 8 & 0x00ff00ff)
	return (bits & 0x0000ffff) + (bits >> 16 & 0x0000ffff)
}

func popCountBig(bits *big.Int) int {
	result := 0
	for _, v := range bits.Bytes() {
		result += int(popCount(uint32(v)))
	}
	return result
}

// http://en.wikipedia.org/wiki/Universal_hashing
type permutation func(uint32) uint32

func createPermutation(a, b uint32, p uint64, m int) permutation {
	return func(x uint32) uint32 {
		return uint32((uint(a*x+b) % uint(p)) % uint(m))
	}
}

// The MinHash signagure
type MinHash struct {
	Permutations []permutation
	HashValues   []uint32
	Seed         int64
}

// Create a new MinHash signature.
// `seed` is used to generate random permutation functions.
// `numHashFunctions` number of permuation functions will
// be generated.
// Higher number of permutations results in better estimation,
// but reduces performance. 128 is a good number to start.
func New(seed int64, numHashFunctions int) (*MinHash, error) {
	if numHashFunctions <= 0 {
		return nil, errors.New("Cannot have non-positive number of permutations")
	}
	s := new(MinHash)
	s.HashValues = make([]uint32, numHashFunctions)
	s.Permutations = make([]permutation, numHashFunctions)
	s.Seed = seed
	rand.Seed(s.Seed)
	var a uint32
	for i := 0; i < numHashFunctions; i++ {
		s.HashValues[i] = math.MaxUint32
		for {
			a = rand.Uint32()
			if a != 0 {
				break
			}
		}
		s.Permutations[i] = createPermutation(a,
			rand.Uint32(), mersennePrime, (1 << 32))
	}
	return s, nil
}

// Consumes a 32-bit hash and then computes all permutations and retains
// the minimum value for each permutations.
// Using a good hash function is decisive in estimation accuracy. See
// http://programmers.stackexchange.com/a/145633.
func (sig *MinHash) Digest(hv uint32) {
	var phv uint32
	for i := range sig.Permutations {
		phv = (sig.Permutations[i])(hv)
		if phv < sig.HashValues[i] {
			sig.HashValues[i] = phv
		}
	}
}

// Compute the estimation of Jaccard Similarity among MinHash signatures.
func EstimateJaccard(sigs ...*MinHash) (float64, error) {
	if sigs == nil || len(sigs) == 0 {
		return 0.0, errors.New("Less than 2 MinHash signatures were given")
	}
	numPerm := len(sigs[0].Permutations)
	for _, sig := range sigs[1:] {
		if sigs[0].Seed != sig.Seed {
			return 0.0, errors.New("Cannot compare MinHash signatures with different seed")
		}
		if numPerm != len(sig.Permutations) {
			return 0.0, errors.New("Cannot compare MinHash signatures with different numbers of permutations")
		}
	}
	intersection := 0
	var currRowAgree int
	for i := 0; i < numPerm; i++ {
		currRowAgree = 1
		for _, sig := range sigs[1:] {
			if sigs[0].HashValues[i] != sig.HashValues[i] {
				currRowAgree = 0
				break
			}
		}
		intersection += currRowAgree
	}
	return float64(intersection) / float64(numPerm), nil
}

// For b-Bit MinHash see:
// http://research.microsoft.com/pubs/120078/wfc0398-lips.pdf
// This is the 1-bit signature that can be used to compute
// similarity with other 1-bit signatures.
// Using signature incurs some loss of precision, but reduced storage cost.
// Usually a good idea to use this when the actual Jaccard is > 0.5.
type OneBitMinHash struct {
	Size     int
	BitArray *big.Int
	Seed     int64
}

// Export the full MinHash signature to OneBitMinHash.
// Keeping only the lowest bit of every hash value.
// If the number of hash functions exceeds the maximum size
// of the bit array `BIT_ARRAY_SIZE`, only the first
// `BIT_ARRAY_SIZE` number of hash values will be exported.
func (sig *MinHash) ExportOneBit() *OneBitMinHash {
	var numExportedHashValues int
	if len(sig.Permutations) > BIT_ARRAY_SIZE {
		numExportedHashValues = BIT_ARRAY_SIZE
	} else {
		numExportedHashValues = len(sig.Permutations)
	}
	sigOneBit := OneBitMinHash{
		BitArray: big.NewInt(0),
		Seed:     sig.Seed,
		Size:     numExportedHashValues,
	}
	for i := 0; i < numExportedHashValues; i++ {
		sigOneBit.BitArray.SetBit(sigOneBit.BitArray, i, uint(sig.HashValues[i]&onebitMask))
	}
	return &sigOneBit
}

// Estimate Jaccard similarity of OneBitMinHash signatures
func EstimateJaccardOnBit(sigs ...*OneBitMinHash) (float64, error) {
	if sigs == nil || len(sigs) == 0 {
		return 0.0, errors.New("Less than 2 OneBitMinHash signatures were given")
	}
	for _, sig := range sigs[1:] {
		if sigs[0].Seed != sig.Seed {
			return 0.0, errors.New("Cannot compare OneBitMinHash signatures with different seed")
		}
		if sigs[0].Size != sig.Size {
			return 0.0, errors.New("Cannot compare OneBitMinHash signatures with different numbers of permutations")
		}
	}
	commonBits := big.NewInt(0)
	for _, sig := range sigs {
		commonBits.Xor(commonBits, sig.BitArray)
	}
	return 2.0 * (float64((sigs[0].Size-popCountBig(commonBits)))/float64(sigs[0].Size) - 0.5), nil
}
<|endoftext|>


<commit_before>// Commons for HTTP handling
package rpcserver

import (
	"bufio"
	"encoding/json"
	"fmt"
	"net"
	"net/http"
	"runtime/debug"
	"strings"
	"time"

	"github.com/pkg/errors"
	"golang.org/x/net/netutil"

	types "github.com/tendermint/tendermint/rpc/lib/types"
	"github.com/tendermint/tmlibs/log"
)

// Config is an RPC server configuration.
type Config struct {
	MaxOpenConnections int
}

// StartHTTPServer starts an HTTP server on listenAddr with the given handler.
// It wraps handler with RecoverAndLogHandler.
func StartHTTPServer(listenAddr string, handler http.Handler, logger log.Logger, config Config) (listener net.Listener, err error) {
	var proto, addr string
	parts := strings.SplitN(listenAddr, "://", 2)
	if len(parts) != 2 {
		return nil, errors.Errorf("Invalid listening address %s (use fully formed addresses, including the tcp:// or unix:// prefix)", listenAddr)
	}
	proto, addr = parts[0], parts[1]

	logger.Info(fmt.Sprintf("Starting RPC HTTP server on %s", listenAddr))
	listener, err = net.Listen(proto, addr)
	if err != nil {
		return nil, errors.Errorf("Failed to listen on %v: %v", listenAddr, err)
	}
	if config.MaxOpenConnections > 0 {
		listener = netutil.LimitListener(listener, config.MaxOpenConnections)
	}

	go func() {
		err := http.Serve(
			listener,
			RecoverAndLogHandler(handler, logger),
		)
		logger.Error("RPC HTTP server stopped", "err", err)
	}()
	return listener, nil
}

// StartHTTPAndTLSServer starts an HTTPS server on listenAddr with the given
// handler.
// It wraps handler with RecoverAndLogHandler.
func StartHTTPAndTLSServer(listenAddr string, handler http.Handler, certFile, keyFile string, logger log.Logger, config Config) (listener net.Listener, err error) {
	var proto, addr string
	parts := strings.SplitN(listenAddr, "://", 2)
	if len(parts) != 2 {
		return nil, errors.Errorf("Invalid listening address %s (use fully formed addresses, including the tcp:// or unix:// prefix)", listenAddr)
	}
	proto, addr = parts[0], parts[1]

	logger.Info(fmt.Sprintf("Starting RPC HTTPS server on %s (cert: %q, key: %q)", listenAddr, certFile, keyFile))
	listener, err = net.Listen(proto, addr)
	if err != nil {
		return nil, errors.Errorf("Failed to listen on %v: %v", listenAddr, err)
	}
	if config.MaxOpenConnections > 0 {
		listener = netutil.LimitListener(listener, config.MaxOpenConnections)
	}

	go func() {
		err := http.ServeTLS(
			listener,
			RecoverAndLogHandler(handler, logger),
			certFile,
			keyFile,
		)
		logger.Error("RPC HTTPS server stopped", "err", err)
	}()
	return listener, nil
}

func WriteRPCResponseHTTPError(w http.ResponseWriter, httpCode int, res types.RPCResponse) {
	jsonBytes, err := json.MarshalIndent(res, "", "  ")
	if err != nil {
		panic(err)
	}

	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(httpCode)
	w.Write(jsonBytes) // nolint: errcheck, gas
}

func WriteRPCResponseHTTP(w http.ResponseWriter, res types.RPCResponse) {
	jsonBytes, err := json.MarshalIndent(res, "", "  ")
	if err != nil {
		panic(err)
	}
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(200)
	w.Write(jsonBytes) // nolint: errcheck, gas
}

//-----------------------------------------------------------------------------

// Wraps an HTTP handler, adding error logging.
// If the inner function panics, the outer function recovers, logs, sends an
// HTTP 500 error response.
func RecoverAndLogHandler(handler http.Handler, logger log.Logger) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// Wrap the ResponseWriter to remember the status
		rww := &ResponseWriterWrapper{-1, w}
		begin := time.Now()

		// Common headers
		origin := r.Header.Get("Origin")
		rww.Header().Set("Access-Control-Allow-Origin", origin)
		rww.Header().Set("Access-Control-Allow-Credentials", "true")
		rww.Header().Set("Access-Control-Expose-Headers", "X-Server-Time")
		rww.Header().Set("X-Server-Time", fmt.Sprintf("%v", begin.Unix()))

		defer func() {
			// Send a 500 error if a panic happens during a handler.
			// Without this, Chrome & Firefox were retrying aborted ajax requests,
			// at least to my localhost.
			if e := recover(); e != nil {

				// If RPCResponse
				if res, ok := e.(types.RPCResponse); ok {
					WriteRPCResponseHTTP(rww, res)
				} else {
					// For the rest,
					logger.Error("Panic in RPC HTTP handler", "err", e, "stack", string(debug.Stack()))
					rww.WriteHeader(http.StatusInternalServerError)
					WriteRPCResponseHTTP(rww, types.RPCInternalError("", e.(error)))
				}
			}

			// Finally, log.
			durationMS := time.Since(begin).Nanoseconds() / 1000000
			if rww.Status == -1 {
				rww.Status = 200
			}
			logger.Info("Served RPC HTTP response",
				"method", r.Method, "url", r.URL,
				"status", rww.Status, "duration", durationMS,
				"remoteAddr", r.RemoteAddr,
			)
		}()

		handler.ServeHTTP(rww, r)
	})
}

// Remember the status for logging
type ResponseWriterWrapper struct {
	Status int
	http.ResponseWriter
}

func (w *ResponseWriterWrapper) WriteHeader(status int) {
	w.Status = status
	w.ResponseWriter.WriteHeader(status)
}

// implements http.Hijacker
func (w *ResponseWriterWrapper) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	return w.ResponseWriter.(http.Hijacker).Hijack()
}
<commit_msg>rpc: Break up long lines<commit_after>// Commons for HTTP handling
package rpcserver

import (
	"bufio"
	"encoding/json"
	"fmt"
	"net"
	"net/http"
	"runtime/debug"
	"strings"
	"time"

	"github.com/pkg/errors"
	"golang.org/x/net/netutil"

	types "github.com/tendermint/tendermint/rpc/lib/types"
	"github.com/tendermint/tmlibs/log"
)

// Config is an RPC server configuration.
type Config struct {
	MaxOpenConnections int
}

// StartHTTPServer starts an HTTP server on listenAddr with the given handler.
// It wraps handler with RecoverAndLogHandler.
func StartHTTPServer(
	listenAddr string,
	handler http.Handler,
	logger log.Logger,
	config Config,
) (listener net.Listener, err error) {
	var proto, addr string
	parts := strings.SplitN(listenAddr, "://", 2)
	if len(parts) != 2 {
		return nil, errors.Errorf(
			"Invalid listening address %s (use fully formed addresses, including the tcp:// or unix:// prefix)",
			listenAddr,
		)
	}
	proto, addr = parts[0], parts[1]

	logger.Info(fmt.Sprintf("Starting RPC HTTP server on %s", listenAddr))
	listener, err = net.Listen(proto, addr)
	if err != nil {
		return nil, errors.Errorf("Failed to listen on %v: %v", listenAddr, err)
	}
	if config.MaxOpenConnections > 0 {
		listener = netutil.LimitListener(listener, config.MaxOpenConnections)
	}

	go func() {
		err := http.Serve(
			listener,
			RecoverAndLogHandler(handler, logger),
		)
		logger.Error("RPC HTTP server stopped", "err", err)
	}()
	return listener, nil
}

// StartHTTPAndTLSServer starts an HTTPS server on listenAddr with the given
// handler.
// It wraps handler with RecoverAndLogHandler.
func StartHTTPAndTLSServer(
	listenAddr string,
	handler http.Handler,
	certFile, keyFile string,
	logger log.Logger,
	config Config,
) (listener net.Listener, err error) {
	var proto, addr string
	parts := strings.SplitN(listenAddr, "://", 2)
	if len(parts) != 2 {
		return nil, errors.Errorf(
			"Invalid listening address %s (use fully formed addresses, including the tcp:// or unix:// prefix)",
			listenAddr,
		)
	}
	proto, addr = parts[0], parts[1]

	logger.Info(
		fmt.Sprintf(
			"Starting RPC HTTPS server on %s (cert: %q, key: %q)",
			listenAddr,
			certFile,
			keyFile,
		),
	)
	listener, err = net.Listen(proto, addr)
	if err != nil {
		return nil, errors.Errorf("Failed to listen on %v: %v", listenAddr, err)
	}
	if config.MaxOpenConnections > 0 {
		listener = netutil.LimitListener(listener, config.MaxOpenConnections)
	}

	go func() {
		err := http.ServeTLS(
			listener,
			RecoverAndLogHandler(handler, logger),
			certFile,
			keyFile,
		)
		logger.Error("RPC HTTPS server stopped", "err", err)
	}()
	return listener, nil
}

func WriteRPCResponseHTTPError(
	w http.ResponseWriter,
	httpCode int,
	res types.RPCResponse,
) {
	jsonBytes, err := json.MarshalIndent(res, "", "  ")
	if err != nil {
		panic(err)
	}

	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(httpCode)
	w.Write(jsonBytes) // nolint: errcheck, gas
}

func WriteRPCResponseHTTP(w http.ResponseWriter, res types.RPCResponse) {
	jsonBytes, err := json.MarshalIndent(res, "", "  ")
	if err != nil {
		panic(err)
	}
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(200)
	w.Write(jsonBytes) // nolint: errcheck, gas
}

//-----------------------------------------------------------------------------

// Wraps an HTTP handler, adding error logging.
// If the inner function panics, the outer function recovers, logs, sends an
// HTTP 500 error response.
func RecoverAndLogHandler(handler http.Handler, logger log.Logger) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// Wrap the ResponseWriter to remember the status
		rww := &ResponseWriterWrapper{-1, w}
		begin := time.Now()

		// Common headers
		origin := r.Header.Get("Origin")
		rww.Header().Set("Access-Control-Allow-Origin", origin)
		rww.Header().Set("Access-Control-Allow-Credentials", "true")
		rww.Header().Set("Access-Control-Expose-Headers", "X-Server-Time")
		rww.Header().Set("X-Server-Time", fmt.Sprintf("%v", begin.Unix()))

		defer func() {
			// Send a 500 error if a panic happens during a handler.
			// Without this, Chrome & Firefox were retrying aborted ajax requests,
			// at least to my localhost.
			if e := recover(); e != nil {

				// If RPCResponse
				if res, ok := e.(types.RPCResponse); ok {
					WriteRPCResponseHTTP(rww, res)
				} else {
					// For the rest,
					logger.Error(
						"Panic in RPC HTTP handler", "err", e, "stack",
						string(debug.Stack()),
					)
					rww.WriteHeader(http.StatusInternalServerError)
					WriteRPCResponseHTTP(rww, types.RPCInternalError("", e.(error)))
				}
			}

			// Finally, log.
			durationMS := time.Since(begin).Nanoseconds() / 1000000
			if rww.Status == -1 {
				rww.Status = 200
			}
			logger.Info("Served RPC HTTP response",
				"method", r.Method, "url", r.URL,
				"status", rww.Status, "duration", durationMS,
				"remoteAddr", r.RemoteAddr,
			)
		}()

		handler.ServeHTTP(rww, r)
	})
}

// Remember the status for logging
type ResponseWriterWrapper struct {
	Status int
	http.ResponseWriter
}

func (w *ResponseWriterWrapper) WriteHeader(status int) {
	w.Status = status
	w.ResponseWriter.WriteHeader(status)
}

// implements http.Hijacker
func (w *ResponseWriterWrapper) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	return w.ResponseWriter.(http.Hijacker).Hijack()
}
<|endoftext|>


<commit_before>package permissions

import (
	"testing"
)

func TestPerm(t *testing.T) {
	userstate := NewUserStateSimple()

	userstate.AddUser("bob", "hunter1", "bob@zombo.com")

	if !userstate.HasUser("bob") {
		t.Error("Error, user bob should exist")
	}

	if userstate.IsConfirmed("bob") {
		t.Error("Error, user bob should not be confirmed right now.")
	}

	userstate.MarkConfirmed("bob")

	if !userstate.IsConfirmed("bob") {
		t.Error("Error, user bob should be marked as confirmed right now.")
	}

	if userstate.IsAdmin("bob") {
		t.Error("Error, user bob should not have admin rights")
	}

	userstate.SetAdminStatus("bob")

	if !userstate.IsAdmin("bob") {
		t.Error("Error, user bob should have admin rights")
	}

	userstate.RemoveUser("bob")

	if userstate.HasUser("bob") {
		t.Error("Error, user bob should not exist")
	}
}

func TestPasswordBasic(t *testing.T) {
	userstate := NewUserStateSimple()

	// Assert that the default password algorithm is "bcrypt+"
	if userstate.PasswordAlgo() != "bcrypt+" {
		t.Error("Error, bcrypt+ should be the default password algorithm")
	}

	// Set password algorithm
	userstate.SetPasswordAlgo("sha256")

	// Assert that the algorithm is now sha256
	if userstate.PasswordAlgo() != "sha256" {
		t.Error("Error, setting password algorithm failed")
	}

}

func TestPasswordBackward(t *testing.T) {
	userstate := NewUserStateSimple()
	userstate.SetPasswordAlgo("sha256")
	userstate.AddUser("bob", "hunter1", "bob@zombo.com")
	if !userstate.HasUser("bob") {
		t.Error("Error, user bob should exist")
	}
	userstate.SetPasswordAlgo("sha256")
	if !userstate.CorrectPassword("bob", "hunter1") {
		t.Error("Error, the sha256 password really is correct")
	}

	userstate.SetPasswordAlgo("bcrypt")
	if userstate.CorrectPassword("bob", "hunter1") {
		t.Error("Error, the password as stored as sha256, not bcrypt")
	}

	userstate.SetPasswordAlgo("bcrypt+")
	if !userstate.CorrectPassword("bob", "hunter1") {
		t.Error("Error, the sha256 password is not correct when checking with bcrypt+")
	}
}

func TestPasswordAlgoMatching(t *testing.T) {
	userstate := NewUserStateSimple()
	// generate two different password using the same credentials but different algos
	userstate.SetPasswordAlgo("sha256")
	sha256_hash := userstate.HashPassword("testuser@example.com", "textpassword")
	userstate.SetPasswordAlgo("bcrypt")
	bcrypt_hash := userstate.HashPassword("testuser@example.com", "textpassword")

	// they shouldn't match
	if sha256_hash == bcrypt_hash {
		t.Error("Error, different algorithms should not have a password match")
	}
}

func TestUserStateKeeper(t *testing.T) {
	userstate := NewUserStateSimple()
	// Check that the userstate qualifies for the UserStateKeeper interface
	var _ UserStateKeeper = userstate
}
<commit_msg>Additional tests<commit_after>package permissions

import (
	"testing"
)

func TestPerm(t *testing.T) {
	userstate := NewUserStateSimple()

	userstate.AddUser("bob", "hunter1", "bob@zombo.com")

	if !userstate.HasUser("bob") {
		t.Error("Error, user bob should exist")
	}

	if userstate.IsConfirmed("bob") {
		t.Error("Error, user bob should not be confirmed right now.")
	}

	userstate.MarkConfirmed("bob")

	if !userstate.IsConfirmed("bob") {
		t.Error("Error, user bob should be marked as confirmed right now.")
	}

	if userstate.IsAdmin("bob") {
		t.Error("Error, user bob should not have admin rights")
	}

	userstate.SetAdminStatus("bob")

	if !userstate.IsAdmin("bob") {
		t.Error("Error, user bob should have admin rights")
	}

	userstate.RemoveUser("bob")

	if userstate.HasUser("bob") {
		t.Error("Error, user bob should not exist")
	}
}

func TestPasswordBasic(t *testing.T) {
	userstate := NewUserStateSimple()

	// Assert that the default password algorithm is "bcrypt+"
	if userstate.PasswordAlgo() != "bcrypt+" {
		t.Error("Error, bcrypt+ should be the default password algorithm")
	}

	// Set password algorithm
	userstate.SetPasswordAlgo("sha256")

	// Assert that the algorithm is now sha256
	if userstate.PasswordAlgo() != "sha256" {
		t.Error("Error, setting password algorithm failed")
	}

}

// Check if the functionality for backwards compatible hashing works
func TestPasswordBackward(t *testing.T) {
	userstate := NewUserStateSimple()
	userstate.SetPasswordAlgo("sha256")
	userstate.AddUser("bob", "hunter1", "bob@zombo.com")
	if !userstate.HasUser("bob") {
		t.Error("Error, user bob should exist")
	}
	userstate.SetPasswordAlgo("sha256")
	if !userstate.CorrectPassword("bob", "hunter1") {
		t.Error("Error, the sha256 password really is correct")
	}

	userstate.SetPasswordAlgo("bcrypt")
	if userstate.CorrectPassword("bob", "hunter1") {
		t.Error("Error, the password as stored as sha256, not bcrypt")
	}

	userstate.SetPasswordAlgo("bcrypt+")
	if !userstate.CorrectPassword("bob", "hunter1") {
		t.Error("Error, the sha256 password is not correct when checking with bcrypt+")
	}

	userstate.RemoveUser("bob")
}

// Check if the functionality for backwards compatible hashing works
func TestPasswordNotBackward(t *testing.T) {
	userstate := NewUserStateSimple()
	userstate.SetPasswordAlgo("bcrypt")
	userstate.AddUser("bob", "hunter1", "bob@zombo.com")
	if !userstate.HasUser("bob") {
		t.Error("Error, user bob should exist")
	}
	userstate.SetPasswordAlgo("sha256")
	if userstate.CorrectPassword("bob", "hunter1") {
		t.Error("Error, the password is stored as bcrypt, should not be okay with sha256")
	}

	userstate.SetPasswordAlgo("bcrypt")
	if !userstate.CorrectPassword("bob", "hunter1") {
		t.Error("Error, the password should be correct when checking with bcrypt")
	}

	userstate.RemoveUser("bob")
}

func TestPasswordAlgoMatching(t *testing.T) {
	userstate := NewUserStateSimple()
	// generate two different password using the same credentials but different algos
	userstate.SetPasswordAlgo("sha256")
	sha256_hash := userstate.HashPassword("testuser@example.com", "textpassword")
	userstate.SetPasswordAlgo("bcrypt")
	bcrypt_hash := userstate.HashPassword("testuser@example.com", "textpassword")

	// they shouldn't match
	if sha256_hash == bcrypt_hash {
		t.Error("Error, different algorithms should not have a password match")
	}
}

func TestUserStateKeeper(t *testing.T) {
	userstate := NewUserStateSimple()
	// Check that the userstate qualifies for the UserStateKeeper interface
	var _ UserStateKeeper = userstate
}
<|endoftext|>


<commit_before>package permissions

import (
	"testing"

	"github.com/xyproto/db"
)

func TestPerm(t *testing.T) {
	//db.Verbose = true

	//userstate := NewUserStateSimple() // for localhost
	userstate := NewUserState("travis:@127.0.0.1/", true) // for travis-ci

	userstate.AddUser("bob", "hunter1", "bob@zombo.com")

	if !userstate.HasUser("bob") {
		t.Error("Error, user bob should exist")
	}

	if userstate.IsConfirmed("bob") {
		t.Error("Error, user bob should not be confirmed right now.")
	}

	userstate.MarkConfirmed("bob")

	if !userstate.IsConfirmed("bob") {
		t.Error("Error, user bob should be marked as confirmed right now.")
	}

	if userstate.IsAdmin("bob") {
		t.Error("Error, user bob should not have admin rights")
	}

	userstate.SetAdminStatus("bob")

	if !userstate.IsAdmin("bob") {
		t.Error("Error, user bob should have admin rights")
	}

	userstate.RemoveUser("bob")

	if userstate.HasUser("bob") {
		t.Error("Error, user bob should not exist")
	}
}

func TestPasswordBasic(t *testing.T) {
	userstate := NewUserStateSimple()

	// Assert that the default password algorithm is "bcrypt+"
	if userstate.PasswordAlgo() != "bcrypt+" {
		t.Error("Error, bcrypt+ should be the default password algorithm")
	}

	// Set password algorithm
	userstate.SetPasswordAlgo("sha256")

	// Assert that the algorithm is now sha256
	if userstate.PasswordAlgo() != "sha256" {
		t.Error("Error, setting password algorithm failed")
	}

}

// Check if the functionality for backwards compatible hashing works
func TestPasswordBackward(t *testing.T) {
	userstate := NewUserStateSimple()
	userstate.SetPasswordAlgo("sha256")
	userstate.AddUser("bob", "hunter1", "bob@zombo.com")
	if !userstate.HasUser("bob") {
		t.Error("Error, user bob should exist")
	}
	userstate.SetPasswordAlgo("sha256")
	if !userstate.CorrectPassword("bob", "hunter1") {
		t.Error("Error, the sha256 password really is correct")
	}

	userstate.SetPasswordAlgo("bcrypt")
	if userstate.CorrectPassword("bob", "hunter1") {
		t.Error("Error, the password as stored as sha256, not bcrypt")
	}

	userstate.SetPasswordAlgo("bcrypt+")
	if !userstate.CorrectPassword("bob", "hunter1") {
		t.Error("Error, the sha256 password is not correct when checking with bcrypt+")
	}

	userstate.RemoveUser("bob")
}

// Check if the functionality for backwards compatible hashing works
func TestPasswordNotBackward(t *testing.T) {
	userstate := NewUserStateSimple()
	userstate.SetPasswordAlgo("bcrypt")
	userstate.AddUser("bob", "hunter1", "bob@zombo.com")
	if !userstate.HasUser("bob") {
		t.Error("Error, user bob should exist")
	}
	userstate.SetPasswordAlgo("sha256")
	if userstate.CorrectPassword("bob", "hunter1") {
		t.Error("Error, the password is stored as bcrypt, should not be okay with sha256")
	}

	userstate.SetPasswordAlgo("bcrypt")
	if !userstate.CorrectPassword("bob", "hunter1") {
		t.Error("Error, the password should be correct when checking with bcrypt")
	}

	userstate.RemoveUser("bob")
}

func TestPasswordAlgoMatching(t *testing.T) {
	userstate := NewUserStateSimple()
	// generate two different password using the same credentials but different algos
	userstate.SetPasswordAlgo("sha256")
	sha256_hash := userstate.HashPassword("testuser@example.com", "textpassword")
	userstate.SetPasswordAlgo("bcrypt")
	bcrypt_hash := userstate.HashPassword("testuser@example.com", "textpassword")

	// they shouldn't match
	if sha256_hash == bcrypt_hash {
		t.Error("Error, different algorithms should not have a password match")
	}
}

func TestIUserState(t *testing.T) {
	userstate := NewUserStateSimple()
	// Check that the userstate qualifies for the IUserState interface
	var _ db.IUserState = userstate
}

func TestHostPassword(t *testing.T) {
	userstate := NewUserState(":@localhost/", true)

	userstate.AddUser("bob", "hunter1", "bob@zombo.com")
	if !userstate.HasUser("bob") {
		t.Error("Error, user bob should exist")
	}

	userstate.RemoveUser("bob")
	if userstate.HasUser("bob") {
		t.Error("Error, user bob should not exist")
	}
}
<commit_msg>For Travis CI<commit_after>package permissions

import (
	"testing"

	"github.com/xyproto/db"
)

func TestPerm(t *testing.T) {
	//db.Verbose = true

	//userstate := NewUserStateSimple() // for localhost
	userstate := NewUserState("travis:@127.0.0.1/", true) // for travis-ci

	userstate.AddUser("bob", "hunter1", "bob@zombo.com")

	if !userstate.HasUser("bob") {
		t.Error("Error, user bob should exist")
	}

	if userstate.IsConfirmed("bob") {
		t.Error("Error, user bob should not be confirmed right now.")
	}

	userstate.MarkConfirmed("bob")

	if !userstate.IsConfirmed("bob") {
		t.Error("Error, user bob should be marked as confirmed right now.")
	}

	if userstate.IsAdmin("bob") {
		t.Error("Error, user bob should not have admin rights")
	}

	userstate.SetAdminStatus("bob")

	if !userstate.IsAdmin("bob") {
		t.Error("Error, user bob should have admin rights")
	}

	userstate.RemoveUser("bob")

	if userstate.HasUser("bob") {
		t.Error("Error, user bob should not exist")
	}
}

func TestPasswordBasic(t *testing.T) {
	//userstate := NewUserStateSimple() // for localhost
	userstate := NewUserState("travis:@127.0.0.1/", true) // for travis-ci

	// Assert that the default password algorithm is "bcrypt+"
	if userstate.PasswordAlgo() != "bcrypt+" {
		t.Error("Error, bcrypt+ should be the default password algorithm")
	}

	// Set password algorithm
	userstate.SetPasswordAlgo("sha256")

	// Assert that the algorithm is now sha256
	if userstate.PasswordAlgo() != "sha256" {
		t.Error("Error, setting password algorithm failed")
	}

}

// Check if the functionality for backwards compatible hashing works
func TestPasswordBackward(t *testing.T) {
	//userstate := NewUserStateSimple() // for localhost
	userstate := NewUserState("travis:@127.0.0.1/", true) // for travis-ci

	userstate.SetPasswordAlgo("sha256")
	userstate.AddUser("bob", "hunter1", "bob@zombo.com")
	if !userstate.HasUser("bob") {
		t.Error("Error, user bob should exist")
	}
	userstate.SetPasswordAlgo("sha256")
	if !userstate.CorrectPassword("bob", "hunter1") {
		t.Error("Error, the sha256 password really is correct")
	}

	userstate.SetPasswordAlgo("bcrypt")
	if userstate.CorrectPassword("bob", "hunter1") {
		t.Error("Error, the password as stored as sha256, not bcrypt")
	}

	userstate.SetPasswordAlgo("bcrypt+")
	if !userstate.CorrectPassword("bob", "hunter1") {
		t.Error("Error, the sha256 password is not correct when checking with bcrypt+")
	}

	userstate.RemoveUser("bob")
}

// Check if the functionality for backwards compatible hashing works
func TestPasswordNotBackward(t *testing.T) {
	//userstate := NewUserStateSimple() // for localhost
	userstate := NewUserState("travis:@127.0.0.1/", true) // for travis-ci

	userstate.SetPasswordAlgo("bcrypt")
	userstate.AddUser("bob", "hunter1", "bob@zombo.com")
	if !userstate.HasUser("bob") {
		t.Error("Error, user bob should exist")
	}
	userstate.SetPasswordAlgo("sha256")
	if userstate.CorrectPassword("bob", "hunter1") {
		t.Error("Error, the password is stored as bcrypt, should not be okay with sha256")
	}

	userstate.SetPasswordAlgo("bcrypt")
	if !userstate.CorrectPassword("bob", "hunter1") {
		t.Error("Error, the password should be correct when checking with bcrypt")
	}

	userstate.RemoveUser("bob")
}

func TestPasswordAlgoMatching(t *testing.T) {
	//userstate := NewUserStateSimple() // for localhost
	userstate := NewUserState("travis:@127.0.0.1/", true) // for travis-ci

	// generate two different password using the same credentials but different algos
	userstate.SetPasswordAlgo("sha256")
	sha256_hash := userstate.HashPassword("testuser@example.com", "textpassword")
	userstate.SetPasswordAlgo("bcrypt")
	bcrypt_hash := userstate.HashPassword("testuser@example.com", "textpassword")

	// they shouldn't match
	if sha256_hash == bcrypt_hash {
		t.Error("Error, different algorithms should not have a password match")
	}
}

func TestIUserState(t *testing.T) {
	//userstate := NewUserStateSimple() // for localhost
	userstate := NewUserState("travis:@127.0.0.1/", true) // for travis-ci

	// Check that the userstate qualifies for the IUserState interface
	var _ db.IUserState = userstate
}

func TestHostPassword(t *testing.T) {
	//userstate := NewUserStateSimple() // for localhost
	userstate := NewUserState("travis:@127.0.0.1/", true) // for travis-ci

	userstate.AddUser("bob", "hunter1", "bob@zombo.com")
	if !userstate.HasUser("bob") {
		t.Error("Error, user bob should exist")
	}

	userstate.RemoveUser("bob")
	if userstate.HasUser("bob") {
		t.Error("Error, user bob should not exist")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
)

const (
	VERSION = "0.1"
)

var (
	formatFlag = flag.String("format", "csv", "Name of formatter")
	fatalFlag  = flag.Bool("fatal", false, "Do not continue on error")
	helpFlag   = flag.Bool("help", false, "Show verbose help")
)

func main() {
	flag.Parse()

	if *helpFlag || flag.NArg() != 1 {
		fmt.Println("Usage: jsonformat [options] <format string>")
		flag.PrintDefaults()
		fmt.Println("Formatters:")
		for name, format := range Formats {
			fmt.Printf("\t\"%s\": %s\n", name, format.Description)
		}
		fmt.Printf("Version %s\n", VERSION)
		return
	}

	format, ok := Formats[*formatFlag]
	if !ok {
		log.Fatalf("Unknown format %s", *formatFlag)
	}
	f, err := format.Compiler(flag.Arg(0))
	if err != nil {
		log.Fatalf("Template invalid: %s", err)
	}

	dec := json.NewDecoder(os.Stdin)
	logFn := NewLogFn(*fatalFlag)
	for {
		var input interface{}
		err := dec.Decode(&input)
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatalf("Could not decode input: %s", err)
			continue
		}
		err = f.Execute(os.Stdout, input)
		if err != nil {
			logFn("Could not apply input to template: %s", err)
			continue
		}
		io.WriteString(os.Stdout, "\n")
	}
}

type LogFn func(format string, v ...interface{})

func NewLogFn(fatal bool) LogFn {
	if fatal {
		return func(format string, v ...interface{}) {
			log.Fatalf(format, v...)
		}
	}
	return func(format string, v ...interface{}) {
		log.Printf(format, v...)
	}
}
<commit_msg>Add support for file input<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
)

const (
	VERSION = "0.1"
)

var (
	formatFlag = flag.String("format", "csv", "Name of formatter")
	fatalFlag  = flag.Bool("fatal", false, "Do not continue on error")
	inputFlag  = flag.String("input", "", "Read input from file instead of stdin")
	helpFlag   = flag.Bool("help", false, "Show verbose help")
)

func main() {
	flag.Parse()

	if *helpFlag || (flag.NArg() != 1 && len(*inputFlag) <= 0) {
		fmt.Println("Usage: jsonformat [options] <format string>")
		flag.PrintDefaults()
		fmt.Println("Formatters:")
		for name, format := range Formats {
			fmt.Printf("\t\"%s\": %s\n", name, format.Description)
		}
		fmt.Printf("Version %s\n", VERSION)
		return
	}

	format, ok := Formats[*formatFlag]
	if !ok {
		log.Fatalf("Unknown format %s", *formatFlag)
	}

	f, err := format.Compiler(formatString())
	if err != nil {
		log.Fatalf("Template invalid: %s", err)
	}

	dec := json.NewDecoder(os.Stdin)
	logFn := NewLogFn(*fatalFlag)
	for {
		var input interface{}
		err := dec.Decode(&input)
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatalf("Could not decode input: %s", err)
			continue
		}
		err = f.Execute(os.Stdout, input)
		if err != nil {
			logFn("Could not apply input to template: %s", err)
			continue
		}
		io.WriteString(os.Stdout, "\n")
	}
}

type LogFn func(format string, v ...interface{})

func NewLogFn(fatal bool) LogFn {
	if fatal {
		return func(format string, v ...interface{}) {
			log.Fatalf(format, v...)
		}
	}
	return func(format string, v ...interface{}) {
		log.Printf(format, v...)
	}
}

func formatString() string {
	if len(*inputFlag) > 0 {
		d, e := ioutil.ReadFile(*inputFlag)
		if e != nil {
			log.Fatalf("Could not read file \"%s\": %s", *inputFlag, e)
		}
		return string(d)
	}
	return flag.Arg(0)
}
<|endoftext|>


<commit_before>package kafka_avro

import (
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"
	"time"

	avro "github.com/elodina/go-avro"
	kafkaavro "github.com/elodina/go-kafka-avro"
	"github.com/gliderlabs/logspout/router"
	"gopkg.in/Shopify/sarama.v1"
)

var messageSchema = `{
  "type": "record",
  "name": "LogLine",
  "fields": [
    {"name": "timestamp", "type": "string"},
    {"name": "container_name", "type": "string"},
    {"name": "source", "type": "string"},
    {"name": "line", "type": "string"}
  ]
}`

func init() {
	router.AdapterFactories.Register(NewKafkaAvroAdapter, "kafka_avro")
}

type KafkaAvroAdapter struct {
	route    *router.Route
	brokers  []string
	topic    string
	schema   *avro.Schema
	registry *kafkaavro.KafkaAvroEncoder
	producer *sarama.AsyncProducer
}

func NewKafkaAvroAdapter(route *router.Route) (router.LogAdapter, error) {
	brokers := readBrokers(route.Address)
	if len(brokers) == 0 {
		return nil, errorf("The Kafka broker host:port is missing. Did you specify it as a route address?")
	}

	topic := readTopic(route.Address, route.Options)
	if topic == "" {
		return nil, errorf("The Kafka topic is missing. Did you specify it as a route option?")
	}

	schemaUrl := readSchemaRegistryUrl(route.Options)
	if schemaUrl == "" {
		return nil, errorf("The schema registry url is missing. Did you specify it as a route option?")
	}

	registry := kafkaavro.NewKafkaAvroEncoder(schemaUrl)

	schema, err := avro.ParseSchema(messageSchema)
	if err != nil {
		return nil, errorf("The schema could not be parsed")
	}

	if os.Getenv("DEBUG") != "" {
		log.Printf("Starting Kafka producer for address: %s, topic: %s.\n", brokers, topic)
	}

	var retries int
	retries, err = strconv.Atoi(os.Getenv("KAFKA_CONNECT_RETRIES"))
	if err != nil {
		retries = 3
	}
	var producer *sarama.AsyncProducer
	for i := 0; i < retries; i++ {
		producer, err = sarama.NewAsyncProducer(brokers, newConfig())
		if err != nil {
			if os.Getenv("DEBUG") != "" {
				log.Println("Couldn't create Kafka producer. Retrying...", err)
			}
			if i == retries-1 {
				return nil, errorf("Couldn't create Kafka producer. %v", err)
			}
		} else {
			time.Sleep(1 * time.Second)
		}
	}

	return &KafkaAvroAdapter{
		route:    route,
		brokers:  brokers,
		topic:    topic,
		registry: registry,
		schema:   schema,
		producer: producer,
	}, nil
}

func (a *KafkaAvroAdapter) Stream(logstream chan *router.Message) {
	defer a.producer.Close()
	for rm := range logstream {
		message, err := a.formatMessage(rm)
		if err != nil {
			log.Println("kafka:", err)
			a.route.Close()
			break
		}

		a.producer.Input() <- message
	}
}

func newConfig() *sarama.Config {
	config := sarama.NewConfig()
	config.ClientID = "logspout"
	config.Producer.Return.Errors = false
	config.Producer.Return.Successes = false
	config.Producer.Flush.Frequency = 1 * time.Second
	config.Producer.RequiredAcks = sarama.WaitForLocal

	if opt := os.Getenv("KAFKA_COMPRESSION_CODEC"); opt != "" {
		switch opt {
		case "gzip":
			config.Producer.Compression = sarama.CompressionGZIP
		case "snappy":
			config.Producer.Compression = sarama.CompressionSnappy
		}
	}

	return config
}

func (a *KafkaAvroAdapter) formatMessage(message *router.Message) (*sarama.ProducerMessage, error) {
	var encoder sarama.Encoder

	record := avro.NewGenericRecord(a.schema)
	record.Set("timestamp", message.Time)
	record.Set("container_name", message.Container.Name)
	record.Set("source", message.Source)
	record.Set("line", message.Data)

	var b []byte
	b, err = a.registry.Encode(record)
	if err != nil {
		return nil, err
	}

	encoder = sarama.ByteEncoder(b)

	return &sarama.ProducerMessage{
		Topic: a.topic,
		Value: encoder,
	}, nil
}

func readBrokers(address string) []string {
	if strings.Contains(address, "/") {
		slash := strings.Index(address, "/")
		address = address[:slash]
	}

	return strings.Split(address, ",")
}

func readTopic(address string, options map[string]string) string {
	var topic string
	if !strings.Contains(address, "/") {
		topic = options["topic"]
	} else {
		slash := strings.Index(address, "/")
		topic = address[slash+1:]
	}

	return topic
}

func readSchemaRegistryUrl(options map[string]string) string {
	var url string
	if _, ok := options["schema_registry_url"]; ok {
		url = options["schema_registry_url"]
	}
	return url
}

func errorf(format string, a ...interface{}) (err error) {
	err = fmt.Errorf(format, a...)
	if os.Getenv("DEBUG") != "" {
		fmt.Println(err.Error())
	}
	return
}
<commit_msg>fix compile errors<commit_after>package kafka_avro

import (
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"
	"time"

	avro "github.com/elodina/go-avro"
	kafkaavro "github.com/elodina/go-kafka-avro"
	"github.com/gliderlabs/logspout/router"
	"gopkg.in/Shopify/sarama.v1"
)

var messageSchema = `{
  "type": "record",
  "name": "LogLine",
  "fields": [
    {"name": "timestamp", "type": "string"},
    {"name": "container_name", "type": "string"},
    {"name": "source", "type": "string"},
    {"name": "line", "type": "string"}
  ]
}`

func init() {
	router.AdapterFactories.Register(NewKafkaAvroAdapter, "kafka_avro")
}

type KafkaAvroAdapter struct {
	route    *router.Route
	brokers  []string
	topic    string
	schema   avro.Schema
	registry *kafkaavro.KafkaAvroEncoder
	producer sarama.AsyncProducer
}

func NewKafkaAvroAdapter(route *router.Route) (router.LogAdapter, error) {
	brokers := readBrokers(route.Address)
	if len(brokers) == 0 {
		return nil, errorf("The Kafka broker host:port is missing. Did you specify it as a route address?")
	}

	topic := readTopic(route.Address, route.Options)
	if topic == "" {
		return nil, errorf("The Kafka topic is missing. Did you specify it as a route option?")
	}

	schemaUrl := readSchemaRegistryUrl(route.Options)
	if schemaUrl == "" {
		return nil, errorf("The schema registry url is missing. Did you specify it as a route option?")
	}

	registry := kafkaavro.NewKafkaAvroEncoder(schemaUrl)

	var schema avro.Schema
	schema, err := avro.ParseSchema(messageSchema)
	if err != nil {
		return nil, errorf("The schema could not be parsed")
	}

	if os.Getenv("DEBUG") != "" {
		log.Printf("Starting Kafka producer for address: %s, topic: %s.\n", brokers, topic)
	}

	var retries int
	retries, err = strconv.Atoi(os.Getenv("KAFKA_CONNECT_RETRIES"))
	if err != nil {
		retries = 3
	}
	var producer sarama.AsyncProducer
	for i := 0; i < retries; i++ {
		producer, err = sarama.NewAsyncProducer(brokers, newConfig())
		if err != nil {
			if os.Getenv("DEBUG") != "" {
				log.Println("Couldn't create Kafka producer. Retrying...", err)
			}
			if i == retries-1 {
				return nil, errorf("Couldn't create Kafka producer. %v", err)
			}
		} else {
			time.Sleep(1 * time.Second)
		}
	}

	return &KafkaAvroAdapter{
		route:    route,
		brokers:  brokers,
		topic:    topic,
		registry: registry,
		schema:   schema,
		producer: producer,
	}, nil
}

func (a *KafkaAvroAdapter) Stream(logstream chan *router.Message) {
	defer a.producer.Close()
	for rm := range logstream {
		message, err := a.formatMessage(rm)
		if err != nil {
			log.Println("kafka:", err)
			a.route.Close()
			break
		}

		a.producer.Input() <- message
	}
}

func newConfig() *sarama.Config {
	config := sarama.NewConfig()
	config.ClientID = "logspout"
	config.Producer.Return.Errors = false
	config.Producer.Return.Successes = false
	config.Producer.Flush.Frequency = 1 * time.Second
	config.Producer.RequiredAcks = sarama.WaitForLocal

	if opt := os.Getenv("KAFKA_COMPRESSION_CODEC"); opt != "" {
		switch opt {
		case "gzip":
			config.Producer.Compression = sarama.CompressionGZIP
		case "snappy":
			config.Producer.Compression = sarama.CompressionSnappy
		}
	}

	return config
}

func (a *KafkaAvroAdapter) formatMessage(message *router.Message) (*sarama.ProducerMessage, error) {
	var encoder sarama.Encoder

	record := avro.NewGenericRecord(a.schema)
	record.Set("timestamp", message.Time)
	record.Set("container_name", message.Container.Name)
	record.Set("source", message.Source)
	record.Set("line", message.Data)

	var b []byte
	b, err = a.registry.Encode(record)
	if err != nil {
		return nil, err
	}

	encoder = sarama.ByteEncoder(b)

	return &sarama.ProducerMessage{
		Topic: a.topic,
		Value: encoder,
	}, nil
}

func readBrokers(address string) []string {
	if strings.Contains(address, "/") {
		slash := strings.Index(address, "/")
		address = address[:slash]
	}

	return strings.Split(address, ",")
}

func readTopic(address string, options map[string]string) string {
	var topic string
	if !strings.Contains(address, "/") {
		topic = options["topic"]
	} else {
		slash := strings.Index(address, "/")
		topic = address[slash+1:]
	}

	return topic
}

func readSchemaRegistryUrl(options map[string]string) string {
	var url string
	if _, ok := options["schema_registry_url"]; ok {
		url = options["schema_registry_url"]
	}
	return url
}

func errorf(format string, a ...interface{}) (err error) {
	err = fmt.Errorf(format, a...)
	if os.Getenv("DEBUG") != "" {
		fmt.Println(err.Error())
	}
	return
}
<|endoftext|>


<commit_before>package notice

import (
	"context"
	"io"

	"github.com/altairsix/pkg/tracer"
	"github.com/nats-io/go-nats"
	"github.com/opentracing/opentracing-go/log"
	"github.com/pkg/errors"
)

const (
	// Group provides the name of the nats group
	Group = "vavende-identities-dbase"
)

// Notice contains a request by the submitted of a command to update a specific read model
type Message interface {
	AggregateID() string
}

type MessageCloser interface {
	Message
	io.Closer
}

type message struct {
	nc  *nats.Conn
	msg *nats.Msg
}

// Close sends the response back to the original requester.  This should be call prior to disposing the Notice
func (m *message) Close() error {
	if m.msg.Reply == "" {
		return nil
	}

	return m.nc.Publish(m.msg.Reply, m.msg.Data)
}

// AggregateID refers the aggregate that was recently updated
func (m *message) AggregateID() string {
	return string(m.msg.Data)
}

// Subscribe listens for notices on the nats subject provided
func Subscribe(ctx context.Context, nc *nats.Conn, subject string, bufferSize int) (<-chan MessageCloser, error) {
	segment, _ := tracer.NewSegment(ctx, "nats.notice_listener")
	defer segment.Finish()

	ch := make(chan MessageCloser, bufferSize)
	sub, err := nc.QueueSubscribe(subject, Group, func(msg *nats.Msg) {
		select {
		case ch <- &message{nc: nc, msg: msg}:
			segment.Info("nats.notice_received",
				log.String("subject", msg.Subject),
				log.String("id", string(msg.Data)),
			)
		default:
		}
	})
	if err != nil {
		defer close(ch)
		return nil, errors.Wrapf(err, "unable to subscribe to subject, %v", subject)
	}

	go func() {
		defer close(ch)
		defer sub.Unsubscribe()
		<-ctx.Done()
	}()

	return ch, nil
}
<commit_msg>- fixed logging around nats subscriber<commit_after>package notice

import (
	"context"
	"io"

	"github.com/altairsix/pkg/tracer"
	"github.com/nats-io/go-nats"
	"github.com/opentracing/opentracing-go/log"
	"github.com/pkg/errors"
)

const (
	// Group provides the name of the nats group
	Group = "vavende-identities-dbase"
)

// Notice contains a request by the submitted of a command to update a specific read model
type Message interface {
	AggregateID() string
}

type MessageCloser interface {
	Message
	io.Closer
}

type message struct {
	nc  *nats.Conn
	msg *nats.Msg
}

// Close sends the response back to the original requester.  This should be call prior to disposing the Notice
func (m *message) Close() error {
	if m.msg.Reply == "" {
		return nil
	}

	return m.nc.Publish(m.msg.Reply, m.msg.Data)
}

// AggregateID refers the aggregate that was recently updated
func (m *message) AggregateID() string {
	return string(m.msg.Data)
}

// Subscribe listens for notices on the nats subject provided
func Subscribe(ctx context.Context, nc *nats.Conn, subject string, bufferSize int) (<-chan MessageCloser, error) {
	segment, _ := tracer.NewSegment(ctx, "nats.notice_listener")

	ch := make(chan MessageCloser, bufferSize)
	sub, err := nc.QueueSubscribe(subject, Group, func(msg *nats.Msg) {
		select {
		case ch <- &message{nc: nc, msg: msg}:
			segment.Info("nats.notice_received",
				log.String("subject", msg.Subject),
				log.String("id", string(msg.Data)),
			)
		default:
		}
	})
	if err != nil {
		defer close(ch)
		return nil, errors.Wrapf(err, "unable to subscribe to subject, %v", subject)
	}

	go func() {
		defer segment.Finish()
		defer close(ch)
		defer sub.Unsubscribe()
		<-ctx.Done()
	}()

	return ch, nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tar

// TODO(dsymonds):
//   - pax extensions

import (
	"bytes"
	"io"
	"os"
	"strconv"
)

var (
	HeaderError os.Error = os.ErrorString("invalid tar header")
)

// A Reader provides sequential access to the contents of a tar archive.
// A tar archive consists of a sequence of files.
// The Next method advances to the next file in the archive (including the first),
// and then it can be treated as an io.Reader to access the file's data.
//
// Example:
//	tr := tar.NewReader(r)
//	for {
//		hdr, err := tr.Next()
//		if err != nil {
//			// handle error
//		}
//		if hdr == nil {
//			// end of tar archive
//			break
//		}
//		io.Copy(data, tr)
//	}
type Reader struct {
	r   io.Reader
	err os.Error
	nb  int64 // number of unread bytes for current file entry
	pad int64 // amount of padding (ignored) after current file entry
}

// NewReader creates a new Reader reading from r.
func NewReader(r io.Reader) *Reader { return &Reader{r: r} }

// Next advances to the next entry in the tar archive.
func (tr *Reader) Next() (*Header, os.Error) {
	var hdr *Header
	if tr.err == nil {
		tr.skipUnread()
	}
	if tr.err == nil {
		hdr = tr.readHeader()
	}
	return hdr, tr.err
}

// Parse bytes as a NUL-terminated C-style string.
// If a NUL byte is not found then the whole slice is returned as a string.
func cString(b []byte) string {
	n := 0
	for n < len(b) && b[n] != 0 {
		n++
	}
	return string(b[0:n])
}

func (tr *Reader) octal(b []byte) int64 {
	// Removing leading spaces.
	for len(b) > 0 && b[0] == ' ' {
		b = b[1:]
	}
	// Removing trailing NULs and spaces.
	for len(b) > 0 && (b[len(b)-1] == ' ' || b[len(b)-1] == '\x00') {
		b = b[0 : len(b)-1]
	}
	x, err := strconv.Btoui64(cString(b), 8)
	if err != nil {
		tr.err = err
	}
	return int64(x)
}

type ignoreWriter struct{}

func (ignoreWriter) Write(b []byte) (n int, err os.Error) {
	return len(b), nil
}

// Skip any unread bytes in the existing file entry, as well as any alignment padding.
func (tr *Reader) skipUnread() {
	nr := tr.nb + tr.pad // number of bytes to skip
	tr.nb, tr.pad = 0, 0
	if sr, ok := tr.r.(io.Seeker); ok {
		if _, err := sr.Seek(nr, os.SEEK_CUR); err == nil {
			return
		}
	}
	_, tr.err = io.Copyn(ignoreWriter{}, tr.r, nr)
}

func (tr *Reader) verifyChecksum(header []byte) bool {
	if tr.err != nil {
		return false
	}

	given := tr.octal(header[148:156])
	unsigned, signed := checksum(header)
	return given == unsigned || given == signed
}

func (tr *Reader) readHeader() *Header {
	header := make([]byte, blockSize)
	if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil {
		return nil
	}

	// Two blocks of zero bytes marks the end of the archive.
	if bytes.Equal(header, zeroBlock[0:blockSize]) {
		if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil {
			return nil
		}
		if bytes.Equal(header, zeroBlock[0:blockSize]) {
			tr.err = os.EOF
		} else {
			tr.err = HeaderError // zero block and then non-zero block
		}
		return nil
	}

	if !tr.verifyChecksum(header) {
		tr.err = HeaderError
		return nil
	}

	// Unpack
	hdr := new(Header)
	s := slicer(header)

	hdr.Name = cString(s.next(100))
	hdr.Mode = tr.octal(s.next(8))
	hdr.Uid = int(tr.octal(s.next(8)))
	hdr.Gid = int(tr.octal(s.next(8)))
	hdr.Size = tr.octal(s.next(12))
	hdr.Mtime = tr.octal(s.next(12))
	s.next(8) // chksum
	hdr.Typeflag = s.next(1)[0]
	hdr.Linkname = cString(s.next(100))

	// The remainder of the header depends on the value of magic.
	// The original (v7) version of tar had no explicit magic field,
	// so its magic bytes, like the rest of the block, are NULs.
	magic := string(s.next(8)) // contains version field as well.
	var format string
	switch magic {
	case "ustar\x0000": // POSIX tar (1003.1-1988)
		if string(header[508:512]) == "tar\x00" {
			format = "star"
		} else {
			format = "posix"
		}
	case "ustar  \x00": // old GNU tar
		format = "gnu"
	}

	switch format {
	case "posix", "gnu", "star":
		hdr.Uname = cString(s.next(32))
		hdr.Gname = cString(s.next(32))
		devmajor := s.next(8)
		devminor := s.next(8)
		if hdr.Typeflag == TypeChar || hdr.Typeflag == TypeBlock {
			hdr.Devmajor = tr.octal(devmajor)
			hdr.Devminor = tr.octal(devminor)
		}
		var prefix string
		switch format {
		case "posix", "gnu":
			prefix = cString(s.next(155))
		case "star":
			prefix = cString(s.next(131))
			hdr.Atime = tr.octal(s.next(12))
			hdr.Ctime = tr.octal(s.next(12))
		}
		if len(prefix) > 0 {
			hdr.Name = prefix + "/" + hdr.Name
		}
	}

	if tr.err != nil {
		tr.err = HeaderError
		return nil
	}

	// Maximum value of hdr.Size is 64 GB (12 octal digits),
	// so there's no risk of int64 overflowing.
	tr.nb = int64(hdr.Size)
	tr.pad = -tr.nb & (blockSize - 1) // blockSize is a power of two

	return hdr
}

// Read reads from the current entry in the tar archive.
// It returns 0, os.EOF when it reaches the end of that entry,
// until Next is called to advance to the next entry.
func (tr *Reader) Read(b []byte) (n int, err os.Error) {
	if tr.nb == 0 {
		// file consumed
		return 0, os.EOF
	}

	if int64(len(b)) > tr.nb {
		b = b[0:tr.nb]
	}
	n, err = tr.r.Read(b)
	tr.nb -= int64(n)

	if err == os.EOF && tr.nb > 0 {
		err = io.ErrUnexpectedEOF
	}
	tr.err = err
	return
}
<commit_msg>archive/tar: fix example's handling of os.EOF.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tar

// TODO(dsymonds):
//   - pax extensions

import (
	"bytes"
	"io"
	"os"
	"strconv"
)

var (
	HeaderError os.Error = os.ErrorString("invalid tar header")
)

// A Reader provides sequential access to the contents of a tar archive.
// A tar archive consists of a sequence of files.
// The Next method advances to the next file in the archive (including the first),
// and then it can be treated as an io.Reader to access the file's data.
//
// Example:
//	tr := tar.NewReader(r)
//	for {
//		hdr, err := tr.Next()
//		if err == os.EOF {
//			// end of tar archive
//			break
//		}
//		if err != nil {
//			// handle error
//		}
//		io.Copy(data, tr)
//	}
type Reader struct {
	r   io.Reader
	err os.Error
	nb  int64 // number of unread bytes for current file entry
	pad int64 // amount of padding (ignored) after current file entry
}

// NewReader creates a new Reader reading from r.
func NewReader(r io.Reader) *Reader { return &Reader{r: r} }

// Next advances to the next entry in the tar archive.
func (tr *Reader) Next() (*Header, os.Error) {
	var hdr *Header
	if tr.err == nil {
		tr.skipUnread()
	}
	if tr.err == nil {
		hdr = tr.readHeader()
	}
	return hdr, tr.err
}

// Parse bytes as a NUL-terminated C-style string.
// If a NUL byte is not found then the whole slice is returned as a string.
func cString(b []byte) string {
	n := 0
	for n < len(b) && b[n] != 0 {
		n++
	}
	return string(b[0:n])
}

func (tr *Reader) octal(b []byte) int64 {
	// Removing leading spaces.
	for len(b) > 0 && b[0] == ' ' {
		b = b[1:]
	}
	// Removing trailing NULs and spaces.
	for len(b) > 0 && (b[len(b)-1] == ' ' || b[len(b)-1] == '\x00') {
		b = b[0 : len(b)-1]
	}
	x, err := strconv.Btoui64(cString(b), 8)
	if err != nil {
		tr.err = err
	}
	return int64(x)
}

type ignoreWriter struct{}

func (ignoreWriter) Write(b []byte) (n int, err os.Error) {
	return len(b), nil
}

// Skip any unread bytes in the existing file entry, as well as any alignment padding.
func (tr *Reader) skipUnread() {
	nr := tr.nb + tr.pad // number of bytes to skip
	tr.nb, tr.pad = 0, 0
	if sr, ok := tr.r.(io.Seeker); ok {
		if _, err := sr.Seek(nr, os.SEEK_CUR); err == nil {
			return
		}
	}
	_, tr.err = io.Copyn(ignoreWriter{}, tr.r, nr)
}

func (tr *Reader) verifyChecksum(header []byte) bool {
	if tr.err != nil {
		return false
	}

	given := tr.octal(header[148:156])
	unsigned, signed := checksum(header)
	return given == unsigned || given == signed
}

func (tr *Reader) readHeader() *Header {
	header := make([]byte, blockSize)
	if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil {
		return nil
	}

	// Two blocks of zero bytes marks the end of the archive.
	if bytes.Equal(header, zeroBlock[0:blockSize]) {
		if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil {
			return nil
		}
		if bytes.Equal(header, zeroBlock[0:blockSize]) {
			tr.err = os.EOF
		} else {
			tr.err = HeaderError // zero block and then non-zero block
		}
		return nil
	}

	if !tr.verifyChecksum(header) {
		tr.err = HeaderError
		return nil
	}

	// Unpack
	hdr := new(Header)
	s := slicer(header)

	hdr.Name = cString(s.next(100))
	hdr.Mode = tr.octal(s.next(8))
	hdr.Uid = int(tr.octal(s.next(8)))
	hdr.Gid = int(tr.octal(s.next(8)))
	hdr.Size = tr.octal(s.next(12))
	hdr.Mtime = tr.octal(s.next(12))
	s.next(8) // chksum
	hdr.Typeflag = s.next(1)[0]
	hdr.Linkname = cString(s.next(100))

	// The remainder of the header depends on the value of magic.
	// The original (v7) version of tar had no explicit magic field,
	// so its magic bytes, like the rest of the block, are NULs.
	magic := string(s.next(8)) // contains version field as well.
	var format string
	switch magic {
	case "ustar\x0000": // POSIX tar (1003.1-1988)
		if string(header[508:512]) == "tar\x00" {
			format = "star"
		} else {
			format = "posix"
		}
	case "ustar  \x00": // old GNU tar
		format = "gnu"
	}

	switch format {
	case "posix", "gnu", "star":
		hdr.Uname = cString(s.next(32))
		hdr.Gname = cString(s.next(32))
		devmajor := s.next(8)
		devminor := s.next(8)
		if hdr.Typeflag == TypeChar || hdr.Typeflag == TypeBlock {
			hdr.Devmajor = tr.octal(devmajor)
			hdr.Devminor = tr.octal(devminor)
		}
		var prefix string
		switch format {
		case "posix", "gnu":
			prefix = cString(s.next(155))
		case "star":
			prefix = cString(s.next(131))
			hdr.Atime = tr.octal(s.next(12))
			hdr.Ctime = tr.octal(s.next(12))
		}
		if len(prefix) > 0 {
			hdr.Name = prefix + "/" + hdr.Name
		}
	}

	if tr.err != nil {
		tr.err = HeaderError
		return nil
	}

	// Maximum value of hdr.Size is 64 GB (12 octal digits),
	// so there's no risk of int64 overflowing.
	tr.nb = int64(hdr.Size)
	tr.pad = -tr.nb & (blockSize - 1) // blockSize is a power of two

	return hdr
}

// Read reads from the current entry in the tar archive.
// It returns 0, os.EOF when it reaches the end of that entry,
// until Next is called to advance to the next entry.
func (tr *Reader) Read(b []byte) (n int, err os.Error) {
	if tr.nb == 0 {
		// file consumed
		return 0, os.EOF
	}

	if int64(len(b)) > tr.nb {
		b = b[0:tr.nb]
	}
	n, err = tr.r.Read(b)
	tr.nb -= int64(n)

	if err == os.EOF && tr.nb > 0 {
		err = io.ErrUnexpectedEOF
	}
	tr.err = err
	return
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package ecdsa implements the Elliptic Curve Digital Signature Algorithm, as
// defined in FIPS 186-3.
package ecdsa

// References:
//   [NSA]: Suite B implementer's guide to FIPS 186-3,
//     http://www.nsa.gov/ia/_files/ecdsa.pdf
//   [SECG]: SECG, SEC1
//     http://www.secg.org/download/aid-780/sec1-v2.pdf

import (
	"crypto/elliptic"
	"io"
	"math/big"
)

// PublicKey represents an ECDSA public key.
type PublicKey struct {
	elliptic.Curve
	X, Y *big.Int
}

// PrivateKey represents a ECDSA private key.
type PrivateKey struct {
	PublicKey
	D *big.Int
}

var one = new(big.Int).SetInt64(1)

// randFieldElement returns a random element of the field underlying the given
// curve using the procedure given in [NSA] A.2.1.
func randFieldElement(c elliptic.Curve, rand io.Reader) (k *big.Int, err error) {
	params := c.Params()
	b := make([]byte, params.BitSize/8+8)
	_, err = io.ReadFull(rand, b)
	if err != nil {
		return
	}

	k = new(big.Int).SetBytes(b)
	n := new(big.Int).Sub(params.N, one)
	k.Mod(k, n)
	k.Add(k, one)
	return
}

// GenerateKey generates a public&private key pair.
func GenerateKey(c elliptic.Curve, rand io.Reader) (priv *PrivateKey, err error) {
	k, err := randFieldElement(c, rand)
	if err != nil {
		return
	}

	priv = new(PrivateKey)
	priv.PublicKey.Curve = c
	priv.D = k
	priv.PublicKey.X, priv.PublicKey.Y = c.ScalarBaseMult(k.Bytes())
	return
}

// hashToInt converts a hash value to an integer. There is some disagreement
// about how this is done. [NSA] suggests that this is done in the obvious
// manner, but [SECG] truncates the hash to the bit-length of the curve order
// first. We follow [SECG] because that's what OpenSSL does. Additionally,
// OpenSSL right shifts excess bits from the number if the hash is too large
// and we mirror that too.
func hashToInt(hash []byte, c elliptic.Curve) *big.Int {
	orderBits := c.Params().N.BitLen()
	orderBytes := (orderBits + 7) / 8
	if len(hash) > orderBytes {
		hash = hash[:orderBytes]
	}

	ret := new(big.Int).SetBytes(hash)
	excess := len(hash)*8 - orderBits
	if excess > 0 {
		ret.Rsh(ret, uint(excess))
	}
	return ret
}

// Sign signs an arbitrary length hash (which should be the result of hashing a
// larger message) using the private key, priv. It returns the signature as a
// pair of integers. The security of the private key depends on the entropy of
// rand.
func Sign(rand io.Reader, priv *PrivateKey, hash []byte) (r, s *big.Int, err error) {
	// See [NSA] 3.4.1
	c := priv.PublicKey.Curve
	N := c.Params().N

	var k, kInv *big.Int
	for {
		for {
			k, err = randFieldElement(c, rand)
			if err != nil {
				r = nil
				return
			}

			kInv = new(big.Int).ModInverse(k, N)
			r, _ = priv.Curve.ScalarBaseMult(k.Bytes())
			r.Mod(r, N)
			if r.Sign() != 0 {
				break
			}
		}

		e := hashToInt(hash, c)
		s = new(big.Int).Mul(priv.D, r)
		s.Add(s, e)
		s.Mul(s, kInv)
		s.Mod(s, N)
		if s.Sign() != 0 {
			break
		}
	}

	return
}

// Verify verifies the signature in r, s of hash using the public key, pub. It
// returns true iff the signature is valid.
func Verify(pub *PublicKey, hash []byte, r, s *big.Int) bool {
	// See [NSA] 3.4.2
	c := pub.Curve
	N := c.Params().N

	if r.Sign() == 0 || s.Sign() == 0 {
		return false
	}
	if r.Cmp(N) >= 0 || s.Cmp(N) >= 0 {
		return false
	}
	e := hashToInt(hash, c)
	w := new(big.Int).ModInverse(s, N)

	u1 := e.Mul(e, w)
	u1.Mod(u1, N)
	u2 := w.Mul(r, w)
	u2.Mod(u2, N)

	x1, y1 := c.ScalarBaseMult(u1.Bytes())
	x2, y2 := c.ScalarMult(pub.X, pub.Y, u2.Bytes())
	x, y := c.Add(x1, y1, x2, y2)
	if x.Sign() == 0 && y.Sign() == 0 {
		return false
	}
	x.Mod(x, N)
	return x.Cmp(r) == 0
}
<commit_msg>crypto/ecdsa: doc cleanup<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package ecdsa implements the Elliptic Curve Digital Signature Algorithm, as
// defined in FIPS 186-3.
package ecdsa

// References:
//   [NSA]: Suite B implementer's guide to FIPS 186-3,
//     http://www.nsa.gov/ia/_files/ecdsa.pdf
//   [SECG]: SECG, SEC1
//     http://www.secg.org/download/aid-780/sec1-v2.pdf

import (
	"crypto/elliptic"
	"io"
	"math/big"
)

// PublicKey represents an ECDSA public key.
type PublicKey struct {
	elliptic.Curve
	X, Y *big.Int
}

// PrivateKey represents a ECDSA private key.
type PrivateKey struct {
	PublicKey
	D *big.Int
}

var one = new(big.Int).SetInt64(1)

// randFieldElement returns a random element of the field underlying the given
// curve using the procedure given in [NSA] A.2.1.
func randFieldElement(c elliptic.Curve, rand io.Reader) (k *big.Int, err error) {
	params := c.Params()
	b := make([]byte, params.BitSize/8+8)
	_, err = io.ReadFull(rand, b)
	if err != nil {
		return
	}

	k = new(big.Int).SetBytes(b)
	n := new(big.Int).Sub(params.N, one)
	k.Mod(k, n)
	k.Add(k, one)
	return
}

// GenerateKey generates a public and private key pair.
func GenerateKey(c elliptic.Curve, rand io.Reader) (priv *PrivateKey, err error) {
	k, err := randFieldElement(c, rand)
	if err != nil {
		return
	}

	priv = new(PrivateKey)
	priv.PublicKey.Curve = c
	priv.D = k
	priv.PublicKey.X, priv.PublicKey.Y = c.ScalarBaseMult(k.Bytes())
	return
}

// hashToInt converts a hash value to an integer. There is some disagreement
// about how this is done. [NSA] suggests that this is done in the obvious
// manner, but [SECG] truncates the hash to the bit-length of the curve order
// first. We follow [SECG] because that's what OpenSSL does. Additionally,
// OpenSSL right shifts excess bits from the number if the hash is too large
// and we mirror that too.
func hashToInt(hash []byte, c elliptic.Curve) *big.Int {
	orderBits := c.Params().N.BitLen()
	orderBytes := (orderBits + 7) / 8
	if len(hash) > orderBytes {
		hash = hash[:orderBytes]
	}

	ret := new(big.Int).SetBytes(hash)
	excess := len(hash)*8 - orderBits
	if excess > 0 {
		ret.Rsh(ret, uint(excess))
	}
	return ret
}

// Sign signs an arbitrary length hash (which should be the result of hashing a
// larger message) using the private key, priv. It returns the signature as a
// pair of integers. The security of the private key depends on the entropy of
// rand.
func Sign(rand io.Reader, priv *PrivateKey, hash []byte) (r, s *big.Int, err error) {
	// See [NSA] 3.4.1
	c := priv.PublicKey.Curve
	N := c.Params().N

	var k, kInv *big.Int
	for {
		for {
			k, err = randFieldElement(c, rand)
			if err != nil {
				r = nil
				return
			}

			kInv = new(big.Int).ModInverse(k, N)
			r, _ = priv.Curve.ScalarBaseMult(k.Bytes())
			r.Mod(r, N)
			if r.Sign() != 0 {
				break
			}
		}

		e := hashToInt(hash, c)
		s = new(big.Int).Mul(priv.D, r)
		s.Add(s, e)
		s.Mul(s, kInv)
		s.Mod(s, N)
		if s.Sign() != 0 {
			break
		}
	}

	return
}

// Verify verifies the signature in r, s of hash using the public key, pub. It
// returns true iff the signature is valid.
func Verify(pub *PublicKey, hash []byte, r, s *big.Int) bool {
	// See [NSA] 3.4.2
	c := pub.Curve
	N := c.Params().N

	if r.Sign() == 0 || s.Sign() == 0 {
		return false
	}
	if r.Cmp(N) >= 0 || s.Cmp(N) >= 0 {
		return false
	}
	e := hashToInt(hash, c)
	w := new(big.Int).ModInverse(s, N)

	u1 := e.Mul(e, w)
	u1.Mod(u1, N)
	u2 := w.Mul(r, w)
	u2.Mod(u2, N)

	x1, y1 := c.ScalarBaseMult(u1.Bytes())
	x2, y2 := c.ScalarMult(pub.X, pub.Y, u2.Bytes())
	x, y := c.Add(x1, y1, x2, y2)
	if x.Sign() == 0 && y.Sign() == 0 {
		return false
	}
	x.Mod(x, N)
	return x.Cmp(r) == 0
}
<|endoftext|>