<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The /doc/codewalk/ tree is synthesized from codewalk descriptions,
// files named $GOROOT/doc/codewalk/*.xml.
// For an example and a description of the format, see
// http://golang.org/doc/codewalk/codewalk or run godoc -http=:6060
// and see http://localhost:6060/doc/codewalk/codewalk .
// That page is itself a codewalk; the source code for it is
// $GOROOT/doc/codewalk/codewalk.xml.

package main

import (
	"log"
	"net/http"
	"regexp"
	"text/template"

	"code.google.com/p/go.tools/godoc"
	"code.google.com/p/go.tools/godoc/vfs"
)

var (
	pres *godoc.Presentation
	fs   = vfs.NameSpace{}
)

func registerHandlers(pres *godoc.Presentation) {
	if pres == nil {
		panic("nil Presentation")
	}
	http.HandleFunc("/doc/codewalk/", codewalk)
	http.Handle("/doc/play/", pres.FileServer())
	http.Handle("/robots.txt", pres.FileServer())
	http.Handle("/", pres)
	handlePathRedirects(pkgRedirects, "/pkg/")
	handlePathRedirects(cmdRedirects, "/cmd/")
	for prefix, redirect := range prefixHelpers {
		p := "/" + prefix + "/"
		h := makePrefixRedirectHandler(p, redirect)
		http.HandleFunc(p, h)
	}
	for path, redirect := range redirects {
		h := makeRedirectHandler(redirect)
		http.HandleFunc(path, h)
	}
}

func readTemplate(name string) *template.Template {
	if pres == nil {
		panic("no global Presentation set yet")
	}
	path := "lib/godoc/" + name

	// use underlying file system fs to read the template file
	// (cannot use template ParseFile functions directly)
	data, err := vfs.ReadFile(fs, path)
	if err != nil {
		log.Fatal("readTemplate: ", err)
	}
	// be explicit with errors (for app engine use)
	t, err := template.New(name).Funcs(pres.FuncMap()).Parse(string(data))
	if err != nil {
		log.Fatal("readTemplate: ", err)
	}
	return t
}

func readTemplates(p *godoc.Presentation, html bool) {
	p.PackageText = readTemplate("package.txt")
	p.SearchText = readTemplate("search.txt")

	if html {
		codewalkHTML = readTemplate("codewalk.html")
		codewalkdirHTML = readTemplate("codewalkdir.html")
		p.DirlistHTML = readTemplate("dirlist.html")
		p.ErrorHTML = readTemplate("error.html")
		p.ExampleHTML = readTemplate("example.html")
		p.GodocHTML = readTemplate("godoc.html")
		p.PackageHTML = readTemplate("package.html")
		p.SearchHTML = readTemplate("search.html")
		p.SearchDescXML = readTemplate("opensearch.xml")
	}
}

// Packages that were renamed between r60 and go1.
var pkgRedirects = map[string]string{
	"asn1":              "encoding/asn1",
	"big":               "math/big",
	"cmath":             "math/cmplx",
	"csv":               "encoding/csv",
	"exec":              "os/exec",
	"exp/template/html": "html/template",
	"gob":               "encoding/gob",
	"http":              "net/http",
	"http/cgi":          "net/http/cgi",
	"http/fcgi":         "net/http/fcgi",
	"http/httptest":     "net/http/httptest",
	"http/pprof":        "net/http/pprof",
	"json":              "encoding/json",
	"mail":              "net/mail",
	"rand":              "math/rand",
	"rpc":               "net/rpc",
	"rpc/jsonrpc":       "net/rpc/jsonrpc",
	"scanner":           "text/scanner",
	"smtp":              "net/smtp",
	"tabwriter":         "text/tabwriter",
	"template":          "text/template",
	"template/parse":    "text/template/parse",
	"url":               "net/url",
	"utf16":             "unicode/utf16",
	"utf8":              "unicode/utf8",
	"xml":               "encoding/xml",
}

// Commands that were renamed between r60 and go1.
var cmdRedirects = map[string]string{
	"gofix":     "fix",
	"goinstall": "go",
	"gopack":    "pack",
	"govet":     "vet",
	"goyacc":    "yacc",
}

var redirects = map[string]string{
	"/blog":      "http://blog.golang.org",
	"/build":     "http://build.golang.org",
	"/change":    "https://code.google.com/p/go/source/list",
	"/cl":        "https://gocodereview.appspot.com/",
	"/issue":     "https://code.google.com/p/go/issues",
	"/issue/new": "https://code.google.com/p/go/issues/entry",
	"/issues":    "https://code.google.com/p/go/issues",
	"/play":      "http://play.golang.org",
	"/ref":       "/doc/#references",
	"/ref/":      "/doc/#references",
	"/ref/mem":   "/doc/mem",
	"/ref/spec":  "/doc/spec",
	"/talks":     "http://talks.golang.org",
	"/tour":      "http://tour.golang.org",
	"/wiki":      "https://code.google.com/p/go-wiki/w/list",

	"/doc/articles/c_go_cgo.html":                    "/blog/c-go-cgo",
	"/doc/articles/concurrency_patterns.html":        "/blog/go-concurrency-patterns-timing-out-and",
	"/doc/articles/defer_panic_recover.html":         "/blog/defer-panic-and-recover",
	"/doc/articles/error_handling.html":              "/blog/error-handling-and-go",
	"/doc/articles/gobs_of_data.html":                "/blog/gobs-of-data",
	"/doc/articles/godoc_documenting_go_code.html":   "/blog/godoc-documenting-go-code",
	"/doc/articles/gos_declaration_syntax.html":      "/blog/gos-declaration-syntax",
	"/doc/articles/image_draw.html":                  "/blog/go-imagedraw-package",
	"/doc/articles/image_package.html":               "/blog/go-image-package",
	"/doc/articles/json_and_go.html":                 "/blog/json-and-go",
	"/doc/articles/json_rpc_tale_of_interfaces.html": "/blog/json-rpc-tale-of-interfaces",
	"/doc/articles/laws_of_reflection.html":          "/blog/laws-of-reflection",
	"/doc/articles/race_detector.html":               "/blog/race-detector",
	"/doc/articles/slices_usage_and_internals.html":  "/blog/go-slices-usage-and-internals",
	"/doc/go_for_cpp_programmers.html":               "https://code.google.com/p/go-wiki/wiki/GoForCPPProgrammers",
	"/doc/go_tutorial.html":                          "http://tour.golang.org/",
}

var prefixHelpers = map[string]string{
	"blog":   "http://blog.golang.org/",
	"change": "https://code.google.com/p/go/source/detail?r=",
	"cl":     "https://codereview.appspot.com/",
	"issue":  "https://code.google.com/p/go/issues/detail?id=",
	"play":   "http://play.golang.org/",
	"talks":  "http://talks.golang.org/",
	"wiki":   "https://code.google.com/p/go-wiki/wiki/",
}

func handlePathRedirects(redirects map[string]string, prefix string) {
	for source, target := range pkgRedirects {
		h := makeRedirectHandler(prefix + target + "/")
		p := prefix + source
		http.HandleFunc(p, h)
		http.HandleFunc(p+"/", h)
	}
}

func makeRedirectHandler(target string) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, target, http.StatusMovedPermanently)
	}
}

var validId = regexp.MustCompile(`^[A-Za-z0-9-]*$`)

func makePrefixRedirectHandler(prefix, baseURL string) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if p := r.URL.Path; p == prefix {
			// redirect /prefix/ to /prefix
			http.Redirect(w, r, p[:len(p)-1], http.StatusFound)
			return
		}
		id := r.URL.Path[len(prefix):]
		if !validId.MatchString(id) {
			http.Error(w, "Not found", http.StatusNotFound)
			return
		}
		target := baseURL + id
		http.Redirect(w, r, target, http.StatusFound)
	}
}
<commit_msg>go.tools/cmd/godoc: add redirects for godoc, vet, and gotest<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The /doc/codewalk/ tree is synthesized from codewalk descriptions,
// files named $GOROOT/doc/codewalk/*.xml.
// For an example and a description of the format, see
// http://golang.org/doc/codewalk/codewalk or run godoc -http=:6060
// and see http://localhost:6060/doc/codewalk/codewalk .
// That page is itself a codewalk; the source code for it is
// $GOROOT/doc/codewalk/codewalk.xml.

package main

import (
	"log"
	"net/http"
	"regexp"
	"text/template"

	"code.google.com/p/go.tools/godoc"
	"code.google.com/p/go.tools/godoc/vfs"
)

var (
	pres *godoc.Presentation
	fs   = vfs.NameSpace{}
)

func registerHandlers(pres *godoc.Presentation) {
	if pres == nil {
		panic("nil Presentation")
	}
	http.HandleFunc("/doc/codewalk/", codewalk)
	http.Handle("/doc/play/", pres.FileServer())
	http.Handle("/robots.txt", pres.FileServer())
	http.Handle("/", pres)
	handlePathRedirects(pkgRedirects, "/pkg/")
	handlePathRedirects(cmdRedirects, "/cmd/")
	for prefix, redirect := range prefixHelpers {
		p := "/" + prefix + "/"
		h := makePrefixRedirectHandler(p, redirect)
		http.HandleFunc(p, h)
	}
	for path, redirect := range redirects {
		h := makeRedirectHandler(redirect)
		http.HandleFunc(path, h)
	}
}

func readTemplate(name string) *template.Template {
	if pres == nil {
		panic("no global Presentation set yet")
	}
	path := "lib/godoc/" + name

	// use underlying file system fs to read the template file
	// (cannot use template ParseFile functions directly)
	data, err := vfs.ReadFile(fs, path)
	if err != nil {
		log.Fatal("readTemplate: ", err)
	}
	// be explicit with errors (for app engine use)
	t, err := template.New(name).Funcs(pres.FuncMap()).Parse(string(data))
	if err != nil {
		log.Fatal("readTemplate: ", err)
	}
	return t
}

func readTemplates(p *godoc.Presentation, html bool) {
	p.PackageText = readTemplate("package.txt")
	p.SearchText = readTemplate("search.txt")

	if html {
		codewalkHTML = readTemplate("codewalk.html")
		codewalkdirHTML = readTemplate("codewalkdir.html")
		p.DirlistHTML = readTemplate("dirlist.html")
		p.ErrorHTML = readTemplate("error.html")
		p.ExampleHTML = readTemplate("example.html")
		p.GodocHTML = readTemplate("godoc.html")
		p.PackageHTML = readTemplate("package.html")
		p.SearchHTML = readTemplate("search.html")
		p.SearchDescXML = readTemplate("opensearch.xml")
	}
}

// Packages that were renamed between r60 and go1.
var pkgRedirects = map[string]string{
	"asn1":              "encoding/asn1",
	"big":               "math/big",
	"cmath":             "math/cmplx",
	"csv":               "encoding/csv",
	"exec":              "os/exec",
	"exp/template/html": "html/template",
	"gob":               "encoding/gob",
	"http":              "net/http",
	"http/cgi":          "net/http/cgi",
	"http/fcgi":         "net/http/fcgi",
	"http/httptest":     "net/http/httptest",
	"http/pprof":        "net/http/pprof",
	"json":              "encoding/json",
	"mail":              "net/mail",
	"rand":              "math/rand",
	"rpc":               "net/rpc",
	"rpc/jsonrpc":       "net/rpc/jsonrpc",
	"scanner":           "text/scanner",
	"smtp":              "net/smtp",
	"tabwriter":         "text/tabwriter",
	"template":          "text/template",
	"template/parse":    "text/template/parse",
	"url":               "net/url",
	"utf16":             "unicode/utf16",
	"utf8":              "unicode/utf8",
	"xml":               "encoding/xml",
}

// Commands that were renamed between r60 and go1.
var cmdRedirects = map[string]string{
	"gofix":     "fix",
	"goinstall": "go",
	"gopack":    "pack",
	"gotest":    "go",
	"govet":     "vet",
	"goyacc":    "yacc",
}

var redirects = map[string]string{
	"/blog":       "http://blog.golang.org",
	"/build":      "http://build.golang.org",
	"/change":     "https://code.google.com/p/go/source/list",
	"/cl":         "https://gocodereview.appspot.com/",
	"/cmd/godoc/": "http://godoc.org/code.google.com/p/go.tools/cmd/godoc/",
	"/cmd/vet/":   "http://godoc.org/code.google.com/p/go.tools/cmd/vet/",
	"/issue":      "https://code.google.com/p/go/issues",
	"/issue/new":  "https://code.google.com/p/go/issues/entry",
	"/issues":     "https://code.google.com/p/go/issues",
	"/play":       "http://play.golang.org",
	"/ref":        "/doc/#references",
	"/ref/":       "/doc/#references",
	"/ref/mem":    "/doc/mem",
	"/ref/spec":   "/doc/spec",
	"/talks":      "http://talks.golang.org",
	"/tour":       "http://tour.golang.org",
	"/wiki":       "https://code.google.com/p/go-wiki/w/list",

	"/doc/articles/c_go_cgo.html":                    "/blog/c-go-cgo",
	"/doc/articles/concurrency_patterns.html":        "/blog/go-concurrency-patterns-timing-out-and",
	"/doc/articles/defer_panic_recover.html":         "/blog/defer-panic-and-recover",
	"/doc/articles/error_handling.html":              "/blog/error-handling-and-go",
	"/doc/articles/gobs_of_data.html":                "/blog/gobs-of-data",
	"/doc/articles/godoc_documenting_go_code.html":   "/blog/godoc-documenting-go-code",
	"/doc/articles/gos_declaration_syntax.html":      "/blog/gos-declaration-syntax",
	"/doc/articles/image_draw.html":                  "/blog/go-imagedraw-package",
	"/doc/articles/image_package.html":               "/blog/go-image-package",
	"/doc/articles/json_and_go.html":                 "/blog/json-and-go",
	"/doc/articles/json_rpc_tale_of_interfaces.html": "/blog/json-rpc-tale-of-interfaces",
	"/doc/articles/laws_of_reflection.html":          "/blog/laws-of-reflection",
	"/doc/articles/race_detector.html":               "/blog/race-detector",
	"/doc/articles/slices_usage_and_internals.html":  "/blog/go-slices-usage-and-internals",
	"/doc/go_for_cpp_programmers.html":               "https://code.google.com/p/go-wiki/wiki/GoForCPPProgrammers",
	"/doc/go_tutorial.html":                          "http://tour.golang.org/",
}

var prefixHelpers = map[string]string{
	"blog":   "http://blog.golang.org/",
	"change": "https://code.google.com/p/go/source/detail?r=",
	"cl":     "https://codereview.appspot.com/",
	"issue":  "https://code.google.com/p/go/issues/detail?id=",
	"play":   "http://play.golang.org/",
	"talks":  "http://talks.golang.org/",
	"wiki":   "https://code.google.com/p/go-wiki/wiki/",
}

func handlePathRedirects(redirects map[string]string, prefix string) {
	for source, target := range pkgRedirects {
		h := makeRedirectHandler(prefix + target + "/")
		p := prefix + source
		http.HandleFunc(p, h)
		http.HandleFunc(p+"/", h)
	}
}

func makeRedirectHandler(target string) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, target, http.StatusMovedPermanently)
	}
}

var validId = regexp.MustCompile(`^[A-Za-z0-9-]*$`)

func makePrefixRedirectHandler(prefix, baseURL string) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if p := r.URL.Path; p == prefix {
			// redirect /prefix/ to /prefix
			http.Redirect(w, r, p[:len(p)-1], http.StatusFound)
			return
		}
		id := r.URL.Path[len(prefix):]
		if !validId.MatchString(id) {
			http.Error(w, "Not found", http.StatusNotFound)
			return
		}
		target := baseURL + id
		http.Redirect(w, r, target, http.StatusFound)
	}
}
<|endoftext|>


<commit_before>package handler

import (
	"log"
	"net/http"
	"regexp"
	"strings"

	"github.com/drone/drone/plugin/remote"
	"github.com/drone/drone/server/datastore"
	"github.com/drone/drone/server/worker"
	"github.com/drone/drone/shared/build/script"
	"github.com/drone/drone/shared/httputil"
	"github.com/drone/drone/shared/model"
	"github.com/goji/context"
	"github.com/zenazn/goji/web"
)

// PostHook accepts a post-commit hook and parses the payload
// in order to trigger a build. The payload is specified to the
// remote system (ie GitHub) and will therefore get parsed by
// the appropriate remote plugin.
//
//     GET /api/hook/:host
//
func PostHook(c web.C, w http.ResponseWriter, r *http.Request) {
	var ctx = context.FromC(c)
	var host = c.URLParams["host"]
	var token = c.URLParams["token"]
	var remote = remote.Lookup(host)
	if remote == nil {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	// parse the hook payload
	hook, err := remote.ParseHook(r)
	if err != nil {
		log.Printf("Unable to parse hook. %s\n", err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	// in some cases we have neither a hook nor error. An example
	// would be GitHub sending a ping request to the URL, in which
	// case we'll just exit quiely with an 'OK'
	shouldSkip, _ := regexp.MatchString(`\[(?i:ci *skip|skip *ci)\]`, hook.Message)
	if hook == nil || shouldSkip {
		w.WriteHeader(http.StatusOK)
		return
	}

	// fetch the repository from the database
	repo, err := datastore.GetRepoName(ctx, remote.GetHost(), hook.Owner, hook.Repo)
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	// each hook contains a token to verify the sender. If the token
	// is not provided or does not match, exit
	if len(repo.Token) == 0 || repo.Token != token {
		log.Printf("Rejected post commit hook for %s. Token mismatch\n", repo.Name)
		w.WriteHeader(http.StatusUnauthorized)
		return
	}

	if repo.Active == false ||
		(repo.PostCommit == false && len(hook.PullRequest) == 0) ||
		(repo.PullRequest == false && len(hook.PullRequest) != 0) {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	// fetch the user from the database that owns this repo
	user, err := datastore.GetUser(ctx, repo.UserID)
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	// Request a new token and update
	user_token, err := remote.GetToken(user)
	if user_token != nil {
		user.Access = user_token.AccessToken
		user.Secret = user_token.RefreshToken
		user.TokenExpiry = user_token.Expiry
		datastore.PutUser(ctx, user)
	} else if err != nil {
		log.Printf("Unable to refresh token. %s\n", err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	// fetch the .drone.yml file from the database
	yml, err := remote.GetScript(user, repo, hook)
	if err != nil {
		log.Printf("Unable to fetch .drone.yml file. %s\n", err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	// verify the commit hooks branch matches the list of approved
	// branches (unless it is a pull request). Note that we don't really
	// care if parsing the yaml fails here.
	s, _ := script.ParseBuild(string(yml))
	if len(hook.PullRequest) == 0 && !s.MatchBranch(hook.Branch) {
		w.WriteHeader(http.StatusOK)
		return
	}

	commit := model.Commit{
		RepoID:      repo.ID,
		Status:      model.StatusEnqueue,
		Sha:         hook.Sha,
		Branch:      hook.Branch,
		PullRequest: hook.PullRequest,
		Timestamp:   hook.Timestamp,
		Message:     hook.Message,
		Config:      string(yml),
	}
	commit.SetAuthor(hook.Author)

	// inserts the commit into the database
	if err := datastore.PostCommit(ctx, &commit); err != nil {
		log.Printf("Unable to persist commit %s@%s. %s\n", commit.Sha, commit.Branch, err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	owner, err := datastore.GetUser(ctx, repo.UserID)
	if err != nil {
		log.Printf("Unable to retrieve repository owner. %s.\n", err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	// drop the items on the queue
	go worker.Do(ctx, &worker.Work{
		User:   owner,
		Repo:   repo,
		Commit: &commit,
		Host:   httputil.GetURL(r),
	})

	w.WriteHeader(http.StatusOK)
}
<commit_msg>remove unused import<commit_after>package handler

import (
	"log"
	"net/http"
	"regexp"

	"github.com/drone/drone/plugin/remote"
	"github.com/drone/drone/server/datastore"
	"github.com/drone/drone/server/worker"
	"github.com/drone/drone/shared/build/script"
	"github.com/drone/drone/shared/httputil"
	"github.com/drone/drone/shared/model"
	"github.com/goji/context"
	"github.com/zenazn/goji/web"
)

// PostHook accepts a post-commit hook and parses the payload
// in order to trigger a build. The payload is specified to the
// remote system (ie GitHub) and will therefore get parsed by
// the appropriate remote plugin.
//
//     GET /api/hook/:host
//
func PostHook(c web.C, w http.ResponseWriter, r *http.Request) {
	var ctx = context.FromC(c)
	var host = c.URLParams["host"]
	var token = c.URLParams["token"]
	var remote = remote.Lookup(host)
	if remote == nil {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	// parse the hook payload
	hook, err := remote.ParseHook(r)
	if err != nil {
		log.Printf("Unable to parse hook. %s\n", err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	// in some cases we have neither a hook nor error. An example
	// would be GitHub sending a ping request to the URL, in which
	// case we'll just exit quiely with an 'OK'
	shouldSkip, _ := regexp.MatchString(`\[(?i:ci *skip|skip *ci)\]`, hook.Message)
	if hook == nil || shouldSkip {
		w.WriteHeader(http.StatusOK)
		return
	}

	// fetch the repository from the database
	repo, err := datastore.GetRepoName(ctx, remote.GetHost(), hook.Owner, hook.Repo)
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	// each hook contains a token to verify the sender. If the token
	// is not provided or does not match, exit
	if len(repo.Token) == 0 || repo.Token != token {
		log.Printf("Rejected post commit hook for %s. Token mismatch\n", repo.Name)
		w.WriteHeader(http.StatusUnauthorized)
		return
	}

	if repo.Active == false ||
		(repo.PostCommit == false && len(hook.PullRequest) == 0) ||
		(repo.PullRequest == false && len(hook.PullRequest) != 0) {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	// fetch the user from the database that owns this repo
	user, err := datastore.GetUser(ctx, repo.UserID)
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	// Request a new token and update
	user_token, err := remote.GetToken(user)
	if user_token != nil {
		user.Access = user_token.AccessToken
		user.Secret = user_token.RefreshToken
		user.TokenExpiry = user_token.Expiry
		datastore.PutUser(ctx, user)
	} else if err != nil {
		log.Printf("Unable to refresh token. %s\n", err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	// fetch the .drone.yml file from the database
	yml, err := remote.GetScript(user, repo, hook)
	if err != nil {
		log.Printf("Unable to fetch .drone.yml file. %s\n", err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	// verify the commit hooks branch matches the list of approved
	// branches (unless it is a pull request). Note that we don't really
	// care if parsing the yaml fails here.
	s, _ := script.ParseBuild(string(yml))
	if len(hook.PullRequest) == 0 && !s.MatchBranch(hook.Branch) {
		w.WriteHeader(http.StatusOK)
		return
	}

	commit := model.Commit{
		RepoID:      repo.ID,
		Status:      model.StatusEnqueue,
		Sha:         hook.Sha,
		Branch:      hook.Branch,
		PullRequest: hook.PullRequest,
		Timestamp:   hook.Timestamp,
		Message:     hook.Message,
		Config:      string(yml),
	}
	commit.SetAuthor(hook.Author)

	// inserts the commit into the database
	if err := datastore.PostCommit(ctx, &commit); err != nil {
		log.Printf("Unable to persist commit %s@%s. %s\n", commit.Sha, commit.Branch, err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	owner, err := datastore.GetUser(ctx, repo.UserID)
	if err != nil {
		log.Printf("Unable to retrieve repository owner. %s.\n", err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	// drop the items on the queue
	go worker.Do(ctx, &worker.Work{
		User:   owner,
		Repo:   repo,
		Commit: &commit,
		Host:   httputil.GetURL(r),
	})

	w.WriteHeader(http.StatusOK)
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"crypto/x509/pkix"
	"io/ioutil"
	"log"
	"math/big"
	"time"
)

func main() {
	ca := &x509.Certificate{
		SerialNumber: big.NewInt(1653),
		Subject: pkix.Name{
			Country:            []string{"China"},
			Organization:       []string{"HoleHUB"},
			OrganizationalUnit: []string{"holehub.com"},
		},
		NotBefore:             time.Now(),
		NotAfter:              time.Now().AddDate(10, 0, 0),
		SubjectKeyId:          []byte{1, 2, 3, 4, 5},
		BasicConstraintsValid: true,
		IsCA:        true,
		ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
		KeyUsage:    x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
	}

	priv, _ := rsa.GenerateKey(rand.Reader, 1024)
	pub := &priv.PublicKey
	ca_b, err := x509.CreateCertificate(rand.Reader, ca, ca, pub, priv)
	if err != nil {
		log.Println("create ca failed", err)
		return
	}
	ca_f := "ca.pem"
	log.Println("write to", ca_f)
	ioutil.WriteFile(ca_f, ca_b, 0777)

	priv_f := "ca.key"
	priv_b := x509.MarshalPKCS1PrivateKey(priv)
	log.Println("write to", priv_f)
	ioutil.WriteFile(priv_f, priv_b, 0777)

	cert2 := &x509.Certificate{
		SerialNumber: big.NewInt(1658),
		Subject: pkix.Name{
			Country:            []string{"China"},
			Organization:       []string{"HoleHUB"},
			OrganizationalUnit: []string{"holehub.com"},
		},
		NotBefore:    time.Now(),
		NotAfter:     time.Now().AddDate(10, 0, 0),
		SubjectKeyId: []byte{1, 2, 3, 4, 6},
		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
		KeyUsage:     x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
	}
	priv2, _ := rsa.GenerateKey(rand.Reader, 1024)
	pub2 := &priv2.PublicKey
	cert2_b, err2 := x509.CreateCertificate(rand.Reader, cert2, ca, pub2, priv)
	if err2 != nil {
		log.Println("create cert2 failed", err2)
		return
	}

	cert2_f := "cert.pem"
	log.Println("write to", cert2_f)
	ioutil.WriteFile(cert2_f, cert2_b, 0777)

	priv2_f := "cert.key"
	priv2_b := x509.MarshalPKCS1PrivateKey(priv2)
	log.Println("write to", priv2_f)
	ioutil.WriteFile(priv2_f, priv2_b, 0777)

	ca_c, _ := x509.ParseCertificate(ca_b)
	cert2_c, _ := x509.ParseCertificate(cert2_b)

	err3 := cert2_c.CheckSignatureFrom(ca_c)
	log.Println("check signature", err3 == nil)
}
<commit_msg>update pem mode<commit_after>package main

import (
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"crypto/x509/pkix"
	"io/ioutil"
	"log"
	"math/big"
	"time"
)

func main() {
	ca := &x509.Certificate{
		SerialNumber: big.NewInt(1653),
		Subject: pkix.Name{
			Country:            []string{"China"},
			Organization:       []string{"HoleHUB"},
			OrganizationalUnit: []string{"holehub.com"},
		},
		NotBefore:             time.Now(),
		NotAfter:              time.Now().AddDate(10, 0, 0),
		SubjectKeyId:          []byte{1, 2, 3, 4, 5},
		BasicConstraintsValid: true,
		IsCA:        true,
		ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
		KeyUsage:    x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
	}

	priv, _ := rsa.GenerateKey(rand.Reader, 1024)
	pub := &priv.PublicKey
	ca_b, err := x509.CreateCertificate(rand.Reader, ca, ca, pub, priv)
	if err != nil {
		log.Println("create ca failed", err)
		return
	}
	ca_f := "ca.pem"
	log.Println("write to", ca_f)
	ioutil.WriteFile(ca_f, ca_b, 0444)

	priv_f := "ca.key"
	priv_b := x509.MarshalPKCS1PrivateKey(priv)
	log.Println("write to", priv_f)
	ioutil.WriteFile(priv_f, priv_b, 0444)

	cert2 := &x509.Certificate{
		SerialNumber: big.NewInt(1658),
		Subject: pkix.Name{
			Country:            []string{"China"},
			Organization:       []string{"HoleHUB"},
			OrganizationalUnit: []string{"holehub.com"},
		},
		NotBefore:    time.Now(),
		NotAfter:     time.Now().AddDate(10, 0, 0),
		SubjectKeyId: []byte{1, 2, 3, 4, 6},
		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
		KeyUsage:     x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
	}
	priv2, _ := rsa.GenerateKey(rand.Reader, 1024)
	pub2 := &priv2.PublicKey
	cert2_b, err2 := x509.CreateCertificate(rand.Reader, cert2, ca, pub2, priv)
	if err2 != nil {
		log.Println("create cert2 failed", err2)
		return
	}

	cert2_f := "cert.pem"
	log.Println("write to", cert2_f)
	ioutil.WriteFile(cert2_f, cert2_b, 0444)

	priv2_f := "cert.key"
	priv2_b := x509.MarshalPKCS1PrivateKey(priv2)
	log.Println("write to", priv2_f)
	ioutil.WriteFile(priv2_f, priv2_b, 0444)

	ca_c, _ := x509.ParseCertificate(ca_b)
	cert2_c, _ := x509.ParseCertificate(cert2_b)

	err3 := cert2_c.CheckSignatureFrom(ca_c)
	log.Println("check signature", err3 == nil)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"fmt"
	"github.com/spf13/cobra"
	"io"
	"k8s.io/kops/cmd/kops/util"
	api "k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/upup/pkg/fi/cloudup"
	"k8s.io/kubernetes/pkg/kubectl/cmd/util/editor"
	"os"
	"path/filepath"
	"strings"
)

type CreateInstanceGroupOptions struct {
	Role string
}

func NewCmdCreateInstanceGroup(f *util.Factory, out io.Writer) *cobra.Command {
	options := &CreateInstanceGroupOptions{
		Role: string(api.InstanceGroupRoleNode),
	}

	cmd := &cobra.Command{
		Use:     "instancegroup",
		Aliases: []string{"instancegroups", "ig"},
		Short:   "Create instancegroup",
		Long:    `Create an instancegroup configuration.`,
		Run: func(cmd *cobra.Command, args []string) {
			err := RunCreateInstanceGroup(f, cmd, args, os.Stdout, options)
			if err != nil {
				exitWithError(err)
			}
		},
	}

	// TODO: Create Enum helper - or is there one in k8s already?
	var allRoles []string
	for _, r := range api.AllInstanceGroupRoles {
		allRoles = append(allRoles, string(r))
	}

	cmd.Flags().StringVar(&options.Role, "role", options.Role, "Type of instance group to create ("+strings.Join(allRoles, ",")+")")

	return cmd
}

func RunCreateInstanceGroup(f *util.Factory, cmd *cobra.Command, args []string, out io.Writer, options *CreateInstanceGroupOptions) error {
	if len(args) == 0 {
		return fmt.Errorf("Specify name of instance group to create")
	}
	if len(args) != 1 {
		return fmt.Errorf("Can only create one instance group at a time!")
	}
	groupName := args[0]

	cluster, err := rootCommand.Cluster()

	clientset, err := rootCommand.Clientset()
	if err != nil {
		return err
	}

	channel, err := cloudup.ChannelForCluster(cluster)
	if err != nil {
		return err
	}

	existing, err := clientset.InstanceGroups(cluster.ObjectMeta.Name).Get(groupName)
	if err != nil {
		return err
	}

	if existing != nil {
		return fmt.Errorf("instance group %q already exists", groupName)
	}

	// Populate some defaults
	ig := &api.InstanceGroup{}
	ig.ObjectMeta.Name = groupName

	role, ok := api.ParseInstanceGroupRole(options.Role, true)
	if !ok {
		return fmt.Errorf("unknown role %q", options.Role)
	}
	ig.Spec.Role = role

	ig, err = cloudup.PopulateInstanceGroupSpec(cluster, ig, channel)
	if err != nil {
		return err
	}

	var (
		edit = editor.NewDefaultEditor(editorEnvs)
	)

	raw, err := api.ToVersionedYaml(ig)
	if err != nil {
		return err
	}
	ext := "yaml"

	// launch the editor
	edited, file, err := edit.LaunchTempFile(fmt.Sprintf("%s-edit-", filepath.Base(os.Args[0])), ext, bytes.NewReader(raw))
	defer func() {
		if file != "" {
			os.Remove(file)
		}
	}()
	if err != nil {
		return fmt.Errorf("error launching editor: %v", err)
	}

	obj, _, err := api.ParseVersionedYaml(edited)
	if err != nil {
		return fmt.Errorf("error parsing yaml: %v", err)
	}
	group, ok := obj.(*api.InstanceGroup)
	if !ok {
		return fmt.Errorf("unexpected object type: %T", obj)
	}

	err = group.Validate()
	if err != nil {
		return err
	}

	_, err = clientset.InstanceGroups(cluster.ObjectMeta.Name).Create(group)
	if err != nil {
		return fmt.Errorf("error storing InstanceGroup: %v", err)
	}

	return nil
}
<commit_msg>Allow specifying subnets on `create ig`<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"fmt"
	"github.com/spf13/cobra"
	"io"
	"k8s.io/kops/cmd/kops/util"
	api "k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/upup/pkg/fi/cloudup"
	"k8s.io/kubernetes/pkg/kubectl/cmd/util/editor"
	"os"
	"path/filepath"
	"strings"
)

type CreateInstanceGroupOptions struct {
	Role    string
	Subnets []string
}

func NewCmdCreateInstanceGroup(f *util.Factory, out io.Writer) *cobra.Command {
	options := &CreateInstanceGroupOptions{
		Role: string(api.InstanceGroupRoleNode),
	}

	cmd := &cobra.Command{
		Use:     "instancegroup",
		Aliases: []string{"instancegroups", "ig"},
		Short:   "Create instancegroup",
		Long:    `Create an instancegroup configuration.`,
		Run: func(cmd *cobra.Command, args []string) {
			err := RunCreateInstanceGroup(f, cmd, args, os.Stdout, options)
			if err != nil {
				exitWithError(err)
			}
		},
	}

	// TODO: Create Enum helper - or is there one in k8s already?
	var allRoles []string
	for _, r := range api.AllInstanceGroupRoles {
		allRoles = append(allRoles, string(r))
	}

	cmd.Flags().StringVar(&options.Role, "role", options.Role, "Type of instance group to create ("+strings.Join(allRoles, ",")+")")
	cmd.Flags().StringSliceVar(&options.Subnets, "subnet", options.Subnets, "Subnets in which to create instance group")

	return cmd
}

func RunCreateInstanceGroup(f *util.Factory, cmd *cobra.Command, args []string, out io.Writer, options *CreateInstanceGroupOptions) error {
	if len(args) == 0 {
		return fmt.Errorf("Specify name of instance group to create")
	}
	if len(args) != 1 {
		return fmt.Errorf("Can only create one instance group at a time!")
	}
	groupName := args[0]

	cluster, err := rootCommand.Cluster()

	clientset, err := rootCommand.Clientset()
	if err != nil {
		return err
	}

	channel, err := cloudup.ChannelForCluster(cluster)
	if err != nil {
		return err
	}

	existing, err := clientset.InstanceGroups(cluster.ObjectMeta.Name).Get(groupName)
	if err != nil {
		return err
	}

	if existing != nil {
		return fmt.Errorf("instance group %q already exists", groupName)
	}

	// Populate some defaults
	ig := &api.InstanceGroup{}
	ig.ObjectMeta.Name = groupName

	role, ok := api.ParseInstanceGroupRole(options.Role, true)
	if !ok {
		return fmt.Errorf("unknown role %q", options.Role)
	}
	ig.Spec.Role = role

	if len(options.Subnets) == 0 {
		return fmt.Errorf("cannot create instance group without subnets; specify --subnet flag(s)")
	}
	ig.Spec.Subnets = options.Subnets

	ig, err = cloudup.PopulateInstanceGroupSpec(cluster, ig, channel)
	if err != nil {
		return err
	}

	var (
		edit = editor.NewDefaultEditor(editorEnvs)
	)

	raw, err := api.ToVersionedYaml(ig)
	if err != nil {
		return err
	}
	ext := "yaml"

	// launch the editor
	edited, file, err := edit.LaunchTempFile(fmt.Sprintf("%s-edit-", filepath.Base(os.Args[0])), ext, bytes.NewReader(raw))
	defer func() {
		if file != "" {
			os.Remove(file)
		}
	}()
	if err != nil {
		return fmt.Errorf("error launching editor: %v", err)
	}

	obj, _, err := api.ParseVersionedYaml(edited)
	if err != nil {
		return fmt.Errorf("error parsing yaml: %v", err)
	}
	group, ok := obj.(*api.InstanceGroup)
	if !ok {
		return fmt.Errorf("unexpected object type: %T", obj)
	}

	err = group.Validate()
	if err != nil {
		return err
	}

	_, err = clientset.InstanceGroups(cluster.ObjectMeta.Name).Create(group)
	if err != nil {
		return fmt.Errorf("error storing InstanceGroup: %v", err)
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	log "github.com/Sirupsen/logrus"
	"github.com/loadimpact/speedboat"
	"github.com/loadimpact/speedboat/js"
	"github.com/loadimpact/speedboat/sampler"
	"github.com/loadimpact/speedboat/sampler/influxdb"
	"github.com/loadimpact/speedboat/sampler/stream"
	"github.com/loadimpact/speedboat/simple"
	"github.com/urfave/cli"
	"golang.org/x/net/context"
	"gopkg.in/yaml.v2"
	"io"
	"io/ioutil"
	"os"
	"strings"
	"time"
)

const (
	typeURL = "url"
	typeYML = "yml"
	typeJS  = "js"
)

// Configure the global logger.
func configureLogging(c *cli.Context) {
	log.SetLevel(log.InfoLevel)
	if c.GlobalBool("verbose") {
		log.SetLevel(log.DebugLevel)
	}
}

// Configure the global sampler.
func configureSampler(c *cli.Context) {
	sampler.DefaultSampler.OnError = func(err error) {
		log.WithError(err).Error("[Sampler error]")
	}

	for _, output := range c.GlobalStringSlice("metrics") {
		parts := strings.SplitN(output, "+", 2)
		switch parts[0] {
		case "influxdb":
			out, err := influxdb.NewFromURL(parts[1])
			if err != nil {
				log.WithError(err).Fatal("Couldn't create InfluxDB client")
			}
			sampler.DefaultSampler.Outputs = append(sampler.DefaultSampler.Outputs, out)
		default:
			var writer io.WriteCloser
			switch output {
			case "stdout", "-":
				writer = os.Stdout
			default:
				file, err := os.Create(output)
				if err != nil {
					log.WithError(err).Fatal("Couldn't create output file")
				}
				writer = file
			}

			var out sampler.Output
			switch c.GlobalString("format") {
			case "json":
				out = &stream.JSONOutput{Output: writer}
			case "csv":
				out = &stream.CSVOutput{Output: writer}
			default:
				log.Fatal("Unknown output format")
			}
			sampler.DefaultSampler.Outputs = append(sampler.DefaultSampler.Outputs, out)
		}
	}
}

func guessType(arg string) string {
	switch {
	case strings.Contains(arg, "://"):
		return typeURL
	case strings.HasSuffix(arg, ".js"):
		return typeJS
	case strings.HasSuffix(arg, ".yml"):
		return typeYML
	}
	return ""
}

func parse(cc *cli.Context) (conf Config, err error) {
	if len(cc.Args()) == 0 {
		return conf, errors.New("Nothing to do!")
	}

	conf.VUs = cc.Int("vus")
	conf.Duration = cc.Duration("duration").String()

	arg := cc.Args()[0]
	argType := cc.String("type")
	if argType == "" {
		argType = guessType(arg)
	}

	switch argType {
	case typeYML:
		bytes, err := ioutil.ReadFile(cc.Args()[0])
		if err != nil {
			return conf, errors.New("Couldn't read config file")
		}
		if err := yaml.Unmarshal(bytes, &conf); err != nil {
			return conf, errors.New("Couldn't parse config file")
		}
	case typeURL:
		conf.URL = arg
	case typeJS:
		conf.Script = arg
	default:
		return conf, errors.New("Unsure of what to do, try specifying --type")
	}

	return conf, nil
}

func dumpTest(t *speedboat.Test) {
	log.WithFields(log.Fields{
		"script": t.Script,
		"url":    t.URL,
	}).Info("General")
	for i, stage := range t.Stages {
		log.WithFields(log.Fields{
			"#":        i,
			"duration": stage.Duration,
			"start":    stage.StartVUs,
			"end":      stage.EndVUs,
		}).Info("Stage")
	}
}

func headlessController(c context.Context, t *speedboat.Test) <-chan int {
	ch := make(chan int)

	go func() {
		defer close(ch)

		select {
		case ch <- t.VUsAt(0):
		case <-c.Done():
			return
		}

		startTime := time.Now()
		ticker := time.NewTicker(100 * time.Millisecond)
		for {
			select {
			case <-ticker.C:
				ch <- t.VUsAt(time.Since(startTime))
			case <-c.Done():
				return
			}
		}
	}()

	return ch
}

func action(cc *cli.Context) error {
	if len(cc.Args()) == 0 {
		cli.ShowAppHelp(cc)
		return nil
	}

	conf, err := parse(cc)
	if err != nil {
		log.WithError(err).Fatal("Invalid arguments; see --help")
	}

	t, err := conf.MakeTest()
	if err != nil {
		log.WithError(err).Fatal("Configuration error")
	}

	if cc.Bool("dump") {
		dumpTest(&t)
		return nil
	}

	// Inspect the test to find a suitable runner; additional ones can easily be added
	var runner speedboat.Runner
	switch {
	case t.Script == "":
		runner = simple.New()
	case strings.HasSuffix(t.Script, ".js"):
		src, err := ioutil.ReadFile(t.Script)
		if err != nil {
			log.WithError(err).Fatal("Couldn't read script")
		}
		runner = js.New(t.Script, string(src))
	default:
		log.Fatal("No suitable runner found!")
	}

	// Context that expires at the end of the test
	ctx, cancel := context.WithTimeout(context.Background(), t.TotalDuration())

	// Configure the VU logger
	logger := &log.Logger{
		Out:       os.Stderr,
		Level:     log.DebugLevel,
		Formatter: &log.TextFormatter{},
	}
	ctx = speedboat.WithLogger(ctx, logger)

	// Store metrics unless the --quiet flag is specified
	quiet := cc.Bool("quiet")
	sampler.DefaultSampler.Accumulate = !quiet

	// Commit metrics to any configured backends once per second
	go func() {
		ticker := time.NewTicker(1 * time.Second)
		for {
			select {
			case <-ticker.C:
				commitMetrics()
			case <-ctx.Done():
				return
			}
		}
	}()

	// Use a "headless controller" to scale VUs by polling the test ramp
	mVUs := sampler.Gauge("vus")
	vus := []context.CancelFunc{}
	for scale := range headlessController(ctx, &t) {
		for i := len(vus); i < scale; i++ {
			log.WithField("id", i).Debug("Spawning VU")
			vuCtx, vuCancel := context.WithCancel(ctx)
			vus = append(vus, vuCancel)
			go func() {
				defer func() {
					if v := recover(); v != nil {
						switch err := v.(type) {
						case speedboat.FlowControl:
							switch err {
							case speedboat.AbortTest:
								log.Error("Test aborted")
								cancel()
							}
						default:
							log.WithFields(log.Fields{
								"id":    i,
								"error": err,
							}).Error("VU crashed!")
						}
					}
				}()
				runner.RunVU(vuCtx, t, len(vus))
			}()
		}
		for i := len(vus); i > scale; i-- {
			log.WithField("id", i-1).Debug("Dropping VU")
			vus[i-1]()
			vus = vus[:i-1]
		}
		mVUs.Int(len(vus))
	}

	// Wait until the end of the test
	<-ctx.Done()

	// Print and commit final metrics
	if !quiet {
		printMetrics()
	}
	commitMetrics()
	closeMetrics()

	return nil
}

func main() {
	// Free up -v and -h for our own flags
	cli.VersionFlag.Name = "version"
	cli.HelpFlag.Name = "help, ?"

	// Bootstrap using action-registered commandline flags
	app := cli.NewApp()
	app.Name = "speedboat"
	app.Usage = "A next-generation load generator"
	app.Version = "0.0.1a1"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "type, t",
			Usage: "Input file type, if not evident (url, yml or js)",
		},
		cli.IntFlag{
			Name:  "vus, u",
			Usage: "Number of VUs to simulate",
			Value: 10,
		},
		cli.DurationFlag{
			Name:  "duration, d",
			Usage: "Test duration",
			Value: time.Duration(10) * time.Second,
		},
		cli.BoolFlag{
			Name:  "verbose, v",
			Usage: "More verbose output",
		},
		cli.BoolFlag{
			Name:  "quiet, q",
			Usage: "Suppress the summary at the end of a test",
		},
		cli.StringSliceFlag{
			Name:  "metrics, m",
			Usage: "Write metrics to a file or database",
		},
		cli.StringFlag{
			Name:  "format, f",
			Usage: "Metric output format (json or csv)",
			Value: "json",
		},
		cli.BoolFlag{
			Name:  "dump",
			Usage: "Dump parsed test and exit",
		},
	}
	app.Before = func(c *cli.Context) error {
		configureLogging(c)
		configureSampler(c)
		return nil
	}
	app.Action = action
	app.Run(os.Args)
}
<commit_msg>[feat] No more --dump<commit_after>package main

import (
	"errors"
	log "github.com/Sirupsen/logrus"
	"github.com/loadimpact/speedboat"
	"github.com/loadimpact/speedboat/js"
	"github.com/loadimpact/speedboat/sampler"
	"github.com/loadimpact/speedboat/sampler/influxdb"
	"github.com/loadimpact/speedboat/sampler/stream"
	"github.com/loadimpact/speedboat/simple"
	"github.com/urfave/cli"
	"golang.org/x/net/context"
	"gopkg.in/yaml.v2"
	"io"
	"io/ioutil"
	"os"
	"strings"
	"time"
)

const (
	typeURL = "url"
	typeYML = "yml"
	typeJS  = "js"
)

// Configure the global logger.
func configureLogging(c *cli.Context) {
	log.SetLevel(log.InfoLevel)
	if c.GlobalBool("verbose") {
		log.SetLevel(log.DebugLevel)
	}
}

// Configure the global sampler.
func configureSampler(c *cli.Context) {
	sampler.DefaultSampler.OnError = func(err error) {
		log.WithError(err).Error("[Sampler error]")
	}

	for _, output := range c.GlobalStringSlice("metrics") {
		parts := strings.SplitN(output, "+", 2)
		switch parts[0] {
		case "influxdb":
			out, err := influxdb.NewFromURL(parts[1])
			if err != nil {
				log.WithError(err).Fatal("Couldn't create InfluxDB client")
			}
			sampler.DefaultSampler.Outputs = append(sampler.DefaultSampler.Outputs, out)
		default:
			var writer io.WriteCloser
			switch output {
			case "stdout", "-":
				writer = os.Stdout
			default:
				file, err := os.Create(output)
				if err != nil {
					log.WithError(err).Fatal("Couldn't create output file")
				}
				writer = file
			}

			var out sampler.Output
			switch c.GlobalString("format") {
			case "json":
				out = &stream.JSONOutput{Output: writer}
			case "csv":
				out = &stream.CSVOutput{Output: writer}
			default:
				log.Fatal("Unknown output format")
			}
			sampler.DefaultSampler.Outputs = append(sampler.DefaultSampler.Outputs, out)
		}
	}
}

func guessType(arg string) string {
	switch {
	case strings.Contains(arg, "://"):
		return typeURL
	case strings.HasSuffix(arg, ".js"):
		return typeJS
	case strings.HasSuffix(arg, ".yml"):
		return typeYML
	}
	return ""
}

func parse(cc *cli.Context) (conf Config, err error) {
	if len(cc.Args()) == 0 {
		return conf, errors.New("Nothing to do!")
	}

	conf.VUs = cc.Int("vus")
	conf.Duration = cc.Duration("duration").String()

	arg := cc.Args()[0]
	argType := cc.String("type")
	if argType == "" {
		argType = guessType(arg)
	}

	switch argType {
	case typeYML:
		bytes, err := ioutil.ReadFile(cc.Args()[0])
		if err != nil {
			return conf, errors.New("Couldn't read config file")
		}
		if err := yaml.Unmarshal(bytes, &conf); err != nil {
			return conf, errors.New("Couldn't parse config file")
		}
	case typeURL:
		conf.URL = arg
	case typeJS:
		conf.Script = arg
	default:
		return conf, errors.New("Unsure of what to do, try specifying --type")
	}

	return conf, nil
}

func headlessController(c context.Context, t *speedboat.Test) <-chan int {
	ch := make(chan int)

	go func() {
		defer close(ch)

		select {
		case ch <- t.VUsAt(0):
		case <-c.Done():
			return
		}

		startTime := time.Now()
		ticker := time.NewTicker(100 * time.Millisecond)
		for {
			select {
			case <-ticker.C:
				ch <- t.VUsAt(time.Since(startTime))
			case <-c.Done():
				return
			}
		}
	}()

	return ch
}

func action(cc *cli.Context) error {
	if len(cc.Args()) == 0 {
		cli.ShowAppHelp(cc)
		return nil
	}

	conf, err := parse(cc)
	if err != nil {
		log.WithError(err).Fatal("Invalid arguments; see --help")
	}

	t, err := conf.MakeTest()
	if err != nil {
		log.WithError(err).Fatal("Configuration error")
	}

	// Inspect the test to find a suitable runner; additional ones can easily be added
	var runner speedboat.Runner
	switch {
	case t.Script == "":
		runner = simple.New()
	case strings.HasSuffix(t.Script, ".js"):
		src, err := ioutil.ReadFile(t.Script)
		if err != nil {
			log.WithError(err).Fatal("Couldn't read script")
		}
		runner = js.New(t.Script, string(src))
	default:
		log.Fatal("No suitable runner found!")
	}

	// Context that expires at the end of the test
	ctx, cancel := context.WithTimeout(context.Background(), t.TotalDuration())

	// Configure the VU logger
	logger := &log.Logger{
		Out:       os.Stderr,
		Level:     log.DebugLevel,
		Formatter: &log.TextFormatter{},
	}
	ctx = speedboat.WithLogger(ctx, logger)

	// Store metrics unless the --quiet flag is specified
	quiet := cc.Bool("quiet")
	sampler.DefaultSampler.Accumulate = !quiet

	// Commit metrics to any configured backends once per second
	go func() {
		ticker := time.NewTicker(1 * time.Second)
		for {
			select {
			case <-ticker.C:
				commitMetrics()
			case <-ctx.Done():
				return
			}
		}
	}()

	// Use a "headless controller" to scale VUs by polling the test ramp
	mVUs := sampler.Gauge("vus")
	vus := []context.CancelFunc{}
	for scale := range headlessController(ctx, &t) {
		for i := len(vus); i < scale; i++ {
			log.WithField("id", i).Debug("Spawning VU")
			vuCtx, vuCancel := context.WithCancel(ctx)
			vus = append(vus, vuCancel)
			go func() {
				defer func() {
					if v := recover(); v != nil {
						switch err := v.(type) {
						case speedboat.FlowControl:
							switch err {
							case speedboat.AbortTest:
								log.Error("Test aborted")
								cancel()
							}
						default:
							log.WithFields(log.Fields{
								"id":    i,
								"error": err,
							}).Error("VU crashed!")
						}
					}
				}()
				runner.RunVU(vuCtx, t, len(vus))
			}()
		}
		for i := len(vus); i > scale; i-- {
			log.WithField("id", i-1).Debug("Dropping VU")
			vus[i-1]()
			vus = vus[:i-1]
		}
		mVUs.Int(len(vus))
	}

	// Wait until the end of the test
	<-ctx.Done()

	// Print and commit final metrics
	if !quiet {
		printMetrics()
	}
	commitMetrics()
	closeMetrics()

	return nil
}

func main() {
	// Free up -v and -h for our own flags
	cli.VersionFlag.Name = "version"
	cli.HelpFlag.Name = "help, ?"

	// Bootstrap using action-registered commandline flags
	app := cli.NewApp()
	app.Name = "speedboat"
	app.Usage = "A next-generation load generator"
	app.Version = "0.0.1a1"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "type, t",
			Usage: "Input file type, if not evident (url, yml or js)",
		},
		cli.IntFlag{
			Name:  "vus, u",
			Usage: "Number of VUs to simulate",
			Value: 10,
		},
		cli.DurationFlag{
			Name:  "duration, d",
			Usage: "Test duration",
			Value: time.Duration(10) * time.Second,
		},
		cli.BoolFlag{
			Name:  "verbose, v",
			Usage: "More verbose output",
		},
		cli.BoolFlag{
			Name:  "quiet, q",
			Usage: "Suppress the summary at the end of a test",
		},
		cli.StringSliceFlag{
			Name:  "metrics, m",
			Usage: "Write metrics to a file or database",
		},
		cli.StringFlag{
			Name:  "format, f",
			Usage: "Metric output format (json or csv)",
			Value: "json",
		},
	}
	app.Before = func(c *cli.Context) error {
		configureLogging(c)
		configureSampler(c)
		return nil
	}
	app.Action = action
	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/fujiwara/stretcher"
	"os"
)

var (
	version   string
	buildDate string
)

func main() {
	var (
		showVersion bool
	)
	flag.BoolVar(&showVersion, "v", false, "show version")
	flag.BoolVar(&showVersion, "version", false, "show version")
	flag.Parse()

	if showVersion {
		fmt.Println("version:", version)
		fmt.Println("build:", buildDate)
		os.Exit(0)
	}
	stretcher.Run()
}
<commit_msg>show version at startup<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/fujiwara/stretcher"
	"log"
)

var (
	version   string
	buildDate string
)

func main() {
	var (
		showVersion bool
	)
	flag.BoolVar(&showVersion, "v", false, "show version")
	flag.BoolVar(&showVersion, "version", false, "show version")
	flag.Parse()

	if showVersion {
		fmt.Println("version:", version)
		fmt.Println("build:", buildDate)
		return
	}
	log.Println("stretcher version:", version)
	stretcher.Run()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"strconv"
	"strings"
	"time"

	"github.com/nelhage/taktician/ai"
	"github.com/nelhage/taktician/playtak"
	"github.com/nelhage/taktician/ptn"
	"github.com/nelhage/taktician/tak"
)

var (
	server   = flag.String("server", "playtak.com:10000", "playtak.com server to connect to")
	depth    = flag.Int("depth", 5, "minimax depth")
	user     = flag.String("user", "", "username for login")
	pass     = flag.String("pass", "", "password for login")
	accept   = flag.String("accept", "", "accept a game from specified user")
	gameTime = flag.Duration("time", 20*time.Minute, "Length of game to offer")
	size     = flag.Int("size", 5, "size of game to offer")
	once     = flag.Bool("once", false, "play a single game and exit")
	takbot   = flag.String("takbot", "", "challenge TakBot AI")
)

const ClientName = "Taktician AI"

func main() {
	flag.Parse()
	if *accept != "" || *takbot != "" {
		*once = true
	}

	backoff := 1 * time.Second
	for {
		client := &playtak.Client{
			Debug: true,
		}
		err := client.Connect(*server)
		if err != nil {
			goto reconnect
		}
		backoff = time.Second
		client.SendClient(ClientName)
		if *user != "" {
			err = client.Login(*user, *pass)
		} else {
			err = client.LoginGuest()
		}
		if err != nil {
			log.Fatal("login: ", err)
		}
		for {
			if *accept != "" {
				for line := range client.Recv {
					if strings.HasPrefix(line, "Seek new") {
						bits := strings.Split(line, " ")
						if bits[3] == *accept {
							log.Printf("accepting game %s from %s", bits[2], bits[3])
							client.SendCommand("Accept", bits[2])
							break
						}
					}
				}
			} else {
				client.SendCommand("Seek", strconv.Itoa(*size), strconv.Itoa(int(gameTime.Seconds())))
				if *takbot != "" {
					client.SendCommand("Shout", "takbot: play", *takbot)
				}
			}
			for line := range client.Recv {
				if strings.HasPrefix(line, "Game Start") {
					playGame(client, line)
					break
				}
			}
			if *once {
				return
			}
			if client.Error() != nil {
				log.Printf("Disconnected: %v", client.Error())
				break
			}
		}
	reconnect:
		log.Printf("sleeping %s before reconnect...", backoff)
		time.Sleep(backoff)
		backoff = backoff * 2
		if backoff > time.Minute {
			backoff = time.Minute
		}
	}
}

func timeBound(remaining time.Duration) time.Duration {
	return time.Minute
}

func playGame(c *playtak.Client, line string) {
	log.Println("New Game", line)
	bits := strings.Split(line, " ")
	size, _ := strconv.Atoi(bits[3])
	ai := ai.NewMinimax(ai.MinimaxConfig{
		Size:  size,
		Depth: *depth,
		Debug: 1,
	})
	p := tak.New(tak.Config{Size: size})
	gameStr := fmt.Sprintf("Game#%s", bits[2])
	var color tak.Color
	switch bits[7] {
	case "white":
		color = tak.White
	case "black":
		color = tak.Black
	default:
		panic(fmt.Sprintf("bad color: %s", bits[7]))
	}
	timeLeft := *gameTime
	for {
		over, _ := p.GameOver()
		if color == p.ToMove() && !over {
			move := ai.GetMove(p, timeBound(timeLeft))
			next, err := p.Move(&move)
			if err != nil {
				log.Printf("ai returned bad move: %s: %s",
					ptn.FormatMove(&move), err)
				continue
			}
			p = next
			c.SendCommand(gameStr, playtak.FormatServer(&move))
		} else {
			var timeout <-chan time.Time
		theirMove:
			for {
				var line string
				select {
				case line = <-c.Recv:
				case <-timeout:
					break theirMove
				}

				if !strings.HasPrefix(line, gameStr) {
					continue
				}
				bits = strings.Split(line, " ")
				switch bits[1] {
				case "P", "M":
					move, err := playtak.ParseServer(strings.Join(bits[1:], " "))
					if err != nil {
						panic(err)
					}
					p, err = p.Move(&move)
					if err != nil {
						panic(err)
					}
					timeout = time.NewTimer(500 * time.Millisecond).C
				case "Abandoned.", "Over":
					return
				case "Time":
					w, b := bits[2], bits[3]
					var secsLeft int
					if color == tak.White {
						secsLeft, _ = strconv.Atoi(w)
					} else {
						secsLeft, _ = strconv.Atoi(b)
					}
					timeLeft = time.Duration(secsLeft) * time.Second
					break theirMove
				}
			}
		}
	}
}
<commit_msg>taktician -debug<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"strconv"
	"strings"
	"time"

	"github.com/nelhage/taktician/ai"
	"github.com/nelhage/taktician/playtak"
	"github.com/nelhage/taktician/ptn"
	"github.com/nelhage/taktician/tak"
)

var (
	server   = flag.String("server", "playtak.com:10000", "playtak.com server to connect to")
	depth    = flag.Int("depth", 5, "minimax depth")
	user     = flag.String("user", "", "username for login")
	pass     = flag.String("pass", "", "password for login")
	accept   = flag.String("accept", "", "accept a game from specified user")
	gameTime = flag.Duration("time", 20*time.Minute, "Length of game to offer")
	size     = flag.Int("size", 5, "size of game to offer")
	once     = flag.Bool("once", false, "play a single game and exit")
	debug    = flag.Int("debug", 1, "debug level")
	takbot   = flag.String("takbot", "", "challenge TakBot AI")
)

const ClientName = "Taktician AI"

func main() {
	flag.Parse()
	if *accept != "" || *takbot != "" {
		*once = true
	}

	backoff := 1 * time.Second
	for {
		client := &playtak.Client{
			Debug: true,
		}
		err := client.Connect(*server)
		if err != nil {
			goto reconnect
		}
		backoff = time.Second
		client.SendClient(ClientName)
		if *user != "" {
			err = client.Login(*user, *pass)
		} else {
			err = client.LoginGuest()
		}
		if err != nil {
			log.Fatal("login: ", err)
		}
		for {
			if *accept != "" {
				for line := range client.Recv {
					if strings.HasPrefix(line, "Seek new") {
						bits := strings.Split(line, " ")
						if bits[3] == *accept {
							log.Printf("accepting game %s from %s", bits[2], bits[3])
							client.SendCommand("Accept", bits[2])
							break
						}
					}
				}
			} else {
				client.SendCommand("Seek", strconv.Itoa(*size), strconv.Itoa(int(gameTime.Seconds())))
				if *takbot != "" {
					client.SendCommand("Shout", "takbot: play", *takbot)
				}
			}
			for line := range client.Recv {
				if strings.HasPrefix(line, "Game Start") {
					playGame(client, line)
					break
				}
			}
			if *once {
				return
			}
			if client.Error() != nil {
				log.Printf("Disconnected: %v", client.Error())
				break
			}
		}
	reconnect:
		log.Printf("sleeping %s before reconnect...", backoff)
		time.Sleep(backoff)
		backoff = backoff * 2
		if backoff > time.Minute {
			backoff = time.Minute
		}
	}
}

func timeBound(remaining time.Duration) time.Duration {
	return time.Minute
}

func playGame(c *playtak.Client, line string) {
	log.Println("New Game", line)
	bits := strings.Split(line, " ")
	size, _ := strconv.Atoi(bits[3])
	ai := ai.NewMinimax(ai.MinimaxConfig{
		Size:  size,
		Depth: *depth,
		Debug: *debug,
	})
	p := tak.New(tak.Config{Size: size})
	gameStr := fmt.Sprintf("Game#%s", bits[2])
	var color tak.Color
	switch bits[7] {
	case "white":
		color = tak.White
	case "black":
		color = tak.Black
	default:
		panic(fmt.Sprintf("bad color: %s", bits[7]))
	}
	timeLeft := *gameTime
	for {
		over, _ := p.GameOver()
		if color == p.ToMove() && !over {
			move := ai.GetMove(p, timeBound(timeLeft))
			next, err := p.Move(&move)
			if err != nil {
				log.Printf("ai returned bad move: %s: %s",
					ptn.FormatMove(&move), err)
				continue
			}
			p = next
			c.SendCommand(gameStr, playtak.FormatServer(&move))
		} else {
			var timeout <-chan time.Time
		theirMove:
			for {
				var line string
				select {
				case line = <-c.Recv:
				case <-timeout:
					break theirMove
				}

				if !strings.HasPrefix(line, gameStr) {
					continue
				}
				bits = strings.Split(line, " ")
				switch bits[1] {
				case "P", "M":
					move, err := playtak.ParseServer(strings.Join(bits[1:], " "))
					if err != nil {
						panic(err)
					}
					p, err = p.Move(&move)
					if err != nil {
						panic(err)
					}
					timeout = time.NewTimer(500 * time.Millisecond).C
				case "Abandoned.", "Over":
					return
				case "Time":
					w, b := bits[2], bits[3]
					var secsLeft int
					if color == tak.White {
						secsLeft, _ = strconv.Atoi(w)
					} else {
						secsLeft, _ = strconv.Atoi(b)
					}
					timeLeft = time.Duration(secsLeft) * time.Second
					break theirMove
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"net/http"
	"time"

	"github.com/NYTimes/gziphandler"
	"github.com/kelseyhightower/envconfig"

	"github.com/brnstz/bus/api"
	"github.com/brnstz/bus/internal/conf"
	"github.com/brnstz/bus/internal/etc"
)

func main() {
	var err error
	log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)

	err = envconfig.Process("bus", &conf.DB)
	if err != nil {
		log.Fatal(err)
	}

	err = envconfig.Process("bus", &conf.API)
	if err != nil {
		log.Fatal(err)
	}

	err = envconfig.Process("bus", &conf.Cache)
	if err != nil {
		log.Fatal(err)
	}

	err = envconfig.Process("bus", &conf.Partner)
	if err != nil {
		log.Fatal(err)
	}

	time.Local, err = time.LoadLocation("America/New_York")
	if err != nil {
		log.Fatal(err)
	}

	etc.DBConn = etc.MustDB()

	if conf.API.BuildTimestamp == 0 {
		conf.API.BuildTimestamp = time.Now().Unix()
	}

	handler := api.NewHandler()

	withgz := gziphandler.GzipHandler(handler)

	err = api.InitRouteCache()
	if err != nil {
		log.Fatal(err)
	}

	log.Fatal(http.ListenAndServe(conf.API.Addr, withgz))
}
<commit_msg>redirect<commit_after>package main

import (
	"log"
	"net/http"
	"time"

	"github.com/NYTimes/gziphandler"
	"github.com/kelseyhightower/envconfig"

	"github.com/brnstz/bus/api"
	"github.com/brnstz/bus/internal/conf"
	"github.com/brnstz/bus/internal/etc"
)

func main() {
	var err error
	log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)

	err = envconfig.Process("bus", &conf.DB)
	if err != nil {
		log.Fatal(err)
	}

	err = envconfig.Process("bus", &conf.API)
	if err != nil {
		log.Fatal(err)
	}

	err = envconfig.Process("bus", &conf.Cache)
	if err != nil {
		log.Fatal(err)
	}

	err = envconfig.Process("bus", &conf.Partner)
	if err != nil {
		log.Fatal(err)
	}

	time.Local, err = time.LoadLocation("America/New_York")
	if err != nil {
		log.Fatal(err)
	}

	etc.DBConn = etc.MustDB()

	if conf.API.BuildTimestamp == 0 {
		conf.API.BuildTimestamp = time.Now().Unix()
	}

	handler := api.NewHandler()

	withgz := gziphandler.GzipHandler(handler)

	err = api.InitRouteCache()
	if err != nil {
		log.Fatal(err)
	}

	// prod http to https redirect
	go func() {
		redirMux := http.NewServeMux()
		redirMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
			http.Redirect(w, r, "https://token.live/", http.StatusMovedPermanently)
		})
		err = http.ListenAndServe(":8001", redirMux)
		if err != nil {
			log.Fatal(err)
		}
	}()

	log.Fatal(http.ListenAndServe(conf.API.Addr, withgz))
}
<|endoftext|>


<commit_before>// Copyright 2019, 2020 Tamás Gulácsi
//
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package grpcer

import (
	"bytes"
	"fmt"
	//json "encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"os"
	"reflect"
	"strings"

	json "github.com/json-iterator/go"
)

var (
	errNewField = errors.New("new field")
	errWrongType = errors.New("wrong type")
)

func mergeStreams(w io.Writer, first interface{}, recv interface{ Recv() (interface{}, error) }, Log func(...interface{}) error) error {
	if Log == nil {
		Log = func(...interface{}) error { return nil }
	}

	slice, notSlice := sliceFields(first)
	if len(slice) == 0 {
		var err error
		part := first
		enc := json.NewEncoder(w)
		for {
			if err := enc.Encode(part); err != nil {
				Log("encode", part, "error", err)
				return fmt.Errorf("encode part: %w", err)
			}

			part, err = recv.Recv()
			if err != nil {
				if err != io.EOF {
					Log("msg", "recv", "error", err)
				}
				break
			}
		}
		Log("slice", len(slice))
		return nil
	}

	names := make(map[string]bool, len(slice)+len(notSlice))

	buf := bufPool.Get().(*bytes.Buffer)
	defer func() {
		buf.Reset()
		bufPool.Put(buf)
	}()
	jenc := json.NewEncoder(buf)

	//Log("slices", slice)
	w.Write([]byte("{"))
	for _, f := range notSlice {
		buf.Reset()
		jenc.Encode(f.JSONName)
		w.Write(bytes.TrimSpace(buf.Bytes()))

		w.Write([]byte{':'})
		buf.Reset()
		jenc.Encode(f.Value)
		w.Write(bytes.TrimSpace(buf.Bytes()))
		w.Write([]byte{','})

		names[f.Name] = false
	}
	buf.Reset()
	jenc.Encode(slice[0].JSONName)
	w.Write(bytes.TrimSpace(buf.Bytes()))
	w.Write([]byte(":"))

	buf.Reset()
	jenc.Encode(slice[0].Value)
	w.Write(bytes.TrimSuffix(bytes.TrimSpace(buf.Bytes()), []byte{']'}))

	names[slice[0].Name] = true
	files := make(map[string]*os.File, len(slice)-1)
	openFile := func(f field) error {
		fh, err := ioutil.TempFile("", "merge-"+f.Name+"-")
		if err != nil {
			Log("tempFile", f.Name, "error", err)
			return fmt.Errorf("%s: %w", f.Name, err)
		}
		os.Remove(fh.Name())
		Log("fn", fh.Name())
		files[f.Name] = fh
		buf.Reset()
		jenc.Encode(f.JSONName)
		fh.Write(bytes.TrimSpace(buf.Bytes()))
		io.WriteString(fh, ":[")

		buf.Reset()
		jenc.Encode(f.Value)
		fh.Write(trimSqBrs(buf.Bytes()))

		names[f.Name] = true
		return nil
	}
	defer func()  {
		for nm, fh := range files {
			fh.Close()
			delete(files, nm)
		}
	}()

	for _, f := range slice[1:] {
		if err := openFile(f); err != nil {
			return err
		}
	}

	var part interface{}
	var err error
	for {
		part, err = recv.Recv()
		if err != nil {
			if err != io.EOF {
				Log("msg", "recv", "error", err)
			}
			break
		}
		buf.Reset()
		jenc.Encode(part)
		Log("part", limitWidth(buf.Bytes(), 256))

		S, nS := sliceFields(part)
		for _, f := range S {
			if isSlice, ok := names[f.Name]; !ok {
				if err = openFile(f); err != nil {
					break
				}
				//err = fmt.Errorf("%s: %w", f.Name, errNewField)
				//break
			} else if !isSlice {
				err = fmt.Errorf("%s not slice: %w", f.Name, errWrongType)
				break
			}
		}
		for _, f := range nS {
			if isSlice, ok := names[f.Name]; !ok {
				err = fmt.Errorf("%s: %w", f.Name, errNewField)
				break
			} else if isSlice {
				err = fmt.Errorf("%s slice: %w", f.Name, errWrongType)
				break
			}
		}
		if len(S) == 0 {
			break
		}
		if err != nil {
			Log("error", err)
			//TODO(tgulacsi): close the merge and send as is
			return err
		}

		if S[0].Name == slice[0].Name {
			w.Write([]byte{','})
			buf.Reset()
			jenc.Encode(S[0].Value)
			w.Write(trimSqBrs(buf.Bytes()))
			S = S[1:]
		}
		for _, f := range S {
			fh := files[f.Name]
			if _, err := fh.Write([]byte{','}); err != nil {
				if Log != nil {
					Log("write", fh.Name(), "error", err)
				}
			}
			buf.Reset()
			jenc.Encode(f.Value)
			fh.Write(trimSqBrs(buf.Bytes()))
		}
	}
	w.Write([]byte("]"))

	for nm, fh := range files {
		if _, err := fh.Seek(0, 0); err != nil {
			Log("Seek", fh.Name(), "error", err)
			continue
		}
		w.Write([]byte{','})
		io.Copy(w, fh)
		w.Write([]byte{']'})
		fh.Close()
		delete(files, nm)
	}
	w.Write([]byte{'}', '\n'})
	return nil
}

type field struct {
	Name     string
	JSONName string
	Value    interface{}
}

func sliceFields(part interface{}) (slice, notSlice []field) {
	rv := reflect.ValueOf(part)
	t := rv.Type()
	if t.Kind() == reflect.Ptr {
		rv = rv.Elem()
		t = rv.Type()
	}
	n := t.NumField()
	for i := 0; i < n; i++ {
		f := rv.Field(i)
		tf := t.Field(i)
		fld := field{Name: tf.Name, Value: f.Interface()}
		fld.JSONName = tf.Tag.Get("json")
		if i := strings.IndexByte(fld.JSONName, ','); i >= 0 {
			fld.JSONName = fld.JSONName[:i]
		}
		if fld.JSONName == "" {
			fld.JSONName = fld.Name
		}

		if f.Type().Kind() != reflect.Slice {
			notSlice = append(notSlice, fld)
			continue
		}
		if f.IsNil() {
			continue
		}
		slice = append(slice, fld)
	}
	return slice, notSlice
}
func trimSqBrs(b []byte) []byte {
	b = bytes.TrimSpace(b)
	if len(b) == 0 {
		return b
	}
	if b[0] == '[' {
		b = b[1:]
	}
	if len(b) == 0 {
		return b
	}
	if b[len(b)-1] == ']' {
		b = b[:len(b)-1]
	}
	return b
}
<commit_msg>Export SliceFields for merging<commit_after>// Copyright 2019, 2021 Tamás Gulácsi
//
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package grpcer

import (
	"bytes"
	"fmt"

	//json "encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"os"
	"reflect"
	"strings"

	json "github.com/json-iterator/go"
)

var (
	errNewField  = errors.New("new field")
	errWrongType = errors.New("wrong type")
)

func mergeStreams(w io.Writer, first interface{}, recv interface{ Recv() (interface{}, error) }, Log func(...interface{}) error) error {
	if Log == nil {
		Log = func(...interface{}) error { return nil }
	}

	slice, notSlice := SliceFields(first, "json")
	if len(slice) == 0 {
		var err error
		part := first
		enc := json.NewEncoder(w)
		for {
			if err := enc.Encode(part); err != nil {
				Log("encode", part, "error", err)
				return fmt.Errorf("encode part: %w", err)
			}

			part, err = recv.Recv()
			if err != nil {
				if err != io.EOF {
					Log("msg", "recv", "error", err)
				}
				break
			}
		}
		Log("slice", len(slice))
		return nil
	}

	names := make(map[string]bool, len(slice)+len(notSlice))

	buf := bufPool.Get().(*bytes.Buffer)
	defer func() {
		buf.Reset()
		bufPool.Put(buf)
	}()
	jenc := json.NewEncoder(buf)

	//Log("slices", slice)
	w.Write([]byte("{"))
	for _, f := range notSlice {
		buf.Reset()
		jenc.Encode(f.TagName)
		w.Write(bytes.TrimSpace(buf.Bytes()))

		w.Write([]byte{':'})
		buf.Reset()
		jenc.Encode(f.Value)
		w.Write(bytes.TrimSpace(buf.Bytes()))
		w.Write([]byte{','})

		names[f.Name] = false
	}
	buf.Reset()
	jenc.Encode(slice[0].TagName)
	w.Write(bytes.TrimSpace(buf.Bytes()))
	w.Write([]byte(":"))

	buf.Reset()
	jenc.Encode(slice[0].Value)
	w.Write(bytes.TrimSuffix(bytes.TrimSpace(buf.Bytes()), []byte{']'}))

	names[slice[0].Name] = true
	files := make(map[string]*os.File, len(slice)-1)
	openFile := func(f Field) error {
		fh, err := ioutil.TempFile("", "merge-"+f.Name+"-")
		if err != nil {
			Log("tempFile", f.Name, "error", err)
			return fmt.Errorf("%s: %w", f.Name, err)
		}
		os.Remove(fh.Name())
		Log("fn", fh.Name())
		files[f.Name] = fh
		buf.Reset()
		jenc.Encode(f.TagName)
		fh.Write(bytes.TrimSpace(buf.Bytes()))
		io.WriteString(fh, ":[")

		buf.Reset()
		jenc.Encode(f.Value)
		fh.Write(trimSqBrs(buf.Bytes()))

		names[f.Name] = true
		return nil
	}
	defer func() {
		for nm, fh := range files {
			fh.Close()
			delete(files, nm)
		}
	}()

	for _, f := range slice[1:] {
		if err := openFile(f); err != nil {
			return err
		}
	}

	var part interface{}
	var err error
	for {
		part, err = recv.Recv()
		if err != nil {
			if err != io.EOF {
				Log("msg", "recv", "error", err)
			}
			break
		}
		buf.Reset()
		jenc.Encode(part)
		Log("part", limitWidth(buf.Bytes(), 256))

		S, nS := SliceFields(part, "json")
		for _, f := range S {
			if isSlice, ok := names[f.Name]; !ok {
				if err = openFile(f); err != nil {
					break
				}
				//err = fmt.Errorf("%s: %w", f.Name, errNewField)
				//break
			} else if !isSlice {
				err = fmt.Errorf("%s not slice: %w", f.Name, errWrongType)
				break
			}
		}
		for _, f := range nS {
			if isSlice, ok := names[f.Name]; !ok {
				err = fmt.Errorf("%s: %w", f.Name, errNewField)
				break
			} else if isSlice {
				err = fmt.Errorf("%s slice: %w", f.Name, errWrongType)
				break
			}
		}
		if len(S) == 0 {
			break
		}
		if err != nil {
			Log("error", err)
			//TODO(tgulacsi): close the merge and send as is
			return err
		}

		if S[0].Name == slice[0].Name {
			w.Write([]byte{','})
			buf.Reset()
			jenc.Encode(S[0].Value)
			w.Write(trimSqBrs(buf.Bytes()))
			S = S[1:]
		}
		for _, f := range S {
			fh := files[f.Name]
			if _, err := fh.Write([]byte{','}); err != nil {
				if Log != nil {
					Log("write", fh.Name(), "error", err)
				}
			}
			buf.Reset()
			jenc.Encode(f.Value)
			fh.Write(trimSqBrs(buf.Bytes()))
		}
	}
	w.Write([]byte("]"))

	for nm, fh := range files {
		if _, err := fh.Seek(0, 0); err != nil {
			Log("Seek", fh.Name(), "error", err)
			continue
		}
		w.Write([]byte{','})
		io.Copy(w, fh)
		w.Write([]byte{']'})
		fh.Close()
		delete(files, nm)
	}
	w.Write([]byte{'}', '\n'})
	return nil
}

type Field struct {
	Name    string
	TagName string
	Value   interface{}
}

func SliceFields(part interface{}, tagName string) (slice, notSlice []Field) {
	rv := reflect.ValueOf(part)
	t := rv.Type()
	if t.Kind() == reflect.Ptr {
		rv = rv.Elem()
		t = rv.Type()
	}
	n := t.NumField()
	for i := 0; i < n; i++ {
		f := rv.Field(i)
		tf := t.Field(i)
		fld := Field{Name: tf.Name, Value: f.Interface(), TagName: tf.Name}
		if tagName != "" {
			if fld.TagName = tf.Tag.Get(tagName); fld.TagName == "" {
				fld.TagName = fld.Name
			} else {
				if i := strings.IndexByte(fld.TagName, ','); i >= 0 {
					fld.TagName = fld.TagName[:i]
				}
				if fld.TagName == "-" { // Skip field
					continue
				}
			}
		}

		if f.Type().Kind() != reflect.Slice {
			notSlice = append(notSlice, fld)
			continue
		}
		if f.IsNil() {
			continue
		}
		slice = append(slice, fld)
	}
	return slice, notSlice
}
func trimSqBrs(b []byte) []byte {
	b = bytes.TrimSpace(b)
	if len(b) == 0 {
		return b
	}
	if b[0] == '[' {
		b = b[1:]
	}
	if len(b) == 0 {
		return b
	}
	if b[len(b)-1] == ']' {
		b = b[:len(b)-1]
	}
	return b
}
<|endoftext|>


<commit_before>package semaphore
<commit_msg>define draft structure<commit_after>package semaphore

import "sync/atomic"

type draft struct {
	state    uint32
	capacity uint32
}

func (semaphore *draft) Release() error {
	panic("implement me")
}

func (semaphore *draft) Acquire(breaker BreakCloser, places ...uint32) (Releaser, error) {
	_ = reduce(places...)
	panic("implement me")
}

func (semaphore *draft) Try(breaker Breaker, places ...uint32) (Releaser, error) {
	_ = reduce(places...)
	panic("implement me")
}

func (semaphore *draft) Signal(breaker Breaker) <-chan Releaser {
	panic("implement me")
}

func (semaphore *draft) Peek() uint32 {
	return atomic.LoadUint32(&semaphore.state)
}

func (semaphore *draft) Size(new uint32) uint32 {
	current := atomic.LoadUint32(&semaphore.capacity)
	if new != 0 {
		atomic.StoreUint32(&semaphore.capacity, new)
	}
	return current
}

func reduce(places ...uint32) uint32 {
	var capacity uint32
	for _, size := range places {
		capacity += size
	}
	if capacity == 0 {
		return 1
	}
	return capacity
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"encoding/base64"
	"errors"
	"log"
	"net/http"
	"strings"
	"time"

	"github.com/bmizerany/lpx"
	"github.com/kr/logfmt"
	metrics "github.com/rcrowley/go-metrics"
)

var (
	TokenPrefix = []byte("t.")
	Heroku      = []byte("heroku")

	// go-metrics Instruments
	wrongMethodErrorCounter    = metrics.NewRegisteredCounter("lumbermill.errors.drain.wrong.method", metrics.DefaultRegistry)
	authFailureCounter         = metrics.NewRegisteredCounter("lumbermill.errors.auth.failure", metrics.DefaultRegistry)
	badRequestCounter          = metrics.NewRegisteredCounter("lumbermill.errors.badrequest", metrics.DefaultRegistry)
	internalServerErrorCounter = metrics.NewRegisteredCounter("lumbermill.errors.internalserver", metrics.DefaultRegistry)
	tokenMissingCounter        = metrics.NewRegisteredCounter("lumbermill.errors.token.missing", metrics.DefaultRegistry)
	timeParsingErrorCounter    = metrics.NewRegisteredCounter("lumbermill.errors.time.parse", metrics.DefaultRegistry)
	logfmtParsingErrorCounter  = metrics.NewRegisteredCounter("lumbermill.errors.logfmt.parse", metrics.DefaultRegistry)
	droppedErrorCounter        = metrics.NewRegisteredCounter("lumbermill.errors.dropped", metrics.DefaultRegistry)
	batchCounter               = metrics.NewRegisteredCounter("lumbermill.batch", metrics.DefaultRegistry)
	linesCounter               = metrics.NewRegisteredCounter("lumbermill.lines", metrics.DefaultRegistry)
	routerErrorLinesCounter    = metrics.NewRegisteredCounter("lumbermill.lines.router.error", metrics.DefaultRegistry)
	routerLinesCounter         = metrics.NewRegisteredCounter("lumbermill.lines.router", metrics.DefaultRegistry)
	dynoErrorLinesCounter      = metrics.NewRegisteredCounter("lumbermill.lines.dyno.error", metrics.DefaultRegistry)
	dynoMemLinesCounter        = metrics.NewRegisteredCounter("lumbermill.lines.dyno.mem", metrics.DefaultRegistry)
	dynoLoadLinesCounter       = metrics.NewRegisteredCounter("lumbermill.lines.dyno.load", metrics.DefaultRegistry)
	unknownHerokuLinesCounter  = metrics.NewRegisteredCounter("lumbermill.lines.unknown.heroku", metrics.DefaultRegistry)
	unknownUserLinesCounter    = metrics.NewRegisteredCounter("lumbermill.lines.unknown.user", metrics.DefaultRegistry)
	parseTimer                 = metrics.NewRegisteredTimer("lumbermill.batches.parse.time", metrics.DefaultRegistry)
	batchSizeHistogram         = metrics.NewRegisteredHistogram("lumbermill.batches.sizes", metrics.DefaultRegistry, metrics.NewUniformSample(100))
)

func checkAuth(r *http.Request) error {
	header := r.Header.Get("Authorization")
	if header == "" {
		return errors.New("Authorization required")
	}
	headerParts := strings.SplitN(header, " ", 2)
	if len(headerParts) != 2 {
		return errors.New("Authorization header is malformed")
	}

	method := headerParts[0]
	if method != "Basic" {
		return errors.New("Only Basic Authorization is accepted")
	}

	encodedUserPass := headerParts[1]
	decodedUserPass, err := base64.StdEncoding.DecodeString(encodedUserPass)
	if err != nil {
		return errors.New("Authorization header is malformed")
	}

	userPassParts := bytes.SplitN(decodedUserPass, []byte{':'}, 2)
	if len(userPassParts) != 2 {
		return errors.New("Authorization header is malformed")
	}

	user := userPassParts[0]
	pass := userPassParts[1]

	if string(user) != User {
		return errors.New("Unknown user")
	}
	if string(pass) != Password {
		return errors.New("Incorrect token")
	}

	return nil
}

// Dyno's are generally reported as "<type>.<#>"
// Extract the <type> and return it
func dynoType(what string) string {
	s := strings.Split(what, ".")
	return s[0]
}

func handleLogFmtParsingError(err error) {
	logfmtParsingErrorCounter.Inc(1)
	log.Printf("logfmt unmarshal error: %s\n", err)
}

// "Parse tree" from hell
func serveDrain(w http.ResponseWriter, r *http.Request) {

	w.Header().Set("Content-Length", "0")

	if r.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		wrongMethodErrorCounter.Inc(1)
		return
	}

	id := r.Header.Get("Logplex-Drain-Token")

	if id == "" {
		if err := checkAuth(r); err != nil {
			w.WriteHeader(http.StatusForbidden)
			authFailureCounter.Inc(1)
			return
		}
	}

	batchCounter.Inc(1)

	parseStart := time.Now()
	lp := lpx.NewReader(bufio.NewReader(r.Body))

	linesCounterInc := 0

	for lp.Next() {
		linesCounterInc += 1
		header := lp.Header()

		// If the syslog Name Header field contains what looks like a log token,
		// let's assume it's an override of the id and we're getting the data from the magic
		// channel
		if bytes.HasPrefix(header.Name, TokenPrefix) {
			id = string(header.Name)
		}

		// If we still don't have an id, throw an error and try the next line
		if id == "" {
			tokenMissingCounter.Inc(1)
			continue
		}

		destination := hashRing.Get(id)

		msg := lp.Bytes()
		switch {
		case bytes.Equal(header.Name, Heroku), bytes.HasPrefix(header.Name, TokenPrefix):
			timeStr := string(lp.Header().Time)
			t, e := time.Parse("2006-01-02T15:04:05.000000+00:00", timeStr)
			if e != nil {
				t, e = time.Parse("2006-01-02T15:04:05+00:00", timeStr)
				if e != nil {
					timeParsingErrorCounter.Inc(1)
					log.Printf("Error Parsing Time(%s): %q\n", string(lp.Header().Time), e)
					continue
				}
			}

			timestamp := t.UnixNano() / int64(time.Microsecond)

			pid := string(header.Procid)
			switch pid {
			case "router":

				switch {
				// router logs with a H error code in them
				case bytes.Contains(msg, keyCodeH):
					routerErrorLinesCounter.Inc(1)
					re := routerError{}
					err := logfmt.Unmarshal(msg, &re)
					if err != nil {
						handleLogFmtParsingError(err)
						continue
					}
					destination.PostPoint(Point{id, EventsRouter, []interface{}{timestamp, re.Code}})

				// likely a standard router log
				default:
					routerLinesCounter.Inc(1)
					rm := routerMsg{}
					err := logfmt.Unmarshal(msg, &rm)
					if err != nil {
						handleLogFmtParsingError(err)
						continue
					}

					destination.PostPoint(Point{id, Router, []interface{}{timestamp, rm.Status, rm.Service}})
				}

				// Non router logs, so either dynos, runtime, etc
			default:
				switch {
				// Dyno error messages
				case bytes.HasPrefix(msg, dynoErrorSentinel):
					dynoErrorLinesCounter.Inc(1)
					de, err := parseBytesToDynoError(msg)
					if err != nil {
						handleLogFmtParsingError(err)
						continue
					}

					what := string(lp.Header().Procid)
					destination.PostPoint(
						Point{id, EventsDyno, []interface{}{timestamp, what, "R", de.Code, string(msg), dynoType(what)}},
					)

				// Dyno log-runtime-metrics memory messages
				case bytes.Contains(msg, dynoMemMsgSentinel):
					dynoMemLinesCounter.Inc(1)
					dm := dynoMemMsg{}
					err := logfmt.Unmarshal(msg, &dm)
					if err != nil {
						handleLogFmtParsingError(err)
						continue
					}
					if dm.Source != "" {
						destination.PostPoint(
							Point{
								id,
								DynoMem,
								[]interface{}{
									timestamp,
									dm.Source,
									dm.MemoryCache,
									dm.MemoryPgpgin,
									dm.MemoryPgpgout,
									dm.MemoryRSS,
									dm.MemorySwap,
									dm.MemoryTotal,
									dynoType(dm.Source),
								},
							},
						)
					}

					// Dyno log-runtime-metrics load messages
				case bytes.Contains(msg, dynoLoadMsgSentinel):
					dynoLoadLinesCounter.Inc(1)
					dm := dynoLoadMsg{}
					err := logfmt.Unmarshal(msg, &dm)
					if err != nil {
						handleLogFmtParsingError(err)
						continue
					}
					if dm.Source != "" {
						destination.PostPoint(
							Point{
								id,
								DynoLoad,
								[]interface{}{timestamp, dm.Source, dm.LoadAvg1Min, dm.LoadAvg5Min, dm.LoadAvg15Min, dynoType(dm.Source)},
							},
						)
					}

				// unknown
				default:
					unknownHerokuLinesCounter.Inc(1)
					if Debug {
						log.Printf("Unknown Heroku Line - Header: PRI: %s, Time: %s, Hostname: %s, Name: %s, ProcId: %s, MsgId: %s - Body: %s",
							header.PrivalVersion,
							header.Time,
							header.Hostname,
							header.Name,
							header.Procid,
							header.Msgid,
							string(msg),
						)
					}
				}
			}

		// non heroku lines
		default:
			unknownUserLinesCounter.Inc(1)
			if Debug {
				log.Printf("Unknown User Line - Header: PRI: %s, Time: %s, Hostname: %s, Name: %s, ProcId: %s, MsgId: %s - Body: %s",
					header.PrivalVersion,
					header.Time,
					header.Hostname,
					header.Name,
					header.Procid,
					header.Msgid,
					string(msg),
				)
			}
		}
	}

	linesCounter.Inc(int64(linesCounterInc))

	batchSizeHistogram.Update(int64(linesCounterInc))

	parseTimer.UpdateSince(parseStart)

	// If we are told to close the connection after the reply, do so.
	select {
	case <-connectionCloser:
		w.Header().Set("Connection", "close")
	default:
		//Nothing
	}

	w.WriteHeader(http.StatusNoContent)
}
<commit_msg>provide more information for logfmt unmarshall error<commit_after>package main

import (
	"bufio"
	"bytes"
	"encoding/base64"
	"errors"
	"log"
	"net/http"
	"strings"
	"time"

	"github.com/bmizerany/lpx"
	"github.com/kr/logfmt"
	metrics "github.com/rcrowley/go-metrics"
)

var (
	TokenPrefix = []byte("t.")
	Heroku      = []byte("heroku")

	// go-metrics Instruments
	wrongMethodErrorCounter    = metrics.NewRegisteredCounter("lumbermill.errors.drain.wrong.method", metrics.DefaultRegistry)
	authFailureCounter         = metrics.NewRegisteredCounter("lumbermill.errors.auth.failure", metrics.DefaultRegistry)
	badRequestCounter          = metrics.NewRegisteredCounter("lumbermill.errors.badrequest", metrics.DefaultRegistry)
	internalServerErrorCounter = metrics.NewRegisteredCounter("lumbermill.errors.internalserver", metrics.DefaultRegistry)
	tokenMissingCounter        = metrics.NewRegisteredCounter("lumbermill.errors.token.missing", metrics.DefaultRegistry)
	timeParsingErrorCounter    = metrics.NewRegisteredCounter("lumbermill.errors.time.parse", metrics.DefaultRegistry)
	logfmtParsingErrorCounter  = metrics.NewRegisteredCounter("lumbermill.errors.logfmt.parse", metrics.DefaultRegistry)
	droppedErrorCounter        = metrics.NewRegisteredCounter("lumbermill.errors.dropped", metrics.DefaultRegistry)
	batchCounter               = metrics.NewRegisteredCounter("lumbermill.batch", metrics.DefaultRegistry)
	linesCounter               = metrics.NewRegisteredCounter("lumbermill.lines", metrics.DefaultRegistry)
	routerErrorLinesCounter    = metrics.NewRegisteredCounter("lumbermill.lines.router.error", metrics.DefaultRegistry)
	routerLinesCounter         = metrics.NewRegisteredCounter("lumbermill.lines.router", metrics.DefaultRegistry)
	dynoErrorLinesCounter      = metrics.NewRegisteredCounter("lumbermill.lines.dyno.error", metrics.DefaultRegistry)
	dynoMemLinesCounter        = metrics.NewRegisteredCounter("lumbermill.lines.dyno.mem", metrics.DefaultRegistry)
	dynoLoadLinesCounter       = metrics.NewRegisteredCounter("lumbermill.lines.dyno.load", metrics.DefaultRegistry)
	unknownHerokuLinesCounter  = metrics.NewRegisteredCounter("lumbermill.lines.unknown.heroku", metrics.DefaultRegistry)
	unknownUserLinesCounter    = metrics.NewRegisteredCounter("lumbermill.lines.unknown.user", metrics.DefaultRegistry)
	parseTimer                 = metrics.NewRegisteredTimer("lumbermill.batches.parse.time", metrics.DefaultRegistry)
	batchSizeHistogram         = metrics.NewRegisteredHistogram("lumbermill.batches.sizes", metrics.DefaultRegistry, metrics.NewUniformSample(100))
)

func checkAuth(r *http.Request) error {
	header := r.Header.Get("Authorization")
	if header == "" {
		return errors.New("Authorization required")
	}
	headerParts := strings.SplitN(header, " ", 2)
	if len(headerParts) != 2 {
		return errors.New("Authorization header is malformed")
	}

	method := headerParts[0]
	if method != "Basic" {
		return errors.New("Only Basic Authorization is accepted")
	}

	encodedUserPass := headerParts[1]
	decodedUserPass, err := base64.StdEncoding.DecodeString(encodedUserPass)
	if err != nil {
		return errors.New("Authorization header is malformed")
	}

	userPassParts := bytes.SplitN(decodedUserPass, []byte{':'}, 2)
	if len(userPassParts) != 2 {
		return errors.New("Authorization header is malformed")
	}

	user := userPassParts[0]
	pass := userPassParts[1]

	if string(user) != User {
		return errors.New("Unknown user")
	}
	if string(pass) != Password {
		return errors.New("Incorrect token")
	}

	return nil
}

// Dyno's are generally reported as "<type>.<#>"
// Extract the <type> and return it
func dynoType(what string) string {
	s := strings.Split(what, ".")
	return s[0]
}

func handleLogFmtParsingError(msg []byte, err error) {
	logfmtParsingErrorCounter.Inc(1)
	log.Printf("logfmt unmarshal error(%q): %q\n", string(msg), err)
}

// "Parse tree" from hell
func serveDrain(w http.ResponseWriter, r *http.Request) {

	w.Header().Set("Content-Length", "0")

	if r.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		wrongMethodErrorCounter.Inc(1)
		return
	}

	id := r.Header.Get("Logplex-Drain-Token")

	if id == "" {
		if err := checkAuth(r); err != nil {
			w.WriteHeader(http.StatusForbidden)
			authFailureCounter.Inc(1)
			return
		}
	}

	batchCounter.Inc(1)

	parseStart := time.Now()
	lp := lpx.NewReader(bufio.NewReader(r.Body))

	linesCounterInc := 0

	for lp.Next() {
		linesCounterInc += 1
		header := lp.Header()

		// If the syslog Name Header field contains what looks like a log token,
		// let's assume it's an override of the id and we're getting the data from the magic
		// channel
		if bytes.HasPrefix(header.Name, TokenPrefix) {
			id = string(header.Name)
		}

		// If we still don't have an id, throw an error and try the next line
		if id == "" {
			tokenMissingCounter.Inc(1)
			continue
		}

		destination := hashRing.Get(id)

		msg := lp.Bytes()
		switch {
		case bytes.Equal(header.Name, Heroku), bytes.HasPrefix(header.Name, TokenPrefix):
			timeStr := string(lp.Header().Time)
			t, e := time.Parse("2006-01-02T15:04:05.000000+00:00", timeStr)
			if e != nil {
				t, e = time.Parse("2006-01-02T15:04:05+00:00", timeStr)
				if e != nil {
					timeParsingErrorCounter.Inc(1)
					log.Printf("Error Parsing Time(%s): %q\n", string(lp.Header().Time), e)
					continue
				}
			}

			timestamp := t.UnixNano() / int64(time.Microsecond)

			pid := string(header.Procid)
			switch pid {
			case "router":

				switch {
				// router logs with a H error code in them
				case bytes.Contains(msg, keyCodeH):
					routerErrorLinesCounter.Inc(1)
					re := routerError{}
					err := logfmt.Unmarshal(msg, &re)
					if err != nil {
						handleLogFmtParsingError(msg, err)
						continue
					}
					destination.PostPoint(Point{id, EventsRouter, []interface{}{timestamp, re.Code}})

				// likely a standard router log
				default:
					routerLinesCounter.Inc(1)
					rm := routerMsg{}
					err := logfmt.Unmarshal(msg, &rm)
					if err != nil {
						handleLogFmtParsingError(msg, err)
						continue
					}

					destination.PostPoint(Point{id, Router, []interface{}{timestamp, rm.Status, rm.Service}})
				}

				// Non router logs, so either dynos, runtime, etc
			default:
				switch {
				// Dyno error messages
				case bytes.HasPrefix(msg, dynoErrorSentinel):
					dynoErrorLinesCounter.Inc(1)
					de, err := parseBytesToDynoError(msg)
					if err != nil {
						handleLogFmtParsingError(msg, err)
						continue
					}

					what := string(lp.Header().Procid)
					destination.PostPoint(
						Point{id, EventsDyno, []interface{}{timestamp, what, "R", de.Code, string(msg), dynoType(what)}},
					)

				// Dyno log-runtime-metrics memory messages
				case bytes.Contains(msg, dynoMemMsgSentinel):
					dynoMemLinesCounter.Inc(1)
					dm := dynoMemMsg{}
					err := logfmt.Unmarshal(msg, &dm)
					if err != nil {
						handleLogFmtParsingError(msg, err)
						continue
					}
					if dm.Source != "" {
						destination.PostPoint(
							Point{
								id,
								DynoMem,
								[]interface{}{
									timestamp,
									dm.Source,
									dm.MemoryCache,
									dm.MemoryPgpgin,
									dm.MemoryPgpgout,
									dm.MemoryRSS,
									dm.MemorySwap,
									dm.MemoryTotal,
									dynoType(dm.Source),
								},
							},
						)
					}

					// Dyno log-runtime-metrics load messages
				case bytes.Contains(msg, dynoLoadMsgSentinel):
					dynoLoadLinesCounter.Inc(1)
					dm := dynoLoadMsg{}
					err := logfmt.Unmarshal(msg, &dm)
					if err != nil {
						handleLogFmtParsingError(msg, err)
						continue
					}
					if dm.Source != "" {
						destination.PostPoint(
							Point{
								id,
								DynoLoad,
								[]interface{}{timestamp, dm.Source, dm.LoadAvg1Min, dm.LoadAvg5Min, dm.LoadAvg15Min, dynoType(dm.Source)},
							},
						)
					}

				// unknown
				default:
					unknownHerokuLinesCounter.Inc(1)
					if Debug {
						log.Printf("Unknown Heroku Line - Header: PRI: %s, Time: %s, Hostname: %s, Name: %s, ProcId: %s, MsgId: %s - Body: %s",
							header.PrivalVersion,
							header.Time,
							header.Hostname,
							header.Name,
							header.Procid,
							header.Msgid,
							string(msg),
						)
					}
				}
			}

		// non heroku lines
		default:
			unknownUserLinesCounter.Inc(1)
			if Debug {
				log.Printf("Unknown User Line - Header: PRI: %s, Time: %s, Hostname: %s, Name: %s, ProcId: %s, MsgId: %s - Body: %s",
					header.PrivalVersion,
					header.Time,
					header.Hostname,
					header.Name,
					header.Procid,
					header.Msgid,
					string(msg),
				)
			}
		}
	}

	linesCounter.Inc(int64(linesCounterInc))

	batchSizeHistogram.Update(int64(linesCounterInc))

	parseTimer.UpdateSince(parseStart)

	// If we are told to close the connection after the reply, do so.
	select {
	case <-connectionCloser:
		w.Header().Set("Connection", "close")
	default:
		//Nothing
	}

	w.WriteHeader(http.StatusNoContent)
}
<|endoftext|>


<commit_before>package immortal

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"sync/atomic"
	"time"
)

// Supervisor for the process
type Supervisor struct {
	daemon  *Daemon
	process *process
	pid     int
	wait    time.Duration
}

// Supervise keep daemon process up and running
func Supervise(d *Daemon) error {
	// start a new process
	p, err := d.Run(NewProcess(d.cfg))
	if err != nil {
		return err
	}
	supervisor := &Supervisor{
		daemon:  d,
		process: p,
	}
	return supervisor.Start()
}

// Start loop forever
func (s *Supervisor) Start() error {
	for {
		select {
		case <-s.daemon.quit:
			s.daemon.wg.Wait()
			return fmt.Errorf("supervisor stopped, count: %d", s.daemon.count)
		case <-s.daemon.run:
			s.ReStart()
		case err := <-s.process.errch:
			// Check for post_exit command
			if len(s.daemon.cfg.PostExit) > 0 {
				var shell = "sh"
				if sh := os.Getenv("SHELL"); sh != "" {
					shell = sh
				}
				if err := exec.Command(shell, "-c", s.daemon.cfg.PostExit).Run(); err != nil {
					log.Printf("post exit command failed: %s", err)
				}
			}
			// stop or exit based on the retries
			if s.Terminate(err) {
				if s.daemon.cfg.cli || os.Getenv("IMMORTAL_EXIT") != "" {
					close(s.daemon.quit)
				} else {
					// stop don't exit
					atomic.StoreUint32(&s.daemon.lock, 1)
				}
			} else {
				// follow the new pid instead of trying to call run again unless the new pid dies
				if s.daemon.cfg.Pid.Follow != "" {
					s.FollowPid(err)
				} else {
					s.ReStart()
				}
			}
		}
	}
}

// ReStart create a new process
func (s *Supervisor) ReStart() {
	var err error
	time.Sleep(s.wait)
	if s.daemon.lock == 0 {
		np := NewProcess(s.daemon.cfg)
		if s.process, err = s.daemon.Run(np); err != nil {
			close(np.quit)
			log.Print(err)
			// loop again but wait 1 seccond before trying
			s.wait = time.Second
			s.daemon.run <- struct{}{}
		}
	}
}

// Terminate handle process termination
func (s *Supervisor) Terminate(err error) bool {
	s.daemon.Lock()
	defer s.daemon.Unlock()

	// set end time
	s.process.eTime = time.Now()
	// unlock, or lock once
	atomic.StoreUint32(&s.daemon.lock, s.daemon.lockOnce)
	// WatchPid returns EXIT
	if err != nil && err.Error() == "EXIT" {
		log.Printf("PID: %d (%s) exited", s.pid, s.process.cmd.Path)
	} else {
		log.Printf("PID %d (%s) terminated, %s [%v user  %v sys  %s up]\n",
			s.process.cmd.ProcessState.Pid(),
			s.process.cmd.Path,
			s.process.cmd.ProcessState,
			s.process.cmd.ProcessState.UserTime(),
			s.process.cmd.ProcessState.SystemTime(),
			time.Since(s.process.sTime),
		)
		// calculate time for next reboot (avoids high CPU usage)
		uptime := s.process.eTime.Sub(s.process.sTime)
		s.wait = 0 * time.Second
		if uptime < time.Second {
			s.wait = time.Second - uptime
		}
	}
	// behavior based on the retries
	if s.daemon.cfg.Retries >= 0 {
		//  0 run only once (don't retry)
		if s.daemon.cfg.Retries == 0 {
			return true
		}
		// +1 run N times
		if s.daemon.count > s.daemon.cfg.Retries {
			return true
		}
	}
	// -1 run forever
	return false
}

// FollowPid check if process still up and running if it is, follow the pid,
// monitor the existing pid created by the process instead of creating
// another process
func (s *Supervisor) FollowPid(err error) {
	s.daemon.Lock()
	defer s.daemon.Unlock()

	s.pid, err = s.daemon.ReadPidFile(s.daemon.cfg.Pid.Follow)
	if err != nil {
		log.Printf("Cannot read pidfile: %s, %s", s.daemon.cfg.Pid.Follow, err)
		s.daemon.run <- struct{}{}
	} else {
		// check if pid in file is valid
		if s.pid > 1 && s.pid != s.process.Pid() && s.daemon.IsRunning(s.pid) {
			log.Printf("Watching pid %d on file: %s", s.pid, s.daemon.cfg.Pid.Follow)
			s.daemon.fpid = true
			// overwrite original (defunct) pid with the fpid in order to be available to send signals
			s.process.cmd.Process.Pid = s.pid
			s.daemon.WatchPid(s.pid, s.process.errch)
		} else {
			// if cmd exits or process is kill
			s.daemon.run <- struct{}{}
		}
	}
}
<commit_msg>return exit code and passe it as an argument to<commit_after>package immortal

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"sync/atomic"
	"time"
)

// Supervisor for the process
type Supervisor struct {
	daemon  *Daemon
	process *process
	pid     int
	wait    time.Duration
}

// Supervise keep daemon process up and running
func Supervise(d *Daemon) error {
	// start a new process
	p, err := d.Run(NewProcess(d.cfg))
	if err != nil {
		return err
	}
	supervisor := &Supervisor{
		daemon:  d,
		process: p,
	}
	return supervisor.Start()
}

// Start loop forever
func (s *Supervisor) Start() error {
	for {
		select {
		case <-s.daemon.quit:
			s.daemon.wg.Wait()
			return fmt.Errorf("supervisor stopped, count: %d", s.daemon.count)
		case <-s.daemon.run:
			s.ReStart()
		case err := <-s.process.errch:
			// get exit code
			// TODO check EXIT from kqueue since we don't know the exit code there
			exitcode := 0
			if exitError, ok := err.(*exec.ExitError); ok {
				exitcode = exitError.ExitCode()
			}
			log.Printf("PID: %d exit code: %d", s.pid, exitcode)
			// Check for post_exit command
			if len(s.daemon.cfg.PostExit) > 0 {
				var shell = "sh"
				if sh := os.Getenv("SHELL"); sh != "" {
					shell = sh
				}
				if err := exec.Command(shell, "-c", s.daemon.cfg.PostExit, fmt.Sprintf("%d", exitcode)).Run(); err != nil {
					log.Printf("post exit command failed: %s", err)
				}
			}
			// stop or exit based on the retries
			if s.Terminate(err) {
				if s.daemon.cfg.cli || os.Getenv("IMMORTAL_EXIT") != "" {
					close(s.daemon.quit)
				} else {
					// stop don't exit
					atomic.StoreUint32(&s.daemon.lock, 1)
				}
			} else {
				// follow the new pid instead of trying to call run again unless the new pid dies
				if s.daemon.cfg.Pid.Follow != "" {
					s.FollowPid(err)
				} else {
					s.ReStart()
				}
			}
		}
	}
}

// ReStart create a new process
func (s *Supervisor) ReStart() {
	var err error
	time.Sleep(s.wait)
	if s.daemon.lock == 0 {
		np := NewProcess(s.daemon.cfg)
		if s.process, err = s.daemon.Run(np); err != nil {
			close(np.quit)
			log.Print(err)
			// loop again but wait 1 seccond before trying
			s.wait = time.Second
			s.daemon.run <- struct{}{}
		}
	}
}

// Terminate handle process termination
func (s *Supervisor) Terminate(err error) bool {
	s.daemon.Lock()
	defer s.daemon.Unlock()

	// set end time
	s.process.eTime = time.Now()
	// unlock, or lock once
	atomic.StoreUint32(&s.daemon.lock, s.daemon.lockOnce)
	// WatchPid returns EXIT
	if err != nil && err.Error() == "EXIT" {
		log.Printf("PID: %d (%s) exited", s.pid, s.process.cmd.Path)
	} else {
		log.Printf("PID %d (%s) terminated, %s [%v user  %v sys  %s up]\n",
			s.process.cmd.ProcessState.Pid(),
			s.process.cmd.Path,
			s.process.cmd.ProcessState,
			s.process.cmd.ProcessState.UserTime(),
			s.process.cmd.ProcessState.SystemTime(),
			time.Since(s.process.sTime),
		)
		// calculate time for next reboot (avoids high CPU usage)
		uptime := s.process.eTime.Sub(s.process.sTime)
		s.wait = 0 * time.Second
		if uptime < time.Second {
			s.wait = time.Second - uptime
		}
	}
	// behavior based on the retries
	if s.daemon.cfg.Retries >= 0 {
		//  0 run only once (don't retry)
		if s.daemon.cfg.Retries == 0 {
			return true
		}
		// +1 run N times
		if s.daemon.count > s.daemon.cfg.Retries {
			return true
		}
	}
	// -1 run forever
	return false
}

// FollowPid check if process still up and running if it is, follow the pid,
// monitor the existing pid created by the process instead of creating
// another process
func (s *Supervisor) FollowPid(err error) {
	s.daemon.Lock()
	defer s.daemon.Unlock()

	s.pid, err = s.daemon.ReadPidFile(s.daemon.cfg.Pid.Follow)
	if err != nil {
		log.Printf("Cannot read pidfile: %s, %s", s.daemon.cfg.Pid.Follow, err)
		s.daemon.run <- struct{}{}
	} else {
		// check if pid in file is valid
		if s.pid > 1 && s.pid != s.process.Pid() && s.daemon.IsRunning(s.pid) {
			log.Printf("Watching pid %d on file: %s", s.pid, s.daemon.cfg.Pid.Follow)
			s.daemon.fpid = true
			// overwrite original (defunct) pid with the fpid in order to be available to send signals
			s.process.cmd.Process.Pid = s.pid
			s.daemon.WatchPid(s.pid, s.process.errch)
		} else {
			// if cmd exits or process is kill
			s.daemon.run <- struct{}{}
		}
	}
}
<|endoftext|>