<commit_before>package main

import (
	"errors"
	"fmt"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/Songmu/gitconfig"
	"github.com/motemen/ghq/logger"
	"github.com/saracen/walker"
)

const envGhqRoot = "GHQ_ROOT"

// LocalRepository represents local repository
type LocalRepository struct {
	FullPath  string
	RelPath   string
	RootPath  string
	PathParts []string

	repoPath   string
	vcsBackend *VCSBackend
}

// RepoPath returns local repository path
func (repo *LocalRepository) RepoPath() string {
	if repo.repoPath != "" {
		return repo.repoPath
	}
	return repo.FullPath
}

// LocalRepositoryFromFullPath resolve LocalRepository from file path
func LocalRepositoryFromFullPath(fullPath string, backend *VCSBackend) (*LocalRepository, error) {
	var relPath string

	roots, err := localRepositoryRoots(true)
	if err != nil {
		return nil, err
	}
	var root string
	for _, root = range roots {
		if !strings.HasPrefix(fullPath, root) {
			continue
		}

		var err error
		relPath, err = filepath.Rel(root, fullPath)
		if err == nil {
			break
		}
	}

	if relPath == "" {
		return nil, fmt.Errorf("no local repository found for: %s", fullPath)
	}

	pathParts := strings.Split(relPath, string(filepath.Separator))

	return &LocalRepository{
		FullPath:   fullPath,
		RelPath:    filepath.ToSlash(relPath),
		RootPath:   root,
		PathParts:  pathParts,
		vcsBackend: backend,
	}, nil
}

// LocalRepositoryFromURL resolve LocalRepository from URL
func LocalRepositoryFromURL(remoteURL *url.URL) (*LocalRepository, error) {
	pathParts := append(
		[]string{remoteURL.Hostname()}, strings.Split(remoteURL.Path, "/")...,
	)
	relPath := strings.TrimSuffix(filepath.Join(pathParts...), ".git")
	pathParts[len(pathParts)-1] = strings.TrimSuffix(pathParts[len(pathParts)-1], ".git")

	var (
		localRepository *LocalRepository
		mu              sync.Mutex
	)
	// Find existing local repository first
	if err := walkAllLocalRepositories(func(repo *LocalRepository) {
		if repo.RelPath == relPath {
			mu.Lock()
			localRepository = repo
			mu.Unlock()
		}
	}); err != nil {
		return nil, err
	}

	if localRepository != nil {
		return localRepository, nil
	}

	prim, err := getRoot(remoteURL.String())
	if err != nil {
		return nil, err
	}

	// No local repository found, returning new one
	return &LocalRepository{
		FullPath:  filepath.Join(prim, relPath),
		RelPath:   relPath,
		RootPath:  prim,
		PathParts: pathParts,
	}, nil
}

func getRoot(u string) (string, error) {
	prim, err := gitconfig.Do("--path", "--get-urlmatch", "ghq.root", u)
	if err != nil && !gitconfig.IsNotFound(err) {
		return "", err
	}
	if prim == "" {
		prim, err = primaryLocalRepositoryRoot()
		if err != nil {
			return "", err
		}
	}
	return prim, nil
}

// Subpaths returns lists of tail parts of relative path from the root directory (shortest first)
// for example, {"ghq", "motemen/ghq", "github.com/motemen/ghq"} for $root/github.com/motemen/ghq.
func (repo *LocalRepository) Subpaths() []string {
	tails := make([]string, len(repo.PathParts))

	for i := range repo.PathParts {
		tails[i] = strings.Join(repo.PathParts[len(repo.PathParts)-(i+1):], "/")
	}

	return tails
}

// NonHostPath returns non host path
func (repo *LocalRepository) NonHostPath() string {
	return strings.Join(repo.PathParts[1:], "/")
}

// list as bellow
// - "$GHQ_ROOT/github.com/motemen/ghq/cmdutil" // repo.FullPath
// - "$GHQ_ROOT/github.com/motemen/ghq"
// - "$GHQ_ROOT/github.com/motemen
func (repo *LocalRepository) repoRootCandidates() []string {
	hostRoot := filepath.Join(repo.RootPath, repo.PathParts[0])
	nonHostParts := repo.PathParts[1:]
	candidates := make([]string, len(nonHostParts))
	for i := 0; i < len(nonHostParts); i++ {
		candidates[i] = filepath.Join(append(
			[]string{hostRoot}, nonHostParts[0:len(nonHostParts)-i]...)...)
	}
	return candidates
}

// IsUnderPrimaryRoot or not
func (repo *LocalRepository) IsUnderPrimaryRoot() bool {
	prim, err := primaryLocalRepositoryRoot()
	if err != nil {
		return false
	}
	return strings.HasPrefix(repo.FullPath, prim)
}

// Matches checks if any subpath of the local repository equals the query.
func (repo *LocalRepository) Matches(pathQuery string) bool {
	for _, p := range repo.Subpaths() {
		if p == pathQuery {
			return true
		}
	}

	return false
}

// VCS returns VCSBackend of the repository
func (repo *LocalRepository) VCS() (*VCSBackend, string) {
	if repo.vcsBackend == nil {
		for _, dir := range repo.repoRootCandidates() {
			backend := findVCSBackend(dir, "")
			if backend != nil {
				repo.vcsBackend = backend
				repo.repoPath = dir
				break
			}
		}
	}
	return repo.vcsBackend, repo.RepoPath()
}

var vcsContentsMap = map[string]*VCSBackend{
	".git":           GitBackend,
	".hg":            MercurialBackend,
	".svn":           SubversionBackend,
	"_darcs":         DarcsBackend,
	".bzr":           BazaarBackend,
	".fslckout":      FossilBackend, // file
	"_FOSSIL_":       FossilBackend, // file
	"CVS/Repository": cvsDummyBackend,
}

var vcsContents = [...]string{
	".git",
	".hg",
	".svn",
	"_darcs",
	".bzr",
	".fslckout",
	"._FOSSIL_",
	"CVS/Repository",
}

func findVCSBackend(fpath, vcs string) *VCSBackend {
	// When vcs is not empty, search only specified contents of vcs
	if vcs != "" {
		vcsBackend, ok := vcsRegistry[vcs]
		if !ok {
			return nil
		}
		for _, d := range vcsBackend.Contents {
			if _, err := os.Stat(filepath.Join(fpath, d)); err == nil {
				return vcsBackend
			}
		}
		return nil
	}
	for _, d := range vcsContents {
		if _, err := os.Stat(filepath.Join(fpath, d)); err == nil {
			return vcsContentsMap[d]
		}
	}
	return nil
}

func walkAllLocalRepositories(callback func(*LocalRepository)) error {
	return walkLocalRepositories("", callback)
}

func walkLocalRepositories(vcs string, callback func(*LocalRepository)) error {
	roots, err := localRepositoryRoots(true)
	if err != nil {
		return err
	}

	walkFn := func(fpath string, fi os.FileInfo) error {
		isSymlink := false
		if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
			isSymlink = true
			realpath, err := filepath.EvalSymlinks(fpath)
			if err != nil {
				return nil
			}
			fi, err = os.Stat(realpath)
			if err != nil {
				return nil
			}
		}
		if !fi.IsDir() {
			return nil
		}
		vcsBackend := findVCSBackend(fpath, vcs)
		if vcsBackend == nil {
			return nil
		}

		repo, err := LocalRepositoryFromFullPath(fpath, vcsBackend)
		if err != nil || repo == nil {
			return nil
		}
		callback(repo)

		if isSymlink {
			return nil
		}
		return filepath.SkipDir
	}

	errCb := walker.WithErrorCallback(func(pathname string, err error) error {
		if os.IsPermission(errors.Unwrap(err)) {
			logger.Log("warning", fmt.Sprintf("%s: Permission denied", pathname))
			return nil
		}
		return err
	})

	for _, root := range roots {
		fi, err := os.Stat(root)
		if err != nil {
			if os.IsNotExist(err) {
				continue
			}
		}
		if fi.Mode()&0444 == 0 {
			logger.Log("warning", fmt.Sprintf("%s: Permission denied", root))
			continue
		}
		if err := walker.Walk(root, walkFn, errCb); err != nil {
			return err
		}
	}
	return nil
}

var _home string

func getHome() (string, error) {
	if _home != "" {
		return _home, nil
	}
	var err error
	_home, err = os.UserHomeDir()
	return _home, err
}

var _localRepositoryRoots []string

// localRepositoryRoots returns locally cloned repositories' root directories.
// The root dirs are determined as following:
//
//   - If GHQ_ROOT environment variable is nonempty, use it as the only root dir.
//   - Otherwise, use the result of `git config --get-all ghq.root` as the dirs.
//   - Otherwise, fallback to the default root, `~/.ghq`.
func localRepositoryRoots(all bool) ([]string, error) {
	if len(_localRepositoryRoots) != 0 {
		return _localRepositoryRoots, nil
	}

	envRoot := os.Getenv(envGhqRoot)
	if envRoot != "" {
		_localRepositoryRoots = filepath.SplitList(envRoot)
	} else {
		var err error
		_localRepositoryRoots, err = gitconfig.PathAll("ghq.root")
		if err != nil && !gitconfig.IsNotFound(err) {
			return nil, err
		}
	}

	if len(_localRepositoryRoots) == 0 {
		homeDir, err := getHome()
		if err != nil {
			return nil, err
		}
		_localRepositoryRoots = []string{filepath.Join(homeDir, ".ghq")}
	}

	if all && envRoot == "" {
		roots, err := urlMatchLocalRepositoryRoots()
		if err != nil {
			return nil, err
		}
		_localRepositoryRoots = append(_localRepositoryRoots, roots...)
	}

	for i, v := range _localRepositoryRoots {
		path := filepath.Clean(v)
		if _, err := os.Stat(path); err == nil {
			if path, err = filepath.EvalSymlinks(path); err != nil {
				return nil, err
			}
		}
		if !filepath.IsAbs(path) {
			var err error
			if path, err = filepath.Abs(path); err != nil {
				return nil, err
			}
		}
		_localRepositoryRoots[i] = path
	}

	return _localRepositoryRoots, nil
}

func urlMatchLocalRepositoryRoots() ([]string, error) {
	out, err := gitconfig.Do("--path", "--get-regexp", `^ghq\..+\.root$`)
	if err != nil {
		if gitconfig.IsNotFound(err) {
			return nil, nil
		}
		return nil, err
	}
	items := strings.Split(out, "\x00")
	ret := make([]string, len(items))
	for i, kvStr := range items {
		kv := strings.SplitN(kvStr, "\n", 2)
		ret[i] = kv[1]
	}
	return ret, nil
}

func primaryLocalRepositoryRoot() (string, error) {
	roots, err := localRepositoryRoots(false)
	if err != nil {
		return "", err
	}
	return roots[0], nil
}
<commit_msg>GHQ_ROOT env is top priority to getRoot<commit_after>package main

import (
	"errors"
	"fmt"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/Songmu/gitconfig"
	"github.com/motemen/ghq/logger"
	"github.com/saracen/walker"
)

const envGhqRoot = "GHQ_ROOT"

// LocalRepository represents local repository
type LocalRepository struct {
	FullPath  string
	RelPath   string
	RootPath  string
	PathParts []string

	repoPath   string
	vcsBackend *VCSBackend
}

// RepoPath returns local repository path
func (repo *LocalRepository) RepoPath() string {
	if repo.repoPath != "" {
		return repo.repoPath
	}
	return repo.FullPath
}

// LocalRepositoryFromFullPath resolve LocalRepository from file path
func LocalRepositoryFromFullPath(fullPath string, backend *VCSBackend) (*LocalRepository, error) {
	var relPath string

	roots, err := localRepositoryRoots(true)
	if err != nil {
		return nil, err
	}
	var root string
	for _, root = range roots {
		if !strings.HasPrefix(fullPath, root) {
			continue
		}

		var err error
		relPath, err = filepath.Rel(root, fullPath)
		if err == nil {
			break
		}
	}

	if relPath == "" {
		return nil, fmt.Errorf("no local repository found for: %s", fullPath)
	}

	pathParts := strings.Split(relPath, string(filepath.Separator))

	return &LocalRepository{
		FullPath:   fullPath,
		RelPath:    filepath.ToSlash(relPath),
		RootPath:   root,
		PathParts:  pathParts,
		vcsBackend: backend,
	}, nil
}

// LocalRepositoryFromURL resolve LocalRepository from URL
func LocalRepositoryFromURL(remoteURL *url.URL) (*LocalRepository, error) {
	pathParts := append(
		[]string{remoteURL.Hostname()}, strings.Split(remoteURL.Path, "/")...,
	)
	relPath := strings.TrimSuffix(filepath.Join(pathParts...), ".git")
	pathParts[len(pathParts)-1] = strings.TrimSuffix(pathParts[len(pathParts)-1], ".git")

	var (
		localRepository *LocalRepository
		mu              sync.Mutex
	)
	// Find existing local repository first
	if err := walkAllLocalRepositories(func(repo *LocalRepository) {
		if repo.RelPath == relPath {
			mu.Lock()
			localRepository = repo
			mu.Unlock()
		}
	}); err != nil {
		return nil, err
	}

	if localRepository != nil {
		return localRepository, nil
	}

	prim, err := getRoot(remoteURL.String())
	if err != nil {
		return nil, err
	}

	// No local repository found, returning new one
	return &LocalRepository{
		FullPath:  filepath.Join(prim, relPath),
		RelPath:   relPath,
		RootPath:  prim,
		PathParts: pathParts,
	}, nil
}

func getRoot(u string) (string, error) {
	prim := os.Getenv(envGhqRoot)
	if prim != "" {
		return prim, nil
	}
	prim, err := gitconfig.Do("--path", "--get-urlmatch", "ghq.root", u)
	if err != nil && !gitconfig.IsNotFound(err) {
		return "", err
	}
	if prim == "" {
		prim, err = primaryLocalRepositoryRoot()
		if err != nil {
			return "", err
		}
	}
	return prim, nil
}

// Subpaths returns lists of tail parts of relative path from the root directory (shortest first)
// for example, {"ghq", "motemen/ghq", "github.com/motemen/ghq"} for $root/github.com/motemen/ghq.
func (repo *LocalRepository) Subpaths() []string {
	tails := make([]string, len(repo.PathParts))

	for i := range repo.PathParts {
		tails[i] = strings.Join(repo.PathParts[len(repo.PathParts)-(i+1):], "/")
	}

	return tails
}

// NonHostPath returns non host path
func (repo *LocalRepository) NonHostPath() string {
	return strings.Join(repo.PathParts[1:], "/")
}

// list as bellow
// - "$GHQ_ROOT/github.com/motemen/ghq/cmdutil" // repo.FullPath
// - "$GHQ_ROOT/github.com/motemen/ghq"
// - "$GHQ_ROOT/github.com/motemen
func (repo *LocalRepository) repoRootCandidates() []string {
	hostRoot := filepath.Join(repo.RootPath, repo.PathParts[0])
	nonHostParts := repo.PathParts[1:]
	candidates := make([]string, len(nonHostParts))
	for i := 0; i < len(nonHostParts); i++ {
		candidates[i] = filepath.Join(append(
			[]string{hostRoot}, nonHostParts[0:len(nonHostParts)-i]...)...)
	}
	return candidates
}

// IsUnderPrimaryRoot or not
func (repo *LocalRepository) IsUnderPrimaryRoot() bool {
	prim, err := primaryLocalRepositoryRoot()
	if err != nil {
		return false
	}
	return strings.HasPrefix(repo.FullPath, prim)
}

// Matches checks if any subpath of the local repository equals the query.
func (repo *LocalRepository) Matches(pathQuery string) bool {
	for _, p := range repo.Subpaths() {
		if p == pathQuery {
			return true
		}
	}

	return false
}

// VCS returns VCSBackend of the repository
func (repo *LocalRepository) VCS() (*VCSBackend, string) {
	if repo.vcsBackend == nil {
		for _, dir := range repo.repoRootCandidates() {
			backend := findVCSBackend(dir, "")
			if backend != nil {
				repo.vcsBackend = backend
				repo.repoPath = dir
				break
			}
		}
	}
	return repo.vcsBackend, repo.RepoPath()
}

var vcsContentsMap = map[string]*VCSBackend{
	".git":           GitBackend,
	".hg":            MercurialBackend,
	".svn":           SubversionBackend,
	"_darcs":         DarcsBackend,
	".bzr":           BazaarBackend,
	".fslckout":      FossilBackend, // file
	"_FOSSIL_":       FossilBackend, // file
	"CVS/Repository": cvsDummyBackend,
}

var vcsContents = [...]string{
	".git",
	".hg",
	".svn",
	"_darcs",
	".bzr",
	".fslckout",
	"._FOSSIL_",
	"CVS/Repository",
}

func findVCSBackend(fpath, vcs string) *VCSBackend {
	// When vcs is not empty, search only specified contents of vcs
	if vcs != "" {
		vcsBackend, ok := vcsRegistry[vcs]
		if !ok {
			return nil
		}
		for _, d := range vcsBackend.Contents {
			if _, err := os.Stat(filepath.Join(fpath, d)); err == nil {
				return vcsBackend
			}
		}
		return nil
	}
	for _, d := range vcsContents {
		if _, err := os.Stat(filepath.Join(fpath, d)); err == nil {
			return vcsContentsMap[d]
		}
	}
	return nil
}

func walkAllLocalRepositories(callback func(*LocalRepository)) error {
	return walkLocalRepositories("", callback)
}

func walkLocalRepositories(vcs string, callback func(*LocalRepository)) error {
	roots, err := localRepositoryRoots(true)
	if err != nil {
		return err
	}

	walkFn := func(fpath string, fi os.FileInfo) error {
		isSymlink := false
		if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
			isSymlink = true
			realpath, err := filepath.EvalSymlinks(fpath)
			if err != nil {
				return nil
			}
			fi, err = os.Stat(realpath)
			if err != nil {
				return nil
			}
		}
		if !fi.IsDir() {
			return nil
		}
		vcsBackend := findVCSBackend(fpath, vcs)
		if vcsBackend == nil {
			return nil
		}

		repo, err := LocalRepositoryFromFullPath(fpath, vcsBackend)
		if err != nil || repo == nil {
			return nil
		}
		callback(repo)

		if isSymlink {
			return nil
		}
		return filepath.SkipDir
	}

	errCb := walker.WithErrorCallback(func(pathname string, err error) error {
		if os.IsPermission(errors.Unwrap(err)) {
			logger.Log("warning", fmt.Sprintf("%s: Permission denied", pathname))
			return nil
		}
		return err
	})

	for _, root := range roots {
		fi, err := os.Stat(root)
		if err != nil {
			if os.IsNotExist(err) {
				continue
			}
		}
		if fi.Mode()&0444 == 0 {
			logger.Log("warning", fmt.Sprintf("%s: Permission denied", root))
			continue
		}
		if err := walker.Walk(root, walkFn, errCb); err != nil {
			return err
		}
	}
	return nil
}

var _home string

func getHome() (string, error) {
	if _home != "" {
		return _home, nil
	}
	var err error
	_home, err = os.UserHomeDir()
	return _home, err
}

var _localRepositoryRoots []string

// localRepositoryRoots returns locally cloned repositories' root directories.
// The root dirs are determined as following:
//
//   - If GHQ_ROOT environment variable is nonempty, use it as the only root dir.
//   - Otherwise, use the result of `git config --get-all ghq.root` as the dirs.
//   - Otherwise, fallback to the default root, `~/.ghq`.
func localRepositoryRoots(all bool) ([]string, error) {
	if len(_localRepositoryRoots) != 0 {
		return _localRepositoryRoots, nil
	}

	envRoot := os.Getenv(envGhqRoot)
	if envRoot != "" {
		_localRepositoryRoots = filepath.SplitList(envRoot)
	} else {
		var err error
		_localRepositoryRoots, err = gitconfig.PathAll("ghq.root")
		if err != nil && !gitconfig.IsNotFound(err) {
			return nil, err
		}
	}

	if len(_localRepositoryRoots) == 0 {
		homeDir, err := getHome()
		if err != nil {
			return nil, err
		}
		_localRepositoryRoots = []string{filepath.Join(homeDir, ".ghq")}
	}

	if all && envRoot == "" {
		roots, err := urlMatchLocalRepositoryRoots()
		if err != nil {
			return nil, err
		}
		_localRepositoryRoots = append(_localRepositoryRoots, roots...)
	}

	for i, v := range _localRepositoryRoots {
		path := filepath.Clean(v)
		if _, err := os.Stat(path); err == nil {
			if path, err = filepath.EvalSymlinks(path); err != nil {
				return nil, err
			}
		}
		if !filepath.IsAbs(path) {
			var err error
			if path, err = filepath.Abs(path); err != nil {
				return nil, err
			}
		}
		_localRepositoryRoots[i] = path
	}

	return _localRepositoryRoots, nil
}

func urlMatchLocalRepositoryRoots() ([]string, error) {
	out, err := gitconfig.Do("--path", "--get-regexp", `^ghq\..+\.root$`)
	if err != nil {
		if gitconfig.IsNotFound(err) {
			return nil, nil
		}
		return nil, err
	}
	items := strings.Split(out, "\x00")
	ret := make([]string, len(items))
	for i, kvStr := range items {
		kv := strings.SplitN(kvStr, "\n", 2)
		ret[i] = kv[1]
	}
	return ret, nil
}

func primaryLocalRepositoryRoot() (string, error) {
	roots, err := localRepositoryRoots(false)
	if err != nil {
		return "", err
	}
	return roots[0], nil
}
<|endoftext|>


<commit_before>package webHandlers

import (
	"context"
	"net/http"
	"strconv"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/gin-gonic/gin"
	"github.com/molecul/qa_portal/checker"
	"github.com/molecul/qa_portal/model"
	"github.com/molecul/qa_portal/web/middleware"
	"github.com/zalando/gin-oauth2/google"
)

func doError(ctx *gin.Context, err error) {
	ctx.JSON(http.StatusInternalServerError, gin.H{"Internal error": err.Error()})
	panic(err)
}

func UserLoginHandler(ctx *gin.Context) {
	gu := ctx.MustGet("user").(google.User)
	usr, err := model.GetUserByEmail(gu.Email)
	if err != nil {
		doError(ctx, err)
		return
	}
	if usr == nil {
		if err = model.CreateUser(new(model.User).FillFromGoogle(&gu)); err != nil {
			doError(ctx, err)
			return
		}
		// TODO ONLY FOR DEBUG. NEED REMOVE
		usr, err = model.GetUserByEmail(gu.Email)
		if err != nil {
			doError(ctx, err)
			return
		}
		if usr == nil {
			doError(ctx, nil)
		}
		// ================================
	} else {
		if err = usr.FillFromGoogle(&gu).Update(); err != nil {
			doError(ctx, err)
			return
		}
	}
	middleware.UserSessionSet(ctx, usr.ID)
	ctx.JSON(http.StatusOK, gin.H{"Hello": "from private", "user": gu, "internal_user": usr})
}

func MainPageHandler(c *gin.Context) {
	c.HTML(http.StatusOK, "index.html", gin.H{
		"title": "Main website",
	})
}

func DockerHealthCheckHandler(c *gin.Context) {
	temp_id, _ := strconv.Atoi(time.Now().Format("20060102150405"))
	task := checker.Get().NewTask(&model.Challenge{
		ID: 	    int64(temp_id),
		Image:      "python:2.7",
		TargetPath: "/tmp/task.py",
		Cmd:        "echo \"Inside $CHECKER_FILE:\"; cat $CHECKER_FILE",
		InternalName: "test",
	}, &model.Test{
		ID:          int64(temp_id),
		ChallengeID: int64(temp_id),
		InputFile:   "healthcheck_"+strconv.FormatInt(int64(temp_id), 10),
	})

	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
	if err := task.Do(ctx); err != nil {
		c.JSON(http.StatusBadGateway, err)
		logrus.Print(err)
	} else {

		c.JSON(http.StatusOK,
			gin.H{
				"exitcode": task.Result.ExitCode,
				"stdout": task.Result.Stdout,
				"stderr": task.Result.Stderr})
	}
	cancel()
}<commit_msg>Change return type of stdout/stderr in DockerHealthCheckHandler<commit_after>package webHandlers

import (
	"context"
	"net/http"
	"strconv"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/gin-gonic/gin"
	"github.com/molecul/qa_portal/checker"
	"github.com/molecul/qa_portal/model"
	"github.com/molecul/qa_portal/web/middleware"
	"github.com/zalando/gin-oauth2/google"
)

func doError(ctx *gin.Context, err error) {
	ctx.JSON(http.StatusInternalServerError, gin.H{"Internal error": err.Error()})
	panic(err)
}

func UserLoginHandler(ctx *gin.Context) {
	gu := ctx.MustGet("user").(google.User)
	usr, err := model.GetUserByEmail(gu.Email)
	if err != nil {
		doError(ctx, err)
		return
	}
	if usr == nil {
		if err = model.CreateUser(new(model.User).FillFromGoogle(&gu)); err != nil {
			doError(ctx, err)
			return
		}
		// TODO ONLY FOR DEBUG. NEED REMOVE
		usr, err = model.GetUserByEmail(gu.Email)
		if err != nil {
			doError(ctx, err)
			return
		}
		if usr == nil {
			doError(ctx, nil)
		}
		// ================================
	} else {
		if err = usr.FillFromGoogle(&gu).Update(); err != nil {
			doError(ctx, err)
			return
		}
	}
	middleware.UserSessionSet(ctx, usr.ID)
	ctx.JSON(http.StatusOK, gin.H{"Hello": "from private", "user": gu, "internal_user": usr})
}

func MainPageHandler(c *gin.Context) {
	c.HTML(http.StatusOK, "index.html", gin.H{
		"title": "Main website",
	})
}

func DockerHealthCheckHandler(c *gin.Context) {
	temp_id, _ := strconv.Atoi(time.Now().Format("20060102150405"))
	task := checker.Get().NewTask(&model.Challenge{
		ID: 	    int64(temp_id),
		Image:      "python:2.7",
		TargetPath: "/tmp/task.py",
		Cmd:        "echo \"Inside $CHECKER_FILE:\"; cat $CHECKER_FILE",
		InternalName: "test",
	}, &model.Test{
		ID:          int64(temp_id),
		ChallengeID: int64(temp_id),
		InputFile:   "healthcheck_"+strconv.FormatInt(int64(temp_id), 10),
	})

	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
	if err := task.Do(ctx); err != nil {
		c.JSON(http.StatusBadGateway, err)
		logrus.Print(err)
	} else {

		c.JSON(http.StatusOK,
			gin.H{
				"exitcode": task.Result.ExitCode,
				"stdout": task.Result.Stdout.String(),
				"stderr": task.Result.Stderr.String()})
	}
	cancel()
}<|endoftext|>


<commit_before>package feature

import (
	"fmt"

	"github.com/lightningnetwork/lnd/lnwire"
)

// Config houses any runtime modifications to the default set descriptors. For
// our purposes, this typically means disabling certain features to test legacy
// protocol interoperability or functionality.
type Config struct {
	// NoTLVOnion unsets any optional or required TLVOnionPaylod bits from
	// all feature sets.
	NoTLVOnion bool

	// NoStaticRemoteKey unsets any optional or required StaticRemoteKey
	// bits from all feature sets.
	NoStaticRemoteKey bool
}

// Manager is responsible for generating feature vectors for different requested
// feature sets.
type Manager struct {
	// fsets is a static map of feature set to raw feature vectors. Requests
	// are fulfilled by cloning these interal feature vectors.
	fsets map[Set]*lnwire.RawFeatureVector
}

// NewManager creates a new feature Manager, applying any custom modifications
// to its feature sets before returning.
func NewManager(cfg Config) (*Manager, error) {
	return newManager(cfg, defaultSetDesc)
}

// newManager creates a new feeature Manager, applying any custom modifications
// to its feature sets before returning. This method accepts the setDesc as its
// own parameter so that it can be unit tested.
func newManager(cfg Config, desc setDesc) (*Manager, error) {
	// First build the default feature vector for all known sets.
	fsets := make(map[Set]*lnwire.RawFeatureVector)
	for bit, sets := range desc {
		for set := range sets {
			// Fetch the feature vector for this set, allocating a
			// new one if it doesn't exist.
			fv, ok := fsets[set]
			if !ok {
				fv = lnwire.NewRawFeatureVector()
			}

			// Set the configured bit on the feature vector,
			// ensuring that we don't set two feature bits for the
			// same pair.
			err := fv.SafeSet(bit)
			if err != nil {
				return nil, fmt.Errorf("unable to set "+
					"%v in %v: %v", bit, set, err)
			}

			// Write the updated feature vector under its set.
			fsets[set] = fv
		}
	}

	// Now, remove any features as directed by the config.
	for set, raw := range fsets {
		if cfg.NoTLVOnion {
			raw.Unset(lnwire.TLVOnionPayloadOptional)
			raw.Unset(lnwire.TLVOnionPayloadRequired)
			raw.Unset(lnwire.PaymentAddrOptional)
			raw.Unset(lnwire.PaymentAddrRequired)
			raw.Unset(lnwire.MPPOptional)
			raw.Unset(lnwire.MPPRequired)
		}
		if cfg.NoStaticRemoteKey {
			raw.Unset(lnwire.StaticRemoteKeyOptional)
			raw.Unset(lnwire.StaticRemoteKeyRequired)
		}

		// Ensure that all of our feature sets properly set any
		// dependent features.
		fv := lnwire.NewFeatureVector(raw, lnwire.Features)
		err := ValidateDeps(fv)
		if err != nil {
			return nil, fmt.Errorf("invalid feature set %v: %v",
				set, err)
		}
	}

	return &Manager{
		fsets: fsets,
	}, nil
}

// GetRaw returns a raw feature vector for the passed set. If no set is known,
// an empty raw feature vector is returned.
func (m *Manager) GetRaw(set Set) *lnwire.RawFeatureVector {
	if fv, ok := m.fsets[set]; ok {
		return fv.Clone()
	}

	return lnwire.NewRawFeatureVector()
}

// Get returns a feature vector for the passed set. If no set is known, an empty
// feature vector is returned.
func (m *Manager) Get(set Set) *lnwire.FeatureVector {
	raw := m.GetRaw(set)
	return lnwire.NewFeatureVector(raw, lnwire.Features)
}
<commit_msg>feature: add ListSets function<commit_after>package feature

import (
	"fmt"

	"github.com/lightningnetwork/lnd/lnwire"
)

// Config houses any runtime modifications to the default set descriptors. For
// our purposes, this typically means disabling certain features to test legacy
// protocol interoperability or functionality.
type Config struct {
	// NoTLVOnion unsets any optional or required TLVOnionPaylod bits from
	// all feature sets.
	NoTLVOnion bool

	// NoStaticRemoteKey unsets any optional or required StaticRemoteKey
	// bits from all feature sets.
	NoStaticRemoteKey bool
}

// Manager is responsible for generating feature vectors for different requested
// feature sets.
type Manager struct {
	// fsets is a static map of feature set to raw feature vectors. Requests
	// are fulfilled by cloning these interal feature vectors.
	fsets map[Set]*lnwire.RawFeatureVector
}

// NewManager creates a new feature Manager, applying any custom modifications
// to its feature sets before returning.
func NewManager(cfg Config) (*Manager, error) {
	return newManager(cfg, defaultSetDesc)
}

// newManager creates a new feeature Manager, applying any custom modifications
// to its feature sets before returning. This method accepts the setDesc as its
// own parameter so that it can be unit tested.
func newManager(cfg Config, desc setDesc) (*Manager, error) {
	// First build the default feature vector for all known sets.
	fsets := make(map[Set]*lnwire.RawFeatureVector)
	for bit, sets := range desc {
		for set := range sets {
			// Fetch the feature vector for this set, allocating a
			// new one if it doesn't exist.
			fv, ok := fsets[set]
			if !ok {
				fv = lnwire.NewRawFeatureVector()
			}

			// Set the configured bit on the feature vector,
			// ensuring that we don't set two feature bits for the
			// same pair.
			err := fv.SafeSet(bit)
			if err != nil {
				return nil, fmt.Errorf("unable to set "+
					"%v in %v: %v", bit, set, err)
			}

			// Write the updated feature vector under its set.
			fsets[set] = fv
		}
	}

	// Now, remove any features as directed by the config.
	for set, raw := range fsets {
		if cfg.NoTLVOnion {
			raw.Unset(lnwire.TLVOnionPayloadOptional)
			raw.Unset(lnwire.TLVOnionPayloadRequired)
			raw.Unset(lnwire.PaymentAddrOptional)
			raw.Unset(lnwire.PaymentAddrRequired)
			raw.Unset(lnwire.MPPOptional)
			raw.Unset(lnwire.MPPRequired)
		}
		if cfg.NoStaticRemoteKey {
			raw.Unset(lnwire.StaticRemoteKeyOptional)
			raw.Unset(lnwire.StaticRemoteKeyRequired)
		}

		// Ensure that all of our feature sets properly set any
		// dependent features.
		fv := lnwire.NewFeatureVector(raw, lnwire.Features)
		err := ValidateDeps(fv)
		if err != nil {
			return nil, fmt.Errorf("invalid feature set %v: %v",
				set, err)
		}
	}

	return &Manager{
		fsets: fsets,
	}, nil
}

// GetRaw returns a raw feature vector for the passed set. If no set is known,
// an empty raw feature vector is returned.
func (m *Manager) GetRaw(set Set) *lnwire.RawFeatureVector {
	if fv, ok := m.fsets[set]; ok {
		return fv.Clone()
	}

	return lnwire.NewRawFeatureVector()
}

// Get returns a feature vector for the passed set. If no set is known, an empty
// feature vector is returned.
func (m *Manager) Get(set Set) *lnwire.FeatureVector {
	raw := m.GetRaw(set)
	return lnwire.NewFeatureVector(raw, lnwire.Features)
}

// ListSets returns a list of the feature sets that our node supports.
func (m *Manager) ListSets() []Set {
	var sets []Set

	for set := range m.fsets {
		sets = append(sets, set)
	}

	return sets
}
<|endoftext|>


<commit_before>package middleware

import (
	"crypto/rand"
	"encoding/base64"
	"fmt"
	"net/http"
	"os"
	"strings"
	"sync/atomic"

	"github.com/zenazn/goji/web"
)

// Key to use when setting the request ID.
const RequestIDKey = "reqID"

var prefix string
var reqid uint64

func init() {
	hostname, err := os.Hostname()
	if hostname == "" || err != nil {
		hostname = "localhost"
	}
	var buf [12]byte
	rand.Read(buf[:])
	b64 := base64.StdEncoding.EncodeToString(buf[:])
	// Strip out annoying characters. We have something like a billion to
	// one chance of having enough from 12 bytes of entropy
	b64 = strings.NewReplacer("+", "", "/", "").Replace(b64)

	prefix = fmt.Sprintf("%s/%s", hostname, b64[0:8])
}

// RequestID is a middleware that injects a request ID into the context of each
// request. A request ID is a string of the form "host.example.com/random-0001",
// where "random" is a base62 random string that uniquely identifies this go
// process, and where the last number is an atomically incremented request
// counter.
func RequestID(c *web.C, h http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {
		if c.Env == nil {
			c.Env = make(map[string]interface{})
		}
		myid := atomic.AddUint64(&reqid, 1)
		c.Env[RequestIDKey] = fmt.Sprintf("%s-%06d", prefix, myid)

		h.ServeHTTP(w, r)
	}

	return http.HandlerFunc(fn)
}

// GetReqID returns a request ID from the given context if one is present.
// Returns the empty string if a request ID cannot be found.
func GetReqID(c web.C) string {
	if c.Env == nil {
		return ""
	}
	v, ok := c.Env[RequestIDKey]
	if !ok {
		return ""
	}
	if reqID, ok := v.(string); ok {
		return reqID
	}
	return ""
}
<commit_msg>Increase size of per-process request ID nonce<commit_after>package middleware

import (
	"crypto/rand"
	"encoding/base64"
	"fmt"
	"net/http"
	"os"
	"strings"
	"sync/atomic"

	"github.com/zenazn/goji/web"
)

// Key to use when setting the request ID.
const RequestIDKey = "reqID"

var prefix string
var reqid uint64

/*
A quick note on the statistics here: we're trying to calculate the chance that
two randomly generated base62 prefixes will collide. We use the formula from
http://en.wikipedia.org/wiki/Birthday_problem

P[m, n] \approx 1 - e^{-m^2/2n}

We ballpark an upper bound for $m$ by imagining (for whatever reason) a server
that restarts every second over 10 years, for $m = 86400 * 365 * 10 = 315360000$

For a $k$ character base-62 identifier, we have $n(k) = 62^k$

Plugging this in, we find $P[m, n(10)] \approx 5.75%$, which is good enough for
our purposes, and is surely more than anyone would ever need in practice -- a
process that is rebooted a handful of times a day for a hundred years has less
than a millionth of a percent chance of generating two colliding IDs.
*/

func init() {
	hostname, err := os.Hostname()
	if hostname == "" || err != nil {
		hostname = "localhost"
	}
	var buf [12]byte
	var b64 string
	for len(b64) < 10 {
		rand.Read(buf[:])
		b64 = base64.StdEncoding.EncodeToString(buf[:])
		b64 = strings.NewReplacer("+", "", "/", "").Replace(b64)
	}

	prefix = fmt.Sprintf("%s/%s", hostname, b64[0:10])
}

// RequestID is a middleware that injects a request ID into the context of each
// request. A request ID is a string of the form "host.example.com/random-0001",
// where "random" is a base62 random string that uniquely identifies this go
// process, and where the last number is an atomically incremented request
// counter.
func RequestID(c *web.C, h http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {
		if c.Env == nil {
			c.Env = make(map[string]interface{})
		}
		myid := atomic.AddUint64(&reqid, 1)
		c.Env[RequestIDKey] = fmt.Sprintf("%s-%06d", prefix, myid)

		h.ServeHTTP(w, r)
	}

	return http.HandlerFunc(fn)
}

// GetReqID returns a request ID from the given context if one is present.
// Returns the empty string if a request ID cannot be found.
func GetReqID(c web.C) string {
	if c.Env == nil {
		return ""
	}
	v, ok := c.Env[RequestIDKey]
	if !ok {
		return ""
	}
	if reqID, ok := v.(string); ok {
		return reqID
	}
	return ""
}
<|endoftext|>


<commit_before>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"testing"

	"github.com/google/mtail/metrics"
	"github.com/google/mtail/mtail"
	"github.com/google/mtail/vm"
	"github.com/google/mtail/watcher"
	"github.com/kylelemons/godebug/pretty"
)

var (
	recordBenchmark = flag.Bool("record_benchmark", false, "Record the benchmark results to 'benchmark_results.csv'.")
)

var exampleProgramTests = []struct {
	programfile string // Example program file.
	logfile     string // Sample log input.
	jsonfile    string // Expected metrics after processing.
}{
	{
		"examples/rsyncd.mtail",
		"testdata/rsyncd.log",
		"testdata/rsyncd.json",
	},
	{
		"examples/sftp.mtail",
		"testdata/sftp_chroot.log",
		"testdata/sftp_chroot.json",
	},
	{
		"examples/dhcpd.mtail",
		"testdata/anonymised_dhcpd_log",
		"testdata/anonymised_dhcpd_log.json",
	},
}

func CompileAndLoad(programfile string, ms *metrics.Store, lines chan string) (*vm.Loader, error) {
	p, err := os.Open(programfile)
	if err != nil {
		return nil, fmt.Errorf("%s: could not open program file: %s", programfile, err)
	}
	defer p.Close()

	name := filepath.Base(programfile)
	w := watcher.NewFakeWatcher()
	o := vm.LoaderOptions{W: w, Store: ms, Lines: lines, SyslogUseCurrentYear: false}

	l, err := vm.NewLoader(o)
	if err != nil {
		return nil, fmt.Errorf("couldn't create program loader: %s", err)
	}
	if pErr := l.CompileAndRun(name, p); pErr != nil {
		return nil, fmt.Errorf("couldn't compile program: %s: %s", programfile, pErr)
	}
	return l, nil
}

func TestExamplePrograms(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping test in short mode")
	}
	for _, tc := range exampleProgramTests {
		w := watcher.NewFakeWatcher()
		o := mtail.Options{Progs: tc.programfile, W: w}
		mtail, err := mtail.New(o)
		if err != nil {
			t.Fatalf("create mtail failed: %s", err)
		}

		if err := mtail.OneShot(tc.logfile); err != nil {
			t.Errorf("Oneshot failed for %s: %s", tc.logfile, err)
			continue
		}

		// Dirty hack to create json files :)
		if false {
			j, err := os.Create(tc.jsonfile)
			if err != nil {
				t.Errorf("%s: could not open json file: %s", tc.jsonfile, err)
				continue
			}
			defer j.Close()
			if err := mtail.WriteMetrics(j); err != nil {
				t.Errorf("couldn't marshall metrics: %q", err)
				continue
			}
		}

		j, err := os.Open(tc.jsonfile)
		if err != nil {
			t.Fatalf("%s: could not open json file: %s", tc.jsonfile, err)
		}
		defer j.Close()

		var m, ex bytes.Buffer
		mtail.Close()
		mtail.WriteMetrics(&m)
		m.WriteString("\n") // Golden data has trailing newline.

		if _, err := ex.ReadFrom(j); err != nil {
			t.Fatalf("Coldn't read from json: %s", err)
		}
		diff := pretty.Compare(m.String(), ex.String())
		if len(diff) > 0 {
			t.Errorf("%s: metrics don't match:\n%s", tc.programfile, diff)
		}
	}
}

func benchmarkProgram(b *testing.B, programfile string, logfile string) {
	w := watcher.NewFakeWatcher()
	o := mtail.Options{Progs: programfile, W: w}
	mtail, err := mtail.New(o)
	if err != nil {
		b.Fatalf("Failed to create mtail: %s", err)
	}

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		if err := mtail.OneShot(logfile); err != nil {
			b.Errorf("OneShot log parse failed: %s", err)
		}
	}
	b.StopTimer()
	mtail.Close()
	l, err := strconv.ParseInt(vm.LineCount.String(), 10, 64)
	if err != nil {
		b.Fatalf("strconv.ParseInt failed: %s", err)
		return
	}
	b.SetBytes(l)
}

func BenchmarkRsyncdProgram(b *testing.B) {
	benchmarkProgram(b, "examples/rsync.mtail", "testdata/rsyncd.log")
}
func BenchmarkSftpProgram(b *testing.B) {
	benchmarkProgram(b, "examples/sftp.mtail", "testdata/sftp_chroot.log")
}
func BenchmarkDhcpdProgram(b *testing.B) {
	benchmarkProgram(b, "examples/dhcpd.mtail", "testdata/anonymised_dhcpd_log")
}
<commit_msg>Fix order of parameters to pretty.Compare so expected is first.<commit_after>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"testing"

	"github.com/google/mtail/metrics"
	"github.com/google/mtail/mtail"
	"github.com/google/mtail/vm"
	"github.com/google/mtail/watcher"
	"github.com/kylelemons/godebug/pretty"
)

var (
	recordBenchmark = flag.Bool("record_benchmark", false, "Record the benchmark results to 'benchmark_results.csv'.")
)

var exampleProgramTests = []struct {
	programfile string // Example program file.
	logfile     string // Sample log input.
	jsonfile    string // Expected metrics after processing.
}{
	{
		"examples/rsyncd.mtail",
		"testdata/rsyncd.log",
		"testdata/rsyncd.json",
	},
	{
		"examples/sftp.mtail",
		"testdata/sftp_chroot.log",
		"testdata/sftp_chroot.json",
	},
	{
		"examples/dhcpd.mtail",
		"testdata/anonymised_dhcpd_log",
		"testdata/anonymised_dhcpd_log.json",
	},
}

func CompileAndLoad(programfile string, ms *metrics.Store, lines chan string) (*vm.Loader, error) {
	p, err := os.Open(programfile)
	if err != nil {
		return nil, fmt.Errorf("%s: could not open program file: %s", programfile, err)
	}
	defer p.Close()

	name := filepath.Base(programfile)
	w := watcher.NewFakeWatcher()
	o := vm.LoaderOptions{W: w, Store: ms, Lines: lines, SyslogUseCurrentYear: false}

	l, err := vm.NewLoader(o)
	if err != nil {
		return nil, fmt.Errorf("couldn't create program loader: %s", err)
	}
	if pErr := l.CompileAndRun(name, p); pErr != nil {
		return nil, fmt.Errorf("couldn't compile program: %s: %s", programfile, pErr)
	}
	return l, nil
}

func TestExamplePrograms(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping test in short mode")
	}
	for _, tc := range exampleProgramTests {
		w := watcher.NewFakeWatcher()
		o := mtail.Options{Progs: tc.programfile, W: w}
		mtail, err := mtail.New(o)
		if err != nil {
			t.Fatalf("create mtail failed: %s", err)
		}

		if err := mtail.OneShot(tc.logfile); err != nil {
			t.Errorf("Oneshot failed for %s: %s", tc.logfile, err)
			continue
		}

		// Dirty hack to create json files :)
		if false {
			j, err := os.Create(tc.jsonfile)
			if err != nil {
				t.Errorf("%s: could not open json file: %s", tc.jsonfile, err)
				continue
			}
			defer j.Close()
			if err := mtail.WriteMetrics(j); err != nil {
				t.Errorf("couldn't marshall metrics: %q", err)
				continue
			}
		}

		j, err := os.Open(tc.jsonfile)
		if err != nil {
			t.Fatalf("%s: could not open json file: %s", tc.jsonfile, err)
		}
		defer j.Close()

		var m, ex bytes.Buffer
		mtail.Close()
		mtail.WriteMetrics(&m)
		m.WriteString("\n") // Golden data has trailing newline.

		if _, err := ex.ReadFrom(j); err != nil {
			t.Fatalf("Coldn't read from json: %s", err)
		}
		diff := pretty.Compare(ex.String(), m.String())
		if len(diff) > 0 {
			t.Errorf("%s: metrics don't match:\n%s", tc.programfile, diff)
		}
	}
}

func benchmarkProgram(b *testing.B, programfile string, logfile string) {
	w := watcher.NewFakeWatcher()
	o := mtail.Options{Progs: programfile, W: w}
	mtail, err := mtail.New(o)
	if err != nil {
		b.Fatalf("Failed to create mtail: %s", err)
	}

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		if err := mtail.OneShot(logfile); err != nil {
			b.Errorf("OneShot log parse failed: %s", err)
		}
	}
	b.StopTimer()
	mtail.Close()
	l, err := strconv.ParseInt(vm.LineCount.String(), 10, 64)
	if err != nil {
		b.Fatalf("strconv.ParseInt failed: %s", err)
		return
	}
	b.SetBytes(l)
}

func BenchmarkRsyncdProgram(b *testing.B) {
	benchmarkProgram(b, "examples/rsync.mtail", "testdata/rsyncd.log")
}
func BenchmarkSftpProgram(b *testing.B) {
	benchmarkProgram(b, "examples/sftp.mtail", "testdata/sftp_chroot.log")
}
func BenchmarkDhcpdProgram(b *testing.B) {
	benchmarkProgram(b, "examples/dhcpd.mtail", "testdata/anonymised_dhcpd_log")
}
<|endoftext|>


<commit_before>package pbf

import (
	"bytes"
	"compress/zlib"
	"io"
	"os"
	"testing"

	"github.com/gogo/protobuf/proto"

	"github.com/omniscale/imposm3/element"
	"github.com/omniscale/imposm3/log"
	"github.com/omniscale/imposm3/parser/pbf/internal/osmpbf"
)

func BenchmarkHello(b *testing.B) {
	b.StopTimer()
	pbf, err := open("./monaco-20150428.osm.pbf")
	if err != nil {
		panic(err)
	}

	for pos := range pbf.BlockPositions() {
		b.StartTimer()
		for i := 0; i < b.N; i++ {
			readPrimitiveBlock(pos)
		}
		return
		// for {
		// 	stringtable := NewStringTable(block.GetStringtable())

		// 	for _, group := range block.Primitivegroup {
		// 		dense := group.GetDense()
		// 		ReadDenseNodes(dense, block, stringtable)
		// 	}
		// }
		// return
	}

}

func BenchmarkParser(b *testing.B) {
	b.ReportAllocs()
	for i := 0; i < b.N; i++ {
		p, err := NewParser("./monaco-20150428.osm.pbf")
		if err != nil {
			b.Fatal(err)
		}

		coords := make(chan []element.Node)
		nodes := make(chan []element.Node)
		ways := make(chan []element.Way)
		relations := make(chan []element.Relation)
		go func() {
			for _ = range coords {
			}
		}()
		go func() {
			for _ = range nodes {
			}
		}()
		go func() {
			for _ = range ways {
			}
		}()
		go func() {
			for _ = range relations {
			}
		}()

		p.Parse(coords, nodes, ways, relations)

		close(coords)
		close(nodes)
		close(ways)
		close(relations)
	}
}

func BenchmarkPrimitiveBlock(b *testing.B) {
	b.StopTimer()

	file, err := os.Open("./monaco-20150428.osm.pbf")
	if err != nil {
		log.Panic(err)
	}
	defer file.Close()

	var block = &osmpbf.PrimitiveBlock{}
	var blob = &osmpbf.Blob{}

	var size = 79566
	var offset int64 = 155

	blobData := make([]byte, size)
	file.Seek(offset, 0)
	io.ReadFull(file, blobData)
	err = proto.Unmarshal(blobData, blob)
	if err != nil {
		log.Panic("unmarshaling error blob: ", err)
	}

	b.StartTimer()
	for i := 0; i < b.N; i++ {
		buf := bytes.NewBuffer(blob.GetZlibData())
		r, err := zlib.NewReader(buf)
		if err != nil {
			log.Panic("zlib error: ", err)
		}
		raw := make([]byte, blob.GetRawSize())
		io.ReadFull(r, raw)
		err = proto.Unmarshal(raw, block)
		if err != nil {
			log.Panic("unmarshaling error: ", err)
		}
	}
}
<commit_msg>test: use b.Fatal instead of panic<commit_after>package pbf

import (
	"bytes"
	"compress/zlib"
	"io"
	"os"
	"testing"

	"github.com/gogo/protobuf/proto"

	"github.com/omniscale/imposm3/element"
	"github.com/omniscale/imposm3/parser/pbf/internal/osmpbf"
)

func BenchmarkHello(b *testing.B) {
	b.StopTimer()
	pbf, err := open("./monaco-20150428.osm.pbf")
	if err != nil {
		b.Fatal(err)
	}

	for pos := range pbf.BlockPositions() {
		b.StartTimer()
		for i := 0; i < b.N; i++ {
			readPrimitiveBlock(pos)
		}
		return
		// for {
		// 	stringtable := NewStringTable(block.GetStringtable())

		// 	for _, group := range block.Primitivegroup {
		// 		dense := group.GetDense()
		// 		ReadDenseNodes(dense, block, stringtable)
		// 	}
		// }
		// return
	}

}

func BenchmarkParser(b *testing.B) {
	b.ReportAllocs()
	for i := 0; i < b.N; i++ {
		p, err := NewParser("./monaco-20150428.osm.pbf")
		if err != nil {
			b.Fatal(err)
		}

		coords := make(chan []element.Node)
		nodes := make(chan []element.Node)
		ways := make(chan []element.Way)
		relations := make(chan []element.Relation)
		go func() {
			for _ = range coords {
			}
		}()
		go func() {
			for _ = range nodes {
			}
		}()
		go func() {
			for _ = range ways {
			}
		}()
		go func() {
			for _ = range relations {
			}
		}()

		p.Parse(coords, nodes, ways, relations)

		close(coords)
		close(nodes)
		close(ways)
		close(relations)
	}
}

func BenchmarkPrimitiveBlock(b *testing.B) {
	b.StopTimer()

	file, err := os.Open("./monaco-20150428.osm.pbf")
	if err != nil {
		b.Fatal(err)
	}
	defer file.Close()

	var block = &osmpbf.PrimitiveBlock{}
	var blob = &osmpbf.Blob{}

	var size = 79566
	var offset int64 = 155

	blobData := make([]byte, size)
	file.Seek(offset, 0)
	io.ReadFull(file, blobData)
	err = proto.Unmarshal(blobData, blob)
	if err != nil {
		b.Fatal("unmarshaling error blob: ", err)
	}

	b.StartTimer()
	for i := 0; i < b.N; i++ {
		buf := bytes.NewBuffer(blob.GetZlibData())
		r, err := zlib.NewReader(buf)
		if err != nil {
			b.Fatal("zlib error: ", err)
		}
		raw := make([]byte, blob.GetRawSize())
		io.ReadFull(r, raw)
		err = proto.Unmarshal(raw, block)
		if err != nil {
			b.Fatal("unmarshaling error: ", err)
		}
	}
}
<|endoftext|>


<commit_before>package fetcheddit

import (
	"code.leeclagett.com/grokeddit"
	"errors"
	"net/url"
	"strings"
)

const (
	redditJsonSuffix     = ".json"
	redditBeforeModifier = "before"
	redditAfterModifier  = "after"
)

type Path string

func CreatePath(redditPath string) (Path, error) {

	redditUrl, error := url.Parse(redditPath)
	if error != nil {
		return "", errors.New("Bad reddit path: " + error.Error())
	}

	redditUrl.Scheme = ""
	redditUrl.Host = ""

	if !strings.HasSuffix(redditUrl.Path, redditJsonSuffix) {
		redditUrl.Path = redditUrl.Path + redditJsonSuffix
	}

	// this is added at the last stage
	queries := redditUrl.Query()
	queries.Del(redditBeforeModifier)
	queries.Del(redditAfterModifier)
	redditUrl.RawQuery = queries.Encode()

	// flatten once to (hopefully) reduce garbage
	flattenedPath := redditUrl.String()

	// this is the sketchiest part -- assume dangling ? and & are
	// acceptable, and assume they won't be present in these situations ...
	// unittests can at least catch unexpected string values.
	if len(redditUrl.Query()) == 0 {
		flattenedPath = flattenedPath + "?"
	} else {
		flattenedPath = flattenedPath + "&"
	}

	return Path(flattenedPath), nil
}

func (path Path) FetchGrokedListing(contentFetcher Fetcher, anchor *AnchorPoint) (grokeddit.Groked, error) {
	retrievePath := string(path)

	if anchor != nil {
		retrievePath = retrievePath + anchor.String()
	}

	fetched, error := contentFetcher.Fetch(retrievePath)

	if error != nil {
		return grokeddit.Groked{}, errors.New("Unable to retrieve listing [" + retrievePath + "]: " + error.Error())
	}

	defer fetched.Close()
	return grokeddit.GrokListing(fetched)
}
<commit_msg>CreatePath should be slightly more efficient.<commit_after>package fetcheddit

import (
	"code.leeclagett.com/grokeddit"
	"errors"
	"net/url"
	"strings"
)

const (
	redditJsonSuffix     = ".json"
	redditBeforeModifier = "before"
	redditAfterModifier  = "after"
)

type Path string

func CreatePath(redditPath string) (Path, error) {

	redditUrl, error := url.Parse(redditPath)
	if error != nil {
		return "", errors.New("Bad reddit path: " + error.Error())
	}

	redditUrl.Scheme = ""
	redditUrl.Host = ""

	if !strings.HasSuffix(redditUrl.Path, redditJsonSuffix) {
		redditUrl.Path = redditUrl.Path + redditJsonSuffix
	}

	// before/after is added when path is fetched
	if len(redditUrl.RawQuery) != 0 {
		queries := redditUrl.Query()
		queries.Del(redditBeforeModifier)
		queries.Del(redditAfterModifier)
		redditUrl.RawQuery = queries.Encode()
	}

	// flatten once to (hopefully) reduce garbage
	flattenedPath := redditUrl.String()

	// this is the sketchiest part -- assume dangling ? and & are
	// acceptable, and assume they won't be present in these situations ...
	// unittests can at least catch unexpected string values.
	if len(redditUrl.RawQuery) == 0 { // len can change after removal above
		flattenedPath = flattenedPath + "?"
	} else {
		flattenedPath = flattenedPath + "&"
	}

	return Path(flattenedPath), nil
}

func (path Path) FetchGrokedListing(contentFetcher Fetcher, anchor *AnchorPoint) (grokeddit.Groked, error) {
	retrievePath := string(path)

	if anchor != nil {
		retrievePath = retrievePath + anchor.String()
	}

	fetched, error := contentFetcher.Fetch(retrievePath)

	if error != nil {
		return grokeddit.Groked{}, errors.New("Unable to retrieve listing [" + retrievePath + "]: " + error.Error())
	}

	defer fetched.Close()
	return grokeddit.GrokListing(fetched)
}
<|endoftext|>


<commit_before>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ogletest

// FailureRecord represents a single failed expectation or assertion for a
// test. Most users don't want to interact with these directly; they are
// generated implicitly using ExpectThat, AssertThat, ExpectLt, etc.
type FailureRecord struct {
	// The file name within which the expectation failed, e.g. "foo_test.go".
	FileName string

	// The line number at which the expectation failed.
	LineNumber int

	// The error associated with the file:line pair above. For example, the
	// following expectation:
	//
	//     ExpectEq(17, "taco")"
	//
	// May cause this error:
	//
	//     Expected: 17
	//     Actual:   "taco", which is not numeric
	//
	Error string
}

// Record a failure for the currently running test (and continue running it).
// Most users will want to use ExpectThat, ExpectEq, etc. instead of this
// function. Those that do want to report arbitrary errors will probably be
// satisfied with AddFailure, which is easier to use.
func AddFailureRecord(r FailureRecord) {
	currentlyRunningTest.mu.Lock()
	defer currentlyRunningTest.mu.Unlock()

	currentlyRunningTest.failureRecords = append(
		currentlyRunningTest.failureRecords,
		r)
}

// Call AddFailureRecord with a record whose file name and line number come
// from the caller of this function, and whose error string is created by
// calling fmt.Sprintf using the arguments to this function.
func AddFailure(format string, a ...interface{})

// Immediately stop executing the running test, causing it to fail with the
// failures previously recorded. Behavior is undefined if no failures have been
// recorded.
func AbortTest()
<commit_msg>Implemented AddFailure.<commit_after>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ogletest

import (
	"fmt"
	"runtime"
)

// FailureRecord represents a single failed expectation or assertion for a
// test. Most users don't want to interact with these directly; they are
// generated implicitly using ExpectThat, AssertThat, ExpectLt, etc.
type FailureRecord struct {
	// The file name within which the expectation failed, e.g. "foo_test.go".
	FileName string

	// The line number at which the expectation failed.
	LineNumber int

	// The error associated with the file:line pair above. For example, the
	// following expectation:
	//
	//     ExpectEq(17, "taco")"
	//
	// May cause this error:
	//
	//     Expected: 17
	//     Actual:   "taco", which is not numeric
	//
	Error string
}

// Record a failure for the currently running test (and continue running it).
// Most users will want to use ExpectThat, ExpectEq, etc. instead of this
// function. Those that do want to report arbitrary errors will probably be
// satisfied with AddFailure, which is easier to use.
func AddFailureRecord(r FailureRecord) {
	currentlyRunningTest.mu.Lock()
	defer currentlyRunningTest.mu.Unlock()

	currentlyRunningTest.failureRecords = append(
		currentlyRunningTest.failureRecords,
		r)
}

// Call AddFailureRecord with a record whose file name and line number come
// from the caller of this function, and whose error string is created by
// calling fmt.Sprintf using the arguments to this function.
func AddFailure(format string, a ...interface{}) {
	r := FailureRecord{
		Error: fmt.Sprintf(format, a...),
	}

	// Get information about the call site.
	var ok bool
	if _, r.FileName, r.LineNumber, ok = runtime.Caller(1); !ok {
		panic("Can't find caller")
	}

	AddFailureRecord(r)
}

// Immediately stop executing the running test, causing it to fail with the
// failures previously recorded. Behavior is undefined if no failures have been
// recorded.
func AbortTest()
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"net/http"
	"github.com/advancedlogic/GoOse"
)

type Article struct {
	Url         string
	Title       string
	Description string
	Body        string
	RawHTML     string
	StatusCode  int
}

func GetArticle(url string) Article {
	g := goose.New()
	resp, _ := http.Get(url)
	defer resp.Body.Close()

	html, _ := ioutil.ReadAll(resp.Body)
	article, err := g.ExtractFromRawHTML(url, string(html))
	if err != nil {
		return Article{StatusCode:resp.StatusCode}
	}

	return Article{url, article.Title, article.MetaDescription, article.CleanedText, article.RawHTML, resp.StatusCode}
}
<commit_msg>エラー無視しない<commit_after>package main

import (
	"io/ioutil"
	"net/http"
	"github.com/advancedlogic/GoOse"
)

type Article struct {
	Url         string
	Title       string
	Description string
	Body        string
	RawHTML     string
	StatusCode  int
}

func GetArticle(url string) Article {
	g := goose.New()
	resp, err := http.Get(url)
	if err != nil {
		return Article{StatusCode:resp.StatusCode}
	}
	defer resp.Body.Close()

	html, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return Article{StatusCode:resp.StatusCode}
	}

	article, err := g.ExtractFromRawHTML(url, string(html))
	if err != nil {
		return Article{StatusCode:resp.StatusCode}
	}

	return Article{url, article.Title, article.MetaDescription, article.CleanedText, article.RawHTML, resp.StatusCode}
}
<|endoftext|>


<commit_before>package antibody

import (
	"io/ioutil"
	"os"
	"strings"
	"testing"
)

func home() string {
	file, err := ioutil.TempDir(os.TempDir(), "antibody")
	if err != nil {
		panic(err.Error())
	}
	defer os.RemoveAll(file)
	os.Setenv("ANTIBODY_HOME", file+"/")
	return file + "/"
}

func TestProcessesArgsBunde(t *testing.T) {
	home := home()
	ProcessArgs([]string{"bundle", "caarlos0/zsh-pg"}, home)
}

func TestUpdate(t *testing.T) {
	home := home()
	ProcessArgs([]string{"update"}, home)
}

func TestBundlesSinglePlugin(t *testing.T) {
	home := home()
	Bundle("caarlos0/zsh-pg", home)
}

func TestLoadsDefaultHome(t *testing.T) {
	os.Unsetenv("ANTIBODY_HOME")
	if !strings.HasSuffix(Home(), "/.antibody/") {
		t.Error("Expected default ANTIBODY_HOME")
	}
}

func TestLoadsCustomHome(t *testing.T) {
	home := home()
	if home != Home() {
		t.Error("Expected custom ANTIBODY_HOME")
	}
}

func TestFailsToBundleInvalidRepos(t *testing.T) {
	home := home()
	defer func() {
		if err := recover(); err != nil {
			t.Log("Recovered from expected error")
		} else {
			t.Error("Expected a panic hence an invalid bundle was passed")
		}
	}()
	Bundle("csadsadp", home)
}
<commit_msg>test api.ProcessArgs with invalid arg<commit_after>package antibody

import (
	"io/ioutil"
	"os"
	"strings"
	"testing"
)

func home() string {
	file, err := ioutil.TempDir(os.TempDir(), "antibody")
	if err != nil {
		panic(err.Error())
	}
	defer os.RemoveAll(file)
	os.Setenv("ANTIBODY_HOME", file+"/")
	return file + "/"
}

func expectError(t *testing.T) {
	if err := recover(); err != nil {
		t.Log("Recovered from expected error")
	} else {
		t.Error("Expected an error here!")
	}
}

func TestProcessesArgsBunde(t *testing.T) {
	home := home()
	ProcessArgs([]string{"bundle", "caarlos0/zsh-pg"}, home)
}

func TestUpdate(t *testing.T) {
	home := home()
	ProcessArgs([]string{"update"}, home)
}

func TestBundlesSinglePlugin(t *testing.T) {
	home := home()
	Bundle("caarlos0/zsh-pg", home)
}

func TestLoadsDefaultHome(t *testing.T) {
	os.Unsetenv("ANTIBODY_HOME")
	if !strings.HasSuffix(Home(), "/.antibody/") {
		t.Error("Expected default ANTIBODY_HOME")
	}
}

func TestLoadsCustomHome(t *testing.T) {
	home := home()
	if home != Home() {
		t.Error("Expected custom ANTIBODY_HOME")
	}
}

func TestFailsToBundleInvalidRepos(t *testing.T) {
	home := home()
	defer expectError(t)
	Bundle("csadsadp", home)
}

func TestFailsToProcessInvalidArgs(t *testing.T) {
	home := home()
	defer expectError(t)
	ProcessArgs([]string{"nope", "caarlos0/zsh-pg"}, home)
}
<|endoftext|>


<commit_before>package structtag

import (
	"errors"
	"reflect"
)

// ReflectFunc type used to create funcrions to parse struct and tags
type ReflectFunc func(
	field *reflect.StructField,
	value *reflect.Value,
	tag string) (err error)

var (
	// ErrNotAPointer error when not a pointer
	ErrNotAPointer = errors.New("Not a pointer")

	// ErrNotAStruct error when not a struct
	ErrNotAStruct = errors.New("Not a struct")

	// ErrTypeNotSupported error when type not supported
	ErrTypeNotSupported = errors.New("Type not supported")

	// ErrUndefinedTag error when Tag var is not defined
	ErrUndefinedTag = errors.New("Undefined tag")

	// Tag set the main tag
	Tag string

	// TagDefault set tag default
	TagDefault string

	// TagHelper set tag usage
	TagHelper string

	// TagDisabled used to not process an input
	TagDisabled string

	// TagSeparator separe names on environment variables
	TagSeparator string

	// Prefix is a string that would be placed at the beginning of the generated tags.
	Prefix string

	// ParseMap points to each of the supported types
	ParseMap map[reflect.Kind]ReflectFunc
)

// Setup maps and variables
func Setup() {
	TagDisabled = "-"
	TagSeparator = "_"

	ParseMap = make(map[reflect.Kind]ReflectFunc)

	ParseMap[reflect.Struct] = ReflectStruct
}

// Reset maps caling setup function
func Reset() {
	Setup()
}

//Parse tags on struct instance
func Parse(s interface{}, superTag string) (err error) {
	if Tag == "" {
		err = ErrUndefinedTag
		return
	}

	st := reflect.TypeOf(s)
	if st.Kind() != reflect.Ptr {
		err = ErrNotAPointer
		return
	}

	refField := st.Elem()
	if refField.Kind() != reflect.Struct {
		err = ErrNotAStruct
		return
	}

	refValue := reflect.ValueOf(s).Elem()
	for i := 0; i < refField.NumField(); i++ {
		field := refField.Field(i)
		value := refValue.Field(i)
		kind := field.Type.Kind()

		if field.PkgPath != "" {
			continue
		}

		t := updateTag(&field, superTag)
		if t == "" {
			continue
		}

		f, ok := ParseMap[kind]
		if !ok {
			err = ErrTypeNotSupported
			return
		}

		err = f(&field, &value, t)
		if err != nil {
			return
		}
	}
	return
}

// SetBoolDefaults populates the boolean fields of 's' with cfgDefault values
func SetBoolDefaults(s interface{}, superTag string) (err error) {
	if Tag == "" {
		err = ErrUndefinedTag
		return
	}

	st := reflect.TypeOf(s)
	if st.Kind() != reflect.Ptr {
		err = ErrNotAPointer
		return
	}

	refField := st.Elem()
	if refField.Kind() != reflect.Struct {
		err = ErrNotAStruct
		return
	}

	refValue := reflect.ValueOf(s).Elem()
	for i := 0; i < refField.NumField(); i++ {
		field := refField.Field(i)
		kind := field.Type.Kind()
		value := refValue.Field(i)

		if kind == reflect.Bool {

			if field.PkgPath != "" {
				continue
			}

			t := updateTag(&field, superTag)
			if t == "" {
				continue
			}

			defaultValue := field.Tag.Get(TagDefault)
			v := defaultValue == "true" || defaultValue == "t"
			value.SetBool(v)
		} else if kind == reflect.Struct {
			t := updateTag(&field, superTag)
			if t != "" {
				err := SetBoolDefaults(value.Addr().Interface(), "")
				if err != nil {
					return err
				}
			}
		}
	}
	return
}

func updateTag(field *reflect.StructField, superTag string) (ret string) {
	ret = field.Tag.Get(Tag)
	if ret == TagDisabled {
		ret = ""
		return
	}
	if ret == "" {
		ret = field.Name
	}
	if superTag != "" {
		ret = superTag + TagSeparator + ret
		return
	}
	if Prefix != "" {
		ret = Prefix + TagSeparator + ret
	}
	return
}

// ReflectStruct is called when the Parse encounters a sub-structure in the current structure and then calls Parsr again to treat the fields of the sub-structure.
func ReflectStruct(field *reflect.StructField, value *reflect.Value, tag string) (err error) {
	err = Parse(value.Addr().Interface(), tag)
	return
}
<commit_msg>fix typo<commit_after>package structtag

import (
	"errors"
	"reflect"
)

// ReflectFunc type used to create funcrions to parse struct and tags
type ReflectFunc func(
	field *reflect.StructField,
	value *reflect.Value,
	tag string) (err error)

var (
	// ErrNotAPointer error when not a pointer
	ErrNotAPointer = errors.New("Not a pointer")

	// ErrNotAStruct error when not a struct
	ErrNotAStruct = errors.New("Not a struct")

	// ErrTypeNotSupported error when type not supported
	ErrTypeNotSupported = errors.New("Type not supported")

	// ErrUndefinedTag error when Tag var is not defined
	ErrUndefinedTag = errors.New("Undefined tag")

	// Tag set the main tag
	Tag string

	// TagDefault set tag default
	TagDefault string

	// TagHelper set tag usage
	TagHelper string

	// TagDisabled used to not process an input
	TagDisabled string

	// TagSeparator separe names on environment variables
	TagSeparator string

	// Prefix is a string that would be placed at the beginning of the generated tags.
	Prefix string

	// ParseMap points to each of the supported types
	ParseMap map[reflect.Kind]ReflectFunc
)

// Setup maps and variables
func Setup() {
	TagDisabled = "-"
	TagSeparator = "_"

	ParseMap = make(map[reflect.Kind]ReflectFunc)

	ParseMap[reflect.Struct] = ReflectStruct
}

// Reset maps caling setup function
func Reset() {
	Setup()
}

//Parse tags on struct instance
func Parse(s interface{}, superTag string) (err error) {
	if Tag == "" {
		err = ErrUndefinedTag
		return
	}

	st := reflect.TypeOf(s)
	if st.Kind() != reflect.Ptr {
		err = ErrNotAPointer
		return
	}

	refField := st.Elem()
	if refField.Kind() != reflect.Struct {
		err = ErrNotAStruct
		return
	}

	refValue := reflect.ValueOf(s).Elem()
	for i := 0; i < refField.NumField(); i++ {
		field := refField.Field(i)
		value := refValue.Field(i)
		kind := field.Type.Kind()

		if field.PkgPath != "" {
			continue
		}

		t := updateTag(&field, superTag)
		if t == "" {
			continue
		}

		f, ok := ParseMap[kind]
		if !ok {
			err = ErrTypeNotSupported
			return
		}

		err = f(&field, &value, t)
		if err != nil {
			return
		}
	}
	return
}

// SetBoolDefaults populates the boolean fields of 's' with cfgDefault values
func SetBoolDefaults(s interface{}, superTag string) (err error) {
	if Tag == "" {
		err = ErrUndefinedTag
		return
	}

	st := reflect.TypeOf(s)
	if st.Kind() != reflect.Ptr {
		err = ErrNotAPointer
		return
	}

	refField := st.Elem()
	if refField.Kind() != reflect.Struct {
		err = ErrNotAStruct
		return
	}

	refValue := reflect.ValueOf(s).Elem()
	for i := 0; i < refField.NumField(); i++ {
		field := refField.Field(i)
		kind := field.Type.Kind()
		value := refValue.Field(i)

		if kind == reflect.Bool {

			if field.PkgPath != "" {
				continue
			}

			t := updateTag(&field, superTag)
			if t == "" {
				continue
			}

			defaultValue := field.Tag.Get(TagDefault)
			v := defaultValue == "true" || defaultValue == "t"
			value.SetBool(v)
		} else if kind == reflect.Struct {
			t := updateTag(&field, superTag)
			if t != "" {
				err := SetBoolDefaults(value.Addr().Interface(), "")
				if err != nil {
					return err
				}
			}
		}
	}
	return
}

func updateTag(field *reflect.StructField, superTag string) (ret string) {
	ret = field.Tag.Get(Tag)
	if ret == TagDisabled {
		ret = ""
		return
	}
	if ret == "" {
		ret = field.Name
	}
	if superTag != "" {
		ret = superTag + TagSeparator + ret
		return
	}
	if Prefix != "" {
		ret = Prefix + TagSeparator + ret
	}
	return
}

// ReflectStruct is called when the Parse encounters a sub-structure in the current structure and then calls Parser again to treat the fields of the sub-structure.
func ReflectStruct(field *reflect.StructField, value *reflect.Value, tag string) (err error) {
	err = Parse(value.Addr().Interface(), tag)
	return
}
<|endoftext|>


<commit_before>// Copyright 2017 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

// Git remote helper for the Keybase file system.

package main

import (
	"context"
	"flag"
	"fmt"
	"os"
	"path/filepath"

	"github.com/keybase/client/go/libkb"
	"github.com/keybase/kbfs/env"
	"github.com/keybase/kbfs/kbfsgit"
	"github.com/keybase/kbfs/libfs"
	"github.com/keybase/kbfs/libgit"
	"github.com/keybase/kbfs/libkbfs"
	"github.com/pkg/errors"
)

var version = flag.Bool("version", false, "Print version")

const usageFormatStr = `Usage:
  git-remote-keybase -version

To run against remote KBFS servers:
  git-remote-keybase %s <remote> [keybase://<repo>]

To run in a local testing environment:
  git-remote-keybase %s <remote> [keybase://<repo>]

Defaults:
%s
`

func getUsageString(ctx libkbfs.Context) string {
	remoteUsageStr := libkbfs.GetRemoteUsageString()
	localUsageStr := libkbfs.GetLocalUsageString()
	defaultUsageStr := libkbfs.GetDefaultsUsageString(ctx)
	return fmt.Sprintf(
		usageFormatStr, remoteUsageStr, localUsageStr, defaultUsageStr)
}

func getLocalGitDir() (gitDir string, err error) {
	gitDir = os.Getenv("GIT_DIR")
	fi, err := os.Stat(gitDir)
	if err != nil {
		return "", err
	}
	if !fi.IsDir() {
		return "", errors.Errorf("GIT_DIR=%s, but is not a dir", gitDir)
	}
	// On Windows, git annoyingly puts normal slashes in the
	// environment variable.
	return filepath.FromSlash(gitDir), nil
}

func start() (startErr *libfs.Error) {
	kbCtx := env.NewContext()

	defaultParams, storageRoot, err := libgit.Params(kbCtx, kbCtx.GetDataDir())
	if err != nil {
		return libfs.InitError(err.Error())
	}
	defer func() {
		rmErr := os.RemoveAll(storageRoot)
		if rmErr != nil {
			fmt.Fprintf(os.Stderr,
				"Error cleaning storage dir %s: %+v\n", storageRoot, rmErr)
		}
	}()
	defaultLogPath := filepath.Join(kbCtx.GetLogDir(), libkb.GitLogFileName)

	// Duplicate the stderr fd, so that when the logger closes it when
	// redirecting log messages to a file, we will still be able to
	// write status updates back to the git process.
	stderrFile, err := dupStderr()
	if err != nil {
		return libfs.InitError(err.Error())
	}
	defer stderrFile.Close()

	defer func() {
		// Now that the stderr has been duplicated, print all errors
		// to the duplicate as well as to the new `os.Stderr` down in
		// `main()`, so that the error shows up both in the log and to
		// the user.
		if startErr != nil {
			fmt.Fprintf(stderrFile, "git-remote-keybase error: (%d) %s\n",
				startErr.Code, startErr.Message)
		}
	}()

	kbfsParams := libkbfs.AddFlagsWithDefaults(
		flag.CommandLine, defaultParams, defaultLogPath)
	flag.Parse()

	if *version {
		fmt.Printf("%s\n", libkbfs.VersionString())
		return nil
	}

	if len(flag.Args()) < 1 {
		fmt.Print(getUsageString(kbCtx))
		return libfs.InitError("no remote repo specified")
	}

	remote := flag.Arg(0)
	var repo string
	if len(flag.Args()) > 1 {
		repo = flag.Arg(1)
	}

	if len(flag.Args()) > 2 {
		fmt.Print(getUsageString(kbCtx))
		return libfs.InitError("extra arguments specified (flags go before the first argument)")
	}

	gitDir, err := getLocalGitDir()
	if err != nil {
		return libfs.InitError(err.Error())
	}

	options := kbfsgit.StartOptions{
		KbfsParams: *kbfsParams,
		Remote:     remote,
		Repo:       repo,
		GitDir:     gitDir,
	}

	ctx := context.Background()
	return kbfsgit.Start(
		ctx, options, kbCtx, defaultLogPath, os.Stdin, os.Stdout, stderrFile)
}

func main() {
	err := start()
	if err != nil {
		fmt.Fprintf(os.Stderr, "git-remote-keybase error: (%d) %s\n",
			err.Code, err.Message)
		os.Exit(err.Code)
	}
	os.Exit(0)
}
<commit_msg>git-remote-keybase: default to production run mode<commit_after>// Copyright 2017 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

// Git remote helper for the Keybase file system.

package main

import (
	"context"
	"flag"
	"fmt"
	"os"
	"path/filepath"

	"github.com/keybase/client/go/libkb"
	"github.com/keybase/kbfs/env"
	"github.com/keybase/kbfs/kbfsgit"
	"github.com/keybase/kbfs/libfs"
	"github.com/keybase/kbfs/libgit"
	"github.com/keybase/kbfs/libkbfs"
	"github.com/pkg/errors"
)

var version = flag.Bool("version", false, "Print version")

const usageFormatStr = `Usage:
  git-remote-keybase -version

To run against remote KBFS servers:
  git-remote-keybase %s <remote> [keybase://<repo>]

To run in a local testing environment:
  git-remote-keybase %s <remote> [keybase://<repo>]

Defaults:
%s
`

func getUsageString(ctx libkbfs.Context) string {
	remoteUsageStr := libkbfs.GetRemoteUsageString()
	localUsageStr := libkbfs.GetLocalUsageString()
	defaultUsageStr := libkbfs.GetDefaultsUsageString(ctx)
	return fmt.Sprintf(
		usageFormatStr, remoteUsageStr, localUsageStr, defaultUsageStr)
}

func getLocalGitDir() (gitDir string, err error) {
	gitDir = os.Getenv("GIT_DIR")
	fi, err := os.Stat(gitDir)
	if err != nil {
		return "", err
	}
	if !fi.IsDir() {
		return "", errors.Errorf("GIT_DIR=%s, but is not a dir", gitDir)
	}
	// On Windows, git annoyingly puts normal slashes in the
	// environment variable.
	return filepath.FromSlash(gitDir), nil
}

func start() (startErr *libfs.Error) {
	kbCtx := env.NewContext()

	switch kbCtx.GetRunMode() {
	case libkb.StagingRunMode:
		fmt.Fprintf(os.Stderr, "Running in staging mode\n")
	case libkb.DevelRunMode:
		fallthrough
	case libkb.NoRunMode:
		fmt.Fprintf(os.Stderr, "Running in devel mode\n")
	}

	defaultParams, storageRoot, err := libgit.Params(kbCtx, kbCtx.GetDataDir())
	if err != nil {
		return libfs.InitError(err.Error())
	}
	defer func() {
		rmErr := os.RemoveAll(storageRoot)
		if rmErr != nil {
			fmt.Fprintf(os.Stderr,
				"Error cleaning storage dir %s: %+v\n", storageRoot, rmErr)
		}
	}()
	defaultLogPath := filepath.Join(kbCtx.GetLogDir(), libkb.GitLogFileName)

	// Duplicate the stderr fd, so that when the logger closes it when
	// redirecting log messages to a file, we will still be able to
	// write status updates back to the git process.
	stderrFile, err := dupStderr()
	if err != nil {
		return libfs.InitError(err.Error())
	}
	defer stderrFile.Close()

	defer func() {
		// Now that the stderr has been duplicated, print all errors
		// to the duplicate as well as to the new `os.Stderr` down in
		// `main()`, so that the error shows up both in the log and to
		// the user.
		if startErr != nil {
			fmt.Fprintf(stderrFile, "git-remote-keybase error: (%d) %s\n",
				startErr.Code, startErr.Message)
		}
	}()

	kbfsParams := libkbfs.AddFlagsWithDefaults(
		flag.CommandLine, defaultParams, defaultLogPath)
	flag.Parse()

	if *version {
		fmt.Printf("%s\n", libkbfs.VersionString())
		return nil
	}

	if len(flag.Args()) < 1 {
		fmt.Print(getUsageString(kbCtx))
		return libfs.InitError("no remote repo specified")
	}

	remote := flag.Arg(0)
	var repo string
	if len(flag.Args()) > 1 {
		repo = flag.Arg(1)
	}

	if len(flag.Args()) > 2 {
		fmt.Print(getUsageString(kbCtx))
		return libfs.InitError("extra arguments specified (flags go before the first argument)")
	}

	gitDir, err := getLocalGitDir()
	if err != nil {
		return libfs.InitError(err.Error())
	}

	options := kbfsgit.StartOptions{
		KbfsParams: *kbfsParams,
		Remote:     remote,
		Repo:       repo,
		GitDir:     gitDir,
	}

	ctx := context.Background()
	return kbfsgit.Start(
		ctx, options, kbCtx, defaultLogPath, os.Stdin, os.Stdout, stderrFile)
}

func main() {
	runMode := os.Getenv("KEYBASE_RUN_MODE")
	if len(runMode) == 0 {
		// Default to prod.
		os.Setenv("KEYBASE_RUN_MODE", "prod")
	}

	err := start()
	if err != nil {
		fmt.Fprintf(os.Stderr, "git-remote-keybase error: (%d) %s\n",
			err.Code, err.Message)
		os.Exit(err.Code)
	}
	os.Exit(0)
}
<|endoftext|>


<commit_before>/*
Package summarize implements functions for analyzing readability and usage statistics of text.
*/
package summarize

import (
	"unicode"

	"github.com/jdkato/prose/internal/util"
	"github.com/jdkato/prose/tokenize"
)

// A Document represents a collection of text to be analyzed.
//
// A Document's calculations depend on its word and sentence tokenizers. You
// can use the defaults by invoking NewDocument, choose another implemention
// from the tokenize package, or use your own (as long as it implements the
// ProseTokenizer interface). For example,
//
//    d := Document{Content: ..., WordTokenizer: ..., SentenceTokenizer: ...}
//    d.Initialize()
//
// TODO: There should be a way to efficiently add or remove text from a the
// content of a Document (e.g., we should be able to build it incrementally).
// Perhaps we should look into using a rope as our underlying data structure?
type Document struct {
	Content           string
	NumCharacters     float64
	NumComplexWords   float64
	NumPolysylWords   float64
	NumSentences      float64
	NumSyllables      float64
	NumWords          float64
	Sentences         map[string]int
	SentenceTokenizer tokenize.ProseTokenizer
	WordFrequency     map[string]int
	Words             []string
	WordTokenizer     tokenize.ProseTokenizer
}

// An Assessment provides comprehensive access to a Document's metrics.
type Assessment struct {
	AutomatedReadability float64
	FleschKincaid        float64
	ReadingEase          float64
	GunningFog           float64
	SMOG                 float64
}

// NewDocument is a Document constructor that takes a string as an argument. It
// then calculates the data necessary for computing readability and usage
// statistics.
//
// This is a convenience wrapper around the Document initialization process
// that defaults to using a WordBoundaryTokenizer and a PunktSentenceTokenizer
// as its word and sentence tokenizers, respectively.
func NewDocument(text string) *Document {
	wTok := tokenize.NewWordBoundaryTokenizer()
	sTok := tokenize.NewPunktSentenceTokenizer()
	doc := Document{Content: text, WordTokenizer: wTok, SentenceTokenizer: sTok}
	doc.Initialize()
	return &doc
}

// Initialize calculates the data necessary for computing readability and usage
// statistics.
func (d *Document) Initialize() {
	d.WordFrequency = make(map[string]int)
	for _, s := range d.SentenceTokenizer.Tokenize(d.Content) {
		wordCount := d.NumWords
		d.NumSentences++
		for _, word := range d.WordTokenizer.Tokenize(s) {
			if count, found := d.WordFrequency[word]; found {
				d.WordFrequency[word] = count + 1
			} else {
				d.WordFrequency[word] = 1
			}
			d.NumCharacters += countChars(word)
			syllables := Syllables(word)
			d.NumSyllables += float64(syllables)
			if syllables > 2 {
				d.NumPolysylWords++
			}
			if isComplex(word, syllables) {
				d.NumComplexWords++
			}
			d.Words = append(d.Words, word)
			d.NumWords++
		}
		d.Sentences[s] = int(d.NumWords - wordCount)
	}
}

// Assess returns an Assessment for the Document d.
func (d *Document) Assess() *Assessment {
	return &Assessment{
		FleschKincaid: d.FleschKincaid(), ReadingEase: d.ReadingEase(),
		GunningFog: d.Gunningfog(), SMOG: d.SMOG(),
		AutomatedReadability: d.AutomatedReadability()}
}

// Syllables returns the number of syllables in the string word.
func Syllables(word string) int {
	vowels := []rune{'a', 'e', 'i', 'o', 'u', 'y'}
	vowelCount := 0
	ext := len(word)

	lastWasVowel := false
	for _, c := range word {
		found := false
		for _, v := range vowels {
			if v == c {
				found = true
				if !lastWasVowel {
					vowelCount++
				}
				break
			}
		}
		lastWasVowel = found
	}
	if (ext > 2 && word[ext-2:] == "es") || (ext > 1 && word[ext-1:] == "e") {
		vowelCount--
	}

	return vowelCount
}

func isComplex(word string, syllables int) bool {
	if util.HasAnySuffix(word, []string{"es", "ed", "ing"}) {
		syllables--
	}
	return syllables > 2
}

func countChars(word string) float64 {
	count := 0
	for _, c := range word {
		if unicode.IsLetter(c) || unicode.IsNumber(c) {
			count++
		}
	}
	return float64(count)
}
<commit_msg>Fix [summarize]: Don't assign to a nil map<commit_after>/*
Package summarize implements functions for analyzing readability and usage statistics of text.
*/
package summarize

import (
	"unicode"

	"github.com/jdkato/prose/internal/util"
	"github.com/jdkato/prose/tokenize"
)

// A Document represents a collection of text to be analyzed.
//
// A Document's calculations depend on its word and sentence tokenizers. You
// can use the defaults by invoking NewDocument, choose another implemention
// from the tokenize package, or use your own (as long as it implements the
// ProseTokenizer interface). For example,
//
//    d := Document{Content: ..., WordTokenizer: ..., SentenceTokenizer: ...}
//    d.Initialize()
//
// TODO: There should be a way to efficiently add or remove text from a the
// content of a Document (e.g., we should be able to build it incrementally).
// Perhaps we should look into using a rope as our underlying data structure?
type Document struct {
	Content           string
	NumCharacters     float64
	NumComplexWords   float64
	NumPolysylWords   float64
	NumSentences      float64
	NumSyllables      float64
	NumWords          float64
	Sentences         map[string]int
	SentenceTokenizer tokenize.ProseTokenizer
	WordFrequency     map[string]int
	Words             []string
	WordTokenizer     tokenize.ProseTokenizer
}

// An Assessment provides comprehensive access to a Document's metrics.
type Assessment struct {
	AutomatedReadability float64
	FleschKincaid        float64
	ReadingEase          float64
	GunningFog           float64
	SMOG                 float64
}

// NewDocument is a Document constructor that takes a string as an argument. It
// then calculates the data necessary for computing readability and usage
// statistics.
//
// This is a convenience wrapper around the Document initialization process
// that defaults to using a WordBoundaryTokenizer and a PunktSentenceTokenizer
// as its word and sentence tokenizers, respectively.
func NewDocument(text string) *Document {
	wTok := tokenize.NewWordBoundaryTokenizer()
	sTok := tokenize.NewPunktSentenceTokenizer()
	doc := Document{Content: text, WordTokenizer: wTok, SentenceTokenizer: sTok}
	doc.Initialize()
	return &doc
}

// Initialize calculates the data necessary for computing readability and usage
// statistics.
func (d *Document) Initialize() {
	d.WordFrequency = make(map[string]int)
	d.Sentences = make(map[string]int)
	for _, s := range d.SentenceTokenizer.Tokenize(d.Content) {
		wordCount := d.NumWords
		d.NumSentences++
		for _, word := range d.WordTokenizer.Tokenize(s) {
			if count, found := d.WordFrequency[word]; found {
				d.WordFrequency[word] = count + 1
			} else {
				d.WordFrequency[word] = 1
			}
			d.NumCharacters += countChars(word)
			syllables := Syllables(word)
			d.NumSyllables += float64(syllables)
			if syllables > 2 {
				d.NumPolysylWords++
			}
			if isComplex(word, syllables) {
				d.NumComplexWords++
			}
			d.Words = append(d.Words, word)
			d.NumWords++
		}
		d.Sentences[s] = int(d.NumWords - wordCount)
	}
}

// Assess returns an Assessment for the Document d.
func (d *Document) Assess() *Assessment {
	return &Assessment{
		FleschKincaid: d.FleschKincaid(), ReadingEase: d.ReadingEase(),
		GunningFog: d.Gunningfog(), SMOG: d.SMOG(),
		AutomatedReadability: d.AutomatedReadability()}
}

// Syllables returns the number of syllables in the string word.
func Syllables(word string) int {
	vowels := []rune{'a', 'e', 'i', 'o', 'u', 'y'}
	vowelCount := 0
	ext := len(word)

	lastWasVowel := false
	for _, c := range word {
		found := false
		for _, v := range vowels {
			if v == c {
				found = true
				if !lastWasVowel {
					vowelCount++
				}
				break
			}
		}
		lastWasVowel = found
	}
	if (ext > 2 && word[ext-2:] == "es") || (ext > 1 && word[ext-1:] == "e") {
		vowelCount--
	}

	return vowelCount
}

func isComplex(word string, syllables int) bool {
	if util.HasAnySuffix(word, []string{"es", "ed", "ing"}) {
		syllables--
	}
	return syllables > 2
}

func countChars(word string) float64 {
	count := 0
	for _, c := range word {
		if unicode.IsLetter(c) || unicode.IsNumber(c) {
			count++
		}
	}
	return float64(count)
}
<|endoftext|>