<commit_before>package apparmor

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
)

const DefaultProfilePath = "/etc/apparmor.d/docker"
const DefaultProfile = `
# AppArmor profile from lxc for containers.

#include <tunables/global>
profile docker-default flags=(attach_disconnected,mediate_deleted) {
  #include <abstractions/base>
  network,
  capability,
  file,
  umount,

  # ignore DENIED message on / remount
  deny mount options=(ro, remount) -> /,

  # allow tmpfs mounts everywhere
  mount fstype=tmpfs,

  # allow mqueue mounts everywhere
  mount fstype=mqueue,

  # allow fuse mounts everywhere
  mount fstype=fuse.*,

  # allow bind mount of /lib/init/fstab for lxcguest
  mount options=(rw, bind) /lib/init/fstab.lxc/ -> /lib/init/fstab/,

  # deny writes in /proc/sys/fs but allow binfmt_misc to be mounted
  mount fstype=binfmt_misc -> /proc/sys/fs/binfmt_misc/,
  deny @{PROC}/sys/fs/** wklx,

  # allow efivars to be mounted, writing to it will be blocked though
  mount fstype=efivarfs -> /sys/firmware/efi/efivars/,

  # block some other dangerous paths
  deny @{PROC}/sysrq-trigger rwklx,
  deny @{PROC}/mem rwklx,
  deny @{PROC}/kmem rwklx,
  deny @{PROC}/sys/kernel/[^s][^h][^m]* wklx,
  deny @{PROC}/sys/kernel/*/** wklx,

  # deny writes in /sys except for /sys/fs/cgroup, also allow
  # fusectl, securityfs and debugfs to be mounted there (read-only)
  mount fstype=fusectl -> /sys/fs/fuse/connections/,
  mount fstype=securityfs -> /sys/kernel/security/,
  mount fstype=debugfs -> /sys/kernel/debug/,
  deny mount fstype=debugfs -> /var/lib/ureadahead/debugfs/,
  mount fstype=proc -> /proc/,
  mount fstype=sysfs -> /sys/,
  deny /sys/[^f]*/** wklx,
  deny /sys/f[^s]*/** wklx,
  deny /sys/fs/[^c]*/** wklx,
  deny /sys/fs/c[^g]*/** wklx,
  deny /sys/fs/cg[^r]*/** wklx,
  deny /sys/firmware/efi/efivars/** rwklx,
  deny /sys/kernel/security/** rwklx,
  mount options=(move) /sys/fs/cgroup/cgmanager/ -> /sys/fs/cgroup/cgmanager.lower/,

  # the container may never be allowed to mount devpts.  If it does, it
  # will remount the host's devpts.  We could allow it to do it with
  # the newinstance option (but, right now, we don't).
  deny mount fstype=devpts,
}
`

func InstallDefaultProfile() error {
	if !IsEnabled() {
		return nil
	}

	// If the profile already exists, let it be.
	if _, err := os.Stat(DefaultProfilePath); err == nil {
		return nil
	}

	// Make sure /etc/apparmor.d exists
	if err := os.MkdirAll(path.Dir(DefaultProfilePath), 0755); err != nil {
		return err
	}

	if err := ioutil.WriteFile(DefaultProfilePath, []byte(DefaultProfile), 0644); err != nil {
		return err
	}

	output, err := exec.Command("/lib/init/apparmor-profile-load", "docker").CombinedOutput()
	if err != nil {
		return fmt.Errorf("Error loading docker profile: %s (%s)", err, output)
	}
	return nil
}
<commit_msg>Backup current docker apparmor profile and replace it with the new one<commit_after>package apparmor

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
)

const (
	DefaultProfilePath = "/etc/apparmor.d/docker"
)

const DefaultProfile = `
# AppArmor profile from lxc for containers.

#include <tunables/global>
profile docker-default flags=(attach_disconnected,mediate_deleted) {
  #include <abstractions/base>
  network,
  capability,
  file,
  umount,

  # ignore DENIED message on / remount
  deny mount options=(ro, remount) -> /,

  # allow tmpfs mounts everywhere
  mount fstype=tmpfs,

  # allow mqueue mounts everywhere
  mount fstype=mqueue,

  # allow fuse mounts everywhere
  mount fstype=fuse.*,

  # allow bind mount of /lib/init/fstab for lxcguest
  mount options=(rw, bind) /lib/init/fstab.lxc/ -> /lib/init/fstab/,

  # deny writes in /proc/sys/fs but allow binfmt_misc to be mounted
  mount fstype=binfmt_misc -> /proc/sys/fs/binfmt_misc/,
  deny @{PROC}/sys/fs/** wklx,

  # allow efivars to be mounted, writing to it will be blocked though
  mount fstype=efivarfs -> /sys/firmware/efi/efivars/,

  # block some other dangerous paths
  deny @{PROC}/sysrq-trigger rwklx,
  deny @{PROC}/mem rwklx,
  deny @{PROC}/kmem rwklx,
  deny @{PROC}/sys/kernel/[^s][^h][^m]* wklx,
  deny @{PROC}/sys/kernel/*/** wklx,

  # deny writes in /sys except for /sys/fs/cgroup, also allow
  # fusectl, securityfs and debugfs to be mounted there (read-only)
  mount fstype=fusectl -> /sys/fs/fuse/connections/,
  mount fstype=securityfs -> /sys/kernel/security/,
  mount fstype=debugfs -> /sys/kernel/debug/,
  deny mount fstype=debugfs -> /var/lib/ureadahead/debugfs/,
  mount fstype=proc -> /proc/,
  mount fstype=sysfs -> /sys/,
  deny /sys/[^f]*/** wklx,
  deny /sys/f[^s]*/** wklx,
  deny /sys/fs/[^c]*/** wklx,
  deny /sys/fs/c[^g]*/** wklx,
  deny /sys/fs/cg[^r]*/** wklx,
  deny /sys/firmware/efi/efivars/** rwklx,
  deny /sys/kernel/security/** rwklx,
  mount options=(move) /sys/fs/cgroup/cgmanager/ -> /sys/fs/cgroup/cgmanager.lower/,

  # the container may never be allowed to mount devpts.  If it does, it
  # will remount the host's devpts.  We could allow it to do it with
  # the newinstance option (but, right now, we don't).
  deny mount fstype=devpts,
}
`

func InstallDefaultProfile(backupPath string) error {
	if !IsEnabled() {
		return nil
	}

	// If the profile already exists, check if we already have a backup
	// if not, do the backup and override it. (docker 0.10 upgrade changed the apparmor profile)
	// see gh#5049, apparmor blocks signals in ubuntu 14.04
	if _, err := os.Stat(DefaultProfilePath); err == nil {
		if _, err := os.Stat(backupPath); err == nil {
			// If both the profile and the backup are present, do nothing
			return nil
		}
		// Make sure the directory exists
		if err := os.MkdirAll(path.Dir(backupPath), 0755); err != nil {
			return err
		}

		// Create the backup file
		f, err := os.Create(backupPath)
		if err != nil {
			return err
		}
		defer f.Close()
		src, err := os.Open(DefaultProfilePath)
		if err != nil {
			return err
		}
		defer src.Close()
		if _, err := io.Copy(f, src); err != nil {
			return err
		}
	}

	// Make sure /etc/apparmor.d exists
	if err := os.MkdirAll(path.Dir(DefaultProfilePath), 0755); err != nil {
		return err
	}

	if err := ioutil.WriteFile(DefaultProfilePath, []byte(DefaultProfile), 0644); err != nil {
		return err
	}

	output, err := exec.Command("/lib/init/apparmor-profile-load", "docker").CombinedOutput()
	if err != nil {
		return fmt.Errorf("Error loading docker profile: %s (%s)", err, output)
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright © 2014 Alienero. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"errors"
	"net"
	"net/http"
	"net/rpc"
	"time"

	myrpc "github.com/Alienero/quick-know/rpc"
	"github.com/Alienero/quick-know/store"
	"github.com/Alienero/quick-know/store/define"

	"github.com/golang/glog"
)

type Comet_RPC struct {
}

func (*Comet_RPC) Relogin(id string, r *myrpc.Reply) error {
	c := Users.Get(id)
	if c == nil {
		// r.IsRe = true
	} else {
		c.lock.Lock()
		if !c.isLetClose {
			c.isLetClose = true
			c.lock.Unlock()
			select {
			case c.CloseChan <- 1:
				<-c.CloseChan
				r.IsOk = true
				glog.Info("RPC: Ok will be relogin.")
			case <-time.After(2 * time.Second):
				// Timeout.
				if c := Users.Get(id); c != nil {
					return errors.New("Close the logon user timeout")
				}
				// Has been esc.
			}
		} else {
			c.lock.Unlock()
		}
	}
	return nil
}

func (*Comet_RPC) WriteOnlineMsg(msg *define.Msg, r *myrpc.Reply) (err error) {
	defer func() {
		if err == nil {
			r.IsOk = true
		}
	}()
	glog.Infof("Get a Write msg RPC,msg is :%v", string(msg.Body))
	msg.Dup = 0
	// fix the Expired
	if msg.Expired > 0 {
		msg.Expired = time.Now().UTC().Add(time.Duration(msg.Expired)).Unix()
	}

	c := Users.Get(msg.To_id)
	if c == nil {
		msg.Typ = OFFLINE
		// Get the offline msg id
		err = store.Manager.InsertOfflineMsg(msg)
		return
	}

	c.lock.Lock()
	if len(c.onlines) == Conf.MaxCacheMsg {
		c.lock.Unlock()
		msg.Typ = OFFLINE
		err = store.Manager.InsertOfflineMsg(msg)
		return
	} else {
		c.lock.Unlock()
	}
	c.lock.Lock()
	if c.isStop {
		c.lock.Unlock()
		msg.Typ = OFFLINE
		err = store.Manager.InsertOfflineMsg(msg)
	} else {
		msg.Typ = ONLINE
		c.onlines <- msg
		c.lock.Unlock()
	}
	return
}

func listenRPC() {
	comet := new(Comet_RPC)
	if err := rpc.Register(comet); err != nil {
		glog.Fatal(err)
	}
	rpc.HandleHTTP()
	l, err := net.Listen("tcp", Conf.RPC_addr)
	if err != nil {
		glog.Fatal(err)
	}
	if err = http.Serve(l, nil); err != nil {
		glog.Error(err)
	}
}
<commit_msg>comet: add RPC ping method<commit_after>// Copyright © 2014 Alienero. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"errors"
	"net"
	"net/http"
	"net/rpc"
	"time"

	myrpc "github.com/Alienero/quick-know/rpc"
	"github.com/Alienero/quick-know/store"
	"github.com/Alienero/quick-know/store/define"

	"github.com/golang/glog"
)

type Comet_RPC struct {
}

func (*Comet_RPC) Relogin(id string, r *myrpc.Reply) error {
	c := Users.Get(id)
	if c == nil {
		// r.IsRe = true
	} else {
		c.lock.Lock()
		if !c.isLetClose {
			c.isLetClose = true
			c.lock.Unlock()
			select {
			case c.CloseChan <- 1:
				<-c.CloseChan
				r.IsOk = true
				glog.Info("RPC: Ok will be relogin.")
			case <-time.After(2 * time.Second):
				// Timeout.
				if c := Users.Get(id); c != nil {
					return errors.New("Close the logon user timeout")
				}
				// Has been esc.
			}
		} else {
			c.lock.Unlock()
		}
	}
	return nil
}

func (*Comet_RPC) WriteOnlineMsg(msg *define.Msg, r *myrpc.Reply) (err error) {
	defer func() {
		if err == nil {
			r.IsOk = true
		}
	}()
	glog.Infof("Get a Write msg RPC,msg is :%v", string(msg.Body))
	msg.Dup = 0
	// fix the Expired
	if msg.Expired > 0 {
		msg.Expired = time.Now().UTC().Add(time.Duration(msg.Expired)).Unix()
	}

	c := Users.Get(msg.To_id)
	if c == nil {
		msg.Typ = OFFLINE
		// Get the offline msg id
		err = store.Manager.InsertOfflineMsg(msg)
		return
	}

	c.lock.Lock()
	if len(c.onlines) == Conf.MaxCacheMsg {
		c.lock.Unlock()
		msg.Typ = OFFLINE
		err = store.Manager.InsertOfflineMsg(msg)
		return
	} else {
		c.lock.Unlock()
	}
	c.lock.Lock()
	if c.isStop {
		c.lock.Unlock()
		msg.Typ = OFFLINE
		err = store.Manager.InsertOfflineMsg(msg)
	} else {
		msg.Typ = ONLINE
		c.onlines <- msg
		c.lock.Unlock()
	}
	return
}

func (*Comet_RPC) Ping(total *int, r *myrpc.Reply) (err error) {
	r.IsOk = true
	total = len(Users)
}

func listenRPC() {
	comet := new(Comet_RPC)
	if err := rpc.Register(comet); err != nil {
		glog.Fatal(err)
	}
	rpc.HandleHTTP()
	l, err := net.Listen("tcp", Conf.RPC_addr)
	if err != nil {
		glog.Fatal(err)
	}
	if err = http.Serve(l, nil); err != nil {
		glog.Error(err)
	}
}
<|endoftext|>


<commit_before>package github

import (
	"context"
	"errors"
	"fmt"
	"log"
	"path/filepath"
	"strings"
	"sync"

	"github.com/google/go-github/v32/github"

	"github.com/reviewdog/reviewdog"
	"github.com/reviewdog/reviewdog/cienv"
	"github.com/reviewdog/reviewdog/proto/rdf"
	"github.com/reviewdog/reviewdog/service/commentutil"
	"github.com/reviewdog/reviewdog/service/github/githubutils"
	"github.com/reviewdog/reviewdog/service/serviceutil"
)

var _ reviewdog.CommentService = &GitHubPullRequest{}
var _ reviewdog.DiffService = &GitHubPullRequest{}

const maxCommentsPerRequest = 30

// GitHubPullRequest is a comment and diff service for GitHub PullRequest.
//
// API:
//	https://developer.github.com/v3/pulls/comments/#create-a-comment
//	POST /repos/:owner/:repo/pulls/:number/comments
type GitHubPullRequest struct {
	cli   *github.Client
	owner string
	repo  string
	pr    int
	sha   string

	muComments   sync.Mutex
	postComments []*reviewdog.Comment

	postedcs commentutil.PostedComments

	// wd is working directory relative to root of repository.
	wd string
}

// NewGitHubPullRequest returns a new GitHubPullRequest service.
// GitHubPullRequest service needs git command in $PATH.
func NewGitHubPullRequest(cli *github.Client, owner, repo string, pr int, sha string) (*GitHubPullRequest, error) {
	workDir, err := serviceutil.GitRelWorkdir()
	if err != nil {
		return nil, fmt.Errorf("GitHubPullRequest needs 'git' command: %w", err)
	}
	return &GitHubPullRequest{
		cli:   cli,
		owner: owner,
		repo:  repo,
		pr:    pr,
		sha:   sha,
		wd:    workDir,
	}, nil
}

// Post accepts a comment and holds it. Flush method actually posts comments to
// GitHub in parallel.
func (g *GitHubPullRequest) Post(_ context.Context, c *reviewdog.Comment) error {
	c.Result.Diagnostic.GetLocation().Path = filepath.ToSlash(filepath.Join(g.wd,
		c.Result.Diagnostic.GetLocation().GetPath()))
	g.muComments.Lock()
	defer g.muComments.Unlock()
	g.postComments = append(g.postComments, c)
	return nil
}

// Flush posts comments which has not been posted yet.
func (g *GitHubPullRequest) Flush(ctx context.Context) error {
	g.muComments.Lock()
	defer g.muComments.Unlock()

	if err := g.setPostedComment(ctx); err != nil {
		return err
	}
	return g.postAsReviewComment(ctx)
}

func (g *GitHubPullRequest) postAsReviewComment(ctx context.Context) error {
	comments := make([]*github.DraftReviewComment, 0, len(g.postComments))
	remaining := make([]*reviewdog.Comment, 0)
	for _, c := range g.postComments {
		if !c.Result.InDiffContext {
			// GitHub Review API cannot report results outside diff. If it's running
			// in GitHub Actions, fallback to GitHub Actions log as report .
			if cienv.IsInGitHubAction() {
				githubutils.ReportAsGitHubActionsLog(c.ToolName, "warning", c.Result.Diagnostic)
			}
			continue
		}
		body := buildBody(c)
		if g.postedcs.IsPosted(c, githubCommentLine(c), body) {
			continue
		}
		// Only posts maxCommentsPerRequest comments per 1 request to avoid spammy
		// review comments. An example GitHub error if we don't limit the # of
		// review comments.
		//
		// > 403 You have triggered an abuse detection mechanism and have been
		// > temporarily blocked from content creation. Please retry your request
		// > again later.
		// https://developer.github.com/v3/#abuse-rate-limits
		if len(comments) >= maxCommentsPerRequest {
			remaining = append(remaining, c)
			continue
		}
		comments = append(comments, buildDraftReviewComment(c, body))
	}

	if len(comments) == 0 {
		return nil
	}

	review := &github.PullRequestReviewRequest{
		CommitID: &g.sha,
		Event:    github.String("COMMENT"),
		Comments: comments,
		Body:     github.String(g.remainingCommentsSummary(remaining)),
	}
	_, _, err := g.cli.PullRequests.CreateReview(ctx, g.owner, g.repo, g.pr, review)
	return err
}

// Document: https://docs.github.com/en/rest/reference/pulls#create-a-review-comment-for-a-pull-request
func buildDraftReviewComment(c *reviewdog.Comment, body string) *github.DraftReviewComment {
	loc := c.Result.Diagnostic.GetLocation()
	line := githubCommentLine(c)
	r := &github.DraftReviewComment{
		Path: github.String(loc.GetPath()),
		Side: github.String("RIGHT"),
		Body: github.String(body),
		Line: github.Int(line),
	}
	// GitHub API: Start line must precede the end line.
	if startLine := int(loc.GetRange().GetStart().GetLine()); startLine < line {
		r.StartSide = github.String("RIGHT")
		r.StartLine = github.Int(startLine)
	}
	return r
}

// line represents end line if it's a multiline comment in GitHub, otherwise
// it's start line.
// Document: https://docs.github.com/en/rest/reference/pulls#create-a-review-comment-for-a-pull-request
func githubCommentLine(c *reviewdog.Comment) int {
	loc := c.Result.Diagnostic.GetLocation()
	line := loc.GetRange().GetEnd().GetLine()
	// End position with column == 1 means range to the end of the previous lines
	// including line-break.
	if loc.GetRange().GetEnd().GetColumn() == 1 {
		line--
	}
	if line == 0 {
		line = loc.GetRange().GetStart().GetLine()
	}
	return int(line)
}

func (g *GitHubPullRequest) remainingCommentsSummary(remaining []*reviewdog.Comment) string {
	if len(remaining) == 0 {
		return ""
	}
	perTool := make(map[string][]*reviewdog.Comment)
	for _, c := range remaining {
		perTool[c.ToolName] = append(perTool[c.ToolName], c)
	}
	var sb strings.Builder
	sb.WriteString("Remaining comments which cannot be posted as a review comment to avoid GitHub Rate Limit\n")
	sb.WriteString("\n")
	for tool, comments := range perTool {
		sb.WriteString("\n")
		sb.WriteString(fmt.Sprintf("%s\n", tool))
		sb.WriteString("\n")
		for _, c := range comments {
			sb.WriteString(githubutils.LinkedMarkdownDiagnostic(g.owner, g.repo, g.sha, c.Result.Diagnostic))
			sb.WriteString("\n")
		}
		sb.WriteString("\n")
	}
	return sb.String()
}

func (g *GitHubPullRequest) setPostedComment(ctx context.Context) error {
	g.postedcs = make(commentutil.PostedComments)
	cs, err := g.comment(ctx)
	if err != nil {
		return err
	}
	for _, c := range cs {
		if c.Line == nil || c.Path == nil || c.Body == nil {
			continue
		}
		g.postedcs.AddPostedComment(c.GetPath(), c.GetLine(), c.GetBody())
	}
	return nil
}

// Diff returns a diff of PullRequest.
func (g *GitHubPullRequest) Diff(ctx context.Context) ([]byte, error) {
	opt := github.RawOptions{Type: github.Diff}
	d, _, err := g.cli.PullRequests.GetRaw(ctx, g.owner, g.repo, g.pr, opt)
	if err != nil {
		return nil, err
	}
	return []byte(d), nil
}

// Strip returns 1 as a strip of git diff.
func (g *GitHubPullRequest) Strip() int {
	return 1
}

func (g *GitHubPullRequest) comment(ctx context.Context) ([]*github.PullRequestComment, error) {
	// https://developer.github.com/v3/guides/traversing-with-pagination/
	opts := &github.PullRequestListCommentsOptions{
		ListOptions: github.ListOptions{
			PerPage: 100,
		},
	}
	comments, err := listAllPullRequestsComments(ctx, g.cli, g.owner, g.repo, g.pr, opts)
	if err != nil {
		return nil, err
	}
	return comments, nil
}

func listAllPullRequestsComments(ctx context.Context, cli *github.Client,
	owner, repo string, pr int, opts *github.PullRequestListCommentsOptions) ([]*github.PullRequestComment, error) {
	comments, resp, err := cli.PullRequests.ListComments(ctx, owner, repo, pr, opts)
	if err != nil {
		return nil, err
	}
	if resp.NextPage == 0 {
		return comments, nil
	}
	newOpts := &github.PullRequestListCommentsOptions{
		ListOptions: github.ListOptions{
			Page:    resp.NextPage,
			PerPage: opts.PerPage,
		},
	}
	restComments, err := listAllPullRequestsComments(ctx, cli, owner, repo, pr, newOpts)
	if err != nil {
		return nil, err
	}
	return append(comments, restComments...), nil
}

func buildBody(c *reviewdog.Comment) string {
	cbody := commentutil.CommentBody(c)
	if suggestion := buildSuggestions(c); suggestion != "" {
		cbody += "\n" + suggestion
	}
	return cbody
}

func buildSuggestions(c *reviewdog.Comment) string {
	var sb strings.Builder
	for _, s := range c.Result.Diagnostic.GetSuggestions() {
		txt, err := buildSingleSuggestion(c, s)
		if err != nil {
			log.Printf("reviewdog: %v", err)
			continue
		}
		sb.WriteString(txt)
		sb.WriteString("\n")
	}
	return sb.String()
}

func buildSingleSuggestion(c *reviewdog.Comment, s *rdf.Suggestion) (string, error) {
	start := s.GetRange().GetStart()
	if start.GetLine() != c.Result.Diagnostic.GetLocation().GetRange().GetStart().GetLine() {
		return "", errors.New("Diagnostic and Suggestion lines must be the same.")
	}
	end := s.GetRange().GetEnd()
	if !(end.GetLine() == 0 ||
		(start.GetLine() == end.GetLine() && end.GetColumn() == 0) ||
		(start.GetLine() == end.GetLine()+1 && end.GetColumn() == 1)) {
		// It must be a suggestion for a single line change due to GitHub API
		// restriction. Create a review for a pull request API [1] doesn't support
		// comments to multi lines as of writing (2020-07-21).
		// [1]: https://docs.github.com/en/rest/reference/pulls#create-a-review-for-a-pull-request
		return "", errors.New("non single line")
	}
	if start.GetColumn() > 1 {
		// TODO(haya14busa): Support non-line based suggestion.
		return "", errors.New("non line based")
	}
	var sb strings.Builder
	sb.WriteString("```suggestion\n")
	if txt := s.GetText(); txt != "" {
		sb.WriteString(txt)
		sb.WriteString("\n")
	}
	sb.WriteString("```")
	return sb.String(), nil
}
<commit_msg>github-pr-review: support multiline suggestion<commit_after>package github

import (
	"context"
	"errors"
	"fmt"
	"log"
	"path/filepath"
	"strings"
	"sync"

	"github.com/google/go-github/v32/github"

	"github.com/reviewdog/reviewdog"
	"github.com/reviewdog/reviewdog/cienv"
	"github.com/reviewdog/reviewdog/proto/rdf"
	"github.com/reviewdog/reviewdog/service/commentutil"
	"github.com/reviewdog/reviewdog/service/github/githubutils"
	"github.com/reviewdog/reviewdog/service/serviceutil"
)

var _ reviewdog.CommentService = &GitHubPullRequest{}
var _ reviewdog.DiffService = &GitHubPullRequest{}

const maxCommentsPerRequest = 30

// GitHubPullRequest is a comment and diff service for GitHub PullRequest.
//
// API:
//	https://developer.github.com/v3/pulls/comments/#create-a-comment
//	POST /repos/:owner/:repo/pulls/:number/comments
type GitHubPullRequest struct {
	cli   *github.Client
	owner string
	repo  string
	pr    int
	sha   string

	muComments   sync.Mutex
	postComments []*reviewdog.Comment

	postedcs commentutil.PostedComments

	// wd is working directory relative to root of repository.
	wd string
}

// NewGitHubPullRequest returns a new GitHubPullRequest service.
// GitHubPullRequest service needs git command in $PATH.
func NewGitHubPullRequest(cli *github.Client, owner, repo string, pr int, sha string) (*GitHubPullRequest, error) {
	workDir, err := serviceutil.GitRelWorkdir()
	if err != nil {
		return nil, fmt.Errorf("GitHubPullRequest needs 'git' command: %w", err)
	}
	return &GitHubPullRequest{
		cli:   cli,
		owner: owner,
		repo:  repo,
		pr:    pr,
		sha:   sha,
		wd:    workDir,
	}, nil
}

// Post accepts a comment and holds it. Flush method actually posts comments to
// GitHub in parallel.
func (g *GitHubPullRequest) Post(_ context.Context, c *reviewdog.Comment) error {
	c.Result.Diagnostic.GetLocation().Path = filepath.ToSlash(filepath.Join(g.wd,
		c.Result.Diagnostic.GetLocation().GetPath()))
	g.muComments.Lock()
	defer g.muComments.Unlock()
	g.postComments = append(g.postComments, c)
	return nil
}

// Flush posts comments which has not been posted yet.
func (g *GitHubPullRequest) Flush(ctx context.Context) error {
	g.muComments.Lock()
	defer g.muComments.Unlock()

	if err := g.setPostedComment(ctx); err != nil {
		return err
	}
	return g.postAsReviewComment(ctx)
}

func (g *GitHubPullRequest) postAsReviewComment(ctx context.Context) error {
	comments := make([]*github.DraftReviewComment, 0, len(g.postComments))
	remaining := make([]*reviewdog.Comment, 0)
	for _, c := range g.postComments {
		if !c.Result.InDiffContext {
			// GitHub Review API cannot report results outside diff. If it's running
			// in GitHub Actions, fallback to GitHub Actions log as report .
			if cienv.IsInGitHubAction() {
				githubutils.ReportAsGitHubActionsLog(c.ToolName, "warning", c.Result.Diagnostic)
			}
			continue
		}
		body := buildBody(c)
		if g.postedcs.IsPosted(c, githubCommentLine(c), body) {
			continue
		}
		// Only posts maxCommentsPerRequest comments per 1 request to avoid spammy
		// review comments. An example GitHub error if we don't limit the # of
		// review comments.
		//
		// > 403 You have triggered an abuse detection mechanism and have been
		// > temporarily blocked from content creation. Please retry your request
		// > again later.
		// https://developer.github.com/v3/#abuse-rate-limits
		if len(comments) >= maxCommentsPerRequest {
			remaining = append(remaining, c)
			continue
		}
		comments = append(comments, buildDraftReviewComment(c, body))
	}

	if len(comments) == 0 {
		return nil
	}

	review := &github.PullRequestReviewRequest{
		CommitID: &g.sha,
		Event:    github.String("COMMENT"),
		Comments: comments,
		Body:     github.String(g.remainingCommentsSummary(remaining)),
	}
	_, _, err := g.cli.PullRequests.CreateReview(ctx, g.owner, g.repo, g.pr, review)
	return err
}

// Document: https://docs.github.com/en/rest/reference/pulls#create-a-review-comment-for-a-pull-request
func buildDraftReviewComment(c *reviewdog.Comment, body string) *github.DraftReviewComment {
	loc := c.Result.Diagnostic.GetLocation()
	line := githubCommentLine(c)
	r := &github.DraftReviewComment{
		Path: github.String(loc.GetPath()),
		Side: github.String("RIGHT"),
		Body: github.String(body),
		Line: github.Int(line),
	}
	// GitHub API: Start line must precede the end line.
	if startLine := int(loc.GetRange().GetStart().GetLine()); startLine < line {
		r.StartSide = github.String("RIGHT")
		r.StartLine = github.Int(startLine)
	}
	return r
}

// line represents end line if it's a multiline comment in GitHub, otherwise
// it's start line.
// Document: https://docs.github.com/en/rest/reference/pulls#create-a-review-comment-for-a-pull-request
func githubCommentLine(c *reviewdog.Comment) int {
	loc := c.Result.Diagnostic.GetLocation()
	line := loc.GetRange().GetEnd().GetLine()
	// End position with column == 1 means range to the end of the previous lines
	// including line-break.
	if loc.GetRange().GetEnd().GetColumn() == 1 {
		line--
	}
	if line == 0 {
		line = loc.GetRange().GetStart().GetLine()
	}
	return int(line)
}

func (g *GitHubPullRequest) remainingCommentsSummary(remaining []*reviewdog.Comment) string {
	if len(remaining) == 0 {
		return ""
	}
	perTool := make(map[string][]*reviewdog.Comment)
	for _, c := range remaining {
		perTool[c.ToolName] = append(perTool[c.ToolName], c)
	}
	var sb strings.Builder
	sb.WriteString("Remaining comments which cannot be posted as a review comment to avoid GitHub Rate Limit\n")
	sb.WriteString("\n")
	for tool, comments := range perTool {
		sb.WriteString("\n")
		sb.WriteString(fmt.Sprintf("%s\n", tool))
		sb.WriteString("\n")
		for _, c := range comments {
			sb.WriteString(githubutils.LinkedMarkdownDiagnostic(g.owner, g.repo, g.sha, c.Result.Diagnostic))
			sb.WriteString("\n")
		}
		sb.WriteString("\n")
	}
	return sb.String()
}

func (g *GitHubPullRequest) setPostedComment(ctx context.Context) error {
	g.postedcs = make(commentutil.PostedComments)
	cs, err := g.comment(ctx)
	if err != nil {
		return err
	}
	for _, c := range cs {
		if c.Line == nil || c.Path == nil || c.Body == nil {
			continue
		}
		g.postedcs.AddPostedComment(c.GetPath(), c.GetLine(), c.GetBody())
	}
	return nil
}

// Diff returns a diff of PullRequest.
func (g *GitHubPullRequest) Diff(ctx context.Context) ([]byte, error) {
	opt := github.RawOptions{Type: github.Diff}
	d, _, err := g.cli.PullRequests.GetRaw(ctx, g.owner, g.repo, g.pr, opt)
	if err != nil {
		return nil, err
	}
	return []byte(d), nil
}

// Strip returns 1 as a strip of git diff.
func (g *GitHubPullRequest) Strip() int {
	return 1
}

func (g *GitHubPullRequest) comment(ctx context.Context) ([]*github.PullRequestComment, error) {
	// https://developer.github.com/v3/guides/traversing-with-pagination/
	opts := &github.PullRequestListCommentsOptions{
		ListOptions: github.ListOptions{
			PerPage: 100,
		},
	}
	comments, err := listAllPullRequestsComments(ctx, g.cli, g.owner, g.repo, g.pr, opts)
	if err != nil {
		return nil, err
	}
	return comments, nil
}

func listAllPullRequestsComments(ctx context.Context, cli *github.Client,
	owner, repo string, pr int, opts *github.PullRequestListCommentsOptions) ([]*github.PullRequestComment, error) {
	comments, resp, err := cli.PullRequests.ListComments(ctx, owner, repo, pr, opts)
	if err != nil {
		return nil, err
	}
	if resp.NextPage == 0 {
		return comments, nil
	}
	newOpts := &github.PullRequestListCommentsOptions{
		ListOptions: github.ListOptions{
			Page:    resp.NextPage,
			PerPage: opts.PerPage,
		},
	}
	restComments, err := listAllPullRequestsComments(ctx, cli, owner, repo, pr, newOpts)
	if err != nil {
		return nil, err
	}
	return append(comments, restComments...), nil
}

func buildBody(c *reviewdog.Comment) string {
	cbody := commentutil.CommentBody(c)
	if suggestion := buildSuggestions(c); suggestion != "" {
		cbody += "\n" + suggestion
	}
	return cbody
}

func buildSuggestions(c *reviewdog.Comment) string {
	var sb strings.Builder
	for _, s := range c.Result.Diagnostic.GetSuggestions() {
		txt, err := buildSingleSuggestion(c, s)
		if err != nil {
			log.Printf("reviewdog: %v", err)
			continue
		}
		sb.WriteString(txt)
		sb.WriteString("\n")
	}
	return sb.String()
}

func buildSingleSuggestion(c *reviewdog.Comment, s *rdf.Suggestion) (string, error) {
	start := s.GetRange().GetStart()
	if start.GetLine() != c.Result.Diagnostic.GetLocation().GetRange().GetStart().GetLine() {
		return "", errors.New("Diagnostic and Suggestion lines must be the same.")
	}
	if start.GetColumn() > 1 || s.GetRange().GetEnd().GetColumn() > 1 {
		// TODO(haya14busa): Support non-line based suggestion.
		return "", errors.New("non line based")
	}
	var sb strings.Builder
	sb.WriteString("```suggestion\n")
	if txt := s.GetText(); txt != "" {
		sb.WriteString(txt)
		sb.WriteString("\n")
	}
	sb.WriteString("```")
	return sb.String(), nil
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package collector

import (
	"time"

	"github.com/blevesearch/bleve/index"
	"github.com/blevesearch/bleve/search"
	"golang.org/x/net/context"
)

// PreAllocSizeSkipCap will cap preallocation to this amount when
// size+skip exceeds this value
var PreAllocSizeSkipCap = 1000

type collectorCompare func(i, j *search.DocumentMatch) int

type collectorFixup func(d *search.DocumentMatch) error

// TopNCollector collects the top N hits, optionally skipping some results
type TopNCollector struct {
	size          int
	skip          int
	total         uint64
	maxScore      float64
	took          time.Duration
	sort          search.SortOrder
	results       search.DocumentMatchCollection
	facetsBuilder *search.FacetsBuilder

	store *collectStoreSlice

	needDocIds    bool
	neededFields  []string
	cachedScoring []bool
	cachedDesc    []bool

	lowestMatchOutsideResults *search.DocumentMatch
}

// CheckDoneEvery controls how frequently we check the context deadline
const CheckDoneEvery = uint64(1024)

// NewTopNCollector builds a collector to find the top 'size' hits
// skipping over the first 'skip' hits
// ordering hits by the provided sort order
func NewTopNCollector(size int, skip int, sort search.SortOrder) *TopNCollector {
	hc := &TopNCollector{size: size, skip: skip, sort: sort}

	// pre-allocate space on the store to avoid reslicing
	// unless the size + skip is too large, then cap it
	// everything should still work, just reslices as necessary
	backingSize := size + skip + 1
	if size+skip > PreAllocSizeSkipCap {
		backingSize = PreAllocSizeSkipCap + 1
	}

	hc.store = newStoreSlice(backingSize, func(i, j *search.DocumentMatch) int {
		return hc.sort.Compare(hc.cachedScoring, hc.cachedDesc, i, j)
	})

	// these lookups traverse an interface, so do once up-front
	if sort.RequiresDocID() {
		hc.needDocIds = true
	}
	hc.neededFields = sort.RequiredFields()
	hc.cachedScoring = sort.CacheIsScore()
	hc.cachedDesc = sort.CacheDescending()

	return hc
}

// Collect goes to the index to find the matching documents
func (hc *TopNCollector) Collect(ctx context.Context, searcher search.Searcher, reader index.IndexReader) error {
	startTime := time.Now()
	var err error
	var next *search.DocumentMatch

	// pre-allocate enough space in the DocumentMatchPool
	// unless the size + skip is too large, then cap it
	// everything should still work, just allocates DocumentMatches on demand
	backingSize := hc.size + hc.skip + 1
	if hc.size+hc.skip > PreAllocSizeSkipCap {
		backingSize = PreAllocSizeSkipCap + 1
	}
	searchContext := &search.SearchContext{
		DocumentMatchPool: search.NewDocumentMatchPool(backingSize+searcher.DocumentMatchPoolSize(), len(hc.sort)),
	}

	select {
	case <-ctx.Done():
		return ctx.Err()
	default:
		next, err = searcher.Next(searchContext)
	}
	for err == nil && next != nil {
		if hc.total%CheckDoneEvery == 0 {
			select {
			case <-ctx.Done():
				return ctx.Err()
			default:
			}
		}

		err = hc.collectSingle(searchContext, reader, next)
		if err != nil {
			break
		}

		next, err = searcher.Next(searchContext)
	}
	// compute search duration
	hc.took = time.Since(startTime)
	if err != nil {
		return err
	}
	// finalize actual results
	err = hc.finalizeResults(reader)
	if err != nil {
		return err
	}
	return nil
}

var sortByScoreOpt = []string{"_score"}

func (hc *TopNCollector) collectSingle(ctx *search.SearchContext, reader index.IndexReader, d *search.DocumentMatch) error {
	var err error

	// visit field terms for features that require it (sort, facets)
	if len(hc.neededFields) > 0 {
		err = hc.visitFieldTerms(reader, d)
		if err != nil {
			return err
		}
	}

	// increment total hits
	hc.total++
	d.HitNumber = hc.total

	// update max score
	if d.Score > hc.maxScore {
		hc.maxScore = d.Score
	}

	// see if we need to load ID (at this early stage, for example to sort on it)
	if hc.needDocIds {
		d.ID, err = reader.ExternalID(d.IndexInternalID)
		if err != nil {
			return err
		}
	}

	// compute this hits sort value
	if len(hc.sort) == 1 && hc.cachedScoring[0] {
		d.Sort = sortByScoreOpt
	} else {
		hc.sort.Value(d)
	}

	// optimization, we track lowest sorting hit already removed from heap
	// with this one comparison, we can avoid all heap operations if
	// this hit would have been added and then immediately removed
	if hc.lowestMatchOutsideResults != nil {
		cmp := hc.sort.Compare(hc.cachedScoring, hc.cachedDesc, d, hc.lowestMatchOutsideResults)
		if cmp >= 0 {
			// this hit can't possibly be in the result set, so avoid heap ops
			ctx.DocumentMatchPool.Put(d)
			return nil
		}
	}

	hc.store.Add(d)
	if hc.store.Len() > hc.size+hc.skip {
		removed := hc.store.RemoveLast()
		if hc.lowestMatchOutsideResults == nil {
			hc.lowestMatchOutsideResults = removed
		} else {
			cmp := hc.sort.Compare(hc.cachedScoring, hc.cachedDesc, removed, hc.lowestMatchOutsideResults)
			if cmp < 0 {
				tmp := hc.lowestMatchOutsideResults
				hc.lowestMatchOutsideResults = removed
				ctx.DocumentMatchPool.Put(tmp)
			}
		}
	}

	return nil
}

// visitFieldTerms is responsible for visiting the field terms of the
// search hit, and passing visited terms to the sort and facet builder
func (hc *TopNCollector) visitFieldTerms(reader index.IndexReader, d *search.DocumentMatch) error {
	if hc.facetsBuilder != nil {
		hc.facetsBuilder.StartDoc()
	}

	err := reader.DocumentVisitFieldTerms(d.IndexInternalID, hc.neededFields, func(field string, term []byte) {
		if hc.facetsBuilder != nil {
			hc.facetsBuilder.UpdateVisitor(field, term)
		}
		hc.sort.UpdateVisitor(field, term)
	})

	if hc.facetsBuilder != nil {
		hc.facetsBuilder.EndDoc()
	}

	return err
}

// SetFacetsBuilder registers a facet builder for this collector
func (hc *TopNCollector) SetFacetsBuilder(facetsBuilder *search.FacetsBuilder) {
	hc.facetsBuilder = facetsBuilder
	hc.neededFields = append(hc.neededFields, hc.facetsBuilder.RequiredFields()...)
}

// finalizeResults starts with the heap containing the final top size+skip
// it now throws away the results to be skipped
// and does final doc id lookup (if necessary)
func (hc *TopNCollector) finalizeResults(r index.IndexReader) error {
	var err error
	hc.results, err = hc.store.Final(hc.skip, func(doc *search.DocumentMatch) error {
		if doc.ID == "" {
			// look up the id since we need it for lookup
			var err error
			doc.ID, err = r.ExternalID(doc.IndexInternalID)
			if err != nil {
				return err
			}
		}
		return nil
	})

	return err
}

// Results returns the collected hits
func (hc *TopNCollector) Results() search.DocumentMatchCollection {
	return hc.results
}

// Total returns the total number of hits
func (hc *TopNCollector) Total() uint64 {
	return hc.total
}

// MaxScore returns the maximum score seen across all the hits
func (hc *TopNCollector) MaxScore() float64 {
	return hc.maxScore
}

// Took returns the time spent collecting hits
func (hc *TopNCollector) Took() time.Duration {
	return hc.took
}

// FacetResults returns the computed facets results
func (hc *TopNCollector) FacetResults() search.FacetResults {
	if hc.facetsBuilder != nil {
		return hc.facetsBuilder.Results()
	}
	return search.FacetResults{}
}
<commit_msg>switch collector store impl from slice to heap<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package collector

import (
	"time"

	"github.com/blevesearch/bleve/index"
	"github.com/blevesearch/bleve/search"
	"golang.org/x/net/context"
)

// PreAllocSizeSkipCap will cap preallocation to this amount when
// size+skip exceeds this value
var PreAllocSizeSkipCap = 1000

type collectorCompare func(i, j *search.DocumentMatch) int

type collectorFixup func(d *search.DocumentMatch) error

// TopNCollector collects the top N hits, optionally skipping some results
type TopNCollector struct {
	size          int
	skip          int
	total         uint64
	maxScore      float64
	took          time.Duration
	sort          search.SortOrder
	results       search.DocumentMatchCollection
	facetsBuilder *search.FacetsBuilder

	store *collectStoreHeap

	needDocIds    bool
	neededFields  []string
	cachedScoring []bool
	cachedDesc    []bool

	lowestMatchOutsideResults *search.DocumentMatch
}

// CheckDoneEvery controls how frequently we check the context deadline
const CheckDoneEvery = uint64(1024)

// NewTopNCollector builds a collector to find the top 'size' hits
// skipping over the first 'skip' hits
// ordering hits by the provided sort order
func NewTopNCollector(size int, skip int, sort search.SortOrder) *TopNCollector {
	hc := &TopNCollector{size: size, skip: skip, sort: sort}

	// pre-allocate space on the store to avoid reslicing
	// unless the size + skip is too large, then cap it
	// everything should still work, just reslices as necessary
	backingSize := size + skip + 1
	if size+skip > PreAllocSizeSkipCap {
		backingSize = PreAllocSizeSkipCap + 1
	}

	hc.store = newStoreHeap(backingSize, func(i, j *search.DocumentMatch) int {
		return hc.sort.Compare(hc.cachedScoring, hc.cachedDesc, i, j)
	})

	// these lookups traverse an interface, so do once up-front
	if sort.RequiresDocID() {
		hc.needDocIds = true
	}
	hc.neededFields = sort.RequiredFields()
	hc.cachedScoring = sort.CacheIsScore()
	hc.cachedDesc = sort.CacheDescending()

	return hc
}

// Collect goes to the index to find the matching documents
func (hc *TopNCollector) Collect(ctx context.Context, searcher search.Searcher, reader index.IndexReader) error {
	startTime := time.Now()
	var err error
	var next *search.DocumentMatch

	// pre-allocate enough space in the DocumentMatchPool
	// unless the size + skip is too large, then cap it
	// everything should still work, just allocates DocumentMatches on demand
	backingSize := hc.size + hc.skip + 1
	if hc.size+hc.skip > PreAllocSizeSkipCap {
		backingSize = PreAllocSizeSkipCap + 1
	}
	searchContext := &search.SearchContext{
		DocumentMatchPool: search.NewDocumentMatchPool(backingSize+searcher.DocumentMatchPoolSize(), len(hc.sort)),
	}

	select {
	case <-ctx.Done():
		return ctx.Err()
	default:
		next, err = searcher.Next(searchContext)
	}
	for err == nil && next != nil {
		if hc.total%CheckDoneEvery == 0 {
			select {
			case <-ctx.Done():
				return ctx.Err()
			default:
			}
		}

		err = hc.collectSingle(searchContext, reader, next)
		if err != nil {
			break
		}

		next, err = searcher.Next(searchContext)
	}
	// compute search duration
	hc.took = time.Since(startTime)
	if err != nil {
		return err
	}
	// finalize actual results
	err = hc.finalizeResults(reader)
	if err != nil {
		return err
	}
	return nil
}

var sortByScoreOpt = []string{"_score"}

func (hc *TopNCollector) collectSingle(ctx *search.SearchContext, reader index.IndexReader, d *search.DocumentMatch) error {
	var err error

	// visit field terms for features that require it (sort, facets)
	if len(hc.neededFields) > 0 {
		err = hc.visitFieldTerms(reader, d)
		if err != nil {
			return err
		}
	}

	// increment total hits
	hc.total++
	d.HitNumber = hc.total

	// update max score
	if d.Score > hc.maxScore {
		hc.maxScore = d.Score
	}

	// see if we need to load ID (at this early stage, for example to sort on it)
	if hc.needDocIds {
		d.ID, err = reader.ExternalID(d.IndexInternalID)
		if err != nil {
			return err
		}
	}

	// compute this hits sort value
	if len(hc.sort) == 1 && hc.cachedScoring[0] {
		d.Sort = sortByScoreOpt
	} else {
		hc.sort.Value(d)
	}

	// optimization, we track lowest sorting hit already removed from heap
	// with this one comparison, we can avoid all heap operations if
	// this hit would have been added and then immediately removed
	if hc.lowestMatchOutsideResults != nil {
		cmp := hc.sort.Compare(hc.cachedScoring, hc.cachedDesc, d, hc.lowestMatchOutsideResults)
		if cmp >= 0 {
			// this hit can't possibly be in the result set, so avoid heap ops
			ctx.DocumentMatchPool.Put(d)
			return nil
		}
	}

	hc.store.Add(d)
	if hc.store.Len() > hc.size+hc.skip {
		removed := hc.store.RemoveLast()
		if hc.lowestMatchOutsideResults == nil {
			hc.lowestMatchOutsideResults = removed
		} else {
			cmp := hc.sort.Compare(hc.cachedScoring, hc.cachedDesc, removed, hc.lowestMatchOutsideResults)
			if cmp < 0 {
				tmp := hc.lowestMatchOutsideResults
				hc.lowestMatchOutsideResults = removed
				ctx.DocumentMatchPool.Put(tmp)
			}
		}
	}

	return nil
}

// visitFieldTerms is responsible for visiting the field terms of the
// search hit, and passing visited terms to the sort and facet builder
func (hc *TopNCollector) visitFieldTerms(reader index.IndexReader, d *search.DocumentMatch) error {
	if hc.facetsBuilder != nil {
		hc.facetsBuilder.StartDoc()
	}

	err := reader.DocumentVisitFieldTerms(d.IndexInternalID, hc.neededFields, func(field string, term []byte) {
		if hc.facetsBuilder != nil {
			hc.facetsBuilder.UpdateVisitor(field, term)
		}
		hc.sort.UpdateVisitor(field, term)
	})

	if hc.facetsBuilder != nil {
		hc.facetsBuilder.EndDoc()
	}

	return err
}

// SetFacetsBuilder registers a facet builder for this collector
func (hc *TopNCollector) SetFacetsBuilder(facetsBuilder *search.FacetsBuilder) {
	hc.facetsBuilder = facetsBuilder
	hc.neededFields = append(hc.neededFields, hc.facetsBuilder.RequiredFields()...)
}

// finalizeResults starts with the heap containing the final top size+skip
// it now throws away the results to be skipped
// and does final doc id lookup (if necessary)
func (hc *TopNCollector) finalizeResults(r index.IndexReader) error {
	var err error
	hc.results, err = hc.store.Final(hc.skip, func(doc *search.DocumentMatch) error {
		if doc.ID == "" {
			// look up the id since we need it for lookup
			var err error
			doc.ID, err = r.ExternalID(doc.IndexInternalID)
			if err != nil {
				return err
			}
		}
		return nil
	})

	return err
}

// Results returns the collected hits
func (hc *TopNCollector) Results() search.DocumentMatchCollection {
	return hc.results
}

// Total returns the total number of hits
func (hc *TopNCollector) Total() uint64 {
	return hc.total
}

// MaxScore returns the maximum score seen across all the hits
func (hc *TopNCollector) MaxScore() float64 {
	return hc.maxScore
}

// Took returns the time spent collecting hits
func (hc *TopNCollector) Took() time.Duration {
	return hc.took
}

// FacetResults returns the computed facets results
func (hc *TopNCollector) FacetResults() search.FacetResults {
	if hc.facetsBuilder != nil {
		return hc.facetsBuilder.Results()
	}
	return search.FacetResults{}
}
<|endoftext|>


<commit_before>/*
 * This file is part of easyKV.
 * © 2022 The easyKV Authors
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

package nats

import (
	"context"
	"fmt"
	"strings"

	"github.com/HeavyHorst/easykv"
	"github.com/nats-io/nats.go"
)

var cleanReplacer = strings.NewReplacer(".", "/")

// Client provides a shell for the env client
type Client struct {
	nc *nats.Conn
	kv nats.KeyValue
}

// New returns a new client
func New(nodes []string, bucket string, opts ...Option) (*Client, error) {
	var options Options
	for _, o := range opts {
		o(&options)
	}

	if len(nodes) == 0 {
		nodes = append(nodes, nats.DefaultURL)
	}

	natsOptions := []nats.Option{nats.MaxReconnects(-1)}

	// override authentication, if any was specified
	if options.Auth.Username != "" && options.Auth.Password != "" {
		natsOptions = append(natsOptions, nats.UserInfo(options.Auth.Username, options.Auth.Password))
	}

	if options.Token != "" {
		natsOptions = append(natsOptions, nats.Token(options.Token))
	}

	if options.Creds != "" {
		natsOptions = append(natsOptions, nats.UserCredentials(options.Creds))
	}

	nc, err := nats.Connect(strings.Join(nodes, ","), natsOptions...)
	if err != nil {
		return nil, fmt.Errorf("could't connect to nats: %w", err)
	}

	js, err := nc.JetStream()
	if err != nil {
		return nil, fmt.Errorf("could't initialize jetstream: %w", err)
	}

	kv, err := js.KeyValue(bucket)
	if err != nil {
		return nil, fmt.Errorf("could't open kv bucket: %w", err)
	}

	return &Client{
		nc: nc,
		kv: kv,
	}, nil
}

// Close is only meant to fulfill the easykv.ReadWatcher interface.
// Does nothing.
func (c *Client) Close() {
	c.nc.Close()
}

func clean(key string) string {
	newKey := "/" + key
	return cleanReplacer.Replace(strings.ToLower(newKey))
}

// GetValues is used to lookup all keys with a prefix.
// Several prefixes can be specified in the keys array.
func (c *Client) GetValues(keys []string) (map[string]string, error) {
	allKeys, err := c.kv.Keys()
	if err != nil {
		return nil, fmt.Errorf("couldn't get keys: %w", err)
	}

	// filter keys
	var filteredKeys []string
	for _, key := range keys {
		for _, k := range allKeys {
			if strings.HasPrefix(clean(k), key) {
				filteredKeys = append(filteredKeys, k)
			}
		}
	}

	vars := make(map[string]string)
	for _, key := range filteredKeys {
		val, err := c.kv.Get(key)
		if err != nil {
			return nil, fmt.Errorf("couldn't get key: %v %w", key, err)
		}
		vars[clean(key)] = string(val.Value())
	}

	return vars, nil
}

// WatchPrefix
func (c *Client) WatchPrefix(ctx context.Context, prefix string, opts ...easykv.WatchOption) (uint64, error) {
	var (
		options easykv.WatchOptions
		watcher nats.KeyWatcher
		err     error
	)
	for _, o := range opts {
		o(&options)
	}

	watcher, err = c.kv.WatchAll(nats.Context(ctx))
	if err != nil {
		return 0, fmt.Errorf("couldn't create nats watcher: %w", err)
	}

	defer watcher.Stop()
	for v := range watcher.Updates() {
		if v == nil {
			break
		}
		for _, k := range options.Keys {
			if strings.HasPrefix(clean(string(v.Key())), k) {
				return v.Revision(), nil
			}
		}
	}

	if ctx.Err() == context.Canceled {
		return options.WaitIndex, easykv.ErrWatchCanceled
	}
	return 0, err
}
<commit_msg>don't transfer keys to lowercase<commit_after>/*
 * This file is part of easyKV.
 * © 2022 The easyKV Authors
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

package nats

import (
	"context"
	"fmt"
	"strings"

	"github.com/HeavyHorst/easykv"
	"github.com/nats-io/nats.go"
)

var cleanReplacer = strings.NewReplacer(".", "/")

// Client provides a shell for the env client
type Client struct {
	nc *nats.Conn
	kv nats.KeyValue
}

// New returns a new client
func New(nodes []string, bucket string, opts ...Option) (*Client, error) {
	var options Options
	for _, o := range opts {
		o(&options)
	}

	if len(nodes) == 0 {
		nodes = append(nodes, nats.DefaultURL)
	}

	natsOptions := []nats.Option{nats.MaxReconnects(-1)}

	// override authentication, if any was specified
	if options.Auth.Username != "" && options.Auth.Password != "" {
		natsOptions = append(natsOptions, nats.UserInfo(options.Auth.Username, options.Auth.Password))
	}

	if options.Token != "" {
		natsOptions = append(natsOptions, nats.Token(options.Token))
	}

	if options.Creds != "" {
		natsOptions = append(natsOptions, nats.UserCredentials(options.Creds))
	}

	nc, err := nats.Connect(strings.Join(nodes, ","), natsOptions...)
	if err != nil {
		return nil, fmt.Errorf("could't connect to nats: %w", err)
	}

	js, err := nc.JetStream()
	if err != nil {
		return nil, fmt.Errorf("could't initialize jetstream: %w", err)
	}

	kv, err := js.KeyValue(bucket)
	if err != nil {
		return nil, fmt.Errorf("could't open kv bucket: %w", err)
	}

	return &Client{
		nc: nc,
		kv: kv,
	}, nil
}

// Close is only meant to fulfill the easykv.ReadWatcher interface.
// Does nothing.
func (c *Client) Close() {
	c.nc.Close()
}

func clean(key string) string {
	newKey := "/" + key
	return cleanReplacer.Replace(newKey)
}

// GetValues is used to lookup all keys with a prefix.
// Several prefixes can be specified in the keys array.
func (c *Client) GetValues(keys []string) (map[string]string, error) {
	allKeys, err := c.kv.Keys()
	if err != nil {
		return nil, fmt.Errorf("couldn't get keys: %w", err)
	}

	// filter keys
	var filteredKeys []string
	for _, key := range keys {
		for _, k := range allKeys {
			if strings.HasPrefix(clean(k), key) {
				filteredKeys = append(filteredKeys, k)
			}
		}
	}

	vars := make(map[string]string)
	for _, key := range filteredKeys {
		val, err := c.kv.Get(key)
		if err != nil {
			return nil, fmt.Errorf("couldn't get key: %v %w", key, err)
		}
		vars[clean(key)] = string(val.Value())
	}

	return vars, nil
}

// WatchPrefix
func (c *Client) WatchPrefix(ctx context.Context, prefix string, opts ...easykv.WatchOption) (uint64, error) {
	var (
		options easykv.WatchOptions
		watcher nats.KeyWatcher
		err     error
	)
	for _, o := range opts {
		o(&options)
	}

	watcher, err = c.kv.WatchAll(nats.Context(ctx))
	if err != nil {
		return 0, fmt.Errorf("couldn't create nats watcher: %w", err)
	}

	defer watcher.Stop()
	for v := range watcher.Updates() {
		if v == nil {
			break
		}
		for _, k := range options.Keys {
			if strings.HasPrefix(clean(string(v.Key())), k) {
				return v.Revision(), nil
			}
		}
	}

	if ctx.Err() == context.Canceled {
		return options.WaitIndex, easykv.ErrWatchCanceled
	}
	return 0, err
}
<|endoftext|>


<commit_before>package tftp

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"math/rand"
	"net"
	"os"
	"testing"
)

var (
	c *Client
	s *Server
)

func TestMain(m *testing.M) {
	addr, _ := net.ResolveUDPAddr("udp", "localhost:12312")

	log := log.New(os.Stderr, "", log.Ldate|log.Ltime)

	s = &Server{addr, handleWrite, handleRead, log}
	go s.Serve()

	c = &Client{addr, log}

	os.Exit(m.Run())
}

func TestSmallWrites(t *testing.T) {
	filename := "small-writes"
	mode := "octet"
	bs := []byte("just write a tftpHandler that writes a file into the pipe one byte at a time")
	c.Put(filename, mode, func(writer *io.PipeWriter) {
		for i := 0; i < len(bs); i++ {
			writer.Write(bs[i : i+1])
		}
		writer.Close()
	})
	buf := new(bytes.Buffer)
	c.Get(filename, mode, func(reader *io.PipeReader) {
		buf.ReadFrom(reader)
	})
	if !bytes.Equal(bs, buf.Bytes()) {
		t.Fatalf("sent: %s, received: %s", string(bs), buf.String())
	}
}

func TestPutGet(t *testing.T) {
	testPutGet(t, "f1", []byte("foobar"), "octet")
	testPutGet(t, "f2", []byte("La sonda New Horizons, a quasi due mesidal passaggio ravvicinato su Plutone, sta iniziando a inviare una dose consistente di immagini ad alta risoluzione del pianeta nano. La Nasa ha diffuso le prime foto il 10 settembre, come questa della Cthulhu Regio, ripresa il 14 luglio da una distanza di 80 mila km. Un’area più scura accanto alla chiara Sputnik Planum."), "octet")
	for i := 500; i < 520; i++ {
		testPutGet(t, fmt.Sprintf("size-%d", i), randomByteArray(i), "octet")
	}
}

func randomByteArray(n int) []byte {
	bs := make([]byte, n)
	for i := 0; i < n; i++ {
		bs[i] = byte(rand.Int63() & 0xff)
	}
	return bs
}

func testPutGet(t *testing.T, filename string, bs []byte, mode string) {
	c.Put(filename, mode, func(writer *io.PipeWriter) {
		writer.Write(bs)
		writer.Close()
	})
	buf := new(bytes.Buffer)
	c.Get(filename, mode, func(reader *io.PipeReader) {
		buf.ReadFrom(reader)
	})
	if !bytes.Equal(bs, buf.Bytes()) {
		t.Fatalf("sent: %s, received: %s", string(bs), buf.String())
	}
}

var m = map[string][]byte{}

func handleWrite(filename string, r *io.PipeReader) {
	_, exists := m[filename]
	if exists {
		r.CloseWithError(fmt.Errorf("File already exists: %s", filename))
		return
	}
	buffer := &bytes.Buffer{}
	c, e := buffer.ReadFrom(r)
	if e != nil {
		fmt.Fprintf(os.Stderr, "Can't receive %s: %v\n", filename, e)
	} else {
		fmt.Fprintf(os.Stderr, "Received %s (%d bytes)\n", filename, c)
		m[filename] = buffer.Bytes()
	}
}

func handleRead(filename string, w *io.PipeWriter) {
	b, exists := m[filename]
	if exists {
		buffer := bytes.NewBuffer(b)
		c, e := buffer.WriteTo(w)
		if e != nil {
			fmt.Fprintf(os.Stderr, "Can't send %s: %v\n", filename, e)
		} else {
			fmt.Fprintf(os.Stderr, "Sent %s (%d bytes)\n", filename, c)
		}
		w.Close()
	} else {
		w.CloseWithError(fmt.Errorf("File not found: %s", filename))
	}
}
<commit_msg>brutal locking to avoid race in tests<commit_after>package tftp

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"math/rand"
	"net"
	"os"
	"sync"
	"testing"
)

var (
	c *Client
	s *Server
)

func TestMain(m *testing.M) {
	addr, _ := net.ResolveUDPAddr("udp", "localhost:12312")

	log := log.New(os.Stderr, "", log.Ldate|log.Ltime)

	s = &Server{addr, handleWrite, handleRead, log}
	go s.Serve()

	c = &Client{addr, log}

	os.Exit(m.Run())
}

func TestSmallWrites(t *testing.T) {
	filename := "small-writes"
	mode := "octet"
	bs := []byte("just write a tftpHandler that writes a file into the pipe one byte at a time")
	c.Put(filename, mode, func(writer *io.PipeWriter) {
		for i := 0; i < len(bs); i++ {
			writer.Write(bs[i : i+1])
		}
		writer.Close()
	})
	buf := new(bytes.Buffer)
	c.Get(filename, mode, func(reader *io.PipeReader) {
		buf.ReadFrom(reader)
	})
	if !bytes.Equal(bs, buf.Bytes()) {
		t.Fatalf("sent: %s, received: %s", string(bs), buf.String())
	}
}

func TestPutGet(t *testing.T) {
	testPutGet(t, "f1", []byte("foobar"), "octet")
	testPutGet(t, "f2", []byte("La sonda New Horizons, a quasi due mesidal passaggio ravvicinato su Plutone, sta iniziando a inviare una dose consistente di immagini ad alta risoluzione del pianeta nano. La Nasa ha diffuso le prime foto il 10 settembre, come questa della Cthulhu Regio, ripresa il 14 luglio da una distanza di 80 mila km. Un’area più scura accanto alla chiara Sputnik Planum."), "octet")
	for i := 500; i < 520; i++ {
		testPutGet(t, fmt.Sprintf("size-%d", i), randomByteArray(i), "octet")
	}
}

func randomByteArray(n int) []byte {
	bs := make([]byte, n)
	for i := 0; i < n; i++ {
		bs[i] = byte(rand.Int63() & 0xff)
	}
	return bs
}

func testPutGet(t *testing.T, filename string, bs []byte, mode string) {
	c.Put(filename, mode, func(writer *io.PipeWriter) {
		writer.Write(bs)
		writer.Close()
	})
	buf := new(bytes.Buffer)
	c.Get(filename, mode, func(reader *io.PipeReader) {
		buf.ReadFrom(reader)
	})
	if !bytes.Equal(bs, buf.Bytes()) {
		t.Fatalf("sent: %s, received: %s", string(bs), buf.String())
	}
}

var m = map[string][]byte{}
var mu sync.Mutex

func handleWrite(filename string, r *io.PipeReader) {
	mu.Lock()
	defer mu.Unlock()
	_, exists := m[filename]
	if exists {
		r.CloseWithError(fmt.Errorf("File already exists: %s", filename))
		return
	}
	buffer := &bytes.Buffer{}
	c, e := buffer.ReadFrom(r)
	if e != nil {
		fmt.Fprintf(os.Stderr, "Can't receive %s: %v\n", filename, e)
	} else {
		fmt.Fprintf(os.Stderr, "Received %s (%d bytes)\n", filename, c)
		m[filename] = buffer.Bytes()
	}
}

func handleRead(filename string, w *io.PipeWriter) {
	mu.Lock()
	defer mu.Unlock()
	b, exists := m[filename]
	if exists {
		buffer := bytes.NewBuffer(b)
		c, e := buffer.WriteTo(w)
		if e != nil {
			fmt.Fprintf(os.Stderr, "Can't send %s: %v\n", filename, e)
		} else {
			fmt.Fprintf(os.Stderr, "Sent %s (%d bytes)\n", filename, c)
		}
		w.Close()
	} else {
		w.CloseWithError(fmt.Errorf("File not found: %s", filename))
	}
}
<|endoftext|>


<commit_before>package jsoncodec

import (
	"github.com/flachnetz/dd-zipkin-proxy/cache"
	"github.com/openzipkin/zipkin-go-opentracing/thrift/gen-go/zipkincore"
)

type SpanV2 struct {
	TraceID  Id  `json:"traceId"`
	ID       Id  `json:"id"`
	ParentID *Id `json:"parentId"`

	Name string `json:"name"`

	Endpoint *Endpoint `json:"localEndpoint"`

	Tags map[string]string `json:"tags"`

	Kind      string `json:"kind"`
	Timestamp int64  `json:"timestamp"`
	Duration  int64  `json:"duration"`
}

func (span *SpanV2) ToZipkincoreSpan() *zipkincore.Span {
	var annotations []*zipkincore.Annotation

	endpoint := endpointToZipkin(span.Endpoint)

	var binaryAnnotations []*zipkincore.BinaryAnnotation
	for key, value := range span.Tags {
		binaryAnnotations = append(binaryAnnotations, &zipkincore.BinaryAnnotation{
			Key:            cache.String(key),
			Value:          toBytesCached(value),
			Host:           endpoint,
			AnnotationType: zipkincore.AnnotationType_STRING,
		})
	}

	// in root spans the traceId equals the span id.
	parentId := span.ParentID
	if span.TraceID == span.ID {
		parentId = nil
	}

	return &zipkincore.Span{
		TraceID: int64(span.TraceID),
		ID:      int64(span.ID),
		Name:    cache.String(span.Name),

		ParentID: (*int64)(parentId),

		Annotations:       annotations,
		BinaryAnnotations: binaryAnnotations,

		Timestamp: &span.Timestamp,
		Duration:  &span.Duration,
	}
}
<commit_msg>Fix v2 bug.<commit_after>package jsoncodec

import (
	"github.com/flachnetz/dd-zipkin-proxy/cache"
	"github.com/openzipkin/zipkin-go-opentracing/thrift/gen-go/zipkincore"
)

type SpanV2 struct {
	TraceID  Id  `json:"traceId"`
	ID       Id  `json:"id"`
	ParentID *Id `json:"parentId"`

	Name string `json:"name"`

	Endpoint *Endpoint `json:"localEndpoint"`

	Tags map[string]string `json:"tags"`

	Kind      string `json:"kind"`
	Timestamp int64  `json:"timestamp"`
	Duration  int64  `json:"duration"`
}

func (span *SpanV2) ToZipkincoreSpan() *zipkincore.Span {
	var annotations []*zipkincore.Annotation

	endpoint := endpointToZipkin(span.Endpoint)

	var binaryAnnotations []*zipkincore.BinaryAnnotation
	for key, value := range span.Tags {
		binaryAnnotations = append(binaryAnnotations, &zipkincore.BinaryAnnotation{
			Key:            cache.String(key),
			Value:          toBytesCached(value),
			Host:           endpoint,
			AnnotationType: zipkincore.AnnotationType_STRING,
		})
	}

	// in root spans the traceId equals the span id.
	parentId := span.ParentID
	if span.TraceID == span.ID {
		parentId = nil
	}

	times := [2]int64{span.Timestamp, span.Duration}

	return &zipkincore.Span{
		TraceID: int64(span.TraceID),
		ID:      int64(span.ID),
		Name:    cache.String(span.Name),

		ParentID: (*int64)(parentId),

		Annotations:       annotations,
		BinaryAnnotations: binaryAnnotations,

		Timestamp: &times[0],
		Duration:  &times[1],
	}
}
<|endoftext|>


<commit_before>package demand

import (
	"fmt"
	"sort"
)

// Returns whether tasks have all drained down to 0
func (tasks *Tasks) Exited() (done bool) {
	tasks.RLock()
	defer tasks.RUnlock()

	done = true
	for _, task := range tasks.Tasks {
		if task.Running > 0 {
			done = false
			log.Debugf("Waiting for %s, still %d running, %d requested", task.Name, task.Running, task.Requested)
		}
	}

	return done
}

// Returns number of containers we have space for
func (tasks *Tasks) CheckCapacity() int {
	// TODO!! For now we are simply going to say there is a maximum total number of containers this deployment can handle
	// TODO!! It should really look at the available CPU / mem / bw in / out
	totalRequested := 0
	for _, t := range tasks.Tasks {
		totalRequested += t.Requested
	}

	return tasks.MaxContainers - totalRequested
}

// implements sort.Interface tasks based on priority
type byPriority []*Task

func (p byPriority) Len() int           { return len(p) }
func (p byPriority) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p byPriority) Less(i, j int) bool { return p[i].Priority < p[j].Priority }

func (t *Tasks) PrioritySort(reverse bool) {
	if reverse {
		sort.Sort(sort.Reverse(byPriority(t.Tasks)))
	} else {
		sort.Sort(byPriority(t.Tasks))
	}
}

func (t *Tasks) GetTask(name string) (task *Task, err error) {
	for _, task := range t.Tasks {
		if task.Name == name {
			return task, nil
		}
	}

	return nil, fmt.Errorf("No Task with name %s", name)
}
<commit_msg>Since we’re updating Demand as we go through containers, this is what we should look at to see what the available capacity is<commit_after>package demand

import (
	"fmt"
	"sort"
)

// Returns whether tasks have all drained down to 0
func (tasks *Tasks) Exited() (done bool) {
	tasks.RLock()
	defer tasks.RUnlock()

	done = true
	for _, task := range tasks.Tasks {
		if task.Running > 0 {
			done = false
			log.Debugf("Waiting for %s, still %d running, %d requested", task.Name, task.Running, task.Requested)
		}
	}

	return done
}

// Returns number of containers we have space for
func (tasks *Tasks) CheckCapacity() int {
	// TODO!! For now we are simply going to say there is a maximum total number of containers this deployment can handle
	// TODO!! It should really look at the available CPU / mem / bw in / out
	totalDemand := 0
	for _, t := range tasks.Tasks {
		totalDemand += t.Demand
	}

	return tasks.MaxContainers - totalDemand
}

// implements sort.Interface tasks based on priority
type byPriority []*Task

func (p byPriority) Len() int           { return len(p) }
func (p byPriority) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p byPriority) Less(i, j int) bool { return p[i].Priority < p[j].Priority }

func (t *Tasks) PrioritySort(reverse bool) {
	if reverse {
		sort.Sort(sort.Reverse(byPriority(t.Tasks)))
	} else {
		sort.Sort(byPriority(t.Tasks))
	}
}

func (t *Tasks) GetTask(name string) (task *Task, err error) {
	for _, task := range t.Tasks {
		if task.Name == name {
			return task, nil
		}
	}

	return nil, fmt.Errorf("No Task with name %s", name)
}
<|endoftext|>


<commit_before>package aufs

import (
	"github.com/dotcloud/docker/archive"
	"os"
	"path"
	"testing"
)

var (
	tmp = path.Join(os.TempDir(), "aufs-tests", "aufs")
)

func newDriver(t *testing.T) *AufsDriver {
	if err := os.MkdirAll(tmp, 0755); err != nil {
		t.Fatal(err)
	}

	d, err := Init(tmp)
	if err != nil {
		t.Fatal(err)
	}
	return d.(*AufsDriver)
}

func TestNewAufsDriver(t *testing.T) {
	if err := os.MkdirAll(tmp, 0755); err != nil {
		t.Fatal(err)
	}

	d, err := Init(tmp)
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(tmp)
	if d == nil {
		t.Fatalf("Driver should not be nil")
	}
}

func TestAufsString(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)

	if d.String() != "aufs" {
		t.Fatalf("Expected aufs got %s", d.String())
	}
}

func TestCreateDirStructure(t *testing.T) {
	newDriver(t)
	defer os.RemoveAll(tmp)

	paths := []string{
		"mnt",
		"layers",
		"diff",
	}

	for _, p := range paths {
		if _, err := os.Stat(path.Join(tmp, p)); err != nil {
			t.Fatal(err)
		}
	}
}

// We should be able to create two drivers with the same dir structure
func TestNewDriverFromExistingDir(t *testing.T) {
	if err := os.MkdirAll(tmp, 0755); err != nil {
		t.Fatal(err)
	}

	if _, err := Init(tmp); err != nil {
		t.Fatal(err)
	}
	if _, err := Init(tmp); err != nil {
		t.Fatal(err)
	}
	os.RemoveAll(tmp)
}

func TestCreateNewDir(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)

	if err := d.Create("1", ""); err != nil {
		t.Fatal(err)
	}
}

func TestCreateNewDirStructure(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)

	if err := d.Create("1", ""); err != nil {
		t.Fatal(err)
	}

	paths := []string{
		"mnt",
		"diff",
		"layers",
	}

	for _, p := range paths {
		if _, err := os.Stat(path.Join(tmp, p, "1")); err != nil {
			t.Fatal(err)
		}
	}
}

func TestRemoveImage(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)

	if err := d.Create("1", ""); err != nil {
		t.Fatal(err)
	}

	if err := d.Remove("1"); err != nil {
		t.Fatal(err)
	}

	paths := []string{
		"mnt",
		"diff",
		"layers",
	}

	for _, p := range paths {
		if _, err := os.Stat(path.Join(tmp, p, "1")); err == nil {
			t.Fatalf("Error should not be nil because dirs with id 1 should be delted: %s", p)
		}
	}
}

func TestGetWithoutParent(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)

	if err := d.Create("1", ""); err != nil {
		t.Fatal(err)
	}

	diffPath, err := d.Get("1")
	if err != nil {
		t.Fatal(err)
	}
	expected := path.Join(tmp, "diff", "1")
	if diffPath != expected {
		t.Fatalf("Expected path %s got %s", expected, diffPath)
	}
}

func TestCleanupWithNoDirs(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)

	if err := d.Cleanup(); err != nil {
		t.Fatal(err)
	}
}

func TestCleanupWithDir(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)

	if err := d.Create("1", ""); err != nil {
		t.Fatal(err)
	}

	if err := d.Cleanup(); err != nil {
		t.Fatal(err)
	}
}

func TestMountedFalseResponse(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)

	if err := d.Create("1", ""); err != nil {
		t.Fatal(err)
	}

	response, err := d.mounted("1")
	if err != nil {
		t.Fatal(err)
	}

	if response != false {
		t.Fatalf("Response if dir id 1 is mounted should be false")
	}
}

func TestMountedTrueReponse(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)
	defer d.Cleanup()

	if err := d.Create("1", ""); err != nil {
		t.Fatal(err)
	}
	if err := d.Create("2", "1"); err != nil {
		t.Fatal(err)
	}

	_, err := d.Get("2")
	if err != nil {
		t.Fatal(err)
	}

	response, err := d.mounted("2")
	if err != nil {
		t.Fatal(err)
	}

	if response != true {
		t.Fatalf("Response if dir id 2 is mounted should be true")
	}
}

func TestMountWithParent(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)

	if err := d.Create("1", ""); err != nil {
		t.Fatal(err)
	}
	if err := d.Create("2", "1"); err != nil {
		t.Fatal(err)
	}

	defer func() {
		if err := d.Cleanup(); err != nil {
			t.Fatal(err)
		}
	}()

	mntPath, err := d.Get("2")
	if err != nil {
		t.Fatal(err)
	}
	if mntPath == "" {
		t.Fatal("mntPath should not be empty string")
	}

	expected := path.Join(tmp, "mnt", "2")
	if mntPath != expected {
		t.Fatalf("Expected %s got %s", expected, mntPath)
	}
}

func TestRemoveMountedDir(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)

	if err := d.Create("1", ""); err != nil {
		t.Fatal(err)
	}
	if err := d.Create("2", "1"); err != nil {
		t.Fatal(err)
	}

	defer func() {
		if err := d.Cleanup(); err != nil {
			t.Fatal(err)
		}
	}()

	mntPath, err := d.Get("2")
	if err != nil {
		t.Fatal(err)
	}
	if mntPath == "" {
		t.Fatal("mntPath should not be empty string")
	}

	mounted, err := d.mounted("2")
	if err != nil {
		t.Fatal(err)
	}

	if !mounted {
		t.Fatalf("Dir id 2 should be mounted")
	}

	if err := d.Remove("2"); err != nil {
		t.Fatal(err)
	}
}

func TestCreateWithInvalidParent(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)

	if err := d.Create("1", "docker"); err == nil {
		t.Fatalf("Error should not be nil with parent does not exist")
	}
}

func TestGetDiff(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)

	if err := d.Create("1", ""); err != nil {
		t.Fatal(err)
	}

	diffPath, err := d.Get("1")
	if err != nil {
		t.Fatal(err)
	}

	// Add a file to the diff path with a fixed size
	size := int64(1024)

	f, err := os.Create(path.Join(diffPath, "test_file"))
	if err != nil {
		t.Fatal(err)
	}
	if err := f.Truncate(size); err != nil {
		t.Fatal(err)
	}
	f.Close()

	a, err := d.Diff("1")
	if err != nil {
		t.Fatal(err)
	}
	if a == nil {
		t.Fatalf("Archive should not be nil")
	}
}

func TestChanges(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)

	if err := d.Create("1", ""); err != nil {
		t.Fatal(err)
	}
	if err := d.Create("2", "1"); err != nil {
		t.Fatal(err)
	}

	defer func() {
		if err := d.Cleanup(); err != nil {
			t.Fatal(err)
		}
	}()

	mntPoint, err := d.Get("2")
	if err != nil {
		t.Fatal(err)
	}

	// Create a file to save in the mountpoint
	f, err := os.Create(path.Join(mntPoint, "test.txt"))
	if err != nil {
		t.Fatal(err)
	}

	if _, err := f.WriteString("testline"); err != nil {
		t.Fatal(err)
	}
	if err := f.Close(); err != nil {
		t.Fatal(err)
	}

	changes, err := d.Changes("2")
	if err != nil {
		t.Fatal(err)
	}
	if len(changes) != 1 {
		t.Fatalf("Dir 2 should have one change from parent got %d", len(changes))
	}
	change := changes[0]

	expectedPath := "/test.txt"
	if change.Path != expectedPath {
		t.Fatalf("Expected path %s got %s", expectedPath, change.Path)
	}

	if change.Kind != archive.ChangeAdd {
		t.Fatalf("Change kind should be ChangeAdd got %s", change.Kind)
	}
}

/* FIXME: How to properly test this?
func TestDiffSize(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)

	if err := d.Create("1", ""); err != nil {
		t.Fatal(err)
	}

	diffPath, err := d.Get("1")
	if err != nil {
		t.Fatal(err)
	}

	// Add a file to the diff path with a fixed size
	size := int64(1024)

	f, err := os.Create(path.Join(diffPath, "test_file"))
	if err != nil {
		t.Fatal(err)
	}
	f.Truncate(size)
	s, err := f.Stat()
	if err != nil {
		t.Fatal(err)
	}
	size = s.Size()
	if err := f.Close(); err != nil {
		t.Fatal(err)
	}

	diffSize, err := d.DiffSize("1")
	if err != nil {
		t.Fatal(err)
	}
	if diffSize != size {
		t.Fatalf("Expected size to be %d got %d", size, diffSize)
	}
}
*/
<commit_msg>Add unit test for child changes diff in aufs<commit_after>package aufs

import (
	"github.com/dotcloud/docker/archive"
	"os"
	"path"
	"testing"
)

var (
	tmp = path.Join(os.TempDir(), "aufs-tests", "aufs")
)

func newDriver(t *testing.T) *AufsDriver {
	if err := os.MkdirAll(tmp, 0755); err != nil {
		t.Fatal(err)
	}

	d, err := Init(tmp)
	if err != nil {
		t.Fatal(err)
	}
	return d.(*AufsDriver)
}

func TestNewAufsDriver(t *testing.T) {
	if err := os.MkdirAll(tmp, 0755); err != nil {
		t.Fatal(err)
	}

	d, err := Init(tmp)
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(tmp)
	if d == nil {
		t.Fatalf("Driver should not be nil")
	}
}

func TestAufsString(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)

	if d.String() != "aufs" {
		t.Fatalf("Expected aufs got %s", d.String())
	}
}

func TestCreateDirStructure(t *testing.T) {
	newDriver(t)
	defer os.RemoveAll(tmp)

	paths := []string{
		"mnt",
		"layers",
		"diff",
	}

	for _, p := range paths {
		if _, err := os.Stat(path.Join(tmp, p)); err != nil {
			t.Fatal(err)
		}
	}
}

// We should be able to create two drivers with the same dir structure
func TestNewDriverFromExistingDir(t *testing.T) {
	if err := os.MkdirAll(tmp, 0755); err != nil {
		t.Fatal(err)
	}

	if _, err := Init(tmp); err != nil {
		t.Fatal(err)
	}
	if _, err := Init(tmp); err != nil {
		t.Fatal(err)
	}
	os.RemoveAll(tmp)
}

func TestCreateNewDir(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)

	if err := d.Create("1", ""); err != nil {
		t.Fatal(err)
	}
}

func TestCreateNewDirStructure(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)

	if err := d.Create("1", ""); err != nil {
		t.Fatal(err)
	}

	paths := []string{
		"mnt",
		"diff",
		"layers",
	}

	for _, p := range paths {
		if _, err := os.Stat(path.Join(tmp, p, "1")); err != nil {
			t.Fatal(err)
		}
	}
}

func TestRemoveImage(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)

	if err := d.Create("1", ""); err != nil {
		t.Fatal(err)
	}

	if err := d.Remove("1"); err != nil {
		t.Fatal(err)
	}

	paths := []string{
		"mnt",
		"diff",
		"layers",
	}

	for _, p := range paths {
		if _, err := os.Stat(path.Join(tmp, p, "1")); err == nil {
			t.Fatalf("Error should not be nil because dirs with id 1 should be delted: %s", p)
		}
	}
}

func TestGetWithoutParent(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)

	if err := d.Create("1", ""); err != nil {
		t.Fatal(err)
	}

	diffPath, err := d.Get("1")
	if err != nil {
		t.Fatal(err)
	}
	expected := path.Join(tmp, "diff", "1")
	if diffPath != expected {
		t.Fatalf("Expected path %s got %s", expected, diffPath)
	}
}

func TestCleanupWithNoDirs(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)

	if err := d.Cleanup(); err != nil {
		t.Fatal(err)
	}
}

func TestCleanupWithDir(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)

	if err := d.Create("1", ""); err != nil {
		t.Fatal(err)
	}

	if err := d.Cleanup(); err != nil {
		t.Fatal(err)
	}
}

func TestMountedFalseResponse(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)

	if err := d.Create("1", ""); err != nil {
		t.Fatal(err)
	}

	response, err := d.mounted("1")
	if err != nil {
		t.Fatal(err)
	}

	if response != false {
		t.Fatalf("Response if dir id 1 is mounted should be false")
	}
}

func TestMountedTrueReponse(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)
	defer d.Cleanup()

	if err := d.Create("1", ""); err != nil {
		t.Fatal(err)
	}
	if err := d.Create("2", "1"); err != nil {
		t.Fatal(err)
	}

	_, err := d.Get("2")
	if err != nil {
		t.Fatal(err)
	}

	response, err := d.mounted("2")
	if err != nil {
		t.Fatal(err)
	}

	if response != true {
		t.Fatalf("Response if dir id 2 is mounted should be true")
	}
}

func TestMountWithParent(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)

	if err := d.Create("1", ""); err != nil {
		t.Fatal(err)
	}
	if err := d.Create("2", "1"); err != nil {
		t.Fatal(err)
	}

	defer func() {
		if err := d.Cleanup(); err != nil {
			t.Fatal(err)
		}
	}()

	mntPath, err := d.Get("2")
	if err != nil {
		t.Fatal(err)
	}
	if mntPath == "" {
		t.Fatal("mntPath should not be empty string")
	}

	expected := path.Join(tmp, "mnt", "2")
	if mntPath != expected {
		t.Fatalf("Expected %s got %s", expected, mntPath)
	}
}

func TestRemoveMountedDir(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)

	if err := d.Create("1", ""); err != nil {
		t.Fatal(err)
	}
	if err := d.Create("2", "1"); err != nil {
		t.Fatal(err)
	}

	defer func() {
		if err := d.Cleanup(); err != nil {
			t.Fatal(err)
		}
	}()

	mntPath, err := d.Get("2")
	if err != nil {
		t.Fatal(err)
	}
	if mntPath == "" {
		t.Fatal("mntPath should not be empty string")
	}

	mounted, err := d.mounted("2")
	if err != nil {
		t.Fatal(err)
	}

	if !mounted {
		t.Fatalf("Dir id 2 should be mounted")
	}

	if err := d.Remove("2"); err != nil {
		t.Fatal(err)
	}
}

func TestCreateWithInvalidParent(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)

	if err := d.Create("1", "docker"); err == nil {
		t.Fatalf("Error should not be nil with parent does not exist")
	}
}

func TestGetDiff(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)

	if err := d.Create("1", ""); err != nil {
		t.Fatal(err)
	}

	diffPath, err := d.Get("1")
	if err != nil {
		t.Fatal(err)
	}

	// Add a file to the diff path with a fixed size
	size := int64(1024)

	f, err := os.Create(path.Join(diffPath, "test_file"))
	if err != nil {
		t.Fatal(err)
	}
	if err := f.Truncate(size); err != nil {
		t.Fatal(err)
	}
	f.Close()

	a, err := d.Diff("1")
	if err != nil {
		t.Fatal(err)
	}
	if a == nil {
		t.Fatalf("Archive should not be nil")
	}
}

func TestChanges(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)

	if err := d.Create("1", ""); err != nil {
		t.Fatal(err)
	}
	if err := d.Create("2", "1"); err != nil {
		t.Fatal(err)
	}

	defer func() {
		if err := d.Cleanup(); err != nil {
			t.Fatal(err)
		}
	}()

	mntPoint, err := d.Get("2")
	if err != nil {
		t.Fatal(err)
	}

	// Create a file to save in the mountpoint
	f, err := os.Create(path.Join(mntPoint, "test.txt"))
	if err != nil {
		t.Fatal(err)
	}

	if _, err := f.WriteString("testline"); err != nil {
		t.Fatal(err)
	}
	if err := f.Close(); err != nil {
		t.Fatal(err)
	}

	changes, err := d.Changes("2")
	if err != nil {
		t.Fatal(err)
	}
	if len(changes) != 1 {
		t.Fatalf("Dir 2 should have one change from parent got %d", len(changes))
	}
	change := changes[0]

	expectedPath := "/test.txt"
	if change.Path != expectedPath {
		t.Fatalf("Expected path %s got %s", expectedPath, change.Path)
	}

	if change.Kind != archive.ChangeAdd {
		t.Fatalf("Change kind should be ChangeAdd got %s", change.Kind)
	}

	if err := d.Create("3", "2"); err != nil {
		t.Fatal(err)
	}
	mntPoint, err = d.Get("3")
	if err != nil {
		t.Fatal(err)
	}

	// Create a file to save in the mountpoint
	f, err = os.Create(path.Join(mntPoint, "test2.txt"))
	if err != nil {
		t.Fatal(err)
	}

	if _, err := f.WriteString("testline"); err != nil {
		t.Fatal(err)
	}
	if err := f.Close(); err != nil {
		t.Fatal(err)
	}

	changes, err = d.Changes("3")
	if err != nil {
		t.Fatal(err)
	}

	if len(changes) != 1 {
		t.Fatalf("Dir 2 should have one change from parent got %d", len(changes))
	}
	change = changes[0]

	expectedPath = "/test2.txt"
	if change.Path != expectedPath {
		t.Fatalf("Expected path %s got %s", expectedPath, change.Path)
	}

	if change.Kind != archive.ChangeAdd {
		t.Fatalf("Change kind should be ChangeAdd got %s", change.Kind)
	}
}

/* FIXME: How to properly test this?
func TestDiffSize(t *testing.T) {
	d := newDriver(t)
	defer os.RemoveAll(tmp)

	if err := d.Create("1", ""); err != nil {
		t.Fatal(err)
	}

	diffPath, err := d.Get("1")
	if err != nil {
		t.Fatal(err)
	}

	// Add a file to the diff path with a fixed size
	size := int64(1024)

	f, err := os.Create(path.Join(diffPath, "test_file"))
	if err != nil {
		t.Fatal(err)
	}
	f.Truncate(size)
	s, err := f.Stat()
	if err != nil {
		t.Fatal(err)
	}
	size = s.Size()
	if err := f.Close(); err != nil {
		t.Fatal(err)
	}

	diffSize, err := d.DiffSize("1")
	if err != nil {
		t.Fatal(err)
	}
	if diffSize != size {
		t.Fatalf("Expected size to be %d got %d", size, diffSize)
	}
}
*/
<|endoftext|>


<commit_before>package auth

import (
	"encoding/base64"
	"fmt"
	"regexp"
	"strings"
)

// Parse http basic header
type BasicAuth struct {
	Name string
	Pass string
}

var (
	basicAuthRegex = regexp.MustCompile("^([^:]*):(.*)$")
)

func parseAuthHeader(header string) (*BasicAuth, error) {
	parts := strings.SplitN(header, " ", 2)
	if len(parts) > 2 {
		return nil, fmt.Errorf("Invalid authorization header, not enought parts")
	}

	authType := parts[0]
	authData := parts[1]

	if strings.ToLower(authType) != "basic" {
		return nil, fmt.Errorf("Authentication '%s' was not of 'Basic' type", authType)
	}

	data, err := base64.StdEncoding.DecodeString(authData)
	if err != nil {
		return nil, err
	}

	matches := basicAuthRegex.FindStringSubmatch(string(data))
	if matches == nil {
		return nil, fmt.Errorf("Authorization data '%s' did not match auth regexp", data)
	}

	return &BasicAuth{
		Name: matches[1],
		Pass: matches[2],
	}, nil
}
<commit_msg>Fix issue with basicauth's parseAuthHeader<commit_after>package auth

import (
	"encoding/base64"
	"fmt"
	"regexp"
	"strings"
)

// Parse http basic header
type BasicAuth struct {
	Name string
	Pass string
}

var (
	basicAuthRegex = regexp.MustCompile("^([^:]*):(.*)$")
)

func parseAuthHeader(header string) (*BasicAuth, error) {
	parts := strings.SplitN(header, " ", 2)
	if len(parts) < 2 {
		return nil, fmt.Errorf("Invalid authorization header, not enought parts")
	}

	authType := parts[0]
	authData := parts[1]

	if strings.ToLower(authType) != "basic" {
		return nil, fmt.Errorf("Authentication '%s' was not of 'Basic' type", authType)
	}

	data, err := base64.StdEncoding.DecodeString(authData)
	if err != nil {
		return nil, err
	}

	matches := basicAuthRegex.FindStringSubmatch(string(data))
	if matches == nil {
		return nil, fmt.Errorf("Authorization data '%s' did not match auth regexp", data)
	}

	return &BasicAuth{
		Name: matches[1],
		Pass: matches[2],
	}, nil
}
<|endoftext|>


<commit_before>// +build linux

package main

import (
	"fmt"
	"github.com/dotcloud/docker/pkg/libcontainer"
	"github.com/dotcloud/docker/pkg/libcontainer/network"
	"github.com/dotcloud/docker/pkg/libcontainer/utils"
	"github.com/dotcloud/docker/pkg/system"
	"github.com/dotcloud/docker/pkg/term"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"syscall"
)

func execCommand(container *libcontainer.Container, args []string) (int, error) {
	master, console, err := createMasterAndConsole()
	if err != nil {
		return -1, err
	}

	// create a pipe so that we can syncronize with the namespaced process and
	// pass the veth name to the child
	r, w, err := os.Pipe()
	if err != nil {
		return -1, err
	}
	system.UsetCloseOnExec(r.Fd())

	command := createCommand(container, console, r.Fd(), args)

	if err := command.Start(); err != nil {
		return -1, err
	}

	if err := writePidFile(command); err != nil {
		command.Process.Kill()
		return -1, err
	}
	defer deletePidFile()

	// Do this before syncing with child so that no children
	// can escape the cgroup
	if container.Cgroups != nil {
		if err := container.Cgroups.Apply(command.Process.Pid); err != nil {
			command.Process.Kill()
			return -1, err
		}
	}

	if container.Network != nil {
		vethPair, err := initializeContainerVeth(container.Network.Bridge, command.Process.Pid)
		if err != nil {
			return -1, err
		}
		sendVethName(vethPair, w)
	}

	// Sync with child
	w.Close()

	go io.Copy(os.Stdout, master)
	go io.Copy(master, os.Stdin)

	state, err := setupWindow(master)
	if err != nil {
		command.Process.Kill()
		return -1, err
	}
	defer term.RestoreTerminal(os.Stdin.Fd(), state)

	if err := command.Wait(); err != nil {
		if _, ok := err.(*exec.ExitError); !ok {
			return -1, err
		}
	}
	return command.ProcessState.Sys().(syscall.WaitStatus).ExitStatus(), nil
}

// sendVethName writes the veth pair name to the child's stdin then closes the
// pipe so that the child stops waiting for more data
func sendVethName(name string, pipe io.WriteCloser) {
	fmt.Fprint(pipe, name)
}

// initializeContainerVeth will create a veth pair and setup the host's
// side of the pair by setting the specified bridge as the master and bringing
// up the interface.
//
// Then will with set the other side of the veth pair into the container's namespaced
// using the pid and returns the veth's interface name to provide to the container to
// finish setting up the interface inside the namespace
func initializeContainerVeth(bridge string, nspid int) (string, error) {
	name1, name2, err := createVethPair()
	if err != nil {
		return "", err
	}
	if err := network.SetInterfaceMaster(name1, bridge); err != nil {
		return "", err
	}
	if err := network.InterfaceUp(name1); err != nil {
		return "", err
	}
	if err := network.SetInterfaceInNamespacePid(name2, nspid); err != nil {
		return "", err
	}
	return name2, nil
}

func setupWindow(master *os.File) (*term.State, error) {
	ws, err := term.GetWinsize(os.Stdin.Fd())
	if err != nil {
		return nil, err
	}
	if err := term.SetWinsize(master.Fd(), ws); err != nil {
		return nil, err
	}
	return term.SetRawTerminal(os.Stdin.Fd())
}

// createMasterAndConsole will open /dev/ptmx on the host and retreive the
// pts name for use as the pty slave inside the container
func createMasterAndConsole() (*os.File, string, error) {
	master, err := os.OpenFile("/dev/ptmx", syscall.O_RDWR|syscall.O_NOCTTY|syscall.O_CLOEXEC, 0)
	if err != nil {
		return nil, "", err
	}
	console, err := system.Ptsname(master)
	if err != nil {
		return nil, "", err
	}
	if err := system.Unlockpt(master); err != nil {
		return nil, "", err
	}
	return master, console, nil
}

// createVethPair will automatically generage two random names for
// the veth pair and ensure that they have been created
func createVethPair() (name1 string, name2 string, err error) {
	name1, err = utils.GenerateRandomName("dock", 4)
	if err != nil {
		return
	}
	name2, err = utils.GenerateRandomName("dock", 4)
	if err != nil {
		return
	}
	if err = network.CreateVethPair(name1, name2); err != nil {
		return
	}
	return
}

// writePidFile writes the namespaced processes pid to .nspid in the rootfs for the container
func writePidFile(command *exec.Cmd) error {
	return ioutil.WriteFile(".nspid", []byte(fmt.Sprint(command.Process.Pid)), 0655)
}

func deletePidFile() error {
	return os.Remove(".nspid")
}

// createCommand will return an exec.Cmd with the Cloneflags set to the proper namespaces
// defined on the container's configuration and use the current binary as the init with the
// args provided
func createCommand(container *libcontainer.Container, console string, pipe uintptr, args []string) *exec.Cmd {
	command := exec.Command("nsinit", append([]string{"-console", console, "-pipe", fmt.Sprint(pipe), "init"}, args...)...)
	command.SysProcAttr = &syscall.SysProcAttr{
		Cloneflags: uintptr(getNamespaceFlags(container.Namespaces)),
	}
	return command
}
<commit_msg>Minor cleanup<commit_after>// +build linux

package main

import (
	"fmt"
	"github.com/dotcloud/docker/pkg/libcontainer"
	"github.com/dotcloud/docker/pkg/libcontainer/network"
	"github.com/dotcloud/docker/pkg/libcontainer/utils"
	"github.com/dotcloud/docker/pkg/system"
	"github.com/dotcloud/docker/pkg/term"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"syscall"
)

func execCommand(container *libcontainer.Container, args []string) (int, error) {
	master, console, err := createMasterAndConsole()
	if err != nil {
		return -1, err
	}

	// create a pipe so that we can syncronize with the namespaced process and
	// pass the veth name to the child
	r, w, err := os.Pipe()
	if err != nil {
		return -1, err
	}
	system.UsetCloseOnExec(r.Fd())

	command := createCommand(container, console, r.Fd(), args)

	if err := command.Start(); err != nil {
		return -1, err
	}

	if err := writePidFile(command); err != nil {
		command.Process.Kill()
		return -1, err
	}
	defer deletePidFile()

	// Do this before syncing with child so that no children
	// can escape the cgroup
	if container.Cgroups != nil {
		if err := container.Cgroups.Apply(command.Process.Pid); err != nil {
			command.Process.Kill()
			return -1, err
		}
	}

	if container.Network != nil {
		vethPair, err := initializeContainerVeth(container.Network.Bridge, command.Process.Pid)
		if err != nil {
			return -1, err
		}
		sendVethName(w, vethPair)
	}

	// Sync with child
	w.Close()
	r.Close()

	go io.Copy(os.Stdout, master)
	go io.Copy(master, os.Stdin)

	state, err := setupWindow(master)
	if err != nil {
		command.Process.Kill()
		return -1, err
	}
	defer term.RestoreTerminal(os.Stdin.Fd(), state)

	if err := command.Wait(); err != nil {
		if _, ok := err.(*exec.ExitError); !ok {
			return -1, err
		}
	}
	return command.ProcessState.Sys().(syscall.WaitStatus).ExitStatus(), nil
}

// sendVethName writes the veth pair name to the child's stdin then closes the
// pipe so that the child stops waiting for more data
func sendVethName(pipe io.Writer, name string) {
	fmt.Fprint(pipe, name)
}

// initializeContainerVeth will create a veth pair and setup the host's
// side of the pair by setting the specified bridge as the master and bringing
// up the interface.
//
// Then will with set the other side of the veth pair into the container's namespaced
// using the pid and returns the veth's interface name to provide to the container to
// finish setting up the interface inside the namespace
func initializeContainerVeth(bridge string, nspid int) (string, error) {
	name1, name2, err := createVethPair()
	if err != nil {
		return "", err
	}
	if err := network.SetInterfaceMaster(name1, bridge); err != nil {
		return "", err
	}
	if err := network.InterfaceUp(name1); err != nil {
		return "", err
	}
	if err := network.SetInterfaceInNamespacePid(name2, nspid); err != nil {
		return "", err
	}
	return name2, nil
}

func setupWindow(master *os.File) (*term.State, error) {
	ws, err := term.GetWinsize(os.Stdin.Fd())
	if err != nil {
		return nil, err
	}
	if err := term.SetWinsize(master.Fd(), ws); err != nil {
		return nil, err
	}
	return term.SetRawTerminal(os.Stdin.Fd())
}

// createMasterAndConsole will open /dev/ptmx on the host and retreive the
// pts name for use as the pty slave inside the container
func createMasterAndConsole() (*os.File, string, error) {
	master, err := os.OpenFile("/dev/ptmx", syscall.O_RDWR|syscall.O_NOCTTY|syscall.O_CLOEXEC, 0)
	if err != nil {
		return nil, "", err
	}
	console, err := system.Ptsname(master)
	if err != nil {
		return nil, "", err
	}
	if err := system.Unlockpt(master); err != nil {
		return nil, "", err
	}
	return master, console, nil
}

// createVethPair will automatically generage two random names for
// the veth pair and ensure that they have been created
func createVethPair() (name1 string, name2 string, err error) {
	name1, err = utils.GenerateRandomName("dock", 4)
	if err != nil {
		return
	}
	name2, err = utils.GenerateRandomName("dock", 4)
	if err != nil {
		return
	}
	if err = network.CreateVethPair(name1, name2); err != nil {
		return
	}
	return
}

// writePidFile writes the namespaced processes pid to .nspid in the rootfs for the container
func writePidFile(command *exec.Cmd) error {
	return ioutil.WriteFile(".nspid", []byte(fmt.Sprint(command.Process.Pid)), 0655)
}

func deletePidFile() error {
	return os.Remove(".nspid")
}

// createCommand will return an exec.Cmd with the Cloneflags set to the proper namespaces
// defined on the container's configuration and use the current binary as the init with the
// args provided
func createCommand(container *libcontainer.Container, console string, pipe uintptr, args []string) *exec.Cmd {
	command := exec.Command("nsinit", append([]string{"-console", console, "-pipe", fmt.Sprint(pipe), "init"}, args...)...)
	command.SysProcAttr = &syscall.SysProcAttr{
		Cloneflags: uintptr(getNamespaceFlags(container.Namespaces)),
	}
	return command
}
<|endoftext|>


<commit_before>package data

// var queueLength = expvar.NewInt("queue-length")
// var conns = expvar.NewInt("conns")

// Queue is a message queue for queueing and sending messages to users.
type Queue interface {
	// todo: buffered channels or basic locks or a concurrent multimap?
	// todo: at-least-once delivery relaxes things a bit for queueProcessor
	//
	// actually queue should not be interacted with directly, just like DB, it should be an interface
	// and server.send(userID) should use it automatically behind the scenes
}
<commit_msg>add note to move expvar:queueLenght to neptulon<commit_after>package data

// var queueLength = expvar.NewInt("queue-length") // todo: conn ID -> *Conn is tracked by neptulon already so we don't need to dupe this code
// var conns = expvar.NewInt("conns")

// Queue is a message queue for queueing and sending messages to users.
type Queue interface {
	// todo: buffered channels or basic locks or a concurrent multimap?
	// todo: at-least-once delivery relaxes things a bit for queueProcessor
	//
	// actually queue should not be interacted with directly, just like DB, it should be an interface
	// and server.send(userID) should use it automatically behind the scenes
}
<|endoftext|>


<commit_before>package db

import (
	"fmt"
	"github.com/open-falcon/common/model"
	"github.com/toolkits/container/set"
	"log"
	"strings"
	"time"
)

// 获取所有的Strategy列表
func QueryStrategies(tpls map[int]*model.Template) (map[int]*model.Strategy, error) {
	ret := make(map[int]*model.Strategy)

	if tpls == nil || len(tpls) == 0 {
		return ret, fmt.Errorf("illegal argument")
	}

	now := time.Now().Format("15:04")
	sql := fmt.Sprintf(
		"select %s from strategy as s where (s.run_begin='' and s.run_end='') or (s.run_begin <= '%s' and s.run_end > '%s')",
		"s.id, s.metric, s.tags, s.func, s.op, s.right_value, s.max_step, s.priority, s.note, s.tpl_id",
		now,
		now,
	)

	rows, err := DB.Query(sql)
	if err != nil {
		log.Println("ERROR:", err)
		return ret, err
	}

	defer rows.Close()
	for rows.Next() {
		s := model.Strategy{}
		var tags string
		var tid int
		err = rows.Scan(&s.Id, &s.Metric, &tags, &s.Func, &s.Operator, &s.RightValue, &s.MaxStep, &s.Priority, &s.Note, &tid)
		if err != nil {
			log.Println("ERROR:", err)
			continue
		}

		tt := make(map[string]string)

		if tags != "" {
			arr := strings.Split(tags, ",")
			for _, tag := range arr {
				kv := strings.Split(tag, "=")
				if len(kv) != 2 {
					continue
				}
				tt[kv[0]] = kv[1]
			}
		}

		s.Tags = tt
		s.Tpl = tpls[tid]
		if s.Tpl == nil {
			log.Printf("WARN: tpl is nil. strategy id=%d, tpl id=%d", s.Id, tid)
			// 如果Strategy没有对应的Tpl，那就没有action，就没法报警，无需往后传递了
			continue
		}

		ret[s.Id] = &s
	}

	return ret, nil
}

func QueryBuiltinMetrics(tids string) ([]*model.BuiltinMetric, error) {
	sql := fmt.Sprintf(
		"select metric, tags from strategy where tpl_id in (%s) and metric in ('net.port.listen', 'proc.num', 'du.bs')",
		tids,
	)

	ret := []*model.BuiltinMetric{}

	rows, err := DB.Query(sql)
	if err != nil {
		log.Println("ERROR:", err)
		return ret, err
	}

	metricTagsSet := set.NewStringSet()

	defer rows.Close()
	for rows.Next() {
		builtinMetric := model.BuiltinMetric{}
		err = rows.Scan(&builtinMetric.Metric, &builtinMetric.Tags)
		if err != nil {
			log.Println("WARN:", err)
			continue
		}

		k := fmt.Sprintf("%s%s", builtinMetric.Metric, builtinMetric.Tags)
		if metricTagsSet.Exists(k) {
			continue
		}

		ret = append(ret, &builtinMetric)
		metricTagsSet.Add(k)
	}

	return ret, nil
}
<commit_msg>让hbs给agent下发url.check.health这个监控项<commit_after>package db

import (
	"fmt"
	"github.com/open-falcon/common/model"
	"github.com/toolkits/container/set"
	"log"
	"strings"
	"time"
)

// 获取所有的Strategy列表
func QueryStrategies(tpls map[int]*model.Template) (map[int]*model.Strategy, error) {
	ret := make(map[int]*model.Strategy)

	if tpls == nil || len(tpls) == 0 {
		return ret, fmt.Errorf("illegal argument")
	}

	now := time.Now().Format("15:04")
	sql := fmt.Sprintf(
		"select %s from strategy as s where (s.run_begin='' and s.run_end='') or (s.run_begin <= '%s' and s.run_end > '%s')",
		"s.id, s.metric, s.tags, s.func, s.op, s.right_value, s.max_step, s.priority, s.note, s.tpl_id",
		now,
		now,
	)

	rows, err := DB.Query(sql)
	if err != nil {
		log.Println("ERROR:", err)
		return ret, err
	}

	defer rows.Close()
	for rows.Next() {
		s := model.Strategy{}
		var tags string
		var tid int
		err = rows.Scan(&s.Id, &s.Metric, &tags, &s.Func, &s.Operator, &s.RightValue, &s.MaxStep, &s.Priority, &s.Note, &tid)
		if err != nil {
			log.Println("ERROR:", err)
			continue
		}

		tt := make(map[string]string)

		if tags != "" {
			arr := strings.Split(tags, ",")
			for _, tag := range arr {
				kv := strings.Split(tag, "=")
				if len(kv) != 2 {
					continue
				}
				tt[kv[0]] = kv[1]
			}
		}

		s.Tags = tt
		s.Tpl = tpls[tid]
		if s.Tpl == nil {
			log.Printf("WARN: tpl is nil. strategy id=%d, tpl id=%d", s.Id, tid)
			// 如果Strategy没有对应的Tpl，那就没有action，就没法报警，无需往后传递了
			continue
		}

		ret[s.Id] = &s
	}

	return ret, nil
}

func QueryBuiltinMetrics(tids string) ([]*model.BuiltinMetric, error) {
	sql := fmt.Sprintf(
		"select metric, tags from strategy where tpl_id in (%s) and metric in ('net.port.listen', 'proc.num', 'du.bs', 'url.check.health')",
		tids,
	)

	ret := []*model.BuiltinMetric{}

	rows, err := DB.Query(sql)
	if err != nil {
		log.Println("ERROR:", err)
		return ret, err
	}

	metricTagsSet := set.NewStringSet()

	defer rows.Close()
	for rows.Next() {
		builtinMetric := model.BuiltinMetric{}
		err = rows.Scan(&builtinMetric.Metric, &builtinMetric.Tags)
		if err != nil {
			log.Println("WARN:", err)
			continue
		}

		k := fmt.Sprintf("%s%s", builtinMetric.Metric, builtinMetric.Tags)
		if metricTagsSet.Exists(k) {
			continue
		}

		ret = append(ret, &builtinMetric)
		metricTagsSet.Add(k)
	}

	return ret, nil
}
<|endoftext|>