<commit_before>package gokismet

import (
	"io"
	"net/url"
	"strings"
	"time"
)

// A Comment represents an item of user-generated comment to be checked for
// spam, such as a blog comment or forum post. The zero-value object is not
// guaranteed to work. Always use one of the constructors to create Comments.
type Comment struct {
	api    *API
	params *url.Values
}

// NewComment creates a Comment with the provided Akismet API key and
// website. The key and website are verified with Akismet and stored
// for use in subsequent calls to Check, ReportSpam and ReportNotSpam. If
// Akismet fails to verify your key, NewComment returns a nil pointer and
// a non-nil error.
func NewComment(key string, site string) (*Comment, error) {
	return new(key, site, false, "")
}

// NewCommentUA is identical to NewComment except it allows you to specify
// a user agent to send to Akismet in API calls. The user agent should
// be the name of your application, preferably in the format application
// name/version, e.g.
//
//		MyApplication/1.0
//
// Note: This is distinct from SetUserAgent which sets the commenter's
// user agent for a specific comment.
func NewCommentUA(key string, site string, userAgent string) (*Comment, error) {
	return new(key, site, false, userAgent)
}

// NewTestComment creates a Comment in test mode. This tells Akismet
// not to learn from or adapt to any API calls it receives, making
// tests somewhat repeatable. Test mode is recommended (but not
// required) for development.
//
// As with NewComment, the provided API key and website are verified with
// Akismet and stored for subsequent calls to Check, ReportSpam and
// ReportNotSpam. A non-nil error is returned if verification fails.
func NewTestComment(key string, site string) (*Comment, error) {
	return new(key, site, true, "")
}

// NewTestCommentUA is identical to NewTestComment except it allows you to
// specify a user agent to send to Akismet in API calls. The user agent
// should be the name of your application, preferably in the format
// application name/version, e.g.
//
//		MyApplication/1.0
//
// Note: This is distinct from SetUserAgent which sets the commenter's
// user agent for a specific comment.
func NewTestCommentUA(key string, site string, userAgent string) (*Comment, error) {
	return new(key, site, true, userAgent)
}

// new does the heavy lifting for the various versions of the Comment
// constructor. It initialises a new Comment, sets its user agent, and
// verifies the provided Akismet API key. If the key is verified, new
// returns the new Comment, otherwise it returns nil with a non-nil error
// object.
func new(key string, site string, testMode bool, userAgent string) (*Comment, error) {

	comment := &Comment{
		api: &API{
			TestMode:  testMode,
			UserAgent: userAgent,
		},
		params: &url.Values{
			_Site: {site},
			_Type: {"comment"},
		},
	}

	if err := comment.api.VerifyKey(key, site); err != nil {
		return nil, err
	}

	return comment, nil
}

// Check sends a Comment to Akismet for spam checking. If the call is
// successful, the returned status is one of StatusNotSpam,
// StatusProbableSpam or StatusDefiniteSpam and the returned error is nil.
// Otherwise, Check returns StatusUnknown and a non-nil error.
//
// The Akismet docs advise sending as much information about a comment as
// possible. The more data you provide, the more accurate the results. In
// particular, the commenter's IP address must be set (Check will fail
// without it) and the user agent is highly recommended.
func (c *Comment) Check() (SpamStatus, error) {
	return c.api.CheckComment(c.params)
}

// ReportSpam tells Akismet that something it thought was legitimate
// content is actually spam. This implies that a previous call to Check
// returned StatusNotSpam. When calling ReportSpam you should provide as
// much of the comment data from the original Check call as possible.
// You may not be able to resend everything, but any values you do send
// should be identical to the previous values.
func (c *Comment) ReportSpam() error {
	return c.api.SubmitSpam(c.params)
}

// ReportNotSpam tells Akismet that something it thought was spam is
// actually legitimate content. This implies that a previous call to Check
// returned StatusProbableSpam or StatusDefiniteSpam. When calling ReportNotSpam
// you should provide as much of the comment data from the original Check call
// as possible. You may not be able to resend everything, but any values you
// do send should be identical to the previous values.
func (c *Comment) ReportNotSpam() error {
	return c.api.SubmitHam(c.params)
}

// Reset reverts a Comment to its initial state (i.e. just after the call
// to NewComment, NewTestComment etc).
func (c *Comment) Reset() {
	c.params = &url.Values{
		_Site: {c.params.Get(_Site)},
		_Type: {c.params.Get(_Type)},
	}
}

// DebugTo specifies a Writer for debug output. Any HTTP requests sent to
// Akismet and HTTP responses received from Akismet will be logged to this
// Writer. As the name suggests, you should only enable this feature during
// development.
func (c *Comment) DebugTo(writer io.Writer) {
	c.api.DebugWriter = writer
}

// SetType specifies the type of content being checked for spam. The default
// value is "comment". See http://blog.akismet.com/2012/06/19/pro-tip-tell-us-your-comment_type/
// for other options.
func (c *Comment) SetType(s string) {
	c.set(_Type, s)
}

// SetUserIP specifies the IP address of the commenter.
// This is required for calls to Check, ReportSpam and ReportNotSpam.
func (c *Comment) SetUserIP(s string) {
	c.set(_UserIP, s)
}

// SetUserAgent specifies the user agent of the commenter's browser.
// This is not technically required but still highly recommended for
// calls to Check, ReportSpam and ReportNotSpam.
func (c *Comment) SetUserAgent(s string) {
	c.set(_UserAgent, s)
}

// SetReferer specifies the commenter's referring URL.
func (c *Comment) SetReferer(s string) {
	c.set(_Referer, s)
}

// SetPage specifies the URL of the page where the comment was entered.
func (c *Comment) SetPage(s string) {
	c.set(_Page, s)
}

// SetPageTimestamp specifies the publish date of the page where the
// comment was entered.
func (c *Comment) SetPageTimestamp(t time.Time) {
	c.set(_PageTimestamp, formatTime(t))
}

// SetAuthor specifies the name submitted by the commenter.
func (c *Comment) SetAuthor(s string) {
	c.set(_Author, s)
}

// SetEmail specifies the email address submitted by the commenter.
func (c *Comment) SetEmail(s string) {
	c.set(_Email, s)
}

// SetURL specifies the URL submitted by the commenter.
func (c *Comment) SetURL(s string) {
	c.set(_URL, s)
}

// SetContent specifies the content of the comment.
func (c *Comment) SetContent(s string) {
	c.set(_Content, s)
}

// SetTimestamp specifies the creation time of the comment. If this
// is not provided, Akismet uses the time of the API call.
func (c *Comment) SetTimestamp(t time.Time) {
	c.set(_Timestamp, formatTime(t))
}

// SetSiteLanguage specifies the language(s) in use on the site
// where the comment was entered. Format is ISO 639-1, comma-separated
// (e.g. "en, fr_ca").
func (c *Comment) SetSiteLanguage(s string) {
	c.set(_SiteLanguage, s)
}

// SetCharset specifies the character encoding for the comment data
// (e.g. "UTF-8" or "ISO-8859-1").
func (c *Comment) SetCharset(s string) {
	c.set(_Charset, s)
}

// Generic set param function safeguards against blank values
func (c *Comment) set(key string, value string) {
	if s := strings.TrimSpace(value); s != "" {
		c.params.Set(key, s)
	}
}

// formatTime returns a string representation of a Time object,
// formatted for Akismet API calls.
func formatTime(t time.Time) string {
	// Akismet requires UTC time in ISO 8601 format
	// e.g. "2015-04-18T10:30Z"
	return t.UTC().Format(time.RFC3339)
}
<commit_msg>Changed the Comment members from pointers to values.<commit_after>package gokismet

import (
	"io"
	"net/url"
	"strings"
	"time"
)

// A Comment represents an item of user-generated comment to be checked for
// spam, such as a blog comment or forum post. The zero-value object is not
// guaranteed to work. Always use one of the constructors to create Comments.
type Comment struct {
	api    API
	params url.Values
}

// NewComment creates a Comment with the provided Akismet API key and
// website. The key and website are verified with Akismet and stored
// for use in subsequent calls to Check, ReportSpam and ReportNotSpam. If
// Akismet fails to verify your key, NewComment returns a nil pointer and
// a non-nil error.
func NewComment(key string, site string) (*Comment, error) {
	return new(key, site, false, "")
}

// NewCommentUA is identical to NewComment except it allows you to specify
// a user agent to send to Akismet in API calls. The user agent should
// be the name of your application, preferably in the format application
// name/version, e.g.
//
//		MyApplication/1.0
//
// Note: This is distinct from SetUserAgent which sets the commenter's
// user agent for a specific comment.
func NewCommentUA(key string, site string, userAgent string) (*Comment, error) {
	return new(key, site, false, userAgent)
}

// NewTestComment creates a Comment in test mode. This tells Akismet
// not to learn from or adapt to any API calls it receives, making
// tests somewhat repeatable. Test mode is recommended (but not
// required) for development.
//
// As with NewComment, the provided API key and website are verified with
// Akismet and stored for subsequent calls to Check, ReportSpam and
// ReportNotSpam. A non-nil error is returned if verification fails.
func NewTestComment(key string, site string) (*Comment, error) {
	return new(key, site, true, "")
}

// NewTestCommentUA is identical to NewTestComment except it allows you to
// specify a user agent to send to Akismet in API calls. The user agent
// should be the name of your application, preferably in the format
// application name/version, e.g.
//
//		MyApplication/1.0
//
// Note: This is distinct from SetUserAgent which sets the commenter's
// user agent for a specific comment.
func NewTestCommentUA(key string, site string, userAgent string) (*Comment, error) {
	return new(key, site, true, userAgent)
}

// new does the heavy lifting for the various versions of the Comment
// constructor. It initialises a new Comment, sets its user agent, and
// verifies the provided Akismet API key. If the key is verified, new
// returns the new Comment, otherwise it returns nil with a non-nil error
// object.
func new(key string, site string, testMode bool, userAgent string) (*Comment, error) {

	comment := &Comment{
		api: API{
			TestMode:  testMode,
			UserAgent: userAgent,
		},
		params: url.Values{
			_Site: {site},
			_Type: {"comment"},
		},
	}

	if err := comment.api.VerifyKey(key, site); err != nil {
		return nil, err
	}

	return comment, nil
}

// Check sends a Comment to Akismet for spam checking. If the call is
// successful, the returned status is one of StatusNotSpam,
// StatusProbableSpam or StatusDefiniteSpam and the returned error is nil.
// Otherwise, Check returns StatusUnknown and a non-nil error.
//
// The Akismet docs advise sending as much information about a comment as
// possible. The more data you provide, the more accurate the results. In
// particular, the commenter's IP address must be set (Check will fail
// without it) and the user agent is highly recommended.
func (c *Comment) Check() (SpamStatus, error) {
	return c.api.CheckComment(&c.params)
}

// ReportSpam tells Akismet that something it thought was legitimate
// content is actually spam. This implies that a previous call to Check
// returned StatusNotSpam. When calling ReportSpam you should provide as
// much of the comment data from the original Check call as possible.
// You may not be able to resend everything, but any values you do send
// should be identical to the previous values.
func (c *Comment) ReportSpam() error {
	return c.api.SubmitSpam(&c.params)
}

// ReportNotSpam tells Akismet that something it thought was spam is
// actually legitimate content. This implies that a previous call to Check
// returned StatusProbableSpam or StatusDefiniteSpam. When calling ReportNotSpam
// you should provide as much of the comment data from the original Check call
// as possible. You may not be able to resend everything, but any values you
// do send should be identical to the previous values.
func (c *Comment) ReportNotSpam() error {
	return c.api.SubmitHam(&c.params)
}

// Reset reverts a Comment to its initial state (i.e. just after the call
// to NewComment, NewTestComment etc).
func (c *Comment) Reset() {
	c.params = url.Values{
		_Site: {c.params.Get(_Site)},
		_Type: {c.params.Get(_Type)},
	}
}

// DebugTo specifies a Writer for debug output. Any HTTP requests sent to
// Akismet and HTTP responses received from Akismet will be logged to this
// Writer. As the name suggests, you should only enable this feature during
// development.
func (c *Comment) DebugTo(writer io.Writer) {
	c.api.DebugWriter = writer
}

// SetType specifies the type of content being checked for spam. The default
// value is "comment". See http://blog.akismet.com/2012/06/19/pro-tip-tell-us-your-comment_type/
// for other options.
func (c *Comment) SetType(s string) {
	c.set(_Type, s)
}

// SetUserIP specifies the IP address of the commenter.
// This is required for calls to Check, ReportSpam and ReportNotSpam.
func (c *Comment) SetUserIP(s string) {
	c.set(_UserIP, s)
}

// SetUserAgent specifies the user agent of the commenter's browser.
// This is not technically required but still highly recommended for
// calls to Check, ReportSpam and ReportNotSpam.
func (c *Comment) SetUserAgent(s string) {
	c.set(_UserAgent, s)
}

// SetReferer specifies the commenter's referring URL.
func (c *Comment) SetReferer(s string) {
	c.set(_Referer, s)
}

// SetPage specifies the URL of the page where the comment was entered.
func (c *Comment) SetPage(s string) {
	c.set(_Page, s)
}

// SetPageTimestamp specifies the publish date of the page where the
// comment was entered.
func (c *Comment) SetPageTimestamp(t time.Time) {
	c.set(_PageTimestamp, formatTime(t))
}

// SetAuthor specifies the name submitted by the commenter.
func (c *Comment) SetAuthor(s string) {
	c.set(_Author, s)
}

// SetEmail specifies the email address submitted by the commenter.
func (c *Comment) SetEmail(s string) {
	c.set(_Email, s)
}

// SetURL specifies the URL submitted by the commenter.
func (c *Comment) SetURL(s string) {
	c.set(_URL, s)
}

// SetContent specifies the content of the comment.
func (c *Comment) SetContent(s string) {
	c.set(_Content, s)
}

// SetTimestamp specifies the creation time of the comment. If this
// is not provided, Akismet uses the time of the API call.
func (c *Comment) SetTimestamp(t time.Time) {
	c.set(_Timestamp, formatTime(t))
}

// SetSiteLanguage specifies the language(s) in use on the site
// where the comment was entered. Format is ISO 639-1, comma-separated
// (e.g. "en, fr_ca").
func (c *Comment) SetSiteLanguage(s string) {
	c.set(_SiteLanguage, s)
}

// SetCharset specifies the character encoding for the comment data
// (e.g. "UTF-8" or "ISO-8859-1").
func (c *Comment) SetCharset(s string) {
	c.set(_Charset, s)
}

// Generic set param function safeguards against blank values
func (c *Comment) set(key string, value string) {
	if s := strings.TrimSpace(value); s != "" {
		c.params.Set(key, s)
	}
}

// formatTime returns a string representation of a Time object,
// formatted for Akismet API calls.
func formatTime(t time.Time) string {
	// Akismet requires UTC time in ISO 8601 format
	// e.g. "2015-04-18T10:30Z"
	return t.UTC().Format(time.RFC3339)
}
<|endoftext|>


<commit_before>package router_test

import (
	"context"
	"os"
	"path/filepath"
	"strconv"
	"testing"
	"time"

	"v2ray.com/core/common/session"

	proto "github.com/golang/protobuf/proto"
	"v2ray.com/core/app/dispatcher"
	. "v2ray.com/core/app/router"
	"v2ray.com/core/common"
	"v2ray.com/core/common/errors"
	"v2ray.com/core/common/net"
	"v2ray.com/core/common/platform"
	"v2ray.com/core/common/protocol"
	"v2ray.com/core/common/protocol/http"
	. "v2ray.com/ext/assert"
	"v2ray.com/ext/sysio"
)

func withOutbound(outbound *session.Outbound) context.Context {
	return session.ContextWithOutbound(context.Background(), outbound)
}

func withInbound(inbound *session.Inbound) context.Context {
	return session.ContextWithInbound(context.Background(), inbound)
}

func TestRoutingRule(t *testing.T) {
	assert := With(t)

	type ruleTest struct {
		input  context.Context
		output bool
	}

	cases := []struct {
		rule *RoutingRule
		test []ruleTest
	}{
		{
			rule: &RoutingRule{
				Domain: []*Domain{
					{
						Value: "v2ray.com",
						Type:  Domain_Plain,
					},
					{
						Value: "google.com",
						Type:  Domain_Domain,
					},
					{
						Value: "^facebook\\.com$",
						Type:  Domain_Regex,
					},
				},
			},
			test: []ruleTest{
				{
					input:  withOutbound(&session.Outbound{Target: net.TCPDestination(net.DomainAddress("v2ray.com"), 80)}),
					output: true,
				},
				{
					input:  withOutbound(&session.Outbound{Target: net.TCPDestination(net.DomainAddress("www.v2ray.com.www"), 80)}),
					output: true,
				},
				{
					input:  withOutbound(&session.Outbound{Target: net.TCPDestination(net.DomainAddress("v2ray.co"), 80)}),
					output: false,
				},
				{
					input:  withOutbound(&session.Outbound{Target: net.TCPDestination(net.DomainAddress("www.google.com"), 80)}),
					output: true,
				},
				{
					input:  withOutbound(&session.Outbound{Target: net.TCPDestination(net.DomainAddress("facebook.com"), 80)}),
					output: true,
				},
				{
					input:  withOutbound(&session.Outbound{Target: net.TCPDestination(net.DomainAddress("www.facebook.com"), 80)}),
					output: false,
				},
				{
					input:  context.Background(),
					output: false,
				},
			},
		},
		{
			rule: &RoutingRule{
				Cidr: []*CIDR{
					{
						Ip:     []byte{8, 8, 8, 8},
						Prefix: 32,
					},
					{
						Ip:     []byte{8, 8, 8, 8},
						Prefix: 32,
					},
					{
						Ip:     net.ParseAddress("2001:0db8:85a3:0000:0000:8a2e:0370:7334").IP(),
						Prefix: 128,
					},
				},
			},
			test: []ruleTest{
				{
					input:  withOutbound(&session.Outbound{Target: net.TCPDestination(net.ParseAddress("8.8.8.8"), 80)}),
					output: true,
				},
				{
					input:  withOutbound(&session.Outbound{Target: net.TCPDestination(net.ParseAddress("8.8.4.4"), 80)}),
					output: false,
				},
				{
					input:  withOutbound(&session.Outbound{Target: net.TCPDestination(net.ParseAddress("2001:0db8:85a3:0000:0000:8a2e:0370:7334"), 80)}),
					output: true,
				},
				{
					input:  context.Background(),
					output: false,
				},
			},
		},
		{
			rule: &RoutingRule{
				UserEmail: []string{
					"admin@v2ray.com",
				},
			},
			test: []ruleTest{
				{
					input:  withInbound(&session.Inbound{User: &protocol.MemoryUser{Email: "admin@v2ray.com"}}),
					output: true,
				},
				{
					input:  withInbound(&session.Inbound{User: &protocol.MemoryUser{Email: "love@v2ray.com"}}),
					output: false,
				},
				{
					input:  context.Background(),
					output: false,
				},
			},
		},
		{
			rule: &RoutingRule{
				Protocol: []string{"http"},
			},
			test: []ruleTest{
				{
					input:  dispatcher.ContextWithSniffingResult(context.Background(), &http.SniffHeader{}),
					output: true,
				},
			},
		},
	}

	for _, test := range cases {
		cond, err := test.rule.BuildCondition()
		assert(err, IsNil)

		for _, t := range test.test {
			assert(cond.Apply(t.input), Equals, t.output)
		}
	}
}

func loadGeoSite(country string) ([]*Domain, error) {
	geositeBytes, err := sysio.ReadAsset("geosite.dat")
	if err != nil {
		return nil, err
	}
	var geositeList GeoSiteList
	if err := proto.Unmarshal(geositeBytes, &geositeList); err != nil {
		return nil, err
	}

	for _, site := range geositeList.Entry {
		if site.CountryCode == country {
			return site.Domain, nil
		}
	}

	return nil, errors.New("country not found: " + country)
}

func TestChinaSites(t *testing.T) {
	assert := With(t)

	common.Must(sysio.CopyFile(platform.GetAssetLocation("geosite.dat"), filepath.Join(os.Getenv("GOPATH"), "src", "v2ray.com", "core", "release", "config", "geosite.dat")))

	domains, err := loadGeoSite("CN")
	assert(err, IsNil)

	matcher, err := NewDomainMatcher(domains)
	common.Must(err)

	assert(matcher.ApplyDomain("163.com"), IsTrue)
	assert(matcher.ApplyDomain("163.com"), IsTrue)
	assert(matcher.ApplyDomain("164.com"), IsFalse)
	assert(matcher.ApplyDomain("164.com"), IsFalse)

	for i := 0; i < 1024; i++ {
		assert(matcher.ApplyDomain(strconv.Itoa(i)+".not-exists.com"), IsFalse)
	}
	time.Sleep(time.Second * 10)
	for i := 0; i < 1024; i++ {
		assert(matcher.ApplyDomain(strconv.Itoa(i)+".not-exists2.com"), IsFalse)
	}
}
<commit_msg>remove unnecessary sleep<commit_after>package router_test

import (
	"context"
	"os"
	"path/filepath"
	"strconv"
	"testing"

	proto "github.com/golang/protobuf/proto"

	"v2ray.com/core/app/dispatcher"
	. "v2ray.com/core/app/router"
	"v2ray.com/core/common"
	"v2ray.com/core/common/errors"
	"v2ray.com/core/common/net"
	"v2ray.com/core/common/platform"
	"v2ray.com/core/common/protocol"
	"v2ray.com/core/common/protocol/http"
	"v2ray.com/core/common/session"
	. "v2ray.com/ext/assert"
	"v2ray.com/ext/sysio"
)

func withOutbound(outbound *session.Outbound) context.Context {
	return session.ContextWithOutbound(context.Background(), outbound)
}

func withInbound(inbound *session.Inbound) context.Context {
	return session.ContextWithInbound(context.Background(), inbound)
}

func TestRoutingRule(t *testing.T) {
	assert := With(t)

	type ruleTest struct {
		input  context.Context
		output bool
	}

	cases := []struct {
		rule *RoutingRule
		test []ruleTest
	}{
		{
			rule: &RoutingRule{
				Domain: []*Domain{
					{
						Value: "v2ray.com",
						Type:  Domain_Plain,
					},
					{
						Value: "google.com",
						Type:  Domain_Domain,
					},
					{
						Value: "^facebook\\.com$",
						Type:  Domain_Regex,
					},
				},
			},
			test: []ruleTest{
				{
					input:  withOutbound(&session.Outbound{Target: net.TCPDestination(net.DomainAddress("v2ray.com"), 80)}),
					output: true,
				},
				{
					input:  withOutbound(&session.Outbound{Target: net.TCPDestination(net.DomainAddress("www.v2ray.com.www"), 80)}),
					output: true,
				},
				{
					input:  withOutbound(&session.Outbound{Target: net.TCPDestination(net.DomainAddress("v2ray.co"), 80)}),
					output: false,
				},
				{
					input:  withOutbound(&session.Outbound{Target: net.TCPDestination(net.DomainAddress("www.google.com"), 80)}),
					output: true,
				},
				{
					input:  withOutbound(&session.Outbound{Target: net.TCPDestination(net.DomainAddress("facebook.com"), 80)}),
					output: true,
				},
				{
					input:  withOutbound(&session.Outbound{Target: net.TCPDestination(net.DomainAddress("www.facebook.com"), 80)}),
					output: false,
				},
				{
					input:  context.Background(),
					output: false,
				},
			},
		},
		{
			rule: &RoutingRule{
				Cidr: []*CIDR{
					{
						Ip:     []byte{8, 8, 8, 8},
						Prefix: 32,
					},
					{
						Ip:     []byte{8, 8, 8, 8},
						Prefix: 32,
					},
					{
						Ip:     net.ParseAddress("2001:0db8:85a3:0000:0000:8a2e:0370:7334").IP(),
						Prefix: 128,
					},
				},
			},
			test: []ruleTest{
				{
					input:  withOutbound(&session.Outbound{Target: net.TCPDestination(net.ParseAddress("8.8.8.8"), 80)}),
					output: true,
				},
				{
					input:  withOutbound(&session.Outbound{Target: net.TCPDestination(net.ParseAddress("8.8.4.4"), 80)}),
					output: false,
				},
				{
					input:  withOutbound(&session.Outbound{Target: net.TCPDestination(net.ParseAddress("2001:0db8:85a3:0000:0000:8a2e:0370:7334"), 80)}),
					output: true,
				},
				{
					input:  context.Background(),
					output: false,
				},
			},
		},
		{
			rule: &RoutingRule{
				UserEmail: []string{
					"admin@v2ray.com",
				},
			},
			test: []ruleTest{
				{
					input:  withInbound(&session.Inbound{User: &protocol.MemoryUser{Email: "admin@v2ray.com"}}),
					output: true,
				},
				{
					input:  withInbound(&session.Inbound{User: &protocol.MemoryUser{Email: "love@v2ray.com"}}),
					output: false,
				},
				{
					input:  context.Background(),
					output: false,
				},
			},
		},
		{
			rule: &RoutingRule{
				Protocol: []string{"http"},
			},
			test: []ruleTest{
				{
					input:  dispatcher.ContextWithSniffingResult(context.Background(), &http.SniffHeader{}),
					output: true,
				},
			},
		},
	}

	for _, test := range cases {
		cond, err := test.rule.BuildCondition()
		assert(err, IsNil)

		for _, t := range test.test {
			assert(cond.Apply(t.input), Equals, t.output)
		}
	}
}

func loadGeoSite(country string) ([]*Domain, error) {
	geositeBytes, err := sysio.ReadAsset("geosite.dat")
	if err != nil {
		return nil, err
	}
	var geositeList GeoSiteList
	if err := proto.Unmarshal(geositeBytes, &geositeList); err != nil {
		return nil, err
	}

	for _, site := range geositeList.Entry {
		if site.CountryCode == country {
			return site.Domain, nil
		}
	}

	return nil, errors.New("country not found: " + country)
}

func TestChinaSites(t *testing.T) {
	assert := With(t)

	common.Must(sysio.CopyFile(platform.GetAssetLocation("geosite.dat"), filepath.Join(os.Getenv("GOPATH"), "src", "v2ray.com", "core", "release", "config", "geosite.dat")))

	domains, err := loadGeoSite("CN")
	assert(err, IsNil)

	matcher, err := NewDomainMatcher(domains)
	common.Must(err)

	assert(matcher.ApplyDomain("163.com"), IsTrue)
	assert(matcher.ApplyDomain("163.com"), IsTrue)
	assert(matcher.ApplyDomain("164.com"), IsFalse)
	assert(matcher.ApplyDomain("164.com"), IsFalse)

	for i := 0; i < 1024; i++ {
		assert(matcher.ApplyDomain(strconv.Itoa(i)+".not-exists.com"), IsFalse)
	}
}
<|endoftext|>


<commit_before>package grandRepositorySky

//IMPORT parts ----------------------------------------------------------
import (
	"bytes"
	"fmt"
	"github.com/tgermain/grandRepositorySky/dht"
	// "math/big"
)

//Const parts -----------------------------------------------------------
const SPACESIZE = 160

//Objects parts ---------------------------------------------------------
type DHTnode struct {
	id        string
	fingers   []*fingerEntry //Successor is fingers[0].tmp
	successor *fingerEntry
	ip, port  string
}

type fingerEntry struct {
	idKey  string
	idResp string
	tmp    *DHTnode
}

//Method parts ----------------------------------------------------------

func (currentNode *DHTnode) AddToRing(newNode *DHTnode) {
	//furthers comments assume that he current currentNode is named x
	switch {
	case bytes.Compare([]byte(currentNode.id), []byte(currentNode.successor.tmp.id)) == 0:
		{
			//init case : currentNode looping on itself
			// fmt.Println("Init case : 2 node")

			currentNode.chainingToTheRing(newNode)
			//TODO : initialize both fingers tables
		}
	case dht.Between(currentNode.id, currentNode.successor.tmp.id, newNode.id):
		// (currentNode.id < newNode.id) && (newNode.id < currentNode.successor.tmp.id)
		{
			//case of x->(x+2) and we want to add (x+1) node
			// fmt.Println("add in the middle")
			currentNode.chainingToTheRing(newNode)
		}
	case dht.Between(currentNode.successor.tmp.id, newNode.id, currentNode.id):
		// (currentNode.successor.tmp.id < currentNode.id) && (currentNode.id < newNode.id)
		{
			//case of X -> 0 and we want to add (x+1) node
			// fmt.Println("add at the end")
			currentNode.chainingToTheRing(newNode)
		}
	default:
		{
			// fmt.Println("Go to the next")
			currentNode.successor.tmp.AddToRing(newNode)
		}
	}
}

func (currentNode *DHTnode) chainingToTheRing(newNode *DHTnode) {

	// fmt.Println("old node : ")
	// currentNode.PrintNodeInfo()
	// fmt.Println("new node : ")
	// newNode.PrintNodeInfo()

	newNode.successor.tmp = currentNode.successor.tmp

	currentNode.successor.tmp = newNode

	// fmt.Println("============================================")
	// fmt.Println("old node : ")
	// currentNode.PrintNodeInfo()
	// fmt.Println("new node : ")
	// newNode.PrintNodeInfo()

	newNode.initFingersTable()
	// currentNode.initFingersTable()
}

func (currentNode *DHTnode) Lookup(idToSearch string) *DHTnode {
	if dht.Between(currentNode.id, currentNode.fingers[0].tmp.id, idToSearch) {
		// fmt.Printf("We are seeking for %s\n", idToSearch)
		return currentNode
	} else {
		// fmt.Println("go to the next one")
		//TODO use the fingers table here
		return currentNode.findClosestNode(idToSearch).Lookup(idToSearch)
	}
}

func (currentNode *DHTnode) findClosestNode(idToSearch string) *DHTnode {
	bestFinger := currentNode.successor.tmp

	minDistance := dht.Distance([]byte(currentNode.successor.tmp.id), []byte(idToSearch), SPACESIZE)
	for _, v := range currentNode.fingers {
		if v != nil {
			//if a member of finger table brought closer than the actual one, we udate the value of minDistance and of the chosen finger
			currentDistance := dht.Distance([]byte(v.idResp), []byte(idToSearch), SPACESIZE)

			// x.cmp(y)
			// -1 if x <  y
			//  0 if x == y
			// +1 if x >  y

			if minDistance.Cmp(currentDistance) == 1 {
				minDistance = currentDistance
				bestFinger = v.tmp
			}
		}
	}
	fmt.Printf("From [%s] We have found the bes way to go to [%s] : we go throught node [%s]\n", currentNode.id, idToSearch, bestFinger.id)
	return bestFinger
}

func (node *DHTnode) PrintRing() {
	fmt.Printf("%s\n", node.id)
	node.successor.tmp.printRingRec(node.id)
}

func (node *DHTnode) printRingRec(origId string) {
	fmt.Printf("%s\n", node.id)
	if bytes.Compare([]byte(node.successor.tmp.id), []byte(origId)) != 0 {

		node.successor.tmp.printRingRec(origId)
	}
}

func (node *DHTnode) TestCalcFingers(k, m int) {
	fingersId, _ := dht.CalcFinger([]byte(node.id), k, m)
	node.Lookup(fingersId).PrintNodeInfo()
}

func (node *DHTnode) PrintNodeInfo() {
	fmt.Println("---------------------------------")
	fmt.Println("Node info")
	fmt.Println("---------------------------------")
	fmt.Printf("  Id		Ip						Port\n")
	fmt.Printf("  %s		%s 		%s\n", node.id, node.ip, node.port)
	fmt.Println()
	// fmt.Println("  Fingers table :")
	// fmt.Println("  ---------------------------------")
	// fmt.Println("  Index		idkey			idNode ")
	// for i, v := range node.fingers {
	// 	if v != nil {
	// 		fmt.Printf("  %d 		%s					%s\n", i, v.idKey, v.idResp)
	// 	}
	// }
	fmt.Println("---------------------------------")

}

//other functions parts --------------------------------------------------------
func MakeDHTNode(NewId *string, NewIp, NewPort string) *DHTnode {
	if NewId == nil {
		tempId := dht.GenerateNodeId()
		NewId = &tempId
	}
	daNode := DHTnode{
		id:      *NewId,
		fingers: make([]*fingerEntry, 1),
		ip:      NewIp,
		port:    NewPort,
	}
	// initialization of fingers table is done while adding the node to the ring
	// The fingers table of the first node of a ring is initialized when a second node is added to the ring

	//Initialize the finger table with each finger pointing to the node frehly created itself
	return &daNode
}
<commit_msg>correct initialization of a newNode<commit_after>package grandRepositorySky

//IMPORT parts ----------------------------------------------------------
import (
	"bytes"
	"fmt"
	"github.com/tgermain/grandRepositorySky/dht"
	// "math/big"
)

//Const parts -----------------------------------------------------------
const SPACESIZE = 160

//Objects parts ---------------------------------------------------------
type DHTnode struct {
	id        string
	fingers   []*fingerEntry //Successor is fingers[0].tmp
	successor *fingerEntry
	ip, port  string
}

type fingerEntry struct {
	idKey  string
	idResp string
	tmp    *DHTnode
}

//Method parts ----------------------------------------------------------

func (currentNode *DHTnode) AddToRing(newNode *DHTnode) {
	//furthers comments assume that he current currentNode is named x
	switch {
	case bytes.Compare([]byte(currentNode.id), []byte(currentNode.successor.tmp.id)) == 0:
		{
			//init case : currentNode looping on itself
			// fmt.Println("Init case : 2 node")

			currentNode.chainingToTheRing(newNode)
			//TODO : initialize both fingers tables
		}
	case dht.Between(currentNode.id, currentNode.successor.tmp.id, newNode.id):
		// (currentNode.id < newNode.id) && (newNode.id < currentNode.successor.tmp.id)
		{
			//case of x->(x+2) and we want to add (x+1) node
			// fmt.Println("add in the middle")
			currentNode.chainingToTheRing(newNode)
		}
	case dht.Between(currentNode.successor.tmp.id, newNode.id, currentNode.id):
		// (currentNode.successor.tmp.id < currentNode.id) && (currentNode.id < newNode.id)
		{
			//case of X -> 0 and we want to add (x+1) node
			// fmt.Println("add at the end")
			currentNode.chainingToTheRing(newNode)
		}
	default:
		{
			// fmt.Println("Go to the next")
			currentNode.successor.tmp.AddToRing(newNode)
		}
	}
}

func (currentNode *DHTnode) chainingToTheRing(newNode *DHTnode) {

	// fmt.Println("old node : ")
	// currentNode.PrintNodeInfo()
	// fmt.Println("new node : ")
	// newNode.PrintNodeInfo()

	newNode.successor.tmp = currentNode.successor.tmp

	currentNode.successor.tmp = newNode

	// fmt.Println("============================================")
	// fmt.Println("old node : ")
	// currentNode.PrintNodeInfo()
	// fmt.Println("new node : ")
	// newNode.PrintNodeInfo()

	newNode.initFingersTable()
	// currentNode.initFingersTable()
}

func (currentNode *DHTnode) Lookup(idToSearch string) *DHTnode {
	if dht.Between(currentNode.id, currentNode.fingers[0].tmp.id, idToSearch) {
		// fmt.Printf("We are seeking for %s\n", idToSearch)
		return currentNode
	} else {
		// fmt.Println("go to the next one")
		//TODO use the fingers table here
		return currentNode.findClosestNode(idToSearch).Lookup(idToSearch)
	}
}

func (currentNode *DHTnode) findClosestNode(idToSearch string) *DHTnode {
	bestFinger := currentNode.successor.tmp

	minDistance := dht.Distance([]byte(currentNode.successor.tmp.id), []byte(idToSearch), SPACESIZE)
	for _, v := range currentNode.fingers {
		if v != nil {
			//if a member of finger table brought closer than the actual one, we udate the value of minDistance and of the chosen finger
			currentDistance := dht.Distance([]byte(v.idResp), []byte(idToSearch), SPACESIZE)

			// x.cmp(y)
			// -1 if x <  y
			//  0 if x == y
			// +1 if x >  y

			if minDistance.Cmp(currentDistance) == 1 {
				minDistance = currentDistance
				bestFinger = v.tmp
			}
		}
	}
	fmt.Printf("From [%s] We have found the bes way to go to [%s] : we go throught node [%s]\n", currentNode.id, idToSearch, bestFinger.id)
	return bestFinger
}

func (node *DHTnode) PrintRing() {
	fmt.Printf("%s\n", node.id)
	node.successor.tmp.printRingRec(node.id)
}

func (node *DHTnode) printRingRec(origId string) {
	fmt.Printf("%s\n", node.id)
	if bytes.Compare([]byte(node.successor.tmp.id), []byte(origId)) != 0 {

		node.successor.tmp.printRingRec(origId)
	}
}

func (node *DHTnode) TestCalcFingers(k, m int) {
	fingersId, _ := dht.CalcFinger([]byte(node.id), k, m)
	node.Lookup(fingersId).PrintNodeInfo()
}

func (node *DHTnode) PrintNodeInfo() {
	fmt.Println("---------------------------------")
	fmt.Println("Node info")
	fmt.Println("---------------------------------")
	fmt.Printf("  Id		Ip						Port\n")
	fmt.Printf("  %s		%s 		%s\n", node.id, node.ip, node.port)
	fmt.Println()
	// fmt.Println("  Fingers table :")
	// fmt.Println("  ---------------------------------")
	// fmt.Println("  Index		idkey			idNode ")
	// for i, v := range node.fingers {
	// 	if v != nil {
	// 		fmt.Printf("  %d 		%s					%s\n", i, v.idKey, v.idResp)
	// 	}
	// }
	fmt.Println("---------------------------------")

}

//other functions parts --------------------------------------------------------
func MakeDHTNode(NewId *string, NewIp, NewPort string) *DHTnode {
	if NewId == nil {
		tempId := dht.GenerateNodeId()
		NewId = &tempId
	}
	daNode := DHTnode{
		id:      *NewId,
		fingers: make([]*fingerEntry, SPACESIZE),
		ip:      NewIp,
		port:    NewPort,
	}
	daNode.successor = &fingerEntry{"truc", "bidule", nil}
	daNode.successor.tmp = &daNode
	daNode.successor.idResp = daNode.id
	// initialization of fingers table is done while adding the node to the ring
	// The fingers table of the first node of a ring is initialized when a second node is added to the ring

	//Initialize the finger table with each finger pointing to the node frehly created itself
	return &daNode
}
<|endoftext|>


<commit_before>// Package executors provides objects that gathers resource data from a host.
package executors

import (
	"errors"
	"reflect"
	"sync"

	resourced_config "github.com/resourced/resourced/config"
	"github.com/resourced/resourced/queryparser"
)

var executorConstructors = make(map[string]func() IExecutor)

var ConditionMetByPathCounter = make(map[string]int)

// Register makes any executor constructor available by name.
func Register(name string, constructor func() IExecutor) {
	if constructor == nil {
		panic("executor: Register executor constructor is nil")
	}
	if _, dup := executorConstructors[name]; dup {
		panic("executor: Register called twice for executor constructor " + name)
	}
	executorConstructors[name] = constructor
}

func ResetConditionsMetByPath() {
	ConditionMetByPathCounter = make(map[string]int)
}

// NewGoStruct instantiates IExecutor
func NewGoStruct(name string) (IExecutor, error) {
	constructor, ok := executorConstructors[name]
	if !ok {
		return nil, errors.New("GoStruct is undefined.")
	}

	return constructor(), nil
}

// NewGoStructByConfig instantiates IExecutor given Config struct
func NewGoStructByConfig(config resourced_config.Config) (IExecutor, error) {
	executor, err := NewGoStruct(config.GoStruct)
	if err != nil {
		return nil, err
	}

	executor.SetPath(config.Path)
	executor.SetInterval(config.Interval)

	// Populate IExecutor fields dynamically
	if len(config.GoStructFields) > 0 {
		for structFieldInString, value := range config.GoStructFields {
			goStructField := reflect.ValueOf(executor).Elem().FieldByName(structFieldInString)

			if goStructField.IsValid() && goStructField.CanSet() {
				valueOfValue := reflect.ValueOf(value)
				goStructField.Set(valueOfValue)
			}
		}
	}

	return executor, nil
}

// IExecutor is generic interface for all executors.
type IExecutor interface {
	SetPath(string)
	GetPath() string
	SetInterval(string)
	Run() error
	ToJson() ([]byte, error)
	SetQueryParser(map[string][]byte)
	SetReadersDataInBytes(map[string][]byte)
	SetTags(map[string]string)
	IsConditionMet() bool
	LowThresholdExceeded() bool
	HighThresholdExceeded() bool
	ConditionMetCount() int
	OnLowThresholdExceeded() error
	OnHighThresholdExceeded() error
}

type Base struct {
	Command          string
	Path             string
	Interval         string
	LowThreshold     int
	HighThreshold    int
	Conditions       string
	ReadersDataBytes map[string][]byte
	qp               *queryparser.QueryParser
	sync.RWMutex
}

func (b *Base) SetPath(path string) {
	b.Path = path
}

func (b *Base) GetPath() string {
	return b.Path
}

func (b *Base) SetInterval(interval string) {
	b.Interval = interval
}

func (b *Base) SetQueryParser(readersJsonBytes map[string][]byte) {
	b.qp = queryparser.New(readersJsonBytes, nil)
}

// SetReadersDataInBytes pulls readers data and store them on ReadersData field.
func (b *Base) SetReadersDataInBytes(readersJsonBytes map[string][]byte) {
	b.ReadersDataBytes = readersJsonBytes

	b.SetQueryParser(readersJsonBytes)
}

// SetTags assigns all host tags to qp (QueryParser).
func (b *Base) SetTags(tags map[string]string) {
	b.qp.SetTags(tags)
}

func (b *Base) IsConditionMet() bool {
	if b.Conditions == "" {
		b.Conditions = "true"
	}

	result, err := b.qp.Parse(b.Conditions)
	if err != nil {
		return false
	}
	if result == true {
		b.conditionMet()
	} else {
		b.conditionNoLongerMet()
	}
	return result
}

func (b *Base) conditionMet() {
	b.Lock()
	ConditionMetByPathCounter[b.Path] = ConditionMetByPathCounter[b.Path] + 1
	b.Unlock()
}

func (b *Base) conditionNoLongerMet() {
	b.Lock()
	ConditionMetByPathCounter[b.Path] = 0
	b.Unlock()
}

func (b *Base) ConditionMetCount() int {
	return ConditionMetByPathCounter[b.Path]
}

func (b *Base) LowThresholdExceeded() bool {
	if b.LowThreshold == 0 {
		return false
	}
	return ConditionMetByPathCounter[b.Path] > b.LowThreshold
}

func (b *Base) HighThresholdExceeded() bool {
	if b.HighThreshold == 0 {
		return false
	}
	return ConditionMetByPathCounter[b.Path] > b.HighThreshold
}

func (b *Base) OnLowThresholdExceeded() error {
	return nil
}

func (b *Base) OnHighThresholdExceeded() error {
	return nil
}
<commit_msg>We don’t need these 2 methods yet.<commit_after>// Package executors provides objects that gathers resource data from a host.
package executors

import (
	"errors"
	"reflect"
	"sync"

	resourced_config "github.com/resourced/resourced/config"
	"github.com/resourced/resourced/queryparser"
)

var executorConstructors = make(map[string]func() IExecutor)

var ConditionMetByPathCounter = make(map[string]int)

// Register makes any executor constructor available by name.
func Register(name string, constructor func() IExecutor) {
	if constructor == nil {
		panic("executor: Register executor constructor is nil")
	}
	if _, dup := executorConstructors[name]; dup {
		panic("executor: Register called twice for executor constructor " + name)
	}
	executorConstructors[name] = constructor
}

func ResetConditionsMetByPath() {
	ConditionMetByPathCounter = make(map[string]int)
}

// NewGoStruct instantiates IExecutor
func NewGoStruct(name string) (IExecutor, error) {
	constructor, ok := executorConstructors[name]
	if !ok {
		return nil, errors.New("GoStruct is undefined.")
	}

	return constructor(), nil
}

// NewGoStructByConfig instantiates IExecutor given Config struct
func NewGoStructByConfig(config resourced_config.Config) (IExecutor, error) {
	executor, err := NewGoStruct(config.GoStruct)
	if err != nil {
		return nil, err
	}

	executor.SetPath(config.Path)
	executor.SetInterval(config.Interval)

	// Populate IExecutor fields dynamically
	if len(config.GoStructFields) > 0 {
		for structFieldInString, value := range config.GoStructFields {
			goStructField := reflect.ValueOf(executor).Elem().FieldByName(structFieldInString)

			if goStructField.IsValid() && goStructField.CanSet() {
				valueOfValue := reflect.ValueOf(value)
				goStructField.Set(valueOfValue)
			}
		}
	}

	return executor, nil
}

// IExecutor is generic interface for all executors.
type IExecutor interface {
	SetPath(string)
	GetPath() string
	SetInterval(string)
	Run() error
	ToJson() ([]byte, error)
	SetQueryParser(map[string][]byte)
	SetReadersDataInBytes(map[string][]byte)
	SetTags(map[string]string)
	IsConditionMet() bool
	LowThresholdExceeded() bool
	HighThresholdExceeded() bool
	ConditionMetCount() int
}

type Base struct {
	Command          string
	Path             string
	Interval         string
	LowThreshold     int
	HighThreshold    int
	Conditions       string
	ReadersDataBytes map[string][]byte
	qp               *queryparser.QueryParser
	sync.RWMutex
}

func (b *Base) SetPath(path string) {
	b.Path = path
}

func (b *Base) GetPath() string {
	return b.Path
}

func (b *Base) SetInterval(interval string) {
	b.Interval = interval
}

func (b *Base) SetQueryParser(readersJsonBytes map[string][]byte) {
	b.qp = queryparser.New(readersJsonBytes, nil)
}

// SetReadersDataInBytes pulls readers data and store them on ReadersData field.
func (b *Base) SetReadersDataInBytes(readersJsonBytes map[string][]byte) {
	b.ReadersDataBytes = readersJsonBytes

	b.SetQueryParser(readersJsonBytes)
}

// SetTags assigns all host tags to qp (QueryParser).
func (b *Base) SetTags(tags map[string]string) {
	b.qp.SetTags(tags)
}

func (b *Base) IsConditionMet() bool {
	if b.Conditions == "" {
		b.Conditions = "true"
	}

	result, err := b.qp.Parse(b.Conditions)
	if err != nil {
		return false
	}
	if result == true {
		b.conditionMet()
	} else {
		b.conditionNoLongerMet()
	}
	return result
}

func (b *Base) conditionMet() {
	b.Lock()
	ConditionMetByPathCounter[b.Path] = ConditionMetByPathCounter[b.Path] + 1
	b.Unlock()
}

func (b *Base) conditionNoLongerMet() {
	b.Lock()
	ConditionMetByPathCounter[b.Path] = 0
	b.Unlock()
}

func (b *Base) ConditionMetCount() int {
	return ConditionMetByPathCounter[b.Path]
}

func (b *Base) LowThresholdExceeded() bool {
	if b.LowThreshold == 0 {
		return false
	}
	return ConditionMetByPathCounter[b.Path] > b.LowThreshold
}

func (b *Base) HighThresholdExceeded() bool {
	if b.HighThreshold == 0 {
		return false
	}
	return ConditionMetByPathCounter[b.Path] > b.HighThreshold
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package resource

import (
	"strconv"

	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/watch"
)

var metadataAccessor = meta.NewAccessor()

// Helper provides methods for retrieving or mutating a RESTful
// resource.
type Helper struct {
	// The name of this resource as the server would recognize it
	Resource string
	// A RESTClient capable of mutating this resource.
	RESTClient RESTClient
	// True if the resource type is scoped to namespaces
	NamespaceScoped bool
}

// NewHelper creates a Helper from a ResourceMapping
func NewHelper(client RESTClient, mapping *meta.RESTMapping) *Helper {
	return &Helper{
		Resource:        mapping.Resource.Resource,
		RESTClient:      client,
		NamespaceScoped: mapping.Scope.Name() == meta.RESTScopeNameNamespace,
	}
}

func (m *Helper) Get(namespace, name string, export bool) (runtime.Object, error) {
	req := m.RESTClient.Get().
		NamespaceIfScoped(namespace, m.NamespaceScoped).
		Resource(m.Resource).
		Name(name)
	if export {
		// TODO: I should be part of GetOptions
		req.Param("export", strconv.FormatBool(export))
	}
	return req.Do().Get()
}

func (m *Helper) List(namespace, apiVersion string, export bool, options *metav1.ListOptions) (runtime.Object, error) {
	req := m.RESTClient.Get().
		NamespaceIfScoped(namespace, m.NamespaceScoped).
		Resource(m.Resource).
		VersionedParams(options, metav1.ParameterCodec)
	if export {
		// TODO: I should be part of ListOptions
		req.Param("export", strconv.FormatBool(export))
	}
	return req.Do().Get()
}

func (m *Helper) Watch(namespace, apiVersion string, options *metav1.ListOptions) (watch.Interface, error) {
	options.Watch = true
	return m.RESTClient.Get().
		NamespaceIfScoped(namespace, m.NamespaceScoped).
		Resource(m.Resource).
		VersionedParams(options, metav1.ParameterCodec).
		Watch()
}

func (m *Helper) WatchSingle(namespace, name, resourceVersion string) (watch.Interface, error) {
	return m.RESTClient.Get().
		NamespaceIfScoped(namespace, m.NamespaceScoped).
		Resource(m.Resource).
		VersionedParams(&metav1.ListOptions{
			ResourceVersion: resourceVersion,
			Watch:           true,
			FieldSelector:   fields.OneTermEqualSelector("metadata.name", name).String(),
		}, metav1.ParameterCodec).
		Watch()
}

func (m *Helper) Delete(namespace, name string) (runtime.Object, error) {
	return m.DeleteWithOptions(namespace, name, nil)
}

func (m *Helper) DeleteWithOptions(namespace, name string, options *metav1.DeleteOptions) (runtime.Object, error) {
	return m.RESTClient.Delete().
		NamespaceIfScoped(namespace, m.NamespaceScoped).
		Resource(m.Resource).
		Name(name).
		Body(options).
		Do().
		Get()
}

func (m *Helper) Create(namespace string, modify bool, obj runtime.Object, options *metav1.CreateOptions) (runtime.Object, error) {
	if options == nil {
		options = &metav1.CreateOptions{}
	}
	if modify {
		// Attempt to version the object based on client logic.
		version, err := metadataAccessor.ResourceVersion(obj)
		if err != nil {
			// We don't know how to clear the version on this object, so send it to the server as is
			return m.createResource(m.RESTClient, m.Resource, namespace, obj, options)
		}
		if version != "" {
			if err := metadataAccessor.SetResourceVersion(obj, ""); err != nil {
				return nil, err
			}
		}
	}

	return m.createResource(m.RESTClient, m.Resource, namespace, obj, options)
}

func (m *Helper) createResource(c RESTClient, resource, namespace string, obj runtime.Object, options *metav1.CreateOptions) (runtime.Object, error) {
	return c.Post().
		NamespaceIfScoped(namespace, m.NamespaceScoped).
		Resource(resource).
		VersionedParams(options, metav1.ParameterCodec).
		Body(obj).
		Do().
		Get()
}
func (m *Helper) Patch(namespace, name string, pt types.PatchType, data []byte, options *metav1.UpdateOptions) (runtime.Object, error) {
	if options == nil {
		options = &metav1.UpdateOptions{}
	}
	return m.RESTClient.Patch(pt).
		NamespaceIfScoped(namespace, m.NamespaceScoped).
		Resource(m.Resource).
		Name(name).
		VersionedParams(options, metav1.ParameterCodec).
		Body(data).
		Do().
		Get()
}

func (m *Helper) Replace(namespace, name string, overwrite bool, obj runtime.Object) (runtime.Object, error) {
	c := m.RESTClient

	// Attempt to version the object based on client logic.
	version, err := metadataAccessor.ResourceVersion(obj)
	if err != nil {
		// We don't know how to version this object, so send it to the server as is
		return m.replaceResource(c, m.Resource, namespace, name, obj)
	}
	if version == "" && overwrite {
		// Retrieve the current version of the object to overwrite the server object
		serverObj, err := c.Get().NamespaceIfScoped(namespace, m.NamespaceScoped).Resource(m.Resource).Name(name).Do().Get()
		if err != nil {
			// The object does not exist, but we want it to be created
			return m.replaceResource(c, m.Resource, namespace, name, obj)
		}
		serverVersion, err := metadataAccessor.ResourceVersion(serverObj)
		if err != nil {
			return nil, err
		}
		if err := metadataAccessor.SetResourceVersion(obj, serverVersion); err != nil {
			return nil, err
		}
	}

	return m.replaceResource(c, m.Resource, namespace, name, obj)
}

func (m *Helper) replaceResource(c RESTClient, resource, namespace, name string, obj runtime.Object) (runtime.Object, error) {
	return c.Put().NamespaceIfScoped(namespace, m.NamespaceScoped).Resource(resource).Name(name).Body(obj).Do().Get()
}
<commit_msg>API Machinery, Kubectl and tests<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package resource

import (
	"strconv"

	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/watch"
)

var metadataAccessor = meta.NewAccessor()

// Helper provides methods for retrieving or mutating a RESTful
// resource.
type Helper struct {
	// The name of this resource as the server would recognize it
	Resource string
	// A RESTClient capable of mutating this resource.
	RESTClient RESTClient
	// True if the resource type is scoped to namespaces
	NamespaceScoped bool
}

// NewHelper creates a Helper from a ResourceMapping
func NewHelper(client RESTClient, mapping *meta.RESTMapping) *Helper {
	return &Helper{
		Resource:        mapping.Resource.Resource,
		RESTClient:      client,
		NamespaceScoped: mapping.Scope.Name() == meta.RESTScopeNameNamespace,
	}
}

func (m *Helper) Get(namespace, name string, export bool) (runtime.Object, error) {
	req := m.RESTClient.Get().
		NamespaceIfScoped(namespace, m.NamespaceScoped).
		Resource(m.Resource).
		Name(name)
	if export {
		// TODO: I should be part of GetOptions
		req.Param("export", strconv.FormatBool(export))
	}
	return req.Do().Get()
}

func (m *Helper) List(namespace, apiVersion string, export bool, options *metav1.ListOptions) (runtime.Object, error) {
	req := m.RESTClient.Get().
		NamespaceIfScoped(namespace, m.NamespaceScoped).
		Resource(m.Resource).
		VersionedParams(options, metav1.ParameterCodec)
	if export {
		// TODO: I should be part of ListOptions
		req.Param("export", strconv.FormatBool(export))
	}
	return req.Do().Get()
}

func (m *Helper) Watch(namespace, apiVersion string, options *metav1.ListOptions) (watch.Interface, error) {
	options.Watch = true
	return m.RESTClient.Get().
		NamespaceIfScoped(namespace, m.NamespaceScoped).
		Resource(m.Resource).
		VersionedParams(options, metav1.ParameterCodec).
		Watch()
}

func (m *Helper) WatchSingle(namespace, name, resourceVersion string) (watch.Interface, error) {
	return m.RESTClient.Get().
		NamespaceIfScoped(namespace, m.NamespaceScoped).
		Resource(m.Resource).
		VersionedParams(&metav1.ListOptions{
			ResourceVersion: resourceVersion,
			Watch:           true,
			FieldSelector:   fields.OneTermEqualSelector("metadata.name", name).String(),
		}, metav1.ParameterCodec).
		Watch()
}

func (m *Helper) Delete(namespace, name string) (runtime.Object, error) {
	return m.DeleteWithOptions(namespace, name, nil)
}

func (m *Helper) DeleteWithOptions(namespace, name string, options *metav1.DeleteOptions) (runtime.Object, error) {
	return m.RESTClient.Delete().
		NamespaceIfScoped(namespace, m.NamespaceScoped).
		Resource(m.Resource).
		Name(name).
		Body(options).
		Do().
		Get()
}

func (m *Helper) Create(namespace string, modify bool, obj runtime.Object, options *metav1.CreateOptions) (runtime.Object, error) {
	if options == nil {
		options = &metav1.CreateOptions{}
	}
	if modify {
		// Attempt to version the object based on client logic.
		version, err := metadataAccessor.ResourceVersion(obj)
		if err != nil {
			// We don't know how to clear the version on this object, so send it to the server as is
			return m.createResource(m.RESTClient, m.Resource, namespace, obj, options)
		}
		if version != "" {
			if err := metadataAccessor.SetResourceVersion(obj, ""); err != nil {
				return nil, err
			}
		}
	}

	return m.createResource(m.RESTClient, m.Resource, namespace, obj, options)
}

func (m *Helper) createResource(c RESTClient, resource, namespace string, obj runtime.Object, options *metav1.CreateOptions) (runtime.Object, error) {
	return c.Post().
		NamespaceIfScoped(namespace, m.NamespaceScoped).
		Resource(resource).
		VersionedParams(options, metav1.ParameterCodec).
		Body(obj).
		Do().
		Get()
}
func (m *Helper) Patch(namespace, name string, pt types.PatchType, data []byte, options *metav1.PatchOptions) (runtime.Object, error) {
	if options == nil {
		options = &metav1.PatchOptions{}
	}
	return m.RESTClient.Patch(pt).
		NamespaceIfScoped(namespace, m.NamespaceScoped).
		Resource(m.Resource).
		Name(name).
		VersionedParams(options, metav1.ParameterCodec).
		Body(data).
		Do().
		Get()
}

func (m *Helper) Replace(namespace, name string, overwrite bool, obj runtime.Object) (runtime.Object, error) {
	c := m.RESTClient

	// Attempt to version the object based on client logic.
	version, err := metadataAccessor.ResourceVersion(obj)
	if err != nil {
		// We don't know how to version this object, so send it to the server as is
		return m.replaceResource(c, m.Resource, namespace, name, obj)
	}
	if version == "" && overwrite {
		// Retrieve the current version of the object to overwrite the server object
		serverObj, err := c.Get().NamespaceIfScoped(namespace, m.NamespaceScoped).Resource(m.Resource).Name(name).Do().Get()
		if err != nil {
			// The object does not exist, but we want it to be created
			return m.replaceResource(c, m.Resource, namespace, name, obj)
		}
		serverVersion, err := metadataAccessor.ResourceVersion(serverObj)
		if err != nil {
			return nil, err
		}
		if err := metadataAccessor.SetResourceVersion(obj, serverVersion); err != nil {
			return nil, err
		}
	}

	return m.replaceResource(c, m.Resource, namespace, name, obj)
}

func (m *Helper) replaceResource(c RESTClient, resource, namespace, name string, obj runtime.Object) (runtime.Object, error) {
	return c.Put().NamespaceIfScoped(namespace, m.NamespaceScoped).Resource(resource).Name(name).Body(obj).Do().Get()
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package topologymanager

import (
	"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
	"k8s.io/kubernetes/pkg/kubelet/lifecycle"
)

// Policy interface for Topology Manager Pod Admit Result
type Policy interface {
	// Returns Policy Name
	Name() string
	// Returns a merged TopologyHint based on input from hint providers
	// and a Pod Admit Handler Response based on hints and policy type
	Merge(providersHints []map[string][]TopologyHint) (TopologyHint, lifecycle.PodAdmitResult)
}

// Merge a TopologyHints permutation to a single hint by performing a bitwise-AND
// of their affinity masks. The hint shall be preferred if all hits in the permutation
// are preferred.
func mergePermutation(numaNodes []int, permutation []TopologyHint) TopologyHint {
	// Get the NUMANodeAffinity from each hint in the permutation and see if any
	// of them encode unpreferred allocations.
	defaultAffinity, _ := bitmask.NewBitMask(numaNodes...)
	preferred := true
	var numaAffinities []bitmask.BitMask
	for _, hint := range permutation {
		// Only consider hints that have an actual NUMANodeAffinity set.
		if hint.NUMANodeAffinity == nil {
			numaAffinities = append(numaAffinities, defaultAffinity)
		} else {
			numaAffinities = append(numaAffinities, hint.NUMANodeAffinity)
		}

		if !hint.Preferred {
			preferred = false
		}
	}

	// Merge the affinities using a bitwise-and operation.
	mergedAffinity, _ := bitmask.NewBitMask(numaNodes...)
	mergedAffinity.And(numaAffinities...)
	// Build a mergedHint from the merged affinity mask, indicating if an
	// preferred allocation was used to generate the affinity mask or not.
	return TopologyHint{mergedAffinity, preferred}
}

// Iterate over all permutations of hints in 'allProviderHints [][]TopologyHint'.
//
// This procedure is implemented as a recursive function over the set of hints
// in 'allproviderHints[i]'. It applies the function 'callback' to each
// permutation as it is found. It is the equivalent of:
//
// for i := 0; i < len(providerHints[0]); i++
//     for j := 0; j < len(providerHints[1]); j++
//         for k := 0; k < len(providerHints[2]); k++
//             ...
//             for z := 0; z < len(providerHints[-1]); z++
//                 permutation := []TopologyHint{
//                     providerHints[0][i],
//                     providerHints[1][j],
//                     providerHints[2][k],
//                     ...
//                     providerHints[-1][z]
//                 }
//                 callback(permutation)
func iterateAllProviderTopologyHints(allProviderHints [][]TopologyHint, callback func([]TopologyHint)) {
	// Internal helper function to accumulate the permutation before calling the callback.
	var iterate func(i int, accum []TopologyHint)
	iterate = func(i int, accum []TopologyHint) {
		// Base case: we have looped through all providers and have a full permutation.
		if i == len(allProviderHints) {
			callback(accum)
			return
		}

		// Loop through all hints for provider 'i', and recurse to build the
		// the permutation of this hint with all hints from providers 'i++'.
		for j := range allProviderHints[i] {
			iterate(i+1, append(accum, allProviderHints[i][j]))
		}
	}
	iterate(0, []TopologyHint{})
}
<commit_msg>Cleanup use of defaultAffinity in mergePermutation of TopologyManager<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package topologymanager

import (
	"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
	"k8s.io/kubernetes/pkg/kubelet/lifecycle"
)

// Policy interface for Topology Manager Pod Admit Result
type Policy interface {
	// Returns Policy Name
	Name() string
	// Returns a merged TopologyHint based on input from hint providers
	// and a Pod Admit Handler Response based on hints and policy type
	Merge(providersHints []map[string][]TopologyHint) (TopologyHint, lifecycle.PodAdmitResult)
}

// Merge a TopologyHints permutation to a single hint by performing a bitwise-AND
// of their affinity masks. The hint shall be preferred if all hits in the permutation
// are preferred.
func mergePermutation(numaNodes []int, permutation []TopologyHint) TopologyHint {
	// Get the NUMANodeAffinity from each hint in the permutation and see if any
	// of them encode unpreferred allocations.
	preferred := true
	defaultAffinity, _ := bitmask.NewBitMask(numaNodes...)
	var numaAffinities []bitmask.BitMask
	for _, hint := range permutation {
		// Only consider hints that have an actual NUMANodeAffinity set.
		if hint.NUMANodeAffinity == nil {
			numaAffinities = append(numaAffinities, defaultAffinity)
		} else {
			numaAffinities = append(numaAffinities, hint.NUMANodeAffinity)
		}

		if !hint.Preferred {
			preferred = false
		}
	}

	// Merge the affinities using a bitwise-and operation.
	mergedAffinity := bitmask.And(defaultAffinity, numaAffinities...)
	// Build a mergedHint from the merged affinity mask, indicating if an
	// preferred allocation was used to generate the affinity mask or not.
	return TopologyHint{mergedAffinity, preferred}
}

// Iterate over all permutations of hints in 'allProviderHints [][]TopologyHint'.
//
// This procedure is implemented as a recursive function over the set of hints
// in 'allproviderHints[i]'. It applies the function 'callback' to each
// permutation as it is found. It is the equivalent of:
//
// for i := 0; i < len(providerHints[0]); i++
//     for j := 0; j < len(providerHints[1]); j++
//         for k := 0; k < len(providerHints[2]); k++
//             ...
//             for z := 0; z < len(providerHints[-1]); z++
//                 permutation := []TopologyHint{
//                     providerHints[0][i],
//                     providerHints[1][j],
//                     providerHints[2][k],
//                     ...
//                     providerHints[-1][z]
//                 }
//                 callback(permutation)
func iterateAllProviderTopologyHints(allProviderHints [][]TopologyHint, callback func([]TopologyHint)) {
	// Internal helper function to accumulate the permutation before calling the callback.
	var iterate func(i int, accum []TopologyHint)
	iterate = func(i int, accum []TopologyHint) {
		// Base case: we have looped through all providers and have a full permutation.
		if i == len(allProviderHints) {
			callback(accum)
			return
		}

		// Loop through all hints for provider 'i', and recurse to build the
		// the permutation of this hint with all hints from providers 'i++'.
		for j := range allProviderHints[i] {
			iterate(i+1, append(accum, allProviderHints[i][j]))
		}
	}
	iterate(0, []TopologyHint{})
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package bootstrapper

import (
	"bytes"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"

	"github.com/golang/glog"
	"github.com/pkg/errors"
	"k8s.io/minikube/pkg/minikube/assets"
)

// ExecRunner runs commands using the os/exec package.
//
// It implements the CommandRunner interface.
type ExecRunner struct{}

// Run starts the specified command in a bash shell and waits for it to complete.
func (*ExecRunner) Run(cmd string) error {
	glog.Infoln("Run:", cmd)
	c := exec.Command("/bin/bash", "-c", cmd)
	if err := c.Run(); err != nil {
		return errors.Wrapf(err, "running command: %s", cmd)
	}
	return nil
}

// CombinedOutputTo runs the command and stores both command
// output and error to out.
func (*ExecRunner) CombinedOutputTo(cmd string, out io.Writer) error {
	glog.Infoln("Run with output:", cmd)
	c := exec.Command("/bin/bash", "-c", cmd)
	c.Stdout = out
	c.Stderr = out
	err := c.Run()
	if err != nil {
		return errors.Wrapf(err, "running command: %s\n.", cmd)
	}

	return nil
}

// CombinedOutput runs the command  in a bash shell and returns its
// combined standard output and standard error.
func (e *ExecRunner) CombinedOutput(cmd string) (string, error) {
	var b bytes.Buffer
	err := e.CombinedOutputTo(cmd, &b)
	if err != nil {
		return "", errors.Wrapf(err, "running command: %s\n output: %s", cmd, b.Bytes())
	}
	return b.String(), nil

}

// Copy copies a file and its permissions
func (*ExecRunner) Copy(f assets.CopyableFile) error {
	if err := os.MkdirAll(f.GetTargetDir(), os.ModePerm); err != nil {
		return errors.Wrapf(err, "error making dirs for %s", f.GetTargetDir())
	}
	targetPath := filepath.Join(f.GetTargetDir(), f.GetTargetName())
	if _, err := os.Stat(targetPath); err == nil {
		if err := os.Remove(targetPath); err != nil {
			return errors.Wrapf(err, "error removing file %s", targetPath)
		}

	}
	target, err := os.Create(targetPath)
	if err != nil {
		return errors.Wrapf(err, "error creating file at %s", targetPath)
	}
	perms, err := strconv.Atoi(f.GetPermissions())
	if err != nil {
		return errors.Wrapf(err, "error converting permissions %s to integer", f.GetPermissions())
	}
	if err := os.Chmod(targetPath, os.FileMode(perms)); err != nil {
		return errors.Wrapf(err, "error changing file permissions for %s", targetPath)
	}

	if _, err = io.Copy(target, f); err != nil {
		return errors.Wrapf(err, `error copying file %s to target location:
do you have the correct permissions?`,
			targetPath)
	}
	return target.Close()
}

// Remove removes a file
func (e *ExecRunner) Remove(f assets.CopyableFile) error {
	targetPath := filepath.Join(f.GetTargetDir(), f.GetTargetName())
	return os.Remove(targetPath)
}
<commit_msg>ExecRunner.Copy now parses permissions in octal instead of decimal, as expected<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package bootstrapper

import (
	"bytes"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"

	"github.com/golang/glog"
	"github.com/pkg/errors"
	"k8s.io/minikube/pkg/minikube/assets"
)

// ExecRunner runs commands using the os/exec package.
//
// It implements the CommandRunner interface.
type ExecRunner struct{}

// Run starts the specified command in a bash shell and waits for it to complete.
func (*ExecRunner) Run(cmd string) error {
	glog.Infoln("Run:", cmd)
	c := exec.Command("/bin/bash", "-c", cmd)
	if err := c.Run(); err != nil {
		return errors.Wrapf(err, "running command: %s", cmd)
	}
	return nil
}

// CombinedOutputTo runs the command and stores both command
// output and error to out.
func (*ExecRunner) CombinedOutputTo(cmd string, out io.Writer) error {
	glog.Infoln("Run with output:", cmd)
	c := exec.Command("/bin/bash", "-c", cmd)
	c.Stdout = out
	c.Stderr = out
	err := c.Run()
	if err != nil {
		return errors.Wrapf(err, "running command: %s\n.", cmd)
	}

	return nil
}

// CombinedOutput runs the command  in a bash shell and returns its
// combined standard output and standard error.
func (e *ExecRunner) CombinedOutput(cmd string) (string, error) {
	var b bytes.Buffer
	err := e.CombinedOutputTo(cmd, &b)
	if err != nil {
		return "", errors.Wrapf(err, "running command: %s\n output: %s", cmd, b.Bytes())
	}
	return b.String(), nil

}

// Copy copies a file and its permissions
func (*ExecRunner) Copy(f assets.CopyableFile) error {
	if err := os.MkdirAll(f.GetTargetDir(), os.ModePerm); err != nil {
		return errors.Wrapf(err, "error making dirs for %s", f.GetTargetDir())
	}
	targetPath := filepath.Join(f.GetTargetDir(), f.GetTargetName())
	if _, err := os.Stat(targetPath); err == nil {
		if err := os.Remove(targetPath); err != nil {
			return errors.Wrapf(err, "error removing file %s", targetPath)
		}

	}
	target, err := os.Create(targetPath)
	if err != nil {
		return errors.Wrapf(err, "error creating file at %s", targetPath)
	}
	perms, err := strconv.ParseInt(f.GetPermissions(), 8, 0)
	if err != nil {
		return errors.Wrapf(err, "error converting permissions %s to integer", f.GetPermissions())
	}
	if err := os.Chmod(targetPath, os.FileMode(perms)); err != nil {
		return errors.Wrapf(err, "error changing file permissions for %s", targetPath)
	}

	if _, err = io.Copy(target, f); err != nil {
		return errors.Wrapf(err, `error copying file %s to target location:
do you have the correct permissions?`,
			targetPath)
	}
	return target.Close()
}

// Remove removes a file
func (e *ExecRunner) Remove(f assets.CopyableFile) error {
	targetPath := filepath.Join(f.GetTargetDir(), f.GetTargetName())
	return os.Remove(targetPath)
}
<|endoftext|>


<commit_before>// (c) Rick Arnold. Licensed under the BSD license (see LICENSE).

package cal

import "time"

// DefaultLoc is the default time.Location to use in functions that do not
// require a full time.Time value.
var DefaultLoc = time.Local

// Calendar represents a basic yearly calendar with a list of holidays.
type Calendar struct {
	Name        string           // calendar short name
	Description string           // calendar description
	Locations   []*time.Location // locations where the calendar applies
	Holidays    []*Holiday       // applicable holidays for this calendar
}

// IsApplicable reports whether the calendar is applicable for the given
// location.
//
// If no locations have been specified for the calendar, true is returned.
func (c *Calendar) IsApplicable(loc *time.Location) bool {
	if c.Locations == nil {
		return true
	}

	for _, l := range c.Locations {
		if l == loc {
			return true
		}
	}

	return false
}

// AddHoliday adds a holiday to the calendar's list.
func (c *Calendar) AddHoliday(h ...*Holiday) {
	if c.Holidays == nil {
		c.Holidays = make([]*Holiday, 0, 12)
	}

	for _, hd := range h {
		c.Holidays = append(c.Holidays, hd)
	}
}

// IsHoliday reports whether a given date is a holiday or an observation day.
func (c *Calendar) IsHoliday(date time.Time) (actual, observed bool, h *Holiday) {
	if c.Holidays == nil || !c.IsApplicable(date.Location()) {
		return false, false, nil
	}

	for _, hol := range c.Holidays {
		act, obs := hol.Calc(date.Year())
		actMatch := !act.IsZero() && act.Month() == date.Month() && act.Day() == date.Day()
		obsMatch := !obs.IsZero() && obs.Month() == date.Month() && obs.Day() == date.Day()
		if actMatch || obsMatch {
			return actMatch, obsMatch, hol
		}
	}

	return false, false, nil
}
<commit_msg>Speed up IsHoliday<commit_after>// (c) Rick Arnold. Licensed under the BSD license (see LICENSE).

package cal

import "time"

// DefaultLoc is the default time.Location to use in functions that do not
// require a full time.Time value.
var DefaultLoc = time.Local

// Calendar represents a basic yearly calendar with a list of holidays.
type Calendar struct {
	Name        string           // calendar short name
	Description string           // calendar description
	Locations   []*time.Location // locations where the calendar applies
	Holidays    []*Holiday       // applicable holidays for this calendar
}

// IsApplicable reports whether the calendar is applicable for the given
// location.
//
// If no locations have been specified for the calendar, true is returned.
func (c *Calendar) IsApplicable(loc *time.Location) bool {
	if c.Locations == nil {
		return true
	}

	for _, l := range c.Locations {
		if l == loc {
			return true
		}
	}

	return false
}

// AddHoliday adds a holiday to the calendar's list.
func (c *Calendar) AddHoliday(h ...*Holiday) {
	if c.Holidays == nil {
		c.Holidays = make([]*Holiday, 0, 12)
	}

	for _, hd := range h {
		c.Holidays = append(c.Holidays, hd)
	}
}

// IsHoliday reports whether a given date is a holiday or an observation day.
func (c *Calendar) IsHoliday(date time.Time) (actual, observed bool, h *Holiday) {
	if c.Holidays == nil || !c.IsApplicable(date.Location()) {
		return false, false, nil
	}

	year, month, day := date.Date()
	for _, hol := range c.Holidays {
		act, obs := hol.Calc(year)
		actMatch := !act.IsZero() && act.Month() == month && act.Day() == day
		obsMatch := !obs.IsZero() && obs.Month() == month && obs.Day() == day
		if actMatch || obsMatch {
			return actMatch, obsMatch, hol
		}
	}

	return false, false, nil
}
<|endoftext|>


<commit_before>package vanity // import "kkn.fi/vanity"

import (
	"fmt"
	"net/http"
)

type (
	// Path is the path component of the HTTP request sent by Go tool or browser.
	Path string
	// Package defines Go package that has vanity import defined by Path,
	// VCS system type and VCS URL.
	Package struct {
		// Path is path component of vanity url.
		Path Path
		// VCSSystem is version control system used by the project.
		// Usually 'git' or 'hg'.
		VCSSystem string
		// VCSURL is the HTTP URL for project's version control system.
		VCSURL string
	}
	// Server is the actual HTTP server for Go vanity domains.
	Server struct {
		// Domain is the vanity domain.
		Domain *string
		// Config contains settings for vanity packages.
		Config map[Path]Package
	}
)

// NewPackage returns a new Package given a path, VCS system and VCS URL.
func NewPackage(path, vcssystem, vcsurl string) *Package {
	p := &Package{
		Path:      Path(path),
		VCSSystem: vcssystem,
		VCSURL:    vcsurl,
	}
	return p
}

// NewServer returns a new Vanity Server given domain name and vanity package configuration.
func NewServer(domain string, config map[Path]Package) *Server {
	s := &Server{
		Domain: &domain,
		Config: config,
	}
	return s
}

// GoDocURL returns the HTTP URL to godoc.org.
func (p Package) GoDocURL(domain, path string) string {
	return fmt.Sprintf("https://godoc.org/%v%v", domain, path)
}

// GoImportLink creates the link used in HTML <meta/> tag
// where domain is the domain name of the server.
func (p Package) GoImportLink(domain string) string {
	return fmt.Sprintf("%s%s %s %s", domain, p.Path, p.VCSSystem, p.VCSURL)
}

// GoImportMeta creates the <meta/> HTML tag containing name and content attributes.
func (p Package) GoImportMeta(domain string) string {
	link := p.GoImportLink(domain)
	return fmt.Sprintf(`<meta name="go-import" content="%s">`, link)
}

func (s Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Content-Type", "text/html; charset=utf-8")
	if r.Method != http.MethodGet {
		http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
		return
	}

	pack, ok := s.Config[Path(r.URL.Path)]
	if !ok {
		http.NotFound(w, r)
		return
	}
	if r.FormValue("go-get") != "1" {
		url := pack.GoDocURL(*s.Domain, r.URL.Path)
		http.Redirect(w, r, url, http.StatusTemporaryRedirect)
		return
	}
	fmt.Fprint(w, pack.GoImportMeta(*s.Domain))
}
<commit_msg>Remove pointer from Server.Domain<commit_after>package vanity // import "kkn.fi/vanity"

import (
	"fmt"
	"net/http"
)

type (
	// Path is the path component of the HTTP request sent by Go tool or browser.
	Path string
	// Package defines Go package that has vanity import defined by Path,
	// VCS system type and VCS URL.
	Package struct {
		// Path is path component of vanity url.
		Path Path
		// VCSSystem is version control system used by the project.
		// Usually 'git' or 'hg'.
		VCSSystem string
		// VCSURL is the HTTP URL for project's version control system.
		VCSURL string
	}
	// Server is the actual HTTP server for Go vanity domains.
	Server struct {
		// Domain is the vanity domain.
		Domain string
		// Config contains settings for vanity packages.
		Config map[Path]Package
	}
)

// NewPackage returns a new Package given a path, VCS system and VCS URL.
func NewPackage(path, vcssystem, vcsurl string) *Package {
	p := &Package{
		Path:      Path(path),
		VCSSystem: vcssystem,
		VCSURL:    vcsurl,
	}
	return p
}

// NewServer returns a new Vanity Server given domain name and vanity package configuration.
func NewServer(domain string, config map[Path]Package) *Server {
	s := &Server{
		Domain: domain,
		Config: config,
	}
	return s
}

// GoDocURL returns the HTTP URL to godoc.org.
func (p Package) GoDocURL(domain, path string) string {
	return fmt.Sprintf("https://godoc.org/%v%v", domain, path)
}

// GoImportLink creates the link used in HTML <meta/> tag
// where domain is the domain name of the server.
func (p Package) GoImportLink(domain string) string {
	return fmt.Sprintf("%s%s %s %s", domain, p.Path, p.VCSSystem, p.VCSURL)
}

// GoImportMeta creates the <meta/> HTML tag containing name and content attributes.
func (p Package) GoImportMeta(domain string) string {
	link := p.GoImportLink(domain)
	return fmt.Sprintf(`<meta name="go-import" content="%s">`, link)
}

func (s Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Content-Type", "text/html; charset=utf-8")
	if r.Method != http.MethodGet {
		http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
		return
	}

	pack, ok := s.Config[Path(r.URL.Path)]
	if !ok {
		http.NotFound(w, r)
		return
	}
	if r.FormValue("go-get") != "1" {
		url := pack.GoDocURL(s.Domain, r.URL.Path)
		http.Redirect(w, r, url, http.StatusTemporaryRedirect)
		return
	}
	fmt.Fprint(w, pack.GoImportMeta(s.Domain))
}
<|endoftext|>


<commit_before>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A command that reads all blobs necessary for verifying the directory
// structure rooted at a set of backup root scores, ensuring that the entire
// directory structure is intact in GCS.
//
// Optionally, all file content is also read and verified. This is less
// important than verifying directory connectedness if we trust that GCS does
// not corrupt object metadata (where we store expected CRC32C and MD5) and
// does correctly report the object's CRC32C and MD5 sums in listings,
// verifying them periodically.
//
// Output of the following form is written to stdout and a file in the user's
// home directory:
//
//     <timestamp> <node> [<child node> ...]
//
// where:
//
//  *  Timestamps are formatted according to time.RFC3339.
//
//  *  Node names have one of two forms:
//
//     *   Nodes of the form "d:<hex score>" represent the directory listing
//         contained within the blob of the given score.
//
//     *   Nodes of the form "f:<hex score>" represent a piece of a file,
//         contained within the blob of the given score.
//
// An output line for a directory node means that at the given timestamp we
// certified that a piece of content with the given score was parseable as a
// directory listing that referred to the given scores for its direct children.
//
// An output line for a file node means that at the given timestamp we
// certified that a piece of content with the given score was parseable as a
// piece of a file. File nodes never have children.

package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"os/user"
	"path"
	"runtime"
	"strings"
	"time"

	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/graph"
	"github.com/jacobsa/comeback/util"
	"github.com/jacobsa/comeback/verify"
	"github.com/jacobsa/comeback/wiring"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

var cmdVerify = &Command{
	Name: "verify",
}

// TODO(jacobsa): Get these automatically from the registry.
var fRoots = cmdVerify.Flags.String(
	"roots",
	"",
	"Comma-separated list of backup root scores to verify.")

var fFast = cmdVerify.Flags.Bool(
	"fast",
	false,
	"When set, don't verify file content.")

func init() {
	cmdVerify.Run = runVerify // Break flag-related dependency loop.
}

////////////////////////////////////////////////////////////////////////
// Visitor types
////////////////////////////////////////////////////////////////////////

// A visitor that writes the information it gleans from the wrapped visitor to
// a channel.
type snoopingVisitor struct {
	records chan<- verify.Record
	wrapped graph.Visitor
}

func (v *snoopingVisitor) Visit(
	ctx context.Context,
	node string) (adjacent []string, err error) {
	// Call through.
	adjacent, err = v.wrapped.Visit(ctx, node)
	if err != nil {
		return
	}

	// Build a record.
	r := verify.Record{
		Time: time.Now(),
	}

	r.Node, err = verify.ParseNode(node)
	if err != nil {
		err = fmt.Errorf("ParseNode(%q): %v", node, err)
		return
	}

	for _, a := range adjacent {
		var adjacentNode verify.Node
		adjacentNode, err = verify.ParseNode(a)
		if err != nil {
			err = fmt.Errorf("ParseNode(%q): %v", a, err)
			return
		}

		r.Children = append(r.Children, adjacentNode)
	}

	// Write out the record.
	select {
	case <-ctx.Done():
		err = ctx.Err()
		return

	case v.records <- r:
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Output
////////////////////////////////////////////////////////////////////////

// Parse the supplied output from a run of the verify command, writing records
// into the supplied channel.
func parseVerifyOutput(
	ctx context.Context,
	in io.Reader,
	records chan<- verify.Record) (err error) {
	reader := bufio.NewReader(in)

	for {
		// Find the next line. EOF with no data means we are done; otherwise ignore
		// EOF in case the file doesn't end with a newline.
		var line []byte
		line, err = reader.ReadBytes('\n')
		if err == io.EOF {
			err = nil
			if len(line) == 0 {
				break
			}
		}

		// Propagate other errors.
		if err != nil {
			err = fmt.Errorf("ReadBytes: %v", err)
			return
		}

		// Trim the delimiter.
		line = line[:len(line)-1]

		// Parse the line.
		var r verify.Record
		r, err = verify.ParseRecord(string(line))
		if err != nil {
			err = fmt.Errorf("parseVerifyRecord(%q): %v", line, err)
			return
		}

		// Write out the record.
		select {
		case <-ctx.Done():
			err = ctx.Err()
			return

		case records <- r:
		}
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// List blob.ListScores, but returns a slice instead of writing into a channel.
func listAllScores(
	ctx context.Context,
	bucket gcs.Bucket,
	namePrefix string) (scores []blob.Score, err error) {
	b := syncutil.NewBundle(context.Background())
	defer func() { err = b.Join() }()

	// List scores into a channel.
	scoreChan := make(chan blob.Score, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(scoreChan)
		err = blob.ListScores(ctx, bucket, wiring.BlobObjectNamePrefix, scoreChan)
		if err != nil {
			err = fmt.Errorf("ListScores: %v", err)
			return
		}

		return
	})

	// Accumulate into the slice.
	b.Add(func(ctx context.Context) (err error) {
		for score := range scoreChan {
			scores = append(scores, score)
		}

		return
	})

	return
}

////////////////////////////////////////////////////////////////////////
// Verify
////////////////////////////////////////////////////////////////////////

// Run the verification pipeline. Return a count of the number of scores
// verified and the number skipped due to readFiles being false.
func verifyImpl(
	ctx context.Context,
	readFiles bool,
	rootScores []blob.Score,
	knownScores []blob.Score,
	blobStore blob.Store,
	output io.Writer) (nodesVerified uint64, nodesSkipped uint64, err error) {
	b := syncutil.NewBundle(ctx)

	// Visit every node in the graph, snooping on the graph structure into a
	// channel.
	visitorRecords := make(chan verify.Record, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(visitorRecords)

		visitor := verify.NewVisitor(
			readFiles,
			knownScores,
			blobStore)

		visitor = &snoopingVisitor{
			wrapped: visitor,
			records: visitorRecords,
		}

		// Format root node names.
		var roots []string
		for _, score := range rootScores {
			root := verify.Node{
				Score: score,
				Dir:   true,
			}

			roots = append(roots, root.String())
		}

		// Traverse starting at the specified roots. Use an "experimentally
		// determined" parallelism, which in theory should depend on bandwidth-delay
		// products but in practice comes down to when the OS gets cranky about open
		// files.
		const parallelism = 128

		err = graph.Traverse(
			context.Background(),
			parallelism,
			roots,
			visitor)

		if err != nil {
			err = fmt.Errorf("Traverse: %v", err)
			return
		}

		return
	})

	// Count and output the nodes visited, filtering out file nodes if we're not
	// actually reading and verifying them.
	b.Add(func(ctx context.Context) (err error) {
		for r := range visitorRecords {
			// Skip files if appropriate.
			if !readFiles && !r.Node.Dir {
				nodesSkipped++
				continue
			}

			// Increment the count.
			nodesVerified++

			// Output the information.
			_, err = fmt.Fprintf(output, "%s\n", r.String())
			if err != nil {
				err = fmt.Errorf("Fprintf: %v", err)
				return
			}
		}

		return
	})

	err = b.Join()
	return
}

// Open the file to which we log verify output.
func openVerifyLog() (w io.WriteCloser, err error) {
	// Find the current user.
	u, err := user.Current()
	if err != nil {
		err = fmt.Errorf("user.Current: %v", err)
		return
	}

	// Put the file in her home directory. Append to whatever is already there.
	w, err = os.OpenFile(
		path.Join(u.HomeDir, ".comeback.verify.log"),
		os.O_WRONLY|os.O_APPEND|os.O_CREATE,
		0600)

	if err != nil {
		err = fmt.Errorf("OpenFile: %v", err)
		return
	}

	return
}

func runVerify(args []string) {
	// Allow parallelism.
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Die on error.
	var err error
	defer func() {
		if err != nil {
			log.Fatalln(err)
		}
	}()

	// Read flags.
	readFiles := !*fFast

	if *fRoots == "" {
		err = fmt.Errorf("You must set --roots.")
		return
	}

	rootHexScores := strings.Split(*fRoots, ",")
	var rootScores []blob.Score
	for _, hexScore := range rootHexScores {
		var score blob.Score
		score, err = blob.ParseHexScore(hexScore)
		if err != nil {
			err = fmt.Errorf("Invalid root %q: %v", hexScore, err)
			return
		}

		rootScores = append(rootScores, score)
	}

	// Grab dependencies.
	bucket := getBucket()
	crypter := getCrypter()

	// Open the log file.
	logFile, err := openVerifyLog()
	if err != nil {
		err = fmt.Errorf("openVerifyLog: %v", err)
		return
	}

	defer logFile.Close()

	// Create a blob store.
	blobStore, err := wiring.MakeBlobStore(
		bucket,
		crypter,
		util.NewStringSet())

	if err != nil {
		err = fmt.Errorf("MakeBlobStore: %v", err)
		return
	}

	// List all scores in the bucket, verifying the object record metadata in the
	// process.
	log.Println("Listing scores...")
	knownScores, err := listAllScores(
		context.Background(),
		bucket,
		wiring.BlobObjectNamePrefix)

	if err != nil {
		err = fmt.Errorf("listAllScores: %v", err)
		return
	}

	log.Printf("Listed %d scores.", len(knownScores))

	// Run the rest of the pipeline.
	nodesVerified, nodesSkipped, err := verifyImpl(
		context.Background(),
		readFiles,
		rootScores,
		knownScores,
		blobStore,
		io.MultiWriter(logFile, os.Stdout))

	if err != nil {
		err = fmt.Errorf("verifyImpl: %v", err)
		return
	}

	log.Printf(
		"Successfully verified %d nodes (%d skipped due to fast mode).",
		nodesVerified,
		nodesSkipped)

	return
}
<commit_msg>Killed snoopingVisitor.<commit_after>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A command that reads all blobs necessary for verifying the directory
// structure rooted at a set of backup root scores, ensuring that the entire
// directory structure is intact in GCS.
//
// Optionally, all file content is also read and verified. This is less
// important than verifying directory connectedness if we trust that GCS does
// not corrupt object metadata (where we store expected CRC32C and MD5) and
// does correctly report the object's CRC32C and MD5 sums in listings,
// verifying them periodically.
//
// Output of the following form is written to stdout and a file in the user's
// home directory:
//
//     <timestamp> <node> [<child node> ...]
//
// where:
//
//  *  Timestamps are formatted according to time.RFC3339.
//
//  *  Node names have one of two forms:
//
//     *   Nodes of the form "d:<hex score>" represent the directory listing
//         contained within the blob of the given score.
//
//     *   Nodes of the form "f:<hex score>" represent a piece of a file,
//         contained within the blob of the given score.
//
// An output line for a directory node means that at the given timestamp we
// certified that a piece of content with the given score was parseable as a
// directory listing that referred to the given scores for its direct children.
//
// An output line for a file node means that at the given timestamp we
// certified that a piece of content with the given score was parseable as a
// piece of a file. File nodes never have children.

package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"os/user"
	"path"
	"runtime"
	"strings"

	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/graph"
	"github.com/jacobsa/comeback/util"
	"github.com/jacobsa/comeback/verify"
	"github.com/jacobsa/comeback/wiring"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

var cmdVerify = &Command{
	Name: "verify",
}

// TODO(jacobsa): Get these automatically from the registry.
var fRoots = cmdVerify.Flags.String(
	"roots",
	"",
	"Comma-separated list of backup root scores to verify.")

var fFast = cmdVerify.Flags.Bool(
	"fast",
	false,
	"When set, don't verify file content.")

func init() {
	cmdVerify.Run = runVerify // Break flag-related dependency loop.
}

////////////////////////////////////////////////////////////////////////
// Output
////////////////////////////////////////////////////////////////////////

// Parse the supplied output from a run of the verify command, writing records
// into the supplied channel.
func parseVerifyOutput(
	ctx context.Context,
	in io.Reader,
	records chan<- verify.Record) (err error) {
	reader := bufio.NewReader(in)

	for {
		// Find the next line. EOF with no data means we are done; otherwise ignore
		// EOF in case the file doesn't end with a newline.
		var line []byte
		line, err = reader.ReadBytes('\n')
		if err == io.EOF {
			err = nil
			if len(line) == 0 {
				break
			}
		}

		// Propagate other errors.
		if err != nil {
			err = fmt.Errorf("ReadBytes: %v", err)
			return
		}

		// Trim the delimiter.
		line = line[:len(line)-1]

		// Parse the line.
		var r verify.Record
		r, err = verify.ParseRecord(string(line))
		if err != nil {
			err = fmt.Errorf("parseVerifyRecord(%q): %v", line, err)
			return
		}

		// Write out the record.
		select {
		case <-ctx.Done():
			err = ctx.Err()
			return

		case records <- r:
		}
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// List blob.ListScores, but returns a slice instead of writing into a channel.
func listAllScores(
	ctx context.Context,
	bucket gcs.Bucket,
	namePrefix string) (scores []blob.Score, err error) {
	b := syncutil.NewBundle(context.Background())
	defer func() { err = b.Join() }()

	// List scores into a channel.
	scoreChan := make(chan blob.Score, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(scoreChan)
		err = blob.ListScores(ctx, bucket, wiring.BlobObjectNamePrefix, scoreChan)
		if err != nil {
			err = fmt.Errorf("ListScores: %v", err)
			return
		}

		return
	})

	// Accumulate into the slice.
	b.Add(func(ctx context.Context) (err error) {
		for score := range scoreChan {
			scores = append(scores, score)
		}

		return
	})

	return
}

////////////////////////////////////////////////////////////////////////
// Verify
////////////////////////////////////////////////////////////////////////

// Run the verification pipeline. Return a count of the number of scores
// verified and the number skipped due to readFiles being false.
func verifyImpl(
	ctx context.Context,
	readFiles bool,
	rootScores []blob.Score,
	knownScores []blob.Score,
	blobStore blob.Store,
	output io.Writer) (nodesVerified uint64, nodesSkipped uint64, err error) {
	b := syncutil.NewBundle(ctx)

	// Visit every node in the graph, snooping on the graph structure into a
	// channel.
	visitorRecords := make(chan verify.Record, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(visitorRecords)

		visitor := verify.NewVisitor(
			readFiles,
			knownScores,
			blobStore)

		visitor = &snoopingVisitor{
			wrapped: visitor,
			records: visitorRecords,
		}

		// Format root node names.
		var roots []string
		for _, score := range rootScores {
			root := verify.Node{
				Score: score,
				Dir:   true,
			}

			roots = append(roots, root.String())
		}

		// Traverse starting at the specified roots. Use an "experimentally
		// determined" parallelism, which in theory should depend on bandwidth-delay
		// products but in practice comes down to when the OS gets cranky about open
		// files.
		const parallelism = 128

		err = graph.Traverse(
			context.Background(),
			parallelism,
			roots,
			visitor)

		if err != nil {
			err = fmt.Errorf("Traverse: %v", err)
			return
		}

		return
	})

	// Count and output the nodes visited, filtering out file nodes if we're not
	// actually reading and verifying them.
	b.Add(func(ctx context.Context) (err error) {
		for r := range visitorRecords {
			// Skip files if appropriate.
			if !readFiles && !r.Node.Dir {
				nodesSkipped++
				continue
			}

			// Increment the count.
			nodesVerified++

			// Output the information.
			_, err = fmt.Fprintf(output, "%s\n", r.String())
			if err != nil {
				err = fmt.Errorf("Fprintf: %v", err)
				return
			}
		}

		return
	})

	err = b.Join()
	return
}

// Open the file to which we log verify output.
func openVerifyLog() (w io.WriteCloser, err error) {
	// Find the current user.
	u, err := user.Current()
	if err != nil {
		err = fmt.Errorf("user.Current: %v", err)
		return
	}

	// Put the file in her home directory. Append to whatever is already there.
	w, err = os.OpenFile(
		path.Join(u.HomeDir, ".comeback.verify.log"),
		os.O_WRONLY|os.O_APPEND|os.O_CREATE,
		0600)

	if err != nil {
		err = fmt.Errorf("OpenFile: %v", err)
		return
	}

	return
}

func runVerify(args []string) {
	// Allow parallelism.
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Die on error.
	var err error
	defer func() {
		if err != nil {
			log.Fatalln(err)
		}
	}()

	// Read flags.
	readFiles := !*fFast

	if *fRoots == "" {
		err = fmt.Errorf("You must set --roots.")
		return
	}

	rootHexScores := strings.Split(*fRoots, ",")
	var rootScores []blob.Score
	for _, hexScore := range rootHexScores {
		var score blob.Score
		score, err = blob.ParseHexScore(hexScore)
		if err != nil {
			err = fmt.Errorf("Invalid root %q: %v", hexScore, err)
			return
		}

		rootScores = append(rootScores, score)
	}

	// Grab dependencies.
	bucket := getBucket()
	crypter := getCrypter()

	// Open the log file.
	logFile, err := openVerifyLog()
	if err != nil {
		err = fmt.Errorf("openVerifyLog: %v", err)
		return
	}

	defer logFile.Close()

	// Create a blob store.
	blobStore, err := wiring.MakeBlobStore(
		bucket,
		crypter,
		util.NewStringSet())

	if err != nil {
		err = fmt.Errorf("MakeBlobStore: %v", err)
		return
	}

	// List all scores in the bucket, verifying the object record metadata in the
	// process.
	log.Println("Listing scores...")
	knownScores, err := listAllScores(
		context.Background(),
		bucket,
		wiring.BlobObjectNamePrefix)

	if err != nil {
		err = fmt.Errorf("listAllScores: %v", err)
		return
	}

	log.Printf("Listed %d scores.", len(knownScores))

	// Run the rest of the pipeline.
	nodesVerified, nodesSkipped, err := verifyImpl(
		context.Background(),
		readFiles,
		rootScores,
		knownScores,
		blobStore,
		io.MultiWriter(logFile, os.Stdout))

	if err != nil {
		err = fmt.Errorf("verifyImpl: %v", err)
		return
	}

	log.Printf(
		"Successfully verified %d nodes (%d skipped due to fast mode).",
		nodesVerified,
		nodesSkipped)

	return
}
<|endoftext|>