<commit_before>package restfulspec

import (
	"testing"

	restful "github.com/emicklei/go-restful"
	"github.com/go-openapi/spec"
)

func TestRouteToPath(t *testing.T) {
	description := "get the a b test\nthis is the test description"

	ws := new(restful.WebService)
	ws.Path("/tests/{v}")
	ws.Param(ws.PathParameter("v", "value of v").DefaultValue("default-v"))
	ws.Consumes(restful.MIME_JSON)
	ws.Produces(restful.MIME_XML)
	ws.Route(ws.GET("/a/{b}").To(dummy).
		Doc(description).
		Param(ws.PathParameter("b", "value of b").DefaultValue("default-b")).
		Param(ws.QueryParameter("q", "value of q").DefaultValue("default-q")).
		Returns(200, "list of a b tests", []Sample{}).
		Writes([]Sample{}))
	ws.Route(ws.GET("/a/{b}/{c:[a-z]+}/{d:[1-9]+}/e").To(dummy).
		Doc("get the a b test").
		Param(ws.PathParameter("b", "value of b").DefaultValue("default-b")).
		Param(ws.PathParameter("c", "with regex").DefaultValue("abc")).
		Param(ws.PathParameter("d", "with regex").DefaultValue("abcef")).
		Param(ws.QueryParameter("q", "value of q").DefaultValue("default-q")).
		Returns(200, "list of a b tests", []Sample{}).
		Writes([]Sample{}))

	p := buildPaths(ws, Config{})
	t.Log(asJSON(p))

	if p.Paths["/tests/{v}/a/{b}"].Get.Parameters[0].Type != "string" {
		t.Error("Parameter type is not set.")
	}
	if _, exists := p.Paths["/tests/{v}/a/{b}/{c}/{d}/e"]; !exists {
		t.Error("Expected path to exist after it was sanitized.")
	}

	if p.Paths["/tests/{v}/a/{b}"].Get.Description != description {
		t.Errorf("GET description incorrect")
	}
	if p.Paths["/tests/{v}/a/{b}"].Get.Summary != "get the a b test" {
		t.Errorf("GET summary incorrect")
	}
	response := p.Paths["/tests/{v}/a/{b}"].Get.Responses.StatusCodeResponses[200]
	if response.Schema.Type[0] != "array" {
		t.Errorf("response type incorrect")
	}
	if response.Schema.Items.Schema.Ref.String() != "#/definitions/restfulspec.Sample" {
		t.Errorf("response element type incorrect")
	}

	// Test for patterns
	path := p.Paths["/tests/{v}/a/{b}/{c}/{d}/e"]
	checkPattern(t, path, "c", "[a-z]+")
	checkPattern(t, path, "d", "[1-9]+")
	checkPattern(t, path, "v", "")
}

func getParameter(path spec.PathItem, name string) (*spec.Parameter, bool) {
	for _, param := range path.Get.Parameters {
		if param.Name == name {
			return &param, true
		}
	}
	return nil, false
}

func checkPattern(t *testing.T, path spec.PathItem, paramName string, pattern string) {
	param, exists := getParameter(path, paramName)
	if !exists {
		t.Error("Expected Parameter %s to exist", paramName)
	}
	if param.Pattern != pattern {
		t.Error("Expected pattern %s to equal %s", param.Pattern, pattern)
	}
}

func TestMultipleMethodsRouteToPath(t *testing.T) {
	ws := new(restful.WebService)
	ws.Path("/tests/a")
	ws.Consumes(restful.MIME_JSON)
	ws.Produces(restful.MIME_XML)
	ws.Route(ws.GET("/a/b").To(dummy).
		Doc("get a b test").
		Returns(200, "list of a b tests", []Sample{}).
		Writes([]Sample{}))
	ws.Route(ws.POST("/a/b").To(dummy).
		Doc("post a b test").
		Returns(200, "list of a b tests", []Sample{}).
		Returns(500, "internal server error", []Sample{}).
		Reads(Sample{}).
		Writes([]Sample{}))

	p := buildPaths(ws, Config{})
	t.Log(asJSON(p))

	if p.Paths["/tests/a/a/b"].Get.Description != "get a b test" {
		t.Errorf("GET description incorrect")
	}
	if p.Paths["/tests/a/a/b"].Post.Description != "post a b test" {
		t.Errorf("POST description incorrect")
	}
	if _, exists := p.Paths["/tests/a/a/b"].Post.Responses.StatusCodeResponses[500]; !exists {
		t.Errorf("Response code 500 not added to spec.")
	}

	expectedRef := spec.MustCreateRef("#/definitions/restfulspec.Sample")
	postBodyparam := p.Paths["/tests/a/a/b"].Post.Parameters[0]
	postBodyRef := postBodyparam.Schema.Ref
	if postBodyRef.String() != expectedRef.String() {
		t.Errorf("Expected: %s, Got: %s", expectedRef.String(), postBodyRef.String())
	}

	if postBodyparam.Format != "" || postBodyparam.Type != "" || postBodyparam.Default != nil {
		t.Errorf("Invalid parameter property is set on body property")
	}
}
<commit_msg>add unit test for reading array type in body.<commit_after>package restfulspec

import (
	"testing"

	restful "github.com/emicklei/go-restful"
	"github.com/go-openapi/spec"
)

func TestRouteToPath(t *testing.T) {
	description := "get the a b test\nthis is the test description"

	ws := new(restful.WebService)
	ws.Path("/tests/{v}")
	ws.Param(ws.PathParameter("v", "value of v").DefaultValue("default-v"))
	ws.Consumes(restful.MIME_JSON)
	ws.Produces(restful.MIME_XML)
	ws.Route(ws.GET("/a/{b}").To(dummy).
		Doc(description).
		Param(ws.PathParameter("b", "value of b").DefaultValue("default-b")).
		Param(ws.QueryParameter("q", "value of q").DefaultValue("default-q")).
		Returns(200, "list of a b tests", []Sample{}).
		Writes([]Sample{}))
	ws.Route(ws.GET("/a/{b}/{c:[a-z]+}/{d:[1-9]+}/e").To(dummy).
		Doc("get the a b test").
		Param(ws.PathParameter("b", "value of b").DefaultValue("default-b")).
		Param(ws.PathParameter("c", "with regex").DefaultValue("abc")).
		Param(ws.PathParameter("d", "with regex").DefaultValue("abcef")).
		Param(ws.QueryParameter("q", "value of q").DefaultValue("default-q")).
		Returns(200, "list of a b tests", []Sample{}).
		Writes([]Sample{}))

	p := buildPaths(ws, Config{})
	t.Log(asJSON(p))

	if p.Paths["/tests/{v}/a/{b}"].Get.Parameters[0].Type != "string" {
		t.Error("Parameter type is not set.")
	}
	if _, exists := p.Paths["/tests/{v}/a/{b}/{c}/{d}/e"]; !exists {
		t.Error("Expected path to exist after it was sanitized.")
	}

	if p.Paths["/tests/{v}/a/{b}"].Get.Description != description {
		t.Errorf("GET description incorrect")
	}
	if p.Paths["/tests/{v}/a/{b}"].Get.Summary != "get the a b test" {
		t.Errorf("GET summary incorrect")
	}
	response := p.Paths["/tests/{v}/a/{b}"].Get.Responses.StatusCodeResponses[200]
	if response.Schema.Type[0] != "array" {
		t.Errorf("response type incorrect")
	}
	if response.Schema.Items.Schema.Ref.String() != "#/definitions/restfulspec.Sample" {
		t.Errorf("response element type incorrect")
	}

	// Test for patterns
	path := p.Paths["/tests/{v}/a/{b}/{c}/{d}/e"]
	checkPattern(t, path, "c", "[a-z]+")
	checkPattern(t, path, "d", "[1-9]+")
	checkPattern(t, path, "v", "")
}

func getParameter(path spec.PathItem, name string) (*spec.Parameter, bool) {
	for _, param := range path.Get.Parameters {
		if param.Name == name {
			return &param, true
		}
	}
	return nil, false
}

func checkPattern(t *testing.T, path spec.PathItem, paramName string, pattern string) {
	param, exists := getParameter(path, paramName)
	if !exists {
		t.Error("Expected Parameter %s to exist", paramName)
	}
	if param.Pattern != pattern {
		t.Error("Expected pattern %s to equal %s", param.Pattern, pattern)
	}
}

func TestMultipleMethodsRouteToPath(t *testing.T) {
	ws := new(restful.WebService)
	ws.Path("/tests/a")
	ws.Consumes(restful.MIME_JSON)
	ws.Produces(restful.MIME_XML)
	ws.Route(ws.GET("/a/b").To(dummy).
		Doc("get a b test").
		Returns(200, "list of a b tests", []Sample{}).
		Writes([]Sample{}))
	ws.Route(ws.POST("/a/b").To(dummy).
		Doc("post a b test").
		Returns(200, "list of a b tests", []Sample{}).
		Returns(500, "internal server error", []Sample{}).
		Reads(Sample{}).
		Writes([]Sample{}))

	p := buildPaths(ws, Config{})
	t.Log(asJSON(p))

	if p.Paths["/tests/a/a/b"].Get.Description != "get a b test" {
		t.Errorf("GET description incorrect")
	}
	if p.Paths["/tests/a/a/b"].Post.Description != "post a b test" {
		t.Errorf("POST description incorrect")
	}
	if _, exists := p.Paths["/tests/a/a/b"].Post.Responses.StatusCodeResponses[500]; !exists {
		t.Errorf("Response code 500 not added to spec.")
	}

	expectedRef := spec.MustCreateRef("#/definitions/restfulspec.Sample")
	postBodyparam := p.Paths["/tests/a/a/b"].Post.Parameters[0]
	postBodyRef := postBodyparam.Schema.Ref
	if postBodyRef.String() != expectedRef.String() {
		t.Errorf("Expected: %s, Got: %s", expectedRef.String(), postBodyRef.String())
	}

	if postBodyparam.Format != "" || postBodyparam.Type != "" || postBodyparam.Default != nil {
		t.Errorf("Invalid parameter property is set on body property")
	}
}

func TestReadArrayObjectInBody(t *testing.T) {
	ws := new(restful.WebService)
	ws.Path("/tests/a")
	ws.Consumes(restful.MIME_JSON)
	ws.Produces(restful.MIME_XML)

	ws.Route(ws.POST("/a/b").To(dummy).
		Doc("post a b test with array in body").
		Returns(200, "list of a b tests", []Sample{}).
		Returns(500, "internal server error", []Sample{}).
		Reads([]Sample{}).
		Writes([]Sample{}))

	p := buildPaths(ws, Config{})
	t.Log(asJSON(p))

	postInfo := p.Paths["/tests/a/a/b"].Post

	if postInfo.Description != "post a b test with array in body" {
		t.Errorf("POST description incorrect")
	}
	if _, exists := postInfo.Responses.StatusCodeResponses[500]; !exists {
		t.Errorf("Response code 500 not added to spec.")
	}
	// indentify  element model type in body array
	expectedItemRef := spec.MustCreateRef("#/definitions/restfulspec.Sample")
	postBody := postInfo.Parameters[0]
	if postBody.Schema.Ref.String() != "" {
		t.Errorf("you shouldn't have body Ref setting when using array in body!")
	}
	// check body array dy item ref
	postBodyitems := postBody.Schema.Items.Schema.Ref
	if postBodyitems.String() != expectedItemRef.String() {
		t.Errorf("Expected: %s, Got: %s", expectedItemRef.String(), expectedItemRef.String())
	}

	if postBody.Format != "" || postBody.Type != "" || postBody.Default != nil {
		t.Errorf("Invalid parameter property is set on body property")
	}
}
<|endoftext|>


<commit_before>/*
 *
 * Copyright 2021 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

// Package googledirectpath implements a resolver that configures xds to make
// cloud to prod directpath connection.
//
// It's a combo of DNS and xDS resolvers. It delegates to DNS if
// - not on GCE, or
// - xDS bootstrap env var is set (so this client needs to do normal xDS, not
// direct path, and clients with this scheme is not part of the xDS mesh).
package googledirectpath

import (
	"fmt"
	"time"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials/google"
	"google.golang.org/grpc/grpclog"
	"google.golang.org/grpc/internal/envconfig"
	"google.golang.org/grpc/internal/googlecloud"
	internalgrpclog "google.golang.org/grpc/internal/grpclog"
	"google.golang.org/grpc/internal/grpcrand"
	"google.golang.org/grpc/resolver"
	_ "google.golang.org/grpc/xds" // To register xds resolvers and balancers.
	"google.golang.org/grpc/xds/internal/xdsclient"
	"google.golang.org/grpc/xds/internal/xdsclient/bootstrap"
	"google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version"
	"google.golang.org/protobuf/types/known/structpb"

	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
)

const (
	c2pScheme             = "google-c2p"
	c2pExperimentalScheme = "google-c2p-experimental"

	tdURL          = "dns:///directpath-pa.googleapis.com"
	httpReqTimeout = 10 * time.Second
	zoneURL        = "http://metadata.google.internal/computeMetadata/v1/instance/zone"
	ipv6URL        = "http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/ipv6s"

	gRPCUserAgentName               = "gRPC Go"
	clientFeatureNoOverprovisioning = "envoy.lb.does_not_support_overprovisioning"
	ipv6CapableMetadataName         = "TRAFFICDIRECTOR_DIRECTPATH_C2P_IPV6_CAPABLE"

	logPrefix = "[google-c2p-resolver]"

	dnsName, xdsName = "dns", "xds"
)

// For overriding in unittests.
var (
	onGCE = googlecloud.OnGCE

	newClientWithConfig = func(config *bootstrap.Config) (xdsclient.XDSClient, error) {
		return xdsclient.NewWithConfig(config)
	}

	logger = internalgrpclog.NewPrefixLogger(grpclog.Component("directpath"), logPrefix)
)

func init() {
	resolver.Register(c2pResolverBuilder{
		scheme: c2pScheme,
	})
	// TODO(apolcyn): remove this experimental scheme before the 1.52 release
	resolver.Register(c2pResolverBuilder{
		scheme: c2pExperimentalScheme,
	})
}

type c2pResolverBuilder struct {
	scheme string
}

func (c2pResolverBuilder) Build(t resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) {
	if !runDirectPath() {
		// If not xDS, fallback to DNS.
		t.Scheme = dnsName
		return resolver.Get(dnsName).Build(t, cc, opts)
	}

	// Note that the following calls to getZone() and getIPv6Capable() does I/O,
	// and has 10 seconds timeout each.
	//
	// This should be fine in most of the cases. In certain error cases, this
	// could block Dial() for up to 10 seconds (each blocking call has its own
	// goroutine).
	zoneCh, ipv6CapableCh := make(chan string), make(chan bool)
	go func() { zoneCh <- getZone(httpReqTimeout) }()
	go func() { ipv6CapableCh <- getIPv6Capable(httpReqTimeout) }()

	balancerName := envconfig.C2PResolverTestOnlyTrafficDirectorURI
	if balancerName == "" {
		balancerName = tdURL
	}
	serverConfig := &bootstrap.ServerConfig{
		ServerURI:    balancerName,
		Creds:        grpc.WithCredentialsBundle(google.NewDefaultCredentials()),
		TransportAPI: version.TransportV3,
		NodeProto:    newNode(<-zoneCh, <-ipv6CapableCh),
	}
	config := &bootstrap.Config{
		XDSServer: serverConfig,
		ClientDefaultListenerResourceNameTemplate: "%s",
		Authorities: map[string]*bootstrap.Authority{
			"traffic-director-c2p.xds.googleapis.com": {
				XDSServer: serverConfig,
			},
		},
	}

	// Create singleton xds client with this config. The xds client will be
	// used by the xds resolver later.
	xdsC, err := newClientWithConfig(config)
	if err != nil {
		return nil, fmt.Errorf("failed to start xDS client: %v", err)
	}

	// Create and return an xDS resolver.
	t.Scheme = xdsName
	xdsR, err := resolver.Get(xdsName).Build(t, cc, opts)
	if err != nil {
		xdsC.Close()
		return nil, err
	}
	return &c2pResolver{
		Resolver: xdsR,
		client:   xdsC,
	}, nil
}

func (b c2pResolverBuilder) Scheme() string {
	return b.scheme
}

type c2pResolver struct {
	resolver.Resolver
	client xdsclient.XDSClient
}

func (r *c2pResolver) Close() {
	r.Resolver.Close()
	r.client.Close()
}

var ipv6EnabledMetadata = &structpb.Struct{
	Fields: map[string]*structpb.Value{
		ipv6CapableMetadataName: structpb.NewBoolValue(true),
	},
}

var id = fmt.Sprintf("C2P-%d", grpcrand.Int())

// newNode makes a copy of defaultNode, and populate it's Metadata and
// Locality fields.
func newNode(zone string, ipv6Capable bool) *v3corepb.Node {
	ret := &v3corepb.Node{
		// Not all required fields are set in defaultNote. Metadata will be set
		// if ipv6 is enabled. Locality will be set to the value from metadata.
		Id:                   id,
		UserAgentName:        gRPCUserAgentName,
		UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version},
		ClientFeatures:       []string{clientFeatureNoOverprovisioning},
	}
	ret.Locality = &v3corepb.Locality{Zone: zone}
	if ipv6Capable {
		ret.Metadata = ipv6EnabledMetadata
	}
	return ret
}

// runDirectPath returns whether this resolver should use direct path.
//
// direct path is enabled if this client is running on GCE, and the normal xDS
// is not used (bootstrap env vars are not set) or federation is enabled.
func runDirectPath() bool {
	if !onGCE() {
		return false
	}
	return envconfig.XDSFederation || envconfig.XDSBootstrapFileName == "" && envconfig.XDSBootstrapFileContent == ""
}
<commit_msg>google-c2p: use new-style resource name for LDS subscription (#5743)<commit_after>/*
 *
 * Copyright 2021 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

// Package googledirectpath implements a resolver that configures xds to make
// cloud to prod directpath connection.
//
// It's a combo of DNS and xDS resolvers. It delegates to DNS if
// - not on GCE, or
// - xDS bootstrap env var is set (so this client needs to do normal xDS, not
// direct path, and clients with this scheme is not part of the xDS mesh).
package googledirectpath

import (
	"fmt"
	"net/url"
	"time"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials/google"
	"google.golang.org/grpc/grpclog"
	"google.golang.org/grpc/internal/envconfig"
	"google.golang.org/grpc/internal/googlecloud"
	internalgrpclog "google.golang.org/grpc/internal/grpclog"
	"google.golang.org/grpc/internal/grpcrand"
	"google.golang.org/grpc/resolver"
	_ "google.golang.org/grpc/xds" // To register xds resolvers and balancers.
	"google.golang.org/grpc/xds/internal/xdsclient"
	"google.golang.org/grpc/xds/internal/xdsclient/bootstrap"
	"google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version"
	"google.golang.org/protobuf/types/known/structpb"

	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
)

const (
	c2pScheme             = "google-c2p"
	c2pExperimentalScheme = "google-c2p-experimental"
	c2pAuthority          = "traffic-director-c2p.xds.googleapis.com"

	tdURL          = "dns:///directpath-pa.googleapis.com"
	httpReqTimeout = 10 * time.Second
	zoneURL        = "http://metadata.google.internal/computeMetadata/v1/instance/zone"
	ipv6URL        = "http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/ipv6s"

	gRPCUserAgentName               = "gRPC Go"
	clientFeatureNoOverprovisioning = "envoy.lb.does_not_support_overprovisioning"
	ipv6CapableMetadataName         = "TRAFFICDIRECTOR_DIRECTPATH_C2P_IPV6_CAPABLE"

	logPrefix = "[google-c2p-resolver]"

	dnsName, xdsName = "dns", "xds"
)

// For overriding in unittests.
var (
	onGCE = googlecloud.OnGCE

	newClientWithConfig = func(config *bootstrap.Config) (xdsclient.XDSClient, error) {
		return xdsclient.NewWithConfig(config)
	}

	logger = internalgrpclog.NewPrefixLogger(grpclog.Component("directpath"), logPrefix)
)

func init() {
	resolver.Register(c2pResolverBuilder{
		scheme: c2pScheme,
	})
	// TODO(apolcyn): remove this experimental scheme before the 1.52 release
	resolver.Register(c2pResolverBuilder{
		scheme: c2pExperimentalScheme,
	})
}

type c2pResolverBuilder struct {
	scheme string
}

func (c2pResolverBuilder) Build(t resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) {
	if !runDirectPath() {
		// If not xDS, fallback to DNS.
		t.Scheme = dnsName
		t.URL.Scheme = dnsName
		return resolver.Get(dnsName).Build(t, cc, opts)
	}

	// Note that the following calls to getZone() and getIPv6Capable() does I/O,
	// and has 10 seconds timeout each.
	//
	// This should be fine in most of the cases. In certain error cases, this
	// could block Dial() for up to 10 seconds (each blocking call has its own
	// goroutine).
	zoneCh, ipv6CapableCh := make(chan string), make(chan bool)
	go func() { zoneCh <- getZone(httpReqTimeout) }()
	go func() { ipv6CapableCh <- getIPv6Capable(httpReqTimeout) }()

	balancerName := envconfig.C2PResolverTestOnlyTrafficDirectorURI
	if balancerName == "" {
		balancerName = tdURL
	}
	serverConfig := &bootstrap.ServerConfig{
		ServerURI:    balancerName,
		Creds:        grpc.WithCredentialsBundle(google.NewDefaultCredentials()),
		TransportAPI: version.TransportV3,
		NodeProto:    newNode(<-zoneCh, <-ipv6CapableCh),
	}
	config := &bootstrap.Config{
		XDSServer: serverConfig,
		ClientDefaultListenerResourceNameTemplate: "%s",
		Authorities: map[string]*bootstrap.Authority{
			c2pAuthority: {
				XDSServer: serverConfig,
			},
		},
	}

	// Create singleton xds client with this config. The xds client will be
	// used by the xds resolver later.
	xdsC, err := newClientWithConfig(config)
	if err != nil {
		return nil, fmt.Errorf("failed to start xDS client: %v", err)
	}

	// Create and return an xDS resolver.
	t.Scheme = xdsName
	t.URL.Scheme = xdsName
	if envconfig.XDSFederation {
		t = resolver.Target{
			URL: url.URL{
				Scheme: xdsName,
				Host:   c2pAuthority,
				Path:   t.URL.Path,
			},
		}
	}
	xdsR, err := resolver.Get(xdsName).Build(t, cc, opts)
	if err != nil {
		xdsC.Close()
		return nil, err
	}
	return &c2pResolver{
		Resolver: xdsR,
		client:   xdsC,
	}, nil
}

func (b c2pResolverBuilder) Scheme() string {
	return b.scheme
}

type c2pResolver struct {
	resolver.Resolver
	client xdsclient.XDSClient
}

func (r *c2pResolver) Close() {
	r.Resolver.Close()
	r.client.Close()
}

var ipv6EnabledMetadata = &structpb.Struct{
	Fields: map[string]*structpb.Value{
		ipv6CapableMetadataName: structpb.NewBoolValue(true),
	},
}

var id = fmt.Sprintf("C2P-%d", grpcrand.Int())

// newNode makes a copy of defaultNode, and populate it's Metadata and
// Locality fields.
func newNode(zone string, ipv6Capable bool) *v3corepb.Node {
	ret := &v3corepb.Node{
		// Not all required fields are set in defaultNote. Metadata will be set
		// if ipv6 is enabled. Locality will be set to the value from metadata.
		Id:                   id,
		UserAgentName:        gRPCUserAgentName,
		UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version},
		ClientFeatures:       []string{clientFeatureNoOverprovisioning},
	}
	ret.Locality = &v3corepb.Locality{Zone: zone}
	if ipv6Capable {
		ret.Metadata = ipv6EnabledMetadata
	}
	return ret
}

// runDirectPath returns whether this resolver should use direct path.
//
// direct path is enabled if this client is running on GCE, and the normal xDS
// is not used (bootstrap env vars are not set) or federation is enabled.
func runDirectPath() bool {
	if !onGCE() {
		return false
	}
	return envconfig.XDSFederation || envconfig.XDSBootstrapFileName == "" && envconfig.XDSBootstrapFileContent == ""
}
<|endoftext|>


<commit_before>/*
 *
 * Copyright 2021 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

// Package googledirectpath implements a resolver that configures xds to make
// cloud to prod directpath connection.
//
// It's a combo of DNS and xDS resolvers. It delegates to DNS if
// - not on GCE, or
// - xDS bootstrap env var is set (so this client needs to do normal xDS, not
// direct path, and clients with this scheme is not part of the xDS mesh).
package googledirectpath

import (
	"fmt"
	"time"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials/google"
	"google.golang.org/grpc/grpclog"
	"google.golang.org/grpc/internal/envconfig"
	"google.golang.org/grpc/internal/googlecloud"
	internalgrpclog "google.golang.org/grpc/internal/grpclog"
	"google.golang.org/grpc/internal/grpcrand"
	"google.golang.org/grpc/resolver"
	_ "google.golang.org/grpc/xds" // To register xds resolvers and balancers.
	"google.golang.org/grpc/xds/internal/xdsclient"
	"google.golang.org/grpc/xds/internal/xdsclient/bootstrap"
	"google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version"
	"google.golang.org/protobuf/types/known/structpb"

	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
)

const (
	c2pScheme = "google-c2p-experimental"

	tdURL          = "dns:///directpath-pa.googleapis.com"
	httpReqTimeout = 10 * time.Second
	zoneURL        = "http://metadata.google.internal/computeMetadata/v1/instance/zone"
	ipv6URL        = "http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/ipv6s"

	gRPCUserAgentName               = "gRPC Go"
	clientFeatureNoOverprovisioning = "envoy.lb.does_not_support_overprovisioning"
	ipv6CapableMetadataName         = "TRAFFICDIRECTOR_DIRECTPATH_C2P_IPV6_CAPABLE"

	logPrefix = "[google-c2p-resolver]"

	dnsName, xdsName = "dns", "xds"
)

// For overriding in unittests.
var (
	onGCE = googlecloud.OnGCE

	newClientWithConfig = func(config *bootstrap.Config) (xdsclient.XDSClient, error) {
		return xdsclient.NewWithConfig(config)
	}

	logger = internalgrpclog.NewPrefixLogger(grpclog.Component("directpath"), logPrefix)
)

func init() {
	resolver.Register(c2pResolverBuilder{})
}

type c2pResolverBuilder struct{}

func (c2pResolverBuilder) Build(t resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) {
	if !runDirectPath() {
		// If not xDS, fallback to DNS.
		t.Scheme = dnsName
		return resolver.Get(dnsName).Build(t, cc, opts)
	}

	// Note that the following calls to getZone() and getIPv6Capable() does I/O,
	// and has 10 seconds timeout each.
	//
	// This should be fine in most of the cases. In certain error cases, this
	// could block Dial() for up to 10 seconds (each blocking call has its own
	// goroutine).
	zoneCh, ipv6CapableCh := make(chan string), make(chan bool)
	go func() { zoneCh <- getZone(httpReqTimeout) }()
	go func() { ipv6CapableCh <- getIPv6Capable(httpReqTimeout) }()

	balancerName := envconfig.C2PResolverTestOnlyTrafficDirectorURI
	if balancerName == "" {
		balancerName = tdURL
	}
	config := &bootstrap.Config{
		XDSServer: &bootstrap.ServerConfig{
			ServerURI:    balancerName,
			Creds:        grpc.WithCredentialsBundle(google.NewDefaultCredentials()),
			TransportAPI: version.TransportV3,
			NodeProto:    newNode(<-zoneCh, <-ipv6CapableCh),
		},
		ClientDefaultListenerResourceNameTemplate: "%s",
	}

	// Create singleton xds client with this config. The xds client will be
	// used by the xds resolver later.
	xdsC, err := newClientWithConfig(config)
	if err != nil {
		return nil, fmt.Errorf("failed to start xDS client: %v", err)
	}

	// Create and return an xDS resolver.
	t.Scheme = xdsName
	xdsR, err := resolver.Get(xdsName).Build(t, cc, opts)
	if err != nil {
		xdsC.Close()
		return nil, err
	}
	return &c2pResolver{
		Resolver: xdsR,
		client:   xdsC,
	}, nil
}

func (c2pResolverBuilder) Scheme() string {
	return c2pScheme
}

type c2pResolver struct {
	resolver.Resolver
	client xdsclient.XDSClient
}

func (r *c2pResolver) Close() {
	r.Resolver.Close()
	r.client.Close()
}

var ipv6EnabledMetadata = &structpb.Struct{
	Fields: map[string]*structpb.Value{
		ipv6CapableMetadataName: structpb.NewBoolValue(true),
	},
}

var id = fmt.Sprintf("C2P-%d", grpcrand.Int())

// newNode makes a copy of defaultNode, and populate it's Metadata and
// Locality fields.
func newNode(zone string, ipv6Capable bool) *v3corepb.Node {
	ret := &v3corepb.Node{
		// Not all required fields are set in defaultNote. Metadata will be set
		// if ipv6 is enabled. Locality will be set to the value from metadata.
		Id:                   id,
		UserAgentName:        gRPCUserAgentName,
		UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version},
		ClientFeatures:       []string{clientFeatureNoOverprovisioning},
	}
	ret.Locality = &v3corepb.Locality{Zone: zone}
	if ipv6Capable {
		ret.Metadata = ipv6EnabledMetadata
	}
	return ret
}

// runDirectPath returns whether this resolver should use direct path.
//
// direct path is enabled if this client is running on GCE, and the normal xDS
// is not used (bootstrap env vars are not set).
func runDirectPath() bool {
	return envconfig.XDSBootstrapFileName == "" && envconfig.XDSBootstrapFileContent == "" && onGCE()
}
<commit_msg>xds: de-experimentalize google c2p resolver (#5707)<commit_after>/*
 *
 * Copyright 2021 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

// Package googledirectpath implements a resolver that configures xds to make
// cloud to prod directpath connection.
//
// It's a combo of DNS and xDS resolvers. It delegates to DNS if
// - not on GCE, or
// - xDS bootstrap env var is set (so this client needs to do normal xDS, not
// direct path, and clients with this scheme is not part of the xDS mesh).
package googledirectpath

import (
	"fmt"
	"time"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials/google"
	"google.golang.org/grpc/grpclog"
	"google.golang.org/grpc/internal/envconfig"
	"google.golang.org/grpc/internal/googlecloud"
	internalgrpclog "google.golang.org/grpc/internal/grpclog"
	"google.golang.org/grpc/internal/grpcrand"
	"google.golang.org/grpc/resolver"
	_ "google.golang.org/grpc/xds" // To register xds resolvers and balancers.
	"google.golang.org/grpc/xds/internal/xdsclient"
	"google.golang.org/grpc/xds/internal/xdsclient/bootstrap"
	"google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version"
	"google.golang.org/protobuf/types/known/structpb"

	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
)

const (
	c2pScheme             = "google-c2p"
	c2pExperimentalScheme = "google-c2p-experimental"

	tdURL          = "dns:///directpath-pa.googleapis.com"
	httpReqTimeout = 10 * time.Second
	zoneURL        = "http://metadata.google.internal/computeMetadata/v1/instance/zone"
	ipv6URL        = "http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/ipv6s"

	gRPCUserAgentName               = "gRPC Go"
	clientFeatureNoOverprovisioning = "envoy.lb.does_not_support_overprovisioning"
	ipv6CapableMetadataName         = "TRAFFICDIRECTOR_DIRECTPATH_C2P_IPV6_CAPABLE"

	logPrefix = "[google-c2p-resolver]"

	dnsName, xdsName = "dns", "xds"
)

// For overriding in unittests.
var (
	onGCE = googlecloud.OnGCE

	newClientWithConfig = func(config *bootstrap.Config) (xdsclient.XDSClient, error) {
		return xdsclient.NewWithConfig(config)
	}

	logger = internalgrpclog.NewPrefixLogger(grpclog.Component("directpath"), logPrefix)
)

func init() {
	resolver.Register(c2pResolverBuilder{
		scheme: c2pScheme,
	})
	// TODO(apolcyn): remove this experimental scheme before the 1.52 release
	resolver.Register(c2pResolverBuilder{
		scheme: c2pExperimentalScheme,
	})
}

type c2pResolverBuilder struct {
	scheme string
}

func (c2pResolverBuilder) Build(t resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) {
	if !runDirectPath() {
		// If not xDS, fallback to DNS.
		t.Scheme = dnsName
		return resolver.Get(dnsName).Build(t, cc, opts)
	}

	// Note that the following calls to getZone() and getIPv6Capable() does I/O,
	// and has 10 seconds timeout each.
	//
	// This should be fine in most of the cases. In certain error cases, this
	// could block Dial() for up to 10 seconds (each blocking call has its own
	// goroutine).
	zoneCh, ipv6CapableCh := make(chan string), make(chan bool)
	go func() { zoneCh <- getZone(httpReqTimeout) }()
	go func() { ipv6CapableCh <- getIPv6Capable(httpReqTimeout) }()

	balancerName := envconfig.C2PResolverTestOnlyTrafficDirectorURI
	if balancerName == "" {
		balancerName = tdURL
	}
	config := &bootstrap.Config{
		XDSServer: &bootstrap.ServerConfig{
			ServerURI:    balancerName,
			Creds:        grpc.WithCredentialsBundle(google.NewDefaultCredentials()),
			TransportAPI: version.TransportV3,
			NodeProto:    newNode(<-zoneCh, <-ipv6CapableCh),
		},
		ClientDefaultListenerResourceNameTemplate: "%s",
	}

	// Create singleton xds client with this config. The xds client will be
	// used by the xds resolver later.
	xdsC, err := newClientWithConfig(config)
	if err != nil {
		return nil, fmt.Errorf("failed to start xDS client: %v", err)
	}

	// Create and return an xDS resolver.
	t.Scheme = xdsName
	xdsR, err := resolver.Get(xdsName).Build(t, cc, opts)
	if err != nil {
		xdsC.Close()
		return nil, err
	}
	return &c2pResolver{
		Resolver: xdsR,
		client:   xdsC,
	}, nil
}

func (b c2pResolverBuilder) Scheme() string {
	return b.scheme
}

type c2pResolver struct {
	resolver.Resolver
	client xdsclient.XDSClient
}

func (r *c2pResolver) Close() {
	r.Resolver.Close()
	r.client.Close()
}

var ipv6EnabledMetadata = &structpb.Struct{
	Fields: map[string]*structpb.Value{
		ipv6CapableMetadataName: structpb.NewBoolValue(true),
	},
}

var id = fmt.Sprintf("C2P-%d", grpcrand.Int())

// newNode makes a copy of defaultNode, and populate it's Metadata and
// Locality fields.
func newNode(zone string, ipv6Capable bool) *v3corepb.Node {
	ret := &v3corepb.Node{
		// Not all required fields are set in defaultNote. Metadata will be set
		// if ipv6 is enabled. Locality will be set to the value from metadata.
		Id:                   id,
		UserAgentName:        gRPCUserAgentName,
		UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version},
		ClientFeatures:       []string{clientFeatureNoOverprovisioning},
	}
	ret.Locality = &v3corepb.Locality{Zone: zone}
	if ipv6Capable {
		ret.Metadata = ipv6EnabledMetadata
	}
	return ret
}

// runDirectPath returns whether this resolver should use direct path.
//
// direct path is enabled if this client is running on GCE, and the normal xDS
// is not used (bootstrap env vars are not set).
func runDirectPath() bool {
	return envconfig.XDSBootstrapFileName == "" && envconfig.XDSBootstrapFileContent == "" && onGCE()
}
<|endoftext|>


<commit_before>/*
Package builtin implements some built-in functions for gosl (Go Language
Script Language, github.com/daviddengcn/gosl)

For use of convinience as a script language, the parameters are commonly
defined as an interface{}.
*/
package builtin

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"path"
	"sort"
	"strconv"
	"strings"
	"syscall"
)

/*
S converts anything into a string. If args is specified, v is used as a format
string.
*/
func S(v interface{}, args ...interface{}) string {
	return fmt.Sprintf(fmt.Sprint(v), args...)
}

/*
I converts anything into an int. When the value is malformed, if the optional
default value is specified, it is converted to int and returned; otherwise,
0 is returned.
*/
func I(v interface{}, def ...interface{}) int {
	if i, ok := v.(int); ok {
		return i
	}
	if i, ok := v.(int64); ok {
		return int(i)
	}

	i, err := strconv.Atoi(S(v))
	if err != nil && len(def) > 0 {
		return I(def[0])
	}
	return i
}

func execCode(err error) int {
	if exiterr, ok := err.(*exec.ExitError); ok {
		if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
			return status.ExitStatus()
		}
	}
	return 0
}

/*
Exec runs a command. exe is the path to the executable and args are arugment
passed to it.

If the command is executed successfuly without mistake, (nil, 0)
will be returned. Otherwise, the error and error code will be returned.

Stdout/stderr are directed the current stdout/stderr.
*/
func Exec(exe interface{}, args ...string) (error, int) {
	cmd := exec.Command(S(exe), args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	err := cmd.Run()
	return err, execCode(err)
}

/*
ExecWithStdout is similar to Exec but the stdout is captured and returned as
the first return value.
*/
func ExecWithStdout(exe interface{}, args ...string) (stdout string, err error, errCode int) {
	var stdoutBuf bytes.Buffer

	cmd := exec.Command(S(exe), args...)
	cmd.Stdout = &stdoutBuf
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	err = cmd.Run()

	return string(stdoutBuf.Bytes()), err, execCode(err)
}

/*
ExecWithStdout is similar to Exec but the stdout/stderr are captured and
returned as the first/second return values.
*/
func ExecWithStdErrOut(exe interface{}, args ...string) (stdout, stderr string, err error, errCode int) {
	var stdoutBuf, stderrBuf bytes.Buffer

	cmd := exec.Command(S(exe), args...)
	cmd.Stdout = &stdoutBuf
	cmd.Stderr = &stderrBuf
	cmd.Stdin = os.Stdin
	err = cmd.Run()

	return string(stdoutBuf.Bytes()), string(stderrBuf.Bytes()), err, execCode(err)
}

/*
Eval is similar to ExecWithStdout but with stdout captured and returned as a
string. Trainling newlines are deleted.
*/
func Eval(exe interface{}, args ...string) string {
	out, _, _ := ExecWithStdout(exe, args...)
	return strings.TrimRight(out, "\r\n")
}

/*
Bash runs a command with bash. Return values are defined in Exec.
*/
func Bash(cmd interface{}, args ...interface{}) (error, int) {
	return Exec("bash", "-c", S(cmd, args...))
}

/*
BashWithStdout is similar to Bash but with stdout captured and returned as a
string.
*/
func BashWithStdout(cmd interface{}, args ...interface{}) (string, error, int) {
	return ExecWithStdout("bash", "-c", S(cmd, args...))
}

/*
BashEval is similar to BashWithStdout but with stdout captured and returned
as a string. Trainling newlines are deleted.
*/
func BashEval(cmd interface{}, args ...interface{}) string {
	out, _, _ := BashWithStdout(cmd, args...)
	return strings.TrimRight(out, "\r\n")
}

/*
Similar to os.Getwd() but no error returned.
*/
func Pwd() string {
	pwd, _ := os.Getwd()
	return pwd
}

/*
DefExitCode is the default exit code.
*/
var DefExitCode = 1

/*
Fatalf print a message and exit the program with DefExitCode.
*/
func Fatalf(msg interface{}, args ...interface{}) {
	fmt.Fprintln(os.Stderr, S(msg, args...))
	os.Exit(DefExitCode)
}

/*
Eprintf is similar to fmt.Printf but output is stderr.
*/
func Eprintf(format interface{}, args ...interface{}) {
	fmt.Fprint(os.Stderr, S(format, args...))
}

/*
Eprint is similar to fmt.Print but output is stderr.
*/
func Eprint(args ...interface{}) {
	fmt.Fprint(os.Stderr, args...)
}

/*
Eprintln is similar to fmt.Println but output is stderr.
*/
func Eprintln(args ...interface{}) {
	fmt.Fprintln(os.Stderr, args...)
}

/*
Eprintfln is similar to Eprintf but with a trailing new-line printed
*/
func Eprintfln(format interface{}, args ...interface{}) {
	fmt.Fprintln(os.Stderr, S(format, args...))
}

/*
Printfln is similar to Eprintf but with a trailing new-line printed
*/
func Printfln(format interface{}, args ...interface{}) {
	fmt.Fprintln(S(format, args...))
}

/*
MustSucc checks the result of Exec/Bash. If not succeed, exit the application.
*/
func MustSucc(err error, code int) {
	if err == nil {
		return
	}

	if code != 0 {
		Fatalf("Failed with error code: %d", code)
	}

	Fatalf("Failed with error: %v", err)
}

type sortI struct {
	l    int
	less func(int, int) bool
	swap func(int, int)
}

func (s *sortI) Len() int {
	return s.l
}

func (s *sortI) Less(i, j int) bool {
	return s.less(i, j)
}

func (s *sortI) Swap(i, j int) {
	s.swap(i, j)
}

/*
SortF sorts the data defined by the length, Less and Swap functions.
*/
func SortF(Len int, Less func(int, int) bool, Swap func(int, int)) {
	sort.Sort(&sortI{l: Len, less: Less, swap: Swap})
}

/*
ScriptDir returns the folder of the current script.
*/
func ScriptDir() string {
	return path.Dir(os.Args[0])
}

/*
Exists checks whether the path exists
*/
func Exists(p interface{}, args ...interface{}) bool {
	_, err := os.Stat(S(p, args...))
	return err == nil
}

/*
IsDir returns true only if the path exists and indicates a directory
*/
func IsDir(p interface{}, args ...interface{}) bool {
	info, err := os.Stat(S(p, args...))
	if err != nil {
		// the path does not exist
		return false
	}
	return info.Mode().IsDir()
}

/*
IsFile returns true only if the path exists and indicates a file
*/
func IsFile(p interface{}, args ...interface{}) bool {
	info, err := os.Stat(S(p, args...))
	if err != nil {
		// the path does not exist
		return false
	}
	return !info.Mode().IsDir()
}
<commit_msg>Fix Printfln<commit_after>/*
Package builtin implements some built-in functions for gosl (Go Language
Script Language, github.com/daviddengcn/gosl)

For use of convinience as a script language, the parameters are commonly
defined as an interface{}.
*/
package builtin

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"path"
	"sort"
	"strconv"
	"strings"
	"syscall"
)

/*
S converts anything into a string. If args is specified, v is used as a format
string.
*/
func S(v interface{}, args ...interface{}) string {
	return fmt.Sprintf(fmt.Sprint(v), args...)
}

/*
I converts anything into an int. When the value is malformed, if the optional
default value is specified, it is converted to int and returned; otherwise,
0 is returned.
*/
func I(v interface{}, def ...interface{}) int {
	if i, ok := v.(int); ok {
		return i
	}
	if i, ok := v.(int64); ok {
		return int(i)
	}

	i, err := strconv.Atoi(S(v))
	if err != nil && len(def) > 0 {
		return I(def[0])
	}
	return i
}

func execCode(err error) int {
	if exiterr, ok := err.(*exec.ExitError); ok {
		if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
			return status.ExitStatus()
		}
	}
	return 0
}

/*
Exec runs a command. exe is the path to the executable and args are arugment
passed to it.

If the command is executed successfuly without mistake, (nil, 0)
will be returned. Otherwise, the error and error code will be returned.

Stdout/stderr are directed the current stdout/stderr.
*/
func Exec(exe interface{}, args ...string) (error, int) {
	cmd := exec.Command(S(exe), args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	err := cmd.Run()
	return err, execCode(err)
}

/*
ExecWithStdout is similar to Exec but the stdout is captured and returned as
the first return value.
*/
func ExecWithStdout(exe interface{}, args ...string) (stdout string, err error, errCode int) {
	var stdoutBuf bytes.Buffer

	cmd := exec.Command(S(exe), args...)
	cmd.Stdout = &stdoutBuf
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	err = cmd.Run()

	return string(stdoutBuf.Bytes()), err, execCode(err)
}

/*
ExecWithStdout is similar to Exec but the stdout/stderr are captured and
returned as the first/second return values.
*/
func ExecWithStdErrOut(exe interface{}, args ...string) (stdout, stderr string, err error, errCode int) {
	var stdoutBuf, stderrBuf bytes.Buffer

	cmd := exec.Command(S(exe), args...)
	cmd.Stdout = &stdoutBuf
	cmd.Stderr = &stderrBuf
	cmd.Stdin = os.Stdin
	err = cmd.Run()

	return string(stdoutBuf.Bytes()), string(stderrBuf.Bytes()), err, execCode(err)
}

/*
Eval is similar to ExecWithStdout but with stdout captured and returned as a
string. Trainling newlines are deleted.
*/
func Eval(exe interface{}, args ...string) string {
	out, _, _ := ExecWithStdout(exe, args...)
	return strings.TrimRight(out, "\r\n")
}

/*
Bash runs a command with bash. Return values are defined in Exec.
*/
func Bash(cmd interface{}, args ...interface{}) (error, int) {
	return Exec("bash", "-c", S(cmd, args...))
}

/*
BashWithStdout is similar to Bash but with stdout captured and returned as a
string.
*/
func BashWithStdout(cmd interface{}, args ...interface{}) (string, error, int) {
	return ExecWithStdout("bash", "-c", S(cmd, args...))
}

/*
BashEval is similar to BashWithStdout but with stdout captured and returned
as a string. Trainling newlines are deleted.
*/
func BashEval(cmd interface{}, args ...interface{}) string {
	out, _, _ := BashWithStdout(cmd, args...)
	return strings.TrimRight(out, "\r\n")
}

/*
Similar to os.Getwd() but no error returned.
*/
func Pwd() string {
	pwd, _ := os.Getwd()
	return pwd
}

/*
DefExitCode is the default exit code.
*/
var DefExitCode = 1

/*
Fatalf print a message and exit the program with DefExitCode.
*/
func Fatalf(msg interface{}, args ...interface{}) {
	fmt.Fprintln(os.Stderr, S(msg, args...))
	os.Exit(DefExitCode)
}

/*
Eprintf is similar to fmt.Printf but output is stderr.
*/
func Eprintf(format interface{}, args ...interface{}) {
	fmt.Fprint(os.Stderr, S(format, args...))
}

/*
Eprint is similar to fmt.Print but output is stderr.
*/
func Eprint(args ...interface{}) {
	fmt.Fprint(os.Stderr, args...)
}

/*
Eprintln is similar to fmt.Println but output is stderr.
*/
func Eprintln(args ...interface{}) {
	fmt.Fprintln(os.Stderr, args...)
}

/*
Eprintfln is similar to Eprintf but with a trailing new-line printed
*/
func Eprintfln(format interface{}, args ...interface{}) {
	fmt.Fprintln(os.Stderr, S(format, args...))
}

/*
Printfln is similar to Eprintf but with a trailing new-line printed
*/
func Printfln(format interface{}, args ...interface{}) {
	fmt.Println(S(format, args...))
}

/*
MustSucc checks the result of Exec/Bash. If not succeed, exit the application.
*/
func MustSucc(err error, code int) {
	if err == nil {
		return
	}

	if code != 0 {
		Fatalf("Failed with error code: %d", code)
	}

	Fatalf("Failed with error: %v", err)
}

type sortI struct {
	l    int
	less func(int, int) bool
	swap func(int, int)
}

func (s *sortI) Len() int {
	return s.l
}

func (s *sortI) Less(i, j int) bool {
	return s.less(i, j)
}

func (s *sortI) Swap(i, j int) {
	s.swap(i, j)
}

/*
SortF sorts the data defined by the length, Less and Swap functions.
*/
func SortF(Len int, Less func(int, int) bool, Swap func(int, int)) {
	sort.Sort(&sortI{l: Len, less: Less, swap: Swap})
}

/*
ScriptDir returns the folder of the current script.
*/
func ScriptDir() string {
	return path.Dir(os.Args[0])
}

/*
Exists checks whether the path exists
*/
func Exists(p interface{}, args ...interface{}) bool {
	_, err := os.Stat(S(p, args...))
	return err == nil
}

/*
IsDir returns true only if the path exists and indicates a directory
*/
func IsDir(p interface{}, args ...interface{}) bool {
	info, err := os.Stat(S(p, args...))
	if err != nil {
		// the path does not exist
		return false
	}
	return info.Mode().IsDir()
}

/*
IsFile returns true only if the path exists and indicates a file
*/
func IsFile(p interface{}, args ...interface{}) bool {
	info, err := os.Stat(S(p, args...))
	if err != nil {
		// the path does not exist
		return false
	}
	return !info.Mode().IsDir()
}
<|endoftext|>


<commit_before>package etcdserver

import (
	"code.google.com/p/go.net/context"
	"github.com/coreos/etcd/raft"
	"github.com/coreos/etcd/store"
	"github.com/coreos/etcd/wait"
)

type Response struct {
	// The last seen term raft was at when this request was built.
	Term int

	// The last seen index raft was at when this request was built.
	Index int

	*store.Event
	*store.Watcher

	err error
}

type Server struct {
	n raft.Node
	w wait.List

	msgsc chan raft.Message

	st store.Store

	// Send specifies the send function for sending msgs to peers. Send
	// MUST NOT block. It is okay to drop messages, since clients should
	// timeout and reissue their messages.  If Send is nil, Server will
	// panic.
	Send func(msgs []raft.Message)

	// Save specifies the save function for saving ents to stable storage.
	// Save MUST block until st and ents are on stable storage.  If Send is
	// nil, Server will panic.
	Save func(st raft.State, ents []raft.Entry)
}

func (s *Server) Run(ctx context.Context) {
	for {
		select {
		case rd := <-s.n.Ready():
			s.Save(rd.State, rd.Entries)
			s.Send(rd.Messages)
			go func() {
				for _, e := range rd.CommittedEntries {
					var r Request
					r.Unmarshal(e.Data)
					s.w.Trigger(r.Id, s.apply(r))
				}
			}()
		case <-ctx.Done():
			return
		}

	}
}

func (s *Server) Do(ctx context.Context, r Request) (Response, error) {
	if r.Id == 0 {
		panic("r.Id cannot be 0")
	}
	switch r.Method {
	case "POST", "PUT", "DELETE":
		data, err := r.Marshal()
		if err != nil {
			return Response{}, err
		}
		ch := s.w.Register(r.Id)
		s.n.Propose(ctx, data)
		select {
		case x := <-ch:
			resp := x.(Response)
			return resp, resp.err
		case <-ctx.Done():
			s.w.Trigger(r.Id, nil) // GC wait
			return Response{}, ctx.Err()
		}
	case "GET":
		switch {
		case r.Wait:
			wc, err := s.st.Watch(r.Path, r.Recursive, false, r.Since)
			if err != nil {
				return Response{}, err
			}
			return Response{Watcher: wc}, nil
		default:
			ev, err := s.st.Get(r.Path, r.Recursive, r.Sorted)
			if err != nil {
				return Response{}, err
			}
			return Response{Event: ev}, nil
		}
	}
	panic("not reached") // for some reason the compiler wants this... :/
}

// apply interprets r as a call to store.X and returns an Response interpreted from store.Event
func (s *Server) apply(r Request) Response {
	panic("not implmented")
}
<commit_msg>etcdserver: remove panic and return default err<commit_after>package etcdserver

import (
	"errors"

	"code.google.com/p/go.net/context"
	"github.com/coreos/etcd/raft"
	"github.com/coreos/etcd/store"
	"github.com/coreos/etcd/wait"
)

var ErrUnknownMethod = errors.New("etcdserver: unknown method")

type Response struct {
	// The last seen term raft was at when this request was built.
	Term int

	// The last seen index raft was at when this request was built.
	Index int

	*store.Event
	*store.Watcher

	err error
}

type Server struct {
	n raft.Node
	w wait.List

	msgsc chan raft.Message

	st store.Store

	// Send specifies the send function for sending msgs to peers. Send
	// MUST NOT block. It is okay to drop messages, since clients should
	// timeout and reissue their messages.  If Send is nil, Server will
	// panic.
	Send func(msgs []raft.Message)

	// Save specifies the save function for saving ents to stable storage.
	// Save MUST block until st and ents are on stable storage.  If Send is
	// nil, Server will panic.
	Save func(st raft.State, ents []raft.Entry)
}

func (s *Server) Run(ctx context.Context) {
	for {
		select {
		case rd := <-s.n.Ready():
			s.Save(rd.State, rd.Entries)
			s.Send(rd.Messages)
			go func() {
				for _, e := range rd.CommittedEntries {
					var r Request
					r.Unmarshal(e.Data)
					s.w.Trigger(r.Id, s.apply(r))
				}
			}()
		case <-ctx.Done():
			return
		}

	}
}

func (s *Server) Do(ctx context.Context, r Request) (Response, error) {
	if r.Id == 0 {
		panic("r.Id cannot be 0")
	}
	switch r.Method {
	case "POST", "PUT", "DELETE":
		data, err := r.Marshal()
		if err != nil {
			return Response{}, err
		}
		ch := s.w.Register(r.Id)
		s.n.Propose(ctx, data)
		select {
		case x := <-ch:
			resp := x.(Response)
			return resp, resp.err
		case <-ctx.Done():
			s.w.Trigger(r.Id, nil) // GC wait
			return Response{}, ctx.Err()
		}
	case "GET":
		switch {
		case r.Wait:
			wc, err := s.st.Watch(r.Path, r.Recursive, false, r.Since)
			if err != nil {
				return Response{}, err
			}
			return Response{Watcher: wc}, nil
		default:
			ev, err := s.st.Get(r.Path, r.Recursive, r.Sorted)
			if err != nil {
				return Response{}, err
			}
			return Response{Event: ev}, nil
		}
	default:
		return Response{}, ErrUnknownMethod
	}
}

// apply interprets r as a call to store.X and returns an Response interpreted from store.Event
func (s *Server) apply(r Request) Response {
	panic("not implmented")
}
<|endoftext|>


<commit_before>// Package fuzz provides primitives to generate random geom geometry types.
package fuzz

import (
	"math/rand"
	"time"

	"github.com/terranodo/tegola/geom"
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

func genNil(withNil bool) bool { return withNil && rand.Intn(100) < 2 }

// GenRandPoint will generate a random point point. It is possible that the point may be nil.
func GenRandPoint() *geom.Point {
	if genNil(true) {
		return (*geom.Point)(nil)
	}
	return &geom.Point{rand.NormFloat64(), rand.NormFloat64()}
}

func genRandSlicePoint(size int) (pts [][2]float64) {
	for i := 0; i < size; i++ {
		pts = append(pts, [2]float64{rand.NormFloat64(), rand.NormFloat64()})
	}
	return pts
}

// GenRandMultiPoint will generate a MultiPoint that may be nil, and will have a random number of points. There is no guarantee that all points are unique.
func GenRandMultiPoint() *geom.MultiPoint {
	if genNil(true) {
		return (*geom.MultiPoint)(nil)
	}
	mp := geom.MultiPoint(genRandSlicePoint(rand.Intn(1000)))
	return &mp
}

// GenRandLineString will generate a random LineString (that may be nil depending on withNil), and a randome number of points. There is no guarantee that the line string is simple.
func GenRandLineString(withNil bool) *geom.LineString {
	if genNil(withNil) {
		return (*geom.LineString)(nil)
	}
	ls := geom.LineString(genRandSlicePoint(rand.Intn(1000)))
	return &ls
}

// GenRandMultiLineString will generate a random MultiLineString (that may be nil depending on withNil), and a random number of linestrings. There is no gaurantee that the line strings are simple.
func GenRandMultiLineString(withNil bool) *geom.MultiLineString {
	if genNil(withNil) {
		return (*geom.MultiLineString)(nil)
	}
	num := rand.Intn(1000)
	var ml geom.MultiLineString
	for i := 0; i < num; i++ {
		ls := GenRandLineString(false)
		ml = append(ml, [][2]float64(*ls))
	}
	return &ml
}

// GenRandPolygon will generate a random Polygon (that may be nil depending on withNil). The Polygon may not be valid or simple.
func GenRandPolygon(withNil bool) *geom.Polygon {
	if genNil(withNil) {
		return (*geom.Polygon)(nil)
	}
	num := rand.Intn(100)
	var p geom.Polygon
	for i := 0; i < num; i++ {
		ls := GenRandLineString(false)
		p = append(p, [][2]float64(*ls))
	}
	return &p
}

// GenRandMultiPolygon will generate a random MultiPolygon (that may be nil depending on withNil). The Polygons may not be valid or simple.
func GenRandMultiPolygon(withNil bool) *geom.MultiPolygon {
	if genNil(withNil) {
		return (*geom.MultiPolygon)(nil)
	}
	num := rand.Intn(10)
	var mp geom.MultiPolygon
	for i := 0; i < num; i++ {
		p := GenRandPolygon(false)
		mp = append(mp, [][][2]float64(*p))
	}
	return &mp
}

// GenRandCollection will generate a random Collection (that may be nil depending on withNil).
func GenRandCollection(withNil bool) *geom.Collection {
	if genNil(withNil) {
		return (*geom.Collection)(nil)
	}
	num := rand.Intn(10)
	var col geom.Collection
	for i := 0; i < num; i++ {
		col = append(col, GenGeometry())
	}
	return &col
}

// GenGenometry will generate a random Geometry. The geometry may be nil.
func GenGeometry() geom.Geometry {
	switch rand.Intn(22) {
	default:
		return nil
	case 0, 13, 20:
		return GenRandPoint()
	case 2, 11, 19:
		return GenRandMultiPoint()
	case 4, 9, 18:
		return GenRandLineString(true)
	case 6, 7, 17:
		return GenRandMultiLineString(true)
	case 8, 5, 16:
		return GenRandPolygon(true)
	case 10, 3, 15:
		return GenRandMultiPolygon(true)
	case 12, 1, 14:
		return GenRandCollection(true)
	}

}
<commit_msg>Update fuzz.go<commit_after>// Package fuzz provides primitives to generate random geom geometry types.
package fuzz

import (
	"math/rand"
	"time"

	"github.com/terranodo/tegola/geom"
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

func genNil(withNil bool) bool { return withNil && rand.Intn(100) < 2 }

// GenRandPoint will generate a random point. It is possible that the point may be nil.
func GenRandPoint() *geom.Point {
	if genNil(true) {
		return (*geom.Point)(nil)
	}
	return &geom.Point{rand.NormFloat64(), rand.NormFloat64()}
}

func genRandSlicePoint(size int) (pts [][2]float64) {
	for i := 0; i < size; i++ {
		pts = append(pts, [2]float64{rand.NormFloat64(), rand.NormFloat64()})
	}
	return pts
}

// GenRandMultiPoint will generate a MultiPoint that may be nil, and will have a random number of points. There is no guarantee that all points are unique.
func GenRandMultiPoint() *geom.MultiPoint {
	if genNil(true) {
		return (*geom.MultiPoint)(nil)
	}
	mp := geom.MultiPoint(genRandSlicePoint(rand.Intn(1000)))
	return &mp
}

// GenRandLineString will generate a random LineString (that may be nil depending on withNil), and a randome number of points. There is no guarantee that the line string is simple.
func GenRandLineString(withNil bool) *geom.LineString {
	if genNil(withNil) {
		return (*geom.LineString)(nil)
	}
	ls := geom.LineString(genRandSlicePoint(rand.Intn(1000)))
	return &ls
}

// GenRandMultiLineString will generate a random MultiLineString (that may be nil depending on withNil), and a random number of linestrings. There is no gaurantee that the line strings are simple.
func GenRandMultiLineString(withNil bool) *geom.MultiLineString {
	if genNil(withNil) {
		return (*geom.MultiLineString)(nil)
	}
	num := rand.Intn(1000)
	var ml geom.MultiLineString
	for i := 0; i < num; i++ {
		ls := GenRandLineString(false)
		ml = append(ml, [][2]float64(*ls))
	}
	return &ml
}

// GenRandPolygon will generate a random Polygon (that may be nil depending on withNil). The Polygon may not be valid or simple.
func GenRandPolygon(withNil bool) *geom.Polygon {
	if genNil(withNil) {
		return (*geom.Polygon)(nil)
	}
	num := rand.Intn(100)
	var p geom.Polygon
	for i := 0; i < num; i++ {
		ls := GenRandLineString(false)
		p = append(p, [][2]float64(*ls))
	}
	return &p
}

// GenRandMultiPolygon will generate a random MultiPolygon (that may be nil depending on withNil). The Polygons may not be valid or simple.
func GenRandMultiPolygon(withNil bool) *geom.MultiPolygon {
	if genNil(withNil) {
		return (*geom.MultiPolygon)(nil)
	}
	num := rand.Intn(10)
	var mp geom.MultiPolygon
	for i := 0; i < num; i++ {
		p := GenRandPolygon(false)
		mp = append(mp, [][][2]float64(*p))
	}
	return &mp
}

// GenRandCollection will generate a random Collection (that may be nil depending on withNil).
func GenRandCollection(withNil bool) *geom.Collection {
	if genNil(withNil) {
		return (*geom.Collection)(nil)
	}
	num := rand.Intn(10)
	var col geom.Collection
	for i := 0; i < num; i++ {
		col = append(col, GenGeometry())
	}
	return &col
}

// GenGenometry will generate a random Geometry. The geometry may be nil.
func GenGeometry() geom.Geometry {
	switch rand.Intn(22) {
	default:
		return nil
	case 0, 13, 20:
		return GenRandPoint()
	case 2, 11, 19:
		return GenRandMultiPoint()
	case 4, 9, 18:
		return GenRandLineString(true)
	case 6, 7, 17:
		return GenRandMultiLineString(true)
	case 8, 5, 16:
		return GenRandPolygon(true)
	case 10, 3, 15:
		return GenRandMultiPolygon(true)
	case 12, 1, 14:
		return GenRandCollection(true)
	}

}
<|endoftext|>


<commit_before>// Copyright 2012 Gonéri Le Bouder. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import "notmuch"
import "log"
import "encoding/json"
import "os"
import "io"
import "fmt"
import "regexp"
import "net/mail"
import "path"

type Filter struct {
	Field   string
	Pattern string
	Re      *regexp.Regexp
	Tags    string
}

type Result struct {
	MessageID string
	Tags      string
	Die       bool
        Filename  string
}

const NCPU = 4 // number of CPU cores

func getMaildirLoc() string {
	// honor NOTMUCH_CONFIG
	home := os.Getenv("NOTMUCH_CONFIG")
	if home == "" {
		home = os.Getenv("HOME")
	}

	return path.Join(home, "Maildir")
}

func RefreshFlags(nmdb *notmuch.Database) {

	query := nmdb.CreateQuery("tag:inbox and tag:delete")
	msgs := query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:archive")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:seen and not tag:list")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.AddTag("archive")
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:seen and tag:list")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:seen and tag:bug")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:killed")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		threadId := msg.GetThreadId()
		filter := fmt.Sprintf("thread:%s", threadId)

		query := nmdb.CreateQuery(filter)
		msgs := query.SearchMessages()
		for ; msgs.Valid(); msgs.MoveToNext() {
			msg := msgs.Get()
			msg.RemoveTag("inbox")
		}
	}

	query = nmdb.CreateQuery("tag:inbox")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		threadId := msg.GetThreadId()
		filter := fmt.Sprintf("thread:%s", threadId)

		query := nmdb.CreateQuery(filter)
		msgs := query.SearchMessages()
		for ; msgs.Valid(); msgs.MoveToNext() {
			msg := msgs.Get()
			msg.AddTag("inbox")
		}
	}

	nmdb.Close()
	fmt.Print("Ok\n")

}

func studyMsg(filter []Filter, filenameIn chan string, resultOut chan Result, quit chan bool) {
	for {
		filename := <-filenameIn

		if filename == "" {
			var result Result
			result.Die = true
			resultOut <- result

			return
		}
		// We can use Notmuch for this directly because Xappian will
		// fails as soon as we have 2 concurrent goroutine
		file, err := os.Open(filename) // For read access.
		if err != nil {
			log.Fatal(err)
		}
		var msg *mail.Message
		msg, err = mail.ReadMessage(file)

		if err != nil {
			log.Fatal(err)
		}

		var result Result
		result.MessageID = msg.Header.Get("Message-Id")
                if (result.MessageID == "") {
                    fmt.Printf("No message ID for %s\n", filename)
                    continue;
                }
		result.Filename = filename
		for _, f := range filter {
			if f.Re.MatchString(msg.Header.Get(f.Field)) {
				result.Tags += " "
				result.Tags += f.Tags
			}

		}
		file.Close()

		resultOut <- result
	}
}

func loadFilter() (filter []Filter) {

	file, err := os.Open(fmt.Sprintf("/%s/notmuch-filter.json", getMaildirLoc())) // For read access.
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	dec := json.NewDecoder(file)
	for {
		var f Filter
		if err := dec.Decode(&f); err == io.EOF {
			break
		} else if err != nil {
			log.Fatal(err)
		}

		var err error = nil
		if f.Re, err = regexp.Compile(f.Pattern); err != nil {
			log.Printf("error: %v\n", err)
		}

		filter = append(filter, f)
	}

	return filter
}

func studyMsgs(resultOut chan Result, quit chan bool, filenames []string) {

	filter := loadFilter()

	filenameIn := make(chan string)
	for i := 0; i < NCPU+1; i++ {
		go studyMsg(filter, filenameIn, resultOut, quit)
	}
	for _, filename := range filenames {
		filenameIn <- filename
	}

	for i := 0; i < NCPU+1; i++ {
		filenameIn <- ""
	}

	quit <- true
}

func main() {
	var query *notmuch.Query
	var nmdb *notmuch.Database

	if db, status := notmuch.OpenDatabase(getMaildirLoc(),
		notmuch.DATABASE_MODE_READ_ONLY); status == notmuch.STATUS_SUCCESS {
		nmdb = db
	} else {
		log.Fatalf("Failed to open the database: %v\n", status)
	}

	quit := make(chan bool)
	resultOut := make(chan Result)

	query = nmdb.CreateQuery("tag:new")

	println(">", query.CountMessages(), "<")
	msgs := query.SearchMessages()

	var filenames []string
	if query.CountMessages() > 0 {
		for ; msgs.Valid(); msgs.MoveToNext() {
			msg := msgs.Get()

			filenames = append(filenames, msg.GetFileName())
		}
	}

	go studyMsgs(resultOut, quit, filenames)

	//	var query *notmuch.Query
	var msgIDRegexp = regexp.MustCompile("^<(.*)>$")
	var tagRegexp = regexp.MustCompile("([\\+-])(\\S+)")

	// open the database
	if db, status := notmuch.OpenDatabase(getMaildirLoc(),
		1); status == notmuch.STATUS_SUCCESS {
		nmdb = db
	} else {
		log.Fatalf("Failed to open the database: %v\n", status)
	}
	defer nmdb.Close()

	var running int = NCPU + 1
	for {
		result := <-resultOut

		if result.Die {

			running--

			if running > 0 {
				continue
			} else {
                            break
			}
		}

		// Message-ID without the <>
                fmt.Printf("MessageID: %s\n", result.MessageID)
		msgID := msgIDRegexp.FindStringSubmatch(result.MessageID)[1]
		filter := "id:"
		filter += msgID
		query := nmdb.CreateQuery(filter)
		msgs := query.SearchMessages()
		msg := msgs.Get()
		if msg == nil {
                    fmt.Printf("Can't find MessageID %s for mail %s\n", msgID, result.Filename)
                    continue
                }

		fmt.Printf("%s, tags: %s\n", msgID, result.Tags)
		msg.Freeze()
		for _, v := range tagRegexp.FindAllStringSubmatch(result.Tags, -1) {
			if v[1] == "+" {
				msg.AddTag(v[2])
			} else if v[1] == "-" {
				msg.RemoveTag(v[2])
			}
		}
		msg.Thaw()

	}
        RefreshFlags(nmdb)
        fmt.Printf("exit\n")
        os.Exit(0)

}
<commit_msg>hides killed messages only at the end<commit_after>// Copyright 2012 Gonéri Le Bouder. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import "notmuch"
import "log"
import "encoding/json"
import "os"
import "io"
import "fmt"
import "regexp"
import "net/mail"
import "path"

type Filter struct {
	Field   string
	Pattern string
	Re      *regexp.Regexp
	Tags    string
}

type Result struct {
	MessageID string
	Tags      string
	Die       bool
        Filename  string
}

const NCPU = 4 // number of CPU cores

func getMaildirLoc() string {
	// honor NOTMUCH_CONFIG
	home := os.Getenv("NOTMUCH_CONFIG")
	if home == "" {
		home = os.Getenv("HOME")
	}

	return path.Join(home, "Maildir")
}

func RefreshFlags(nmdb *notmuch.Database) {

	query := nmdb.CreateQuery("tag:inbox and tag:delete")
	msgs := query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:archive")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:seen and not tag:list")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.AddTag("archive")
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:seen and tag:list")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:seen and tag:bug")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		threadId := msg.GetThreadId()
		filter := fmt.Sprintf("thread:%s", threadId)

		query := nmdb.CreateQuery(filter)
		msgs := query.SearchMessages()
		for ; msgs.Valid(); msgs.MoveToNext() {
			msg := msgs.Get()
			msg.AddTag("inbox")
		}
	}

	query = nmdb.CreateQuery("tag:inbox and tag:killed")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		threadId := msg.GetThreadId()
		filter := fmt.Sprintf("thread:%s", threadId)

		query := nmdb.CreateQuery(filter)
		msgs := query.SearchMessages()
		for ; msgs.Valid(); msgs.MoveToNext() {
			msg := msgs.Get()
			msg.RemoveTag("inbox")
		}
	}

	nmdb.Close()
	fmt.Print("Ok\n")

}

func studyMsg(filter []Filter, filenameIn chan string, resultOut chan Result, quit chan bool) {
	for {
		filename := <-filenameIn

		if filename == "" {
			var result Result
			result.Die = true
			resultOut <- result

			return
		}
		// We can use Notmuch for this directly because Xappian will
		// fails as soon as we have 2 concurrent goroutine
		file, err := os.Open(filename) // For read access.
		if err != nil {
			log.Fatal(err)
		}
		var msg *mail.Message
		msg, err = mail.ReadMessage(file)

		if err != nil {
			log.Fatal(err)
		}

		var result Result
		result.MessageID = msg.Header.Get("Message-Id")
                if (result.MessageID == "") {
                    fmt.Printf("No message ID for %s\n", filename)
                    continue;
                }
		result.Filename = filename
		for _, f := range filter {
			if f.Re.MatchString(msg.Header.Get(f.Field)) {
				result.Tags += " "
				result.Tags += f.Tags
			}

		}
		file.Close()

		resultOut <- result
	}
}

func loadFilter() (filter []Filter) {

	file, err := os.Open(fmt.Sprintf("/%s/notmuch-filter.json", getMaildirLoc())) // For read access.
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	dec := json.NewDecoder(file)
	for {
		var f Filter
		if err := dec.Decode(&f); err == io.EOF {
			break
		} else if err != nil {
			log.Fatal(err)
		}

		var err error = nil
		if f.Re, err = regexp.Compile(f.Pattern); err != nil {
			log.Printf("error: %v\n", err)
		}

		filter = append(filter, f)
	}

	return filter
}

func studyMsgs(resultOut chan Result, quit chan bool, filenames []string) {

	filter := loadFilter()

	filenameIn := make(chan string)
	for i := 0; i < NCPU+1; i++ {
		go studyMsg(filter, filenameIn, resultOut, quit)
	}
	for _, filename := range filenames {
		filenameIn <- filename
	}

	for i := 0; i < NCPU+1; i++ {
		filenameIn <- ""
	}

	quit <- true
}

func main() {
	var query *notmuch.Query
	var nmdb *notmuch.Database

	if db, status := notmuch.OpenDatabase(getMaildirLoc(),
		notmuch.DATABASE_MODE_READ_ONLY); status == notmuch.STATUS_SUCCESS {
		nmdb = db
	} else {
		log.Fatalf("Failed to open the database: %v\n", status)
	}

	quit := make(chan bool)
	resultOut := make(chan Result)

	query = nmdb.CreateQuery("tag:new")

	println(">", query.CountMessages(), "<")
	msgs := query.SearchMessages()

	var filenames []string
	if query.CountMessages() > 0 {
		for ; msgs.Valid(); msgs.MoveToNext() {
			msg := msgs.Get()

			filenames = append(filenames, msg.GetFileName())
		}
	}

	go studyMsgs(resultOut, quit, filenames)

	//	var query *notmuch.Query
	var msgIDRegexp = regexp.MustCompile("^<(.*)>$")
	var tagRegexp = regexp.MustCompile("([\\+-])(\\S+)")

	// open the database
	if db, status := notmuch.OpenDatabase(getMaildirLoc(),
		1); status == notmuch.STATUS_SUCCESS {
		nmdb = db
	} else {
		log.Fatalf("Failed to open the database: %v\n", status)
	}
	defer nmdb.Close()

	var running int = NCPU + 1
	for {
		result := <-resultOut

		if result.Die {

			running--

			if running > 0 {
				continue
			} else {
                            break
			}
		}

		// Message-ID without the <>
                fmt.Printf("MessageID: %s\n", result.MessageID)
		msgID := msgIDRegexp.FindStringSubmatch(result.MessageID)[1]
		filter := "id:"
		filter += msgID
		query := nmdb.CreateQuery(filter)
		msgs := query.SearchMessages()
		msg := msgs.Get()
		if msg == nil {
                    fmt.Printf("Can't find MessageID %s for mail %s\n", msgID, result.Filename)
                    continue
                }

		fmt.Printf("%s, tags: %s\n", msgID, result.Tags)
		msg.Freeze()
		for _, v := range tagRegexp.FindAllStringSubmatch(result.Tags, -1) {
			if v[1] == "+" {
				msg.AddTag(v[2])
			} else if v[1] == "-" {
				msg.RemoveTag(v[2])
			}
		}
		msg.Thaw()

	}
        RefreshFlags(nmdb)
        fmt.Printf("exit\n")
        os.Exit(0)

}
<|endoftext|>


<commit_before>/*
copyright 2019 google llc
licensed under the apache license, version 2.0 (the "license");
you may not use this file except in compliance with the license.
you may obtain a copy of the license at
    http://www.apache.org/licenses/license-2.0
unless required by applicable law or agreed to in writing, software
distributed under the license is distributed on an "as is" basis,
without warranties or conditions of any kind, either express or implied.
see the license for the specific language governing permissions and
limitations under the license.
*/

package ops

import (
	"encoding/base64"
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/google/uuid"
)

//Used for String Splitting Pre and Post Hash
const seperator string = "#!#"

type SUID struct {
<<<<<<< HEAD
	UUID          string    `json:"link"`
	Name          string    `json:"name"`
	TimeStamp     time.Time `json:"timestamp"`
	DisplayName   string    `json:"displayName"`
	DirectoryName string    `json:"directoryName"`
	DownloadId    string    `json:"downloadId"`
	// Private
	longname string
	nameHash string
=======
	UUID        string    `json:"link"`
	Name        string    `json:"name"`
	TimeStamp   time.Time `json:"timestamp"`
	DisplayName string    `json:"displayName"`
	// Private
	longname    string
	nameHash    string
	OutLongname string
>>>>>>> 85e899b (Landed the encoding of conversion objects metadata into foldername)
}

func (s *SUID) name() string {
	return s.Name
}

func (s *SUID) hash() {
	s.nameHash = base64.StdEncoding.EncodeToString([]byte(s.longname))
}

func (s *SUID) Meta() {
	fmt.Println("_+_+_+_+_+_+_+_+_+_+_+_+_+_")
	fmt.Println("_+_+_+_+_+_+_+_+_+_+_+_+_+_")
	fmt.Println(s.UUID)
	fmt.Println(s.Name)
	fmt.Println(s.TimeStamp)
	fmt.Println(s.DisplayName)
	fmt.Println(s.longname)
	fmt.Println(s.nameHash)
	fmt.Println("_+_+_+_+_+_+_+_+_+_+_+_+_+_")
	fmt.Println("_+_+_+_+_+_+_+_+_+_+_+_+_+_")
	fmt.Println("")
}

func CreateSUID(customName string) SUID {

	// Create SUID Object
	suid := SUID{}
	// Assign Variables for Uniqueness
	suid.UUID = uuid.New().String()
	suid.TimeStamp = time.Now()

	// Ensure We always have a Conversion Name
	if customName == "" {
		// Configure Custom Name
		suid.Name = "Shifter Conversion"
	} else {
		// TODO Clean Name Here
		suid.Name = customName
	}

	// String Format - (Timestamp + UUID + Custom Name)
	suid.longname = fmt.Sprintf("%s%s%s%s%s", suid.TimeStamp.Format(time.RFC1123),
		seperator, suid.UUID, seperator, suid.Name)

	suid.hash()
<<<<<<< HEAD
	suid.DirectoryName = suid.nameHash
	suid.DownloadId = suid.nameHash
	suid.DisplayName = fmt.Sprintf("%s - %s", suid.TimeStamp.Format(time.RFC1123), suid.Name)
	return suid
}

func ResolveSUID(downloadId string) (SUID, error) {
	// Create New SUID Object
	suid := SUID{}
	if downloadId == "" {
		return suid, errors.New("Download ID or Filename Hash must be provided when Resolving SUID")
	}
	suid.nameHash = downloadId
	decoded, err := base64.StdEncoding.DecodeString(suid.nameHash)
=======

	suid.OutLongname = suid.nameHash
	return suid
}

func ResolveSUID(hash string) (SUID, error) {
	// Create New SUID Object
	suid := SUID{}
	if hash == "" {
		return suid, errors.New("Filename Hash must be provided when Resolving SUID")
	}
	suid.nameHash = hash
	decoded, err := base64.StdEncoding.DecodeString(hash)
>>>>>>> 85e899b (Landed the encoding of conversion objects metadata into foldername)
	if err != nil {
		return suid, err
	}
	suid.longname = string(decoded)
	items := strings.Split(suid.longname, seperator)
	t, err := time.Parse(time.RFC1123, items[0])
	if err != nil {
		return suid, err
	}
	suid.TimeStamp = t
	suid.UUID = items[1]
	suid.Name = items[2]
<<<<<<< HEAD
	suid.DisplayName = fmt.Sprintf("%s - %s", suid.TimeStamp.Format(time.RFC1123), suid.Name)
	suid.DirectoryName = suid.nameHash
	suid.DownloadId = suid.DirectoryName
=======

>>>>>>> 85e899b (Landed the encoding of conversion objects metadata into foldername)
	return suid, nil

}
<commit_msg>Switching from uuid to suid<commit_after>/*
copyright 2019 google llc
licensed under the apache license, version 2.0 (the "license");
you may not use this file except in compliance with the license.
you may obtain a copy of the license at
    http://www.apache.org/licenses/license-2.0
unless required by applicable law or agreed to in writing, software
distributed under the license is distributed on an "as is" basis,
without warranties or conditions of any kind, either express or implied.
see the license for the specific language governing permissions and
limitations under the license.
*/

package ops

import (
	"encoding/base64"
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/google/uuid"
)

//Used for String Splitting Pre and Post Hash
const seperator string = "#!#"

type SUID struct {
	UUID          string    `json:"link"`
	Name          string    `json:"name"`
	TimeStamp     time.Time `json:"timestamp"`
	DisplayName   string    `json:"displayName"`
	DirectoryName string    `json:"directoryName"`
	DownloadId    string    `json:"downloadId"`
	// Private
	longname string
	nameHash string
}

func (s *SUID) name() string {
	return s.Name
}

func (s *SUID) hash() {
	s.nameHash = base64.StdEncoding.EncodeToString([]byte(s.longname))
}

func (s *SUID) Meta() {
	fmt.Println("_+_+_+_+_+_+_+_+_+_+_+_+_+_")
	fmt.Println("_+_+_+_+_+_+_+_+_+_+_+_+_+_")
	fmt.Println(s.UUID)
	fmt.Println(s.Name)
	fmt.Println(s.TimeStamp)
	fmt.Println(s.DisplayName)
	fmt.Println(s.longname)
	fmt.Println(s.nameHash)
	fmt.Println("_+_+_+_+_+_+_+_+_+_+_+_+_+_")
	fmt.Println("_+_+_+_+_+_+_+_+_+_+_+_+_+_")
	fmt.Println("")
}

func CreateSUID(customName string) SUID {

	// Create SUID Object
	suid := SUID{}
	// Assign Variables for Uniqueness
	suid.UUID = uuid.New().String()
	suid.TimeStamp = time.Now()

	// Ensure We always have a Conversion Name
	if customName == "" {
		// Configure Custom Name
		suid.Name = "Shifter Conversion"
	} else {
		// TODO Clean Name Here
		suid.Name = customName
	}

	// String Format - (Timestamp + UUID + Custom Name)
	suid.longname = fmt.Sprintf("%s%s%s%s%s", suid.TimeStamp.Format(time.RFC1123),
		seperator, suid.UUID, seperator, suid.Name)

	suid.hash()
	suid.DirectoryName = suid.nameHash
	suid.DownloadId = suid.nameHash
	suid.DisplayName = fmt.Sprintf("%s - %s", suid.TimeStamp.Format(time.RFC1123), suid.Name)
	return suid
}

func ResolveSUID(downloadId string) (SUID, error) {
	// Create New SUID Object
	suid := SUID{}
	if downloadId == "" {
		return suid, errors.New("Download ID or Filename Hash must be provided when Resolving SUID")
	}
	suid.nameHash = downloadId
	decoded, err := base64.StdEncoding.DecodeString(suid.nameHash)
	if err != nil {
		return suid, err
	}
	suid.longname = string(decoded)
	items := strings.Split(suid.longname, seperator)
	t, err := time.Parse(time.RFC1123, items[0])
	if err != nil {
		return suid, err
	}
	suid.TimeStamp = t
	suid.UUID = items[1]
	suid.Name = items[2]
	suid.DisplayName = fmt.Sprintf("%s - %s", suid.TimeStamp.Format(time.RFC1123), suid.Name)
	suid.DirectoryName = suid.nameHash
	suid.DownloadId = suid.DirectoryName
	return suid, nil

}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"strconv"
	"strings"
)

func handler(w http.ResponseWriter, r *http.Request) {
	// Workaround for Quentin's system configuration.
	// For some reason, css files are getting served
	// without a content-type...
	if strings.HasSuffix(r.URL.Path, ".css") {
		w.Header().Set("Content-Type", "text/css")
	}

	if strings.HasSuffix(r.URL.Path, ".js") {
		w.Header().Set("Cache-Control: max-age=" + (60 * 60)) //Cache for 1 hour
	}

	url := r.URL.Path

	log.Println(url)

	if url == "" {
		url = "/default.htm"
	}

	http.ServeFile(w, r, "."+url)
}

func main() {
	portNumber := flag.Int("port", 8080, "Sets the port the server listens on for both http requests and websocket connections.")

	flag.Parse()

	http.HandleFunc("/", handler)

	fmt.Println("Beginning HTTP listening on port", *portNumber)
	err := http.ListenAndServe(":"+strconv.Itoa(*portNumber), nil)
	if err != nil {
		log.Fatal("ListenAndServe:", err)
	}
}
<commit_msg>Better root checking.<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"strconv"
	"strings"
)

func handler(w http.ResponseWriter, r *http.Request) {
	// Workaround for Quentin's system configuration.
	// For some reason, css files are getting served
	// without a content-type...
	if strings.HasSuffix(r.URL.Path, ".css") {
		w.Header().Set("Content-Type", "text/css")
	}

	if strings.HasSuffix(r.URL.Path, ".js") {
		w.Header().Set("Cache-Control: max-age=" + (60 * 60)) //Cache for 1 hour
	}

	url := r.URL.Path

	log.Println(url)

	if len(url) <= 1 {
		url = "/default.htm"
	}

	http.ServeFile(w, r, "."+url)
}

func main() {
	portNumber := flag.Int("port", 8080, "Sets the port the server listens on for both http requests and websocket connections.")

	flag.Parse()

	http.HandleFunc("/", handler)

	fmt.Println("Beginning HTTP listening on port", *portNumber)
	err := http.ListenAndServe(":"+strconv.Itoa(*portNumber), nil)
	if err != nil {
		log.Fatal("ListenAndServe:", err)
	}
}
<|endoftext|>


<commit_before>package krpc

type Infohash [20]byte

type CompactInfohashes [][20]byte

func (CompactInfohashes) ElemSize() int { return 20 }

func (me CompactInfohashes) MarshalBinary() ([]byte, error) {
	return marshalBinarySlice(me)
}

func (me CompactInfohashes) MarshalBencode() ([]byte, error) {
	return bencodeBytesResult(me.MarshalBinary())
}

func (me *CompactInfohashes) UnmarshalBinary(b []byte) error {
	return unmarshalBinarySlice(me, b)
}

func (me *CompactInfohashes) UnmarshalBencode(b []byte) error {
	return unmarshalBencodedBinary(me, b)
}<commit_msg>Missing newline<commit_after>package krpc

type Infohash [20]byte

type CompactInfohashes [][20]byte

func (CompactInfohashes) ElemSize() int { return 20 }

func (me CompactInfohashes) MarshalBinary() ([]byte, error) {
	return marshalBinarySlice(me)
}

func (me CompactInfohashes) MarshalBencode() ([]byte, error) {
	return bencodeBytesResult(me.MarshalBinary())
}

func (me *CompactInfohashes) UnmarshalBinary(b []byte) error {
	return unmarshalBinarySlice(me, b)
}

func (me *CompactInfohashes) UnmarshalBencode(b []byte) error {
	return unmarshalBencodedBinary(me, b)
}
<|endoftext|>


<commit_before>package slurp

import (
	"fmt"
	"sync"
	"time"
)

// DataLoader is used to load extra data onto an item.
type DataLoader interface {
	// LoadData will load extra data for an item and
	// return a suggested data key and value.
	// The item itself does NOT get updated directly by
	// calling this function.
	//
	// If their is no data to load and you dont want a nil
	// value added to a data key in the item then this
	// function must return "", nil
	LoadData(*Item) (string, interface{})
}

// DataLoaderFunc is an adapter that allow you to use
// an ordinary function as a DataLoader.
type DataLoaderFunc func(*Item) (string, interface{})

// LoadData calls f(item)
func (f DataLoaderFunc) LoadData(item *Item) (string, interface{}) {
	return f(item)
}

// DataLoaderStatValue provices a standard set of stat counters.
type DataLoaderStatValue struct {
	FirstCallAt   *time.Time    `json:"firstCallAt,omitempty"`
	LastCallAt    *time.Time    `json:"lastCallAt,omitempty"`
	Count         int64         `json:"count"`
	DurationTotal time.Duration `json:"durationTotal"`
	DurationMin   time.Duration `json:"durationMin"`
	DurationAvg   time.Duration `json:"durationAvg"`
	DurationMax   time.Duration `json:"durationMax"`
}

// Called updates *DataLoaderStat
func (s *DataLoaderStatValue) Called(t time.Time, d time.Duration) {
	if s.FirstCallAt == nil || t.Before(*s.FirstCallAt) {
		s.FirstCallAt = &t
	}
	if s.LastCallAt == nil || t.After(*s.LastCallAt) {
		s.LastCallAt = &t
	}
	s.Count++
	s.DurationTotal += d
	if d < s.DurationMin || s.DurationMin == 0 {
		s.DurationMin = d
	}
	s.DurationAvg = time.Duration(int64(s.DurationTotal) / s.Count)
	if d > s.DurationMax {
		s.DurationMax = d
	}
}

// DataLoaderStatWrapper wraps a DataLoader and provides stats about it.
type DataLoaderStatWrapper struct {
	Loader         DataLoader
	mutex          sync.RWMutex
	called         *DataLoaderStatValue
	returnEmptyKey *DataLoaderStatValue
	returnNilData  *DataLoaderStatValue
	returnData     *DataLoaderStatValue
}

// NewDataLoaderStatWrapper allows you to wrap DataLoader for stat collection.
func NewDataLoaderStatWrapper(loader DataLoader) *DataLoaderStatWrapper {
	w := &DataLoaderStatWrapper{
		Loader: loader,
	}
	w.Reset()
	return w
}

// LoadData calls the origional loader and updates its stats.
func (w *DataLoaderStatWrapper) LoadData(item *Item) (string, interface{}) {
	t := time.Now()
	k, v := w.Loader.LoadData(item)
	d := time.Now().Sub(t)
	w.mutex.Lock()
	w.called.Called(t, d)
	if k == "" {
		w.returnEmptyKey.Called(t, d)
	} else {
		if v == nil {
			w.returnNilData.Called(t, d)
		} else {
			w.returnData.Called(t, d)
		}
	}
	w.mutex.Unlock()
	return k, v
}

// Reset clears current stats.
func (w *DataLoaderStatWrapper) Reset() {
	w.mutex.Lock()
	defer w.mutex.Unlock()
	w.called = &DataLoaderStatValue{}
	w.returnEmptyKey = &DataLoaderStatValue{}
	w.returnNilData = &DataLoaderStatValue{}
	w.returnData = &DataLoaderStatValue{}
}

// Stat returns information about the data loader.
func (w *DataLoaderStatWrapper) Stat() *DataLoaderStat {
	w.mutex.RLock()
	defer w.mutex.RUnlock()
	return &DataLoaderStat{
		Called:         *w.called,
		ReturnEmptyKey: *w.returnEmptyKey,
		ReturnNilData:  *w.returnNilData,
		ReturnData:     *w.returnData,
	}
}

// Name ensures that this implements the Describer interface.
func (w *DataLoaderStatWrapper) Name() string {
	if d, ok := w.Loader.(Describer); ok {
		return d.Name()
	}
	return "Anonymous"
}

// Description ensures that this implements the Describer interface.
func (w *DataLoaderStatWrapper) Description() string {
	if d, ok := w.Loader.(Describer); ok {
		return d.Description()
	}
	return fmt.Sprintf("Anonymous %T", w.Loader)
}

// DataLoaderStat is returned from the DataLoaderStatWrapper.Stat method.
type DataLoaderStat struct {
	Called         DataLoaderStatValue `json:"called"`
	ReturnEmptyKey DataLoaderStatValue `json:"returnEmptyKey"`
	ReturnNilData  DataLoaderStatValue `json:"returnNilData"`
	ReturnData     DataLoaderStatValue `json:"returnData"`
}

// LoadData will load data for item concurrently for
// each loader provided. Data is assigned to the item in
// the same order as the arguments provided to this function.
func LoadData(item *Item, loaders ...DataLoader) {
	type data struct {
		k string
		v interface{}
	}
	wg := sync.WaitGroup{}
	newData := make([]*data, len(loaders))
	for offset, loader := range loaders {
		wg.Add(1)
		go func(offset int, loader DataLoader) {
			defer wg.Done()
			k, v := loader.LoadData(item)
			newData[offset] = &data{
				k: k,
				v: v,
			}
		}(offset, loader)
	}
	wg.Wait()
	for _, d := range newData {
		// We still assign nil data values to the map but not empty an key.
		if d.k != "" {
			item.Data[d.k] = d.v
		}
	}
}
<commit_msg>Make DataLoaderStatValue called method is now private.<commit_after>package slurp

import (
	"fmt"
	"sync"
	"time"
)

// DataLoader is used to load extra data onto an item.
type DataLoader interface {
	// LoadData will load extra data for an item and
	// return a suggested data key and value.
	// The item itself does NOT get updated directly by
	// calling this function.
	//
	// If their is no data to load and you dont want a nil
	// value added to a data key in the item then this
	// function must return "", nil
	LoadData(*Item) (string, interface{})
}

// DataLoaderFunc is an adapter that allow you to use
// an ordinary function as a DataLoader.
type DataLoaderFunc func(*Item) (string, interface{})

// LoadData calls f(item)
func (f DataLoaderFunc) LoadData(item *Item) (string, interface{}) {
	return f(item)
}

// DataLoaderStatValue provices a standard set of stat counters.
type DataLoaderStatValue struct {
	FirstCallAt   *time.Time    `json:"firstCallAt,omitempty"`
	LastCallAt    *time.Time    `json:"lastCallAt,omitempty"`
	Count         int64         `json:"count"`
	DurationTotal time.Duration `json:"durationTotal"`
	DurationMin   time.Duration `json:"durationMin"`
	DurationAvg   time.Duration `json:"durationAvg"`
	DurationMax   time.Duration `json:"durationMax"`
}

// Called updates *DataLoaderStat
func (s *DataLoaderStatValue) called(t time.Time, d time.Duration) {
	if s.FirstCallAt == nil || t.Before(*s.FirstCallAt) {
		s.FirstCallAt = &t
	}
	if s.LastCallAt == nil || t.After(*s.LastCallAt) {
		s.LastCallAt = &t
	}
	s.Count++
	s.DurationTotal += d
	if d < s.DurationMin || s.DurationMin == 0 {
		s.DurationMin = d
	}
	s.DurationAvg = time.Duration(int64(s.DurationTotal) / s.Count)
	if d > s.DurationMax {
		s.DurationMax = d
	}
}

// DataLoaderStatWrapper wraps a DataLoader and provides stats about it.
type DataLoaderStatWrapper struct {
	Loader         DataLoader
	mutex          sync.RWMutex
	called         *DataLoaderStatValue
	returnEmptyKey *DataLoaderStatValue
	returnNilData  *DataLoaderStatValue
	returnData     *DataLoaderStatValue
}

// NewDataLoaderStatWrapper allows you to wrap DataLoader for stat collection.
func NewDataLoaderStatWrapper(loader DataLoader) *DataLoaderStatWrapper {
	w := &DataLoaderStatWrapper{
		Loader: loader,
	}
	w.Reset()
	return w
}

// LoadData calls the origional loader and updates its stats.
func (w *DataLoaderStatWrapper) LoadData(item *Item) (string, interface{}) {
	t := time.Now()
	k, v := w.Loader.LoadData(item)
	d := time.Now().Sub(t)
	w.mutex.Lock()
	w.called.called(t, d)
	if k == "" {
		w.returnEmptyKey.called(t, d)
	} else {
		if v == nil {
			w.returnNilData.called(t, d)
		} else {
			w.returnData.called(t, d)
		}
	}
	w.mutex.Unlock()
	return k, v
}

// Reset clears current stats.
func (w *DataLoaderStatWrapper) Reset() {
	w.mutex.Lock()
	defer w.mutex.Unlock()
	w.called = &DataLoaderStatValue{}
	w.returnEmptyKey = &DataLoaderStatValue{}
	w.returnNilData = &DataLoaderStatValue{}
	w.returnData = &DataLoaderStatValue{}
}

// Stat returns information about the data loader.
func (w *DataLoaderStatWrapper) Stat() *DataLoaderStat {
	w.mutex.RLock()
	defer w.mutex.RUnlock()
	return &DataLoaderStat{
		Called:         *w.called,
		ReturnEmptyKey: *w.returnEmptyKey,
		ReturnNilData:  *w.returnNilData,
		ReturnData:     *w.returnData,
	}
}

// Name ensures that this implements the Describer interface.
func (w *DataLoaderStatWrapper) Name() string {
	if d, ok := w.Loader.(Describer); ok {
		return d.Name()
	}
	return "Anonymous"
}

// Description ensures that this implements the Describer interface.
func (w *DataLoaderStatWrapper) Description() string {
	if d, ok := w.Loader.(Describer); ok {
		return d.Description()
	}
	return fmt.Sprintf("Anonymous %T", w.Loader)
}

// DataLoaderStat is returned from the DataLoaderStatWrapper.Stat method.
type DataLoaderStat struct {
	Called         DataLoaderStatValue `json:"called"`
	ReturnEmptyKey DataLoaderStatValue `json:"returnEmptyKey"`
	ReturnNilData  DataLoaderStatValue `json:"returnNilData"`
	ReturnData     DataLoaderStatValue `json:"returnData"`
}

// LoadData will load data for item concurrently for
// each loader provided. Data is assigned to the item in
// the same order as the arguments provided to this function.
func LoadData(item *Item, loaders ...DataLoader) {
	type data struct {
		k string
		v interface{}
	}
	wg := sync.WaitGroup{}
	newData := make([]*data, len(loaders))
	for offset, loader := range loaders {
		wg.Add(1)
		go func(offset int, loader DataLoader) {
			defer wg.Done()
			k, v := loader.LoadData(item)
			newData[offset] = &data{
				k: k,
				v: v,
			}
		}(offset, loader)
	}
	wg.Wait()
	for _, d := range newData {
		// We still assign nil data values to the map but not empty an key.
		if d.k != "" {
			item.Data[d.k] = d.v
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build (darwin && amd64) || linux
// +build darwin,amd64 linux

package unix_test

import (
	"runtime"
	"testing"

	"golang.org/x/sys/unix"
)

func TestSysvSharedMemory(t *testing.T) {
	// create ipc
	id, err := unix.SysvShmGet(unix.IPC_PRIVATE, 1024, unix.IPC_CREAT|unix.IPC_EXCL|0o600)

	// ipc isn't implemented on android, should fail
	if runtime.GOOS == "android" {
		if err != unix.ENOSYS {
			t.Fatalf("expected android to fail, but it didn't")
		}
		return
	}

	if err != nil {
		t.Fatalf("SysvShmGet: %v", err)
	}
	defer func() {
		_, err := unix.SysvShmCtl(id, unix.IPC_RMID, nil)
		if err != nil {
			t.Errorf("Remove failed: %v", err)
		}
	}()

	// attach
	b1, err := unix.SysvShmAttach(id, 0, 0)
	if err != nil {
		t.Fatalf("Attach: %v", err)
	}

	if len(b1) != 1024 {
		t.Fatalf("b1 len = %v, want 1024", len(b1))
	}

	b1[42] = 'x'

	// attach again
	b2, err := unix.SysvShmAttach(id, 0, 0)
	if err != nil {
		t.Fatalf("Attach: %v", err)
	}

	if len(b2) != 1024 {
		t.Fatalf("b2 len = %v, want 1024", len(b1))
	}

	b2[43] = 'y'
	if b2[42] != 'x' || b1[43] != 'y' {
		t.Fatalf("shared memory isn't shared")
	}

	// detach
	if err = unix.SysvShmDetach(b2); err != nil {
		t.Fatalf("Detach: %v", err)
	}

	if b1[42] != 'x' || b1[43] != 'y' {
		t.Fatalf("shared memory was invalidated")
	}
}
<commit_msg>unix: skip TestSysvSharedMemory on ENOSYS<commit_after>// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build (darwin && amd64) || linux
// +build darwin,amd64 linux

package unix_test

import (
	"runtime"
	"testing"

	"golang.org/x/sys/unix"
)

func TestSysvSharedMemory(t *testing.T) {
	// create ipc
	id, err := unix.SysvShmGet(unix.IPC_PRIVATE, 1024, unix.IPC_CREAT|unix.IPC_EXCL|0o600)

	// ipc isn't implemented on android, should fail
	if runtime.GOOS == "android" {
		if err != unix.ENOSYS {
			t.Fatalf("expected android to fail, but it didn't")
		}
		return
	}

	// The kernel may have been built without System V IPC support.
	if err == unix.ENOSYS {
		t.Skip("shmget not supported")
	}

	if err != nil {
		t.Fatalf("SysvShmGet: %v", err)
	}
	defer func() {
		_, err := unix.SysvShmCtl(id, unix.IPC_RMID, nil)
		if err != nil {
			t.Errorf("Remove failed: %v", err)
		}
	}()

	// attach
	b1, err := unix.SysvShmAttach(id, 0, 0)
	if err != nil {
		t.Fatalf("Attach: %v", err)
	}

	if len(b1) != 1024 {
		t.Fatalf("b1 len = %v, want 1024", len(b1))
	}

	b1[42] = 'x'

	// attach again
	b2, err := unix.SysvShmAttach(id, 0, 0)
	if err != nil {
		t.Fatalf("Attach: %v", err)
	}

	if len(b2) != 1024 {
		t.Fatalf("b2 len = %v, want 1024", len(b1))
	}

	b2[43] = 'y'
	if b2[42] != 'x' || b1[43] != 'y' {
		t.Fatalf("shared memory isn't shared")
	}

	// detach
	if err = unix.SysvShmDetach(b2); err != nil {
		t.Fatalf("Detach: %v", err)
	}

	if b1[42] != 'x' || b1[43] != 'y' {
		t.Fatalf("shared memory was invalidated")
	}
}
<|endoftext|>