<commit_before>package rel

import (
	"testing"
)

func TestDeleteStatementNodeEq(t *testing.T) {
	ds1 := DeleteStatementNode{
		Wheres: &[]Visitable{Sql("a"), Sql("b"), Sql("c")},
	}

	ds2 := DeleteStatementNode{
		Wheres: &[]Visitable{Sql("a"), Sql("b"), Sql("c")},
	}

	if !ds1.Eq(ds2) {
		t.Fail()
	}
}

func TestDeleteStatementNodeNotEq(t *testing.T) {
	ds1 := DeleteStatementNode{
		Wheres: &[]Visitable{Sql("a"), Sql("b"), Sql("c")},
	}

	ds2 := DeleteStatementNode{
		Wheres: &[]Visitable{Sql("1"), Sql("2"), Sql("3")},
	}

	if ds1.Eq(ds2) {
		t.Fail()
	}
}
<commit_msg>Implemented tests for DeleteStatementNode in Ginkgo<commit_after>package rel_test

import (
	. "."
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("DeleteStatementNode", func() {
	It("can be equal to other DeleteStatementNode's", func() {
		ds1 := DeleteStatementNode{
			Wheres: &[]Visitable{Sql("a"), Sql("b"), Sql("c")},
		}

		ds2 := DeleteStatementNode{
			Wheres: &[]Visitable{Sql("a"), Sql("b"), Sql("c")},
		}
		Expect(ds1.Eq(ds2)).To(BeTrue())
	})
})
<|endoftext|>


<commit_before>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package notify

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strings"

	"golang.org/x/net/context"

	"go.chromium.org/gae/service/datastore"
	"go.chromium.org/luci/appengine/tq"
	"go.chromium.org/luci/buildbucket"
	bbapi "go.chromium.org/luci/common/api/buildbucket/buildbucket/v1"
	"go.chromium.org/luci/common/api/gitiles"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/retry/transient"
	"go.chromium.org/luci/server/router"

	"go.chromium.org/luci/luci_notify/config"
	"go.chromium.org/luci/luci_notify/testutil"
)

var (
	errBuilderDeleted = fmt.Errorf("builder deleted between datastore.Get calls")
)

func getCommitBuildSet(sets []buildbucket.BuildSet) *buildbucket.GitilesCommit {
	for _, set := range sets {
		if commit, ok := set.(*buildbucket.GitilesCommit); ok {
			return commit
		}
	}
	return nil
}

func getBuilderID(b *Build) string {
	return fmt.Sprintf("buildbucket/%s/%s/%s", b.Project, b.Bucket, b.Builder)
}

func commitIndex(commits []gitiles.Commit, revision string) int {
	for i, commit := range commits {
		if commit.Commit == revision {
			return i
		}
	}
	return -1
}

func extractEmailNotifyValues(parametersJson string) ([]string, error) {
	if parametersJson == "" {
		return nil, nil
	}

	// json equivalent: {"email_notify": [{"email": "<address>"}, ...]}
	output := struct {
		EmailNotify []struct {
			Email string `json:"email"`
		} `json:"email_notify"`
	}{}

	if err := json.NewDecoder(strings.NewReader(parametersJson)).Decode(&output); err != nil {
		return nil, errors.Annotate(err, "invalid msg.ParametersJson").Err()
	}

	result := []string{}
	for _, r := range output.EmailNotify {
		result = append(result, r.Email)
	}
	return result, nil
}

// handleBuild processes a Build and sends appropriate notifications.
//
// This function should serve as documentation of the process of going from
// a Build to sent notifications. It also should explicitly handle ACLs and
// stop the process of handling notifications early to avoid wasting compute time.
//
// history is a function that contacts gitiles to obtain the git history for
// revision ordering purposes. It's passed in as a parameter in order to mock it
// for testing.
func handleBuild(c context.Context, d *tq.Dispatcher, build *Build, history testutil.HistoryFunc) error {
	builderID := getBuilderID(build)
	logging.Infof(c, "Finding config for %q, %s", builderID, build.Status)
	notifiers, err := config.LookupNotifiers(c, build.Project, builderID)
	if err != nil {
		return errors.Annotate(err, "looking up notifiers").Err()
	}
	if len(notifiers) == 0 {
		// No configurations were found, but we might need to generate notifications
		// based on properties. Don't fall through to avoid storing build status for
		// builds without configurations.
		return Notify(c, d, notifiers, StatusUnknown, build)
	}

	gCommit := getCommitBuildSet(build.BuildSets)
	if gCommit == nil {
		logging.Infof(c, "No revision information found for this build, ignoring...")
		return nil
	}

	// Get the Builder for the first time, and initialize if there's nothing there.
	builder := &Builder{ID: builderID}
	switch err := datastore.Get(c, builder); {
	case err == datastore.ErrNoSuchEntity:
		// If the Builder isn't found, start a transaction and try to store
		// the builder for the first time.
		keepGoing := false
		err = datastore.RunInTransaction(c, func(c context.Context) error {
			switch err := datastore.Get(c, builder); {
			case err == datastore.ErrNoSuchEntity:
				// Notify, but avoid on_change notification by setting Status to StatusUnknown.
				builder.Status = StatusUnknown
				if err := Notify(c, d, notifiers, builder.Status, build); err != nil {
					return err
				}
				// Initialize the Builder in the datastore.
				if err := datastore.Put(c, NewBuilder(builderID, gCommit.Revision, &build.Build)); err != nil {
					return err
				}
				return nil
			case err != nil:
				return err
			}
			// If we actually managed to get a Builder, we should continue with the
			// full code path.
			keepGoing = true
			return nil
		}, nil)
		if err != nil || !keepGoing {
			return err
		}
	case err != nil:
		return err
	}

	// commits contains a list of commits from builder.StatusRevision..gCommit.Revision (oldest..newest).
	commits, err := history(c, gCommit.RepoURL(), builder.StatusRevision, gCommit.Revision)
	if err != nil {
		return err
	}
	if len(commits) == 0 {
		logging.Debugf(c, "Found build with old commit, ignoring...")
		// Notify about the build, but ignore on_change by creating a builder with an unknown status.
		return Notify(c, d, notifiers, StatusUnknown, build)
	}

	// Update `builder`, and check if we need to store a newer version, then store it.
	return datastore.RunInTransaction(c, func(c context.Context) error {
		switch err := datastore.Get(c, builder); {
		case err == datastore.ErrNoSuchEntity:
			return errBuilderDeleted
		case err != nil:
			return err
		}

		// If there's been no change in status, don't bother updating.
		if build.Status == builder.Status {
			return nil
		}

		index := commitIndex(commits, builder.StatusRevision)
		outOfOrder := false
		switch {
		// If the revision is not found, we can conclude that the Builder has
		// advanced beyond gCommit.Revision. This is because:
		//   1) builder.StatusRevision only ever moves forward.
		//   2) commits contains the git history up to gCommit.Revision.
		case index < 0:
			logging.Debugf(c, "Found build with old commit during transaction.")
			outOfOrder = true

		// If the revision is current, check build creation time.
		case index == 0 && builder.StatusBuildTime.After(build.CreationTime):
			logging.Debugf(c, "Found build with the same commit but an old time.")
			outOfOrder = true
		}

		// If the build is out-of-order, we want to ignore only on_change notifications.
		// Since on_change only occurs when Builder Status != StatusUnknown, set builder
		// to a new builder with exactly this property.
		if outOfOrder {
			return Notify(c, d, notifiers, StatusUnknown, build)
		}
		if err := Notify(c, d, notifiers, builder.Status, build); err != nil {
			return err
		}
		return datastore.Put(c, NewBuilder(builderID, gCommit.Revision, &build.Build))
	}, nil)
}

// BuildbucketPubSubHandler is the main entrypoint for a new update from buildbucket's pubsub.
//
// This handler delegates the actual processing of the build to handleBuild.
// Its primary purpose is to unwrap context boilerplate and deal with progress-stopping errors.
func BuildbucketPubSubHandler(ctx *router.Context, d *tq.Dispatcher) {
	c, h := ctx.Context, ctx.Writer
	build, err := extractBuild(c, ctx.Request)
	switch {
	case err != nil:
		logging.WithError(err).Errorf(c, "error while extracting build")
	case !strings.HasPrefix(build.Bucket, "luci."):
		logging.Infof(c, "Received build that isn't part of LUCI, ignoring...")
	case !build.Status.Completed():
		logging.Infof(c, "Received build that hasn't completed yet, ignoring...")
	default:
		if err := handleBuild(c, d, build, gitilesHistory); err != nil {
			logging.WithError(err).Errorf(c, "error while notifying")
			if transient.Tag.In(err) {
				// Transient errors are 500 so that PubSub retries them.
				h.WriteHeader(http.StatusInternalServerError)
				return
			}
		}
	}
	h.WriteHeader(http.StatusOK)
}

// Build is buildbucket.Build along with the parsed 'email_notify' values.
type Build struct {
	buildbucket.Build
	EmailNotify []string
}

// extractBuild constructs a Build from the PubSub HTTP request.
func extractBuild(c context.Context, r *http.Request) (*Build, error) {
	// sent by pubsub.
	// This struct is just convenient for unwrapping the json message
	var msg struct {
		Message struct {
			Data []byte
		}
	}
	if err := json.NewDecoder(r.Body).Decode(&msg); err != nil {
		return nil, errors.Annotate(err, "could not decode message").Err()
	}

	var message struct {
		Build bbapi.ApiCommonBuildMessage
	}
	if err := json.Unmarshal(msg.Message.Data, &message); err != nil {
		return nil, errors.Annotate(err, "could not parse pubsub message data").Err()
	}
	var build Build
	if err := build.ParseMessage(&message.Build); err != nil {
		return nil, errors.Annotate(err, "could not decode buildbucket build").Err()
	}

	emails, err := extractEmailNotifyValues(message.Build.ParametersJson)
	if err != nil {
		return nil, errors.Annotate(err, "could not decode email_notify").Err()
	}
	build.EmailNotify = emails

	return &build, nil
}
<commit_msg>luci-notify: Address nits from CL:963480<commit_after>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package notify

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strings"

	"golang.org/x/net/context"

	"go.chromium.org/gae/service/datastore"
	"go.chromium.org/luci/appengine/tq"
	"go.chromium.org/luci/buildbucket"
	bbapi "go.chromium.org/luci/common/api/buildbucket/buildbucket/v1"
	"go.chromium.org/luci/common/api/gitiles"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/retry/transient"
	"go.chromium.org/luci/server/router"

	"go.chromium.org/luci/luci_notify/config"
	"go.chromium.org/luci/luci_notify/testutil"
)

var (
	errBuilderDeleted = fmt.Errorf("builder deleted between datastore.Get calls")
)

func getCommitBuildSet(sets []buildbucket.BuildSet) *buildbucket.GitilesCommit {
	for _, set := range sets {
		if commit, ok := set.(*buildbucket.GitilesCommit); ok {
			return commit
		}
	}
	return nil
}

func getBuilderID(b *Build) string {
	return fmt.Sprintf("buildbucket/%s/%s/%s", b.Project, b.Bucket, b.Builder)
}

func commitIndex(commits []gitiles.Commit, revision string) int {
	for i, commit := range commits {
		if commit.Commit == revision {
			return i
		}
	}
	return -1
}

func extractEmailNotifyValues(parametersJSON string) ([]string, error) {
	if parametersJSON == "" {
		return nil, nil
	}

	// json equivalent: {"email_notify": [{"email": "<address>"}, ...]}
	var output struct {
		EmailNotify []struct {
			Email string `json:"email"`
		} `json:"email_notify"`
	}

	if err := json.NewDecoder(strings.NewReader(parametersJSON)).Decode(&output); err != nil {
		return nil, errors.Annotate(err, "invalid msg.ParametersJson").Err()
	}

	result := make([]string, len(output.EmailNotify))
	for i, r := range output.EmailNotify {
		result[i] = r.Email
	}
	return result, nil
}

// handleBuild processes a Build and sends appropriate notifications.
//
// This function should serve as documentation of the process of going from
// a Build to sent notifications. It also should explicitly handle ACLs and
// stop the process of handling notifications early to avoid wasting compute time.
//
// history is a function that contacts gitiles to obtain the git history for
// revision ordering purposes. It's passed in as a parameter in order to mock it
// for testing.
func handleBuild(c context.Context, d *tq.Dispatcher, build *Build, history testutil.HistoryFunc) error {
	builderID := getBuilderID(build)
	logging.Infof(c, "Finding config for %q, %s", builderID, build.Status)
	notifiers, err := config.LookupNotifiers(c, build.Project, builderID)
	if err != nil {
		return errors.Annotate(err, "looking up notifiers").Err()
	}
	if len(notifiers) == 0 {
		// No configurations were found, but we might need to generate notifications
		// based on properties. Don't fall through to avoid storing build status for
		// builds without configurations.
		return Notify(c, d, notifiers, StatusUnknown, build)
	}

	gCommit := getCommitBuildSet(build.BuildSets)
	if gCommit == nil {
		logging.Infof(c, "No revision information found for this build, ignoring...")
		return nil
	}

	// Get the Builder for the first time, and initialize if there's nothing there.
	builder := &Builder{ID: builderID}
	switch err := datastore.Get(c, builder); {
	case err == datastore.ErrNoSuchEntity:
		// If the Builder isn't found, start a transaction and try to store
		// the builder for the first time.
		keepGoing := false
		err = datastore.RunInTransaction(c, func(c context.Context) error {
			switch err := datastore.Get(c, builder); {
			case err == datastore.ErrNoSuchEntity:
				// Notify, but avoid on_change notification by setting Status to StatusUnknown.
				builder.Status = StatusUnknown
				if err := Notify(c, d, notifiers, builder.Status, build); err != nil {
					return err
				}
				// Initialize the Builder in the datastore.
				if err := datastore.Put(c, NewBuilder(builderID, gCommit.Revision, &build.Build)); err != nil {
					return err
				}
				return nil
			case err != nil:
				return err
			}
			// If we actually managed to get a Builder, we should continue with the
			// full code path.
			keepGoing = true
			return nil
		}, nil)
		if err != nil || !keepGoing {
			return err
		}
	case err != nil:
		return err
	}

	// commits contains a list of commits from builder.StatusRevision..gCommit.Revision (oldest..newest).
	commits, err := history(c, gCommit.RepoURL(), builder.StatusRevision, gCommit.Revision)
	if err != nil {
		return err
	}
	if len(commits) == 0 {
		logging.Debugf(c, "Found build with old commit, ignoring...")
		// Notify about the build, but ignore on_change by creating a builder with an unknown status.
		return Notify(c, d, notifiers, StatusUnknown, build)
	}

	// Update `builder`, and check if we need to store a newer version, then store it.
	return datastore.RunInTransaction(c, func(c context.Context) error {
		switch err := datastore.Get(c, builder); {
		case err == datastore.ErrNoSuchEntity:
			return errBuilderDeleted
		case err != nil:
			return err
		}

		// If there's been no change in status, don't bother updating.
		if build.Status == builder.Status {
			return nil
		}

		index := commitIndex(commits, builder.StatusRevision)
		outOfOrder := false
		switch {
		// If the revision is not found, we can conclude that the Builder has
		// advanced beyond gCommit.Revision. This is because:
		//   1) builder.StatusRevision only ever moves forward.
		//   2) commits contains the git history up to gCommit.Revision.
		case index < 0:
			logging.Debugf(c, "Found build with old commit during transaction.")
			outOfOrder = true

		// If the revision is current, check build creation time.
		case index == 0 && builder.StatusBuildTime.After(build.CreationTime):
			logging.Debugf(c, "Found build with the same commit but an old time.")
			outOfOrder = true
		}

		// If the build is out-of-order, we want to ignore only on_change notifications.
		// Since on_change only occurs when Builder Status != StatusUnknown, set builder
		// to a new builder with exactly this property.
		if outOfOrder {
			return Notify(c, d, notifiers, StatusUnknown, build)
		}
		if err := Notify(c, d, notifiers, builder.Status, build); err != nil {
			return err
		}
		return datastore.Put(c, NewBuilder(builderID, gCommit.Revision, &build.Build))
	}, nil)
}

// BuildbucketPubSubHandler is the main entrypoint for a new update from buildbucket's pubsub.
//
// This handler delegates the actual processing of the build to handleBuild.
// Its primary purpose is to unwrap context boilerplate and deal with progress-stopping errors.
func BuildbucketPubSubHandler(ctx *router.Context, d *tq.Dispatcher) {
	c, h := ctx.Context, ctx.Writer
	build, err := extractBuild(c, ctx.Request)
	switch {
	case err != nil:
		logging.WithError(err).Errorf(c, "error while extracting build")
	case !strings.HasPrefix(build.Bucket, "luci."):
		logging.Infof(c, "Received build that isn't part of LUCI, ignoring...")
	case !build.Status.Completed():
		logging.Infof(c, "Received build that hasn't completed yet, ignoring...")
	default:
		if err := handleBuild(c, d, build, gitilesHistory); err != nil {
			logging.WithError(err).Errorf(c, "error while notifying")
			if transient.Tag.In(err) {
				// Transient errors are 500 so that PubSub retries them.
				h.WriteHeader(http.StatusInternalServerError)
				return
			}
		}
	}
	h.WriteHeader(http.StatusOK)
}

// Build is buildbucket.Build along with the parsed 'email_notify' values.
type Build struct {
	buildbucket.Build
	EmailNotify []string
}

// extractBuild constructs a Build from the PubSub HTTP request.
func extractBuild(c context.Context, r *http.Request) (*Build, error) {
	// sent by pubsub.
	// This struct is just convenient for unwrapping the json message
	var msg struct {
		Message struct {
			Data []byte
		}
	}
	if err := json.NewDecoder(r.Body).Decode(&msg); err != nil {
		return nil, errors.Annotate(err, "could not decode message").Err()
	}

	var message struct {
		Build bbapi.ApiCommonBuildMessage
	}
	if err := json.Unmarshal(msg.Message.Data, &message); err != nil {
		return nil, errors.Annotate(err, "could not parse pubsub message data").Err()
	}
	var build Build
	if err := build.ParseMessage(&message.Build); err != nil {
		return nil, errors.Annotate(err, "could not decode buildbucket build").Err()
	}

	emails, err := extractEmailNotifyValues(message.Build.ParametersJson)
	if err != nil {
		return nil, errors.Annotate(err, "could not decode email_notify").Err()
	}
	build.EmailNotify = emails

	return &build, nil
}
<|endoftext|>


<commit_before>// Copyright 2018 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package notify

import (
	"bytes"
	"context"
	"net/http"
	"path"
	"strings"

	"github.com/golang/protobuf/proto"

	"go.chromium.org/luci/common/api/gitiles"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/proto/srcman"
	"go.chromium.org/luci/common/retry/transient"
	"go.chromium.org/luci/grpc/grpcutil"
	"go.chromium.org/luci/grpc/prpc"
	"go.chromium.org/luci/logdog/client/coordinator"
	"go.chromium.org/luci/logdog/common/renderer"
	"go.chromium.org/luci/logdog/common/types"
	"go.chromium.org/luci/server/auth"
)

// CheckoutFunc is a function that given a Build, produces a source checkout
// related to that build.
type CheckoutFunc func(context.Context, *Build) (Checkout, error)

// srcmanCheckout is a CheckoutFunc which retrieves a source checkout related
// to a build by querying LogDog for a source manifest stream associated with
// that build. It assumes that the build has exactly one source manifest.
func srcmanCheckout(c context.Context, build *Build) (Checkout, error) {
	transport, err := auth.GetRPCTransport(c, auth.AsSelf)
	if err != nil {
		return nil, errors.Annotate(err, "getting RPC Transport").Err()
	}
	client := coordinator.NewClient(&prpc.Client{
		C:       &http.Client{Transport: transport},
		Host:    build.Infra.Logdog.Hostname,
		Options: prpc.DefaultOptions(),
	})
	qo := coordinator.QueryOptions{
		ContentType: srcman.ContentTypeSourceManifest,
	}
	logProject := types.ProjectName(build.Infra.Logdog.Project)
	logPath := path.Join(build.Infra.Logdog.Prefix, "**")

	// Perform the query, capturing exactly one log stream and erroring otherwise.
	var log *coordinator.LogStream
	err = client.Query(c, logProject, logPath, qo, func(s *coordinator.LogStream) bool {
		log = s
		return false
	})
	switch {
	case err != nil:
		return nil, grpcutil.WrapIfTransient(err)
	case log == nil:
		logging.Infof(c, "unable to find source manifest in project %s at path %s",
			build.Infra.Logdog.Project, logPath)
		return nil, nil
	}

	// Read the source manifest from the log stream.
	var buf bytes.Buffer
	_, err = buf.ReadFrom(&renderer.Renderer{
		Source: client.Stream(logProject, log.Path).Fetcher(c, nil),
		Raw:    true,
	})
	if err != nil {
		return nil, errors.Annotate(err, "failed to read stream").Tag(transient.Tag).Err()
	}

	// Unmarshal the source manifest from the bytes.
	var manifest srcman.Manifest
	if err := proto.Unmarshal(buf.Bytes(), &manifest); err != nil {
		return nil, err
	}

	results := make(Checkout)
	for dirname, dir := range manifest.Directories {
		gitCheckout := dir.GetGitCheckout()
		if gitCheckout == nil {
			continue
		}

		url, err := gitiles.NormalizeRepoURL(gitCheckout.RepoUrl, false)
		if err != nil {
			logging.WithError(err).Warningf(c, "could not parse RepoURL %q for dir %q", gitCheckout.RepoUrl, dirname)
			continue
		}

		if !strings.HasSuffix(url.Host, ".googlesource.com") {
			logging.WithError(err).Warningf(c, "unsupported git host %q for dir %q", gitCheckout.RepoUrl, dirname)
			continue
		}
		results[url.String()] = gitCheckout.Revision
	}
	return results, nil
}
<commit_msg>[notify] Skip builds that do not know their Logdog hostname.<commit_after>// Copyright 2018 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package notify

import (
	"bytes"
	"context"
	"net/http"
	"path"
	"strings"

	"github.com/golang/protobuf/proto"

	"go.chromium.org/luci/common/api/gitiles"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/proto/srcman"
	"go.chromium.org/luci/common/retry/transient"
	"go.chromium.org/luci/grpc/grpcutil"
	"go.chromium.org/luci/grpc/prpc"
	"go.chromium.org/luci/logdog/client/coordinator"
	"go.chromium.org/luci/logdog/common/renderer"
	"go.chromium.org/luci/logdog/common/types"
	"go.chromium.org/luci/server/auth"
)

// CheckoutFunc is a function that given a Build, produces a source checkout
// related to that build.
type CheckoutFunc func(context.Context, *Build) (Checkout, error)

// srcmanCheckout is a CheckoutFunc which retrieves a source checkout related
// to a build by querying LogDog for a source manifest stream associated with
// that build. It assumes that the build has exactly one source manifest.
func srcmanCheckout(c context.Context, build *Build) (Checkout, error) {
	if build.Infra == nil || build.Infra.Logdog == nil || build.Infra.Logdog.Hostname == "" {
		return nil, errors.Reason("logdog hostname is not set in the build proto").Err()
	}
	transport, err := auth.GetRPCTransport(c, auth.AsSelf)
	if err != nil {
		return nil, errors.Annotate(err, "getting RPC Transport").Err()
	}
	client := coordinator.NewClient(&prpc.Client{
		C:       &http.Client{Transport: transport},
		Host:    build.Infra.Logdog.Hostname,
		Options: prpc.DefaultOptions(),
	})
	qo := coordinator.QueryOptions{
		ContentType: srcman.ContentTypeSourceManifest,
	}
	logProject := types.ProjectName(build.Infra.Logdog.Project)
	logPath := path.Join(build.Infra.Logdog.Prefix, "**")

	// Perform the query, capturing exactly one log stream and erroring otherwise.
	var log *coordinator.LogStream
	err = client.Query(c, logProject, logPath, qo, func(s *coordinator.LogStream) bool {
		log = s
		return false
	})
	switch {
	case err != nil:
		return nil, grpcutil.WrapIfTransient(err)
	case log == nil:
		logging.Infof(c, "unable to find source manifest in project %s at path %s",
			build.Infra.Logdog.Project, logPath)
		return nil, nil
	}

	// Read the source manifest from the log stream.
	var buf bytes.Buffer
	_, err = buf.ReadFrom(&renderer.Renderer{
		Source: client.Stream(logProject, log.Path).Fetcher(c, nil),
		Raw:    true,
	})
	if err != nil {
		return nil, errors.Annotate(err, "failed to read stream").Tag(transient.Tag).Err()
	}

	// Unmarshal the source manifest from the bytes.
	var manifest srcman.Manifest
	if err := proto.Unmarshal(buf.Bytes(), &manifest); err != nil {
		return nil, err
	}

	results := make(Checkout)
	for dirname, dir := range manifest.Directories {
		gitCheckout := dir.GetGitCheckout()
		if gitCheckout == nil {
			continue
		}

		url, err := gitiles.NormalizeRepoURL(gitCheckout.RepoUrl, false)
		if err != nil {
			logging.WithError(err).Warningf(c, "could not parse RepoURL %q for dir %q", gitCheckout.RepoUrl, dirname)
			continue
		}

		if !strings.HasSuffix(url.Host, ".googlesource.com") {
			logging.WithError(err).Warningf(c, "unsupported git host %q for dir %q", gitCheckout.RepoUrl, dirname)
			continue
		}
		results[url.String()] = gitCheckout.Revision
	}
	return results, nil
}
<|endoftext|>


<commit_before>package drivers

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	"golang.org/x/sys/unix"

	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/units"
)

// wipeDirectory empties the contents of a directory, but leaves it in place.
func wipeDirectory(path string) error {
	// List all entries.
	entries, err := ioutil.ReadDir(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil
		}
	}

	// Individually wipe all entries.
	for _, entry := range entries {
		entryPath := filepath.Join(path, entry.Name())
		err := os.RemoveAll(entryPath)
		if err != nil {
			return err
		}
	}

	return nil
}

func forceUnmount(path string) (bool, error) {
	unmounted := false

	for {
		// Check if already unmounted
		if !shared.IsMountPoint(path) {
			return unmounted, nil
		}

		// Try a clean unmount first
		err := TryUnmount(path, 0)
		if err != nil {
			// Fallback to lazy unmounting
			err = unix.Unmount(path, unix.MNT_DETACH)
			if err != nil {
				return false, err
			}
		}

		unmounted = true
	}
}

func mountReadOnly(srcPath string, dstPath string) (bool, error) {
	// Check if already mounted.
	if shared.IsMountPoint(dstPath) {
		return false, nil
	}

	// Create a mount entry.
	err := TryMount(srcPath, dstPath, "none", unix.MS_BIND, "")
	if err != nil {
		return false, err
	}

	// Make it read-only.
	err = TryMount("", dstPath, "none", unix.MS_BIND|unix.MS_RDONLY|unix.MS_REMOUNT, "")
	if err != nil {
		forceUnmount(dstPath)
		return false, err
	}

	return true, nil
}

func sameMount(srcPath string, dstPath string) bool {
	// Get the source vfs path information
	var srcFsStat unix.Statfs_t
	err := unix.Statfs(srcPath, &srcFsStat)
	if err != nil {
		return false
	}

	// Get the destination vfs path information
	var dstFsStat unix.Statfs_t
	err = unix.Statfs(dstPath, &dstFsStat)
	if err != nil {
		return false
	}

	// Compare statfs
	if srcFsStat.Type != dstFsStat.Type || srcFsStat.Fsid != dstFsStat.Fsid {
		return false
	}

	// Get the source path information
	var srcStat unix.Stat_t
	err = unix.Stat(srcPath, &srcStat)
	if err != nil {
		return false
	}

	// Get the destination path information
	var dstStat unix.Stat_t
	err = unix.Stat(dstPath, &dstStat)
	if err != nil {
		return false
	}

	// Compare inode
	if srcStat.Ino != dstStat.Ino {
		return false
	}

	return true
}

// TryMount tries mounting a filesystem multiple times. This is useful for unreliable backends.
func TryMount(src string, dst string, fs string, flags uintptr, options string) error {
	var err error

	// Attempt 20 mounts over 10s
	for i := 0; i < 20; i++ {
		err = unix.Mount(src, dst, fs, flags, options)
		if err == nil {
			break
		}

		time.Sleep(500 * time.Millisecond)
	}

	if err != nil {
		return err
	}

	return nil
}

// TryUnmount tries unmounting a filesystem multiple times. This is useful for unreliable backends.
func TryUnmount(path string, flags int) error {
	var err error

	for i := 0; i < 20; i++ {
		err = unix.Unmount(path, flags)
		if err == nil {
			break
		}

		time.Sleep(500 * time.Millisecond)
	}

	if err != nil {
		return err
	}

	return nil
}

func tryExists(path string) bool {
	// Attempt 20 checks over 10s
	for i := 0; i < 20; i++ {
		if shared.PathExists(path) {
			return true
		}

		time.Sleep(500 * time.Millisecond)
	}

	return false
}

func fsUUID(path string) (string, error) {
	return shared.RunCommand("blkid", "-s", "UUID", "-o", "value", path)
}

func hasFilesystem(path string, fsType int64) bool {
	fs := unix.Statfs_t{}

	err := unix.Statfs(path, &fs)
	if err != nil {
		return false
	}

	if int64(fs.Type) != fsType {
		return false
	}

	return true
}

// GetPoolMountPath returns the mountpoint of the given pool.
// {LXD_DIR}/storage-pools/<pool>
func GetPoolMountPath(poolName string) string {
	return shared.VarPath("storage-pools", poolName)
}

// GetVolumeMountPath returns the mount path for a specific volume based on its pool and type and
// whether it is a snapshot or not. For VolumeTypeImage the volName is the image fingerprint.
func GetVolumeMountPath(poolName string, volType VolumeType, volName string) string {
	if shared.IsSnapshot(volName) {
		return shared.VarPath("storage-pools", poolName, fmt.Sprintf("%s-snapshots", string(volType)), volName)
	}

	return shared.VarPath("storage-pools", poolName, string(volType), volName)
}

// GetVolumeSnapshotDir gets the snapshot mount directory for the parent volume.
func GetVolumeSnapshotDir(poolName string, volType VolumeType, volName string) string {
	parent, _, _ := shared.InstanceGetParentAndSnapshotName(volName)
	return shared.VarPath("storage-pools", poolName, fmt.Sprintf("%s-snapshots", string(volType)), parent)
}

// GetSnapshotVolumeName returns the full volume name for a parent volume and snapshot name.
func GetSnapshotVolumeName(parentName, snapshotName string) string {
	return fmt.Sprintf("%s%s%s", parentName, shared.SnapshotDelimiter, snapshotName)
}

// createParentSnapshotDirIfMissing creates the parent directory for volume snapshots
func createParentSnapshotDirIfMissing(poolName string, volType VolumeType, volName string) error {
	snapshotsPath := GetVolumeSnapshotDir(poolName, volType, volName)

	// If it's missing, create it.
	if !shared.PathExists(snapshotsPath) {
		return os.Mkdir(snapshotsPath, 0700)
	}

	return nil
}

// deleteParentSnapshotDirIfEmpty removes the parent snapshot directory if it is empty.
// It accepts the pool name, volume type and parent volume name.
func deleteParentSnapshotDirIfEmpty(poolName string, volType VolumeType, volName string) error {
	snapshotsPath := GetVolumeSnapshotDir(poolName, volType, volName)

	// If it exists, try to delete it.
	if shared.PathExists(snapshotsPath) {
		isEmpty, err := shared.PathIsEmpty(snapshotsPath)
		if err != nil {
			return err
		}

		if isEmpty {
			err := os.Remove(snapshotsPath)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// createSparseFile creates a sparse empty file at specified location with specified size.
func createSparseFile(filePath string, sizeBytes int64) error {
	f, err := os.Create(filePath)
	if err != nil {
		return fmt.Errorf("Failed to open %s: %s", filePath, err)
	}
	defer f.Close()

	err = f.Chmod(0600)
	if err != nil {
		return fmt.Errorf("Failed to chmod %s: %s", filePath, err)
	}

	err = f.Truncate(sizeBytes)
	if err != nil {
		return fmt.Errorf("Failed to create sparse file %s: %s", filePath, err)
	}

	return nil
}

// ensureVolumeBlockFile creates or resizes the raw block file for a volume.
func ensureVolumeBlockFile(vol Volume, path string) error {
	blockSize := vol.ExpandedConfig("size")
	if blockSize == "" {
		blockSize = defaultBlockSize
	}

	blockSizeBytes, err := units.ParseByteSizeString(blockSize)
	if err != nil {
		return err
	}

	if shared.PathExists(path) {
		_, err = shared.RunCommand("qemu-img", "resize", "-f", "raw", path, fmt.Sprintf("%d", blockSizeBytes))
		if err != nil {
			return fmt.Errorf("Failed resizing disk image %s to size %s: %v", path, blockSize, err)
		}
	} else {
		// If path doesn't exist, then there has been no filler function
		// supplied to create it from another source. So instead create an empty
		// volume (use for PXE booting a VM).
		_, err = shared.RunCommand("qemu-img", "create", "-f", "raw", path, fmt.Sprintf("%d", blockSizeBytes))
		if err != nil {
			return fmt.Errorf("Failed creating disk image %s as size %s: %v", path, blockSize, err)
		}
	}

	return nil
}

// mkfsOptions represents options for filesystem creation.
type mkfsOptions struct {
	Label string
}

// makeFSType creates the provided filesystem.
func makeFSType(path string, fsType string, options *mkfsOptions) (string, error) {
	var err error
	var msg string

	fsOptions := options
	if fsOptions == nil {
		fsOptions = &mkfsOptions{}
	}

	cmd := []string{fmt.Sprintf("mkfs.%s", fsType), path}
	if fsOptions.Label != "" {
		cmd = append(cmd, "-L", fsOptions.Label)
	}

	if fsType == "ext4" {
		cmd = append(cmd, "-E", "nodiscard,lazy_itable_init=0,lazy_journal_init=0")
	}

	msg, err = shared.TryRunCommand(cmd[0], cmd[1:]...)
	if err != nil {
		return msg, err
	}

	return "", nil
}

// mountOption represents an individual mount option.
type mountOption struct {
	capture bool
	flag    uintptr
}

// mountOptions represents a list of possible mount options.
var mountOptions = map[string]mountOption{
	"async":         {false, unix.MS_SYNCHRONOUS},
	"atime":         {false, unix.MS_NOATIME},
	"bind":          {true, unix.MS_BIND},
	"defaults":      {true, 0},
	"dev":           {false, unix.MS_NODEV},
	"diratime":      {false, unix.MS_NODIRATIME},
	"dirsync":       {true, unix.MS_DIRSYNC},
	"exec":          {false, unix.MS_NOEXEC},
	"lazytime":      {true, unix.MS_LAZYTIME},
	"mand":          {true, unix.MS_MANDLOCK},
	"noatime":       {true, unix.MS_NOATIME},
	"nodev":         {true, unix.MS_NODEV},
	"nodiratime":    {true, unix.MS_NODIRATIME},
	"noexec":        {true, unix.MS_NOEXEC},
	"nomand":        {false, unix.MS_MANDLOCK},
	"norelatime":    {false, unix.MS_RELATIME},
	"nostrictatime": {false, unix.MS_STRICTATIME},
	"nosuid":        {true, unix.MS_NOSUID},
	"rbind":         {true, unix.MS_BIND | unix.MS_REC},
	"relatime":      {true, unix.MS_RELATIME},
	"remount":       {true, unix.MS_REMOUNT},
	"ro":            {true, unix.MS_RDONLY},
	"rw":            {false, unix.MS_RDONLY},
	"strictatime":   {true, unix.MS_STRICTATIME},
	"suid":          {false, unix.MS_NOSUID},
	"sync":          {true, unix.MS_SYNCHRONOUS},
}

// resolveMountOptions resolves the provided mount options.
func resolveMountOptions(options string) (uintptr, string) {
	mountFlags := uintptr(0)
	tmp := strings.SplitN(options, ",", -1)
	for i := 0; i < len(tmp); i++ {
		opt := tmp[i]
		do, ok := mountOptions[opt]
		if !ok {
			continue
		}

		if do.capture {
			mountFlags |= do.flag
		} else {
			mountFlags &= ^do.flag
		}

		copy(tmp[i:], tmp[i+1:])
		tmp[len(tmp)-1] = ""
		tmp = tmp[:len(tmp)-1]
		i--
	}

	return mountFlags, strings.Join(tmp, ",")
}

// shrinkFileSystem shrinks a filesystem if it is supported. Ext4 volumes will be unmounted temporarily if needed.
func shrinkFileSystem(fsType string, devPath string, vol Volume, byteSize int64) error {
	strSize := fmt.Sprintf("%dK", byteSize/1024)

	switch fsType {
	case "": // if not specified, default to ext4.
		fallthrough
	case "xfs":
		return fmt.Errorf(`Shrinking not supported for filesystem type "%s". A dump, mkfs, and restore are required`, fsType)
	case "ext4":
		return vol.UnmountTask(func(op *operations.Operation) error {
			_, err := shared.TryRunCommand("e2fsck", "-f", "-y", devPath)
			if err != nil {
				return err
			}

			_, err = shared.TryRunCommand("resize2fs", devPath, strSize)
			if err != nil {
				return err
			}

			return nil
		}, nil)
	case "btrfs":
		_, err := shared.TryRunCommand("btrfs", "filesystem", "resize", strSize, vol.MountPath())
		if err != nil {
			return err
		}
	default:
		return fmt.Errorf(`Shrinking not supported for filesystem type "%s"`, fsType)
	}

	return nil
}

// growFileSystem grows a filesystem if it is supported. The volume will be mounted temporarily if needed.
func growFileSystem(fsType string, devPath string, vol Volume) error {
	return vol.MountTask(func(mountPath string, op *operations.Operation) error {
		var msg string
		var err error
		switch fsType {
		case "": // if not specified, default to ext4
			fallthrough
		case "ext4":
			msg, err = shared.TryRunCommand("resize2fs", devPath)
		case "xfs":
			msg, err = shared.TryRunCommand("xfs_growfs", devPath)
		case "btrfs":
			msg, err = shared.TryRunCommand("btrfs", "filesystem", "resize", "max", mountPath)
		default:
			return fmt.Errorf(`Growing not supported for filesystem type "%s"`, fsType)
		}

		if err != nil {
			errorMsg := fmt.Sprintf(`Could not extend underlying %s filesystem for "%s": %s`, fsType, devPath, msg)
			return fmt.Errorf(errorMsg)
		}

		return nil
	}, nil)
}
<commit_msg>lxd/storage/drivers/utils: Comment on shrinkFileSystem<commit_after>package drivers

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	"golang.org/x/sys/unix"

	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/units"
)

// wipeDirectory empties the contents of a directory, but leaves it in place.
func wipeDirectory(path string) error {
	// List all entries.
	entries, err := ioutil.ReadDir(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil
		}
	}

	// Individually wipe all entries.
	for _, entry := range entries {
		entryPath := filepath.Join(path, entry.Name())
		err := os.RemoveAll(entryPath)
		if err != nil {
			return err
		}
	}

	return nil
}

func forceUnmount(path string) (bool, error) {
	unmounted := false

	for {
		// Check if already unmounted
		if !shared.IsMountPoint(path) {
			return unmounted, nil
		}

		// Try a clean unmount first
		err := TryUnmount(path, 0)
		if err != nil {
			// Fallback to lazy unmounting
			err = unix.Unmount(path, unix.MNT_DETACH)
			if err != nil {
				return false, err
			}
		}

		unmounted = true
	}
}

func mountReadOnly(srcPath string, dstPath string) (bool, error) {
	// Check if already mounted.
	if shared.IsMountPoint(dstPath) {
		return false, nil
	}

	// Create a mount entry.
	err := TryMount(srcPath, dstPath, "none", unix.MS_BIND, "")
	if err != nil {
		return false, err
	}

	// Make it read-only.
	err = TryMount("", dstPath, "none", unix.MS_BIND|unix.MS_RDONLY|unix.MS_REMOUNT, "")
	if err != nil {
		forceUnmount(dstPath)
		return false, err
	}

	return true, nil
}

func sameMount(srcPath string, dstPath string) bool {
	// Get the source vfs path information
	var srcFsStat unix.Statfs_t
	err := unix.Statfs(srcPath, &srcFsStat)
	if err != nil {
		return false
	}

	// Get the destination vfs path information
	var dstFsStat unix.Statfs_t
	err = unix.Statfs(dstPath, &dstFsStat)
	if err != nil {
		return false
	}

	// Compare statfs
	if srcFsStat.Type != dstFsStat.Type || srcFsStat.Fsid != dstFsStat.Fsid {
		return false
	}

	// Get the source path information
	var srcStat unix.Stat_t
	err = unix.Stat(srcPath, &srcStat)
	if err != nil {
		return false
	}

	// Get the destination path information
	var dstStat unix.Stat_t
	err = unix.Stat(dstPath, &dstStat)
	if err != nil {
		return false
	}

	// Compare inode
	if srcStat.Ino != dstStat.Ino {
		return false
	}

	return true
}

// TryMount tries mounting a filesystem multiple times. This is useful for unreliable backends.
func TryMount(src string, dst string, fs string, flags uintptr, options string) error {
	var err error

	// Attempt 20 mounts over 10s
	for i := 0; i < 20; i++ {
		err = unix.Mount(src, dst, fs, flags, options)
		if err == nil {
			break
		}

		time.Sleep(500 * time.Millisecond)
	}

	if err != nil {
		return err
	}

	return nil
}

// TryUnmount tries unmounting a filesystem multiple times. This is useful for unreliable backends.
func TryUnmount(path string, flags int) error {
	var err error

	for i := 0; i < 20; i++ {
		err = unix.Unmount(path, flags)
		if err == nil {
			break
		}

		time.Sleep(500 * time.Millisecond)
	}

	if err != nil {
		return err
	}

	return nil
}

func tryExists(path string) bool {
	// Attempt 20 checks over 10s
	for i := 0; i < 20; i++ {
		if shared.PathExists(path) {
			return true
		}

		time.Sleep(500 * time.Millisecond)
	}

	return false
}

func fsUUID(path string) (string, error) {
	return shared.RunCommand("blkid", "-s", "UUID", "-o", "value", path)
}

func hasFilesystem(path string, fsType int64) bool {
	fs := unix.Statfs_t{}

	err := unix.Statfs(path, &fs)
	if err != nil {
		return false
	}

	if int64(fs.Type) != fsType {
		return false
	}

	return true
}

// GetPoolMountPath returns the mountpoint of the given pool.
// {LXD_DIR}/storage-pools/<pool>
func GetPoolMountPath(poolName string) string {
	return shared.VarPath("storage-pools", poolName)
}

// GetVolumeMountPath returns the mount path for a specific volume based on its pool and type and
// whether it is a snapshot or not. For VolumeTypeImage the volName is the image fingerprint.
func GetVolumeMountPath(poolName string, volType VolumeType, volName string) string {
	if shared.IsSnapshot(volName) {
		return shared.VarPath("storage-pools", poolName, fmt.Sprintf("%s-snapshots", string(volType)), volName)
	}

	return shared.VarPath("storage-pools", poolName, string(volType), volName)
}

// GetVolumeSnapshotDir gets the snapshot mount directory for the parent volume.
func GetVolumeSnapshotDir(poolName string, volType VolumeType, volName string) string {
	parent, _, _ := shared.InstanceGetParentAndSnapshotName(volName)
	return shared.VarPath("storage-pools", poolName, fmt.Sprintf("%s-snapshots", string(volType)), parent)
}

// GetSnapshotVolumeName returns the full volume name for a parent volume and snapshot name.
func GetSnapshotVolumeName(parentName, snapshotName string) string {
	return fmt.Sprintf("%s%s%s", parentName, shared.SnapshotDelimiter, snapshotName)
}

// createParentSnapshotDirIfMissing creates the parent directory for volume snapshots
func createParentSnapshotDirIfMissing(poolName string, volType VolumeType, volName string) error {
	snapshotsPath := GetVolumeSnapshotDir(poolName, volType, volName)

	// If it's missing, create it.
	if !shared.PathExists(snapshotsPath) {
		return os.Mkdir(snapshotsPath, 0700)
	}

	return nil
}

// deleteParentSnapshotDirIfEmpty removes the parent snapshot directory if it is empty.
// It accepts the pool name, volume type and parent volume name.
func deleteParentSnapshotDirIfEmpty(poolName string, volType VolumeType, volName string) error {
	snapshotsPath := GetVolumeSnapshotDir(poolName, volType, volName)

	// If it exists, try to delete it.
	if shared.PathExists(snapshotsPath) {
		isEmpty, err := shared.PathIsEmpty(snapshotsPath)
		if err != nil {
			return err
		}

		if isEmpty {
			err := os.Remove(snapshotsPath)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// createSparseFile creates a sparse empty file at specified location with specified size.
func createSparseFile(filePath string, sizeBytes int64) error {
	f, err := os.Create(filePath)
	if err != nil {
		return fmt.Errorf("Failed to open %s: %s", filePath, err)
	}
	defer f.Close()

	err = f.Chmod(0600)
	if err != nil {
		return fmt.Errorf("Failed to chmod %s: %s", filePath, err)
	}

	err = f.Truncate(sizeBytes)
	if err != nil {
		return fmt.Errorf("Failed to create sparse file %s: %s", filePath, err)
	}

	return nil
}

// ensureVolumeBlockFile creates or resizes the raw block file for a volume.
func ensureVolumeBlockFile(vol Volume, path string) error {
	blockSize := vol.ExpandedConfig("size")
	if blockSize == "" {
		blockSize = defaultBlockSize
	}

	blockSizeBytes, err := units.ParseByteSizeString(blockSize)
	if err != nil {
		return err
	}

	if shared.PathExists(path) {
		_, err = shared.RunCommand("qemu-img", "resize", "-f", "raw", path, fmt.Sprintf("%d", blockSizeBytes))
		if err != nil {
			return fmt.Errorf("Failed resizing disk image %s to size %s: %v", path, blockSize, err)
		}
	} else {
		// If path doesn't exist, then there has been no filler function
		// supplied to create it from another source. So instead create an empty
		// volume (use for PXE booting a VM).
		_, err = shared.RunCommand("qemu-img", "create", "-f", "raw", path, fmt.Sprintf("%d", blockSizeBytes))
		if err != nil {
			return fmt.Errorf("Failed creating disk image %s as size %s: %v", path, blockSize, err)
		}
	}

	return nil
}

// mkfsOptions represents options for filesystem creation.
type mkfsOptions struct {
	Label string
}

// makeFSType creates the provided filesystem.
func makeFSType(path string, fsType string, options *mkfsOptions) (string, error) {
	var err error
	var msg string

	fsOptions := options
	if fsOptions == nil {
		fsOptions = &mkfsOptions{}
	}

	cmd := []string{fmt.Sprintf("mkfs.%s", fsType), path}
	if fsOptions.Label != "" {
		cmd = append(cmd, "-L", fsOptions.Label)
	}

	if fsType == "ext4" {
		cmd = append(cmd, "-E", "nodiscard,lazy_itable_init=0,lazy_journal_init=0")
	}

	msg, err = shared.TryRunCommand(cmd[0], cmd[1:]...)
	if err != nil {
		return msg, err
	}

	return "", nil
}

// mountOption represents an individual mount option.
type mountOption struct {
	capture bool
	flag    uintptr
}

// mountOptions represents a list of possible mount options.
var mountOptions = map[string]mountOption{
	"async":         {false, unix.MS_SYNCHRONOUS},
	"atime":         {false, unix.MS_NOATIME},
	"bind":          {true, unix.MS_BIND},
	"defaults":      {true, 0},
	"dev":           {false, unix.MS_NODEV},
	"diratime":      {false, unix.MS_NODIRATIME},
	"dirsync":       {true, unix.MS_DIRSYNC},
	"exec":          {false, unix.MS_NOEXEC},
	"lazytime":      {true, unix.MS_LAZYTIME},
	"mand":          {true, unix.MS_MANDLOCK},
	"noatime":       {true, unix.MS_NOATIME},
	"nodev":         {true, unix.MS_NODEV},
	"nodiratime":    {true, unix.MS_NODIRATIME},
	"noexec":        {true, unix.MS_NOEXEC},
	"nomand":        {false, unix.MS_MANDLOCK},
	"norelatime":    {false, unix.MS_RELATIME},
	"nostrictatime": {false, unix.MS_STRICTATIME},
	"nosuid":        {true, unix.MS_NOSUID},
	"rbind":         {true, unix.MS_BIND | unix.MS_REC},
	"relatime":      {true, unix.MS_RELATIME},
	"remount":       {true, unix.MS_REMOUNT},
	"ro":            {true, unix.MS_RDONLY},
	"rw":            {false, unix.MS_RDONLY},
	"strictatime":   {true, unix.MS_STRICTATIME},
	"suid":          {false, unix.MS_NOSUID},
	"sync":          {true, unix.MS_SYNCHRONOUS},
}

// resolveMountOptions resolves the provided mount options.
func resolveMountOptions(options string) (uintptr, string) {
	mountFlags := uintptr(0)
	tmp := strings.SplitN(options, ",", -1)
	for i := 0; i < len(tmp); i++ {
		opt := tmp[i]
		do, ok := mountOptions[opt]
		if !ok {
			continue
		}

		if do.capture {
			mountFlags |= do.flag
		} else {
			mountFlags &= ^do.flag
		}

		copy(tmp[i:], tmp[i+1:])
		tmp[len(tmp)-1] = ""
		tmp = tmp[:len(tmp)-1]
		i--
	}

	return mountFlags, strings.Join(tmp, ",")
}

// shrinkFileSystem shrinks a filesystem if it is supported. Ext4 volumes will be unmounted temporarily if needed.
func shrinkFileSystem(fsType string, devPath string, vol Volume, byteSize int64) error {
	// The smallest unit that resize2fs accepts in byte size (rather than blocks) is kilobytes.
	strSize := fmt.Sprintf("%dK", byteSize/1024)

	switch fsType {
	case "": // if not specified, default to ext4.
		fallthrough
	case "xfs":
		return fmt.Errorf(`Shrinking not supported for filesystem type "%s". A dump, mkfs, and restore are required`, fsType)
	case "ext4":
		return vol.UnmountTask(func(op *operations.Operation) error {
			_, err := shared.TryRunCommand("e2fsck", "-f", "-y", devPath)
			if err != nil {
				return err
			}

			_, err = shared.TryRunCommand("resize2fs", devPath, strSize)
			if err != nil {
				return err
			}

			return nil
		}, nil)
	case "btrfs":
		_, err := shared.TryRunCommand("btrfs", "filesystem", "resize", strSize, vol.MountPath())
		if err != nil {
			return err
		}
	default:
		return fmt.Errorf(`Shrinking not supported for filesystem type "%s"`, fsType)
	}

	return nil
}

// growFileSystem grows a filesystem if it is supported. The volume will be mounted temporarily if needed.
func growFileSystem(fsType string, devPath string, vol Volume) error {
	return vol.MountTask(func(mountPath string, op *operations.Operation) error {
		var msg string
		var err error
		switch fsType {
		case "": // if not specified, default to ext4
			fallthrough
		case "ext4":
			msg, err = shared.TryRunCommand("resize2fs", devPath)
		case "xfs":
			msg, err = shared.TryRunCommand("xfs_growfs", devPath)
		case "btrfs":
			msg, err = shared.TryRunCommand("btrfs", "filesystem", "resize", "max", mountPath)
		default:
			return fmt.Errorf(`Growing not supported for filesystem type "%s"`, fsType)
		}

		if err != nil {
			errorMsg := fmt.Sprintf(`Could not extend underlying %s filesystem for "%s": %s`, fsType, devPath, msg)
			return fmt.Errorf(errorMsg)
		}

		return nil
	}, nil)
}
<|endoftext|>


<commit_before>package main

import "fmt"

func main() {
	var x int = 100
	var str string = "hello world"
	//声明初始化多个变量
	var i, j, k int = 1, 2, 3

	//不用指明类型，通过初始化值来推导
	var b = true
	// 等价与 var y int = 100
	y := 100

	const s string = "hello world"
	const s1 = "test const 类型推导"
	const pi float32 = 3.14
	fmt.Printf("x: %d\n", x)
	fmt.Printf("str: %s\n", str)
	fmt.Printf("i: %d, j: %d, k: %d\n", i, j, k)
	fmt.Printf("b: %t\n", b)
	fmt.Printf("y: %d\n", y)
	fmt.Printf("s: %s\n", s)
	fmt.Printf("s1: %s\n", s1)
	fmt.Printf("pi: %f\n", pi)
	fmt.Printf("test end!\n")
}
<commit_msg>test float<commit_after>package main

import "fmt"

func main() {
	var x int = 100
	var str string = "hello world"
	//声明初始化多个变量
	var i, j, k int = 1, 2, 3

	//不用指明类型，通过初始化值来推导
	var b = true
	// 等价与 var y int = 100
	y := 100

	const s string = "hello world"
	const s1 = "test const 类型推导"
	const pi float32 = 3.14
	fmt.Printf("x: %d\n", x)
	fmt.Printf("str: %s\n", str)
	fmt.Printf("i: %d, j: %d, k: %d\n", i, j, k)
	fmt.Printf("b: %t\n", b)
	fmt.Printf("y: %d\n", y)
	fmt.Printf("s: %s\n", s)
	fmt.Printf("s1: %s\n", s1)
	fmt.Printf("pi: %f\n", pi)
	fmt.Printf("test end!\n")
	var na float64 = 1.1234567899
	var nb float64 = 1.12345678
	var nc float64 = 1.123456781

	fmt.Println(na==nb, na==nc)
}
<|endoftext|>


<commit_before>package https

import (
	"bufio"
	"context"
	"crypto"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"sync"
	"time"

	log "github.com/golang/glog"
	"github.com/golang/protobuf/proto"
	"github.com/google/fleetspeak/fleetspeak/src/common"
	fspb "github.com/google/fleetspeak/fleetspeak/src/common/proto/fleetspeak"
	"github.com/google/fleetspeak/fleetspeak/src/server/comms"
	"github.com/google/fleetspeak/fleetspeak/src/server/db"
	"github.com/google/fleetspeak/fleetspeak/src/server/stats"
)

const magic = uint32(0xf1ee1001)

type fullResponseWriter interface {
	http.ResponseWriter
	http.CloseNotifier
	http.Flusher
}

func readUint32(body *bufio.Reader) (uint32, error) {
	b := make([]byte, 4)
	_, err := io.ReadAtLeast(body, b, 4)
	if err != nil {
		return 0, err
	}
	return binary.LittleEndian.Uint32(b), nil
}

func writeUint32(res fullResponseWriter, i uint32) error {
	return binary.Write(res, binary.LittleEndian, i)
}

// messageServer wraps a Communicator in order to handle clients polls.
type streamingMessageServer struct {
	*Communicator
}

func (s streamingMessageServer) ServeHTTP(res http.ResponseWriter, req *http.Request) {
	earlyError := func(msg string, status int) {
		log.ErrorDepth(1, fmt.Sprintf("%s: %s", http.StatusText(status), msg))
		s.fs.StatsCollector().ClientPoll(stats.PollInfo{
			CTX:    req.Context(),
			Start:  db.Now(),
			End:    db.Now(),
			Status: status,
		})
	}

	if !s.startProcessing() {
		earlyError("server not ready", http.StatusInternalServerError)
		return
	}
	defer s.stopProcessing()

	fullRes, ok := res.(fullResponseWriter)
	if !ok {
		earlyError("/streaming-message requested, but not supported. ResponseWriter is not a fullResponseWriter", http.StatusNotFound)
		return
	}

	if req.Method != http.MethodPost {
		earlyError(fmt.Sprintf("%v not supported", req.Method), http.StatusBadRequest)
		return
	}

	if req.TLS == nil {
		earlyError("TLS information not found", http.StatusBadRequest)
		return
	}

	if len(req.TLS.PeerCertificates) != 1 {
		earlyError(fmt.Sprintf("expected 1 client cert, received %v", len(req.TLS.PeerCertificates)), http.StatusBadRequest)
		return
	}

	cert := req.TLS.PeerCertificates[0]
	if cert.PublicKey == nil {
		earlyError("public key not present in client cert", http.StatusBadRequest)
		return
	}

	body := bufio.NewReader(req.Body)

	// Set a 10 min overall maximum lifespan of the connection.
	ctx, fin := context.WithTimeout(req.Context(), 10*time.Minute)
	defer fin()

	// Also create a way to terminate early in case of error.
	ctx, cancel := context.WithCancel(ctx)

	info, err := s.initialPoll(ctx, addrFromString(req.RemoteAddr), cert.PublicKey, fullRes, body)
	if err != nil || info == nil {
		return
	}

	m := streamManager{
		ctx:  ctx,
		s:    s,
		info: info,
		res:  fullRes,
		body: body,

		cancel: cancel,
	}
	defer func() {
		// Shutdown is a bit subtle.
		//
		// We get here iff m.ctx is canceled, timed out, etc.
		//
		// Once ctx is canceled, writeLoop will notice, close the outgoing
		// ResponseWriter, and begin blindly draining m.out.
		//
		// Closing ResponseWriter will cause any pending read to error out and
		// readLoop to return.
		//
		// Once the readLoop returns, we can safely close m.out and wait for
		// writeLoop to finish.
		m.reading.Wait()
		close(m.out)
		m.writing.Wait()
	}()

	m.reading.Add(1)
	go m.readLoop()

	m.writing.Add(1)
	go m.writeLoop()

	select {
	case <-ctx.Done():
		return
	case <-fullRes.CloseNotify():
		m.cancel()
		return
	case <-s.stopping:
		// Communicator is shutting down.
		m.cancel()
		return
	}
}

func (s streamingMessageServer) initialPoll(ctx context.Context, addr net.Addr, key crypto.PublicKey, res fullResponseWriter, body *bufio.Reader) (*comms.ConnectionInfo, error) {
	ctx, fin := context.WithTimeout(ctx, time.Minute)

	pi := stats.PollInfo{
		CTX:    ctx,
		Start:  db.Now(),
		Status: http.StatusTeapot, // Should never actually be returned
	}
	defer func() {
		fin()
		if pi.Status == http.StatusTeapot {
			log.Errorf("Forgot to set status, PollInfo: %v", pi)
		}
		pi.End = db.Now()
		s.fs.StatsCollector().ClientPoll(pi)
	}()

	makeError := func(msg string, status int) error {
		log.ErrorDepth(1, fmt.Sprintf("%s: %s", http.StatusText(status), msg))
		pi.Status = status
		return errors.New(msg)
	}

	id, err := common.MakeClientID(key)
	if err != nil {
		return nil, makeError(fmt.Sprintf("unable to create client id from public key: %v", err), http.StatusBadRequest)
	}
	pi.ID = id

	m, err := readUint32(body)
	if err != nil {
		return nil, makeError(fmt.Sprintf("error reading magic number: %v", err), http.StatusBadRequest)
	}
	if m != magic {
		return nil, makeError(fmt.Sprintf("unknown magic number: got %x, expected %x", m, magic), http.StatusBadRequest)
	}

	st := time.Now()
	size, err := binary.ReadUvarint(body)
	if err != nil {
		return nil, makeError(fmt.Sprintf("error reading size: %v", err), http.StatusBadRequest)
	}
	if size > MaxContactSize {
		return nil, makeError(fmt.Sprintf("initial contact size too large: got %d, expected less than %d", size, MaxContactSize), http.StatusBadRequest)
	}

	buf := make([]byte, size)
	_, err = io.ReadFull(body, buf)
	if err != nil {
		return nil, makeError(fmt.Sprintf("error reading body for initial exchange: %v", err), http.StatusBadRequest)
	}
	pi.ReadTime = time.Since(st)
	pi.ReadBytes = int(size)

	var wcd fspb.WrappedContactData
	if err = proto.Unmarshal(buf, &wcd); err != nil {
		return nil, makeError(fmt.Sprintf("error parsing body: %v", err), http.StatusBadRequest)
	}

	info, toSend, err := s.fs.InitializeConnection(ctx, addr, key, &wcd, true)
	if err == comms.ErrNotAuthorized {
		return nil, makeError("not authorized", http.StatusServiceUnavailable)
	}
	if err != nil {
		return nil, makeError(fmt.Sprintf("error processing contact: %v", err), http.StatusInternalServerError)
	}
	pi.CacheHit = info.Client.Cached

	out := proto.NewBuffer(make([]byte, 0, 1024))
	// EncodeMessage prepends the size as a Varint.
	if err := out.EncodeMessage(toSend); err != nil {
		return nil, makeError(fmt.Sprintf("error preparing messages: %v", err), http.StatusInternalServerError)
	}
	st = time.Now()
	sz, err := res.Write(out.Bytes())
	if err != nil {
		return nil, makeError(fmt.Sprintf("error writing body: %v", err), http.StatusInternalServerError)
	}
	res.Flush()

	pi.WriteTime = time.Since(st)
	pi.End = time.Now()
	pi.WriteBytes = sz
	pi.Status = http.StatusOK
	return info, nil
}

type streamManager struct {
	ctx context.Context
	s   streamingMessageServer

	info *comms.ConnectionInfo
	res  fullResponseWriter
	body *bufio.Reader

	// The read- and writeLoop will wait for these. Separate because readloop
	// needs to finish before writeLoop.
	reading sync.WaitGroup
	writing sync.WaitGroup

	out chan *fspb.ContactData

	cancel func() // Shuts down the stream when called.
}

func (m *streamManager) readLoop() {
	defer m.reading.Done()
	defer m.cancel()

	cnt := uint64(0)

	for {
		pi, err := m.readOne()
		if err != nil {
			// If the context has been canceled, it is probably a 'normal' termination
			// - disconnect, max connection durating, etc. But if it is still active,
			// we are going to tear down everything because of an unexpected read
			// error and should log/record why.
			if m.ctx.Err() == nil && pi != nil {
				m.s.fs.StatsCollector().ClientPoll(*pi)
				log.Errorf("Streaming Connection to %v terminated with error: %v", m.info.Client.ID, err)
			}
			return
		}
		m.s.fs.StatsCollector().ClientPoll(*pi)
		cnt += 1
		m.out <- &fspb.ContactData{AckIndex: cnt}
	}
}

func (m *streamManager) readOne() (*stats.PollInfo, error) {
	size, err := binary.ReadUvarint(m.body)
	if err != nil {
		return nil, err
	}
	if size > MaxContactSize {
		return nil, fmt.Errorf("streaming contact size too large: got %d, expected less than %d", size, MaxContactSize)
	}

	pi := stats.PollInfo{
		CTX:      m.ctx,
		ID:       m.info.Client.ID,
		Start:    db.Now(),
		Status:   http.StatusTeapot,
		CacheHit: true,
	}
	defer func() {
		if pi.Status == http.StatusTeapot {
			log.Errorf("Forgot to set status.")
		}
		pi.End = db.Now()
	}()
	buf := make([]byte, size)
	if _, err := io.ReadFull(m.body, buf); err != nil {
		return &pi, fmt.Errorf("Error reading streamed data: %v", err)
	}
	pi.ReadTime = time.Since(pi.Start)
	pi.ReadBytes = int(size)

	var wcd fspb.WrappedContactData
	if err = proto.Unmarshal(buf, &wcd); err != nil {
		pi.Status = http.StatusBadRequest
		return &pi, fmt.Errorf("Error parsing streamed data: %v", err)
	}
	if err := m.s.fs.HandleMessagesFromClient(m.ctx, m.info, &wcd); err != nil {
		if err == comms.ErrNotAuthorized {
			pi.Status = http.StatusServiceUnavailable
		} else {
			pi.Status = http.StatusInternalServerError
			err = fmt.Errorf("Error processing streamed messages: %v", err)
		}
		return &pi, err
	}

	return &pi, nil
}

func (m *streamManager) writeLoop() {
	defer m.writing.Done()
	defer func() {
		for range m.out {
		}
	}()

	for {
		select {
		case cd := <-m.out:
			pi, err := m.writeOne(cd)
			if err != nil {
				if m.ctx.Err() != nil {
					log.Errorf("Error sending ContactData to client [%v]: %v", m.info.Client.ID, err)
					m.cancel()
					m.s.fs.StatsCollector().ClientPoll(pi)
				}
				// ctx was already canceled - more or less normal shutdown, so don't log
				// as a poll.
				return
			}
			m.s.fs.StatsCollector().ClientPoll(pi)
		case <-m.ctx.Done():
			return
		}
	}
}

func (m *streamManager) writeOne(cd *fspb.ContactData) (stats.PollInfo, error) {
	pi := stats.PollInfo{
		CTX:      m.ctx,
		ID:       m.info.Client.ID,
		Start:    db.Now(),
		Status:   http.StatusTeapot,
		CacheHit: true,
	}
	defer func() {
		if pi.Status == http.StatusTeapot {
			log.Errorf("Forgot to set status.")
		}
		pi.End = db.Now()
	}()

	buf := proto.NewBuffer(make([]byte, 0, 1024))
	if err := buf.EncodeMessage(cd); err != nil {
		return pi, err
	}

	sw := time.Now()
	s, err := m.res.Write(buf.Bytes())
	if err != nil {
		return pi, err
	}
	m.res.Flush()
	pi.WriteTime = time.Since(sw)
	pi.WriteBytes = s
	pi.Status = http.StatusOK

	return pi, nil
}
<commit_msg>Minor review fixes.<commit_after>package https

import (
	"bufio"
	"context"
	"crypto"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"sync"
	"time"

	log "github.com/golang/glog"
	"github.com/golang/protobuf/proto"
	"github.com/google/fleetspeak/fleetspeak/src/common"
	fspb "github.com/google/fleetspeak/fleetspeak/src/common/proto/fleetspeak"
	"github.com/google/fleetspeak/fleetspeak/src/server/comms"
	"github.com/google/fleetspeak/fleetspeak/src/server/db"
	"github.com/google/fleetspeak/fleetspeak/src/server/stats"
)

const magic = uint32(0xf1ee1001)

type fullResponseWriter interface {
	http.ResponseWriter
	http.CloseNotifier
	http.Flusher
}

func readUint32(body *bufio.Reader) (uint32, error) {
	b := make([]byte, 4)
	if _, err := io.ReadAtLeast(body, b, 4); err != nil {
		return 0, err
	}
	return binary.LittleEndian.Uint32(b), nil
}

func writeUint32(res fullResponseWriter, i uint32) error {
	return binary.Write(res, binary.LittleEndian, i)
}

// messageServer wraps a Communicator in order to handle clients polls.
type streamingMessageServer struct {
	*Communicator
}

func (s streamingMessageServer) ServeHTTP(res http.ResponseWriter, req *http.Request) {
	earlyError := func(msg string, status int) {
		log.ErrorDepth(1, fmt.Sprintf("%s: %s", http.StatusText(status), msg))
		s.fs.StatsCollector().ClientPoll(stats.PollInfo{
			CTX:    req.Context(),
			Start:  db.Now(),
			End:    db.Now(),
			Status: status,
		})
	}

	if !s.startProcessing() {
		earlyError("server not ready", http.StatusInternalServerError)
		return
	}
	defer s.stopProcessing()

	fullRes, ok := res.(fullResponseWriter)
	if !ok {
		earlyError("/streaming-message requested, but not supported. ResponseWriter is not a fullResponseWriter", http.StatusNotFound)
		return
	}

	if req.Method != http.MethodPost {
		earlyError(fmt.Sprintf("%v not supported", req.Method), http.StatusBadRequest)
		return
	}

	if req.TLS == nil {
		earlyError("TLS information not found", http.StatusBadRequest)
		return
	}

	if len(req.TLS.PeerCertificates) != 1 {
		earlyError(fmt.Sprintf("expected 1 client cert, received %v", len(req.TLS.PeerCertificates)), http.StatusBadRequest)
		return
	}

	cert := req.TLS.PeerCertificates[0]
	if cert.PublicKey == nil {
		earlyError("public key not present in client cert", http.StatusBadRequest)
		return
	}

	body := bufio.NewReader(req.Body)

	// Set a 10 min overall maximum lifespan of the connection.
	ctx, fin := context.WithTimeout(req.Context(), 10*time.Minute)
	defer fin()

	// Also create a way to terminate early in case of error.
	ctx, cancel := context.WithCancel(ctx)

	info, err := s.initialPoll(ctx, addrFromString(req.RemoteAddr), cert.PublicKey, fullRes, body)
	if err != nil || info == nil {
		return
	}

	m := streamManager{
		ctx:  ctx,
		s:    s,
		info: info,
		res:  fullRes,
		body: body,

		cancel: cancel,
	}
	defer func() {
		// Shutdown is a bit subtle.
		//
		// We get here iff m.ctx is canceled, timed out, etc.
		//
		// Once ctx is canceled, writeLoop will notice, close the outgoing
		// ResponseWriter, and begin blindly draining m.out.
		//
		// Closing ResponseWriter will cause any pending read to error out and
		// readLoop to return.
		//
		// Once the readLoop returns, we can safely close m.out and wait for
		// writeLoop to finish.
		m.reading.Wait()
		close(m.out)
		m.writing.Wait()
	}()

	m.reading.Add(1)
	go m.readLoop()

	m.writing.Add(1)
	go m.writeLoop()

	select {
	case <-ctx.Done():
		return
	case <-fullRes.CloseNotify():
		m.cancel()
		return
	case <-s.stopping:
		// Communicator is shutting down.
		m.cancel()
		return
	}
}

func (s streamingMessageServer) initialPoll(ctx context.Context, addr net.Addr, key crypto.PublicKey, res fullResponseWriter, body *bufio.Reader) (*comms.ConnectionInfo, error) {
	ctx, fin := context.WithTimeout(ctx, time.Minute)

	pi := stats.PollInfo{
		CTX:    ctx,
		Start:  db.Now(),
		Status: http.StatusTeapot, // Should never actually be returned
	}
	defer func() {
		fin()
		if pi.Status == http.StatusTeapot {
			log.Errorf("Forgot to set status, PollInfo: %v", pi)
		}
		pi.End = db.Now()
		s.fs.StatsCollector().ClientPoll(pi)
	}()

	makeError := func(msg string, status int) error {
		log.ErrorDepth(1, fmt.Sprintf("%s: %s", http.StatusText(status), msg))
		pi.Status = status
		return errors.New(msg)
	}

	id, err := common.MakeClientID(key)
	if err != nil {
		return nil, makeError(fmt.Sprintf("unable to create client id from public key: %v", err), http.StatusBadRequest)
	}
	pi.ID = id

	m, err := readUint32(body)
	if err != nil {
		return nil, makeError(fmt.Sprintf("error reading magic number: %v", err), http.StatusBadRequest)
	}
	if m != magic {
		return nil, makeError(fmt.Sprintf("unknown magic number: got %x, expected %x", m, magic), http.StatusBadRequest)
	}

	st := time.Now()
	size, err := binary.ReadUvarint(body)
	if err != nil {
		return nil, makeError(fmt.Sprintf("error reading size: %v", err), http.StatusBadRequest)
	}
	if size > MaxContactSize {
		return nil, makeError(fmt.Sprintf("initial contact size too large: got %d, expected at most %d", size, MaxContactSize), http.StatusBadRequest)
	}

	buf := make([]byte, size)
	_, err = io.ReadFull(body, buf)
	if err != nil {
		return nil, makeError(fmt.Sprintf("error reading body for initial exchange: %v", err), http.StatusBadRequest)
	}
	pi.ReadTime = time.Since(st)
	pi.ReadBytes = int(size)

	var wcd fspb.WrappedContactData
	if err = proto.Unmarshal(buf, &wcd); err != nil {
		return nil, makeError(fmt.Sprintf("error parsing body: %v", err), http.StatusBadRequest)
	}

	info, toSend, err := s.fs.InitializeConnection(ctx, addr, key, &wcd, true)
	if err == comms.ErrNotAuthorized {
		return nil, makeError("not authorized", http.StatusServiceUnavailable)
	}
	if err != nil {
		return nil, makeError(fmt.Sprintf("error processing contact: %v", err), http.StatusInternalServerError)
	}
	pi.CacheHit = info.Client.Cached

	out := proto.NewBuffer(make([]byte, 0, 1024))
	// EncodeMessage prepends the size as a Varint.
	if err := out.EncodeMessage(toSend); err != nil {
		return nil, makeError(fmt.Sprintf("error preparing messages: %v", err), http.StatusInternalServerError)
	}
	st = time.Now()
	sz, err := res.Write(out.Bytes())
	if err != nil {
		return nil, makeError(fmt.Sprintf("error writing body: %v", err), http.StatusInternalServerError)
	}
	res.Flush()

	pi.WriteTime = time.Since(st)
	pi.End = time.Now()
	pi.WriteBytes = sz
	pi.Status = http.StatusOK
	return info, nil
}

type streamManager struct {
	ctx context.Context
	s   streamingMessageServer

	info *comms.ConnectionInfo
	res  fullResponseWriter
	body *bufio.Reader

	// The read- and writeLoop will wait for these. Separate because readloop
	// needs to finish before writeLoop.
	reading sync.WaitGroup
	writing sync.WaitGroup

	out chan *fspb.ContactData

	cancel func() // Shuts down the stream when called.
}

func (m *streamManager) readLoop() {
	defer m.reading.Done()
	defer m.cancel()

	cnt := uint64(0)

	for {
		pi, err := m.readOne()
		if err != nil {
			// If the context has been canceled, it is probably a 'normal' termination
			// - disconnect, max connection durating, etc. But if it is still active,
			// we are going to tear down everything because of an unexpected read
			// error and should log/record why.
			if m.ctx.Err() == nil && pi != nil {
				m.s.fs.StatsCollector().ClientPoll(*pi)
				log.Errorf("Streaming Connection to %v terminated with error: %v", m.info.Client.ID, err)
			}
			return
		}
		m.s.fs.StatsCollector().ClientPoll(*pi)
		cnt += 1
		m.out <- &fspb.ContactData{AckIndex: cnt}
	}
}

func (m *streamManager) readOne() (*stats.PollInfo, error) {
	size, err := binary.ReadUvarint(m.body)
	if err != nil {
		return nil, err
	}
	if size > MaxContactSize {
		return nil, fmt.Errorf("streaming contact size too large: got %d, expected at most %d", size, MaxContactSize)
	}

	pi := stats.PollInfo{
		CTX:      m.ctx,
		ID:       m.info.Client.ID,
		Start:    db.Now(),
		Status:   http.StatusTeapot,
		CacheHit: true,
	}
	defer func() {
		if pi.Status == http.StatusTeapot {
			log.Errorf("Forgot to set status.")
		}
		pi.End = db.Now()
	}()
	buf := make([]byte, size)
	if _, err := io.ReadFull(m.body, buf); err != nil {
		return &pi, fmt.Errorf("error reading streamed data: %v", err)
	}
	pi.ReadTime = time.Since(pi.Start)
	pi.ReadBytes = int(size)

	var wcd fspb.WrappedContactData
	if err = proto.Unmarshal(buf, &wcd); err != nil {
		pi.Status = http.StatusBadRequest
		return &pi, fmt.Errorf("error parsing streamed data: %v", err)
	}
	if err := m.s.fs.HandleMessagesFromClient(m.ctx, m.info, &wcd); err != nil {
		if err == comms.ErrNotAuthorized {
			pi.Status = http.StatusServiceUnavailable
		} else {
			pi.Status = http.StatusInternalServerError
			err = fmt.Errorf("error processing streamed messages: %v", err)
		}
		return &pi, err
	}

	return &pi, nil
}

func (m *streamManager) writeLoop() {
	defer m.writing.Done()
	defer func() {
		for range m.out {
		}
	}()

	for {
		select {
		case cd := <-m.out:
			pi, err := m.writeOne(cd)
			if err != nil {
				if m.ctx.Err() != nil {
					log.Errorf("Error sending ContactData to client [%v]: %v", m.info.Client.ID, err)
					m.cancel()
					m.s.fs.StatsCollector().ClientPoll(pi)
				}
				// ctx was already canceled - more or less normal shutdown, so don't log
				// as a poll.
				return
			}
			m.s.fs.StatsCollector().ClientPoll(pi)
		case <-m.ctx.Done():
			return
		}
	}
}

func (m *streamManager) writeOne(cd *fspb.ContactData) (stats.PollInfo, error) {
	pi := stats.PollInfo{
		CTX:      m.ctx,
		ID:       m.info.Client.ID,
		Start:    db.Now(),
		Status:   http.StatusTeapot,
		CacheHit: true,
	}
	defer func() {
		if pi.Status == http.StatusTeapot {
			log.Errorf("Forgot to set status.")
		}
		pi.End = db.Now()
	}()

	buf := proto.NewBuffer(make([]byte, 0, 1024))
	if err := buf.EncodeMessage(cd); err != nil {
		return pi, err
	}

	sw := time.Now()
	s, err := m.res.Write(buf.Bytes())
	if err != nil {
		return pi, err
	}
	m.res.Flush()
	pi.WriteTime = time.Since(sw)
	pi.WriteBytes = s
	pi.Status = http.StatusOK

	return pi, nil
}
<|endoftext|>


<commit_before>package generator

import "text/template"

var NSCopyingText = `
- (instancetype)copyWithZone:(NSZone *)zone
{
    // OCGEN: Autogenerated method. Do not touch
    {{if not .IsDirectChildOfNSObject -}}
    {{.Name}} *copy = nil;
    if([super respondsToSelector:@selector(copyWithZone:)])
    {
        copy = [super copyWithZone:zone];
    }
    else
    {
        copy = [[[self class] allocWithZone:zone] init];
    }
    {{- else -}}
        {{.Name}} *copy = [[[self class] allocWithZone:zone] init];
    {{- end}}
    if (copy != nil)
    { {{range .Properties}}
        copy{{.Accessor}}{{.Name}} = {{if .IsObject -}}
                                          {{if eq .Class "NSArray" -}}
                                              [[NSArray alloc] initWithArray:self{{.Accessor}}{{.Name}} copyItems:YES];
                                          {{- else if .IsWeak -}}
                                              self{{.Accessor}}{{.Name}};
                                          {{- else -}}
                                              [self{{.Accessor}}{{.Name}} copyWithZone:zone];
                                          {{- end}}
                                      {{- else -}}
                                          self{{.Accessor}}{{.Name}};
                                      {{- end}}
    {{- end}}
    }
    return copy;
}`
var NSCopyingTpl = template.Must(template.New("NSCopying").Parse(NSCopyingText))

var NSCodingInitText = `
- (instancetype)initWithCoder:(NSCoder *)decoder
{
    // OCGEN: Autogenerated method. Do not touch
    {{if not .IsDirectChildOfNSObject -}}
    if([super respondsToSelector:@selector(initWithCoder:)])
    {
        self = [super initWithCoder:decoder];
    }
    else
    {
        self = [super init];
    }
    {{- else -}}
    self = [super init];
    {{- end}}
    if(self != nil)
    { {{range .Properties}}
        _{{.Name}} = {{if .IsObject -}}
                         [decoder decodeObjectForKey:@"{{.Name}}"];
                     {{- else -}}
                         [decoder decode{{.CoderType}}ForKey:@"{{.Name}}"];
                     {{- end}}
    {{- end}}
    }
    return self;
}`
var NSCodingInitTpl = template.Must(template.New("NSCodingInit").Parse(NSCodingInitText))

var NSCodingEncodeText = `
- (void)encodeWithCoder:(NSCoder *)coder
{
    // OCGEN: Autogenerated method. Do not touch
    {{- if not .IsDirectChildOfNSObject}}
    if([super respondsToSelector:@selector(encodeWithCoder:)])
    {
        [super encodeWithCoder:coder];
    }
    {{- end}}{{range .Properties}}
    {{if .IsObject -}}
        {{if .IsWeak -}}
            [coder encodeConditionalObject:_{{.Name}} forKey:@"{{.Name}}"];
        {{- else -}}
            [coder encodeObject:_{{.Name}} forKey:@"{{.Name}}"];
        {{- end}}
    {{- else -}}
        [coder encode{{.CoderType}}:_{{.Name}} forKey:@"{{.Name}}"];
    {{- end}}
{{- end}}
}`
var NSCodingEncodeTpl = template.Must(template.New("NSCodingEncode").Parse(NSCodingEncodeText))
<commit_msg>Updated OCGEN message per method<commit_after>package generator

import "text/template"

var NSCopyingText = `
- (instancetype)copyWithZone:(NSZone *)zone
{
    // OCGEN: Autogenerated method
    {{if not .IsDirectChildOfNSObject -}}
    {{.Name}} *copy = nil;
    if([super respondsToSelector:@selector(copyWithZone:)])
    {
        copy = [super copyWithZone:zone];
    }
    else
    {
        copy = [[[self class] allocWithZone:zone] init];
    }
    {{- else -}}
        {{.Name}} *copy = [[[self class] allocWithZone:zone] init];
    {{- end}}
    if (copy != nil)
    { {{range .Properties}}
        copy{{.Accessor}}{{.Name}} = {{if .IsObject -}}
                                          {{if eq .Class "NSArray" -}}
                                              [[NSArray alloc] initWithArray:self{{.Accessor}}{{.Name}} copyItems:YES];
                                          {{- else if .IsWeak -}}
                                              self{{.Accessor}}{{.Name}};
                                          {{- else -}}
                                              [self{{.Accessor}}{{.Name}} copyWithZone:zone];
                                          {{- end}}
                                      {{- else -}}
                                          self{{.Accessor}}{{.Name}};
                                      {{- end}}
    {{- end}}
    }
    return copy;
}`
var NSCopyingTpl = template.Must(template.New("NSCopying").Parse(NSCopyingText))

var NSCodingInitText = `
- (instancetype)initWithCoder:(NSCoder *)decoder
{
    // OCGEN: Autogenerated method
    {{if not .IsDirectChildOfNSObject -}}
    if([super respondsToSelector:@selector(initWithCoder:)])
    {
        self = [super initWithCoder:decoder];
    }
    else
    {
        self = [super init];
    }
    {{- else -}}
    self = [super init];
    {{- end}}
    if(self != nil)
    { {{range .Properties}}
        _{{.Name}} = {{if .IsObject -}}
                         [decoder decodeObjectForKey:@"{{.Name}}"];
                     {{- else -}}
                         [decoder decode{{.CoderType}}ForKey:@"{{.Name}}"];
                     {{- end}}
    {{- end}}
    }
    return self;
}`
var NSCodingInitTpl = template.Must(template.New("NSCodingInit").Parse(NSCodingInitText))

var NSCodingEncodeText = `
- (void)encodeWithCoder:(NSCoder *)coder
{
    // OCGEN: Autogenerated method
    {{- if not .IsDirectChildOfNSObject}}
    if([super respondsToSelector:@selector(encodeWithCoder:)])
    {
        [super encodeWithCoder:coder];
    }
    {{- end}}{{range .Properties}}
    {{if .IsObject -}}
        {{if .IsWeak -}}
            [coder encodeConditionalObject:_{{.Name}} forKey:@"{{.Name}}"];
        {{- else -}}
            [coder encodeObject:_{{.Name}} forKey:@"{{.Name}}"];
        {{- end}}
    {{- else -}}
        [coder encode{{.CoderType}}:_{{.Name}} forKey:@"{{.Name}}"];
    {{- end}}
{{- end}}
}`
var NSCodingEncodeTpl = template.Must(template.New("NSCodingEncode").Parse(NSCodingEncodeText))
<|endoftext|>


<commit_before>// +build js

package poll

import "time"

// pollDesc is a no-op implementation of an I/O poller for GOARCH=js.
//
// Its implementation is based on NaCL in gc compiler (see GOROOT/src/internal/poll/fd_poll_nacl.go),
// but it does even less.
type pollDesc struct {
	closing bool
}

func (pd *pollDesc) init(fd *FD) error { return nil }

func (pd *pollDesc) close() {}

func (pd *pollDesc) evict() { pd.closing = true }

func (pd *pollDesc) prepare(mode int, isFile bool) error {
	if pd.closing {
		return errClosing(isFile)
	}
	return nil
}

func (pd *pollDesc) prepareRead(isFile bool) error { return pd.prepare('r', isFile) }

func (pd *pollDesc) prepareWrite(isFile bool) error { return pd.prepare('w', isFile) }

func (pd *pollDesc) wait(mode int, isFile bool) error {
	if pd.closing {
		return errClosing(isFile)
	}
	return ErrTimeout
}

func (pd *pollDesc) waitRead(isFile bool) error { return pd.wait('r', isFile) }

func (pd *pollDesc) waitWrite(isFile bool) error { return pd.wait('w', isFile) }

func (*pollDesc) waitCanceled(mode int) {}

func (*pollDesc) pollable() bool { return true }

func (*FD) SetDeadline(t time.Time) error { return nil }

func (*FD) SetReadDeadline(t time.Time) error { return nil }

func (*FD) SetWriteDeadline(t time.Time) error { return nil }

// PollDescriptor returns the descriptor being used by the poller,
// or ^uintptr(0) if there isn't one. This is only used for testing.
func PollDescriptor() uintptr {
	return ^uintptr(0)
}
<commit_msg>compiler/natives/src/internal/poll: Implement semaphores.<commit_after>// +build js

package poll

import "time"

// pollDesc is a no-op implementation of an I/O poller for GOARCH=js.
//
// Its implementation is based on NaCL in gc compiler (see GOROOT/src/internal/poll/fd_poll_nacl.go),
// but it does even less.
type pollDesc struct {
	closing bool
}

func (pd *pollDesc) init(fd *FD) error { return nil }

func (pd *pollDesc) close() {}

func (pd *pollDesc) evict() { pd.closing = true }

func (pd *pollDesc) prepare(mode int, isFile bool) error {
	if pd.closing {
		return errClosing(isFile)
	}
	return nil
}

func (pd *pollDesc) prepareRead(isFile bool) error { return pd.prepare('r', isFile) }

func (pd *pollDesc) prepareWrite(isFile bool) error { return pd.prepare('w', isFile) }

func (pd *pollDesc) wait(mode int, isFile bool) error {
	if pd.closing {
		return errClosing(isFile)
	}
	return ErrTimeout
}

func (pd *pollDesc) waitRead(isFile bool) error { return pd.wait('r', isFile) }

func (pd *pollDesc) waitWrite(isFile bool) error { return pd.wait('w', isFile) }

func (*pollDesc) waitCanceled(mode int) {}

func (*pollDesc) pollable() bool { return true }

func (*FD) SetDeadline(t time.Time) error { return nil }

func (*FD) SetReadDeadline(t time.Time) error { return nil }

func (*FD) SetWriteDeadline(t time.Time) error { return nil }

// PollDescriptor returns the descriptor being used by the poller,
// or ^uintptr(0) if there isn't one. This is only used for testing.
func PollDescriptor() uintptr {
	return ^uintptr(0)
}

// Copy of sync.runtime_Semacquire.
func runtime_Semacquire(s *uint32) {
	if *s == 0 {
		ch := make(chan bool)
		semWaiters[s] = append(semWaiters[s], ch)
		<-ch
	}
	*s--
}

// Copy of sync.runtime_Semrelease.
func runtime_Semrelease(s *uint32) {
	*s++

	w := semWaiters[s]
	if len(w) == 0 {
		return
	}

	ch := w[0]
	w = w[1:]
	semWaiters[s] = w
	if len(w) == 0 {
		delete(semWaiters, s)
	}

	ch <- true
}

var semWaiters = make(map[*uint32][]chan bool)
<|endoftext|>


<commit_before>package utils

import (
	"fmt"

	"github.com/racker/perigee"
)

// Version is a supported API version, corresponding to a vN package within the appropriate service.
type Version struct {
	ID       string
	Priority int
}

// ChooseVersion queries the base endpoint of a API to choose the most recent non-experimental alternative from a service's
// published versions.
// It returns the highest-Priority Version among the alternatives that are provided, as well as its corresponding endpoint.
func ChooseVersion(baseEndpoint string, recognized []*Version) (*Version, string, error) {
	type linkResp struct {
		Href string `json:"href"`
		Rel  string `json:"rel"`
	}

	type valueResp struct {
		ID     string     `json:"id"`
		Status string     `json:"status"`
		Links  []linkResp `json:"links"`
	}

	type versionsResp struct {
		Values []valueResp `json:"values"`
	}

	type response struct {
		Versions versionsResp `json:"versions"`
	}

	var resp response
	_, err := perigee.Request("GET", baseEndpoint, perigee.Options{
		Results: &resp,
		OkCodes: []int{200},
	})

	if err != nil {
		return nil, "", err
	}

	byID := make(map[string]*Version)
	for _, version := range recognized {
		byID[version.ID] = version
	}

	var highest *Version
	var endpoint string

	for _, value := range resp.Versions.Values {
		if matching, ok := byID[value.ID]; ok {
			if highest == nil || matching.Priority > highest.Priority {
				highest = matching

				found := false
				for _, link := range value.Links {
					if link.Rel == "self" {
						found = true
						endpoint = link.Href
					}
				}

				if !found {
					return nil, "", fmt.Errorf("Endpoint missing in version %s response from %s", value.ID, baseEndpoint)
				}
			}
		}
	}

	if highest == nil || endpoint == "" {
		return nil, "", fmt.Errorf("No supported version available from endpoint %s", baseEndpoint)
	}

	return highest, endpoint, nil
}
<commit_msg>Don't use experimental versions.<commit_after>package utils

import (
	"fmt"
	"strings"

	"github.com/racker/perigee"
)

// Version is a supported API version, corresponding to a vN package within the appropriate service.
type Version struct {
	ID       string
	Priority int
}

var goodStatus = map[string]bool{
	"current":   true,
	"supported": true,
	"stable":    true,
}

// ChooseVersion queries the base endpoint of a API to choose the most recent non-experimental alternative from a service's
// published versions.
// It returns the highest-Priority Version among the alternatives that are provided, as well as its corresponding endpoint.
func ChooseVersion(baseEndpoint string, recognized []*Version) (*Version, string, error) {
	type linkResp struct {
		Href string `json:"href"`
		Rel  string `json:"rel"`
	}

	type valueResp struct {
		ID     string     `json:"id"`
		Status string     `json:"status"`
		Links  []linkResp `json:"links"`
	}

	type versionsResp struct {
		Values []valueResp `json:"values"`
	}

	type response struct {
		Versions versionsResp `json:"versions"`
	}

	var resp response
	_, err := perigee.Request("GET", baseEndpoint, perigee.Options{
		Results: &resp,
		OkCodes: []int{200},
	})

	if err != nil {
		return nil, "", err
	}

	byID := make(map[string]*Version)
	for _, version := range recognized {
		byID[version.ID] = version
	}

	var highest *Version
	var endpoint string

	for _, value := range resp.Versions.Values {
		if matching, ok := byID[value.ID]; ok && goodStatus[strings.ToLower(value.Status)] {
			if highest == nil || matching.Priority > highest.Priority {
				highest = matching

				found := false
				for _, link := range value.Links {
					if link.Rel == "self" {
						found = true
						endpoint = link.Href
					}
				}

				if !found {
					return nil, "", fmt.Errorf("Endpoint missing in version %s response from %s", value.ID, baseEndpoint)
				}
			}
		}
	}

	if highest == nil || endpoint == "" {
		return nil, "", fmt.Errorf("No supported version available from endpoint %s", baseEndpoint)
	}

	return highest, endpoint, nil
}
<|endoftext|>


<commit_before>package dev_test

import (
	"net"
	"testing"
	"time"

	rpc "github.com/relab/gorums/dev"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
)

//todo(meling) implement functionality to keep certs and keys in appropriate places.
//todo(meling) figure out how to generate certs and keys (currently these are only google's certs and keys)

const (
	tlsDir         = "../testdata/tls/"
	serverCertFile = tlsDir + "server1.pem"
	serverKeyFile  = tlsDir + "server1.key"
)

//todo(meling) clean up this stuff; merge with other test setup function or something smart; perhaps splitting it up into separate functions will help with reuse.

func secsetup(t testing.TB, srvs regServers, remote bool) (func(n int), func(n int)) {
	if len(srvs) == 0 {
		t.Fatal("setupServers: need at least one server")
	}

	if remote {
		return func(int) {}, func(int) {}
	}

	var err error
	servers := make([]*grpc.Server, len(srvs))
	listeners := make([]net.Listener, len(srvs))
	for i, rs := range srvs {
		listeners[i], err = net.Listen("tcp", rs.addr)
		if err != nil {
			t.Fatalf("failed to listen: %v", err)
		}

		//todo(meling) should store credentials in Manager or Node
		creds, err := credentials.NewServerTLSFromFile(serverCertFile, serverKeyFile)
		if err != nil {
			t.Fatalf("failed to generate credentials %v", err)
		}
		opts := []grpc.ServerOption{grpc.Creds(creds)}
		servers[i] = grpc.NewServer(opts...)
		rpc.RegisterRegisterServer(servers[i], srvs[i].implementation)

		go func(i int, server *grpc.Server) {
			_ = server.Serve(listeners[i])
		}(i, servers[i])
		srvs[i].addr = "localhost" + rs.addr
	}

	stopGrpcServeFunc := func(n int) {
		if n < 0 || n > len(servers) {
			for _, s := range servers {
				s.Stop()
			}
		} else {
			servers[n].Stop()
		}
	}

	closeListenersFunc := func(n int) {
		if n < 0 || n > len(listeners) {
			for _, l := range listeners {
				l.Close()
			}
		} else {
			listeners[n].Close()
		}
	}

	return stopGrpcServeFunc, closeListenersFunc
}

func TestSecureRegister(t *testing.T) {
	defer leakCheck(t)()
	servers := regServers{
		{":8080", rpc.NewRegisterBasic()},
		{":8081", rpc.NewRegisterBasic()},
		{":8082", rpc.NewRegisterBasic()},
	}
	stopGrpcServe, closeListeners := secsetup(t, servers, false)
	defer stopGrpcServe(allServers)

	//TODO fix hardcoded youtube server name (can we get certificate for localhost servername?)
	clientCreds, err := credentials.NewClientTLSFromFile(tlsDir+"ca.pem", "x.test.youtube.com")
	if err != nil {
		t.Errorf("error creating credentials: %v", err)
	}

	grpcOpts := []grpc.DialOption{
		grpc.WithBlock(),
		grpc.WithTimeout(50 * time.Millisecond),
		grpc.WithTransportCredentials(clientCreds),
	}
	dialOpts := rpc.WithGrpcDialOptions(grpcOpts...)

	mgr, err := rpc.NewManager(servers.addrs(), dialOpts)
	if err != nil {
		t.Fatalf("%v", err)
	}
	defer mgr.Close()
	closeListeners(allServers)

	// Get all all available node ids
	ids := mgr.NodeIDs()

	// Quorum spec: rq=2. wq=3, n=3, sort by timestamp.
	qspec := NewRegisterByTimestampQSpec(2, len(ids))

	config, err := mgr.NewConfiguration(ids, qspec, time.Second)
	if err != nil {
		t.Fatalf("error creating config: %v", err)
	}

	// Test state
	state := &rpc.State{
		Value:     "42",
		Timestamp: time.Now().UnixNano(),
	}

	// Perfomr write call
	wreply, err := config.Write(state)
	if err != nil {
		t.Fatalf("write rpc call error: %v", err)
	}
	t.Logf("wreply: %v\n", wreply)
	if !wreply.Reply.New {
		t.Error("write reply was not marked as new")
	}

	// Do read call
	rreply, err := config.Read(&rpc.ReadRequest{})
	if err != nil {
		t.Fatalf("read rpc call error: %v", err)
	}
	t.Logf("rreply: %v\n", rreply)
	if rreply.Reply.Value != state.Value {
		t.Errorf("read reply: want state %v, got %v", state, rreply.Reply)
	}

	nodes := mgr.Nodes(false)
	for _, m := range nodes {
		t.Logf("%v", m)
	}
}
<commit_msg>dev: TODOs<commit_after>package dev_test

import (
	"net"
	"testing"
	"time"

	rpc "github.com/relab/gorums/dev"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
)

// TODO(meling):
//
// Implement functionality to keep certs and keys in appropriate places.
//
// Figure out how to generate certs and keys (currently these are only
// google's certs and keys).
//
// Clean up this stuff; merge with other test setup function or something
// smart; perhaps splitting it up into separate functions will help with reuse.

const (
	tlsDir         = "../testdata/tls/"
	serverCertFile = tlsDir + "server1.pem"
	serverKeyFile  = tlsDir + "server1.key"
)

func secsetup(t testing.TB, srvs regServers, remote bool) (func(n int), func(n int)) {
	if len(srvs) == 0 {
		t.Fatal("setupServers: need at least one server")
	}

	if remote {
		return func(int) {}, func(int) {}
	}

	var err error
	servers := make([]*grpc.Server, len(srvs))
	listeners := make([]net.Listener, len(srvs))
	for i, rs := range srvs {
		listeners[i], err = net.Listen("tcp", rs.addr)
		if err != nil {
			t.Fatalf("failed to listen: %v", err)
		}

		//todo(meling) should store credentials in Manager or Node
		creds, err := credentials.NewServerTLSFromFile(serverCertFile, serverKeyFile)
		if err != nil {
			t.Fatalf("failed to generate credentials %v", err)
		}
		opts := []grpc.ServerOption{grpc.Creds(creds)}
		servers[i] = grpc.NewServer(opts...)
		rpc.RegisterRegisterServer(servers[i], srvs[i].implementation)

		go func(i int, server *grpc.Server) {
			_ = server.Serve(listeners[i])
		}(i, servers[i])
		srvs[i].addr = "localhost" + rs.addr
	}

	stopGrpcServeFunc := func(n int) {
		if n < 0 || n > len(servers) {
			for _, s := range servers {
				s.Stop()
			}
		} else {
			servers[n].Stop()
		}
	}

	closeListenersFunc := func(n int) {
		if n < 0 || n > len(listeners) {
			for _, l := range listeners {
				l.Close()
			}
		} else {
			listeners[n].Close()
		}
	}

	return stopGrpcServeFunc, closeListenersFunc
}

func TestSecureRegister(t *testing.T) {
	defer leakCheck(t)()
	servers := regServers{
		{":8080", rpc.NewRegisterBasic()},
		{":8081", rpc.NewRegisterBasic()},
		{":8082", rpc.NewRegisterBasic()},
	}
	stopGrpcServe, closeListeners := secsetup(t, servers, false)
	defer stopGrpcServe(allServers)

	//TODO fix hardcoded youtube server name (can we get certificate for localhost servername?)
	clientCreds, err := credentials.NewClientTLSFromFile(tlsDir+"ca.pem", "x.test.youtube.com")
	if err != nil {
		t.Errorf("error creating credentials: %v", err)
	}

	grpcOpts := []grpc.DialOption{
		grpc.WithBlock(),
		grpc.WithTimeout(50 * time.Millisecond),
		grpc.WithTransportCredentials(clientCreds),
	}
	dialOpts := rpc.WithGrpcDialOptions(grpcOpts...)

	mgr, err := rpc.NewManager(servers.addrs(), dialOpts)
	if err != nil {
		t.Fatalf("%v", err)
	}
	defer mgr.Close()
	closeListeners(allServers)

	// Get all all available node ids
	ids := mgr.NodeIDs()

	// Quorum spec: rq=2. wq=3, n=3, sort by timestamp.
	qspec := NewRegisterByTimestampQSpec(2, len(ids))

	config, err := mgr.NewConfiguration(ids, qspec, time.Second)
	if err != nil {
		t.Fatalf("error creating config: %v", err)
	}

	// Test state
	state := &rpc.State{
		Value:     "42",
		Timestamp: time.Now().UnixNano(),
	}

	// Perfomr write call
	wreply, err := config.Write(state)
	if err != nil {
		t.Fatalf("write rpc call error: %v", err)
	}
	t.Logf("wreply: %v\n", wreply)
	if !wreply.Reply.New {
		t.Error("write reply was not marked as new")
	}

	// Do read call
	rreply, err := config.Read(&rpc.ReadRequest{})
	if err != nil {
		t.Fatalf("read rpc call error: %v", err)
	}
	t.Logf("rreply: %v\n", rreply)
	if rreply.Reply.Value != state.Value {
		t.Errorf("read reply: want state %v, got %v", state, rreply.Reply)
	}

	nodes := mgr.Nodes(false)
	for _, m := range nodes {
		t.Logf("%v", m)
	}
}
<|endoftext|>