<commit_before>// Copyright 2021 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bq

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"sort"
	"strconv"

	"golang.org/x/sync/errgroup"
	"google.golang.org/protobuf/types/known/timestamppb"

	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/retry/transient"
	"go.chromium.org/luci/gae/service/datastore"

	cvbqpb "go.chromium.org/luci/cv/api/bigquery/v1"
	"go.chromium.org/luci/cv/internal/common"
	cvbq "go.chromium.org/luci/cv/internal/common/bq"
	"go.chromium.org/luci/cv/internal/migration"
	"go.chromium.org/luci/cv/internal/migration/migrationcfg"
	"go.chromium.org/luci/cv/internal/run"
)

const (
	// CV's own dataset/table.
	CVDataset = "raw"
	CVTable   = "attempts_cv"

	// Legacy CQ dataset.
	legacyProject    = "commit-queue"
	legacyProjectDev = "commit-queue-dev"
	legacyDataset    = "raw"
	legacyTable      = "attempts"
)

func send(ctx context.Context, client cvbq.Client, id common.RunID) error {
	r := run.Run{ID: id}
	switch err := datastore.Get(ctx, &r); {
	case err == datastore.ErrNoSuchEntity:
		return errors.Reason("Run not found").Err()
	case err != nil:
		return errors.Annotate(err, "failed to fetch Run").Tag(transient.Tag).Err()
	case !run.IsEnded(r.Status):
		panic("Run status must be final before sending to BQ.")
	}

	a, err := makeAttempt(ctx, &r)
	if err != nil {
		return errors.Annotate(err, "failed to make Attempt").Err()
	}

	// During the migration period when CQDaemon does most checks and triggers
	// builds, CV can't populate all of the fields of Attempt without the
	// information from CQDaemon; so for finished Attempts reported by
	// CQDaemon, we can fill in the remaining fields.
	switch cqda, err := fetchCQDAttempt(ctx, &r); {
	case err != nil:
		return err
	case cqda != nil:
		a = reconcileAttempts(a, cqda)
	}

	eg, ctx := errgroup.WithContext(ctx)
	defer eg.Wait()

	if !r.FinalizedByCQD {
		// Only export to legacy CQ dataset iff CQDaemon didn't finalize the Run
		// itself, which would have included exporting BQ row.

		// TODO(crbug/1218658): find a proper fix.
		switch yes, err := migrationcfg.IsCVInCharge(ctx, r.ID.LUCIProject()); {
		case err != nil:
			return err
		case !yes:
			logging.Errorf(ctx, "CV is not in charge, but it finalized a Run. Exporting to CV's BQ table only")
		default:
			logging.Debugf(ctx, "CV exporting Run to CQ BQ table")
			eg.Go(func() error {
				project := legacyProject
				if common.IsDev(ctx) {
					project = legacyProjectDev
				}
				return client.SendRow(ctx, cvbq.Row{
					CloudProject: project,
					Dataset:      legacyDataset,
					Table:        legacyTable,
					OperationID:  "run-" + string(id),
					Payload:      a,
				})
			})
		}
	}

	// *Always* export to local CV dataset.
	eg.Go(func() error {
		return client.SendRow(ctx, cvbq.Row{
			Dataset:     CVDataset,
			Table:       CVTable,
			OperationID: "run-" + string(id),
			Payload:     a,
		})
	})

	return eg.Wait()

}

func makeAttempt(ctx context.Context, r *run.Run) (*cvbqpb.Attempt, error) {
	// Load CLs and convert them to GerritChanges including submit status.
	runCLs, err := run.LoadRunCLs(ctx, r.ID, r.CLs)
	if err != nil {
		return nil, err
	}
	submittedSet := make(map[int64]struct{}, len(r.Submission.GetSubmittedCls()))
	for _, clid := range r.Submission.GetSubmittedCls() {
		submittedSet[clid] = struct{}{}
	}
	gerritChanges := make([]*cvbqpb.GerritChange, len(runCLs))
	for i, cl := range runCLs {
		gerritChanges[i] = toGerritChange(cl, submittedSet, r.Mode)
	}

	// TODO(crbug/1173168, crbug/1105669): We want to change the BQ
	// schema so that StartTime is processing start time and CreateTime is
	// trigger time.
	a := &cvbqpb.Attempt{
		Key:                  r.ID.AttemptKey(),
		LuciProject:          r.ID.LUCIProject(),
		ConfigGroup:          r.ConfigGroupID.Name(),
		ClGroupKey:           computeCLGroupKey(runCLs, false),
		EquivalentClGroupKey: computeCLGroupKey(runCLs, true),
		// Run.CreateTime is trigger time, which corresponds to what CQD sends for
		// StartTime.
		StartTime:     timestamppb.New(r.CreateTime),
		EndTime:       timestamppb.New(r.EndTime),
		GerritChanges: gerritChanges,
		// Builds, Substatus and HasCustomRequirement are not known to CV yet
		// during the migration state, so they should be filled in with Attempt
		// from CQD if possible.
		Builds: nil,
		Status: attemptStatus(ctx, r),
		// TODO(crbug/1114686): Add a new FAILED_SUBMIT substatus, which
		// should be used in the case that some CLs failed to submit after
		// passing checks. (In this case, for backwards compatibility, we
		// will set status = SUCCESS, substatus = FAILED_SUBMIT.)
		Substatus: cvbqpb.AttemptSubstatus_NO_SUBSTATUS,
	}
	return a, nil
}

// toGerritChange creates a GerritChange for the given RunCL.
//
// This includes the submit status of the CL.
func toGerritChange(cl *run.RunCL, submitted map[int64]struct{}, mode run.Mode) *cvbqpb.GerritChange {
	detail := cl.Detail
	ci := detail.GetGerrit().GetInfo()
	gc := &cvbqpb.GerritChange{
		Host:                       detail.GetGerrit().Host,
		Project:                    ci.Project,
		Change:                     ci.Number,
		Patchset:                   int64(detail.Patchset),
		EarliestEquivalentPatchset: int64(detail.MinEquivalentPatchset),
		TriggerTime:                cl.Trigger.Time,
		Mode:                       mode.BQAttemptMode(),
		SubmitStatus:               cvbqpb.GerritChange_PENDING,
	}

	if mode == run.FullRun {
		// Mark the CL submit status as success if it appears in the submitted CLs
		// list, and failure if it does not.
		if _, ok := submitted[int64(cl.ID)]; ok {
			gc.SubmitStatus = cvbqpb.GerritChange_SUCCESS
		} else {
			gc.SubmitStatus = cvbqpb.GerritChange_FAILURE
		}
	}

	return gc
}

// fetchCQDAttempt fetches an Attempt from CQDaemon if available.
//
// Returns nil if no Attempt is available.
func fetchCQDAttempt(ctx context.Context, r *run.Run) (*cvbqpb.Attempt, error) {
	if r.FinalizedByCQD {
		f, err := migration.LoadFinishedCQDRun(ctx, r.ID)
		if err != nil {
			return nil, err
		}
		return f.Payload.GetAttempt(), nil
	}
	v := migration.VerifiedCQDRun{ID: r.ID}
	switch err := datastore.Get(ctx, &v); {
	case err == datastore.ErrNoSuchEntity:
		// A Run may end without a VerifiedCQDRun stored if the Run is canceled.
		logging.Debugf(ctx, "no VerifiedCQDRun found for Run %q", r.ID)
	case err != nil:
		return nil, errors.Annotate(err, "failed to fetch VerifiedCQDRun").Tag(transient.Tag).Err()
	}
	return v.Payload.GetRun().GetAttempt(), nil
}

// reconcileAttempts merges the CV Attempt and CQDaemon Attempt.
//
// Modifies and returns the CV Attempt.
//
// Once CV does the relevant work (keeping track of builds, reading the CL
// description footers, and performing checks) these will no longer have to be
// filled in with the CQDaemon Attempt values.
func reconcileAttempts(a, cqda *cvbqpb.Attempt) *cvbqpb.Attempt {
	// The list of Builds will be known to CV after it starts triggering
	// and tracking builds; until then CQD is the source of truth.
	a.Builds = cqda.Builds
	// Substatus generally indicates a failure reason, which is
	// known once one of the checks fails. CQDaemon may specify
	// a substatus in the case of abort (substatus: MANUAL_CANCEL)
	// or failure (FAILED_TRYJOBS etc.).
	if a.Status == cvbqpb.AttemptStatus_ABORTED || a.Status == cvbqpb.AttemptStatus_FAILURE {
		a.Status = cqda.Status
		a.Substatus = cqda.Substatus
	}
	a.Status = cqda.Status
	a.Substatus = cqda.Substatus
	// The HasCustomRequirement is determined by CL description footers.
	a.HasCustomRequirement = cqda.HasCustomRequirement
	return a
}

// attemptStatus converts a Run status to Attempt status.
func attemptStatus(ctx context.Context, r *run.Run) cvbqpb.AttemptStatus {
	switch r.Status {
	case run.Status_SUCCEEDED:
		return cvbqpb.AttemptStatus_SUCCESS
	case run.Status_FAILED:
		// In the case that the checks passed but not all CLs were submitted
		// successfully, the Attempt will still have status set to SUCCESS for
		// backwards compatibility. Note that r.Submission is expected to be
		// set only if a submission is attempted, meaning all checks passed.
		if r.Submission != nil && len(r.Submission.Cls) != len(r.Submission.SubmittedCls) {
			return cvbqpb.AttemptStatus_SUCCESS
		}
		return cvbqpb.AttemptStatus_FAILURE
	case run.Status_CANCELLED:
		return cvbqpb.AttemptStatus_ABORTED
	default:
		logging.Errorf(ctx, "Unexpected attempt status %q", r.Status)
		return cvbqpb.AttemptStatus_ATTEMPT_STATUS_UNSPECIFIED
	}
}

// computeCLGroupKey constructs keys for ClGroupKey and the related
// EquivalentClGroupKey.
//
// These are meant to be opaque keys unique to particular set of CLs and
// patchsets for the purpose of grouping together runs for the same sets of
// patchsets. if isEquivalent is true, then the "min equivalent patchset" is
// used instead of the latest patchset, so that trivial patchsets such as minor
// rebases and CL description updates don't change the key.
func computeCLGroupKey(cls []*run.RunCL, isEquivalent bool) string {
	sort.Slice(cls, func(i, j int) bool {
		// ExternalID includes host and change number but not patchset; but
		// different patchsets of the same CL will never be included in the
		// same list, so sorting on only ExternalID is sufficient.
		return cls[i].ExternalID < cls[j].ExternalID
	})
	h := sha256.New()
	// CL group keys are meant to be opaque keys. We'd like to avoid people
	// depending on CL group key and equivalent CL group key sometimes being
	// equal. We can do this by adding a salt to the hash.
	if isEquivalent {
		h.Write([]byte("equivalent_cl_group_key"))
	}
	separator := []byte{0}
	for i, cl := range cls {
		if i > 0 {
			h.Write(separator)
		}
		h.Write([]byte(cl.Detail.GetGerrit().GetHost()))
		h.Write(separator)
		h.Write([]byte(strconv.FormatInt(cl.Detail.GetGerrit().GetInfo().GetNumber(), 10)))
		h.Write(separator)
		if isEquivalent {
			h.Write([]byte(strconv.FormatInt(int64(cl.Detail.GetMinEquivalentPatchset()), 10)))
		} else {
			h.Write([]byte(strconv.FormatInt(int64(cl.Detail.GetPatchset()), 10)))
		}
	}
	return hex.EncodeToString(h.Sum(nil)[:8])
}
<commit_msg>[cv] reduce log severity of yet another WAI Gerrit/CV/CQD race.<commit_after>// Copyright 2021 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bq

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"sort"
	"strconv"

	"golang.org/x/sync/errgroup"
	"google.golang.org/protobuf/types/known/timestamppb"

	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/retry/transient"
	"go.chromium.org/luci/gae/service/datastore"

	cvbqpb "go.chromium.org/luci/cv/api/bigquery/v1"
	"go.chromium.org/luci/cv/internal/common"
	cvbq "go.chromium.org/luci/cv/internal/common/bq"
	"go.chromium.org/luci/cv/internal/migration"
	"go.chromium.org/luci/cv/internal/migration/migrationcfg"
	"go.chromium.org/luci/cv/internal/run"
)

const (
	// CV's own dataset/table.
	CVDataset = "raw"
	CVTable   = "attempts_cv"

	// Legacy CQ dataset.
	legacyProject    = "commit-queue"
	legacyProjectDev = "commit-queue-dev"
	legacyDataset    = "raw"
	legacyTable      = "attempts"
)

func send(ctx context.Context, client cvbq.Client, id common.RunID) error {
	r := run.Run{ID: id}
	switch err := datastore.Get(ctx, &r); {
	case err == datastore.ErrNoSuchEntity:
		return errors.Reason("Run not found").Err()
	case err != nil:
		return errors.Annotate(err, "failed to fetch Run").Tag(transient.Tag).Err()
	case !run.IsEnded(r.Status):
		panic("Run status must be final before sending to BQ.")
	}

	a, err := makeAttempt(ctx, &r)
	if err != nil {
		return errors.Annotate(err, "failed to make Attempt").Err()
	}

	// During the migration period when CQDaemon does most checks and triggers
	// builds, CV can't populate all of the fields of Attempt without the
	// information from CQDaemon; so for finished Attempts reported by
	// CQDaemon, we can fill in the remaining fields.
	switch cqda, err := fetchCQDAttempt(ctx, &r); {
	case err != nil:
		return err
	case cqda != nil:
		a = reconcileAttempts(a, cqda)
	}

	eg, ctx := errgroup.WithContext(ctx)
	defer eg.Wait()

	if !r.FinalizedByCQD {
		// Only export to legacy CQ dataset iff CQDaemon didn't finalize the Run
		// itself, which would have included exporting BQ row.

		// TODO(crbug/1218658): find a proper fix.
		switch yes, err := migrationcfg.IsCVInCharge(ctx, r.ID.LUCIProject()); {
		case err != nil:
			return err
		case !yes:
			// Per crbug/1220934 investigation, this actually happens due to
			// inevitable races between user updating Gerrit in quick succession and
			// when CV & CQD observe Gerrit.
			logging.Warningf(ctx, "CV is not in charge, but it finalized a Run. Exporting to CV's BQ table only")
		default:
			logging.Debugf(ctx, "CV exporting Run to CQ BQ table")
			eg.Go(func() error {
				project := legacyProject
				if common.IsDev(ctx) {
					project = legacyProjectDev
				}
				return client.SendRow(ctx, cvbq.Row{
					CloudProject: project,
					Dataset:      legacyDataset,
					Table:        legacyTable,
					OperationID:  "run-" + string(id),
					Payload:      a,
				})
			})
		}
	}

	// *Always* export to local CV dataset.
	eg.Go(func() error {
		return client.SendRow(ctx, cvbq.Row{
			Dataset:     CVDataset,
			Table:       CVTable,
			OperationID: "run-" + string(id),
			Payload:     a,
		})
	})

	return eg.Wait()

}

func makeAttempt(ctx context.Context, r *run.Run) (*cvbqpb.Attempt, error) {
	// Load CLs and convert them to GerritChanges including submit status.
	runCLs, err := run.LoadRunCLs(ctx, r.ID, r.CLs)
	if err != nil {
		return nil, err
	}
	submittedSet := make(map[int64]struct{}, len(r.Submission.GetSubmittedCls()))
	for _, clid := range r.Submission.GetSubmittedCls() {
		submittedSet[clid] = struct{}{}
	}
	gerritChanges := make([]*cvbqpb.GerritChange, len(runCLs))
	for i, cl := range runCLs {
		gerritChanges[i] = toGerritChange(cl, submittedSet, r.Mode)
	}

	// TODO(crbug/1173168, crbug/1105669): We want to change the BQ
	// schema so that StartTime is processing start time and CreateTime is
	// trigger time.
	a := &cvbqpb.Attempt{
		Key:                  r.ID.AttemptKey(),
		LuciProject:          r.ID.LUCIProject(),
		ConfigGroup:          r.ConfigGroupID.Name(),
		ClGroupKey:           computeCLGroupKey(runCLs, false),
		EquivalentClGroupKey: computeCLGroupKey(runCLs, true),
		// Run.CreateTime is trigger time, which corresponds to what CQD sends for
		// StartTime.
		StartTime:     timestamppb.New(r.CreateTime),
		EndTime:       timestamppb.New(r.EndTime),
		GerritChanges: gerritChanges,
		// Builds, Substatus and HasCustomRequirement are not known to CV yet
		// during the migration state, so they should be filled in with Attempt
		// from CQD if possible.
		Builds: nil,
		Status: attemptStatus(ctx, r),
		// TODO(crbug/1114686): Add a new FAILED_SUBMIT substatus, which
		// should be used in the case that some CLs failed to submit after
		// passing checks. (In this case, for backwards compatibility, we
		// will set status = SUCCESS, substatus = FAILED_SUBMIT.)
		Substatus: cvbqpb.AttemptSubstatus_NO_SUBSTATUS,
	}
	return a, nil
}

// toGerritChange creates a GerritChange for the given RunCL.
//
// This includes the submit status of the CL.
func toGerritChange(cl *run.RunCL, submitted map[int64]struct{}, mode run.Mode) *cvbqpb.GerritChange {
	detail := cl.Detail
	ci := detail.GetGerrit().GetInfo()
	gc := &cvbqpb.GerritChange{
		Host:                       detail.GetGerrit().Host,
		Project:                    ci.Project,
		Change:                     ci.Number,
		Patchset:                   int64(detail.Patchset),
		EarliestEquivalentPatchset: int64(detail.MinEquivalentPatchset),
		TriggerTime:                cl.Trigger.Time,
		Mode:                       mode.BQAttemptMode(),
		SubmitStatus:               cvbqpb.GerritChange_PENDING,
	}

	if mode == run.FullRun {
		// Mark the CL submit status as success if it appears in the submitted CLs
		// list, and failure if it does not.
		if _, ok := submitted[int64(cl.ID)]; ok {
			gc.SubmitStatus = cvbqpb.GerritChange_SUCCESS
		} else {
			gc.SubmitStatus = cvbqpb.GerritChange_FAILURE
		}
	}

	return gc
}

// fetchCQDAttempt fetches an Attempt from CQDaemon if available.
//
// Returns nil if no Attempt is available.
func fetchCQDAttempt(ctx context.Context, r *run.Run) (*cvbqpb.Attempt, error) {
	if r.FinalizedByCQD {
		f, err := migration.LoadFinishedCQDRun(ctx, r.ID)
		if err != nil {
			return nil, err
		}
		return f.Payload.GetAttempt(), nil
	}
	v := migration.VerifiedCQDRun{ID: r.ID}
	switch err := datastore.Get(ctx, &v); {
	case err == datastore.ErrNoSuchEntity:
		// A Run may end without a VerifiedCQDRun stored if the Run is canceled.
		logging.Debugf(ctx, "no VerifiedCQDRun found for Run %q", r.ID)
	case err != nil:
		return nil, errors.Annotate(err, "failed to fetch VerifiedCQDRun").Tag(transient.Tag).Err()
	}
	return v.Payload.GetRun().GetAttempt(), nil
}

// reconcileAttempts merges the CV Attempt and CQDaemon Attempt.
//
// Modifies and returns the CV Attempt.
//
// Once CV does the relevant work (keeping track of builds, reading the CL
// description footers, and performing checks) these will no longer have to be
// filled in with the CQDaemon Attempt values.
func reconcileAttempts(a, cqda *cvbqpb.Attempt) *cvbqpb.Attempt {
	// The list of Builds will be known to CV after it starts triggering
	// and tracking builds; until then CQD is the source of truth.
	a.Builds = cqda.Builds
	// Substatus generally indicates a failure reason, which is
	// known once one of the checks fails. CQDaemon may specify
	// a substatus in the case of abort (substatus: MANUAL_CANCEL)
	// or failure (FAILED_TRYJOBS etc.).
	if a.Status == cvbqpb.AttemptStatus_ABORTED || a.Status == cvbqpb.AttemptStatus_FAILURE {
		a.Status = cqda.Status
		a.Substatus = cqda.Substatus
	}
	a.Status = cqda.Status
	a.Substatus = cqda.Substatus
	// The HasCustomRequirement is determined by CL description footers.
	a.HasCustomRequirement = cqda.HasCustomRequirement
	return a
}

// attemptStatus converts a Run status to Attempt status.
func attemptStatus(ctx context.Context, r *run.Run) cvbqpb.AttemptStatus {
	switch r.Status {
	case run.Status_SUCCEEDED:
		return cvbqpb.AttemptStatus_SUCCESS
	case run.Status_FAILED:
		// In the case that the checks passed but not all CLs were submitted
		// successfully, the Attempt will still have status set to SUCCESS for
		// backwards compatibility. Note that r.Submission is expected to be
		// set only if a submission is attempted, meaning all checks passed.
		if r.Submission != nil && len(r.Submission.Cls) != len(r.Submission.SubmittedCls) {
			return cvbqpb.AttemptStatus_SUCCESS
		}
		return cvbqpb.AttemptStatus_FAILURE
	case run.Status_CANCELLED:
		return cvbqpb.AttemptStatus_ABORTED
	default:
		logging.Errorf(ctx, "Unexpected attempt status %q", r.Status)
		return cvbqpb.AttemptStatus_ATTEMPT_STATUS_UNSPECIFIED
	}
}

// computeCLGroupKey constructs keys for ClGroupKey and the related
// EquivalentClGroupKey.
//
// These are meant to be opaque keys unique to particular set of CLs and
// patchsets for the purpose of grouping together runs for the same sets of
// patchsets. if isEquivalent is true, then the "min equivalent patchset" is
// used instead of the latest patchset, so that trivial patchsets such as minor
// rebases and CL description updates don't change the key.
func computeCLGroupKey(cls []*run.RunCL, isEquivalent bool) string {
	sort.Slice(cls, func(i, j int) bool {
		// ExternalID includes host and change number but not patchset; but
		// different patchsets of the same CL will never be included in the
		// same list, so sorting on only ExternalID is sufficient.
		return cls[i].ExternalID < cls[j].ExternalID
	})
	h := sha256.New()
	// CL group keys are meant to be opaque keys. We'd like to avoid people
	// depending on CL group key and equivalent CL group key sometimes being
	// equal. We can do this by adding a salt to the hash.
	if isEquivalent {
		h.Write([]byte("equivalent_cl_group_key"))
	}
	separator := []byte{0}
	for i, cl := range cls {
		if i > 0 {
			h.Write(separator)
		}
		h.Write([]byte(cl.Detail.GetGerrit().GetHost()))
		h.Write(separator)
		h.Write([]byte(strconv.FormatInt(cl.Detail.GetGerrit().GetInfo().GetNumber(), 10)))
		h.Write(separator)
		if isEquivalent {
			h.Write([]byte(strconv.FormatInt(int64(cl.Detail.GetMinEquivalentPatchset()), 10)))
		} else {
			h.Write([]byte(strconv.FormatInt(int64(cl.Detail.GetPatchset()), 10)))
		}
	}
	return hex.EncodeToString(h.Sum(nil)[:8])
}
<|endoftext|>


<commit_before>package dev

import (
	"fmt"

	"github.com/relab/gorums"
	"google.golang.org/grpc/encoding"
)

func init() {
	if encoding.GetCodec(gorums.ContentSubtype) == nil {
		encoding.RegisterCodec(gorums.NewCodec())
	}
}

// Manager maintains a connection pool of nodes on
// which quorum calls can be performed.
type Manager struct {
	*gorums.Manager
}

// NewManager returns a new Manager for managing connection to nodes added
// to the manager. This function accepts manager options used to configure
// various aspects of the manager.
func NewManager(opts ...gorums.ManagerOption) (mgr *Manager) {
	mgr = &Manager{}
	mgr.Manager = gorums.NewManager(opts...)
	return mgr
}

// NewConfiguration returns a configuration based on the provided list of nodes
// and a quorum specification. The QuorumSpec must be provided using WithQuorumSpec.
// Nodes can be supplied using WithNodeMap or WithNodeList or WithNodeIDs.
func (m *Manager) NewConfiguration(opts ...gorums.ConfigOption) (c *Configuration, err error) {
	if len(opts) < 1 || len(opts) > 2 {
		return nil, fmt.Errorf("wrong number of options: %d", len(opts))
	}
	c = &Configuration{}
	for _, opt := range opts {
		switch v := opt.(type) {
		case gorums.NodeListOption:
			c.Configuration, err = gorums.NewConfiguration(m.Manager, v)
			if err != nil {
				return nil, err
			}
		case QuorumSpec:
			// Must be last since v may match QuorumSpec if it is interface{}
			c.qspec = v
		default:
			return nil, fmt.Errorf("unknown option type: %v", v)
		}
	}
	return c, nil
}

// Nodes returns a slice of available nodes on this manager.
// IDs are returned in the order they were added at creation of the manager.
func (m *Manager) Nodes() []*Node {
	gorumsNodes := m.Manager.Nodes()
	nodes := make([]*Node, 0, len(gorumsNodes))
	for _, n := range gorumsNodes {
		nodes = append(nodes, &Node{n})
	}
	return nodes
}
<commit_msg>Clarified the NewConfiguration documentation<commit_after>package dev

import (
	"fmt"

	"github.com/relab/gorums"
	"google.golang.org/grpc/encoding"
)

func init() {
	if encoding.GetCodec(gorums.ContentSubtype) == nil {
		encoding.RegisterCodec(gorums.NewCodec())
	}
}

// Manager maintains a connection pool of nodes on
// which quorum calls can be performed.
type Manager struct {
	*gorums.Manager
}

// NewManager returns a new Manager for managing connection to nodes added
// to the manager. This function accepts manager options used to configure
// various aspects of the manager.
func NewManager(opts ...gorums.ManagerOption) (mgr *Manager) {
	mgr = &Manager{}
	mgr.Manager = gorums.NewManager(opts...)
	return mgr
}

// NewConfiguration returns a configuration based on the provided list of nodes (required)
// and an optional quorum specification. The QuorumSpec is require for call types that
// must process replies. For configurations only used for unicast or multicast call types,
// a QuorumSpec is not needed. The QuorumSpec interface is also a ConfigOption.
// Nodes can be supplied using WithNodeMap or WithNodeList or WithNodeIDs.
func (m *Manager) NewConfiguration(opts ...gorums.ConfigOption) (c *Configuration, err error) {
	if len(opts) < 1 || len(opts) > 2 {
		return nil, fmt.Errorf("wrong number of options: %d", len(opts))
	}
	c = &Configuration{}
	for _, opt := range opts {
		switch v := opt.(type) {
		case gorums.NodeListOption:
			c.Configuration, err = gorums.NewConfiguration(m.Manager, v)
			if err != nil {
				return nil, err
			}
		case QuorumSpec:
			// Must be last since v may match QuorumSpec if it is interface{}
			c.qspec = v
		default:
			return nil, fmt.Errorf("unknown option type: %v", v)
		}
	}
	return c, nil
}

// Nodes returns a slice of available nodes on this manager.
// IDs are returned in the order they were added at creation of the manager.
func (m *Manager) Nodes() []*Node {
	gorumsNodes := m.Manager.Nodes()
	nodes := make([]*Node, 0, len(gorumsNodes))
	for _, n := range gorumsNodes {
		nodes = append(nodes, &Node{n})
	}
	return nodes
}
<|endoftext|>


<commit_before>// Copyright 2020 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package executor_test

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	. "github.com/pingcap/check"
	"github.com/pingcap/parser/mysql"
	"github.com/pingcap/tidb/executor"
	"github.com/pingcap/tidb/types"
	"github.com/pingcap/tidb/util/testkit"
)

func cmpAndRm(expected, outfile string, c *C) {
	content, err := ioutil.ReadFile(outfile)
	c.Assert(err, IsNil)
	c.Assert(string(content), Equals, expected)
	c.Assert(os.Remove(outfile), IsNil)
}

func (s *testSuite1) TestSelectIntoFileExists(c *C) {
	tmpDir := os.TempDir()
	outfile := filepath.Join(tmpDir, fmt.Sprintf("TestSelectIntoFileExists-%v.data", time.Now().Nanosecond()))
	tk := testkit.NewTestKit(c, s.store)
	sql := fmt.Sprintf("select 1 into outfile '%v'", outfile)
	tk.MustExec(sql)
	err := tk.ExecToErr(sql)
	c.Assert(err, NotNil)
	strings.Contains(err.Error(), "already exists")
	strings.Contains(err.Error(), outfile)
}

func (s *testSuite1) TestSelectIntoOutfileFromTable(c *C) {
	tmpDir := os.TempDir()
	outfile := filepath.Join(tmpDir, "select-into-outfile.data")
	tk := testkit.NewTestKit(c, s.store)
	tk.MustExec("use test")

	tk.MustExec("drop table if exists t")
	tk.MustExec("create table t (i int, r real, d decimal(10, 5), s varchar(100), dt datetime, ts timestamp, du time, j json)")
	tk.MustExec("insert into t values (1, 1.1, 0.1, 'a', '2000-01-01', '01:01:01', '01:01:01', '[1]')")
	tk.MustExec("insert into t values (2, 2.2, 0.2, 'b', '2000-02-02', '02:02:02', '02:02:02', '[1,2]')")
	tk.MustExec("insert into t values (null, null, null, null, '2000-03-03', '03:03:03', '03:03:03', '[1,2,3]')")
	tk.MustExec("insert into t values (4, 4.4, 0.4, 'd', null, null, null, null)")

	tk.MustExec(fmt.Sprintf("select * from t into outfile %q", outfile))
	cmpAndRm(`1	1.1	0.10000	a	2000-01-01 00:00:00	2001-01-01 00:00:00	01:01:01	[1]
2	2.2	0.20000	b	2000-02-02 00:00:00	2002-02-02 00:00:00	02:02:02	[1, 2]
\N	\N	\N	\N	2000-03-03 00:00:00	2003-03-03 00:00:00	03:03:03	[1, 2, 3]
4	4.4	0.40000	d	\N	\N	\N	\N
`, outfile, c)

	tk.MustExec(fmt.Sprintf("select * from t into outfile %q fields terminated by ',' enclosed by '\"' escaped by '#'", outfile))
	cmpAndRm(`"1","1.1","0.10000","a","2000-01-01 00:00:00","2001-01-01 00:00:00","01:01:01","[1]"
"2","2.2","0.20000","b","2000-02-02 00:00:00","2002-02-02 00:00:00","02:02:02","[1, 2]"
#N,#N,#N,#N,"2000-03-03 00:00:00","2003-03-03 00:00:00","03:03:03","[1, 2, 3]"
"4","4.4","0.40000","d",#N,#N,#N,#N
`, outfile, c)

	tk.MustExec(fmt.Sprintf("select * from t into outfile %q fields terminated by ',' optionally enclosed by '\"' escaped by '#'", outfile))
	cmpAndRm(`1,1.1,0.10000,"a","2000-01-01 00:00:00","2001-01-01 00:00:00","01:01:01","[1]"
2,2.2,0.20000,"b","2000-02-02 00:00:00","2002-02-02 00:00:00","02:02:02","[1, 2]"
#N,#N,#N,#N,"2000-03-03 00:00:00","2003-03-03 00:00:00","03:03:03","[1, 2, 3]"
4,4.4,0.40000,"d",#N,#N,#N,#N
`, outfile, c)

	tk.MustExec(fmt.Sprintf("select * from t into outfile %q fields terminated by ',' optionally enclosed by '\"' escaped by '#' lines terminated by '<<<\n'", outfile))
	cmpAndRm(`1,1.1,0.10000,"a","2000-01-01 00:00:00","2001-01-01 00:00:00","01:01:01","[1]"<<<
2,2.2,0.20000,"b","2000-02-02 00:00:00","2002-02-02 00:00:00","02:02:02","[1, 2]"<<<
#N,#N,#N,#N,"2000-03-03 00:00:00","2003-03-03 00:00:00","03:03:03","[1, 2, 3]"<<<
4,4.4,0.40000,"d",#N,#N,#N,#N<<<
`, outfile, c)
}

func (s *testSuite1) TestSelectIntoOutfileConstant(c *C) {
	tmpDir := os.TempDir()
	outfile := filepath.Join(tmpDir, "select-into-outfile.data")
	tk := testkit.NewTestKit(c, s.store)
	// On windows the outfile name looks like "C:\Users\genius\AppData\Local\Temp\select-into-outfile.data",
	// fmt.Sprintf("%q") is used otherwise the string become
	// "C:UsersgeniusAppDataLocalTempselect-into-outfile.data".
	tk.MustExec(fmt.Sprintf("select 1, 2, 3, '4', '5', '6', 7.7, 8.8, 9.9, null into outfile %q", outfile)) // test constants
	cmpAndRm(`1	2	3	4	5	6	7.7	8.8	9.9	\N
`, outfile, c)

	tk.MustExec(fmt.Sprintf("select 1e10, 1e20, 1.234567e8, 0.000123e3, 1.01234567890123456789, 123456789e-10 into outfile %q", outfile))
	cmpAndRm(`10000000000	1e20	123456700	0.123	1.01234567890123456789	0.0123456789
`, outfile, c)
}

func (s *testSuite1) TestDumpReal(c *C) {
	cases := []struct {
		val    float64
		dec    int
		result string
	}{
		{1.2, 1, "1.2"},
		{1.2, 2, "1.20"},
		{2, 2, "2.00"},
		{2.333, types.UnspecifiedLength, "2.333"},
		{1e14, types.UnspecifiedLength, "100000000000000"},
		{1e15, types.UnspecifiedLength, "1e15"},
		{1e-15, types.UnspecifiedLength, "0.000000000000001"},
		{1e-16, types.UnspecifiedLength, "1e-16"},
	}
	for _, testCase := range cases {
		tp := types.NewFieldType(mysql.TypeDouble)
		tp.Decimal = testCase.dec
		_, buf := executor.DumpRealOutfile(nil, nil, testCase.val, tp)
		c.Assert(string(buf), Equals, testCase.result)
	}
}
<commit_msg>executor: fix unit test of select into file for windows (#14978)<commit_after>// Copyright 2020 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package executor_test

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	. "github.com/pingcap/check"
	"github.com/pingcap/parser/mysql"
	"github.com/pingcap/tidb/executor"
	"github.com/pingcap/tidb/types"
	"github.com/pingcap/tidb/util/testkit"
)

func cmpAndRm(expected, outfile string, c *C) {
	content, err := ioutil.ReadFile(outfile)
	c.Assert(err, IsNil)
	c.Assert(string(content), Equals, expected)
	c.Assert(os.Remove(outfile), IsNil)
}

func (s *testSuite1) TestSelectIntoFileExists(c *C) {
	outfile := filepath.Join(os.TempDir(), fmt.Sprintf("TestSelectIntoFileExists-%v.data", time.Now().Nanosecond()))
	defer func() {
		c.Assert(os.Remove(outfile), IsNil)
	}()
	tk := testkit.NewTestKit(c, s.store)
	sql := fmt.Sprintf("select 1 into outfile %q", outfile)
	tk.MustExec(sql)
	err := tk.ExecToErr(sql)
	c.Assert(err, NotNil)
	c.Assert(strings.Contains(err.Error(), "already exists") ||
		strings.Contains(err.Error(), "file exists"), IsTrue, Commentf("err: %v", err))
	c.Assert(strings.Contains(err.Error(), outfile), IsTrue)
}

func (s *testSuite1) TestSelectIntoOutfileFromTable(c *C) {
	tmpDir := os.TempDir()
	outfile := filepath.Join(tmpDir, "select-into-outfile.data")
	tk := testkit.NewTestKit(c, s.store)
	tk.MustExec("use test")

	tk.MustExec("drop table if exists t")
	tk.MustExec("create table t (i int, r real, d decimal(10, 5), s varchar(100), dt datetime, ts timestamp, du time, j json)")
	tk.MustExec("insert into t values (1, 1.1, 0.1, 'a', '2000-01-01', '01:01:01', '01:01:01', '[1]')")
	tk.MustExec("insert into t values (2, 2.2, 0.2, 'b', '2000-02-02', '02:02:02', '02:02:02', '[1,2]')")
	tk.MustExec("insert into t values (null, null, null, null, '2000-03-03', '03:03:03', '03:03:03', '[1,2,3]')")
	tk.MustExec("insert into t values (4, 4.4, 0.4, 'd', null, null, null, null)")

	tk.MustExec(fmt.Sprintf("select * from t into outfile %q", outfile))
	cmpAndRm(`1	1.1	0.10000	a	2000-01-01 00:00:00	2001-01-01 00:00:00	01:01:01	[1]
2	2.2	0.20000	b	2000-02-02 00:00:00	2002-02-02 00:00:00	02:02:02	[1, 2]
\N	\N	\N	\N	2000-03-03 00:00:00	2003-03-03 00:00:00	03:03:03	[1, 2, 3]
4	4.4	0.40000	d	\N	\N	\N	\N
`, outfile, c)

	tk.MustExec(fmt.Sprintf("select * from t into outfile %q fields terminated by ',' enclosed by '\"' escaped by '#'", outfile))
	cmpAndRm(`"1","1.1","0.10000","a","2000-01-01 00:00:00","2001-01-01 00:00:00","01:01:01","[1]"
"2","2.2","0.20000","b","2000-02-02 00:00:00","2002-02-02 00:00:00","02:02:02","[1, 2]"
#N,#N,#N,#N,"2000-03-03 00:00:00","2003-03-03 00:00:00","03:03:03","[1, 2, 3]"
"4","4.4","0.40000","d",#N,#N,#N,#N
`, outfile, c)

	tk.MustExec(fmt.Sprintf("select * from t into outfile %q fields terminated by ',' optionally enclosed by '\"' escaped by '#'", outfile))
	cmpAndRm(`1,1.1,0.10000,"a","2000-01-01 00:00:00","2001-01-01 00:00:00","01:01:01","[1]"
2,2.2,0.20000,"b","2000-02-02 00:00:00","2002-02-02 00:00:00","02:02:02","[1, 2]"
#N,#N,#N,#N,"2000-03-03 00:00:00","2003-03-03 00:00:00","03:03:03","[1, 2, 3]"
4,4.4,0.40000,"d",#N,#N,#N,#N
`, outfile, c)

	tk.MustExec(fmt.Sprintf("select * from t into outfile %q fields terminated by ',' optionally enclosed by '\"' escaped by '#' lines terminated by '<<<\n'", outfile))
	cmpAndRm(`1,1.1,0.10000,"a","2000-01-01 00:00:00","2001-01-01 00:00:00","01:01:01","[1]"<<<
2,2.2,0.20000,"b","2000-02-02 00:00:00","2002-02-02 00:00:00","02:02:02","[1, 2]"<<<
#N,#N,#N,#N,"2000-03-03 00:00:00","2003-03-03 00:00:00","03:03:03","[1, 2, 3]"<<<
4,4.4,0.40000,"d",#N,#N,#N,#N<<<
`, outfile, c)
}

func (s *testSuite1) TestSelectIntoOutfileConstant(c *C) {
	tmpDir := os.TempDir()
	outfile := filepath.Join(tmpDir, "select-into-outfile.data")
	tk := testkit.NewTestKit(c, s.store)
	// On windows the outfile name looks like "C:\Users\genius\AppData\Local\Temp\select-into-outfile.data",
	// fmt.Sprintf("%q") is used otherwise the string become
	// "C:UsersgeniusAppDataLocalTempselect-into-outfile.data".
	tk.MustExec(fmt.Sprintf("select 1, 2, 3, '4', '5', '6', 7.7, 8.8, 9.9, null into outfile %q", outfile)) // test constants
	cmpAndRm(`1	2	3	4	5	6	7.7	8.8	9.9	\N
`, outfile, c)

	tk.MustExec(fmt.Sprintf("select 1e10, 1e20, 1.234567e8, 0.000123e3, 1.01234567890123456789, 123456789e-10 into outfile %q", outfile))
	cmpAndRm(`10000000000	1e20	123456700	0.123	1.01234567890123456789	0.0123456789
`, outfile, c)
}

func (s *testSuite1) TestDumpReal(c *C) {
	cases := []struct {
		val    float64
		dec    int
		result string
	}{
		{1.2, 1, "1.2"},
		{1.2, 2, "1.20"},
		{2, 2, "2.00"},
		{2.333, types.UnspecifiedLength, "2.333"},
		{1e14, types.UnspecifiedLength, "100000000000000"},
		{1e15, types.UnspecifiedLength, "1e15"},
		{1e-15, types.UnspecifiedLength, "0.000000000000001"},
		{1e-16, types.UnspecifiedLength, "1e-16"},
	}
	for _, testCase := range cases {
		tp := types.NewFieldType(mysql.TypeDouble)
		tp.Decimal = testCase.dec
		_, buf := executor.DumpRealOutfile(nil, nil, testCase.val, tp)
		c.Assert(string(buf), Equals, testCase.result)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"os"
	"strconv"

	"github.com/op/go-nanomsg"
)

func main() {
	flag.Parse()

	if flag.NArg() != 3 {
		println("usage: local_lat <bind-to> <msg-size> <roundtrips")
		os.Exit(1)
	}

	bindTo := flag.Arg(0)
	sz, err := strconv.Atoi(flag.Arg(1))
	if err != nil {
		panic(err)
	}
	rts, err := strconv.Atoi(flag.Arg(2))
	if err != nil {
		panic(err)
	}

	s, err := nanomsg.NewSocket(nanomsg.AF_SP, nanomsg.PAIR)
	if err != nil {
		panic(err)
	}
	if err = s.SetTcpNoDelay(true); err != nil {
		panic(err)
	}
	_, err = s.Bind(bindTo)
	if err != nil {
		panic(err)
	}

	for i := 0; i < rts; i++ {
		buf, err := s.Recv(0)
		if err != nil {
			panic(err)
		} else if len(buf) != sz {
			panic(sz)
		}

		nbytes, err := s.Send(buf, 0)
		if err != nil {
			panic(err)
		} else if nbytes != sz {
			panic(nbytes)
		}
	}

	err = s.Close()
	if err != nil {
		panic(err)
	}
}
<commit_msg>Fixed perf script after last commit<commit_after>package main

import (
	"flag"
	"os"
	"strconv"

	"github.com/op/go-nanomsg"
)

func main() {
	flag.Parse()

	if flag.NArg() != 3 {
		println("usage: local_lat <bind-to> <msg-size> <roundtrips")
		os.Exit(1)
	}

	bindTo := flag.Arg(0)
	sz, err := strconv.Atoi(flag.Arg(1))
	if err != nil {
		panic(err)
	}
	rts, err := strconv.Atoi(flag.Arg(2))
	if err != nil {
		panic(err)
	}

	s, err := nanomsg.NewSocket(nanomsg.AF_SP, nanomsg.PAIR)
	if err != nil {
		panic(err)
	}
	if err = s.SetTCPNoDelay(true); err != nil {
		panic(err)
	}
	_, err = s.Bind(bindTo)
	if err != nil {
		panic(err)
	}

	for i := 0; i < rts; i++ {
		buf, err := s.Recv(0)
		if err != nil {
			panic(err)
		} else if len(buf) != sz {
			panic(sz)
		}

		nbytes, err := s.Send(buf, 0)
		if err != nil {
			panic(err)
		} else if nbytes != sz {
			panic(nbytes)
		}
	}

	err = s.Close()
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"code.google.com/p/go.tools/go/vcs"
)

// These variables are copied from the gobuilder's environment
// to the envv of its subprocesses.
var extraEnv = []string{
	"GOARM",

	// For Unix derivatives.
	"CC",
	"PATH",
	"TMPDIR",
	"USER",

	// For Plan 9.
	"objtype",
	"cputype",
	"path",
}

// builderEnv represents the environment that a Builder will run tests in.
type builderEnv interface {
	// setup sets up the builder environment and returns the directory to run the buildCmd in.
	setup(repo *Repo, workpath, hash string, envv []string) (string, error)
}

// goEnv represents the builderEnv for the main Go repo.
type goEnv struct {
	goos, goarch string
}

func (b *Builder) envv() []string {
	if runtime.GOOS == "windows" {
		return b.envvWindows()
	}

	var e []string
	if *buildTool == "go" {
		e = []string{
			"GOOS=" + b.goos,
			"GOHOSTOS=" + b.goos,
			"GOARCH=" + b.goarch,
			"GOHOSTARCH=" + b.goarch,
			"GOROOT_FINAL=/usr/local/go",
		}
	}

	for _, k := range extraEnv {
		if s, ok := getenvOk(k); ok {
			e = append(e, k+"="+s)
		}
	}
	return e
}

func (b *Builder) envvWindows() []string {
	var start map[string]string
	if *buildTool == "go" {
		start = map[string]string{
			"GOOS":         b.goos,
			"GOHOSTOS":     b.goos,
			"GOARCH":       b.goarch,
			"GOHOSTARCH":   b.goarch,
			"GOROOT_FINAL": `c:\go`,
			"GOBUILDEXIT":  "1", // exit all.bat with completion status.
		}
	}

	for _, name := range extraEnv {
		if s, ok := getenvOk(name); ok {
			start[name] = s
		}
	}
	skip := map[string]bool{
		"GOBIN":   true,
		"GOPATH":  true,
		"GOROOT":  true,
		"INCLUDE": true,
		"LIB":     true,
	}
	var e []string
	for name, v := range start {
		e = append(e, name+"="+v)
		skip[name] = true
	}
	for _, kv := range os.Environ() {
		s := strings.SplitN(kv, "=", 2)
		name := strings.ToUpper(s[0])
		switch {
		case name == "":
			// variables, like "=C:=C:\", just copy them
			e = append(e, kv)
		case !skip[name]:
			e = append(e, kv)
			skip[name] = true
		}
	}
	return e
}

// setup for a goEnv clones the main go repo to workpath/go at the provided hash
// and returns the path workpath/go/src, the location of all go build scripts.
func (env *goEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) {
	goworkpath := filepath.Join(workpath, "go")
	if err := repo.Export(goworkpath, hash); err != nil {
		return "", fmt.Errorf("error exporting repository: %s", err)
	}
	if err := ioutil.WriteFile(filepath.Join(goworkpath, "VERSION"), []byte(hash), 0644); err != nil {
		return "", fmt.Errorf("error writing VERSION file: %s", err)
	}
	return filepath.Join(goworkpath, "src"), nil
}

// gccgoEnv represents the builderEnv for the gccgo compiler.
type gccgoEnv struct{}

// setup for a gccgoEnv clones the gofrontend repo to workpath/go at the hash
// and clones the latest GCC branch to repo.Path/gcc. The gccgo sources are
// replaced with the updated sources in the gofrontend repo and gcc gets
// gets configured and built in workpath/gcc-objdir. The path to
// workpath/gcc-objdir is returned.
func (env *gccgoEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) {
	gofrontendpath := filepath.Join(workpath, "gofrontend")
	gccpath := filepath.Join(repo.Path, "gcc")
	gccgopath := filepath.Join(gccpath, "gcc", "go", "gofrontend")
	gcclibgopath := filepath.Join(gccpath, "libgo")

	// get a handle to SVN vcs.Cmd for pulling down GCC.
	svn := vcs.ByCmd("svn")

	// only pull down gcc if we don't have a local copy.
	if _, err := os.Stat(gccpath); err != nil {
		if err := timeout(*cmdTimeout, func() error {
			// pull down a working copy of GCC.
			return svn.Create(gccpath, *gccPath)
		}); err != nil {
			return "", err
		}
	} else {
		// make sure to remove gccgopath and gcclibgopath before
		// updating the repo to avoid file clobbering.
		if err := os.RemoveAll(gccgopath); err != nil {
			return "", err
		}
		if err := os.RemoveAll(gcclibgopath); err != nil {
			return "", err
		}
	}
	if err := svn.Download(gccpath); err != nil {
		return "", err
	}

	// clone gofrontend repo at specified revision
	if _, err := repo.Clone(gofrontendpath, hash); err != nil {
		return "", err
	}

	// remove gccgopath and gcclibgopath before copying over gofrontend.
	if err := os.RemoveAll(gccgopath); err != nil {
		return "", err
	}
	if err := os.RemoveAll(gcclibgopath); err != nil {
		return "", err
	}

	// copy gofrontend and libgo to appropriate locations
	if err := copyDir(filepath.Join(gofrontendpath, "go"), gccgopath); err != nil {
		return "", fmt.Errorf("Failed to copy gofrontend/go to gcc/go/gofrontend: %s\n", err)
	}
	if err := copyDir(filepath.Join(gofrontendpath, "libgo"), gcclibgopath); err != nil {
		return "", fmt.Errorf("Failed to copy gofrontend/libgo to gcc/libgo: %s\n", err)
	}

	// make objdir to work in
	gccobjdir := filepath.Join(workpath, "gcc-objdir")
	if err := os.Mkdir(gccobjdir, mkdirPerm); err != nil {
		return "", err
	}

	// configure GCC with substituted gofrontend and libgo
	gccConfigCmd := []string{
		filepath.Join(gccpath, "configure"),
		"--enable-languages=c,c++,go",
		"--disable-bootstrap",
		"--disable-multilib",
	}
	if _, err := runOutput(*cmdTimeout, envv, ioutil.Discard, gccobjdir, gccConfigCmd...); err != nil {
		return "", fmt.Errorf("Failed to configure GCC: %s", err)
	}

	// build gcc
	if _, err := runOutput(*buildTimeout, envv, ioutil.Discard, gccobjdir, "make"); err != nil {
		return "", fmt.Errorf("Failed to build GCC: %s", err)
	}

	return gccobjdir, nil
}

// copyDir copies the src directory into the dst
func copyDir(src, dst string) error {
	return filepath.Walk(src, func(path string, f os.FileInfo, err error) error {
		dstPath := strings.Replace(path, src, dst, 1)
		if f.IsDir() {
			return os.Mkdir(dstPath, mkdirPerm)
		}

		srcFile, err := os.Open(path)
		if err != nil {
			return err
		}
		defer srcFile.Close()

		dstFile, err := os.Create(dstPath)
		if err != nil {
			return err
		}

		if _, err := io.Copy(dstFile, srcFile); err != nil {
			return err
		}
		return dstFile.Close()
	})
}

func getenvOk(k string) (v string, ok bool) {
	v = os.Getenv(k)
	if v != "" {
		return v, true
	}
	keq := k + "="
	for _, kv := range os.Environ() {
		if kv == keq {
			return "", true
		}
	}
	return "", false
}
<commit_msg>go.tools/dashboard/builder: respect CGO_ENABLED value from the environment<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"code.google.com/p/go.tools/go/vcs"
)

// These variables are copied from the gobuilder's environment
// to the envv of its subprocesses.
var extraEnv = []string{
	"GOARM",
	"CGO_ENABLED",

	// For Unix derivatives.
	"CC",
	"PATH",
	"TMPDIR",
	"USER",

	// For Plan 9.
	"objtype",
	"cputype",
	"path",
}

// builderEnv represents the environment that a Builder will run tests in.
type builderEnv interface {
	// setup sets up the builder environment and returns the directory to run the buildCmd in.
	setup(repo *Repo, workpath, hash string, envv []string) (string, error)
}

// goEnv represents the builderEnv for the main Go repo.
type goEnv struct {
	goos, goarch string
}

func (b *Builder) envv() []string {
	if runtime.GOOS == "windows" {
		return b.envvWindows()
	}

	var e []string
	if *buildTool == "go" {
		e = []string{
			"GOOS=" + b.goos,
			"GOHOSTOS=" + b.goos,
			"GOARCH=" + b.goarch,
			"GOHOSTARCH=" + b.goarch,
			"GOROOT_FINAL=/usr/local/go",
		}
	}

	for _, k := range extraEnv {
		if s, ok := getenvOk(k); ok {
			e = append(e, k+"="+s)
		}
	}
	return e
}

func (b *Builder) envvWindows() []string {
	var start map[string]string
	if *buildTool == "go" {
		start = map[string]string{
			"GOOS":         b.goos,
			"GOHOSTOS":     b.goos,
			"GOARCH":       b.goarch,
			"GOHOSTARCH":   b.goarch,
			"GOROOT_FINAL": `c:\go`,
			"GOBUILDEXIT":  "1", // exit all.bat with completion status.
		}
	}

	for _, name := range extraEnv {
		if s, ok := getenvOk(name); ok {
			start[name] = s
		}
	}
	skip := map[string]bool{
		"GOBIN":   true,
		"GOPATH":  true,
		"GOROOT":  true,
		"INCLUDE": true,
		"LIB":     true,
	}
	var e []string
	for name, v := range start {
		e = append(e, name+"="+v)
		skip[name] = true
	}
	for _, kv := range os.Environ() {
		s := strings.SplitN(kv, "=", 2)
		name := strings.ToUpper(s[0])
		switch {
		case name == "":
			// variables, like "=C:=C:\", just copy them
			e = append(e, kv)
		case !skip[name]:
			e = append(e, kv)
			skip[name] = true
		}
	}
	return e
}

// setup for a goEnv clones the main go repo to workpath/go at the provided hash
// and returns the path workpath/go/src, the location of all go build scripts.
func (env *goEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) {
	goworkpath := filepath.Join(workpath, "go")
	if err := repo.Export(goworkpath, hash); err != nil {
		return "", fmt.Errorf("error exporting repository: %s", err)
	}
	if err := ioutil.WriteFile(filepath.Join(goworkpath, "VERSION"), []byte(hash), 0644); err != nil {
		return "", fmt.Errorf("error writing VERSION file: %s", err)
	}
	return filepath.Join(goworkpath, "src"), nil
}

// gccgoEnv represents the builderEnv for the gccgo compiler.
type gccgoEnv struct{}

// setup for a gccgoEnv clones the gofrontend repo to workpath/go at the hash
// and clones the latest GCC branch to repo.Path/gcc. The gccgo sources are
// replaced with the updated sources in the gofrontend repo and gcc gets
// gets configured and built in workpath/gcc-objdir. The path to
// workpath/gcc-objdir is returned.
func (env *gccgoEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) {
	gofrontendpath := filepath.Join(workpath, "gofrontend")
	gccpath := filepath.Join(repo.Path, "gcc")
	gccgopath := filepath.Join(gccpath, "gcc", "go", "gofrontend")
	gcclibgopath := filepath.Join(gccpath, "libgo")

	// get a handle to SVN vcs.Cmd for pulling down GCC.
	svn := vcs.ByCmd("svn")

	// only pull down gcc if we don't have a local copy.
	if _, err := os.Stat(gccpath); err != nil {
		if err := timeout(*cmdTimeout, func() error {
			// pull down a working copy of GCC.
			return svn.Create(gccpath, *gccPath)
		}); err != nil {
			return "", err
		}
	} else {
		// make sure to remove gccgopath and gcclibgopath before
		// updating the repo to avoid file clobbering.
		if err := os.RemoveAll(gccgopath); err != nil {
			return "", err
		}
		if err := os.RemoveAll(gcclibgopath); err != nil {
			return "", err
		}
	}
	if err := svn.Download(gccpath); err != nil {
		return "", err
	}

	// clone gofrontend repo at specified revision
	if _, err := repo.Clone(gofrontendpath, hash); err != nil {
		return "", err
	}

	// remove gccgopath and gcclibgopath before copying over gofrontend.
	if err := os.RemoveAll(gccgopath); err != nil {
		return "", err
	}
	if err := os.RemoveAll(gcclibgopath); err != nil {
		return "", err
	}

	// copy gofrontend and libgo to appropriate locations
	if err := copyDir(filepath.Join(gofrontendpath, "go"), gccgopath); err != nil {
		return "", fmt.Errorf("Failed to copy gofrontend/go to gcc/go/gofrontend: %s\n", err)
	}
	if err := copyDir(filepath.Join(gofrontendpath, "libgo"), gcclibgopath); err != nil {
		return "", fmt.Errorf("Failed to copy gofrontend/libgo to gcc/libgo: %s\n", err)
	}

	// make objdir to work in
	gccobjdir := filepath.Join(workpath, "gcc-objdir")
	if err := os.Mkdir(gccobjdir, mkdirPerm); err != nil {
		return "", err
	}

	// configure GCC with substituted gofrontend and libgo
	gccConfigCmd := []string{
		filepath.Join(gccpath, "configure"),
		"--enable-languages=c,c++,go",
		"--disable-bootstrap",
		"--disable-multilib",
	}
	if _, err := runOutput(*cmdTimeout, envv, ioutil.Discard, gccobjdir, gccConfigCmd...); err != nil {
		return "", fmt.Errorf("Failed to configure GCC: %s", err)
	}

	// build gcc
	if _, err := runOutput(*buildTimeout, envv, ioutil.Discard, gccobjdir, "make"); err != nil {
		return "", fmt.Errorf("Failed to build GCC: %s", err)
	}

	return gccobjdir, nil
}

// copyDir copies the src directory into the dst
func copyDir(src, dst string) error {
	return filepath.Walk(src, func(path string, f os.FileInfo, err error) error {
		dstPath := strings.Replace(path, src, dst, 1)
		if f.IsDir() {
			return os.Mkdir(dstPath, mkdirPerm)
		}

		srcFile, err := os.Open(path)
		if err != nil {
			return err
		}
		defer srcFile.Close()

		dstFile, err := os.Create(dstPath)
		if err != nil {
			return err
		}

		if _, err := io.Copy(dstFile, srcFile); err != nil {
			return err
		}
		return dstFile.Close()
	})
}

func getenvOk(k string) (v string, ok bool) {
	v = os.Getenv(k)
	if v != "" {
		return v, true
	}
	keq := k + "="
	for _, kv := range os.Environ() {
		if kv == keq {
			return "", true
		}
	}
	return "", false
}
<|endoftext|>


<commit_before>package http

import (
	"bytes"
	"fmt"
	"io"
	"mime/multipart"
	"net/textproto"

	cmds "github.com/jbenet/go-ipfs/commands"
)

// MultiFileReader reads from a `commands.File` (which can be a directory of files
// or a regular file) as HTTP multipart encoded data.
// WARNING: Not thread-safe!
type MultiFileReader struct {
	io.Reader

	files       cmds.File
	currentFile io.Reader
	buf         bytes.Buffer
	mpWriter    *multipart.Writer
	closed      bool

	// if true, the data will be type 'multipart/form-data'
	// if false, the data will be type 'multipart/mixed'
	form bool
}

// NewMultiFileReader constructs a MultiFileReader. `file` can be any `commands.File`.
// If `form` is set to true, the multipart data will have a Content-Type of 'multipart/form-data',
// if `form` is false, the Content-Type will be 'multipart/mixed'.
func NewMultiFileReader(file cmds.File, form bool) *MultiFileReader {
	mfr := &MultiFileReader{
		files: file,
		form:  form,
	}
	mfr.mpWriter = multipart.NewWriter(&mfr.buf)

	return mfr
}

func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) {
	// if we are closed, end reading
	if mfr.closed && mfr.buf.Len() == 0 {
		return 0, io.EOF
	}

	// if the current file isn't set, advance to the next file
	if mfr.currentFile == nil {
		file, err := mfr.files.NextFile()
		if err == io.EOF {
			mfr.mpWriter.Close()
			mfr.closed = true
		} else if err != nil {
			return 0, err
		}

		// handle starting a new file part
		if !mfr.closed {
			if file.IsDirectory() {
				// if file is a directory, create a multifilereader from it
				// (using 'multipart/mixed')
				mfr.currentFile = NewMultiFileReader(file, false)
			} else {
				// otherwise, use the file as a reader to read its contents
				mfr.currentFile = file
			}

			// write the boundary and headers
			header := make(textproto.MIMEHeader)
			if mfr.form {
				contentDisposition := fmt.Sprintf("form-data; name=\"file\"; filename=\"%s\"", file.FileName())
				header.Set("Content-Disposition", contentDisposition)
			} else {
				header.Set("Content-Disposition", fmt.Sprintf("file; filename=\"%s\"", file.FileName()))
			}

			if file.IsDirectory() {
				boundary := mfr.currentFile.(*MultiFileReader).Boundary()
				header.Set("Content-Type", fmt.Sprintf("multipart/mixed; boundary=%s", boundary))
			} else {
				header.Set("Content-Type", "application/octet-stream")
			}

			_, err := mfr.mpWriter.CreatePart(header)
			if err != nil {
				return 0, err
			}
		}
	}

	var reader io.Reader

	if mfr.buf.Len() > 0 {
		// if the buffer has something in it, read from it
		reader = &mfr.buf

	} else if mfr.currentFile != nil {
		// otherwise, read from file data
		reader = mfr.currentFile
	}

	written, err = reader.Read(buf)
	if err == io.EOF && reader == mfr.currentFile {
		mfr.currentFile = nil
		return mfr.Read(buf)
	}
	return written, err
}

// Boundary returns the boundary string to be used to separate files in the multipart data
func (mfr *MultiFileReader) Boundary() string {
	return mfr.mpWriter.Boundary()
}
<commit_msg>commands/http: Documented MultiFileReader<commit_after>package http

import (
	"bytes"
	"fmt"
	"io"
	"mime/multipart"
	"net/textproto"

	cmds "github.com/jbenet/go-ipfs/commands"
)

// MultiFileReader reads from a `commands.File` (which can be a directory of files
// or a regular file) as HTTP multipart encoded data.
// WARNING: Not thread-safe!
type MultiFileReader struct {
	io.Reader

	files       cmds.File
	currentFile io.Reader
	buf         bytes.Buffer
	mpWriter    *multipart.Writer
	closed      bool

	// if true, the data will be type 'multipart/form-data'
	// if false, the data will be type 'multipart/mixed'
	form bool
}

// NewMultiFileReader constructs a MultiFileReader. `file` can be any `commands.File`.
// If `form` is set to true, the multipart data will have a Content-Type of 'multipart/form-data',
// if `form` is false, the Content-Type will be 'multipart/mixed'.
func NewMultiFileReader(file cmds.File, form bool) *MultiFileReader {
	mfr := &MultiFileReader{
		files: file,
		form:  form,
	}
	mfr.mpWriter = multipart.NewWriter(&mfr.buf)

	return mfr
}

func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) {
	// if we are closed and the buffer is flushed, end reading
	if mfr.closed && mfr.buf.Len() == 0 {
		return 0, io.EOF
	}

	// if the current file isn't set, advance to the next file
	if mfr.currentFile == nil {
		file, err := mfr.files.NextFile()
		if err == io.EOF {
			mfr.mpWriter.Close()
			mfr.closed = true
		} else if err != nil {
			return 0, err
		}

		// handle starting a new file part
		if !mfr.closed {
			if file.IsDirectory() {
				// if file is a directory, create a multifilereader from it
				// (using 'multipart/mixed')
				mfr.currentFile = NewMultiFileReader(file, false)
			} else {
				// otherwise, use the file as a reader to read its contents
				mfr.currentFile = file
			}

			// write the boundary and headers
			header := make(textproto.MIMEHeader)
			if mfr.form {
				contentDisposition := fmt.Sprintf("form-data; name=\"file\"; filename=\"%s\"", file.FileName())
				header.Set("Content-Disposition", contentDisposition)
			} else {
				header.Set("Content-Disposition", fmt.Sprintf("file; filename=\"%s\"", file.FileName()))
			}

			if file.IsDirectory() {
				boundary := mfr.currentFile.(*MultiFileReader).Boundary()
				header.Set("Content-Type", fmt.Sprintf("multipart/mixed; boundary=%s", boundary))
			} else {
				header.Set("Content-Type", "application/octet-stream")
			}

			_, err := mfr.mpWriter.CreatePart(header)
			if err != nil {
				return 0, err
			}
		}
	}

	var reader io.Reader

	if mfr.buf.Len() > 0 {
		// if the buffer has something in it, read from it
		reader = &mfr.buf

	} else if mfr.currentFile != nil {
		// otherwise, read from file data
		reader = mfr.currentFile
	}

	written, err = reader.Read(buf)
	if err == io.EOF && reader == mfr.currentFile {
		mfr.currentFile = nil
		return mfr.Read(buf)
	}
	return written, err
}

// Boundary returns the boundary string to be used to separate files in the multipart data
func (mfr *MultiFileReader) Boundary() string {
	return mfr.mpWriter.Boundary()
}
<|endoftext|>


<commit_before>package gin

import (
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/exec"
	"runtime"
	"sync"
	"time"
)

type Runner interface {
	Run() (*exec.Cmd, error)
	Info() (os.FileInfo, error)
	SetWriter(io.Writer)
	Kill() error
}

type runner struct {
	bin       string
	args      []string
	writer    io.Writer
	command   *exec.Cmd
	starttime time.Time
	mux       sync.Mutex
}

func NewRunner(bin string, args ...string) Runner {
	return &runner{
		bin:       bin,
		args:      args,
		writer:    ioutil.Discard,
		starttime: time.Now(),
	}
}

func (r *runner) Run() (*exec.Cmd, error) {
	if r.needsRefresh() {
		r.Kill()
	}

	r.mux.Lock()
	defer r.mux.Unlock()

	if r.command == nil || r.Exited() {
		err := r.runBin()
		return r.command, err
	} else {
		return r.command, nil
	}

}

func (r *runner) Info() (os.FileInfo, error) {
	return os.Stat(r.bin)
}

func (r *runner) SetWriter(writer io.Writer) {
	r.writer = writer
}

func (r *runner) Kill() error {
	if r.command != nil && r.command.Process != nil {
		done := make(chan error)
		go func() {
			r.command.Wait()
			close(done)
		}()

		//Trying a "soft" kill first
		if runtime.GOOS == "windows" {
			if err := r.command.Process.Kill(); err != nil {
				return err
			}
		} else if err := r.command.Process.Signal(os.Interrupt); err != nil {
			return err
		}

		//Wait for our process to die before we return or hard kill after 3 sec
		select {
		case <-time.After(3 * time.Second):
			if err := r.command.Process.Kill(); err != nil {
				log.Println("failed to kill: ", err)
			}
		case <-done:
		}
		r.command = nil
	}

	return nil
}

func (r *runner) Exited() bool {
	return r.command != nil && r.command.ProcessState != nil && r.command.ProcessState.Exited()
}

func (r *runner) runBin() error {
	r.command = exec.Command(r.bin, r.args...)
	stdout, err := r.command.StdoutPipe()
	if err != nil {
		return err
	}
	stderr, err := r.command.StderrPipe()
	if err != nil {
		return err
	}

	err = r.command.Start()
	if err != nil {
		return err
	}

	r.starttime = time.Now()

	go io.Copy(r.writer, stdout)
	go io.Copy(r.writer, stderr)
	go r.command.Wait()

	_, err = net.Dial("tcp", "localhost:3001")
	for err != nil {
		log.Println("Waiting for 3001")
		time.Sleep(1000 * time.Millisecond)
		_, err = net.Dial("tcp", "localhost:3001")
	}

	return nil
}

func (r *runner) needsRefresh() bool {
	info, err := r.Info()
	if err != nil {
		return false
	} else {
		return info.ModTime().After(r.starttime)
	}
}
<commit_msg>Erroring<commit_after>package gin

import (
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/exec"
	"runtime"
	"sync"
	"time"
)

type Runner interface {
	Run() (*exec.Cmd, error)
	Info() (os.FileInfo, error)
	SetWriter(io.Writer)
	Kill() error
}

type runner struct {
	bin       string
	args      []string
	writer    io.Writer
	command   *exec.Cmd
	starttime time.Time
	mux       sync.Mutex
}

func NewRunner(bin string, args ...string) Runner {
	return &runner{
		bin:       bin,
		args:      args,
		writer:    ioutil.Discard,
		starttime: time.Now(),
	}
}

func (r *runner) Run() (*exec.Cmd, error) {
	if r.needsRefresh() {
		r.Kill()
	}

	r.mux.Lock()
	defer r.mux.Unlock()

	if r.command == nil || r.Exited() {
		err := r.runBin()
		return r.command, err
	} else {
		return r.command, nil
	}

}

func (r *runner) Info() (os.FileInfo, error) {
	return os.Stat(r.bin)
}

func (r *runner) SetWriter(writer io.Writer) {
	r.writer = writer
}

func (r *runner) Kill() error {
	if r.command != nil && r.command.Process != nil {
		done := make(chan error)
		go func() {
			r.command.Wait()
			close(done)
		}()

		//Trying a "soft" kill first
		if runtime.GOOS == "windows" {
			if err := r.command.Process.Kill(); err != nil {
				return err
			}
		} else if err := r.command.Process.Signal(os.Interrupt); err != nil {
			return err
		}

		//Wait for our process to die before we return or hard kill after 3 sec
		select {
		case <-time.After(3 * time.Second):
			if err := r.command.Process.Kill(); err != nil {
				log.Println("failed to kill: ", err)
			}
		case <-done:
		}
		r.command = nil
	}

	return nil
}

func (r *runner) Exited() bool {
	return r.command != nil && r.command.ProcessState != nil && r.command.ProcessState.Exited()
}

func (r *runner) runBin() error {
	r.command = exec.Command(r.bin, r.args...)
	stdout, err := r.command.StdoutPipe()
	if err != nil {
		return err
	}
	stderr, err := r.command.StderrPipe()
	if err != nil {
		return err
	}

	err = r.command.Start()
	if err != nil {
		return err
	}

	r.starttime = time.Now()

	go io.Copy(r.writer, stdout)
	go io.Copy(r.writer, stderr)
	go r.command.Wait()

	conn, err := net.Dial("tcp", ":3001")
	for err != nil {
		log.Printf("Waiting for 3001: %+v, %+v", conn, err)
		time.Sleep(1000 * time.Millisecond)
		conn, err = net.Dial("tcp", ":3001")
	}
	conn.Close()

	return nil
}

func (r *runner) needsRefresh() bool {
	info, err := r.Info()
	if err != nil {
		return false
	} else {
		return info.ModTime().After(r.starttime)
	}
}
<|endoftext|>


<commit_before>package api

import (
	"testing"

	"github.com/grafana/grafana/pkg/api/dtos"
	"github.com/grafana/grafana/pkg/bus"
	m "github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/services/annotations"

	. "github.com/smartystreets/goconvey/convey"
)

func TestAnnotationsApiEndpoint(t *testing.T) {
	Convey("Given an annotation without a dashboard id", t, func() {
		cmd := dtos.PostAnnotationsCmd{
			DashboardId: 1,
			Time:        1000,
			Text:        "annotation text",
			Tags:        []string{"tag1", "tag2"},
			IsRegion:    false,
		}

		updateCmd := dtos.UpdateAnnotationsCmd{
			Time:     1000,
			Text:     "annotation text",
			Tags:     []string{"tag1", "tag2"},
			IsRegion: false,
		}

		Convey("When user is an Org Viewer", func() {
			role := m.ROLE_VIEWER
			Convey("Should not be allowed to save an annotation", func() {
				postAnnotationScenario("When calling POST on", "/api/annotations", "/api/annotations", role, cmd, func(sc *scenarioContext) {
					sc.fakeReqWithParams("POST", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 403)
				})

				putAnnotationScenario("When calling PUT on", "/api/annotations/1", "/api/annotations/:annotationId", role, updateCmd, func(sc *scenarioContext) {
					sc.fakeReqWithParams("PUT", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 403)
				})

				loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/annotations/1", "/api/annotations/:annotationId", role, func(sc *scenarioContext) {
					sc.handlerFunc = DeleteAnnotationById
					sc.fakeReqWithParams("DELETE", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 403)
				})

				loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/annotations/region/1", "/api/annotations/region/:regionId", role, func(sc *scenarioContext) {
					sc.handlerFunc = DeleteAnnotationRegion
					sc.fakeReqWithParams("DELETE", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 403)
				})
			})
		})

		Convey("When user is an Org Editor", func() {
			role := m.ROLE_EDITOR
			Convey("Should be able to save an annotation", func() {
				postAnnotationScenario("When calling POST on", "/api/annotations", "/api/annotations", role, cmd, func(sc *scenarioContext) {
					sc.fakeReqWithParams("POST", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 200)
				})

				putAnnotationScenario("When calling PUT on", "/api/annotations/1", "/api/annotations/:annotationId", role, updateCmd, func(sc *scenarioContext) {
					sc.fakeReqWithParams("PUT", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 200)
				})

				loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/annotations/1", "/api/annotations/:annotationId", role, func(sc *scenarioContext) {
					sc.handlerFunc = DeleteAnnotationById
					sc.fakeReqWithParams("DELETE", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 200)
				})

				loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/annotations/region/1", "/api/annotations/region/:regionId", role, func(sc *scenarioContext) {
					sc.handlerFunc = DeleteAnnotationRegion
					sc.fakeReqWithParams("DELETE", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 200)
				})
			})

			Convey("Should note be able to save an annotation", func() {
				cmd := dtos.PostAnnotationsCmd{
					Time: 1000,
					Text: "annotation text",
				}
				postAnnotationScenario("When calling POST without dashboardId", "/api/annotations", "/api/annotations", role, cmd, func(sc *scenarioContext) {
					sc.fakeReqWithParams("POST", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 500)
				})

				cmd := dtos.PostAnnotationsCmd{
					Time:        1000,
					DashboardId: 3,
				}
				postAnnotationScenario("When calling POST without text", "/api/annotations", "/api/annotations", role, cmd, func(sc *scenarioContext) {
					sc.fakeReqWithParams("POST", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 500)
				})
			})
		})
	})

	Convey("Given an annotation with a dashboard id and the dashboard does not have an acl", t, func() {
		cmd := dtos.PostAnnotationsCmd{
			Time:        1000,
			Text:        "annotation text",
			Tags:        []string{"tag1", "tag2"},
			IsRegion:    false,
			DashboardId: 1,
			PanelId:     1,
		}

		updateCmd := dtos.UpdateAnnotationsCmd{
			Time:     1000,
			Text:     "annotation text",
			Tags:     []string{"tag1", "tag2"},
			IsRegion: false,
			Id:       1,
		}

		viewerRole := m.ROLE_VIEWER
		editorRole := m.ROLE_EDITOR

		aclMockResp := []*m.DashboardAclInfoDTO{
			{Role: &viewerRole, Permission: m.PERMISSION_VIEW},
			{Role: &editorRole, Permission: m.PERMISSION_EDIT},
		}

		bus.AddHandler("test", func(query *m.GetDashboardAclInfoListQuery) error {
			query.Result = aclMockResp
			return nil
		})

		bus.AddHandler("test", func(query *m.GetTeamsByUserQuery) error {
			query.Result = []*m.Team{}
			return nil
		})

		Convey("When user is an Org Viewer", func() {
			role := m.ROLE_VIEWER
			Convey("Should not be allowed to save an annotation", func() {
				postAnnotationScenario("When calling POST on", "/api/annotations", "/api/annotations", role, cmd, func(sc *scenarioContext) {
					sc.fakeReqWithParams("POST", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 403)
				})

				putAnnotationScenario("When calling PUT on", "/api/annotations/1", "/api/annotations/:annotationId", role, updateCmd, func(sc *scenarioContext) {
					sc.fakeReqWithParams("PUT", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 403)
				})

				loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/annotations/1", "/api/annotations/:annotationId", role, func(sc *scenarioContext) {
					sc.handlerFunc = DeleteAnnotationById
					sc.fakeReqWithParams("DELETE", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 403)
				})

				loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/annotations/region/1", "/api/annotations/region/:regionId", role, func(sc *scenarioContext) {
					sc.handlerFunc = DeleteAnnotationRegion
					sc.fakeReqWithParams("DELETE", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 403)
				})
			})
		})

		Convey("When user is an Org Editor", func() {
			role := m.ROLE_EDITOR
			Convey("Should be able to save an annotation", func() {
				postAnnotationScenario("When calling POST on", "/api/annotations", "/api/annotations", role, cmd, func(sc *scenarioContext) {
					sc.fakeReqWithParams("POST", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 200)
				})

				putAnnotationScenario("When calling PUT on", "/api/annotations/1", "/api/annotations/:annotationId", role, updateCmd, func(sc *scenarioContext) {
					sc.fakeReqWithParams("PUT", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 200)
				})

				loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/annotations/1", "/api/annotations/:annotationId", role, func(sc *scenarioContext) {
					sc.handlerFunc = DeleteAnnotationById
					sc.fakeReqWithParams("DELETE", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 200)
				})

				loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/annotations/region/1", "/api/annotations/region/:regionId", role, func(sc *scenarioContext) {
					sc.handlerFunc = DeleteAnnotationRegion
					sc.fakeReqWithParams("DELETE", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 200)
				})
			})
		})
	})
}

type fakeAnnotationsRepo struct {
}

func (repo *fakeAnnotationsRepo) Delete(params *annotations.DeleteParams) error {
	return nil
}
func (repo *fakeAnnotationsRepo) Save(item *annotations.Item) error {
	item.Id = 1
	return nil
}
func (repo *fakeAnnotationsRepo) Update(item *annotations.Item) error {
	return nil
}
func (repo *fakeAnnotationsRepo) Find(query *annotations.ItemQuery) ([]*annotations.ItemDTO, error) {
	annotations := []*annotations.ItemDTO{{Id: 1}}
	return annotations, nil
}

var fakeAnnoRepo *fakeAnnotationsRepo

func postAnnotationScenario(desc string, url string, routePattern string, role m.RoleType, cmd dtos.PostAnnotationsCmd, fn scenarioFunc) {
	Convey(desc+" "+url, func() {
		defer bus.ClearBusHandlers()

		sc := setupScenarioContext(url)
		sc.defaultHandler = wrap(func(c *m.ReqContext) Response {
			sc.context = c
			sc.context.UserId = TestUserID
			sc.context.OrgId = TestOrgID
			sc.context.OrgRole = role

			return PostAnnotation(c, cmd)
		})

		fakeAnnoRepo = &fakeAnnotationsRepo{}
		annotations.SetRepository(fakeAnnoRepo)

		sc.m.Post(routePattern, sc.defaultHandler)

		fn(sc)
	})
}

func putAnnotationScenario(desc string, url string, routePattern string, role m.RoleType, cmd dtos.UpdateAnnotationsCmd, fn scenarioFunc) {
	Convey(desc+" "+url, func() {
		defer bus.ClearBusHandlers()

		sc := setupScenarioContext(url)
		sc.defaultHandler = wrap(func(c *m.ReqContext) Response {
			sc.context = c
			sc.context.UserId = TestUserID
			sc.context.OrgId = TestOrgID
			sc.context.OrgRole = role

			return UpdateAnnotation(c, cmd)
		})

		fakeAnnoRepo = &fakeAnnotationsRepo{}
		annotations.SetRepository(fakeAnnoRepo)

		sc.m.Put(routePattern, sc.defaultHandler)

		fn(sc)
	})
}
<commit_msg>fix operator<commit_after>package api

import (
	"testing"

	"github.com/grafana/grafana/pkg/api/dtos"
	"github.com/grafana/grafana/pkg/bus"
	m "github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/services/annotations"

	. "github.com/smartystreets/goconvey/convey"
)

func TestAnnotationsApiEndpoint(t *testing.T) {
	Convey("Given an annotation without a dashboard id", t, func() {
		cmd := dtos.PostAnnotationsCmd{
			DashboardId: 1,
			Time:        1000,
			Text:        "annotation text",
			Tags:        []string{"tag1", "tag2"},
			IsRegion:    false,
		}

		updateCmd := dtos.UpdateAnnotationsCmd{
			Time:     1000,
			Text:     "annotation text",
			Tags:     []string{"tag1", "tag2"},
			IsRegion: false,
		}

		Convey("When user is an Org Viewer", func() {
			role := m.ROLE_VIEWER
			Convey("Should not be allowed to save an annotation", func() {
				postAnnotationScenario("When calling POST on", "/api/annotations", "/api/annotations", role, cmd, func(sc *scenarioContext) {
					sc.fakeReqWithParams("POST", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 403)
				})

				putAnnotationScenario("When calling PUT on", "/api/annotations/1", "/api/annotations/:annotationId", role, updateCmd, func(sc *scenarioContext) {
					sc.fakeReqWithParams("PUT", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 403)
				})

				loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/annotations/1", "/api/annotations/:annotationId", role, func(sc *scenarioContext) {
					sc.handlerFunc = DeleteAnnotationById
					sc.fakeReqWithParams("DELETE", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 403)
				})

				loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/annotations/region/1", "/api/annotations/region/:regionId", role, func(sc *scenarioContext) {
					sc.handlerFunc = DeleteAnnotationRegion
					sc.fakeReqWithParams("DELETE", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 403)
				})
			})
		})

		Convey("When user is an Org Editor", func() {
			role := m.ROLE_EDITOR
			Convey("Should be able to save an annotation", func() {
				postAnnotationScenario("When calling POST on", "/api/annotations", "/api/annotations", role, cmd, func(sc *scenarioContext) {
					sc.fakeReqWithParams("POST", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 200)
				})

				putAnnotationScenario("When calling PUT on", "/api/annotations/1", "/api/annotations/:annotationId", role, updateCmd, func(sc *scenarioContext) {
					sc.fakeReqWithParams("PUT", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 200)
				})

				loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/annotations/1", "/api/annotations/:annotationId", role, func(sc *scenarioContext) {
					sc.handlerFunc = DeleteAnnotationById
					sc.fakeReqWithParams("DELETE", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 200)
				})

				loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/annotations/region/1", "/api/annotations/region/:regionId", role, func(sc *scenarioContext) {
					sc.handlerFunc = DeleteAnnotationRegion
					sc.fakeReqWithParams("DELETE", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 200)
				})
			})

			Convey("Should note be able to save an annotation", func() {
				cmd = dtos.PostAnnotationsCmd{
					Time: 1000,
					Text: "annotation text",
				}
				postAnnotationScenario("When calling POST without dashboardId", "/api/annotations", "/api/annotations", role, cmd, func(sc *scenarioContext) {
					sc.fakeReqWithParams("POST", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 500)
				})

				cmd = dtos.PostAnnotationsCmd{
					Time:        1000,
					DashboardId: 3,
				}
				postAnnotationScenario("When calling POST without text", "/api/annotations", "/api/annotations", role, cmd, func(sc *scenarioContext) {
					sc.fakeReqWithParams("POST", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 500)
				})
			})
		})
	})

	Convey("Given an annotation with a dashboard id and the dashboard does not have an acl", t, func() {
		cmd := dtos.PostAnnotationsCmd{
			Time:        1000,
			Text:        "annotation text",
			Tags:        []string{"tag1", "tag2"},
			IsRegion:    false,
			DashboardId: 1,
			PanelId:     1,
		}

		updateCmd := dtos.UpdateAnnotationsCmd{
			Time:     1000,
			Text:     "annotation text",
			Tags:     []string{"tag1", "tag2"},
			IsRegion: false,
			Id:       1,
		}

		viewerRole := m.ROLE_VIEWER
		editorRole := m.ROLE_EDITOR

		aclMockResp := []*m.DashboardAclInfoDTO{
			{Role: &viewerRole, Permission: m.PERMISSION_VIEW},
			{Role: &editorRole, Permission: m.PERMISSION_EDIT},
		}

		bus.AddHandler("test", func(query *m.GetDashboardAclInfoListQuery) error {
			query.Result = aclMockResp
			return nil
		})

		bus.AddHandler("test", func(query *m.GetTeamsByUserQuery) error {
			query.Result = []*m.Team{}
			return nil
		})

		Convey("When user is an Org Viewer", func() {
			role := m.ROLE_VIEWER
			Convey("Should not be allowed to save an annotation", func() {
				postAnnotationScenario("When calling POST on", "/api/annotations", "/api/annotations", role, cmd, func(sc *scenarioContext) {
					sc.fakeReqWithParams("POST", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 403)
				})

				putAnnotationScenario("When calling PUT on", "/api/annotations/1", "/api/annotations/:annotationId", role, updateCmd, func(sc *scenarioContext) {
					sc.fakeReqWithParams("PUT", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 403)
				})

				loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/annotations/1", "/api/annotations/:annotationId", role, func(sc *scenarioContext) {
					sc.handlerFunc = DeleteAnnotationById
					sc.fakeReqWithParams("DELETE", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 403)
				})

				loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/annotations/region/1", "/api/annotations/region/:regionId", role, func(sc *scenarioContext) {
					sc.handlerFunc = DeleteAnnotationRegion
					sc.fakeReqWithParams("DELETE", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 403)
				})
			})
		})

		Convey("When user is an Org Editor", func() {
			role := m.ROLE_EDITOR
			Convey("Should be able to save an annotation", func() {
				postAnnotationScenario("When calling POST on", "/api/annotations", "/api/annotations", role, cmd, func(sc *scenarioContext) {
					sc.fakeReqWithParams("POST", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 200)
				})

				putAnnotationScenario("When calling PUT on", "/api/annotations/1", "/api/annotations/:annotationId", role, updateCmd, func(sc *scenarioContext) {
					sc.fakeReqWithParams("PUT", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 200)
				})

				loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/annotations/1", "/api/annotations/:annotationId", role, func(sc *scenarioContext) {
					sc.handlerFunc = DeleteAnnotationById
					sc.fakeReqWithParams("DELETE", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 200)
				})

				loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/annotations/region/1", "/api/annotations/region/:regionId", role, func(sc *scenarioContext) {
					sc.handlerFunc = DeleteAnnotationRegion
					sc.fakeReqWithParams("DELETE", sc.url, map[string]string{}).exec()
					So(sc.resp.Code, ShouldEqual, 200)
				})
			})
		})
	})
}

type fakeAnnotationsRepo struct {
}

func (repo *fakeAnnotationsRepo) Delete(params *annotations.DeleteParams) error {
	return nil
}
func (repo *fakeAnnotationsRepo) Save(item *annotations.Item) error {
	item.Id = 1
	return nil
}
func (repo *fakeAnnotationsRepo) Update(item *annotations.Item) error {
	return nil
}
func (repo *fakeAnnotationsRepo) Find(query *annotations.ItemQuery) ([]*annotations.ItemDTO, error) {
	annotations := []*annotations.ItemDTO{{Id: 1}}
	return annotations, nil
}

var fakeAnnoRepo *fakeAnnotationsRepo

func postAnnotationScenario(desc string, url string, routePattern string, role m.RoleType, cmd dtos.PostAnnotationsCmd, fn scenarioFunc) {
	Convey(desc+" "+url, func() {
		defer bus.ClearBusHandlers()

		sc := setupScenarioContext(url)
		sc.defaultHandler = wrap(func(c *m.ReqContext) Response {
			sc.context = c
			sc.context.UserId = TestUserID
			sc.context.OrgId = TestOrgID
			sc.context.OrgRole = role

			return PostAnnotation(c, cmd)
		})

		fakeAnnoRepo = &fakeAnnotationsRepo{}
		annotations.SetRepository(fakeAnnoRepo)

		sc.m.Post(routePattern, sc.defaultHandler)

		fn(sc)
	})
}

func putAnnotationScenario(desc string, url string, routePattern string, role m.RoleType, cmd dtos.UpdateAnnotationsCmd, fn scenarioFunc) {
	Convey(desc+" "+url, func() {
		defer bus.ClearBusHandlers()

		sc := setupScenarioContext(url)
		sc.defaultHandler = wrap(func(c *m.ReqContext) Response {
			sc.context = c
			sc.context.UserId = TestUserID
			sc.context.OrgId = TestOrgID
			sc.context.OrgRole = role

			return UpdateAnnotation(c, cmd)
		})

		fakeAnnoRepo = &fakeAnnotationsRepo{}
		annotations.SetRepository(fakeAnnoRepo)

		sc.m.Put(routePattern, sc.defaultHandler)

		fn(sc)
	})
}
<|endoftext|>


<commit_before>package network

import (
	"context"
	"fmt"
	"log"
	"net"
	"reflect"

	"github.com/golang/protobuf/proto"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"

	"github.com/golang/protobuf/ptypes/empty"
	"github.com/n0stack/n0core/pkg/datastore"
	"github.com/n0stack/proto.go/budget/v0"
	"github.com/n0stack/proto.go/pool/v0"
)

type NetworkAPI struct {
	dataStore datastore.Datastore
}

func CreateNetworkAPI(ds datastore.Datastore) (*NetworkAPI, error) {
	a := &NetworkAPI{
		dataStore: ds,
	}

	return a, nil
}

func (a NetworkAPI) ListNetworks(ctx context.Context, req *ppool.ListNetworksRequest) (*ppool.ListNetworksResponse, error) {
	res := &ppool.ListNetworksResponse{}
	f := func(s int) []proto.Message {
		res.Networks = make([]*ppool.Network, s)
		for i := range res.Networks {
			res.Networks[i] = &ppool.Network{}
		}

		m := make([]proto.Message, s)
		for i, v := range res.Networks {
			m[i] = v
		}

		return m
	}

	if err := a.dataStore.List(f); err != nil {
		log.Printf("[WARNING] Failed to list data from db: err='%s'", err.Error())
		return nil, grpc.Errorf(codes.Internal, "Failed to list from db, please retry or contact for the administrator of this cluster")
	}
	if len(res.Networks) == 0 {
		return nil, grpc.Errorf(codes.NotFound, "")
	}

	return res, nil
}

func (a NetworkAPI) GetNetwork(ctx context.Context, req *ppool.GetNetworkRequest) (*ppool.Network, error) {
	res := &ppool.Network{}
	if err := a.dataStore.Get(req.Name, res); err != nil {
		log.Printf("[WARNING] Failed to get data from db: err='%s'", err.Error())
		return nil, grpc.Errorf(codes.Internal, "Failed to get '%s' from db, please retry or contact for the administrator of this cluster", req.Name)
	}
	if reflect.ValueOf(res.Metadata).IsNil() {
		return nil, grpc.Errorf(codes.NotFound, "")
	}

	return res, nil
}

func (a NetworkAPI) ApplyNetwork(ctx context.Context, req *ppool.ApplyNetworkRequest) (*ppool.Network, error) {
	res := &ppool.Network{
		Metadata: req.Metadata,
		Spec:     req.Spec,
		Status:   &ppool.NetworkStatus{},
	}

	if _, _, err := net.ParseCIDR(req.Spec.Ipv4Cidr); err != nil {
		return nil, grpc.Errorf(codes.InvalidArgument, "Field 'ipv4_cidr' is invalid : %s", err.Error())
	}

	prev := &ppool.Network{}
	if err := a.dataStore.Get(req.Metadata.Name, prev); err != nil {
		log.Printf("[WARNING] Failed to get data from db: err='%s'", err.Error())
		return nil, grpc.Errorf(codes.Internal, "Failed to get '%s' from db, please retry or contact for the administrator of this cluster", req.Metadata.Name)
	}
	var err error
	res.Metadata.Version, err = datastore.CheckVersion(prev, req)
	if err != nil {
		return nil, grpc.Errorf(codes.InvalidArgument, "Failed to check version: %s", err.Error())
	}

	res.Status.State = ppool.NetworkStatus_AVAILABLE
	if err := a.dataStore.Apply(req.Metadata.Name, res); err != nil {
		log.Printf("[WARNING] Failed to apply data for db: err='%s'", err.Error())
		return nil, grpc.Errorf(codes.Internal, "Failed to store '%s' for db, please retry or contact for the administrator of this cluster", req.Metadata.Name)
	}

	return res, nil
}

func (a NetworkAPI) DeleteNetwork(ctx context.Context, req *ppool.DeleteNetworkRequest) (*empty.Empty, error) {
	if err := a.dataStore.Delete(req.Name); err != nil {
		log.Printf("[WARNING] Failed to delete data from db: err='%s'", err.Error())
		return &empty.Empty{}, grpc.Errorf(codes.Internal, "Failed to delete '%s' from db, please retry or contact for the administrator of this cluster", req.Name)
	}

	return &empty.Empty{}, nil
}

// とりあえず IPv4 のスケジューリングのみに対応
func (a NetworkAPI) ReserveNetworkInterface(ctx context.Context, req *ppool.ReserveNetworkInterfaceRequest) (*ppool.ReserveNetworkInterfaceResponse, error) {
	if req.NetworkInterfaceName == "" {
		return nil, grpc.Errorf(codes.InvalidArgument, "Do not set field 'network_interface_name' as blank")
	}

	n := &ppool.Network{}
	if err := a.dataStore.Get(req.Name, n); err != nil {
		log.Printf("[WARNING] Failed to get data from db: err='%s'", err.Error())
		return nil, grpc.Errorf(codes.Internal, "Failed to get '%s' from db, please retry or contact for the administrator of this cluster", req.Name)
	}
	if n == nil {
		return nil, grpc.Errorf(codes.NotFound, "Node '%s' is not found", req.Name)
	}
	if n.Status.ReservedNetworkInterfaces == nil {
		n.Status.ReservedNetworkInterfaces = make(map[string]*pbudget.NetworkInterface)
	}
	if _, ok := n.Status.ReservedNetworkInterfaces[req.NetworkInterfaceName]; ok {
		return nil, grpc.Errorf(codes.AlreadyExists, "Network interface '%s' is already exists on Node '%s'", req.NetworkInterfaceName, req.Name)
	}

	// 保存する際にパースするのでエラーは発生しない
	_, cidr, _ := net.ParseCIDR(n.Spec.Ipv4Cidr)

	var reqIPv4 net.IP
	if req.NetworkInterface == nil || req.NetworkInterface.Ipv4Address == "" {
		if reqIPv4 = ScheduleNewIPv4(cidr, n.Status.ReservedNetworkInterfaces); reqIPv4 == nil {
			return nil, grpc.Errorf(codes.ResourceExhausted, "ipv4_address is full on Network '%s'", req.Name)
		}
	} else {
		reqIPv4 = net.ParseIP(req.NetworkInterface.Ipv4Address)
		if reqIPv4 == nil {
			return nil, grpc.Errorf(codes.InvalidArgument, "ipv4_address field is invalid")
		}

		if err := CheckIPv4OnCIDR(reqIPv4, cidr); err != nil {
			return nil, grpc.Errorf(codes.InvalidArgument, "ipv4_address field is invalid: %s", err.Error())
		}
		if err := CheckConflictIPv4(reqIPv4, n.Status.ReservedNetworkInterfaces); err != nil {
			return nil, grpc.Errorf(codes.AlreadyExists, "ipv4_address field is invalid: %s", err.Error())
		}
	}

	var reqHW net.HardwareAddr
	if req.NetworkInterface == nil || req.NetworkInterface.HardwareAddress == "" {
		reqHW = GenerateHardwareAddress(fmt.Sprintf("%s/%s", req.Name, req.NetworkInterfaceName))
	} else {
		var err error
		reqHW, err = net.ParseMAC(req.NetworkInterface.HardwareAddress)
		if err != nil {
			return nil, grpc.Errorf(codes.InvalidArgument, "hardware_address field is invalid")
		}
	}

	res := &ppool.ReserveNetworkInterfaceResponse{
		Name:                 req.Name,
		NetworkInterfaceName: req.NetworkInterfaceName,
		NetworkInterface: &pbudget.NetworkInterface{
			Annotations:     req.NetworkInterface.Annotations,
			HardwareAddress: reqHW.String(),
			Ipv4Address:     reqIPv4.String(),
		},
	}
	n.Status.ReservedNetworkInterfaces[req.NetworkInterfaceName] = res.NetworkInterface
	if err := a.dataStore.Apply(req.Name, n); err != nil {
		log.Printf("[WARNING] Failed to store data on db: err='%s'", err.Error())
		return nil, grpc.Errorf(codes.Internal, "Failed to store '%s' on db, please retry or contact for the administrator of this cluster", req.Name)
	}

	return res, nil
}

func (a NetworkAPI) ReleaseNetworkInterface(ctx context.Context, req *ppool.ReleaseNetworkInterfaceRequest) (*empty.Empty, error) {
	n := &ppool.Network{}
	if err := a.dataStore.Get(req.Name, n); err != nil {
		log.Printf("[WARNING] Failed to get data from db: err='%s'", err.Error())
		return nil, grpc.Errorf(codes.Internal, "Failed to get '%s' from db, please retry or contact for the administrator of this cluster", req.Name)
	}
	if n == nil {
		return nil, grpc.Errorf(codes.NotFound, "Do not exists network '%s'", req.Name)
	}

	if _, ok := n.Status.ReservedNetworkInterfaces[req.NetworkInterfaceName]; !ok {
		return nil, grpc.Errorf(codes.NotFound, "Do not exists network interface '%s' on network '%s'", req.NetworkInterfaceName, req.Name)
	}
	delete(n.Status.ReservedNetworkInterfaces, req.NetworkInterfaceName)

	if err := a.dataStore.Apply(req.Name, n); err != nil {
		log.Printf("[WARNING] Failed to apply data for db: err='%s'", err.Error())
		return nil, grpc.Errorf(codes.Internal, "Failed to store '%s' on db, please retry or contact for the administrator of this cluster", req.Name)
	}

	return &empty.Empty{}, nil
}
<commit_msg>req.NetworkInterface on ReserveNetworkInterface<commit_after>package network

import (
	"context"
	"fmt"
	"log"
	"net"
	"reflect"

	"github.com/golang/protobuf/proto"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"

	"github.com/golang/protobuf/ptypes/empty"
	"github.com/n0stack/n0core/pkg/datastore"
	"github.com/n0stack/proto.go/budget/v0"
	"github.com/n0stack/proto.go/pool/v0"
)

type NetworkAPI struct {
	dataStore datastore.Datastore
}

func CreateNetworkAPI(ds datastore.Datastore) (*NetworkAPI, error) {
	a := &NetworkAPI{
		dataStore: ds,
	}

	return a, nil
}

func (a NetworkAPI) ListNetworks(ctx context.Context, req *ppool.ListNetworksRequest) (*ppool.ListNetworksResponse, error) {
	res := &ppool.ListNetworksResponse{}
	f := func(s int) []proto.Message {
		res.Networks = make([]*ppool.Network, s)
		for i := range res.Networks {
			res.Networks[i] = &ppool.Network{}
		}

		m := make([]proto.Message, s)
		for i, v := range res.Networks {
			m[i] = v
		}

		return m
	}

	if err := a.dataStore.List(f); err != nil {
		log.Printf("[WARNING] Failed to list data from db: err='%s'", err.Error())
		return nil, grpc.Errorf(codes.Internal, "Failed to list from db, please retry or contact for the administrator of this cluster")
	}
	if len(res.Networks) == 0 {
		return nil, grpc.Errorf(codes.NotFound, "")
	}

	return res, nil
}

func (a NetworkAPI) GetNetwork(ctx context.Context, req *ppool.GetNetworkRequest) (*ppool.Network, error) {
	res := &ppool.Network{}
	if err := a.dataStore.Get(req.Name, res); err != nil {
		log.Printf("[WARNING] Failed to get data from db: err='%s'", err.Error())
		return nil, grpc.Errorf(codes.Internal, "Failed to get '%s' from db, please retry or contact for the administrator of this cluster", req.Name)
	}
	if reflect.ValueOf(res.Metadata).IsNil() {
		return nil, grpc.Errorf(codes.NotFound, "")
	}

	return res, nil
}

func (a NetworkAPI) ApplyNetwork(ctx context.Context, req *ppool.ApplyNetworkRequest) (*ppool.Network, error) {
	res := &ppool.Network{
		Metadata: req.Metadata,
		Spec:     req.Spec,
		Status:   &ppool.NetworkStatus{},
	}

	if _, _, err := net.ParseCIDR(req.Spec.Ipv4Cidr); err != nil {
		return nil, grpc.Errorf(codes.InvalidArgument, "Field 'ipv4_cidr' is invalid : %s", err.Error())
	}

	prev := &ppool.Network{}
	if err := a.dataStore.Get(req.Metadata.Name, prev); err != nil {
		log.Printf("[WARNING] Failed to get data from db: err='%s'", err.Error())
		return nil, grpc.Errorf(codes.Internal, "Failed to get '%s' from db, please retry or contact for the administrator of this cluster", req.Metadata.Name)
	}
	var err error
	res.Metadata.Version, err = datastore.CheckVersion(prev, req)
	if err != nil {
		return nil, grpc.Errorf(codes.InvalidArgument, "Failed to check version: %s", err.Error())
	}

	res.Status.State = ppool.NetworkStatus_AVAILABLE
	if err := a.dataStore.Apply(req.Metadata.Name, res); err != nil {
		log.Printf("[WARNING] Failed to apply data for db: err='%s'", err.Error())
		return nil, grpc.Errorf(codes.Internal, "Failed to store '%s' for db, please retry or contact for the administrator of this cluster", req.Metadata.Name)
	}

	return res, nil
}

func (a NetworkAPI) DeleteNetwork(ctx context.Context, req *ppool.DeleteNetworkRequest) (*empty.Empty, error) {
	if err := a.dataStore.Delete(req.Name); err != nil {
		log.Printf("[WARNING] Failed to delete data from db: err='%s'", err.Error())
		return &empty.Empty{}, grpc.Errorf(codes.Internal, "Failed to delete '%s' from db, please retry or contact for the administrator of this cluster", req.Name)
	}

	return &empty.Empty{}, nil
}

// とりあえず IPv4 のスケジューリングのみに対応
func (a NetworkAPI) ReserveNetworkInterface(ctx context.Context, req *ppool.ReserveNetworkInterfaceRequest) (*ppool.ReserveNetworkInterfaceResponse, error) {
	if req.NetworkInterfaceName == "" {
		return nil, grpc.Errorf(codes.InvalidArgument, "Do not set field 'network_interface_name' as blank")
	}

	n := &ppool.Network{}
	if err := a.dataStore.Get(req.Name, n); err != nil {
		log.Printf("[WARNING] Failed to get data from db: err='%s'", err.Error())
		return nil, grpc.Errorf(codes.Internal, "Failed to get '%s' from db, please retry or contact for the administrator of this cluster", req.Name)
	}
	if n == nil {
		return nil, grpc.Errorf(codes.NotFound, "Node '%s' is not found", req.Name)
	}
	if n.Status.ReservedNetworkInterfaces == nil {
		n.Status.ReservedNetworkInterfaces = make(map[string]*pbudget.NetworkInterface)
	}
	if _, ok := n.Status.ReservedNetworkInterfaces[req.NetworkInterfaceName]; ok {
		return nil, grpc.Errorf(codes.AlreadyExists, "Network interface '%s' is already exists on Node '%s'", req.NetworkInterfaceName, req.Name)
	}

	// 保存する際にパースするのでエラーは発生しない
	_, cidr, _ := net.ParseCIDR(n.Spec.Ipv4Cidr)
	if req.NetworkInterface == nil {
		req.NetworkInterface = &pbudget.NetworkInterface{}
	}

	var reqIPv4 net.IP
	if req.NetworkInterface.Ipv4Address == "" {
		if reqIPv4 = ScheduleNewIPv4(cidr, n.Status.ReservedNetworkInterfaces); reqIPv4 == nil {
			return nil, grpc.Errorf(codes.ResourceExhausted, "ipv4_address is full on Network '%s'", req.Name)
		}
	} else {
		reqIPv4 = net.ParseIP(req.NetworkInterface.Ipv4Address)
		if reqIPv4 == nil {
			return nil, grpc.Errorf(codes.InvalidArgument, "ipv4_address field is invalid")
		}

		if err := CheckIPv4OnCIDR(reqIPv4, cidr); err != nil {
			return nil, grpc.Errorf(codes.InvalidArgument, "ipv4_address field is invalid: %s", err.Error())
		}
		if err := CheckConflictIPv4(reqIPv4, n.Status.ReservedNetworkInterfaces); err != nil {
			return nil, grpc.Errorf(codes.AlreadyExists, "ipv4_address field is invalid: %s", err.Error())
		}
	}

	var reqHW net.HardwareAddr
	if req.NetworkInterface.HardwareAddress == "" {
		reqHW = GenerateHardwareAddress(fmt.Sprintf("%s/%s", req.Name, req.NetworkInterfaceName))
	} else {
		var err error
		reqHW, err = net.ParseMAC(req.NetworkInterface.HardwareAddress)
		if err != nil {
			return nil, grpc.Errorf(codes.InvalidArgument, "hardware_address field is invalid")
		}
	}

	res := &ppool.ReserveNetworkInterfaceResponse{
		Name:                 req.Name,
		NetworkInterfaceName: req.NetworkInterfaceName,
		NetworkInterface: &pbudget.NetworkInterface{
			Annotations:     req.NetworkInterface.Annotations,
			HardwareAddress: reqHW.String(),
			Ipv4Address:     reqIPv4.String(),
		},
	}
	n.Status.ReservedNetworkInterfaces[req.NetworkInterfaceName] = res.NetworkInterface
	if err := a.dataStore.Apply(req.Name, n); err != nil {
		log.Printf("[WARNING] Failed to store data on db: err='%s'", err.Error())
		return nil, grpc.Errorf(codes.Internal, "Failed to store '%s' on db, please retry or contact for the administrator of this cluster", req.Name)
	}

	return res, nil
}

func (a NetworkAPI) ReleaseNetworkInterface(ctx context.Context, req *ppool.ReleaseNetworkInterfaceRequest) (*empty.Empty, error) {
	n := &ppool.Network{}
	if err := a.dataStore.Get(req.Name, n); err != nil {
		log.Printf("[WARNING] Failed to get data from db: err='%s'", err.Error())
		return nil, grpc.Errorf(codes.Internal, "Failed to get '%s' from db, please retry or contact for the administrator of this cluster", req.Name)
	}
	if n == nil {
		return nil, grpc.Errorf(codes.NotFound, "Do not exists network '%s'", req.Name)
	}

	if _, ok := n.Status.ReservedNetworkInterfaces[req.NetworkInterfaceName]; !ok {
		return nil, grpc.Errorf(codes.NotFound, "Do not exists network interface '%s' on network '%s'", req.NetworkInterfaceName, req.Name)
	}
	delete(n.Status.ReservedNetworkInterfaces, req.NetworkInterfaceName)

	if err := a.dataStore.Apply(req.Name, n); err != nil {
		log.Printf("[WARNING] Failed to apply data for db: err='%s'", err.Error())
		return nil, grpc.Errorf(codes.Internal, "Failed to store '%s' on db, please retry or contact for the administrator of this cluster", req.Name)
	}

	return &empty.Empty{}, nil
}
<|endoftext|>