<commit_before>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Matt Tracy (matt@cockroachlabs.com)

package storage_test

import (
	"fmt"
	"reflect"
	"testing"
	"time"

	"github.com/cockroachdb/cockroach/client"
	"github.com/cockroachdb/cockroach/proto"
	"github.com/cockroachdb/cockroach/storage"
	"github.com/cockroachdb/cockroach/storage/engine"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/leaktest"
)

type storeEventReader struct {
	perStoreFeeds map[proto.StoreID][]string
}

// recordEvent records the events received for all stores. Each event is
// recorded as a simple string value; this is less exhaustive than a full struct
// comparison, but should be easier to correct if future changes slightly modify
// these values.
func (ser *storeEventReader) recordEvent(event interface{}) {
	var sid proto.StoreID
	eventStr := ""
	switch event := event.(type) {
	case *storage.StartStoreEvent:
		sid = event.StoreID
		eventStr = "StartStore"
	case *storage.AddRangeEvent:
		sid = event.StoreID
		eventStr = fmt.Sprintf("AddRange rid=%d, live=%d",
			event.Desc.RaftID, event.Stats.LiveBytes)
	case *storage.UpdateRangeEvent:
		sid = event.StoreID
		eventStr = fmt.Sprintf("UpdateRange rid=%d, live=%d",
			event.Desc.RaftID, event.Diff.LiveBytes)
	case *storage.RemoveRangeEvent:
		sid = event.StoreID
		eventStr = fmt.Sprintf("RemoveRange rid=%d, live=%d",
			event.Desc.RaftID, event.Stats.LiveBytes)
	case *storage.SplitRangeEvent:
		sid = event.StoreID
		eventStr = fmt.Sprintf("SplitRange origId=%d, newId=%d, origLive=%d, newLive=%d",
			event.Original.Desc.RaftID, event.New.Desc.RaftID,
			event.Original.Stats.LiveBytes, event.New.Stats.LiveBytes)
	case *storage.MergeRangeEvent:
		sid = event.StoreID
		eventStr = fmt.Sprintf("MergeRange rid=%d, subId=%d, live=%d, subLive=%d",
			event.Merged.Desc.RaftID, event.Removed.Desc.RaftID,
			event.Merged.Stats.LiveBytes, event.Removed.Stats.LiveBytes)
	case *storage.BeginScanRangesEvent:
		sid = event.StoreID
		eventStr = "BeginScanRanges"
	case *storage.EndScanRangesEvent:
		sid = event.StoreID
		eventStr = "EndScanRanges"
	}
	if sid > 0 {
		ser.perStoreFeeds[sid] = append(ser.perStoreFeeds[sid], eventStr)
	}
}

func (ser *storeEventReader) readEvents(sub *util.Subscription) {
	ser.perStoreFeeds = make(map[proto.StoreID][]string)
	for e := range sub.Events() {
		ser.recordEvent(e)
	}
}

// printValues is used to assist the debugging of tests which use
// storeEventReader.
func (ser *storeEventReader) printValues(t testing.TB) {
	for id, feed := range ser.perStoreFeeds {
		fmt.Printf("proto.StoreID(%d): []string{\n", int64(id))
		for _, evt := range feed {
			fmt.Printf("\t\t\"%s\",\n", evt)
		}
		fmt.Printf("},\n")
	}
}

// TestMultiStoreEventFeed verifies that events on multiple stores are properly
// recieved by a single event reader.
func TestMultiStoreEventFeed(t *testing.T) {
	defer leaktest.AfterTest(t)

	// Create a multiTestContext which publishes all store events to the given
	// feed.
	ser := &storeEventReader{}
	feed := &util.Feed{}
	mtc := &multiTestContext{
		feed: feed,
	}

	// Start reading events from the feed before starting the stores.
	readStopper := util.NewStopper()
	sub := feed.Subscribe()
	readStopper.RunWorker(func() {
		ser.readEvents(sub)
	})

	mtc.Start(t, 3)
	defer mtc.Stop()

	// Replicate the default range.
	raftID := int64(1)
	mtc.replicateRange(raftID, 0, 1, 2)

	// Add some data in a transaction
	err := mtc.db.RunTransaction(nil, func(txn *client.Txn) error {
		return txn.Run(
			client.PutCall(proto.Key("a"), []byte("asdf")),
			client.PutCall(proto.Key("c"), []byte("jkl;")),
		)
	})
	if err != nil {
		t.Fatalf("error putting data to db: %s", err.Error())
	}

	// AdminSplit in between the two ranges.
	err = mtc.db.Run(
		client.Call{
			Args: &proto.AdminSplitRequest{
				RequestHeader: proto.RequestHeader{
					Key: proto.Key("b"),
				},
				SplitKey: proto.Key("b"),
			},
			Reply: &proto.AdminSplitResponse{},
		},
	)
	if err != nil {
		t.Fatalf("error splitting initial: %s", err.Error())
	}

	// AdminSplit an empty range at the end of the second range.
	err = mtc.db.Run(
		client.Call{
			Args: &proto.AdminSplitRequest{
				RequestHeader: proto.RequestHeader{
					Key: proto.Key("z"),
				},
				SplitKey: proto.Key("z"),
			},
			Reply: &proto.AdminSplitResponse{},
		},
	)
	if err != nil {
		t.Fatalf("error splitting second range: %s", err.Error())
	}

	// AdminMerge the empty range back into the second range.
	err = mtc.db.Run(
		client.Call{
			Args: &proto.AdminMergeRequest{
				RequestHeader: proto.RequestHeader{
					Key: proto.Key("c"),
				},
			},
			Reply: &proto.AdminMergeResponse{},
		},
	)
	if err != nil {
		t.Fatalf("error merging final range: %s", err.Error())
	}

	// Add an additional put through the system and wait for all
	// replicas to receive it.
	err = mtc.db.Run(
		client.IncrementCall(proto.Key("aa"), 5),
	)
	if err != nil {
		t.Fatalf("error putting data to db: %s", err.Error())
	}
	util.SucceedsWithin(t, time.Second, func() error {
		for _, eng := range mtc.engines {
			val, err := engine.MVCCGet(eng, proto.Key("aa"), mtc.clock.Now(), true, nil)
			if err != nil {
				return err
			}
			if a, e := val.GetInteger(), int64(5); a != e {
				return util.Errorf("expected aa = %d, got %d", e, a)
			}
		}
		return nil
	})

	// Close feed and wait for reader to receive all events.
	feed.Close()
	readStopper.Stop()
	<-readStopper.IsStopped()

	// Compare events to expected values.
	expected := map[proto.StoreID][]string{
		proto.StoreID(1): []string{
			"StartStore",
			"BeginScanRanges",
			"AddRange rid=1, live=348",
			"EndScanRanges",
			"SplitRange origId=1, newId=2, origLive=938, newLive=42",
			"SplitRange origId=2, newId=3, origLive=42, newLive=0",
			"MergeRange rid=2, subId=3, live=42, subLive=0",
		},
		proto.StoreID(2): []string{
			"StartStore",
			"BeginScanRanges",
			"EndScanRanges",
			"SplitRange origId=1, newId=2, origLive=938, newLive=42",
			"SplitRange origId=2, newId=3, origLive=42, newLive=0",
			"MergeRange rid=2, subId=3, live=42, subLive=0",
		},
		proto.StoreID(3): []string{
			"StartStore",
			"BeginScanRanges",
			"EndScanRanges",
			"SplitRange origId=1, newId=2, origLive=938, newLive=42",
			"SplitRange origId=2, newId=3, origLive=42, newLive=0",
			"MergeRange rid=2, subId=3, live=42, subLive=0",
		},
	}
	if a, e := ser.perStoreFeeds, expected; !reflect.DeepEqual(a, e) {
		t.Errorf("event feed did not meet expected value: printing actual values to compare with above expectation:\n")
		ser.printValues(t)
	}
}
<commit_msg>Fix outdated usage of client *Call invocations.<commit_after>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Matt Tracy (matt@cockroachlabs.com)

package storage_test

import (
	"fmt"
	"reflect"
	"testing"
	"time"

	"github.com/cockroachdb/cockroach/client"
	"github.com/cockroachdb/cockroach/proto"
	"github.com/cockroachdb/cockroach/storage"
	"github.com/cockroachdb/cockroach/storage/engine"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/leaktest"
)

type storeEventReader struct {
	perStoreFeeds map[proto.StoreID][]string
}

// recordEvent records the events received for all stores. Each event is
// recorded as a simple string value; this is less exhaustive than a full struct
// comparison, but should be easier to correct if future changes slightly modify
// these values.
func (ser *storeEventReader) recordEvent(event interface{}) {
	var sid proto.StoreID
	eventStr := ""
	switch event := event.(type) {
	case *storage.StartStoreEvent:
		sid = event.StoreID
		eventStr = "StartStore"
	case *storage.AddRangeEvent:
		sid = event.StoreID
		eventStr = fmt.Sprintf("AddRange rid=%d, live=%d",
			event.Desc.RaftID, event.Stats.LiveBytes)
	case *storage.UpdateRangeEvent:
		sid = event.StoreID
		eventStr = fmt.Sprintf("UpdateRange rid=%d, live=%d",
			event.Desc.RaftID, event.Diff.LiveBytes)
	case *storage.RemoveRangeEvent:
		sid = event.StoreID
		eventStr = fmt.Sprintf("RemoveRange rid=%d, live=%d",
			event.Desc.RaftID, event.Stats.LiveBytes)
	case *storage.SplitRangeEvent:
		sid = event.StoreID
		eventStr = fmt.Sprintf("SplitRange origId=%d, newId=%d, origLive=%d, newLive=%d",
			event.Original.Desc.RaftID, event.New.Desc.RaftID,
			event.Original.Stats.LiveBytes, event.New.Stats.LiveBytes)
	case *storage.MergeRangeEvent:
		sid = event.StoreID
		eventStr = fmt.Sprintf("MergeRange rid=%d, subId=%d, live=%d, subLive=%d",
			event.Merged.Desc.RaftID, event.Removed.Desc.RaftID,
			event.Merged.Stats.LiveBytes, event.Removed.Stats.LiveBytes)
	case *storage.BeginScanRangesEvent:
		sid = event.StoreID
		eventStr = "BeginScanRanges"
	case *storage.EndScanRangesEvent:
		sid = event.StoreID
		eventStr = "EndScanRanges"
	}
	if sid > 0 {
		ser.perStoreFeeds[sid] = append(ser.perStoreFeeds[sid], eventStr)
	}
}

func (ser *storeEventReader) readEvents(sub *util.Subscription) {
	ser.perStoreFeeds = make(map[proto.StoreID][]string)
	for e := range sub.Events() {
		ser.recordEvent(e)
	}
}

// printValues is used to assist the debugging of tests which use
// storeEventReader.
func (ser *storeEventReader) printValues(t testing.TB) {
	for id, feed := range ser.perStoreFeeds {
		fmt.Printf("proto.StoreID(%d): []string{\n", int64(id))
		for _, evt := range feed {
			fmt.Printf("\t\t\"%s\",\n", evt)
		}
		fmt.Printf("},\n")
	}
}

// TestMultiStoreEventFeed verifies that events on multiple stores are properly
// recieved by a single event reader.
func TestMultiStoreEventFeed(t *testing.T) {
	defer leaktest.AfterTest(t)

	// Create a multiTestContext which publishes all store events to the given
	// feed.
	ser := &storeEventReader{}
	feed := &util.Feed{}
	mtc := &multiTestContext{
		feed: feed,
	}

	// Start reading events from the feed before starting the stores.
	readStopper := util.NewStopper()
	sub := feed.Subscribe()
	readStopper.RunWorker(func() {
		ser.readEvents(sub)
	})

	mtc.Start(t, 3)
	defer mtc.Stop()

	// Replicate the default range.
	raftID := int64(1)
	mtc.replicateRange(raftID, 0, 1, 2)

	// Add some data in a transaction
	err := mtc.db.RunTransaction(nil, func(txn *client.Txn) error {
		return txn.Run(
			client.Put(proto.Key("a"), []byte("asdf")),
			client.Put(proto.Key("c"), []byte("jkl;")),
		)
	})
	if err != nil {
		t.Fatalf("error putting data to db: %s", err.Error())
	}

	// AdminSplit in between the two ranges.
	err = mtc.db.Run(
		client.Call{
			Args: &proto.AdminSplitRequest{
				RequestHeader: proto.RequestHeader{
					Key: proto.Key("b"),
				},
				SplitKey: proto.Key("b"),
			},
			Reply: &proto.AdminSplitResponse{},
		},
	)
	if err != nil {
		t.Fatalf("error splitting initial: %s", err.Error())
	}

	// AdminSplit an empty range at the end of the second range.
	err = mtc.db.Run(
		client.Call{
			Args: &proto.AdminSplitRequest{
				RequestHeader: proto.RequestHeader{
					Key: proto.Key("z"),
				},
				SplitKey: proto.Key("z"),
			},
			Reply: &proto.AdminSplitResponse{},
		},
	)
	if err != nil {
		t.Fatalf("error splitting second range: %s", err.Error())
	}

	// AdminMerge the empty range back into the second range.
	err = mtc.db.Run(
		client.Call{
			Args: &proto.AdminMergeRequest{
				RequestHeader: proto.RequestHeader{
					Key: proto.Key("c"),
				},
			},
			Reply: &proto.AdminMergeResponse{},
		},
	)
	if err != nil {
		t.Fatalf("error merging final range: %s", err.Error())
	}

	// Add an additional put through the system and wait for all
	// replicas to receive it.
	err = mtc.db.Run(
		client.Increment(proto.Key("aa"), 5),
	)
	if err != nil {
		t.Fatalf("error putting data to db: %s", err.Error())
	}
	util.SucceedsWithin(t, time.Second, func() error {
		for _, eng := range mtc.engines {
			val, err := engine.MVCCGet(eng, proto.Key("aa"), mtc.clock.Now(), true, nil)
			if err != nil {
				return err
			}
			if a, e := val.GetInteger(), int64(5); a != e {
				return util.Errorf("expected aa = %d, got %d", e, a)
			}
		}
		return nil
	})

	// Close feed and wait for reader to receive all events.
	feed.Close()
	readStopper.Stop()
	<-readStopper.IsStopped()

	// Compare events to expected values.
	expected := map[proto.StoreID][]string{
		proto.StoreID(1): []string{
			"StartStore",
			"BeginScanRanges",
			"AddRange rid=1, live=348",
			"EndScanRanges",
			"SplitRange origId=1, newId=2, origLive=938, newLive=42",
			"SplitRange origId=2, newId=3, origLive=42, newLive=0",
			"MergeRange rid=2, subId=3, live=42, subLive=0",
		},
		proto.StoreID(2): []string{
			"StartStore",
			"BeginScanRanges",
			"EndScanRanges",
			"SplitRange origId=1, newId=2, origLive=938, newLive=42",
			"SplitRange origId=2, newId=3, origLive=42, newLive=0",
			"MergeRange rid=2, subId=3, live=42, subLive=0",
		},
		proto.StoreID(3): []string{
			"StartStore",
			"BeginScanRanges",
			"EndScanRanges",
			"SplitRange origId=1, newId=2, origLive=938, newLive=42",
			"SplitRange origId=2, newId=3, origLive=42, newLive=0",
			"MergeRange rid=2, subId=3, live=42, subLive=0",
		},
	}
	if a, e := ser.perStoreFeeds, expected; !reflect.DeepEqual(a, e) {
		t.Errorf("event feed did not meet expected value: printing actual values to compare with above expectation:\n")
		ser.printValues(t)
	}
}
<|endoftext|>


<commit_before>package migrate

import (
	"database/sql"
	"fmt"
	"strings"

	"github.com/go-gorp/gorp"

	"github.com/ovh/cds/engine/api/application"
	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/engine/api/permission"
	"github.com/ovh/cds/engine/api/pipeline"
	"github.com/ovh/cds/engine/api/workflow"
	"github.com/ovh/cds/sdk"
)

const (
	STATUS_START    = "STARTED"
	STATUS_CLEANING = "CLEANING"
	STATUS_DONE     = "DONE"
)

// ToWorkflow migrates old workflow to new workflow
func ToWorkflow(db gorp.SqlExecutor, store cache.Store, cdTree []sdk.CDPipeline, proj *sdk.Project, u *sdk.User, force, disablePrefix, withCurrentVersion, withRepositoryWebHook bool) ([]sdk.Workflow, error) {
	workflows := make([]sdk.Workflow, len(cdTree))
	for i := range cdTree {
		oldW := cdTree[i]
		name := oldW.Application.Name
		if !disablePrefix {
			name = "w" + name
		}

		if len(cdTree) > 1 {
			name = fmt.Sprintf("%s_%d", name, i)
		}
		newW := sdk.Workflow{
			Name:       name,
			ProjectID:  proj.ID,
			ProjectKey: proj.Key,
		}

		if err := addGroupOnWorkflow(db, &newW, &oldW.Application); err != nil {
			return nil, sdk.WrapError(err, "MigrateToWorkflow")
		}

		currentApplicationID := oldW.Application.ID

		n, err := migratePipeline(db, store, proj, oldW, currentApplicationID, u)
		if err != nil {
			return nil, sdk.WrapError(err, "MigrateToWorkflow migratePipeline>")
		}
		newW.Root = n

		if force {
			w, err := workflow.Load(db, store, proj, newW.Name, u, workflow.LoadOptions{})
			if err == nil {
				if errD := workflow.Delete(db, store, proj, w, u); errD != nil {
					return nil, sdk.WrapError(errD, "MigrateToWorkflow workflow.Load>")
				}
			}
		}

		if withRepositoryWebHook && newW.Root.Context.Application != nil && newW.Root.Context.Application.VCSServer != "" && newW.Root.Context.Application.RepositoryFullname != "" {
			h := &sdk.WorkflowNodeHook{}
			m, err := workflow.LoadHookModelByName(db, sdk.RepositoryWebHookModelName)
			if err != nil {
				return nil, sdk.WrapError(err, "migratePipeline> hook %s not found", h.WorkflowHookModel.Name)
			}
			h.WorkflowHookModel = *m
			h.WorkflowHookModelID = m.ID
			h.Config = make(map[string]sdk.WorkflowNodeHookConfigValue, len(m.DefaultConfig))
			for k, v := range m.DefaultConfig {
				if _, has := h.Config[k]; !has {
					h.Config[k] = v
				}
			}
			newW.Root.Hooks = []sdk.WorkflowNodeHook{*h}

			if errW := workflow.Insert(db, store, &newW, proj, u); errW != nil {
				return nil, sdk.WrapError(errW, "MigrateToWorkflow workflow.Insert>")
			}

			oldW, errO := workflow.Load(db, store, proj, newW.Name, u, workflow.LoadOptions{})
			if errO != nil {
				return nil, sdk.WrapError(errO, "migratePipeline> Unable to load old workflow")
			}

			newW.ID = oldW.ID

			// then register (node is updated in HookRegistration)
			if errHr := workflow.HookRegistration(db, store, nil, *oldW, proj); errHr != nil {
				return nil, sdk.WrapError(errHr, "migratePipeline> Cannot register hook 2")
			}
		} else {
			if errW := workflow.Insert(db, store, &newW, proj, u); errW != nil {
				return nil, sdk.WrapError(errW, "MigrateToWorkflow workflow.Insert>")
			}
		}

		if withCurrentVersion {
			opts := []pipeline.ExecOptionFunc{
				pipeline.LoadPipelineBuildOpts.WithStatus(sdk.StatusSuccess.String()),
			}
			pbs, errPB := pipeline.LoadPipelineBuildsByApplicationAndPipeline(db, oldW.Application.ID, oldW.Pipeline.ID, oldW.Environment.ID, 1, opts...)
			if errPB != nil && errPB != sql.ErrNoRows {
				return nil, sdk.WrapError(err, "migratePipeline> Cannot load pipeline")
			}
			if len(pbs) == 1 {
				if err := workflow.InsertRunNum(db, &newW, pbs[0].Version); err != nil {
					return nil, sdk.WrapError(err, "migratePipeline> Cannot set the version %d", pbs[0].Version)
				}
			}
		}

		for _, g := range newW.Groups {
			if err := workflow.AddGroup(db, &newW, g); err != nil {
				return nil, sdk.WrapError(err, "MigrateToWorkflow> Cannot add group")
			}
		}
		workflows[i] = newW
	}
	return workflows, nil
}

func addGroupOnWorkflow(db gorp.SqlExecutor, w *sdk.Workflow, app *sdk.Application) error {
	if err := application.LoadGroupByApplication(db, app); err != nil {
		return sdk.WrapError(err, "addGroupOnWorkflow> error while LoadGroupByApplication on application %s", app.ID)
	}

	for _, ag := range app.ApplicationGroups {
		if ag.Permission == permission.PermissionReadWriteExecute || ag.Permission == permission.PermissionReadExecute {
			w.Groups = append(w.Groups, ag)
		}
	}
	return nil
}

func migratePipeline(db gorp.SqlExecutor, store cache.Store, p *sdk.Project, oldPipeline sdk.CDPipeline, appID int64, u *sdk.User) (*sdk.WorkflowNode, error) {
	newNode := &sdk.WorkflowNode{
		PipelineID: oldPipeline.Pipeline.ID,
		Context:    &sdk.WorkflowNodeContext{},
	}

	// Check if pipeline use application & env variable
	pip, err := pipeline.LoadPipelineByID(db, oldPipeline.Pipeline.ID, true)
	if err != nil {
		return nil, sdk.WrapError(err, "migratePipeline> Cannot load pipeline")
	}
	foundApp := false
bigloop:
	for _, s := range pip.Stages {
		for _, pre := range s.Prerequisites {
			if strings.Contains(pre.ExpectedValue, "cds.app") {
				foundApp = true
				break bigloop
			}
		}

		for _, j := range s.Jobs {
			for _, r := range j.Action.Requirements {
				if strings.Contains(r.Value, "cds.app") || strings.Contains(r.Value, "git.") {
					foundApp = true
					break bigloop
				}
			}
			for _, step := range j.Action.Actions {
				for _, param := range step.Parameters {
					if strings.Contains(param.Value, "cds.app") || strings.Contains(param.Value, "git.") {
						foundApp = true
						break bigloop
					}
				}
			}
		}
	}
	if foundApp {
		app, err := application.LoadByName(db, store, p.Key, oldPipeline.Application.Name, u, application.LoadOptions.WithClearDeploymentStrategies)
		if err != nil {
			return nil, sdk.WrapError(err, "migratePipeline> Cannot load application")
		}
		newNode.Context.Application = app
	}

	if oldPipeline.Environment.ID != 0 && oldPipeline.Environment.ID != sdk.DefaultEnv.ID {
		newNode.Context.Environment = &oldPipeline.Environment
	}

	// Add trigger
	if len(oldPipeline.SubPipelines) > 0 {
		for _, childPip := range oldPipeline.SubPipelines {
			// Create new trigger
			t := sdk.WorkflowNodeTrigger{}

			// Migrate child pipeline
			n, err := migratePipeline(db, store, p, childPip, appID, u)
			if err != nil {
				return nil, err
			}

			// Migrate pipeline parameter
			n.Context.DefaultPipelineParameters = childPip.Trigger.Parameters

			t.WorkflowDestNode = *n

			// Add Condition on trigger
			for _, c := range childPip.Trigger.Prerequisites {
				t.WorkflowDestNode.Context.Conditions.PlainConditions = append(t.WorkflowDestNode.Context.Conditions.PlainConditions, sdk.WorkflowNodeCondition{
					Variable: c.Parameter,
					Value:    c.ExpectedValue,
					Operator: sdk.WorkflowConditionsOperatorRegex,
				})
			}
			t.WorkflowDestNode.Context.Conditions.PlainConditions = append(t.WorkflowDestNode.Context.Conditions.PlainConditions, sdk.WorkflowNodeCondition{
				Variable: "cds.status",
				Value:    "Success",
				Operator: "eq",
			})
			if childPip.Trigger.Manual {
				t.WorkflowDestNode.Context.Conditions.PlainConditions = append(t.WorkflowDestNode.Context.Conditions.PlainConditions, sdk.WorkflowNodeCondition{
					Variable: "cds.manual",
					Value:    "true",
					Operator: "eq",
				})
			}

			// is sub App
			if childPip.Application.ID != 0 && childPip.Application.ID != appID {
				childPip.Application.WorkflowMigration = STATUS_CLEANING
				childPip.Application.ProjectID = p.ID
				if errA := application.Update(db, store, &childPip.Application, u); errA != nil {
					return nil, sdk.WrapError(errA, "Cannot update subapplication %s", childPip.Application.Name)
				}
			}

			// Add trigger
			newNode.Triggers = append(newNode.Triggers, t)
		}

	}

	return newNode, nil
}
<commit_msg>fix (engine): compute default payload (#2950)<commit_after>package migrate

import (
	"database/sql"
	"fmt"
	"strings"

	"github.com/go-gorp/gorp"

	"github.com/ovh/cds/engine/api/application"
	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/engine/api/permission"
	"github.com/ovh/cds/engine/api/pipeline"
	"github.com/ovh/cds/engine/api/workflow"
	"github.com/ovh/cds/sdk"
)

const (
	STATUS_START    = "STARTED"
	STATUS_CLEANING = "CLEANING"
	STATUS_DONE     = "DONE"
)

// ToWorkflow migrates old workflow to new workflow
func ToWorkflow(db gorp.SqlExecutor, store cache.Store, cdTree []sdk.CDPipeline, proj *sdk.Project, u *sdk.User, force, disablePrefix, withCurrentVersion, withRepositoryWebHook bool) ([]sdk.Workflow, error) {
	workflows := make([]sdk.Workflow, len(cdTree))
	for i := range cdTree {
		oldW := cdTree[i]
		name := oldW.Application.Name
		if !disablePrefix {
			name = "w" + name
		}

		if len(cdTree) > 1 {
			name = fmt.Sprintf("%s_%d", name, i)
		}
		newW := sdk.Workflow{
			Name:       name,
			ProjectID:  proj.ID,
			ProjectKey: proj.Key,
		}

		if err := addGroupOnWorkflow(db, &newW, &oldW.Application); err != nil {
			return nil, sdk.WrapError(err, "MigrateToWorkflow")
		}

		currentApplicationID := oldW.Application.ID

		n, err := migratePipeline(db, store, proj, oldW, currentApplicationID, u)
		if err != nil {
			return nil, sdk.WrapError(err, "MigrateToWorkflow migratePipeline>")
		}
		newW.Root = n

		if force {
			w, err := workflow.Load(db, store, proj, newW.Name, u, workflow.LoadOptions{})
			if err == nil {
				if errD := workflow.Delete(db, store, proj, w, u); errD != nil {
					return nil, sdk.WrapError(errD, "MigrateToWorkflow workflow.Load>")
				}
			}
		}

		if withRepositoryWebHook && newW.Root.Context.Application != nil && newW.Root.Context.Application.VCSServer != "" && newW.Root.Context.Application.RepositoryFullname != "" {
			h := &sdk.WorkflowNodeHook{}
			m, err := workflow.LoadHookModelByName(db, sdk.RepositoryWebHookModelName)
			if err != nil {
				return nil, sdk.WrapError(err, "migratePipeline> hook %s not found", h.WorkflowHookModel.Name)
			}
			h.WorkflowHookModel = *m
			h.WorkflowHookModelID = m.ID
			h.Config = make(map[string]sdk.WorkflowNodeHookConfigValue, len(m.DefaultConfig))
			for k, v := range m.DefaultConfig {
				if _, has := h.Config[k]; !has {
					h.Config[k] = v
				}
			}
			newW.Root.Hooks = []sdk.WorkflowNodeHook{*h}

			if newW.Root != nil && newW.Root.Context != nil && (newW.Root.Context.Application != nil || newW.Root.Context.ApplicationID != 0) {
				var err error
				if newW.Root.Context.DefaultPayload, err = workflow.DefaultPayload(db, store, proj, u, &newW); err != nil {
					return nil, sdk.WrapError(err, "migratePipeline> error compute defaut payload: %s", err)
				}
			}

			if errW := workflow.Insert(db, store, &newW, proj, u); errW != nil {
				return nil, sdk.WrapError(errW, "MigrateToWorkflow workflow.Insert>")
			}

			oldW, errO := workflow.Load(db, store, proj, newW.Name, u, workflow.LoadOptions{})
			if errO != nil {
				return nil, sdk.WrapError(errO, "migratePipeline> Unable to load old workflow")
			}

			newW.ID = oldW.ID

			// then register (node is updated in HookRegistration)
			if errHr := workflow.HookRegistration(db, store, nil, *oldW, proj); errHr != nil {
				return nil, sdk.WrapError(errHr, "migratePipeline> Cannot register hook 2")
			}
		} else {
			if errW := workflow.Insert(db, store, &newW, proj, u); errW != nil {
				return nil, sdk.WrapError(errW, "MigrateToWorkflow workflow.Insert>")
			}
		}

		if withCurrentVersion {
			opts := []pipeline.ExecOptionFunc{
				pipeline.LoadPipelineBuildOpts.WithStatus(sdk.StatusSuccess.String()),
			}
			pbs, errPB := pipeline.LoadPipelineBuildsByApplicationAndPipeline(db, oldW.Application.ID, oldW.Pipeline.ID, oldW.Environment.ID, 1, opts...)
			if errPB != nil && errPB != sql.ErrNoRows {
				return nil, sdk.WrapError(err, "migratePipeline> Cannot load pipeline")
			}
			if len(pbs) == 1 {
				if err := workflow.InsertRunNum(db, &newW, pbs[0].Version); err != nil {
					return nil, sdk.WrapError(err, "migratePipeline> Cannot set the version %d", pbs[0].Version)
				}
			}
		}

		for _, g := range newW.Groups {
			if err := workflow.AddGroup(db, &newW, g); err != nil {
				return nil, sdk.WrapError(err, "MigrateToWorkflow> Cannot add group")
			}
		}
		workflows[i] = newW
	}
	return workflows, nil
}

func addGroupOnWorkflow(db gorp.SqlExecutor, w *sdk.Workflow, app *sdk.Application) error {
	if err := application.LoadGroupByApplication(db, app); err != nil {
		return sdk.WrapError(err, "addGroupOnWorkflow> error while LoadGroupByApplication on application %s", app.ID)
	}

	for _, ag := range app.ApplicationGroups {
		if ag.Permission == permission.PermissionReadWriteExecute || ag.Permission == permission.PermissionReadExecute {
			w.Groups = append(w.Groups, ag)
		}
	}
	return nil
}

func migratePipeline(db gorp.SqlExecutor, store cache.Store, p *sdk.Project, oldPipeline sdk.CDPipeline, appID int64, u *sdk.User) (*sdk.WorkflowNode, error) {
	newNode := &sdk.WorkflowNode{
		PipelineID: oldPipeline.Pipeline.ID,
		Context:    &sdk.WorkflowNodeContext{},
	}

	// Check if pipeline use application & env variable
	pip, err := pipeline.LoadPipelineByID(db, oldPipeline.Pipeline.ID, true)
	if err != nil {
		return nil, sdk.WrapError(err, "migratePipeline> Cannot load pipeline")
	}
	foundApp := false
bigloop:
	for _, s := range pip.Stages {
		for _, pre := range s.Prerequisites {
			if strings.Contains(pre.ExpectedValue, "cds.app") {
				foundApp = true
				break bigloop
			}
		}

		for _, j := range s.Jobs {
			for _, r := range j.Action.Requirements {
				if strings.Contains(r.Value, "cds.app") || strings.Contains(r.Value, "git.") {
					foundApp = true
					break bigloop
				}
			}
			for _, step := range j.Action.Actions {
				for _, param := range step.Parameters {
					if strings.Contains(param.Value, "cds.app") || strings.Contains(param.Value, "git.") {
						foundApp = true
						break bigloop
					}
				}
			}
		}
	}
	if foundApp {
		app, err := application.LoadByName(db, store, p.Key, oldPipeline.Application.Name, u, application.LoadOptions.WithClearDeploymentStrategies)
		if err != nil {
			return nil, sdk.WrapError(err, "migratePipeline> Cannot load application")
		}
		newNode.Context.Application = app
	}

	if oldPipeline.Environment.ID != 0 && oldPipeline.Environment.ID != sdk.DefaultEnv.ID {
		newNode.Context.Environment = &oldPipeline.Environment
	}

	// Add trigger
	if len(oldPipeline.SubPipelines) > 0 {
		for _, childPip := range oldPipeline.SubPipelines {
			// Create new trigger
			t := sdk.WorkflowNodeTrigger{}

			// Migrate child pipeline
			n, err := migratePipeline(db, store, p, childPip, appID, u)
			if err != nil {
				return nil, err
			}

			// Migrate pipeline parameter
			n.Context.DefaultPipelineParameters = childPip.Trigger.Parameters

			t.WorkflowDestNode = *n

			// Add Condition on trigger
			for _, c := range childPip.Trigger.Prerequisites {
				t.WorkflowDestNode.Context.Conditions.PlainConditions = append(t.WorkflowDestNode.Context.Conditions.PlainConditions, sdk.WorkflowNodeCondition{
					Variable: c.Parameter,
					Value:    c.ExpectedValue,
					Operator: sdk.WorkflowConditionsOperatorRegex,
				})
			}
			t.WorkflowDestNode.Context.Conditions.PlainConditions = append(t.WorkflowDestNode.Context.Conditions.PlainConditions, sdk.WorkflowNodeCondition{
				Variable: "cds.status",
				Value:    "Success",
				Operator: "eq",
			})
			if childPip.Trigger.Manual {
				t.WorkflowDestNode.Context.Conditions.PlainConditions = append(t.WorkflowDestNode.Context.Conditions.PlainConditions, sdk.WorkflowNodeCondition{
					Variable: "cds.manual",
					Value:    "true",
					Operator: "eq",
				})
			}

			// is sub App
			if childPip.Application.ID != 0 && childPip.Application.ID != appID {
				childPip.Application.WorkflowMigration = STATUS_CLEANING
				childPip.Application.ProjectID = p.ID
				if errA := application.Update(db, store, &childPip.Application, u); errA != nil {
					return nil, sdk.WrapError(errA, "Cannot update subapplication %s", childPip.Application.Name)
				}
			}

			// Add trigger
			newNode.Triggers = append(newNode.Triggers, t)
		}

	}

	return newNode, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package memfs

import (
	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/timeutil"
)

type memFS struct {
	fuseutil.NotImplementedFileSystem

	/////////////////////////
	// Dependencies
	/////////////////////////

	clock timeutil.Clock

	/////////////////////////
	// Mutable state
	/////////////////////////

	mu syncutil.InvariantMutex

	// The collection of all inodes that have ever been created, indexed by inode
	// ID. Some inodes are not in use if they have been unlinked, and no inode
	// with ID less than fuse.RootInodeID is ever used.
	//
	// INVARIANT: len(inodes) > fuse.RootInodeID
	// INVARIANT: For all i < fuse.RootInodeID, inodes[i].impl == nil
	// INVARIANT: inodes[fuse.RootInodeID].impl is of type *memDir
	inodes []inode // GUARDED_BY(mu)

	// A list of inode IDs within inodes available for reuse, not including the
	// reserved IDs less than fuse.RootInodeID.
	//
	// INVARIANT: This is all and only indices i of inodes such that i >
	// fuse.RootInodeID and inodes[i].impl == nil
	freeInodes []fuse.InodeID // GUARDED_BY(mu)
}

// Create a file system that stores data and metadata in memory.
func NewMemFS(
	clock timeutil.Clock) (fs fuse.FileSystem) {
	fs = &memFS{
		clock: clock,
	}

	fs.(*memFS).mu = syncutil.NewInvariantMutex(fs.(*memFS).checkInvariants)
	return
}

func (fs *memFS) checkInvariants() {
	panic("TODO")
}
<commit_msg>Implemented Init.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package memfs

import (
	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/timeutil"
	"golang.org/x/net/context"
)

type memFS struct {
	fuseutil.NotImplementedFileSystem

	/////////////////////////
	// Dependencies
	/////////////////////////

	clock timeutil.Clock

	/////////////////////////
	// Mutable state
	/////////////////////////

	mu syncutil.InvariantMutex

	// The collection of all inodes that have ever been created, indexed by inode
	// ID. Some inodes are not in use if they have been unlinked, and no inode
	// with ID less than fuse.RootInodeID is ever used.
	//
	// INVARIANT: len(inodes) > fuse.RootInodeID
	// INVARIANT: For all i < fuse.RootInodeID, inodes[i].impl == nil
	// INVARIANT: inodes[fuse.RootInodeID].impl is of type *memDir
	inodes []inode // GUARDED_BY(mu)

	// A list of inode IDs within inodes available for reuse, not including the
	// reserved IDs less than fuse.RootInodeID.
	//
	// INVARIANT: This is all and only indices i of inodes such that i >
	// fuse.RootInodeID and inodes[i].impl == nil
	freeInodes []fuse.InodeID // GUARDED_BY(mu)
}

// Create a file system that stores data and metadata in memory.
func NewMemFS(
	clock timeutil.Clock) (fs fuse.FileSystem) {
	fs = &memFS{
		clock: clock,
	}

	fs.(*memFS).mu = syncutil.NewInvariantMutex(fs.(*memFS).checkInvariants)
	return
}

func (fs *memFS) checkInvariants() {
	panic("TODO")
}

func (fs *memFS) Init(
	ctx context.Context,
	req *fuse.InitRequest) (resp *fuse.InitResponse, err error) {
	resp = &fuse.InitResponse{}
	return
}
<|endoftext|>


<commit_before>package checker

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"mime/multipart"
	"net/http"
	"net/http/cookiejar"
	"net/textproto"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/catatsuy/private-isu/benchmarker/score"
)

const (
	UserAgent = "benchmarker"
)

var (
	targetHost *url.URL
)

type Session struct {
	Client    *http.Client
	Transport *http.Transport

	logger *log.Logger
}

func NewSession() *Session {
	w := &Session{
		logger: log.New(os.Stdout, "", 0),
	}

	jar, _ := cookiejar.New(&cookiejar.Options{})
	w.Transport = &http.Transport{}
	w.Client = &http.Client{
		Transport: w.Transport,
		Jar:       jar,
		Timeout:   time.Duration(10) * time.Second,
	}

	return w
}

func SetTargetHost(host string) (*url.URL, error) {
	parsedURL, err := urlParse(host)
	if err != nil {
		return nil, err
	}

	targetHost = parsedURL
	return targetHost, nil
}

func urlParse(ref string) (*url.URL, error) {
	u, err := url.Parse(ref)
	if err != nil {
		return nil, err
	}

	if u.Host == "" {
		return nil, fmt.Errorf("host is empty")
	}

	if u.Scheme == "" {
		u.Scheme = "http"
	}

	return &url.URL{
		Scheme: u.Scheme,
		Host:   u.Host,
	}, nil
}

func (s *Session) NewRequest(method, uri string, body io.Reader) (*http.Request, error) {
	parsedURL, err := url.Parse(uri)

	if err != nil {
		return nil, err
	}

	if parsedURL.Scheme == "" {
		parsedURL.Scheme = targetHost.Scheme
	}

	if parsedURL.Host == "" {
		parsedURL.Host = targetHost.Host
	}

	req, err := http.NewRequest(method, parsedURL.String(), body)

	if err != nil {
		return nil, err
	}

	return req, err
}

func escapeQuotes(s string) string {
	return strings.NewReplacer("\\", "\\\\", `"`, "\\\"").Replace(s)
}

func (s *Session) NewFileUploadRequest(uri string, params map[string]string, paramName string, asset *Asset) (*http.Request, error) {
	file, err := os.Open(asset.Path)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	// part, err := writer.CreateFormFile(paramName, filepath.Base(path))
	// Content-Typeを指定できないので該当コードから実装
	h := make(textproto.MIMEHeader)
	h.Set("Content-Disposition",
		fmt.Sprintf(`form-data; name="%s"; filename="%s"`,
			escapeQuotes(paramName), escapeQuotes(filepath.Base(asset.Path))))
	h.Set("Content-Type", asset.Type)
	part, err := writer.CreatePart(h)

	if err != nil {
		return nil, err
	}
	_, err = io.Copy(part, file)
	if err != nil {
		return nil, err
	}

	for key, val := range params {
		_ = writer.WriteField(key, val)
	}

	err = writer.Close()
	if err != nil {
		return nil, err
	}

	parsedURL := &url.URL{
		Scheme: targetHost.Scheme,
		Host:   targetHost.Host,
		Path:   uri,
	}

	req, err := http.NewRequest("POST", parsedURL.String(), body)
	if err == nil {
		req.Header.Add("Content-Type", writer.FormDataContentType())
	} else {
		return nil, err
	}

	return req, err
}

func (s *Session) SendRequest(req *http.Request) (*http.Response, error) {
	req.Header.Set("User-Agent", UserAgent)

	return s.Client.Do(req)
}

func (s *Session) Success(point int64) {
	score.GetInstance().SetScore(point)
}

func (s *Session) Fail(point int64, req *http.Request, err error) error {
	score.GetInstance().SetFails(point)
	if req != nil {
		err = fmt.Errorf("%s (%s %s)", err, req.Method, req.URL.Path)
	}

	score.GetFailErrorsInstance().Append(err)
	return err
}
<commit_msg>Revert "Revert "リダイレクトする場合はContent-Typeヘッダを消す""<commit_after>package checker

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"mime/multipart"
	"net/http"
	"net/http/cookiejar"
	"net/textproto"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/catatsuy/private-isu/benchmarker/score"
)

const (
	UserAgent = "benchmarker"
)

var (
	targetHost *url.URL
)

type Session struct {
	Client    *http.Client
	Transport *http.Transport

	logger *log.Logger
}

func NewSession() *Session {
	w := &Session{
		logger: log.New(os.Stdout, "", 0),
	}

	jar, _ := cookiejar.New(&cookiejar.Options{})
	w.Transport = &http.Transport{}
	w.Client = &http.Client{
		Transport: w.Transport,
		Jar:       jar,
		Timeout:   time.Duration(10) * time.Second,
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			if req.Method == http.MethodGet {
				// POST からのリダイレクト GET で content-type: mutilpart/form-data を付けると
				// Rack が EOFError で死ぬのでヘッダを消しておく
				req.Header.Del("Content-Type")
			}
			return nil
		},
	}

	return w
}

func SetTargetHost(host string) (*url.URL, error) {
	parsedURL, err := urlParse(host)
	if err != nil {
		return nil, err
	}

	targetHost = parsedURL
	return targetHost, nil
}

func urlParse(ref string) (*url.URL, error) {
	u, err := url.Parse(ref)
	if err != nil {
		return nil, err
	}

	if u.Host == "" {
		return nil, fmt.Errorf("host is empty")
	}

	if u.Scheme == "" {
		u.Scheme = "http"
	}

	return &url.URL{
		Scheme: u.Scheme,
		Host:   u.Host,
	}, nil
}

func (s *Session) NewRequest(method, uri string, body io.Reader) (*http.Request, error) {
	parsedURL, err := url.Parse(uri)

	if err != nil {
		return nil, err
	}

	if parsedURL.Scheme == "" {
		parsedURL.Scheme = targetHost.Scheme
	}

	if parsedURL.Host == "" {
		parsedURL.Host = targetHost.Host
	}

	req, err := http.NewRequest(method, parsedURL.String(), body)

	if err != nil {
		return nil, err
	}

	return req, err
}

func escapeQuotes(s string) string {
	return strings.NewReplacer("\\", "\\\\", `"`, "\\\"").Replace(s)
}

func (s *Session) NewFileUploadRequest(uri string, params map[string]string, paramName string, asset *Asset) (*http.Request, error) {
	file, err := os.Open(asset.Path)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	// part, err := writer.CreateFormFile(paramName, filepath.Base(path))
	// Content-Typeを指定できないので該当コードから実装
	h := make(textproto.MIMEHeader)
	h.Set("Content-Disposition",
		fmt.Sprintf(`form-data; name="%s"; filename="%s"`,
			escapeQuotes(paramName), escapeQuotes(filepath.Base(asset.Path))))
	h.Set("Content-Type", asset.Type)
	part, err := writer.CreatePart(h)

	if err != nil {
		return nil, err
	}
	_, err = io.Copy(part, file)
	if err != nil {
		return nil, err
	}

	for key, val := range params {
		_ = writer.WriteField(key, val)
	}

	err = writer.Close()
	if err != nil {
		return nil, err
	}

	parsedURL := &url.URL{
		Scheme: targetHost.Scheme,
		Host:   targetHost.Host,
		Path:   uri,
	}

	req, err := http.NewRequest("POST", parsedURL.String(), body)
	if err == nil {
		req.Header.Add("Content-Type", writer.FormDataContentType())
	} else {
		return nil, err
	}

	return req, err
}

func (s *Session) SendRequest(req *http.Request) (*http.Response, error) {
	req.Header.Set("User-Agent", UserAgent)

	return s.Client.Do(req)
}

func (s *Session) Success(point int64) {
	score.GetInstance().SetScore(point)
}

func (s *Session) Fail(point int64, req *http.Request, err error) error {
	score.GetInstance().SetFails(point)
	if req != nil {
		err = fmt.Errorf("%s (%s %s)", err, req.Method, req.URL.Path)
	}

	score.GetFailErrorsInstance().Append(err)
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/ancientlore/vbscribble/vblexer"
	"github.com/ancientlore/vbscribble/vbscanner"
)

var (
	respWrite = flag.Bool("rw", false, "Use Response.Write formatting")
)

func main() {
	flag.Parse()

	for _, pattern := range flag.Args() {
		files, err := filepath.Glob(pattern)
		if err != nil {
			log.Fatal(err)
		}
		for _, f := range files {
			fi, err := os.Stat(f)
			if err != nil {
				log.Fatal(err)
			}
			if !fi.IsDir() {
				fmt.Fprintln(os.Stderr, "\n*** ", f, " ***")
				fil, err := os.Open(f)
				if err != nil {
					log.Fatal(err)
				}
				func(fil io.Reader, f string) {
					var lex vblexer.Lex
					defer func() {
						if r := recover(); r != nil {
							log.Print("PARSE ERROR ", f, ":", lex.Line, ": ", r)
						}
					}()
					lex.Init(fil, f, vbscanner.HTML_MODE)
					aft := ""
					tabs := 0
					startLine := true
					paren := false
					prevK := vblexer.EOF
					var prevT interface{}
					needStarter := false
					if *respWrite {
						fmt.Print("<%")
					}
					for k, t, v := lex.Lex(); k != vblexer.EOF; k, t, v = lex.Lex() {
						if needStarter {
							fmt.Print("<%")
							needStarter = false
						}
						if startLine {
							if k == vblexer.STATEMENT {
								if t == "End" {
									pv := v
									k, t, v = lex.Lex()
									if k != vblexer.EOF {
										t = "End " + t.(string)
										v = pv + " " + v
										tabs--
										if t == "End Select" {
											tabs--
										}
									}
								}
								switch t {
								case "Else", "ElseIf", "Case", "Wend":
									tabs--
								}
							}
							if tabs < 0 {
								tabs = 0
							}
							if prevK != vblexer.HTML {
								fmt.Print(strings.Repeat("\t", tabs))
							}
							startLine = false
							aft = ""
							paren = false
						} else {
							aft = " "
						}
						if paren {
							paren = false
							aft = ""
						}
						if prevK == vblexer.STATEMENT && prevT == "Then" {
							if k != vblexer.EOL && k != vblexer.HTML {
								tabs--
							}
						}
						switch k {
						case vblexer.EOF:
						case vblexer.STATEMENT:
							fmt.Print(aft)
							fmt.Print(t)
							switch t {
							case "If", "Function", "Sub", "Class", "Select", "Property":
								if !(prevK == vblexer.STATEMENT && prevT == "Exit") {
									tabs++
								}
							case "Else":
								if !(prevK == vblexer.STATEMENT && prevT == "Case") {
									tabs++
								}
							case "ElseIf", "Case", "Do", "While":
								tabs++
							}
						case vblexer.FUNCTION:
							fmt.Print(aft)
							fmt.Print(t)
						case vblexer.KEYWORD, vblexer.KEYWORD_BOOL:
							fmt.Print(aft)
							fmt.Print(t)
						case vblexer.COLOR_CONSTANT, vblexer.COMPARE_CONSTANT, vblexer.DATE_CONSTANT, vblexer.DATEFORMAT_CONSTANT, vblexer.MISC_CONSTANT, vblexer.MSGBOX_CONSTANT, vblexer.STRING_CONSTANT, vblexer.TRISTATE_CONSTANT, vblexer.VARTYPE_CONSTANT:
							fmt.Print(aft)
							fmt.Print(t)
						case vblexer.IDENTIFIER:
							fmt.Print(aft)
							fmt.Print(t)
						case vblexer.STRING:
							fmt.Print(aft)
							fmt.Printf("\"%s\"", strings.Replace(v, "\"", "\"\"", -1))
						case vblexer.INT:
							fmt.Print(aft)
							fmt.Print(v)
						case vblexer.FLOAT:
							fmt.Print(aft)
							fmt.Print(v)
						case vblexer.DATE:
							fmt.Print(aft)
							fmt.Print("#", v, "#")
						case vblexer.COMMENT:
							fmt.Print(aft)
							fmt.Printf("' %s", t)
						case vblexer.HTML:
							if *respWrite {
								lines := strings.Split(strings.Replace(v, "\r", "", -1), "\n")
								for index, line := range lines {
									if index == 0 {
										fmt.Println()
										fmt.Print(aft)
										fmt.Print("Response.Write ")
									} else {
										fmt.Print(strings.Repeat("\t", tabs+1))
										fmt.Print("& vbCrLf & ")
									}
									fmt.Printf("\"%s\"\n", strings.Replace(line, "\"", "\"\"", -1))
								}
							} else {
								if prevK != vblexer.EOF {
									fmt.Print(aft)
									fmt.Print("%>")
								}
								fmt.Print(v)
								needStarter = true
							}
							startLine = true
						case vblexer.CHAR:
							fmt.Print(t)
							if t == "(" {
								paren = true
							}
						case vblexer.EOL:
							if t == ":" {
								fmt.Print(aft)
								fmt.Print(t)
								fmt.Print(" ")
							} else {
								fmt.Println()
							}
							startLine = true
						case vblexer.OP:
							fmt.Print(aft)
							fmt.Print(t)
						case vblexer.CONTINUATION:
							fmt.Print(aft)
							fmt.Print(t)
						default:
							panic("Unexpected token type")
						}
						prevK = k
						prevT = t
					}
					if *respWrite {
						fmt.Println("%>")
					}
				}(fil, f)
				fil.Close()
			}
		}
	}
}
<commit_msg>Indent after continuation<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/ancientlore/vbscribble/vblexer"
	"github.com/ancientlore/vbscribble/vbscanner"
)

var (
	respWrite = flag.Bool("rw", false, "Use Response.Write formatting")
)

func main() {
	flag.Parse()

	for _, pattern := range flag.Args() {
		files, err := filepath.Glob(pattern)
		if err != nil {
			log.Fatal(err)
		}
		for _, f := range files {
			fi, err := os.Stat(f)
			if err != nil {
				log.Fatal(err)
			}
			if !fi.IsDir() {
				fmt.Fprintln(os.Stderr, "\n*** ", f, " ***")
				fil, err := os.Open(f)
				if err != nil {
					log.Fatal(err)
				}
				func(fil io.Reader, f string) {
					var lex vblexer.Lex
					defer func() {
						if r := recover(); r != nil {
							log.Print("PARSE ERROR ", f, ":", lex.Line, ": ", r)
						}
					}()
					lex.Init(fil, f, vbscanner.HTML_MODE)
					aft := ""
					tabs := 0
					startLine := true
					paren := false
					prevK := vblexer.EOF
					var prevT interface{}
					needStarter := false
					remTabAfterEOL := false
					if *respWrite {
						fmt.Print("<%")
					}
					for k, t, v := lex.Lex(); k != vblexer.EOF; k, t, v = lex.Lex() {
						if needStarter {
							fmt.Print("<%")
							needStarter = false
						}
						if startLine {
							if k == vblexer.STATEMENT {
								if t == "End" {
									pv := v
									k, t, v = lex.Lex()
									if k != vblexer.EOF {
										t = "End " + t.(string)
										v = pv + " " + v
										tabs--
										if t == "End Select" {
											tabs--
										}
									}
								}
								switch t {
								case "Else", "ElseIf", "Case", "Wend":
									tabs--
								}
							}
							if tabs < 0 {
								tabs = 0
							}
							if prevK != vblexer.HTML {
								fmt.Print(strings.Repeat("\t", tabs))
							}
							if remTabAfterEOL {
								remTabAfterEOL = false
								tabs--
							}
							startLine = false
							aft = ""
							paren = false
						} else {
							aft = " "
						}
						if paren {
							paren = false
							aft = ""
						}
						if prevK == vblexer.STATEMENT && prevT == "Then" {
							if k != vblexer.EOL && k != vblexer.HTML {
								tabs--
							}
						}
						switch k {
						case vblexer.EOF:
						case vblexer.STATEMENT:
							fmt.Print(aft)
							fmt.Print(t)
							switch t {
							case "If", "Function", "Sub", "Class", "Select", "Property":
								if !(prevK == vblexer.STATEMENT && prevT == "Exit") {
									tabs++
								}
							case "Else":
								if !(prevK == vblexer.STATEMENT && prevT == "Case") {
									tabs++
								}
							case "ElseIf", "Case", "Do", "While":
								tabs++
							}
						case vblexer.FUNCTION:
							fmt.Print(aft)
							fmt.Print(t)
						case vblexer.KEYWORD, vblexer.KEYWORD_BOOL:
							fmt.Print(aft)
							fmt.Print(t)
						case vblexer.COLOR_CONSTANT, vblexer.COMPARE_CONSTANT, vblexer.DATE_CONSTANT, vblexer.DATEFORMAT_CONSTANT, vblexer.MISC_CONSTANT, vblexer.MSGBOX_CONSTANT, vblexer.STRING_CONSTANT, vblexer.TRISTATE_CONSTANT, vblexer.VARTYPE_CONSTANT:
							fmt.Print(aft)
							fmt.Print(t)
						case vblexer.IDENTIFIER:
							fmt.Print(aft)
							fmt.Print(t)
						case vblexer.STRING:
							fmt.Print(aft)
							fmt.Printf("\"%s\"", strings.Replace(v, "\"", "\"\"", -1))
						case vblexer.INT:
							fmt.Print(aft)
							fmt.Print(v)
						case vblexer.FLOAT:
							fmt.Print(aft)
							fmt.Print(v)
						case vblexer.DATE:
							fmt.Print(aft)
							fmt.Print("#", v, "#")
						case vblexer.COMMENT:
							fmt.Print(aft)
							fmt.Printf("' %s", t)
						case vblexer.HTML:
							if *respWrite {
								lines := strings.Split(strings.Replace(v, "\r", "", -1), "\n")
								for index, line := range lines {
									if index == 0 {
										fmt.Println()
										fmt.Print(aft)
										fmt.Print("Response.Write ")
									} else {
										fmt.Print(strings.Repeat("\t", tabs+1))
										fmt.Print("& vbCrLf & ")
									}
									fmt.Printf("\"%s\"\n", strings.Replace(line, "\"", "\"\"", -1))
								}
							} else {
								if prevK != vblexer.EOF {
									fmt.Print(aft)
									fmt.Print("%>")
								}
								fmt.Print(v)
								needStarter = true
							}
							startLine = true
						case vblexer.CHAR:
							fmt.Print(t)
							if t == "(" {
								paren = true
							}
						case vblexer.EOL:
							if t == ":" {
								fmt.Print(aft)
								fmt.Print(t)
								fmt.Print(" ")
							} else {
								fmt.Println()
							}
							startLine = true
						case vblexer.OP:
							fmt.Print(aft)
							fmt.Print(t)
						case vblexer.CONTINUATION:
							fmt.Print(aft)
							fmt.Print(t)
							tabs++
							remTabAfterEOL = true
						default:
							panic("Unexpected token type")
						}
						prevK = k
						prevT = t
					}
					if *respWrite {
						fmt.Println("%>")
					}
				}(fil, f)
				fil.Close()
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"runtime/pprof"

	"github.com/dim13/golyb"
	"github.com/dim13/golyb/dynamic"
	"github.com/dim13/golyb/optimize"
	"github.com/dim13/golyb/short"
	"github.com/dim13/golyb/static"
)

func main() {
	var (
		file    = flag.String("file", "", "Source file (required)")
		in      = flag.String("in", "", "Input file")
		out     = flag.String("out", "", "Output file or /dev/null")
		profile = flag.String("profile", "", "Write CPU profile to file")
		tape    = flag.String("tape", "static", "Tape type: static, dynamic or short")
		dump    = flag.Bool("dump", false, "Dump AST and terminate")
		noop    = flag.Bool("noop", false, "Disable optimization")
		show    = flag.Bool("show", false, "Dump tape cells")
		store   = map[string]func(io.ReadWriter) golyb.Storage{
			"static":  static.NewTape,
			"dynamic": dynamic.NewTape,
			"short":   short.NewTape,
		}
	)
	flag.Parse()

	defer func() {
		if r := recover(); r != nil {
			log.Fatal(r)
		}
	}()

	if *profile != "" {
		f, err := os.Create(*profile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *file == "" {
		flag.Usage()
		return
	}

	program, err := golyb.ParseFile(*file)
	if err != nil {
		log.Fatal(err)
	}

	if !*noop {
		program = optimize.All(program)
	}

	if *dump {
		fmt.Print(program)
		return
	}

	o, err := output(*out, *in)
	if err != nil {
		log.Fatal(err)
	}

	storage, ok := store[*tape]
	if !ok {
		flag.Usage()
		return
	}

	st := storage(o)
	program.Execute(st)

	if *show {
		fmt.Println(st)
	}
}
<commit_msg>Move flags back outside<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"runtime/pprof"

	"github.com/dim13/golyb"
	"github.com/dim13/golyb/dynamic"
	"github.com/dim13/golyb/optimize"
	"github.com/dim13/golyb/short"
	"github.com/dim13/golyb/static"
)

var (
	file    = flag.String("file", "", "Source file (required)")
	in      = flag.String("in", "", "Input file")
	out     = flag.String("out", "", "Output file or /dev/null")
	profile = flag.String("profile", "", "Write CPU profile to file")
	tape    = flag.String("tape", "static", "Tape type: static, dynamic or short")
	dump    = flag.Bool("dump", false, "Dump AST and terminate")
	noop    = flag.Bool("noop", false, "Disable optimization")
	show    = flag.Bool("show", false, "Dump tape cells")
	store   = map[string]func(io.ReadWriter) golyb.Storage{
		"static":  static.NewTape,
		"dynamic": dynamic.NewTape,
		"short":   short.NewTape,
	}
)

func main() {
	flag.Parse()

	defer func() {
		if r := recover(); r != nil {
			log.Fatal(r)
		}
	}()

	if *profile != "" {
		f, err := os.Create(*profile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *file == "" {
		flag.Usage()
		return
	}

	program, err := golyb.ParseFile(*file)
	if err != nil {
		log.Fatal(err)
	}

	if !*noop {
		program = optimize.All(program)
	}

	if *dump {
		fmt.Print(program)
		return
	}

	o, err := output(*out, *in)
	if err != nil {
		log.Fatal(err)
	}

	storage, ok := store[*tape]
	if !ok {
		flag.Usage()
		return
	}

	st := storage(o)
	program.Execute(st)

	if *show {
		fmt.Println(st)
	}
}
<|endoftext|>


<commit_before><commit_msg>Update puffs usage message.<commit_after><|endoftext|>


<commit_before>package metainfo

import (
	"encoding/base32"
	"encoding/hex"
	"fmt"
	"net/url"
	"strings"
)

// Magnet link components.
type Magnet struct {
	InfoHash    Hash
	Trackers    []string
	DisplayName string
}

const xtPrefix = "urn:btih:"

func (m Magnet) String() string {
	// net.URL likes to assume //, and encodes ':' on us, so we do most of
	// this manually.
	ret := "magnet:?xt="
	ret += xtPrefix + hex.EncodeToString(m.InfoHash[:])
	if m.DisplayName != "" {
		ret += "&dn=" + url.QueryEscape(m.DisplayName)
	}
	for _, tr := range m.Trackers {
		ret += "&tr=" + url.QueryEscape(tr)
	}
	return ret
}

// ParseMagnetURI parses Magnet-formatted URIs into a Magnet instance
func ParseMagnetURI(uri string) (m Magnet, err error) {
	u, err := url.Parse(uri)
	if err != nil {
		err = fmt.Errorf("error parsing uri: %s", err)
		return
	}
	if u.Scheme != "magnet" {
		err = fmt.Errorf("unexpected scheme: %q", u.Scheme)
		return
	}
	xt := u.Query().Get("xt")
	if !strings.HasPrefix(xt, xtPrefix) {
		err = fmt.Errorf("bad xt parameter")
		return
	}
	infoHash := xt[len(xtPrefix):]

	// BTIH hash can be in HEX or BASE32 encoding
	// will assign appropriate func judging from symbol length
	var decode func(dst, src []byte) (int, error)
	switch len(infoHash) {
	case 40:
		decode = hex.Decode
	case 32:
		decode = base32.StdEncoding.Decode
	}

	if decode == nil {
		err = fmt.Errorf("unhandled xt parameter encoding: encoded length %d", len(infoHash))
		return
	}
	n, err := decode(m.InfoHash[:], []byte(infoHash))
	if err != nil {
		err = fmt.Errorf("error decoding xt: %s", err)
		return
	}
	if n != 20 {
		panic(n)
	}
	m.DisplayName = u.Query().Get("dn")
	m.Trackers = u.Query()["tr"]
	return
}
<commit_msg>metainfo: Add Magnet.Params for more open handling<commit_after>package metainfo

import (
	"encoding/base32"
	"encoding/hex"
	"errors"
	"fmt"
	"net/url"
	"strings"
)

// Magnet link components.
type Magnet struct {
	InfoHash    Hash
	Trackers    []string
	DisplayName string
	Params      url.Values
}

const xtPrefix = "urn:btih:"

func (m Magnet) String() string {
	// Deep-copy m.Params
	vs := make(url.Values, len(m.Params)+len(m.Trackers)+2)
	for k, v := range m.Params {
		vs[k] = append([]string(nil), v...)
	}

	vs.Add("xt", xtPrefix+m.InfoHash.HexString())
	for _, tr := range m.Trackers {
		vs.Add("tr", tr)
	}
	if m.DisplayName != "" {
		vs.Add("dn", m.DisplayName)
	}

	return (&url.URL{
		Scheme:   "magnet",
		RawQuery: vs.Encode(),
	}).String()
}

// ParseMagnetURI parses Magnet-formatted URIs into a Magnet instance
func ParseMagnetURI(uri string) (m Magnet, err error) {
	u, err := url.Parse(uri)
	if err != nil {
		err = fmt.Errorf("error parsing uri: %w", err)
		return
	}
	if u.Scheme != "magnet" {
		err = fmt.Errorf("unexpected scheme %q", u.Scheme)
		return
	}
	q := u.Query()
	xt := q.Get("xt")
	m.InfoHash, err = parseInfohash(q.Get("xt"))
	if err != nil {
		err = fmt.Errorf("error parsing infohash %q: %w", xt, err)
		return
	}
	dropFirst(q, "xt")
	m.DisplayName = q.Get("dn")
	dropFirst(q, "dn")
	m.Trackers = q["tr"]
	delete(q, "tr")
	if len(q) == 0 {
		q = nil
	}
	m.Params = q
	return
}

func parseInfohash(xt string) (ih Hash, err error) {
	if !strings.HasPrefix(xt, xtPrefix) {
		err = errors.New("bad xt parameter prefix")
		return
	}
	encoded := xt[len(xtPrefix):]
	decode := func() func(dst, src []byte) (int, error) {
		switch len(encoded) {
		case 40:
			return hex.Decode
		case 32:
			return base32.StdEncoding.Decode
		}
		return nil
	}()
	if decode == nil {
		err = fmt.Errorf("unhandled xt parameter encoding (encoded length %d)", len(encoded))
		return
	}
	n, err := decode(ih[:], []byte(encoded))
	if err != nil {
		err = fmt.Errorf("error decoding xt: %w", err)
		return
	}
	if n != 20 {
		panic(n)
	}
	return
}

func dropFirst(vs url.Values, key string) {
	sl := vs[key]
	switch len(sl) {
	case 0, 1:
		vs.Del(key)
	default:
		vs[key] = sl[1:]
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Peter Mattis (peter@cockroachlabs.com)

// +build acceptance

package acceptance

import (
	"sync/atomic"
	"testing"
	"time"

	"github.com/cockroachdb/cockroach/acceptance/cluster"
	"github.com/cockroachdb/cockroach/client"
	"github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/util/log"
)

// TestSingleKey stresses the transaction retry machinery by starting
// up an N node cluster and running N workers that are all
// incrementing the value associated with a single key.
func TestSingleKey(t *testing.T) {
	runTestOnConfigs(t, testSingleKeyInner)
}

func testSingleKeyInner(t *testing.T, c cluster.Cluster, cfg cluster.TestConfig) {
	num := c.NumNodes()

	// Initialize the value for our test key to zero.
	const key = "test-key"
	db, initDBStopper := makeClient(t, c.ConnString(0))
	defer initDBStopper.Stop()
	if err := db.Put(key, 0); err != nil {
		t.Fatal(err)
	}

	type result struct {
		err        error
		count      int
		maxLatency time.Duration
	}

	resultCh := make(chan result, num)
	deadline := time.Now().Add(cfg.Duration)
	var expected int64

	// Start up num workers each reading and writing the same
	// key. Each worker is configured to talk to a different node in the
	// cluster.
	for i := 0; i < num; i++ {
		db, dbStopper := makeClient(t, c.ConnString(i))
		defer dbStopper.Stop()
		go func() {
			var r result
			for time.Now().Before(deadline) {
				start := time.Now()
				pErr := db.Txn(func(txn *client.Txn) *roachpb.Error {
					r, pErr := txn.Get(key)
					if pErr != nil {
						return pErr
					}
					b := txn.NewBatch()
					b.Put(key, r.ValueInt()+1)
					return txn.CommitInBatch(b)
				})
				if pErr != nil {
					resultCh <- result{err: pErr.GoError()}
					return
				}
				atomic.AddInt64(&expected, 1)
				r.count++
				latency := time.Since(start)
				if r.maxLatency < latency {
					r.maxLatency = latency
				}
			}
			resultCh <- r
		}()
	}

	// Verify that none of the workers encountered an error.
	var results []result
	for len(results) < num {
		select {
		case <-stopper:
			t.Fatalf("interrupted")
		case r := <-resultCh:
			if r.err != nil {
				t.Fatal(r.err)
			}
			results = append(results, r)
		case <-time.After(1 * time.Second):
			// Periodically print out progress so that we know the test is still
			// running.
			log.Infof("%d", atomic.LoadInt64(&expected))
		}
	}

	// Verify the resulting value stored at the key is what we expect.
	r, err := db.Get(key)
	if err != nil {
		t.Fatal(err)
	}
	v := r.ValueInt()
	if expected != v {
		t.Fatalf("expected %d, but found %d", expected, v)
	}
	var maxLatency []time.Duration
	for _, r := range results {
		maxLatency = append(maxLatency, r.maxLatency)
	}
	log.Infof("%d increments: %s", v, maxLatency)
}
<commit_msg>Add checks to single key test<commit_after>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Peter Mattis (peter@cockroachlabs.com)

// +build acceptance

package acceptance

import (
	"sync/atomic"
	"testing"
	"time"

	"github.com/cockroachdb/cockroach/acceptance/cluster"
	"github.com/cockroachdb/cockroach/client"
	"github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/util/log"
)

// TestSingleKey stresses the transaction retry machinery by starting
// up an N node cluster and running N workers that are all
// incrementing the value associated with a single key.
func TestSingleKey(t *testing.T) {
	runTestOnConfigs(t, testSingleKeyInner)
}

func testSingleKeyInner(t *testing.T, c cluster.Cluster, cfg cluster.TestConfig) {
	num := c.NumNodes()

	// Initialize the value for our test key to zero.
	const key = "test-key"
	db, initDBStopper := makeClient(t, c.ConnString(0))
	defer initDBStopper.Stop()
	if err := db.Put(key, 0); err != nil {
		t.Fatal(err)
	}

	type result struct {
		err        error
		maxLatency time.Duration
	}

	resultCh := make(chan result, num)
	deadline := time.Now().Add(cfg.Duration)
	var expected int64

	// Start up num workers each reading and writing the same
	// key. Each worker is configured to talk to a different node in the
	// cluster.
	for i := 0; i < num; i++ {
		db, dbStopper := makeClient(t, c.ConnString(i))
		defer dbStopper.Stop()
		go func() {
			var r result
			for time.Now().Before(deadline) {
				start := time.Now()
				pErr := db.Txn(func(txn *client.Txn) *roachpb.Error {
					minExp := atomic.LoadInt64(&expected)
					r, pErr := txn.Get(key)
					if pErr != nil {
						return pErr
					}
					b := txn.NewBatch()
					v := r.ValueInt()
					b.Put(key, v+1)
					pErr = txn.CommitInBatch(b)
					// Atomic updates after the fact mean that we should read
					// exp or larger (since concurrent writers might have
					// committed but not yet performed their atomic update).
					if pErr == nil && v < minExp {
						return roachpb.NewErrorf("unexpected read: %d, expected >= %d", v, minExp)
					}
					return pErr
				})
				if pErr != nil {
					resultCh <- result{err: pErr.GoError()}
					return
				}
				atomic.AddInt64(&expected, 1)
				latency := time.Since(start)
				if r.maxLatency < latency {
					r.maxLatency = latency
				}
			}
			resultCh <- r
		}()
	}

	// Verify that none of the workers encountered an error.
	var results []result
	for len(results) < num {
		select {
		case <-stopper:
			t.Fatalf("interrupted")
		case r := <-resultCh:
			if r.err != nil {
				t.Fatal(r.err)
			}
			results = append(results, r)
		case <-time.After(1 * time.Second):
			// Periodically print out progress so that we know the test is still
			// running.
			log.Infof("%d", atomic.LoadInt64(&expected))
		}
	}

	// Verify the resulting value stored at the key is what we expect.
	r, err := db.Get(key)
	if err != nil {
		t.Fatal(err)
	}
	v := r.ValueInt()
	if expected != v {
		t.Fatalf("expected %d, but found %d", expected, v)
	}
	var maxLatency []time.Duration
	for _, r := range results {
		maxLatency = append(maxLatency, r.maxLatency)
	}
	log.Infof("%d increments: %s", v, maxLatency)
}
<|endoftext|>


<commit_before>package metrics

import (
	"fmt"
	"os"

	statsdlib "github.com/CMGS/statsd"
	"github.com/projecteru2/core/types"
	"github.com/projecteru2/core/utils"
	"github.com/prometheus/client_golang/prometheus"
	log "github.com/sirupsen/logrus"
)

const (
	cpuMap           = "core.node.%s.cpu.%s"
	memStats         = "core.node.%s.memory"
	storageStats     = "core.node.%s.storage"
	memUsedStats     = "core.node.%s.memory.used"
	storageUsedStats = "core.node.%s.storage.used"
	cpuUsedStats     = "core.node.%s.cpu.used"
	deployCount      = "core.%s.deploy.count"
)

// Metrics define metrics
type Metrics struct {
	StatsdAddr   string
	Hostname     string
	statsdClient *statsdlib.Client

	MemoryCapacity  *prometheus.GaugeVec
	MemoryUsed      *prometheus.GaugeVec
	StorageCapacity *prometheus.GaugeVec
	StorageUsed     *prometheus.GaugeVec
	CPUMap          *prometheus.GaugeVec
	CPUUsed         *prometheus.GaugeVec
	DeployCount     *prometheus.CounterVec
}

// Lazy connect
func (m *Metrics) checkConn() error {
	if m.statsdClient != nil {
		return nil
	}
	var err error
	// We needn't try to renew/reconnect because of only supporting UDP protocol now
	// We should add an `errorCount` to reconnect when implementing TCP protocol
	if m.statsdClient, err = statsdlib.New(m.StatsdAddr, statsdlib.WithErrorHandler(func(err error) {
		log.Errorf("[statsd] Sending statsd failed: %v", err)
	})); err != nil {
		log.Errorf("[statsd] Connect statsd failed: %v", err)
		return err
	}
	return nil
}

func (m *Metrics) gauge(key string, value float64) error {
	if err := m.checkConn(); err != nil {
		return err
	}
	m.statsdClient.Gauge(key, value)
	return nil
}

func (m *Metrics) count(key string, n int, rate float32) error {
	if err := m.checkConn(); err != nil {
		return err
	}
	m.statsdClient.Count(key, n, rate)
	return nil
}

// SendNodeInfo update node resource capacity
func (m *Metrics) SendNodeInfo(node *types.Node) {
	log.Debugf("[Metrics] Update %s metrics", node.Name)
	nodename := utils.CleanStatsdMetrics(node.Name)
	podname := utils.CleanStatsdMetrics(node.Podname)
	memory := float64(node.MemCap)
	memoryUsed := float64(node.InitMemCap - node.MemCap)
	storage := float64(node.StorageCap)
	storageUsed := float64(node.InitStorageCap - node.StorageCap)
	cpuUsed := node.CPUUsed

	if m.MemoryCapacity != nil {
		m.MemoryCapacity.WithLabelValues(podname, nodename).Set(memory)
	}

	if m.MemoryUsed != nil {
		m.MemoryUsed.WithLabelValues(podname, nodename).Set(memoryUsed)
	}

	if m.StorageCapacity != nil {
		m.StorageCapacity.WithLabelValues(podname, nodename).Set(storage)
	}

	if m.StorageUsed != nil {
		m.StorageUsed.WithLabelValues(podname, nodename).Set(storageUsed)
	}

	if m.CPUUsed != nil {
		m.CPUUsed.WithLabelValues(podname, nodename).Set(cpuUsed)
	}

	for cpuid, value := range node.CPU {
		val := float64(value)

		if m.CPUMap != nil {
			m.CPUMap.WithLabelValues(podname, nodename, cpuid).Set(val)
		}

		if m.StatsdAddr == "" {
			continue
		}

		if err := m.gauge(fmt.Sprintf(cpuMap, nodename, cpuid), val); err != nil {
			log.Errorf("[SendNodeInfo] Error occurred while sending cpu data to statsd: %v", err)
		}
	}

	if m.StatsdAddr == "" {
		return
	}

	if err := m.gauge(fmt.Sprintf(memStats, nodename), memory); err != nil {
		log.Errorf("[SendNodeInfo] Error occurred while sending memory data to statsd: %v", err)
	}

	if err := m.gauge(fmt.Sprintf(storageStats, nodename), storage); err != nil {
		log.Errorf("[SendNodeInfo] Error occurred while sending storage data to statsd: %v", err)
	}

	if err := m.gauge(fmt.Sprintf(memUsedStats, nodename), memoryUsed); err != nil {
		log.Errorf("[SendNodeInfo] Error occurred while sending memory used data to statsd: %v", err)
	}

	if err := m.gauge(fmt.Sprintf(storageUsedStats, nodename), storageUsed); err != nil {
		log.Errorf("[SendNodeInfo] Error occurred while sending storage used data to statsd: %v", err)
	}

	if err := m.gauge(fmt.Sprintf(cpuUsedStats, nodename), cpuUsed); err != nil {
		log.Errorf("[SendNodeInfo] Error occurred while sending cpu used data to statsd: %v", err)
	}
}

// SendDeployCount update deploy counter
func (m *Metrics) SendDeployCount(n int) {
	log.Info("[Metrics] Update deploy counter")
	if m.DeployCount != nil {
		m.DeployCount.WithLabelValues(m.Hostname).Add(float64(n))
	}

	if m.StatsdAddr == "" {
		return
	}
	key := fmt.Sprintf(deployCount, m.Hostname)
	if err := m.count(key, n, 1.0); err != nil {
		log.Errorf("[SendDeployCount] Error occurred while counting: %v", err)
	}
}

// Client is a metrics obj
var Client = Metrics{}

// InitMetrics new a metrics obj
func InitMetrics(statsd string) error {
	hostname, err := os.Hostname()
	if err != nil {
		return err
	}
	Client = Metrics{StatsdAddr: statsd, Hostname: utils.CleanStatsdMetrics(hostname)}

	Client.MemoryCapacity = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "memory_capacity",
		Help: "node available memory.",
	}, []string{"podname", "nodename"})

	Client.MemoryUsed = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "memory_used",
		Help: "node used memory.",
	}, []string{"podname", "nodename"})

	Client.StorageCapacity = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "storage_capacity",
		Help: "node available storage.",
	}, []string{"podname", "nodename"})

	Client.StorageUsed = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "storage_used",
		Help: "node used storage.",
	}, []string{"podname", "nodename"})

	Client.CPUMap = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "cpu_map",
		Help: "node available cpu.",
	}, []string{"podname", "nodename", "cpuid"})

	Client.CPUUsed = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "cpu_used",
		Help: "node used cpu.",
	}, []string{"podname", "nodename"})

	Client.DeployCount = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "core_deploy",
		Help: "core deploy counter",
	}, []string{"hostname"})

	prometheus.MustRegister(
		Client.DeployCount, Client.MemoryCapacity,
		Client.StorageCapacity, Client.CPUMap,
		Client.MemoryUsed, Client.StorageUsed, Client.CPUUsed,
	)
	return nil
}
<commit_msg>metrics: Prometheus metrics names remain origin. (#196)<commit_after>package metrics

import (
	"fmt"
	"os"

	statsdlib "github.com/CMGS/statsd"
	"github.com/projecteru2/core/types"
	"github.com/projecteru2/core/utils"
	"github.com/prometheus/client_golang/prometheus"
	log "github.com/sirupsen/logrus"
)

const (
	cpuMap           = "core.node.%s.cpu.%s"
	memStats         = "core.node.%s.memory"
	storageStats     = "core.node.%s.storage"
	memUsedStats     = "core.node.%s.memory.used"
	storageUsedStats = "core.node.%s.storage.used"
	cpuUsedStats     = "core.node.%s.cpu.used"
	deployCount      = "core.%s.deploy.count"
)

// Metrics define metrics
type Metrics struct {
	StatsdAddr   string
	Hostname     string
	statsdClient *statsdlib.Client

	MemoryCapacity  *prometheus.GaugeVec
	MemoryUsed      *prometheus.GaugeVec
	StorageCapacity *prometheus.GaugeVec
	StorageUsed     *prometheus.GaugeVec
	CPUMap          *prometheus.GaugeVec
	CPUUsed         *prometheus.GaugeVec
	DeployCount     *prometheus.CounterVec
}

// Lazy connect
func (m *Metrics) checkConn() error {
	if m.statsdClient != nil {
		return nil
	}
	var err error
	// We needn't try to renew/reconnect because of only supporting UDP protocol now
	// We should add an `errorCount` to reconnect when implementing TCP protocol
	if m.statsdClient, err = statsdlib.New(m.StatsdAddr, statsdlib.WithErrorHandler(func(err error) {
		log.Errorf("[statsd] Sending statsd failed: %v", err)
	})); err != nil {
		log.Errorf("[statsd] Connect statsd failed: %v", err)
		return err
	}
	return nil
}

func (m *Metrics) gauge(key string, value float64) error {
	if err := m.checkConn(); err != nil {
		return err
	}
	m.statsdClient.Gauge(key, value)
	return nil
}

func (m *Metrics) count(key string, n int, rate float32) error {
	if err := m.checkConn(); err != nil {
		return err
	}
	m.statsdClient.Count(key, n, rate)
	return nil
}

// SendNodeInfo update node resource capacity
func (m *Metrics) SendNodeInfo(node *types.Node) {
	log.Debugf("[Metrics] Update %s metrics", node.Name)
	nodename := node.Name
	podname := node.Podname

	memory := float64(node.MemCap)
	memoryUsed := float64(node.InitMemCap - node.MemCap)
	storage := float64(node.StorageCap)
	storageUsed := float64(node.InitStorageCap - node.StorageCap)
	cpuUsed := node.CPUUsed

	if m.MemoryCapacity != nil {
		m.MemoryCapacity.WithLabelValues(podname, nodename).Set(memory)
	}

	if m.MemoryUsed != nil {
		m.MemoryUsed.WithLabelValues(podname, nodename).Set(memoryUsed)
	}

	if m.StorageCapacity != nil {
		m.StorageCapacity.WithLabelValues(podname, nodename).Set(storage)
	}

	if m.StorageUsed != nil {
		m.StorageUsed.WithLabelValues(podname, nodename).Set(storageUsed)
	}

	if m.CPUUsed != nil {
		m.CPUUsed.WithLabelValues(podname, nodename).Set(cpuUsed)
	}

	cleanedNodeName := utils.CleanStatsdMetrics(nodename)
	for cpuid, value := range node.CPU {
		val := float64(value)

		if m.CPUMap != nil {
			m.CPUMap.WithLabelValues(podname, nodename, cpuid).Set(val)
		}

		if m.StatsdAddr == "" {
			continue
		}

		if err := m.gauge(fmt.Sprintf(cpuMap, cleanedNodeName, cpuid), val); err != nil {
			log.Errorf("[SendNodeInfo] Error occurred while sending cpu data to statsd: %v", err)
		}
	}

	if m.StatsdAddr == "" {
		return
	}

	if err := m.gauge(fmt.Sprintf(memStats, cleanedNodeName), memory); err != nil {
		log.Errorf("[SendNodeInfo] Error occurred while sending memory data to statsd: %v", err)
	}

	if err := m.gauge(fmt.Sprintf(storageStats, cleanedNodeName), storage); err != nil {
		log.Errorf("[SendNodeInfo] Error occurred while sending storage data to statsd: %v", err)
	}

	if err := m.gauge(fmt.Sprintf(memUsedStats, cleanedNodeName), memoryUsed); err != nil {
		log.Errorf("[SendNodeInfo] Error occurred while sending memory used data to statsd: %v", err)
	}

	if err := m.gauge(fmt.Sprintf(storageUsedStats, cleanedNodeName), storageUsed); err != nil {
		log.Errorf("[SendNodeInfo] Error occurred while sending storage used data to statsd: %v", err)
	}

	if err := m.gauge(fmt.Sprintf(cpuUsedStats, cleanedNodeName), cpuUsed); err != nil {
		log.Errorf("[SendNodeInfo] Error occurred while sending cpu used data to statsd: %v", err)
	}
}

// SendDeployCount update deploy counter
func (m *Metrics) SendDeployCount(n int) {
	log.Info("[Metrics] Update deploy counter")
	if m.DeployCount != nil {
		m.DeployCount.WithLabelValues(m.Hostname).Add(float64(n))
	}

	if m.StatsdAddr == "" {
		return
	}
	key := fmt.Sprintf(deployCount, m.Hostname)
	if err := m.count(key, n, 1.0); err != nil {
		log.Errorf("[SendDeployCount] Error occurred while counting: %v", err)
	}
}

// Client is a metrics obj
var Client = Metrics{}

// InitMetrics new a metrics obj
func InitMetrics(statsd string) error {
	hostname, err := os.Hostname()
	if err != nil {
		return err
	}
	Client = Metrics{StatsdAddr: statsd, Hostname: utils.CleanStatsdMetrics(hostname)}

	Client.MemoryCapacity = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "memory_capacity",
		Help: "node available memory.",
	}, []string{"podname", "nodename"})

	Client.MemoryUsed = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "memory_used",
		Help: "node used memory.",
	}, []string{"podname", "nodename"})

	Client.StorageCapacity = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "storage_capacity",
		Help: "node available storage.",
	}, []string{"podname", "nodename"})

	Client.StorageUsed = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "storage_used",
		Help: "node used storage.",
	}, []string{"podname", "nodename"})

	Client.CPUMap = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "cpu_map",
		Help: "node available cpu.",
	}, []string{"podname", "nodename", "cpuid"})

	Client.CPUUsed = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "cpu_used",
		Help: "node used cpu.",
	}, []string{"podname", "nodename"})

	Client.DeployCount = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "core_deploy",
		Help: "core deploy counter",
	}, []string{"hostname"})

	prometheus.MustRegister(
		Client.DeployCount, Client.MemoryCapacity,
		Client.StorageCapacity, Client.CPUMap,
		Client.MemoryUsed, Client.StorageUsed, Client.CPUUsed,
	)
	return nil
}
<|endoftext|>


<commit_before>package commands

import "github.com/concourse/fly/rc"

type FlyCommand struct {
	Help HelpCommand `command:"help" description:"Print this help message"`

	Target  rc.TargetName  `short:"t" long:"target" description:"Concourse target name"`
	Targets TargetsCommand `command:"targets" alias:"ts" description:"List saved targets"`

	Version func() `short:"v" long:"version" description:"Print the version of Fly and exit"`

	Verbose bool `long:"verbose" description:"Print API requests and responses"`

	PrintTableHeaders bool `long:"print-table-headers" description:"Print table headers even for redirected output"`

	Login  LoginCommand  `command:"login" alias:"l" description:"Authenticate with the target"`
	Logout LogoutCommand `command:"logout" alias:"o" description:"Release authentication with the target"`
	Status StatusCommand `command:"status" alias:"st" description:"Login status"`
	Sync   SyncCommand   `command:"sync"  alias:"s" description:"Download and replace the current fly from the target"`

	Teams       TeamsCommand       `command:"teams" alias:"t" description:"List the configured teams"`
	SetTeam     SetTeamCommand     `command:"set-team"  alias:"st" description:"Create or modify a team to have the given credentials"`
	RenameTeam  RenameTeamCommand  `command:"rename-team"   alias:"rt" description:"Rename a team"`
	DestroyTeam DestroyTeamCommand `command:"destroy-team"  alias:"dt" description:"Destroy a team and delete all of its data"`

	Checklist ChecklistCommand `command:"checklist" alias:"cl" description:"Print a Checkfile of the given pipeline"`

	Execute ExecuteCommand `command:"execute" alias:"e" description:"Execute a one-off build using local bits"`
	Watch   WatchCommand   `command:"watch"   alias:"w" description:"Stream a build's output"`

	Containers ContainersCommand `command:"containers" alias:"cs" description:"Print the active containers"`
	Hijack     HijackCommand     `command:"hijack"     alias:"intercept" alias:"i" description:"Execute a command in a container"`

	Jobs       JobsCommand       `command:"jobs"      alias:"js" description:"List the jobs in the pipelines"`
	PauseJob   PauseJobCommand   `command:"pause-job" alias:"pj" description:"Pause a job"`
	UnpauseJob UnpauseJobCommand `command:"unpause-job" alias:"uj" description:"Unpause a job"`

	Pipelines        PipelinesCommand        `command:"pipelines"         alias:"ps" description:"List the configured pipelines"`
	DestroyPipeline  DestroyPipelineCommand  `command:"destroy-pipeline"  alias:"dp" description:"Destroy a pipeline"`
	GetPipeline      GetPipelineCommand      `command:"get-pipeline"      alias:"gp" description:"Get a pipeline's current configuration"`
	SetPipeline      SetPipelineCommand      `command:"set-pipeline"      alias:"sp" description:"Create or update a pipeline's configuration"`
	PausePipeline    PausePipelineCommand    `command:"pause-pipeline"    alias:"pp" description:"Pause a pipeline"`
	UnpausePipeline  UnpausePipelineCommand  `command:"unpause-pipeline"  alias:"up" description:"Un-pause a pipeline"`
	ExposePipeline   ExposePipelineCommand   `command:"expose-pipeline"   alias:"ep" description:"Make a pipeline publicly viewable"`
	HidePipeline     HidePipelineCommand     `command:"hide-pipeline"     alias:"hp" description:"Hide a pipeline from the public"`
	RenamePipeline   RenamePipelineCommand   `command:"rename-pipeline"   alias:"rp" description:"Rename a pipeline"`
	ValidatePipeline ValidatePipelineCommand `command:"validate-pipeline" alias:"vp" description:"Validate a pipeline config"`
	FormatPipeline   FormatPipelineCommand   `command:"format-pipeline"   alias:"fp" description:"Format a pipeline config"`
	OrderPipelines   OrderPipelinesCommand   `command:"order-pipelines" alias:"op" description:"Orders pipelines"`
	CheckResource    CheckResourceCommand    `command:"check-resource"    alias:"cr" description:"Check a resource"`
	PauseResource    PauseResourceCommand    `command:"pause-resource"    alias:"pr" description:"Pause a resource"`
	UnpauseResource  UnpauseResourceCommand  `command:"unpause-resource"  alias:"ur" description:"Unpause a resource"`

	Builds     BuildsCommand     `command:"builds"      alias:"bs" description:"List builds data"`
	AbortBuild AbortBuildCommand `command:"abort-build" alias:"ab" description:"Abort a build"`

	TriggerJob TriggerJobCommand `command:"trigger-job" alias:"tj" description:"Start a job in a pipeline"`

	Volumes VolumesCommand `command:"volumes" alias:"vs" description:"List the active volumes"`

	Workers     WorkersCommand     `command:"workers" alias:"ws" description:"List the registered workers"`
	PruneWorker PruneWorkerCommand `command:"prune-worker" alias:"pw" description:"Prune a stalled, landing, landed, or retiring worker"`
}

var Fly FlyCommand
<commit_msg>Remove shortcut for status<commit_after>package commands

import "github.com/concourse/fly/rc"

type FlyCommand struct {
	Help HelpCommand `command:"help" description:"Print this help message"`

	Target  rc.TargetName  `short:"t" long:"target" description:"Concourse target name"`
	Targets TargetsCommand `command:"targets" alias:"ts" description:"List saved targets"`

	Version func() `short:"v" long:"version" description:"Print the version of Fly and exit"`

	Verbose bool `long:"verbose" description:"Print API requests and responses"`

	PrintTableHeaders bool `long:"print-table-headers" description:"Print table headers even for redirected output"`

	Login  LoginCommand  `command:"login" alias:"l" description:"Authenticate with the target"`
	Logout LogoutCommand `command:"logout" alias:"o" description:"Release authentication with the target"`
	Status StatusCommand `command:"status" description:"Login status"`
	Sync   SyncCommand   `command:"sync"  alias:"s" description:"Download and replace the current fly from the target"`

	Teams       TeamsCommand       `command:"teams" alias:"t" description:"List the configured teams"`
	SetTeam     SetTeamCommand     `command:"set-team"  alias:"st" description:"Create or modify a team to have the given credentials"`
	RenameTeam  RenameTeamCommand  `command:"rename-team"   alias:"rt" description:"Rename a team"`
	DestroyTeam DestroyTeamCommand `command:"destroy-team"  alias:"dt" description:"Destroy a team and delete all of its data"`

	Checklist ChecklistCommand `command:"checklist" alias:"cl" description:"Print a Checkfile of the given pipeline"`

	Execute ExecuteCommand `command:"execute" alias:"e" description:"Execute a one-off build using local bits"`
	Watch   WatchCommand   `command:"watch"   alias:"w" description:"Stream a build's output"`

	Containers ContainersCommand `command:"containers" alias:"cs" description:"Print the active containers"`
	Hijack     HijackCommand     `command:"hijack"     alias:"intercept" alias:"i" description:"Execute a command in a container"`

	Jobs       JobsCommand       `command:"jobs"      alias:"js" description:"List the jobs in the pipelines"`
	PauseJob   PauseJobCommand   `command:"pause-job" alias:"pj" description:"Pause a job"`
	UnpauseJob UnpauseJobCommand `command:"unpause-job" alias:"uj" description:"Unpause a job"`

	Pipelines        PipelinesCommand        `command:"pipelines"         alias:"ps" description:"List the configured pipelines"`
	DestroyPipeline  DestroyPipelineCommand  `command:"destroy-pipeline"  alias:"dp" description:"Destroy a pipeline"`
	GetPipeline      GetPipelineCommand      `command:"get-pipeline"      alias:"gp" description:"Get a pipeline's current configuration"`
	SetPipeline      SetPipelineCommand      `command:"set-pipeline"      alias:"sp" description:"Create or update a pipeline's configuration"`
	PausePipeline    PausePipelineCommand    `command:"pause-pipeline"    alias:"pp" description:"Pause a pipeline"`
	UnpausePipeline  UnpausePipelineCommand  `command:"unpause-pipeline"  alias:"up" description:"Un-pause a pipeline"`
	ExposePipeline   ExposePipelineCommand   `command:"expose-pipeline"   alias:"ep" description:"Make a pipeline publicly viewable"`
	HidePipeline     HidePipelineCommand     `command:"hide-pipeline"     alias:"hp" description:"Hide a pipeline from the public"`
	RenamePipeline   RenamePipelineCommand   `command:"rename-pipeline"   alias:"rp" description:"Rename a pipeline"`
	ValidatePipeline ValidatePipelineCommand `command:"validate-pipeline" alias:"vp" description:"Validate a pipeline config"`
	FormatPipeline   FormatPipelineCommand   `command:"format-pipeline"   alias:"fp" description:"Format a pipeline config"`
	OrderPipelines   OrderPipelinesCommand   `command:"order-pipelines" alias:"op" description:"Orders pipelines"`
	CheckResource    CheckResourceCommand    `command:"check-resource"    alias:"cr" description:"Check a resource"`
	PauseResource    PauseResourceCommand    `command:"pause-resource"    alias:"pr" description:"Pause a resource"`
	UnpauseResource  UnpauseResourceCommand  `command:"unpause-resource"  alias:"ur" description:"Unpause a resource"`

	Builds     BuildsCommand     `command:"builds"      alias:"bs" description:"List builds data"`
	AbortBuild AbortBuildCommand `command:"abort-build" alias:"ab" description:"Abort a build"`

	TriggerJob TriggerJobCommand `command:"trigger-job" alias:"tj" description:"Start a job in a pipeline"`

	Volumes VolumesCommand `command:"volumes" alias:"vs" description:"List the active volumes"`

	Workers     WorkersCommand     `command:"workers" alias:"ws" description:"List the registered workers"`
	PruneWorker PruneWorkerCommand `command:"prune-worker" alias:"pw" description:"Prune a stalled, landing, landed, or retiring worker"`
}

var Fly FlyCommand
<|endoftext|>


<commit_before>package middleware

import (
	"crypto/hmac"
	"crypto/rand"
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/labstack/echo"
)

type (
	// CSRFConfig defines the config for CSRF middleware.
	CSRFConfig struct {
		// Key to create CSRF token.
		Secret []byte `json:"secret"`

		// TokenLookup is a string in the form of "<source>:<key>" that is used
		// to extract token from the request.
		// Optional. Default value "header:X-CSRF-Token".
		// Possible values:
		// - "header:<name>"
		// - "form:<name>"
		// - "header:<name>"
		TokenLookup string `json:"token_lookup"`

		// Context key to store generated CSRF token into context.
		// Optional. Default value "csrf".
		ContextKey string `json:"context_key"`

		// Name of the CSRF cookie. This cookie will store CSRF token.
		// Optional. Default value "csrf".
		CookieName string `json:"cookie_name"`

		// Domain of the CSRF cookie.
		// Optional. Default value none.
		CookieDomain string `json:"cookie_domain"`

		// Path of the CSRF cookie.
		// Optional. Default value none.
		CookiePath string `json:"cookie_path"`

		// Expiration time of the CSRF cookie.
		// Optional. Default value 24H.
		CookieExpires time.Time `json:"cookie_expires"`

		// Indicates if CSRF cookie is secure.
		CookieSecure bool `json:"cookie_secure"`
		// Optional. Default value false.

		// Indicates if CSRF cookie is HTTP only.
		// Optional. Default value false.
		CookieHTTPOnly bool `json:"cookie_http_only"`
	}

	// csrfTokenExtractor defines a function that takes `echo.Context` and returns
	// either a token or an error.
	csrfTokenExtractor func(echo.Context) (string, error)
)

var (
	// DefaultCSRFConfig is the default CSRF middleware config.
	DefaultCSRFConfig = CSRFConfig{
		TokenLookup:   "header:" + echo.HeaderXCSRFToken,
		ContextKey:    "csrf",
		CookieName:    "csrf",
		CookieExpires: time.Now().Add(24 * time.Hour),
	}
)

// CSRF returns a Cross-Site Request Forgery (CSRF) middleware.
// See: https://en.wikipedia.org/wiki/Cross-site_request_forgery
func CSRF(secret []byte) echo.MiddlewareFunc {
	c := DefaultCSRFConfig
	c.Secret = secret
	return CSRFWithConfig(c)
}

// CSRFWithConfig returns a CSRF middleware from config.
// See `CSRF()`.
func CSRFWithConfig(config CSRFConfig) echo.MiddlewareFunc {
	// Defaults
	if config.Secret == nil {
		panic("csrf secret must be provided")
	}
	if config.TokenLookup == "" {
		config.TokenLookup = DefaultCSRFConfig.TokenLookup
	}
	if config.ContextKey == "" {
		config.ContextKey = DefaultCSRFConfig.ContextKey
	}
	if config.CookieName == "" {
		config.CookieName = DefaultCSRFConfig.CookieName
	}
	if config.CookieExpires.IsZero() {
		config.CookieExpires = DefaultCSRFConfig.CookieExpires
	}

	// Initialize
	parts := strings.Split(config.TokenLookup, ":")
	extractor := csrfTokenFromHeader(parts[1])
	switch parts[0] {
	case "form":
		extractor = csrfTokenFromForm(parts[1])
	case "query":
		extractor = csrfTokenFromQuery(parts[1])
	}

	return func(next echo.HandlerFunc) echo.HandlerFunc {
		return func(c echo.Context) error {
			req := c.Request()

			// Set CSRF token
			salt, err := generateSalt(8)
			if err != nil {
				return err
			}
			token := generateCSRFToken(config.Secret, salt)
			c.Set(config.ContextKey, token)
			cookie := new(echo.Cookie)
			cookie.SetName(config.CookieName)
			cookie.SetValue(token)
			if config.CookiePath != "" {
				cookie.SetPath(config.CookiePath)
			}
			if config.CookieDomain != "" {
				cookie.SetDomain(config.CookieDomain)
			}
			cookie.SetExpires(config.CookieExpires)
			cookie.SetSecure(config.CookieSecure)
			cookie.SetHTTPOnly(config.CookieHTTPOnly)
			c.SetCookie(cookie)

			switch req.Method() {
			case echo.GET, echo.HEAD, echo.OPTIONS, echo.TRACE:
			default:
				token, err := extractor(c)
				if err != nil {
					return err
				}
				ok, err := validateCSRFToken(token, config.Secret)
				if err != nil {
					return err
				}
				if !ok {
					return echo.NewHTTPError(http.StatusForbidden, "invalid csrf token")
				}
			}
			return next(c)
		}
	}
}

// csrfTokenFromForm returns a `csrfTokenExtractor` that extracts token from the
// provided request header.
func csrfTokenFromHeader(header string) csrfTokenExtractor {
	return func(c echo.Context) (string, error) {
		return c.Request().Header().Get(header), nil
	}
}

// csrfTokenFromForm returns a `csrfTokenExtractor` that extracts token from the
// provided form parameter.
func csrfTokenFromForm(param string) csrfTokenExtractor {
	return func(c echo.Context) (string, error) {
		token := c.FormValue(param)
		if token == "" {
			return "", errors.New("empty csrf token in form param")
		}
		return token, nil
	}
}

// csrfTokenFromQuery returns a `csrfTokenExtractor` that extracts token from the
// provided query parameter.
func csrfTokenFromQuery(param string) csrfTokenExtractor {
	return func(c echo.Context) (string, error) {
		token := c.QueryParam(param)
		if token == "" {
			return "", errors.New("empty csrf token in query param")
		}
		return token, nil
	}
}

func generateCSRFToken(secret, salt []byte) string {
	h := hmac.New(sha1.New, secret)
	h.Write(salt)
	return fmt.Sprintf("%s:%s", hex.EncodeToString(h.Sum(nil)), hex.EncodeToString(salt))
}

func validateCSRFToken(token string, secret []byte) (bool, error) {
	sep := strings.Index(token, ":")
	if sep < 0 {
		return false, nil
	}
	salt, err := hex.DecodeString(token[sep+1:])
	if err != nil {
		return false, err
	}
	return token == generateCSRFToken(secret, salt), nil
}

func generateSalt(len uint8) (salt []byte, err error) {
	salt = make([]byte, len)
	_, err = rand.Read(salt)
	return
}
<commit_msg>Fixed docs<commit_after>package middleware

import (
	"crypto/hmac"
	"crypto/rand"
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/labstack/echo"
)

type (
	// CSRFConfig defines the config for CSRF middleware.
	CSRFConfig struct {
		// Key to create CSRF token.
		Secret []byte `json:"secret"`

		// TokenLookup is a string in the form of "<source>:<key>" that is used
		// to extract token from the request.
		// Optional. Default value "header:X-CSRF-Token".
		// Possible values:
		// - "header:<name>"
		// - "form:<name>"
		// - "query:<name>"
		TokenLookup string `json:"token_lookup"`

		// Context key to store generated CSRF token into context.
		// Optional. Default value "csrf".
		ContextKey string `json:"context_key"`

		// Name of the CSRF cookie. This cookie will store CSRF token.
		// Optional. Default value "csrf".
		CookieName string `json:"cookie_name"`

		// Domain of the CSRF cookie.
		// Optional. Default value none.
		CookieDomain string `json:"cookie_domain"`

		// Path of the CSRF cookie.
		// Optional. Default value none.
		CookiePath string `json:"cookie_path"`

		// Expiration time of the CSRF cookie.
		// Optional. Default value 24H.
		CookieExpires time.Time `json:"cookie_expires"`

		// Indicates if CSRF cookie is secure.
		CookieSecure bool `json:"cookie_secure"`
		// Optional. Default value false.

		// Indicates if CSRF cookie is HTTP only.
		// Optional. Default value false.
		CookieHTTPOnly bool `json:"cookie_http_only"`
	}

	// csrfTokenExtractor defines a function that takes `echo.Context` and returns
	// either a token or an error.
	csrfTokenExtractor func(echo.Context) (string, error)
)

var (
	// DefaultCSRFConfig is the default CSRF middleware config.
	DefaultCSRFConfig = CSRFConfig{
		TokenLookup:   "header:" + echo.HeaderXCSRFToken,
		ContextKey:    "csrf",
		CookieName:    "csrf",
		CookieExpires: time.Now().Add(24 * time.Hour),
	}
)

// CSRF returns a Cross-Site Request Forgery (CSRF) middleware.
// See: https://en.wikipedia.org/wiki/Cross-site_request_forgery
func CSRF(secret []byte) echo.MiddlewareFunc {
	c := DefaultCSRFConfig
	c.Secret = secret
	return CSRFWithConfig(c)
}

// CSRFWithConfig returns a CSRF middleware from config.
// See `CSRF()`.
func CSRFWithConfig(config CSRFConfig) echo.MiddlewareFunc {
	// Defaults
	if config.Secret == nil {
		panic("csrf secret must be provided")
	}
	if config.TokenLookup == "" {
		config.TokenLookup = DefaultCSRFConfig.TokenLookup
	}
	if config.ContextKey == "" {
		config.ContextKey = DefaultCSRFConfig.ContextKey
	}
	if config.CookieName == "" {
		config.CookieName = DefaultCSRFConfig.CookieName
	}
	if config.CookieExpires.IsZero() {
		config.CookieExpires = DefaultCSRFConfig.CookieExpires
	}

	// Initialize
	parts := strings.Split(config.TokenLookup, ":")
	extractor := csrfTokenFromHeader(parts[1])
	switch parts[0] {
	case "form":
		extractor = csrfTokenFromForm(parts[1])
	case "query":
		extractor = csrfTokenFromQuery(parts[1])
	}

	return func(next echo.HandlerFunc) echo.HandlerFunc {
		return func(c echo.Context) error {
			req := c.Request()

			// Set CSRF token
			salt, err := generateSalt(8)
			if err != nil {
				return err
			}
			token := generateCSRFToken(config.Secret, salt)
			c.Set(config.ContextKey, token)
			cookie := new(echo.Cookie)
			cookie.SetName(config.CookieName)
			cookie.SetValue(token)
			if config.CookiePath != "" {
				cookie.SetPath(config.CookiePath)
			}
			if config.CookieDomain != "" {
				cookie.SetDomain(config.CookieDomain)
			}
			cookie.SetExpires(config.CookieExpires)
			cookie.SetSecure(config.CookieSecure)
			cookie.SetHTTPOnly(config.CookieHTTPOnly)
			c.SetCookie(cookie)

			switch req.Method() {
			case echo.GET, echo.HEAD, echo.OPTIONS, echo.TRACE:
			default:
				token, err := extractor(c)
				if err != nil {
					return err
				}
				ok, err := validateCSRFToken(token, config.Secret)
				if err != nil {
					return err
				}
				if !ok {
					return echo.NewHTTPError(http.StatusForbidden, "invalid csrf token")
				}
			}
			return next(c)
		}
	}
}

// csrfTokenFromForm returns a `csrfTokenExtractor` that extracts token from the
// provided request header.
func csrfTokenFromHeader(header string) csrfTokenExtractor {
	return func(c echo.Context) (string, error) {
		return c.Request().Header().Get(header), nil
	}
}

// csrfTokenFromForm returns a `csrfTokenExtractor` that extracts token from the
// provided form parameter.
func csrfTokenFromForm(param string) csrfTokenExtractor {
	return func(c echo.Context) (string, error) {
		token := c.FormValue(param)
		if token == "" {
			return "", errors.New("empty csrf token in form param")
		}
		return token, nil
	}
}

// csrfTokenFromQuery returns a `csrfTokenExtractor` that extracts token from the
// provided query parameter.
func csrfTokenFromQuery(param string) csrfTokenExtractor {
	return func(c echo.Context) (string, error) {
		token := c.QueryParam(param)
		if token == "" {
			return "", errors.New("empty csrf token in query param")
		}
		return token, nil
	}
}

func generateCSRFToken(secret, salt []byte) string {
	h := hmac.New(sha1.New, secret)
	h.Write(salt)
	return fmt.Sprintf("%s:%s", hex.EncodeToString(h.Sum(nil)), hex.EncodeToString(salt))
}

func validateCSRFToken(token string, secret []byte) (bool, error) {
	sep := strings.Index(token, ":")
	if sep < 0 {
		return false, nil
	}
	salt, err := hex.DecodeString(token[sep+1:])
	if err != nil {
		return false, err
	}
	return token == generateCSRFToken(secret, salt), nil
}

func generateSalt(len uint8) (salt []byte, err error) {
	salt = make([]byte, len)
	_, err = rand.Read(salt)
	return
}
<|endoftext|>