<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package libkb

import (
	"encoding/json"
	"fmt"
	"reflect"
	"sort"
	"testing"
	"time"

	keybase1 "github.com/keybase/client/go/protocol/keybase1"
	jsonw "github.com/keybase/go-jsonw"
	testvectors "github.com/keybase/keybase-test-vectors/go"
	"golang.org/x/net/context"
)

// Returns a map from error name strings to sets of Go error types. If a test
// returns any error type in the corresponding set, it's a pass. (The reason
// the types aren't one-to-one here is that implementation differences between
// the Go and JS sigchains make that more trouble than it's worth.)
func getErrorTypesMap() map[string]map[reflect.Type]bool {
	return map[string]map[reflect.Type]bool{
		"CTIME_MISMATCH": {
			reflect.TypeOf(CtimeMismatchError{}): true,
		},
		"EXPIRED_SIBKEY": {
			reflect.TypeOf(KeyExpiredError{}): true,
		},
		"FINGERPRINT_MISMATCH": {
			reflect.TypeOf(ChainLinkFingerprintMismatchError{}): true,
		},
		"INVALID_SIBKEY": {
			reflect.TypeOf(KeyRevokedError{}): true,
		},
		"NO_KEY_WITH_THIS_HASH": {
			reflect.TypeOf(NoKeyError{}): true,
		},
		"KEY_OWNERSHIP": {
			reflect.TypeOf(KeyFamilyError{}): true,
		},
		"KID_MISMATCH": {
			reflect.TypeOf(ChainLinkKIDMismatchError{}): true,
		},
		"NONEXISTENT_KID": {
			reflect.TypeOf(KeyFamilyError{}): true,
		},
		"NOT_LATEST_SUBCHAIN": {
			reflect.TypeOf(NotLatestSubchainError{}): true,
		},
		"REVERSE_SIG_VERIFY_FAILED": {
			reflect.TypeOf(ReverseSigError{}): true,
		},
		"VERIFY_FAILED": {
			reflect.TypeOf(BadSigError{}): true,
		},
		"WRONG_UID": {
			reflect.TypeOf(UIDMismatchError{}): true,
		},
		"WRONG_USERNAME": {
			reflect.TypeOf(BadUsernameError{}): true,
		},
		"WRONG_SEQNO": {
			reflect.TypeOf(ChainLinkWrongSeqnoError{}): true,
		},
		"WRONG_PREV": {
			reflect.TypeOf(ChainLinkPrevHashMismatchError{}): true,
		},
		"SIGCHAIN_V2_STUBBED_SIGNATURE_NEEDED": {
			reflect.TypeOf(SigchainV2StubbedSignatureNeededError{}): true,
		},
		"SIGCHAIN_V2_STUBBED_FIRST_LINK": {
			reflect.TypeOf(SigchainV2StubbedFirstLinkError{}): true,
		},
		"SIGCHAIN_V2_MISMATCHED_FIELD": {
			reflect.TypeOf(SigchainV2MismatchedFieldError{}): true,
		},
		"SIGCHAIN_V2_MISMATCHED_HASH": {
			reflect.TypeOf(SigchainV2MismatchedHashError{}): true,
		},
	}
}

// One of the test cases from the JSON list of all tests.
type TestCase struct {
	Input   string `json:"input"`
	Len     int    `json:"len"`
	Sibkeys int    `json:"sibkeys"`
	Subkeys int    `json:"subkeys"`
	ErrType string `json:"err_type"`
	Eldest  string `json:"eldest"`
}

// The JSON list of all test cases.
type TestList struct {
	Tests      map[string]TestCase `json:"tests"`
	ErrorTypes []string            `json:"error_types"`
}

// The input data for a single test. Each tests has its own input JSON file.
type TestInput struct {
	// We omit the "chain" member here, because we need it in blob form.
	Username  string            `json:"username"`
	UID       string            `json:"uid"`
	Keys      []string          `json:"keys"`
	LabelKids map[string]string `json:"label_kids"`
	LabelSigs map[string]string `json:"label_sigs"`
}

func TestAllChains(t *testing.T) {
	tc := SetupTest(t, "test_all_chains", 1)
	defer tc.Cleanup()

	var testList TestList
	json.Unmarshal([]byte(testvectors.ChainTests), &testList)
	// Always do the tests in alphabetical order.
	testNames := []string{}
	for name := range testList.Tests {
		testNames = append(testNames, name)
	}
	sort.Strings(testNames)
	for _, name := range testNames {
		testCase := testList.Tests[name]
		G.Log.Info("starting sigchain test case %s (%s)", name, testCase.Input)
		doChainTest(t, tc, testCase)
	}
}

func doChainTest(t *testing.T, tc TestContext, testCase TestCase) {
	inputJSON, exists := testvectors.ChainTestInputs[testCase.Input]
	if !exists {
		t.Fatal("missing test input: " + testCase.Input)
	}
	// Unmarshal test input in two ways: once for the structured data and once
	// for the chain link blobs.
	var input TestInput
	err := json.Unmarshal([]byte(inputJSON), &input)
	if err != nil {
		t.Fatal(err)
	}
	inputBlob, err := jsonw.Unmarshal([]byte(inputJSON))
	if err != nil {
		t.Fatal(err)
	}
	uid, err := UIDFromHex(input.UID)
	if err != nil {
		t.Fatal(err)
	}
	chainLen, err := inputBlob.AtKey("chain").Len()
	if err != nil {
		t.Fatal(err)
	}

	// Get the eldest key. This is assumed to be the first key in the list of
	// bundles, unless the "eldest" field is given in the test description, in
	// which case the eldest key is specified by name.
	var eldestKID keybase1.KID
	if testCase.Eldest == "" {
		eldestKey, _, err := ParseGenericKey(input.Keys[0])
		if err != nil {
			t.Fatal(err)
		}
		eldestKID = eldestKey.GetKID()
	} else {
		eldestKIDStr, found := input.LabelKids[testCase.Eldest]
		if !found {
			t.Fatalf("No KID found for label %s", testCase.Eldest)
		}
		eldestKID = keybase1.KIDFromString(eldestKIDStr)
	}

	// Parse all the key bundles.
	keyFamily, err := createKeyFamily(input.Keys)
	if err != nil {
		t.Fatal(err)
	}

	// Run the actual sigchain parsing and verification. This is most of the
	// code that's actually being tested.
	var sigchainErr error
	ckf := ComputedKeyFamily{kf: keyFamily}
	sigchain := SigChain{
		username:          NewNormalizedUsername(input.Username),
		uid:               uid,
		loadedFromLinkOne: true,
		Contextified:      NewContextified(tc.G),
	}
	for i := 0; i < chainLen; i++ {
		linkBlob := inputBlob.AtKey("chain").AtIndex(i)
		link, err := ImportLinkFromServer(nil, &sigchain, linkBlob, uid)
		if err != nil {
			sigchainErr = err
			break
		}
		sigchain.chainLinks = append(sigchain.chainLinks, link)
	}
	if sigchainErr == nil {
		_, sigchainErr = sigchain.VerifySigsAndComputeKeys(nil, eldestKID, &ckf)
	}

	// Some tests expect an error. If we get one, make sure it's the right
	// type.
	if testCase.ErrType != "" {
		if sigchainErr == nil {
			t.Fatalf("Expected %s error from VerifySigsAndComputeKeys. No error returned.", testCase.ErrType)
		}
		foundType := reflect.TypeOf(sigchainErr)
		expectedTypes := getErrorTypesMap()[testCase.ErrType]
		if len(expectedTypes) == 0 {
			msg := "No Go error types defined for expected failure %s.\n" +
				"This could be because of new test cases in github.com/keybase/keybase-test-vectors.\n" +
				"Go error returned: %s"
			t.Fatalf(msg, testCase.ErrType, foundType)
		}
		if expectedTypes[foundType] {
			// Success! We found the error we expected. This test is done.
			G.Log.Debug("EXPECTED error encountered: %s", sigchainErr)
			return
		}

		// Got an error, but one of the wrong type. Tests with error names
		// that are missing from the map (maybe because we add new test
		// cases in the future) will also hit this branch.
		t.Fatalf("Wrong error type encountered. Expected %v (%s), got %s: %s",
			expectedTypes, testCase.ErrType, foundType, sigchainErr)

	}

	// Tests that expected an error terminated above. Tests that get here
	// should succeed without errors.
	if sigchainErr != nil {
		t.Fatal(sigchainErr)
	}

	// Check the expected results: total unrevoked links, sibkeys, and subkeys.
	unrevokedCount := 0

	// XXX we should really contextify this
	idtable, err := NewIdentityTable(nil, eldestKID, &sigchain, nil)
	if err != nil {
		t.Fatal(err)
	}
	for _, link := range idtable.links {
		if !link.IsRevoked() {
			unrevokedCount++
		}
	}

	fatalStr := ""
	if unrevokedCount != testCase.Len {
		fatalStr += fmt.Sprintf("Expected %d unrevoked links, but found %d.\n", testCase.Len, unrevokedCount)
	}
	// Don't use the current time to get keys, because that will cause test
	// failures 5 years from now :-D
	testTime := getCurrentTimeForTest(sigchain, keyFamily)
	numSibkeys := len(ckf.GetAllActiveSibkeysAtTime(testTime))
	if numSibkeys != testCase.Sibkeys {
		fatalStr += fmt.Sprintf("Expected %d sibkeys, got %d\n", testCase.Sibkeys, numSibkeys)
	}
	numSubkeys := len(ckf.GetAllActiveSubkeysAtTime(testTime))
	if numSubkeys != testCase.Subkeys {
		fatalStr += fmt.Sprintf("Expected %d subkeys, got %d\n", testCase.Subkeys, numSubkeys)
	}

	if fatalStr != "" {
		t.Fatal(fatalStr)
	}

	storeAndLoad(t, tc, &sigchain)
	// Success!
}

func storeAndLoad(t *testing.T, tc TestContext, chain *SigChain) {
	nLinks := chain.Len()
	err := chain.Store(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	sgl := SigChainLoader{
		user: &User{
			name: chain.username.String(),
			id:   chain.uid,
		},
		self:    false,
		allKeys: true,
		leaf: &MerkleUserLeaf{
			public: chain.GetCurrentTailTriple(),
			uid:    chain.uid,
		},
		chainType:    PublicChain,
		Contextified: NewContextified(tc.G),
		ctx:          context.Background(),
	}
	sgl.chain = chain
	sgl.dirtyTail = chain.GetCurrentTailTriple()
	err = sgl.Store()
	if err != nil {
		t.Fatal(err)
	}
	sgl.chain = nil
	sgl.dirtyTail = nil
	var sc2 *SigChain
	sc2, err = sgl.Load()
	if err != nil {
		t.Fatal(err)
	}
	if nLinks != sc2.Len() {
		t.Fatalf("lost some links: %d != %d", nLinks, sc2.Len())
	}
}

func createKeyFamily(bundles []string) (*KeyFamily, error) {
	allKeys := jsonw.NewArray(len(bundles))
	for i, bundle := range bundles {
		err := allKeys.SetIndex(i, jsonw.NewString(bundle))
		if err != nil {
			return nil, err
		}
	}
	publicKeys := jsonw.NewDictionary()
	publicKeys.SetKey("all_bundles", allKeys)
	return ParseKeyFamily(G, publicKeys)
}

func getCurrentTimeForTest(sigChain SigChain, keyFamily *KeyFamily) time.Time {
	// Pick a test time that's the latest ctime of all links and PGP keys.
	var t time.Time
	for _, link := range sigChain.chainLinks {
		linkCTime := time.Unix(link.unpacked.ctime, 0)
		if linkCTime.After(t) {
			t = linkCTime
		}
	}
	for _, ks := range keyFamily.PGPKeySets {
		keyCTime := ks.PermissivelyMergedKey.PrimaryKey.CreationTime
		if keyCTime.After(t) {
			t = keyCTime
		}
	}
	return t
}
<commit_msg>make TestAllChains stop expecting exact equality in cached chains<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package libkb

import (
	"encoding/json"
	"fmt"
	"reflect"
	"sort"
	"testing"
	"time"

	keybase1 "github.com/keybase/client/go/protocol/keybase1"
	jsonw "github.com/keybase/go-jsonw"
	testvectors "github.com/keybase/keybase-test-vectors/go"
	"golang.org/x/net/context"
)

// Returns a map from error name strings to sets of Go error types. If a test
// returns any error type in the corresponding set, it's a pass. (The reason
// the types aren't one-to-one here is that implementation differences between
// the Go and JS sigchains make that more trouble than it's worth.)
func getErrorTypesMap() map[string]map[reflect.Type]bool {
	return map[string]map[reflect.Type]bool{
		"CTIME_MISMATCH": {
			reflect.TypeOf(CtimeMismatchError{}): true,
		},
		"EXPIRED_SIBKEY": {
			reflect.TypeOf(KeyExpiredError{}): true,
		},
		"FINGERPRINT_MISMATCH": {
			reflect.TypeOf(ChainLinkFingerprintMismatchError{}): true,
		},
		"INVALID_SIBKEY": {
			reflect.TypeOf(KeyRevokedError{}): true,
		},
		"NO_KEY_WITH_THIS_HASH": {
			reflect.TypeOf(NoKeyError{}): true,
		},
		"KEY_OWNERSHIP": {
			reflect.TypeOf(KeyFamilyError{}): true,
		},
		"KID_MISMATCH": {
			reflect.TypeOf(ChainLinkKIDMismatchError{}): true,
		},
		"NONEXISTENT_KID": {
			reflect.TypeOf(KeyFamilyError{}): true,
		},
		"NOT_LATEST_SUBCHAIN": {
			reflect.TypeOf(NotLatestSubchainError{}): true,
		},
		"REVERSE_SIG_VERIFY_FAILED": {
			reflect.TypeOf(ReverseSigError{}): true,
		},
		"VERIFY_FAILED": {
			reflect.TypeOf(BadSigError{}): true,
		},
		"WRONG_UID": {
			reflect.TypeOf(UIDMismatchError{}): true,
		},
		"WRONG_USERNAME": {
			reflect.TypeOf(BadUsernameError{}): true,
		},
		"WRONG_SEQNO": {
			reflect.TypeOf(ChainLinkWrongSeqnoError{}): true,
		},
		"WRONG_PREV": {
			reflect.TypeOf(ChainLinkPrevHashMismatchError{}): true,
		},
		"SIGCHAIN_V2_STUBBED_SIGNATURE_NEEDED": {
			reflect.TypeOf(SigchainV2StubbedSignatureNeededError{}): true,
		},
		"SIGCHAIN_V2_STUBBED_FIRST_LINK": {
			reflect.TypeOf(SigchainV2StubbedFirstLinkError{}): true,
		},
		"SIGCHAIN_V2_MISMATCHED_FIELD": {
			reflect.TypeOf(SigchainV2MismatchedFieldError{}): true,
		},
		"SIGCHAIN_V2_MISMATCHED_HASH": {
			reflect.TypeOf(SigchainV2MismatchedHashError{}): true,
		},
	}
}

// One of the test cases from the JSON list of all tests.
type TestCase struct {
	Input   string `json:"input"`
	Len     int    `json:"len"`
	Sibkeys int    `json:"sibkeys"`
	Subkeys int    `json:"subkeys"`
	ErrType string `json:"err_type"`
	Eldest  string `json:"eldest"`
}

// The JSON list of all test cases.
type TestList struct {
	Tests      map[string]TestCase `json:"tests"`
	ErrorTypes []string            `json:"error_types"`
}

// The input data for a single test. Each tests has its own input JSON file.
type TestInput struct {
	// We omit the "chain" member here, because we need it in blob form.
	Username  string            `json:"username"`
	UID       string            `json:"uid"`
	Keys      []string          `json:"keys"`
	LabelKids map[string]string `json:"label_kids"`
	LabelSigs map[string]string `json:"label_sigs"`
}

func TestAllChains(t *testing.T) {
	tc := SetupTest(t, "test_all_chains", 1)
	defer tc.Cleanup()

	var testList TestList
	json.Unmarshal([]byte(testvectors.ChainTests), &testList)
	// Always do the tests in alphabetical order.
	testNames := []string{}
	for name := range testList.Tests {
		testNames = append(testNames, name)
	}
	sort.Strings(testNames)
	for _, name := range testNames {
		testCase := testList.Tests[name]
		G.Log.Info("starting sigchain test case %s (%s)", name, testCase.Input)
		doChainTest(t, tc, testCase)
	}
}

func doChainTest(t *testing.T, tc TestContext, testCase TestCase) {
	inputJSON, exists := testvectors.ChainTestInputs[testCase.Input]
	if !exists {
		t.Fatal("missing test input: " + testCase.Input)
	}
	// Unmarshal test input in two ways: once for the structured data and once
	// for the chain link blobs.
	var input TestInput
	err := json.Unmarshal([]byte(inputJSON), &input)
	if err != nil {
		t.Fatal(err)
	}
	inputBlob, err := jsonw.Unmarshal([]byte(inputJSON))
	if err != nil {
		t.Fatal(err)
	}
	uid, err := UIDFromHex(input.UID)
	if err != nil {
		t.Fatal(err)
	}
	chainLen, err := inputBlob.AtKey("chain").Len()
	if err != nil {
		t.Fatal(err)
	}

	// Get the eldest key. This is assumed to be the first key in the list of
	// bundles, unless the "eldest" field is given in the test description, in
	// which case the eldest key is specified by name.
	var eldestKID keybase1.KID
	if testCase.Eldest == "" {
		eldestKey, _, err := ParseGenericKey(input.Keys[0])
		if err != nil {
			t.Fatal(err)
		}
		eldestKID = eldestKey.GetKID()
	} else {
		eldestKIDStr, found := input.LabelKids[testCase.Eldest]
		if !found {
			t.Fatalf("No KID found for label %s", testCase.Eldest)
		}
		eldestKID = keybase1.KIDFromString(eldestKIDStr)
	}

	// Parse all the key bundles.
	keyFamily, err := createKeyFamily(input.Keys)
	if err != nil {
		t.Fatal(err)
	}

	// Run the actual sigchain parsing and verification. This is most of the
	// code that's actually being tested.
	var sigchainErr error
	ckf := ComputedKeyFamily{kf: keyFamily}
	sigchain := SigChain{
		username:          NewNormalizedUsername(input.Username),
		uid:               uid,
		loadedFromLinkOne: true,
		Contextified:      NewContextified(tc.G),
	}
	for i := 0; i < chainLen; i++ {
		linkBlob := inputBlob.AtKey("chain").AtIndex(i)
		link, err := ImportLinkFromServer(nil, &sigchain, linkBlob, uid)
		if err != nil {
			sigchainErr = err
			break
		}
		sigchain.chainLinks = append(sigchain.chainLinks, link)
	}
	if sigchainErr == nil {
		_, sigchainErr = sigchain.VerifySigsAndComputeKeys(nil, eldestKID, &ckf)
	}

	// Some tests expect an error. If we get one, make sure it's the right
	// type.
	if testCase.ErrType != "" {
		if sigchainErr == nil {
			t.Fatalf("Expected %s error from VerifySigsAndComputeKeys. No error returned.", testCase.ErrType)
		}
		foundType := reflect.TypeOf(sigchainErr)
		expectedTypes := getErrorTypesMap()[testCase.ErrType]
		if len(expectedTypes) == 0 {
			msg := "No Go error types defined for expected failure %s.\n" +
				"This could be because of new test cases in github.com/keybase/keybase-test-vectors.\n" +
				"Go error returned: %s"
			t.Fatalf(msg, testCase.ErrType, foundType)
		}
		if expectedTypes[foundType] {
			// Success! We found the error we expected. This test is done.
			G.Log.Debug("EXPECTED error encountered: %s", sigchainErr)
			return
		}

		// Got an error, but one of the wrong type. Tests with error names
		// that are missing from the map (maybe because we add new test
		// cases in the future) will also hit this branch.
		t.Fatalf("Wrong error type encountered. Expected %v (%s), got %s: %s",
			expectedTypes, testCase.ErrType, foundType, sigchainErr)

	}

	// Tests that expected an error terminated above. Tests that get here
	// should succeed without errors.
	if sigchainErr != nil {
		t.Fatal(sigchainErr)
	}

	// Check the expected results: total unrevoked links, sibkeys, and subkeys.
	unrevokedCount := 0

	// XXX we should really contextify this
	idtable, err := NewIdentityTable(nil, eldestKID, &sigchain, nil)
	if err != nil {
		t.Fatal(err)
	}
	for _, link := range idtable.links {
		if !link.IsRevoked() {
			unrevokedCount++
		}
	}

	fatalStr := ""
	if unrevokedCount != testCase.Len {
		fatalStr += fmt.Sprintf("Expected %d unrevoked links, but found %d.\n", testCase.Len, unrevokedCount)
	}
	if testCase.Len > 0 && sigchain.currentSubchainStart == 0 {
		fatalStr += fmt.Sprintf("Expected nonzero currentSubchainStart, but found %d.\n", sigchain.currentSubchainStart)
	}
	// Don't use the current time to get keys, because that will cause test
	// failures 5 years from now :-D
	testTime := getCurrentTimeForTest(sigchain, keyFamily)
	numSibkeys := len(ckf.GetAllActiveSibkeysAtTime(testTime))
	if numSibkeys != testCase.Sibkeys {
		fatalStr += fmt.Sprintf("Expected %d sibkeys, got %d\n", testCase.Sibkeys, numSibkeys)
	}
	numSubkeys := len(ckf.GetAllActiveSubkeysAtTime(testTime))
	if numSubkeys != testCase.Subkeys {
		fatalStr += fmt.Sprintf("Expected %d subkeys, got %d\n", testCase.Subkeys, numSubkeys)
	}

	if fatalStr != "" {
		t.Fatal(fatalStr)
	}

	storeAndLoad(t, tc, &sigchain)
	// Success!
}

func storeAndLoad(t *testing.T, tc TestContext, chain *SigChain) {
	err := chain.Store(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	sgl := SigChainLoader{
		user: &User{
			name: chain.username.String(),
			id:   chain.uid,
		},
		self:    false,
		allKeys: true,
		leaf: &MerkleUserLeaf{
			public: chain.GetCurrentTailTriple(),
			uid:    chain.uid,
		},
		chainType:    PublicChain,
		Contextified: NewContextified(tc.G),
		ctx:          context.Background(),
	}
	sgl.chain = chain
	sgl.dirtyTail = chain.GetCurrentTailTriple()
	err = sgl.Store()
	if err != nil {
		t.Fatal(err)
	}
	sgl.chain = nil
	sgl.dirtyTail = nil
	var sc2 *SigChain
	sc2, err = sgl.Load()
	if err != nil {
		t.Fatal(err)
	}

	// Loading a chain from cache won't load all the links, if there's a
	// sigchain reset in the middle. So we don't want to check the entire
	// loaded chain for equality with what we saved. But we can make sure it's
	// loaded all the links of the *current subchain*. HOWEVER: Loading
	// sigchains from cache doesn't benefit from knowing the current eldest KID
	// from the Merkle tree. That means if the account just reset, for example,
	// loading from cache will still fetch the old chain. Avoid failing on this
	// case by skipping the comparison when `currentSubchainStart` is 0
	// (invalid) in the original chain.
	if chain.currentSubchainStart == 0 {
		// As described above, short circuit when we know loading from cache
		// would give us a different answer.
		return
	}
	if chain.currentSubchainStart != sc2.currentSubchainStart {
		t.Fatalf("disagreement about currentSubchainStart: %d != %d", chain.currentSubchainStart, sc2.currentSubchainStart)
	}
	subchainSeqnos1 := enumerateCurrentSubchain(chain)
	subchainSeqnos2 := enumerateCurrentSubchain(sc2)
	if len(subchainSeqnos1) != len(subchainSeqnos2) {
		t.Fatalf("subchains don't have the same length: %d != %d", len(subchainSeqnos1), len(subchainSeqnos2))
	}
	equal := true
	for i := 0; i < len(subchainSeqnos1); i++ {
		if subchainSeqnos1[i] != subchainSeqnos2[i] {
			equal = false
			break
		}
	}
	if !equal {
		t.Fatalf("subchains don't match: %#v != %#v", subchainSeqnos1, subchainSeqnos2)
	}
}

func createKeyFamily(bundles []string) (*KeyFamily, error) {
	allKeys := jsonw.NewArray(len(bundles))
	for i, bundle := range bundles {
		err := allKeys.SetIndex(i, jsonw.NewString(bundle))
		if err != nil {
			return nil, err
		}
	}
	publicKeys := jsonw.NewDictionary()
	publicKeys.SetKey("all_bundles", allKeys)
	return ParseKeyFamily(G, publicKeys)
}

func getCurrentTimeForTest(sigChain SigChain, keyFamily *KeyFamily) time.Time {
	// Pick a test time that's the latest ctime of all links and PGP keys.
	var t time.Time
	for _, link := range sigChain.chainLinks {
		linkCTime := time.Unix(link.unpacked.ctime, 0)
		if linkCTime.After(t) {
			t = linkCTime
		}
	}
	for _, ks := range keyFamily.PGPKeySets {
		keyCTime := ks.PermissivelyMergedKey.PrimaryKey.CreationTime
		if keyCTime.After(t) {
			t = keyCTime
		}
	}
	return t
}

func enumerateCurrentSubchain(chain *SigChain) (seqnoList []Seqno) {
	for _, link := range chain.chainLinks {
		if link.GetSeqno() < chain.currentSubchainStart {
			continue
		}
		seqnoList = append(seqnoList, link.GetSeqno())
	}
	return
}
<|endoftext|>


<commit_before>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package worker

import (
	"fmt"
	"time"

	"vitess.io/vitess/go/vt/proto/vtrpc"
	"vitess.io/vitess/go/vt/vterrors"

	"golang.org/x/net/context"

	"vitess.io/vitess/go/sqltypes"
	"vitess.io/vitess/go/stats"
	"vitess.io/vitess/go/vt/key"
	"vitess.io/vitess/go/vt/topo"

	querypb "vitess.io/vitess/go/vt/proto/query"
	tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata"
	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
)

// DiffType specifies why a specific row was found as different when comparing
// a left and right side.
type DiffType int

const (
	// DiffMissing is returned when the row is missing on the right side.
	DiffMissing DiffType = iota
	// DiffNotEqual is returned when the row on the left and right side are
	// not equal.
	DiffNotEqual
	// DiffExtraneous is returned when the row exists on the right side, but not
	// on the left side.
	DiffExtraneous
	// DiffEqual is returned when the rows left and right are equal.
	DiffEqual
)

// DiffTypes has the list of available DiffType values, ordered by their value.
var DiffTypes = []DiffType{DiffMissing, DiffNotEqual, DiffExtraneous, DiffEqual}

// DiffFoundTypes has the list of DiffType values which represent that a
// difference was found. The list is ordered by the values of the types.
var DiffFoundTypes = []DiffType{DiffMissing, DiffNotEqual, DiffExtraneous}

// RowDiffer2 will compare and reconcile two sides. It assumes that the left
// side is the source of truth and necessary reconciliations have to be applied
// to the right side.
// It also assumes left and right are sorted by ascending primary key.
type RowDiffer2 struct {
	left         *RowReader
	right        *RowReader
	pkFieldCount int
	// tableStatusList is used to report the number of reconciled rows.
	tableStatusList *tableStatusList
	// tableIndex is the index of the table in the schema. It is required for
	// reporting the number of reconciled rows to tableStatusList.
	tableIndex int
	// router returns for a row to which destination shard index it should go.
	router RowRouter
	// aggregators are keyed by destination shard and DiffType.
	aggregators [][]*RowAggregator
	// equalRowsStatsCounters tracks per table how many rows are equal.
	equalRowsStatsCounters *stats.CountersWithSingleLabel
	// tableName is required to update "equalRowsStatsCounters".
	tableName string
}

// NewRowDiffer2 returns a new RowDiffer2.
// We assume that the indexes of the slice parameters always correspond to the
// same shard e.g. insertChannels[0] refers to destinationShards[0] and so on.
// The column list td.Columns must be have all primary key columns first and
// then the non-primary-key columns. The columns in the rows returned by
// both ResultReader must have the same order as td.Columns.
func NewRowDiffer2(ctx context.Context, left, right ResultReader, td *tabletmanagerdatapb.TableDefinition, tableStatusList *tableStatusList, tableIndex int,
	// Parameters required by RowRouter.
	destinationShards []*topo.ShardInfo, keyResolver keyspaceIDResolver,
	// Parameters required by RowAggregator.
	insertChannels []chan string, abort <-chan struct{}, dbNames []string, writeQueryMaxRows, writeQueryMaxSize int, statsCounters []*stats.CountersWithSingleLabel) (*RowDiffer2, error) {

	if len(statsCounters) != len(DiffTypes) {
		panic(fmt.Sprintf("statsCounter has the wrong number of elements. got = %v, want = %v", len(statsCounters), len(DiffTypes)))
	}

	if err := compareFields(left.Fields(), right.Fields()); err != nil {
		return nil, err
	}

	// Create a RowAggregator for each destination shard and DiffType.
	aggregators := make([][]*RowAggregator, len(destinationShards))
	for i := range destinationShards {
		aggregators[i] = make([]*RowAggregator, len(DiffFoundTypes))
		for _, typ := range DiffFoundTypes {
			maxRows := writeQueryMaxRows
			aggregators[i][typ] = NewRowAggregator(ctx, maxRows, writeQueryMaxSize,
				insertChannels[i], dbNames[i], td, typ, statsCounters[typ])
		}
	}

	return &RowDiffer2{
		left:                   NewRowReader(left),
		right:                  NewRowReader(right),
		pkFieldCount:           len(td.PrimaryKeyColumns),
		tableStatusList:        tableStatusList,
		tableIndex:             tableIndex,
		router:                 NewRowRouter(destinationShards, keyResolver),
		aggregators:            aggregators,
		equalRowsStatsCounters: statsCounters[DiffEqual],
		tableName:              td.Name,
	}, nil
}

func compareFields(left, right []*querypb.Field) error {
	if len(left) != len(right) {
		return vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "cannot diff inputs with different number of fields: left: %v right: %v", left, right)
	}
	for i, field := range left {
		if field.Type != right[i].Type {
			return vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "cannot diff inputs with different types: field %v types are %v and %v", i, field.Type, right[i].Type)
		}
	}
	return nil
}

// Diff runs the diff and reconcile.
// If an error occurs, it will return and stop.
func (rd *RowDiffer2) Diff() (DiffReport, error) {
	var dr DiffReport
	var err error

	dr.startingTime = time.Now()
	defer dr.ComputeQPS()

	fields := rd.left.Fields()
	var left []sqltypes.Value
	var right []sqltypes.Value
	advanceLeft := true
	advanceRight := true
	for {
		if advanceLeft {
			if left, err = rd.left.Next(); err != nil {
				return dr, err
			}
			advanceLeft = false
		}
		if advanceRight {
			if right, err = rd.right.Next(); err != nil {
				return dr, err
			}
			advanceRight = false
		}
		dr.processedRows++
		if left == nil && right == nil {
			// No more rows from either side. We're done.
			break
		}
		if left == nil {
			// No more rows on the left side.
			// We know we have at least one row on the right side left.
			// Delete the row from the destination.
			if err := rd.reconcileRow(right, DiffExtraneous); err != nil {
				return dr, err
			}
			dr.extraRowsRight++
			advanceRight = true
			continue
		}
		if right == nil {
			// No more rows on the right side.
			// We know we have at least one row on the left side left.
			// Add the row on the destination.
			if err := rd.reconcileRow(left, DiffMissing); err != nil {
				return dr, err
			}
			dr.extraRowsLeft++
			advanceLeft = true
			continue
		}

		// we have both left and right, compare
		f := RowsEqual(left, right)
		if f == -1 {
			// rows are the same, next
			dr.matchingRows++
			advanceLeft = true
			advanceRight = true
			rd.skipRow()
			continue
		}

		if f >= rd.pkFieldCount {
			// rows have the same primary key, only content is different
			dr.mismatchedRows++
			advanceLeft = true
			advanceRight = true
			// Update the row on the destination.
			if err := rd.updateRow(left, right, DiffNotEqual); err != nil {
				return dr, err
			}
			continue
		}

		// have to find the 'smallest' row and advance it
		c, err := CompareRows(fields, rd.pkFieldCount, left, right)
		if err != nil {
			return dr, err
		}
		if c < 0 {
			dr.extraRowsLeft++
			advanceLeft = true
			// Add the row on the destination.
			if err := rd.reconcileRow(left, DiffMissing); err != nil {
				return dr, err
			}
			continue
		} else if c > 0 {
			dr.extraRowsRight++
			advanceRight = true
			// Delete the row from the destination.
			if err := rd.reconcileRow(right, DiffExtraneous); err != nil {
				return dr, err
			}
			continue
		}

		// Values of the primary key columns were not binary equal but their parsed
		// values are.
		// This happens when the raw values returned by MySQL were different but
		// they became equal after we parsed them into ints/floats
		// (due to leading/trailing zeros, for example). So this can happen if MySQL
		// is inconsistent in how it prints a given number.
		dr.mismatchedRows++
		advanceLeft = true
		advanceRight = true
		// Update the row on the destination.
		if err := rd.updateRow(left, right, DiffNotEqual); err != nil {
			return dr, err
		}
	}

	// Flush all aggregators in case they have buffered queries left.
	for i := range rd.aggregators {
		for _, aggregator := range rd.aggregators[i] {
			if err := aggregator.Flush(); err != nil {
				return dr, err
			}
		}
	}

	return dr, nil
}

// skipRow is used for the DiffType DiffEqual.
// Currently, it only updates the statistics and therefore does not require the
// row as input.
func (rd *RowDiffer2) skipRow() {
	rd.equalRowsStatsCounters.Add(rd.tableName, 1)
	rd.tableStatusList.addCopiedRows(rd.tableIndex, 1)
}

// reconcileRow is used for the DiffType DiffMissing and DiffExtraneous.
func (rd *RowDiffer2) reconcileRow(row []sqltypes.Value, typ DiffType) error {
	if typ == DiffNotEqual {
		panic(fmt.Sprintf("reconcileRow() called with wrong type: %v", typ))
	}
	destShardIndex, err := rd.router.Route(row)
	if err != nil {
		return vterrors.Wrapf(err, "failed to route row (%v) to correct shard", row)
	}

	if err := rd.aggregators[destShardIndex][typ].Add(row); err != nil {
		return vterrors.Wrap(err, "failed to add row update to RowAggregator")
	}
	// TODO(mberlin): Add more fine granular stats here.
	rd.tableStatusList.addCopiedRows(rd.tableIndex, 1)
	return nil
}

// updateRow is used for the DiffType DiffNotEqual.
// It needs to look at the row of the source (newRow) and destination (oldRow)
// to detect if the keyspace_id has changed in the meantime.
// If that's the case, we cannot UPDATE the row. Instead, we must DELETE
// the old row and INSERT the new row to the respective destination shards.
func (rd *RowDiffer2) updateRow(newRow, oldRow []sqltypes.Value, typ DiffType) error {
	if typ != DiffNotEqual {
		panic(fmt.Sprintf("updateRow() called with wrong type: %v", typ))
	}
	destShardIndexOld, err := rd.router.Route(oldRow)
	if err != nil {
		return vterrors.Wrapf(err, "failed to route old row (%v) to correct shard", oldRow)
	}
	destShardIndexNew, err := rd.router.Route(newRow)
	if err != nil {
		return vterrors.Wrapf(err, "failed to route new row (%v) to correct shard", newRow)
	}

	if destShardIndexOld == destShardIndexNew {
		// keyspace_id has not changed. Update the row in place on the destination.
		if err := rd.aggregators[destShardIndexNew][typ].Add(newRow); err != nil {
			return vterrors.Wrap(err, "failed to add row update to RowAggregator (keyspace_id not changed)")
		}
	} else {
		// keyspace_id has changed. Delete the old row and insert the new one.
		if err := rd.aggregators[destShardIndexOld][DiffExtraneous].Add(oldRow); err != nil {
			return vterrors.Wrap(err, "failed to add row update to RowAggregator (keyspace_id changed, deleting old row)")
		}
		if err := rd.aggregators[destShardIndexNew][DiffMissing].Add(newRow); err != nil {
			return vterrors.Wrap(err, "failed to add row update to RowAggregator (keyspace_id changed, inserting new row)")
		}
	}

	// TODO(mberlin): Add more fine granular stats here.
	rd.tableStatusList.addCopiedRows(rd.tableIndex, 1)
	return nil
}

// RowRouter allows to find out which shard's key range contains a given
// keyspace ID.
type RowRouter struct {
	keyResolver keyspaceIDResolver
	keyRanges   []*topodatapb.KeyRange
}

// NewRowRouter creates a RowRouter.
// We assume that the key ranges in shardInfo cover the full keyrange i.e.
// for any possible keyspaceID there is a shard we can route to.
func NewRowRouter(shardInfos []*topo.ShardInfo, keyResolver keyspaceIDResolver) RowRouter {
	keyRanges := make([]*topodatapb.KeyRange, len(shardInfos))
	for i, si := range shardInfos {
		keyRanges[i] = si.KeyRange
	}
	return RowRouter{keyResolver, keyRanges}
}

// Route returns which shard (specified by the index of the list of shards
// passed in NewRowRouter) contains the given row.
func (rr *RowRouter) Route(row []sqltypes.Value) (int, error) {
	if len(rr.keyRanges) == 1 {
		// Fast path when there is only one destination shard.
		return 0, nil
	}

	k, err := rr.keyResolver.keyspaceID(row)
	if err != nil {
		return -1, err
	}
	for i, kr := range rr.keyRanges {
		if key.KeyRangeContains(kr, k) {
			return i, nil
		}
	}
	return -1, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "no shard's key range includes the keyspace id: %v for the row: %#v", k, row)
}
<commit_msg>Fix off-by-1 error for row_differ counting rows processed<commit_after>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package worker

import (
	"fmt"
	"time"

	"vitess.io/vitess/go/vt/proto/vtrpc"
	"vitess.io/vitess/go/vt/vterrors"

	"golang.org/x/net/context"

	"vitess.io/vitess/go/sqltypes"
	"vitess.io/vitess/go/stats"
	"vitess.io/vitess/go/vt/key"
	"vitess.io/vitess/go/vt/topo"

	querypb "vitess.io/vitess/go/vt/proto/query"
	tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata"
	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
)

// DiffType specifies why a specific row was found as different when comparing
// a left and right side.
type DiffType int

const (
	// DiffMissing is returned when the row is missing on the right side.
	DiffMissing DiffType = iota
	// DiffNotEqual is returned when the row on the left and right side are
	// not equal.
	DiffNotEqual
	// DiffExtraneous is returned when the row exists on the right side, but not
	// on the left side.
	DiffExtraneous
	// DiffEqual is returned when the rows left and right are equal.
	DiffEqual
)

// DiffTypes has the list of available DiffType values, ordered by their value.
var DiffTypes = []DiffType{DiffMissing, DiffNotEqual, DiffExtraneous, DiffEqual}

// DiffFoundTypes has the list of DiffType values which represent that a
// difference was found. The list is ordered by the values of the types.
var DiffFoundTypes = []DiffType{DiffMissing, DiffNotEqual, DiffExtraneous}

// RowDiffer2 will compare and reconcile two sides. It assumes that the left
// side is the source of truth and necessary reconciliations have to be applied
// to the right side.
// It also assumes left and right are sorted by ascending primary key.
type RowDiffer2 struct {
	left         *RowReader
	right        *RowReader
	pkFieldCount int
	// tableStatusList is used to report the number of reconciled rows.
	tableStatusList *tableStatusList
	// tableIndex is the index of the table in the schema. It is required for
	// reporting the number of reconciled rows to tableStatusList.
	tableIndex int
	// router returns for a row to which destination shard index it should go.
	router RowRouter
	// aggregators are keyed by destination shard and DiffType.
	aggregators [][]*RowAggregator
	// equalRowsStatsCounters tracks per table how many rows are equal.
	equalRowsStatsCounters *stats.CountersWithSingleLabel
	// tableName is required to update "equalRowsStatsCounters".
	tableName string
}

// NewRowDiffer2 returns a new RowDiffer2.
// We assume that the indexes of the slice parameters always correspond to the
// same shard e.g. insertChannels[0] refers to destinationShards[0] and so on.
// The column list td.Columns must be have all primary key columns first and
// then the non-primary-key columns. The columns in the rows returned by
// both ResultReader must have the same order as td.Columns.
func NewRowDiffer2(ctx context.Context, left, right ResultReader, td *tabletmanagerdatapb.TableDefinition, tableStatusList *tableStatusList, tableIndex int,
	// Parameters required by RowRouter.
	destinationShards []*topo.ShardInfo, keyResolver keyspaceIDResolver,
	// Parameters required by RowAggregator.
	insertChannels []chan string, abort <-chan struct{}, dbNames []string, writeQueryMaxRows, writeQueryMaxSize int, statsCounters []*stats.CountersWithSingleLabel) (*RowDiffer2, error) {

	if len(statsCounters) != len(DiffTypes) {
		panic(fmt.Sprintf("statsCounter has the wrong number of elements. got = %v, want = %v", len(statsCounters), len(DiffTypes)))
	}

	if err := compareFields(left.Fields(), right.Fields()); err != nil {
		return nil, err
	}

	// Create a RowAggregator for each destination shard and DiffType.
	aggregators := make([][]*RowAggregator, len(destinationShards))
	for i := range destinationShards {
		aggregators[i] = make([]*RowAggregator, len(DiffFoundTypes))
		for _, typ := range DiffFoundTypes {
			maxRows := writeQueryMaxRows
			aggregators[i][typ] = NewRowAggregator(ctx, maxRows, writeQueryMaxSize,
				insertChannels[i], dbNames[i], td, typ, statsCounters[typ])
		}
	}

	return &RowDiffer2{
		left:                   NewRowReader(left),
		right:                  NewRowReader(right),
		pkFieldCount:           len(td.PrimaryKeyColumns),
		tableStatusList:        tableStatusList,
		tableIndex:             tableIndex,
		router:                 NewRowRouter(destinationShards, keyResolver),
		aggregators:            aggregators,
		equalRowsStatsCounters: statsCounters[DiffEqual],
		tableName:              td.Name,
	}, nil
}

func compareFields(left, right []*querypb.Field) error {
	if len(left) != len(right) {
		return vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "cannot diff inputs with different number of fields: left: %v right: %v", left, right)
	}
	for i, field := range left {
		if field.Type != right[i].Type {
			return vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "cannot diff inputs with different types: field %v types are %v and %v", i, field.Type, right[i].Type)
		}
	}
	return nil
}

// Diff runs the diff and reconcile.
// If an error occurs, it will return and stop.
func (rd *RowDiffer2) Diff() (DiffReport, error) {
	var dr DiffReport
	var err error

	dr.startingTime = time.Now()
	defer dr.ComputeQPS()

	fields := rd.left.Fields()
	var left []sqltypes.Value
	var right []sqltypes.Value
	advanceLeft := true
	advanceRight := true
	for {
		if advanceLeft {
			if left, err = rd.left.Next(); err != nil {
				return dr, err
			}
			advanceLeft = false
		}
		if advanceRight {
			if right, err = rd.right.Next(); err != nil {
				return dr, err
			}
			advanceRight = false
		}
		if left == nil && right == nil {
			// No more rows from either side. We're done.
			break
		}
		dr.processedRows++
		if left == nil {
			// No more rows on the left side.
			// We know we have at least one row on the right side left.
			// Delete the row from the destination.
			if err := rd.reconcileRow(right, DiffExtraneous); err != nil {
				return dr, err
			}
			dr.extraRowsRight++
			advanceRight = true
			continue
		}
		if right == nil {
			// No more rows on the right side.
			// We know we have at least one row on the left side left.
			// Add the row on the destination.
			if err := rd.reconcileRow(left, DiffMissing); err != nil {
				return dr, err
			}
			dr.extraRowsLeft++
			advanceLeft = true
			continue
		}

		// we have both left and right, compare
		f := RowsEqual(left, right)
		if f == -1 {
			// rows are the same, next
			dr.matchingRows++
			advanceLeft = true
			advanceRight = true
			rd.skipRow()
			continue
		}

		if f >= rd.pkFieldCount {
			// rows have the same primary key, only content is different
			dr.mismatchedRows++
			advanceLeft = true
			advanceRight = true
			// Update the row on the destination.
			if err := rd.updateRow(left, right, DiffNotEqual); err != nil {
				return dr, err
			}
			continue
		}

		// have to find the 'smallest' row and advance it
		c, err := CompareRows(fields, rd.pkFieldCount, left, right)
		if err != nil {
			return dr, err
		}
		if c < 0 {
			dr.extraRowsLeft++
			advanceLeft = true
			// Add the row on the destination.
			if err := rd.reconcileRow(left, DiffMissing); err != nil {
				return dr, err
			}
			continue
		} else if c > 0 {
			dr.extraRowsRight++
			advanceRight = true
			// Delete the row from the destination.
			if err := rd.reconcileRow(right, DiffExtraneous); err != nil {
				return dr, err
			}
			continue
		}

		// Values of the primary key columns were not binary equal but their parsed
		// values are.
		// This happens when the raw values returned by MySQL were different but
		// they became equal after we parsed them into ints/floats
		// (due to leading/trailing zeros, for example). So this can happen if MySQL
		// is inconsistent in how it prints a given number.
		dr.mismatchedRows++
		advanceLeft = true
		advanceRight = true
		// Update the row on the destination.
		if err := rd.updateRow(left, right, DiffNotEqual); err != nil {
			return dr, err
		}
	}

	// Flush all aggregators in case they have buffered queries left.
	for i := range rd.aggregators {
		for _, aggregator := range rd.aggregators[i] {
			if err := aggregator.Flush(); err != nil {
				return dr, err
			}
		}
	}

	return dr, nil
}

// skipRow is used for the DiffType DiffEqual.
// Currently, it only updates the statistics and therefore does not require the
// row as input.
func (rd *RowDiffer2) skipRow() {
	rd.equalRowsStatsCounters.Add(rd.tableName, 1)
	rd.tableStatusList.addCopiedRows(rd.tableIndex, 1)
}

// reconcileRow is used for the DiffType DiffMissing and DiffExtraneous.
func (rd *RowDiffer2) reconcileRow(row []sqltypes.Value, typ DiffType) error {
	if typ == DiffNotEqual {
		panic(fmt.Sprintf("reconcileRow() called with wrong type: %v", typ))
	}
	destShardIndex, err := rd.router.Route(row)
	if err != nil {
		return vterrors.Wrapf(err, "failed to route row (%v) to correct shard", row)
	}

	if err := rd.aggregators[destShardIndex][typ].Add(row); err != nil {
		return vterrors.Wrap(err, "failed to add row update to RowAggregator")
	}
	// TODO(mberlin): Add more fine granular stats here.
	rd.tableStatusList.addCopiedRows(rd.tableIndex, 1)
	return nil
}

// updateRow is used for the DiffType DiffNotEqual.
// It needs to look at the row of the source (newRow) and destination (oldRow)
// to detect if the keyspace_id has changed in the meantime.
// If that's the case, we cannot UPDATE the row. Instead, we must DELETE
// the old row and INSERT the new row to the respective destination shards.
func (rd *RowDiffer2) updateRow(newRow, oldRow []sqltypes.Value, typ DiffType) error {
	if typ != DiffNotEqual {
		panic(fmt.Sprintf("updateRow() called with wrong type: %v", typ))
	}
	destShardIndexOld, err := rd.router.Route(oldRow)
	if err != nil {
		return vterrors.Wrapf(err, "failed to route old row (%v) to correct shard", oldRow)
	}
	destShardIndexNew, err := rd.router.Route(newRow)
	if err != nil {
		return vterrors.Wrapf(err, "failed to route new row (%v) to correct shard", newRow)
	}

	if destShardIndexOld == destShardIndexNew {
		// keyspace_id has not changed. Update the row in place on the destination.
		if err := rd.aggregators[destShardIndexNew][typ].Add(newRow); err != nil {
			return vterrors.Wrap(err, "failed to add row update to RowAggregator (keyspace_id not changed)")
		}
	} else {
		// keyspace_id has changed. Delete the old row and insert the new one.
		if err := rd.aggregators[destShardIndexOld][DiffExtraneous].Add(oldRow); err != nil {
			return vterrors.Wrap(err, "failed to add row update to RowAggregator (keyspace_id changed, deleting old row)")
		}
		if err := rd.aggregators[destShardIndexNew][DiffMissing].Add(newRow); err != nil {
			return vterrors.Wrap(err, "failed to add row update to RowAggregator (keyspace_id changed, inserting new row)")
		}
	}

	// TODO(mberlin): Add more fine granular stats here.
	rd.tableStatusList.addCopiedRows(rd.tableIndex, 1)
	return nil
}

// RowRouter allows to find out which shard's key range contains a given
// keyspace ID.
type RowRouter struct {
	keyResolver keyspaceIDResolver
	keyRanges   []*topodatapb.KeyRange
}

// NewRowRouter creates a RowRouter.
// We assume that the key ranges in shardInfo cover the full keyrange i.e.
// for any possible keyspaceID there is a shard we can route to.
func NewRowRouter(shardInfos []*topo.ShardInfo, keyResolver keyspaceIDResolver) RowRouter {
	keyRanges := make([]*topodatapb.KeyRange, len(shardInfos))
	for i, si := range shardInfos {
		keyRanges[i] = si.KeyRange
	}
	return RowRouter{keyResolver, keyRanges}
}

// Route returns which shard (specified by the index of the list of shards
// passed in NewRowRouter) contains the given row.
func (rr *RowRouter) Route(row []sqltypes.Value) (int, error) {
	if len(rr.keyRanges) == 1 {
		// Fast path when there is only one destination shard.
		return 0, nil
	}

	k, err := rr.keyResolver.keyspaceID(row)
	if err != nil {
		return -1, err
	}
	for i, kr := range rr.keyRanges {
		if key.KeyRangeContains(kr, k) {
			return i, nil
		}
	}
	return -1, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "no shard's key range includes the keyspace id: %v for the row: %#v", k, row)
}
<|endoftext|>


<commit_before>package collections

import "sort"

// Sorter represents a collection that can be sorted.
type Sorter interface {
	Bounded
	Comparable
	Swappable
}

// stdlibSorter is a wrapper around a Sorter which can be used directly in
// sort.Sort.
type stdlibSorter struct {
	Sorter
}

// newSorter creates a new stdlibSorter containing the provided Sorter.
func newSorter(s Sorter) *stdlibSorter {
	return &stdlibSorter{s}
}

// Len returns the length of the underlying Sorter. If the Sorter returns
// a negative length, Len will panic.
func (s *stdlibSorter) Len() int {
	n := s.Sorter.Len()
	if n < 0 {
		panic("Len: negative length")
	}

	return n
}

// Less returns the result of the underlying Sorter's compare operation. If
// the Sorter returns an invalid Ord value, Less will panic.
func (s *stdlibSorter) Less(i, j int) bool {
	o := s.Compare(i, j)
	checkOrd("Less", o)

	return o == Less
}

// sort uses the Go stdlib sort to sort the items in this collection.
func (s *stdlibSorter) sort() {
	sort.Sort(s)
}

// Sort sorts the items in the collection using the standard Go sorting function.
func Sort(s Sorter) {
	newSorter(s).sort()
}
<commit_msg>Added reverse sort.<commit_after>package collections

import "sort"

// Sorter represents a collection that can be sorted.
type Sorter interface {
	Bounded
	Comparable
	Swappable
}

// reverseSorter is a wrapper around a Sorter that will always return results
// in reverse order.
type reverseSorter struct {
	Sorter
}

func (r reverseSorter) Compare(i, j int) Ord {
	o := r.Sorter.Compare(i, j)
	switch o {
	case Less:
		return Greater
	case Greater:
		return Less
	default:
		return o
	}
}

// Reverse creates a Sorter that will produce results in reverse order from
// the sorter passed in.
func Reverse(s Sorter) Sorter {
	return reverseSorter{
		Sorter: s,
	}

}

// stdlibSorter is a wrapper around a Sorter which can be used directly in
// sort.Sort.
type stdlibSorter struct {
	Sorter
}

// newSorter creates a new stdlibSorter containing the provided Sorter.
func newSorter(s Sorter) *stdlibSorter {
	return &stdlibSorter{s}
}

// Len returns the length of the underlying Sorter. If the Sorter returns
// a negative length, Len will panic.
func (s *stdlibSorter) Len() int {
	n := s.Sorter.Len()
	if n < 0 {
		panic("Len: negative length")
	}

	return n
}

// Less returns the result of the underlying Sorter's compare operation. If
// the Sorter returns an invalid Ord value, Less will panic.
func (s *stdlibSorter) Less(i, j int) bool {
	o := s.Compare(i, j)
	checkOrd("Less", o)

	return o == Less
}

// sort uses the Go stdlib sort to sort the items in this collection.
func (s *stdlibSorter) sort() {
	sort.Sort(s)
}

// Sort sorts the items in the collection using the standard Go sorting function.
func Sort(s Sorter) {
	newSorter(s).sort()
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package redirect provides hooks to register HTTP handlers that redirect old
// godoc paths to their new equivalents and assist in accessing the issue
// tracker, wiki, code review system, etc.
package redirect // import "golang.org/x/tools/godoc/redirect"

import (
	"fmt"
	"net/http"
	"os"
	"regexp"
	"strconv"
	"strings"
)

// Register registers HTTP handlers that redirect old godoc paths to their new
// equivalents and assist in accessing the issue tracker, wiki, code review
// system, etc. If mux is nil it uses http.DefaultServeMux.
func Register(mux *http.ServeMux) {
	if mux == nil {
		mux = http.DefaultServeMux
	}
	handlePathRedirects(mux, pkgRedirects, "/pkg/")
	handlePathRedirects(mux, cmdRedirects, "/cmd/")
	for prefix, redirect := range prefixHelpers {
		p := "/" + prefix + "/"
		mux.Handle(p, PrefixHandler(p, redirect))
	}
	for path, redirect := range redirects {
		mux.Handle(path, Handler(redirect))
	}
	// NB: /src/pkg (sans trailing slash) is the index of packages.
	mux.HandleFunc("/src/pkg/", srcPkgHandler)
	mux.HandleFunc("/cl/", clHandler)
	mux.HandleFunc("/change/", changeHandler)
}

func handlePathRedirects(mux *http.ServeMux, redirects map[string]string, prefix string) {
	for source, target := range redirects {
		h := Handler(prefix + target + "/")
		p := prefix + source
		mux.Handle(p, h)
		mux.Handle(p+"/", h)
	}
}

// Packages that were renamed between r60 and go1.
var pkgRedirects = map[string]string{
	"asn1":              "encoding/asn1",
	"big":               "math/big",
	"cmath":             "math/cmplx",
	"csv":               "encoding/csv",
	"exec":              "os/exec",
	"exp/template/html": "html/template",
	"gob":               "encoding/gob",
	"http":              "net/http",
	"http/cgi":          "net/http/cgi",
	"http/fcgi":         "net/http/fcgi",
	"http/httptest":     "net/http/httptest",
	"http/pprof":        "net/http/pprof",
	"json":              "encoding/json",
	"mail":              "net/mail",
	"rand":              "math/rand",
	"rpc":               "net/rpc",
	"rpc/jsonrpc":       "net/rpc/jsonrpc",
	"scanner":           "text/scanner",
	"smtp":              "net/smtp",
	"tabwriter":         "text/tabwriter",
	"template":          "text/template",
	"template/parse":    "text/template/parse",
	"url":               "net/url",
	"utf16":             "unicode/utf16",
	"utf8":              "unicode/utf8",
	"xml":               "encoding/xml",
}

// Commands that were renamed between r60 and go1.
var cmdRedirects = map[string]string{
	"gofix":     "fix",
	"goinstall": "go",
	"gopack":    "pack",
	"gotest":    "go",
	"govet":     "vet",
	"goyacc":    "yacc",
}

var redirects = map[string]string{
	"/blog":       "/blog/",
	"/build":      "http://build.golang.org",
	"/change":     "https://go.googlesource.com/go",
	"/cl":         "https://go-review.googlesource.com",
	"/cmd/godoc/": "http://godoc.org/golang.org/x/tools/cmd/godoc/",
	"/cmd/vet/":   "http://godoc.org/golang.org/x/tools/cmd/vet/",
	"/issue":      "https://github.com/golang/go/issues",
	"/issue/new":  "https://github.com/golang/go/issues/new",
	"/issues":     "https://github.com/golang/go/issues",
	"/play":       "http://play.golang.org",

	// In Go 1.2 the references page is part of /doc/.
	"/ref": "/doc/#references",
	// This next rule clobbers /ref/spec and /ref/mem.
	// TODO(adg): figure out what to do here, if anything.
	// "/ref/": "/doc/#references",

	// Be nice to people who are looking in the wrong place.
	"/doc/mem":  "/ref/mem",
	"/doc/spec": "/ref/spec",

	"/talks": "http://talks.golang.org",
	"/tour":  "http://tour.golang.org",
	"/wiki":  "https://github.com/golang/go/wiki",

	"/doc/articles/c_go_cgo.html":                    "/blog/c-go-cgo",
	"/doc/articles/concurrency_patterns.html":        "/blog/go-concurrency-patterns-timing-out-and",
	"/doc/articles/defer_panic_recover.html":         "/blog/defer-panic-and-recover",
	"/doc/articles/error_handling.html":              "/blog/error-handling-and-go",
	"/doc/articles/gobs_of_data.html":                "/blog/gobs-of-data",
	"/doc/articles/godoc_documenting_go_code.html":   "/blog/godoc-documenting-go-code",
	"/doc/articles/gos_declaration_syntax.html":      "/blog/gos-declaration-syntax",
	"/doc/articles/image_draw.html":                  "/blog/go-imagedraw-package",
	"/doc/articles/image_package.html":               "/blog/go-image-package",
	"/doc/articles/json_and_go.html":                 "/blog/json-and-go",
	"/doc/articles/json_rpc_tale_of_interfaces.html": "/blog/json-rpc-tale-of-interfaces",
	"/doc/articles/laws_of_reflection.html":          "/blog/laws-of-reflection",
	"/doc/articles/slices_usage_and_internals.html":  "/blog/go-slices-usage-and-internals",
	"/doc/go_for_cpp_programmers.html":               "/wiki/GoForCPPProgrammers",
	"/doc/go_tutorial.html":                          "http://tour.golang.org/",
}

var prefixHelpers = map[string]string{
	"issue": "https://github.com/golang/go/issues/",
	"play":  "http://play.golang.org/",
	"talks": "http://talks.golang.org/",
	"wiki":  "https://github.com/golang/go/wiki/",
}

func Handler(target string) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, target, http.StatusMovedPermanently)
	})
}

var validId = regexp.MustCompile(`^[A-Za-z0-9-]*$`)

func PrefixHandler(prefix, baseURL string) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if p := r.URL.Path; p == prefix {
			// redirect /prefix/ to /prefix
			http.Redirect(w, r, p[:len(p)-1], http.StatusFound)
			return
		}
		id := r.URL.Path[len(prefix):]
		if !validId.MatchString(id) {
			http.Error(w, "Not found", http.StatusNotFound)
			return
		}
		target := baseURL + id
		http.Redirect(w, r, target, http.StatusFound)
	})
}

// Redirect requests from the old "/src/pkg/foo" to the new "/src/foo".
// See http://golang.org/s/go14nopkg
func srcPkgHandler(w http.ResponseWriter, r *http.Request) {
	r.URL.Path = "/src/" + r.URL.Path[len("/src/pkg/"):]
	http.Redirect(w, r, r.URL.String(), http.StatusMovedPermanently)
}

func clHandler(w http.ResponseWriter, r *http.Request) {
	const prefix = "/cl/"
	if p := r.URL.Path; p == prefix {
		// redirect /prefix/ to /prefix
		http.Redirect(w, r, p[:len(p)-1], http.StatusFound)
		return
	}
	id := r.URL.Path[len(prefix):]
	// support /cl/152700045/, which is used in commit 0edafefc36.
	id = strings.TrimSuffix(id, "/")
	if !validId.MatchString(id) {
		http.Error(w, "Not found", http.StatusNotFound)
		return
	}
	target := ""
	// the first CL in rietveld is about 152046, so only treat the id as
	// a rietveld CL if it is larger than 150000.
	if n, err := strconv.Atoi(id); err == nil && n > 150000 {
		target = "https://codereview.appspot.com/" + id
	} else {
		target = "https://go-review.googlesource.com/r/" + id
	}
	http.Redirect(w, r, target, http.StatusFound)
}

var changeMap *hashMap

// LoadChangeMap loads the specified map of Mercurial to Git revisions,
// which is used by the /change/ handler to intelligently map old hg
// revisions to their new git equivalents.
// It should be called before calling Register.
// The file should remain open as long as the process is running.
// See the implementation of this package for details.
func LoadChangeMap(filename string) error {
	f, err := os.Open(filename)
	if err != nil {
		return err
	}
	m, err := newHashMap(f)
	if err != nil {
		return err
	}
	changeMap = m
	return nil
}

func changeHandler(w http.ResponseWriter, r *http.Request) {
	const prefix = "/change/"
	if p := r.URL.Path; p == prefix {
		// redirect /prefix/ to /prefix
		http.Redirect(w, r, p[:len(p)-1], http.StatusFound)
		return
	}
	hash := r.URL.Path[len(prefix):]
	target := "https://go.googlesource.com/go/+/" + hash
	if git := changeMap.Lookup(hash); git > 0 {
		target = fmt.Sprintf("https://go.googlesource.com/%v/+/%v", git.Repo(), git.Hash())
	}
	http.Redirect(w, r, target, http.StatusFound)
}
<commit_msg>godoc/redirect: make /issue and /issues identical<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package redirect provides hooks to register HTTP handlers that redirect old
// godoc paths to their new equivalents and assist in accessing the issue
// tracker, wiki, code review system, etc.
package redirect // import "golang.org/x/tools/godoc/redirect"

import (
	"fmt"
	"net/http"
	"os"
	"regexp"
	"strconv"
	"strings"
)

// Register registers HTTP handlers that redirect old godoc paths to their new
// equivalents and assist in accessing the issue tracker, wiki, code review
// system, etc. If mux is nil it uses http.DefaultServeMux.
func Register(mux *http.ServeMux) {
	if mux == nil {
		mux = http.DefaultServeMux
	}
	handlePathRedirects(mux, pkgRedirects, "/pkg/")
	handlePathRedirects(mux, cmdRedirects, "/cmd/")
	for prefix, redirect := range prefixHelpers {
		p := "/" + prefix + "/"
		mux.Handle(p, PrefixHandler(p, redirect))
	}
	for path, redirect := range redirects {
		mux.Handle(path, Handler(redirect))
	}
	// NB: /src/pkg (sans trailing slash) is the index of packages.
	mux.HandleFunc("/src/pkg/", srcPkgHandler)
	mux.HandleFunc("/cl/", clHandler)
	mux.HandleFunc("/change/", changeHandler)
}

func handlePathRedirects(mux *http.ServeMux, redirects map[string]string, prefix string) {
	for source, target := range redirects {
		h := Handler(prefix + target + "/")
		p := prefix + source
		mux.Handle(p, h)
		mux.Handle(p+"/", h)
	}
}

// Packages that were renamed between r60 and go1.
var pkgRedirects = map[string]string{
	"asn1":              "encoding/asn1",
	"big":               "math/big",
	"cmath":             "math/cmplx",
	"csv":               "encoding/csv",
	"exec":              "os/exec",
	"exp/template/html": "html/template",
	"gob":               "encoding/gob",
	"http":              "net/http",
	"http/cgi":          "net/http/cgi",
	"http/fcgi":         "net/http/fcgi",
	"http/httptest":     "net/http/httptest",
	"http/pprof":        "net/http/pprof",
	"json":              "encoding/json",
	"mail":              "net/mail",
	"rand":              "math/rand",
	"rpc":               "net/rpc",
	"rpc/jsonrpc":       "net/rpc/jsonrpc",
	"scanner":           "text/scanner",
	"smtp":              "net/smtp",
	"tabwriter":         "text/tabwriter",
	"template":          "text/template",
	"template/parse":    "text/template/parse",
	"url":               "net/url",
	"utf16":             "unicode/utf16",
	"utf8":              "unicode/utf8",
	"xml":               "encoding/xml",
}

// Commands that were renamed between r60 and go1.
var cmdRedirects = map[string]string{
	"gofix":     "fix",
	"goinstall": "go",
	"gopack":    "pack",
	"gotest":    "go",
	"govet":     "vet",
	"goyacc":    "yacc",
}

var redirects = map[string]string{
	"/blog":       "/blog/",
	"/build":      "http://build.golang.org",
	"/change":     "https://go.googlesource.com/go",
	"/cl":         "https://go-review.googlesource.com",
	"/cmd/godoc/": "http://godoc.org/golang.org/x/tools/cmd/godoc/",
	"/cmd/vet/":   "http://godoc.org/golang.org/x/tools/cmd/vet/",
	"/issue":      "https://github.com/golang/go/issues",
	"/issue/new":  "https://github.com/golang/go/issues/new",
	"/issues":     "https://github.com/golang/go/issues",
	"/issues/new": "https://github.com/golang/go/issues/new",
	"/play":       "http://play.golang.org",

	// In Go 1.2 the references page is part of /doc/.
	"/ref": "/doc/#references",
	// This next rule clobbers /ref/spec and /ref/mem.
	// TODO(adg): figure out what to do here, if anything.
	// "/ref/": "/doc/#references",

	// Be nice to people who are looking in the wrong place.
	"/doc/mem":  "/ref/mem",
	"/doc/spec": "/ref/spec",

	"/talks": "http://talks.golang.org",
	"/tour":  "http://tour.golang.org",
	"/wiki":  "https://github.com/golang/go/wiki",

	"/doc/articles/c_go_cgo.html":                    "/blog/c-go-cgo",
	"/doc/articles/concurrency_patterns.html":        "/blog/go-concurrency-patterns-timing-out-and",
	"/doc/articles/defer_panic_recover.html":         "/blog/defer-panic-and-recover",
	"/doc/articles/error_handling.html":              "/blog/error-handling-and-go",
	"/doc/articles/gobs_of_data.html":                "/blog/gobs-of-data",
	"/doc/articles/godoc_documenting_go_code.html":   "/blog/godoc-documenting-go-code",
	"/doc/articles/gos_declaration_syntax.html":      "/blog/gos-declaration-syntax",
	"/doc/articles/image_draw.html":                  "/blog/go-imagedraw-package",
	"/doc/articles/image_package.html":               "/blog/go-image-package",
	"/doc/articles/json_and_go.html":                 "/blog/json-and-go",
	"/doc/articles/json_rpc_tale_of_interfaces.html": "/blog/json-rpc-tale-of-interfaces",
	"/doc/articles/laws_of_reflection.html":          "/blog/laws-of-reflection",
	"/doc/articles/slices_usage_and_internals.html":  "/blog/go-slices-usage-and-internals",
	"/doc/go_for_cpp_programmers.html":               "/wiki/GoForCPPProgrammers",
	"/doc/go_tutorial.html":                          "http://tour.golang.org/",
}

var prefixHelpers = map[string]string{
	"issue":  "https://github.com/golang/go/issues/",
	"issues": "https://github.com/golang/go/issues/",
	"play":   "http://play.golang.org/",
	"talks":  "http://talks.golang.org/",
	"wiki":   "https://github.com/golang/go/wiki/",
}

func Handler(target string) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, target, http.StatusMovedPermanently)
	})
}

var validId = regexp.MustCompile(`^[A-Za-z0-9-]*$`)

func PrefixHandler(prefix, baseURL string) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if p := r.URL.Path; p == prefix {
			// redirect /prefix/ to /prefix
			http.Redirect(w, r, p[:len(p)-1], http.StatusFound)
			return
		}
		id := r.URL.Path[len(prefix):]
		if !validId.MatchString(id) {
			http.Error(w, "Not found", http.StatusNotFound)
			return
		}
		target := baseURL + id
		http.Redirect(w, r, target, http.StatusFound)
	})
}

// Redirect requests from the old "/src/pkg/foo" to the new "/src/foo".
// See http://golang.org/s/go14nopkg
func srcPkgHandler(w http.ResponseWriter, r *http.Request) {
	r.URL.Path = "/src/" + r.URL.Path[len("/src/pkg/"):]
	http.Redirect(w, r, r.URL.String(), http.StatusMovedPermanently)
}

func clHandler(w http.ResponseWriter, r *http.Request) {
	const prefix = "/cl/"
	if p := r.URL.Path; p == prefix {
		// redirect /prefix/ to /prefix
		http.Redirect(w, r, p[:len(p)-1], http.StatusFound)
		return
	}
	id := r.URL.Path[len(prefix):]
	// support /cl/152700045/, which is used in commit 0edafefc36.
	id = strings.TrimSuffix(id, "/")
	if !validId.MatchString(id) {
		http.Error(w, "Not found", http.StatusNotFound)
		return
	}
	target := ""
	// the first CL in rietveld is about 152046, so only treat the id as
	// a rietveld CL if it is larger than 150000.
	if n, err := strconv.Atoi(id); err == nil && n > 150000 {
		target = "https://codereview.appspot.com/" + id
	} else {
		target = "https://go-review.googlesource.com/r/" + id
	}
	http.Redirect(w, r, target, http.StatusFound)
}

var changeMap *hashMap

// LoadChangeMap loads the specified map of Mercurial to Git revisions,
// which is used by the /change/ handler to intelligently map old hg
// revisions to their new git equivalents.
// It should be called before calling Register.
// The file should remain open as long as the process is running.
// See the implementation of this package for details.
func LoadChangeMap(filename string) error {
	f, err := os.Open(filename)
	if err != nil {
		return err
	}
	m, err := newHashMap(f)
	if err != nil {
		return err
	}
	changeMap = m
	return nil
}

func changeHandler(w http.ResponseWriter, r *http.Request) {
	const prefix = "/change/"
	if p := r.URL.Path; p == prefix {
		// redirect /prefix/ to /prefix
		http.Redirect(w, r, p[:len(p)-1], http.StatusFound)
		return
	}
	hash := r.URL.Path[len(prefix):]
	target := "https://go.googlesource.com/go/+/" + hash
	if git := changeMap.Lookup(hash); git > 0 {
		target = fmt.Sprintf("https://go.googlesource.com/%v/+/%v", git.Repo(), git.Hash())
	}
	http.Redirect(w, r, target, http.StatusFound)
}
<|endoftext|>


<commit_before>/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package portgroup

import (
	"context"
	"flag"
	"fmt"
	"strconv"
	"strings"

	"github.com/vmware/govmomi/govc/flags"
	"github.com/vmware/govmomi/vim25/types"
)

type VLANMode string

const (
	VLAN      = VLANMode("vlan")
	TrunkVLAN = VLANMode("trunking")
)

type DVPortgroupConfigSpec struct {
	types.DVPortgroupConfigSpec
}

var (
	vlanMode string
	vlanId int32
	vlanRange string
	vlanSpec types.BaseVmwareDistributedVirtualSwitchVlanSpec
)

func (spec *DVPortgroupConfigSpec) Register(ctx context.Context, f *flag.FlagSet) {
	ptypes := []string{
		string(types.DistributedVirtualPortgroupPortgroupTypeEarlyBinding),
		string(types.DistributedVirtualPortgroupPortgroupTypeLateBinding),
		string(types.DistributedVirtualPortgroupPortgroupTypeEphemeral),
	}

	vlanModes := []string{
		string(VLAN),
		string(TrunkVLAN),
	}

	f.StringVar(&spec.Type, "type", ptypes[0],
		fmt.Sprintf("Portgroup type (%s)", strings.Join(ptypes, "|")))
	f.Var(flags.NewInt32(&spec.NumPorts), "nports", "Number of ports")
	f.Var(flags.NewInt32(&vlanId), "vlan", "VLAN ID")
	f.StringVar(&vlanRange, "vlan-range", "0-4094", "VLAN Ranges with comma delimited")
	f.StringVar(&vlanMode, "vlan-mode", "vlan", fmt.Sprintf("vlan mode (%s)", strings.Join(vlanModes, "|")))
}

func getRange(vlanRange string) []types.NumericRange {
	nRanges := make([]types.NumericRange, 0)
	strRanges := strings.Split(vlanRange, ",")
	for _, v := range strRanges {
		vlans := strings.Split(v, "-")
		if len(vlans) != 2 {
			panic(fmt.Sprintf("range %s does not follow format with vlanId-vlanId", v))
		}
		start, err := strconv.Atoi(vlans[0])
		if err != nil {
			panic(fmt.Sprintf("range %s does not follow format with vlanId-vlanId error: %v", v, err))
		}
		end, err := strconv.Atoi(vlans[1])
		if err != nil {
			panic(fmt.Sprintf("range %s does not follow format with vlanId-vlanId error: %v", v, err))
		}
		nRanges = append(nRanges, types.NumericRange{
			Start: int32(start),
			End:   int32(end),
		})
	}
	return nRanges
}

func (spec *DVPortgroupConfigSpec) Spec() types.DVPortgroupConfigSpec {
	config := new(types.VMwareDVSPortSetting)
	switch VLANMode(vlanMode) {
	case VLAN:
		spec := new(types.VmwareDistributedVirtualSwitchVlanIdSpec)
		spec.VlanId = vlanId
		vlanSpec = spec
	case TrunkVLAN:
		spec := new(types.VmwareDistributedVirtualSwitchTrunkVlanSpec)
		spec.VlanId = getRange(vlanRange)
		vlanSpec = spec
	}
	config.Vlan = vlanSpec
	spec.DefaultPortConfig = config
	return spec.DVPortgroupConfigSpec
}
<commit_msg>fix the goimports validation error<commit_after>/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package portgroup

import (
	"context"
	"flag"
	"fmt"
	"strconv"
	"strings"

	"github.com/vmware/govmomi/govc/flags"
	"github.com/vmware/govmomi/vim25/types"
)

type VLANMode string

const (
	VLAN      = VLANMode("vlan")
	TrunkVLAN = VLANMode("trunking")
)

type DVPortgroupConfigSpec struct {
	types.DVPortgroupConfigSpec
}

var (
	vlanMode  string
	vlanId    int32
	vlanRange string
	vlanSpec  types.BaseVmwareDistributedVirtualSwitchVlanSpec
)

func (spec *DVPortgroupConfigSpec) Register(ctx context.Context, f *flag.FlagSet) {
	ptypes := []string{
		string(types.DistributedVirtualPortgroupPortgroupTypeEarlyBinding),
		string(types.DistributedVirtualPortgroupPortgroupTypeLateBinding),
		string(types.DistributedVirtualPortgroupPortgroupTypeEphemeral),
	}

	vlanModes := []string{
		string(VLAN),
		string(TrunkVLAN),
	}

	f.StringVar(&spec.Type, "type", ptypes[0],
		fmt.Sprintf("Portgroup type (%s)", strings.Join(ptypes, "|")))
	f.Var(flags.NewInt32(&spec.NumPorts), "nports", "Number of ports")
	f.Var(flags.NewInt32(&vlanId), "vlan", "VLAN ID")
	f.StringVar(&vlanRange, "vlan-range", "0-4094", "VLAN Ranges with comma delimited")
	f.StringVar(&vlanMode, "vlan-mode", "vlan", fmt.Sprintf("vlan mode (%s)", strings.Join(vlanModes, "|")))
}

func getRange(vlanRange string) []types.NumericRange {
	nRanges := make([]types.NumericRange, 0)
	strRanges := strings.Split(vlanRange, ",")
	for _, v := range strRanges {
		vlans := strings.Split(v, "-")
		if len(vlans) != 2 {
			panic(fmt.Sprintf("range %s does not follow format with vlanId-vlanId", v))
		}
		start, err := strconv.Atoi(vlans[0])
		if err != nil {
			panic(fmt.Sprintf("range %s does not follow format with vlanId-vlanId error: %v", v, err))
		}
		end, err := strconv.Atoi(vlans[1])
		if err != nil {
			panic(fmt.Sprintf("range %s does not follow format with vlanId-vlanId error: %v", v, err))
		}
		nRanges = append(nRanges, types.NumericRange{
			Start: int32(start),
			End:   int32(end),
		})
	}
	return nRanges
}

func (spec *DVPortgroupConfigSpec) Spec() types.DVPortgroupConfigSpec {
	config := new(types.VMwareDVSPortSetting)
	switch VLANMode(vlanMode) {
	case VLAN:
		spec := new(types.VmwareDistributedVirtualSwitchVlanIdSpec)
		spec.VlanId = vlanId
		vlanSpec = spec
	case TrunkVLAN:
		spec := new(types.VmwareDistributedVirtualSwitchTrunkVlanSpec)
		spec.VlanId = getRange(vlanRange)
		vlanSpec = spec
	}
	config.Vlan = vlanSpec
	spec.DefaultPortConfig = config
	return spec.DVPortgroupConfigSpec
}
<|endoftext|>


<commit_before>package spec

import (
	"strings"
	"testing"
)

type G func(string, func(), ...Option)

func (g G) Pend(text string, f func(), _ ...Option) {
	g(text, f, func(c *config) { c.pend = true })
}

func (g G) Focus(text string, f func(), opts ...Option) {
	g(text, f, append(opts, func(c *config) { c.focus = true })...)
}

type S func(string, func(), ...Option)

func (s S) Before(f func()) {
	s("", f, func(c *config) { c.before = true })
}

func (s S) After(f func()) {
	s("", f, func(c *config) { c.after = true })
}

func (s S) Pend(text string, f func(), _ ...Option) {
	s(text, f, func(c *config) { c.pend = true })
}

func (s S) Focus(text string, f func(), opts ...Option) {
	s(text, f, append(opts, func(c *config) { c.focus = true })...)
}

func Run(t *testing.T, f func(*testing.T, G, S), opts ...Option) bool {
	success := true
	specs, focused := parse(f, opts...)

	for _, s := range specs {
		s := s
		name := strings.Join(s.name, "/")
		success = success && t.Run(name, func(t *testing.T) {
			switch {
			case s.pend, focused && !s.focus:
				t.SkipNow()
			case s.parallel:
				t.Parallel()
			}
			var (
				spec          func()
				before, after []func()
			)
			f(t, func(_ string, f func(), _ ...Option) {
				switch {
				case len(s.groups) == 0:
				case s.groups[0] > 0:
					s.groups[0]--
				default:
					s.groups = s.groups[1:]
					f()
				}
			}, func(text string, f func(), opts ...Option) {
				cfg := options(opts).apply()
				switch {
				case cfg.before:
					before = append(before, f)
				case cfg.after:
					after = append([]func(){f}, after...)
				case spec != nil || len(s.groups) > 0:
				case s.index > 0:
					s.index--
				default:
					spec = f
				}
			})

			if spec == nil {
				t.Fatalf("Failed to parse: %s", name)
			}

			run(before...)
			defer run(after...)
			run(spec)
		})
	}

	return success
}

func run(fs ...func()) {
	for _, f := range fs {
		f()
	}
}

type specInfo struct {
	name     []string
	pend     bool
	focus    bool
	parallel bool
	groups   []uint64
	index    uint64
}

func parse(f func(*testing.T, G, S), opts ...Option) (specs []specInfo, focused bool) {
	var (
		global     = options(opts).apply()
		groups     groupStack
		groupIndex uint64
		specIndex  uint64
	)

	f(nil, func(text string, f func(), opts ...Option) {
		cfg := options(opts).apply()

		groups = groups.push(cfg, text, groupIndex, specIndex)
		defer func() { groups, groupIndex, specIndex = groups.pop() }()

		groupIndex, specIndex = 0, 0
		focused = focused || groups.focused()

		f()
	}, func(text string, _ func(), opts ...Option) {
		cfg := options(opts).apply()
		if cfg.before || cfg.after {
			return
		}
		spec := groups.spec(cfg, text, specIndex)

		spec.parallel = spec.parallel || global.parallel
		focused = focused || spec.focus && !spec.pend

		specs = append(specs, spec)
		specIndex++
	})
	return specs, focused
}

type groupInfo struct {
	text       string
	pend       bool
	focus      bool
	parallel   bool
	groupIndex uint64
	specIndex  uint64
}

type groupStack []groupInfo

func (g groupStack) push(cfg *config, text string, groupIndex, specIndex uint64) groupStack {
	group := g.last()
	return append(g, groupInfo{
		text:       text,
		pend:       group.pend || cfg.pend,
		focus:      group.focus || cfg.focus,
		parallel:   group.parallel || cfg.parallel,
		groupIndex: groupIndex,
		specIndex:  specIndex,
	})
}

func (g groupStack) pop() (stack groupStack, groupIndex, specIndex uint64) {
	group := g.last()
	return g[:len(g)-1], group.groupIndex + 1, group.specIndex
}

func (g groupStack) last() groupInfo {
	if len(g) == 0 {
		return groupInfo{}
	}
	return g[len(g)-1]
}

func (g groupStack) focused() bool {
	group := g.last()
	return group.focus && !group.pend
}

func (g groupStack) spec(cfg *config, text string, specIndex uint64) specInfo {
	group := g.last()
	spec := specInfo{
		pend:     cfg.pend || group.pend,
		focus:    cfg.focus || group.focus,
		parallel: cfg.parallel || group.parallel,
		index:    specIndex,
	}
	for _, group := range g {
		spec.name = append(spec.name, group.text)
		spec.groups = append(spec.groups, group.groupIndex)
	}
	spec.name = append(spec.name, text)
	return spec
}

type Option func(*config)

func Parallel() Option {
	return func(c *config) {
		c.parallel = true
	}
}

type config struct {
	pend     bool
	focus    bool
	parallel bool
	before   bool
	after    bool
}

type options []Option

func (o options) apply() *config {
	cfg := &config{}
	for _, opt := range o {
		opt(cfg)
	}
	return cfg
}
<commit_msg>Continued group refactor<commit_after>package spec

import (
	"strings"
	"testing"
)

type G func(string, func(), ...Option)

func (g G) Pend(text string, f func(), _ ...Option) {
	g(text, f, func(c *config) { c.pend = true })
}

func (g G) Focus(text string, f func(), opts ...Option) {
	g(text, f, append(opts, func(c *config) { c.focus = true })...)
}

type S func(string, func(), ...Option)

func (s S) Before(f func()) {
	s("", f, func(c *config) { c.before = true })
}

func (s S) After(f func()) {
	s("", f, func(c *config) { c.after = true })
}

func (s S) Pend(text string, f func(), _ ...Option) {
	s(text, f, func(c *config) { c.pend = true })
}

func (s S) Focus(text string, f func(), opts ...Option) {
	s(text, f, append(opts, func(c *config) { c.focus = true })...)
}

func Run(t *testing.T, f func(*testing.T, G, S), opts ...Option) bool {
	success := true
	specs, focused := parse(f, opts...)

	for _, s := range specs {
		s := s
		name := strings.Join(s.name, "/")
		success = success && t.Run(name, func(t *testing.T) {
			switch {
			case s.pend, focused && !s.focus:
				t.SkipNow()
			case s.parallel:
				t.Parallel()
			}
			var (
				spec          func()
				before, after []func()
			)
			f(t, func(_ string, f func(), _ ...Option) {
				switch {
				case len(s.groups) == 0:
				case s.groups[0] > 0:
					s.groups[0]--
				default:
					s.groups = s.groups[1:]
					f()
				}
			}, func(text string, f func(), opts ...Option) {
				cfg := options(opts).apply()
				switch {
				case cfg.before:
					before = append(before, f)
				case cfg.after:
					after = append([]func(){f}, after...)
				case spec != nil || len(s.groups) > 0:
				case s.index > 0:
					s.index--
				default:
					spec = f
				}
			})

			if spec == nil {
				t.Fatalf("Failed to parse: %s", name)
			}

			run(before...)
			defer run(after...)
			run(spec)
		})
	}

	return success
}

func run(fs ...func()) {
	for _, f := range fs {
		f()
	}
}

type specInfo struct {
	name     []string
	pend     bool
	focus    bool
	parallel bool
	groups   []uint64
	index    uint64
}

func parse(f func(*testing.T, G, S), opts ...Option) (specs []specInfo, focused bool) {
	var groups groupStack
	global := options(opts).apply()

	f(nil, func(text string, f func(), opts ...Option) {
		cfg := options(opts).apply()
		groups.push(cfg, text)
		defer groups.pop()
		focused = focused || groups.focused()
		f()
	}, func(text string, _ func(), opts ...Option) {
		cfg := options(opts).apply()
		cfg.parallel = cfg.parallel || global.parallel
		spec, ok := groups.spec(cfg, text)
		if !ok {
			return
		}
		focused = focused || spec.focus && !spec.pend
		specs = append(specs, spec)
	})
	return specs, focused
}

type groupInfo struct {
	text       string
	pend       bool
	focus      bool
	parallel   bool
	groupIndex uint64
	specIndex  uint64
}

type groupStack struct {
	groups     []groupInfo
	groupIndex uint64
	specIndex  uint64
}

func (g *groupStack) push(cfg *config, text string) {
	last := g.last()
	g.groups = append(g.groups, groupInfo{
		text:       text,
		pend:       last.pend || cfg.pend,
		focus:      last.focus || cfg.focus,
		parallel:   last.parallel || cfg.parallel,
		groupIndex: g.groupIndex,
		specIndex:  g.specIndex,
	})
	g.groupIndex, g.specIndex = 0, 0
}

func (g *groupStack) pop() {
	l := len(g.groups) - 1
	g.groupIndex = g.groups[l].groupIndex + 1
	g.specIndex = g.groups[l].specIndex
	g.groups = g.groups[:l]
}

func (g *groupStack) last() groupInfo {
	if len(g.groups) == 0 {
		return groupInfo{}
	}
	return g.groups[len(g.groups)-1]
}

func (g *groupStack) focused() bool {
	last := g.last()
	return last.focus && !last.pend
}

func (g *groupStack) spec(cfg *config, text string) (specInfo, bool) {
	if cfg.before || cfg.after {
		return specInfo{}, false
	}
	last := g.last()
	spec := specInfo{
		pend:     cfg.pend || last.pend,
		focus:    cfg.focus || last.focus,
		parallel: cfg.parallel || last.parallel,
		index:    g.specIndex,
	}
	for _, group := range g.groups {
		spec.name = append(spec.name, group.text)
		spec.groups = append(spec.groups, group.groupIndex)
	}
	spec.name = append(spec.name, text)
	g.specIndex++
	return spec, true
}

type Option func(*config)

func Parallel() Option {
	return func(c *config) {
		c.parallel = true
	}
}

type config struct {
	pend     bool
	focus    bool
	parallel bool
	before   bool
	after    bool
}

type options []Option

func (o options) apply() *config {
	cfg := &config{}
	for _, opt := range o {
		opt(cfg)
	}
	return cfg
}
<|endoftext|>


<commit_before>// Listener capture TCP traffic using RAW SOCKETS.
// Note: it requires sudo or root access.
//
// Rigt now it suport only HTTP
package listener

import (
	"bufio"
	"bytes"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"strconv"
	"time"
)

// Enable debug logging only if "--verbose" flag passed
func Debug(v ...interface{}) {
	if Settings.Verbose {
		log.Println(v...)
	}
}

func ReplayServer() (conn net.Conn, err error) {
	// Connection to reaplay server
	conn, err = net.Dial("tcp", Settings.ReplayServer())

	if err != nil {
		log.Println("Connection error ", err, Settings.ReplayAddress)
	}

	return
}

// Because its sub-program, Run acts as `main`
func Run() {
	if os.Getuid() != 0 {
		fmt.Println("Please start the listener as root or sudo!")
		fmt.Println("This is required since listener sniff traffic on given port.")
		os.Exit(1)
	}

	fmt.Println("Listening for HTTP traffic on", Settings.Address+":"+strconv.Itoa(Settings.Port))
	fmt.Println("Forwarding requests to replay server:", Settings.ReplayServer(), "Limit:", Settings.ReplayLimit)

	// Sniffing traffic from given address
	listener := RAWTCPListen(Settings.Address, Settings.Port)

	currentTime := time.Now().UnixNano()
	currentRPS := 0

	var messageLogger *MessageLogger

	if Settings.FileToReplyPath != "" {
		messageLogger = NewLog(Settings.FileToReplyPath)
	}

	for {
		// Receiving TCPMessage object
		fmt.Println("FILE: ", Settings.FileToReplyPath)
		m := listener.Receive()
		fmt.Println("bla bla bla: ", messageLogger)

		if Settings.ReplayLimit != 0 {
			if (time.Now().UnixNano() - currentTime) > time.Second.Nanoseconds() {
				currentTime = time.Now().UnixNano()
				currentRPS = 0
			}

			if currentRPS >= Settings.ReplayLimit {
				continue
			}

			currentRPS++
		}

		fmt.Println("FOO BAR: ", messageLogger)
		if messageLogger != nil {
			go func() {
				messageBuffer := new(bytes.Buffer)
				messageWriter := bufio.NewWriter(messageBuffer)

				fmt.Fprintf(messageWriter, "\n")
				fmt.Fprintf(messageWriter, "------------------------------------------------\n")
				fmt.Fprintf(messageWriter, "%s", string(m.Bytes()))
				fmt.Fprintf(messageWriter, "\n")

				messageWriter.Flush()
				messageLogger.messageChannel <- messageBuffer.String()
			}()
		}
		fmt.Println("bla bla bla: ", messageLogger)
		go sendMessage(m)
	}
}

func sendMessage(m *TCPMessage) {
	conn, err := ReplayServer()

	if err != nil {
		log.Println("Failed to send message. Replay server not respond.")
		return
	} else {
		defer conn.Close()
	}

	// For debugging purpose
	// Usually request parsing happens in replay part
	if Settings.Verbose {
		buf := bytes.NewBuffer(m.Bytes())
		reader := bufio.NewReader(buf)

		request, err := http.ReadRequest(reader)

		if err != nil {
			Debug("Error while parsing request:", err, string(m.Bytes()))
		} else {
			request.ParseMultipartForm(32 << 20)
			Debug("Forwarding request:", request)
		}
	}

	_, err = conn.Write(m.Bytes())

	if err != nil {
		log.Println("Error while sending requests", err)
	}
}
<commit_msg>better file formating<commit_after>// Listener capture TCP traffic using RAW SOCKETS.
// Note: it requires sudo or root access.
//
// Rigt now it suport only HTTP
package listener

import (
	"bufio"
	"bytes"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"strconv"
	"time"
)

// Enable debug logging only if "--verbose" flag passed
func Debug(v ...interface{}) {
	if Settings.Verbose {
		log.Println(v...)
	}
}

func ReplayServer() (conn net.Conn, err error) {
	// Connection to reaplay server
	conn, err = net.Dial("tcp", Settings.ReplayServer())

	if err != nil {
		log.Println("Connection error ", err, Settings.ReplayAddress)
	}

	return
}

// Because its sub-program, Run acts as `main`
func Run() {
	if os.Getuid() != 0 {
		fmt.Println("Please start the listener as root or sudo!")
		fmt.Println("This is required since listener sniff traffic on given port.")
		os.Exit(1)
	}

	fmt.Println("Listening for HTTP traffic on", Settings.Address+":"+strconv.Itoa(Settings.Port))
	fmt.Println("Forwarding requests to replay server:", Settings.ReplayServer(), "Limit:", Settings.ReplayLimit)

	// Sniffing traffic from given address
	listener := RAWTCPListen(Settings.Address, Settings.Port)

	currentTime := time.Now().UnixNano()
	currentRPS := 0

	var messageLogger *MessageLogger

	if Settings.FileToReplyPath != "" {
		messageLogger = NewLog(Settings.FileToReplyPath)
	}

	for {
		// Receiving TCPMessage object
		m := listener.Receive()

		if Settings.ReplayLimit != 0 {
			if (time.Now().UnixNano() - currentTime) > time.Second.Nanoseconds() {
				currentTime = time.Now().UnixNano()
				currentRPS = 0
			}

			if currentRPS >= Settings.ReplayLimit {
				continue
			}

			currentRPS++
		}

		if messageLogger != nil {
      fmt.Println("FILE: ", Settings.FileToReplyPath)
			go func() {
				messageBuffer := new(bytes.Buffer)
				messageWriter := bufio.NewWriter(messageBuffer)

				// fmt.Fprintf(messageWriter, "------------------------------------------------\n")
				fmt.Fprintf(messageWriter, "%s", string(m.Bytes()))

				messageWriter.Flush()
				messageLogger.messageChannel <- messageBuffer.String()
			}()
		}

		go sendMessage(m)
	}
}

func sendMessage(m *TCPMessage) {
	conn, err := ReplayServer()

	if err != nil {
		log.Println("Failed to send message. Replay server not respond.")
		return
	} else {
		defer conn.Close()
	}

	// For debugging purpose
	// Usually request parsing happens in replay part
	if Settings.Verbose {
		buf := bytes.NewBuffer(m.Bytes())
		reader := bufio.NewReader(buf)

		request, err := http.ReadRequest(reader)

		if err != nil {
			Debug("Error while parsing request:", err, string(m.Bytes()))
		} else {
			request.ParseMultipartForm(32 << 20)
			Debug("Forwarding request:", request)
		}
	}

	_, err = conn.Write(m.Bytes())

	if err != nil {
		log.Println("Error while sending requests", err)
	}
}
<|endoftext|>


<commit_before>package peco

import (
	"unicode/utf8"

	"github.com/mattn/go-runewidth"
	"github.com/nsf/termbox-go"
)

// termbox always gives us some sort of warning when we run
// go run -race cmd/peco/peco.go
var termboxMutex = newMutex()

// These functions are here so that we can test
var (
	termboxClose     = termbox.Close
	termboxFlush     = termbox.Flush
	termboxInit      = termbox.Init
	termboxPollEvent = termbox.PollEvent
	termboxSetCell   = termbox.SetCell
	termboxSize      = termbox.Size
)

func (t Termbox) Init() error {
	trace("initializing termbox")
	if err := termboxInit(); err != nil {
		return err
	}

	return t.PostInit()
}

func (t Termbox) Close() error {
	termboxClose()
	return nil
}

// SendEvent is used to allow programmers generate random
// events, but it's only useful for testing purposes.
// When interactiving with termbox-go, this method is a noop
func (t Termbox) SendEvent(_ termbox.Event) {
	// no op
}

// Flush calls termbox.Flush
func (t Termbox) Flush() error {
	termboxMutex.Lock()
	defer termboxMutex.Unlock()
	return termboxFlush()
}

// PollEvent returns a channel that you can listen to for
// termbox's events. The actual polling is done in a
// separate gouroutine
func (t Termbox) PollEvent() chan termbox.Event {
	// XXX termbox.PollEvent() can get stuck on unexpected signal
	// handling cases. We still would like to wait until the user
	// (termbox) has some event for us to process, but we don't
	// want to allow termbox to control/block our input loop.
	//
	// Solution: put termbox polling in a separate goroutine,
	// and we just watch for a channel. The loop can now
	// safely be implemented in terms of select {} which is
	// safe from being stuck.
	evCh := make(chan termbox.Event)
	go func() {
		defer func() { recover() }()
		defer func() { close(evCh) }()
		for {
			evCh <- termboxPollEvent()
		}
	}()
	return evCh

}

// SetCell writes to the terminal
func (t Termbox) SetCell(x, y int, ch rune, fg, bg termbox.Attribute) {
	termboxMutex.Lock()
	defer termboxMutex.Unlock()
	termboxSetCell(x, y, ch, fg, bg)
}

// Size returns the dimensions of the current terminal
func (t Termbox) Size() (int, int) {
	termboxMutex.Lock()
	defer termboxMutex.Unlock()
	return termboxSize()
}

type PrintArgs struct {
	X       int
	XOffset int
	Y       int
	Fg      termbox.Attribute
	Bg      termbox.Attribute
	Msg     string
	Fill    bool
}

func (t Termbox) Print(args PrintArgs) int {
	return screenPrint(t, args)
}

func screenPrint(t Screen, args PrintArgs) int {
	var written int

	bg := args.Bg
	fg := args.Fg
	msg := args.Msg
	x := args.X
	y := args.Y
	xOffset := args.XOffset
	for len(msg) > 0 {
		c, w := utf8.DecodeRuneInString(msg)
		if c == utf8.RuneError {
			c = '?'
			w = 1
		}
		msg = msg[w:]
		if c == '\t' {
			// In case we found a tab, we draw it as 4 spaces
			n := 4 - (x+xOffset)%4
			for i := int(0); i <= n; i++ {
				t.SetCell(int(x+i), int(y), ' ', fg, bg)
			}
			written += n
			x += n
		} else {
			t.SetCell(int(x), int(y), c, fg, bg)
			n := int(runewidth.RuneWidth(c))
			x += n
			written += n
		}
	}

	if !args.Fill {
		return written
	}

	width, _ := t.Size()
	for ; x < int(width); x++ {
		t.SetCell(int(x), int(y), ' ', fg, bg)
	}
	written += int(width) - x
	return written
}
<commit_msg>remove unused code<commit_after>package peco

import (
	"unicode/utf8"

	"github.com/mattn/go-runewidth"
	"github.com/nsf/termbox-go"
)

// termbox always gives us some sort of warning when we run
// go run -race cmd/peco/peco.go
var termboxMutex = newMutex()

func (t Termbox) Init() error {
	trace("initializing termbox")
	if err := termbox.Init(); err != nil {
		return err
	}

	return t.PostInit()
}

func (t Termbox) Close() error {
	termbox.Close()
	return nil
}

// SendEvent is used to allow programmers generate random
// events, but it's only useful for testing purposes.
// When interactiving with termbox-go, this method is a noop
func (t Termbox) SendEvent(_ termbox.Event) {
	// no op
}

// Flush calls termbox.Flush
func (t Termbox) Flush() error {
	termboxMutex.Lock()
	defer termboxMutex.Unlock()
	return termbox.Flush()
}

// PollEvent returns a channel that you can listen to for
// termbox's events. The actual polling is done in a
// separate gouroutine
func (t Termbox) PollEvent() chan termbox.Event {
	// XXX termbox.PollEvent() can get stuck on unexpected signal
	// handling cases. We still would like to wait until the user
	// (termbox) has some event for us to process, but we don't
	// want to allow termbox to control/block our input loop.
	//
	// Solution: put termbox polling in a separate goroutine,
	// and we just watch for a channel. The loop can now
	// safely be implemented in terms of select {} which is
	// safe from being stuck.
	evCh := make(chan termbox.Event)
	go func() {
		defer func() { recover() }()
		defer func() { close(evCh) }()
		for {
			evCh <- termbox.PollEvent()
		}
	}()
	return evCh

}

// SetCell writes to the terminal
func (t Termbox) SetCell(x, y int, ch rune, fg, bg termbox.Attribute) {
	termboxMutex.Lock()
	defer termboxMutex.Unlock()
	termbox.SetCell(x, y, ch, fg, bg)
}

// Size returns the dimensions of the current terminal
func (t Termbox) Size() (int, int) {
	termboxMutex.Lock()
	defer termboxMutex.Unlock()
	return termbox.Size()
}

type PrintArgs struct {
	X       int
	XOffset int
	Y       int
	Fg      termbox.Attribute
	Bg      termbox.Attribute
	Msg     string
	Fill    bool
}

func (t Termbox) Print(args PrintArgs) int {
	return screenPrint(t, args)
}

func screenPrint(t Screen, args PrintArgs) int {
	var written int

	bg := args.Bg
	fg := args.Fg
	msg := args.Msg
	x := args.X
	y := args.Y
	xOffset := args.XOffset
	for len(msg) > 0 {
		c, w := utf8.DecodeRuneInString(msg)
		if c == utf8.RuneError {
			c = '?'
			w = 1
		}
		msg = msg[w:]
		if c == '\t' {
			// In case we found a tab, we draw it as 4 spaces
			n := 4 - (x+xOffset)%4
			for i := int(0); i <= n; i++ {
				t.SetCell(int(x+i), int(y), ' ', fg, bg)
			}
			written += n
			x += n
		} else {
			t.SetCell(int(x), int(y), c, fg, bg)
			n := int(runewidth.RuneWidth(c))
			x += n
			written += n
		}
	}

	if !args.Fill {
		return written
	}

	width, _ := t.Size()
	for ; x < int(width); x++ {
		t.SetCell(int(x), int(y), ' ', fg, bg)
	}
	written += int(width) - x
	return written
}
<|endoftext|>


<commit_before>package compose

import (
	"net/http"
	"time"

	"github.com/ory-am/fosite"
	foauth2 "github.com/ory-am/fosite/handler/oauth2"
	"github.com/ory-am/hydra/firewall"
	. "github.com/ory-am/hydra/oauth2"
	"github.com/ory-am/hydra/pkg"
	"github.com/ory-am/hydra/warden"
	"github.com/ory-am/ladon"
	"golang.org/x/oauth2"
)

func NewFirewall(issuer string, subject string, scopes fosite.Arguments, p ...ladon.Policy) (firewall.Firewall, *http.Client) {
	tokens := pkg.Tokens(1)

	fositeStore := pkg.FositeStore()
	ps := map[string]ladon.Policy{}

	for _, x := range p {
		ps[x.GetID()] = x
	}
	ladonWarden := pkg.LadonWarden(ps)

	ar := fosite.NewAccessRequest(NewSession(subject))
	ar.GrantedScopes = scopes
	fositeStore.CreateAccessTokenSession(nil, tokens[0][0], ar)

	conf := &oauth2.Config{Scopes: scopes, Endpoint: oauth2.Endpoint{}}

	return &warden.LocalWarden{
			Warden: ladonWarden,
			OAuth2: &fosite.Fosite{
				Store: fositeStore,
				TokenIntrospectionHandlers: fosite.TokenIntrospectionHandlers{
					&foauth2.CoreValidator{
						CoreStrategy:  pkg.HMACStrategy,
						CoreStorage:   fositeStore,
						ScopeStrategy: fosite.HierarchicScopeStrategy,
					},
				},
				ScopeStrategy: fosite.HierarchicScopeStrategy,
			},
			Issuer:              issuer,
			AccessTokenLifespan: time.Hour,
		}, conf.Client(oauth2.NoContext, &oauth2.Token{
			AccessToken: tokens[0][1],
			Expiry:      time.Now().Add(time.Hour),
			TokenType:   "bearer",
		})
}
<commit_msg>compose: resolve issue with firewall set up<commit_after>package compose

import (
	"net/http"
	"time"

	"github.com/ory-am/fosite"
	foauth2 "github.com/ory-am/fosite/handler/oauth2"
	"github.com/ory-am/hydra/firewall"
	. "github.com/ory-am/hydra/oauth2"
	"github.com/ory-am/hydra/pkg"
	"github.com/ory-am/hydra/warden"
	"github.com/ory-am/ladon"
	"golang.org/x/oauth2"
	"github.com/ory-am/hydra/warden/group"
)

func NewFirewall(issuer string, subject string, scopes fosite.Arguments, p ...ladon.Policy) (firewall.Firewall, *http.Client) {
	tokens := pkg.Tokens(1)

	fositeStore := pkg.FositeStore()
	ps := map[string]ladon.Policy{}

	for _, x := range p {
		ps[x.GetID()] = x
	}
	ladonWarden := pkg.LadonWarden(ps)

	ar := fosite.NewAccessRequest(NewSession(subject))
	ar.GrantedScopes = scopes
	fositeStore.CreateAccessTokenSession(nil, tokens[0][0], ar)

	conf := &oauth2.Config{Scopes: scopes, Endpoint: oauth2.Endpoint{}}

	return &warden.LocalWarden{
		Warden: ladonWarden,
		OAuth2: &fosite.Fosite{
			Store: fositeStore,
			TokenIntrospectionHandlers: fosite.TokenIntrospectionHandlers{
				&foauth2.CoreValidator{
					CoreStrategy:  pkg.HMACStrategy,
					CoreStorage:   fositeStore,
					ScopeStrategy: fosite.HierarchicScopeStrategy,
				},
			},
			ScopeStrategy: fosite.HierarchicScopeStrategy,
		},
		Issuer:              issuer,
		AccessTokenLifespan: time.Hour,
		Groups: group.NewMemoryManager(),
	}, conf.Client(oauth2.NoContext, &oauth2.Token{
		AccessToken: tokens[0][1],
		Expiry:      time.Now().Add(time.Hour),
		TokenType:   "bearer",
	})
}
<|endoftext|>