<commit_before>package deprecatedapirequest

import (
	"context"
	"strings"
	"sync"
	"time"

	apiv1 "github.com/openshift/api/apiserver/v1"

	apiclientv1 "github.com/openshift/client-go/apiserver/clientset/versioned/typed/apiserver/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/klog/v2"
	"k8s.io/kubernetes/openshift-kube-apiserver/filters/deprecatedapirequest/v1helpers"
)

// NewController returns a controller
func NewController(client apiclientv1.APIRequestCountInterface, nodeName string) *controller {
	ret := &controller{
		client:       client,
		nodeName:     nodeName,
		updatePeriod: 5 * time.Minute,
	}
	ret.resetRequestCount()
	return ret
}

// APIRequestLogger support logging deprecated API requests.
type APIRequestLogger interface {
	IsDeprecated(resource, version, group string) bool
	LogRequest(resource schema.GroupVersionResource, timestamp time.Time, user, userAgent, verb string)
	Start(stop <-chan struct{})
}

type controller struct {
	client       apiclientv1.APIRequestCountInterface
	nodeName     string
	updatePeriod time.Duration

	requestCountLock sync.RWMutex
	requestCounts    *apiRequestCounts
}

// IsDeprecated return true if the resource is deprecated.
func (c *controller) IsDeprecated(resource, version, group string) bool {
	_, ok := deprecatedApiRemovedRelease[schema.GroupVersionResource{
		Group:    group,
		Version:  version,
		Resource: resource,
	}]
	return ok
}

// LogRequest queues an api request for logging
func (c *controller) LogRequest(resource schema.GroupVersionResource, timestamp time.Time, user, userAgent, verb string) {
	c.requestCountLock.RLock()
	defer c.requestCountLock.RUnlock()
	// we snip user agents to reduce cardinality and unique keys.  For well behaved agents, we see useragents about like
	// kube-controller-manager/v1.21.0 (linux/amd64) kubernetes/743bd58/kube-controller-manager
	// so we will snip at the first space.
	snippedUserAgent := userAgent
	if i := strings.Index(userAgent, " "); i > 0 {
		snippedUserAgent = userAgent[:i]
	}
	userKey := userKey{
		user:      user,
		userAgent: snippedUserAgent,
	}
	c.requestCounts.IncrementRequestCount(resource, timestamp.Hour(), userKey, verb, 1)
}

// resetCount returns the current count and creates a new requestCount instance var
func (c *controller) resetRequestCount() *apiRequestCounts {
	c.requestCountLock.Lock()
	defer c.requestCountLock.Unlock()
	existing := c.requestCounts
	c.requestCounts = newAPIRequestCounts(c.nodeName)
	return existing
}

// Start the controller
func (c *controller) Start(stop <-chan struct{}) {
	klog.Infof("Starting DeprecatedAPIRequest controller.")

	// create a context.Context needed for some API calls
	ctx, cancel := context.WithCancel(context.Background())
	go func() {
		<-stop
		klog.Infof("Shutting down DeprecatedAPIRequest controller.")
		cancel()
	}()

	// write out logs every c.updatePeriod
	go wait.UntilWithContext(ctx, c.persistRequestCountForAllResources, c.updatePeriod)
}

func (c *controller) persistRequestCountForAllResources(ctx context.Context) {
	klog.V(2).Infof("updating top APIRequest counts")
	defer klog.V(2).Infof("finished updating top APIRequest counts")

	// get the current count to persist, start a new in-memory count
	countsToPersist := c.resetRequestCount()

	// remove stale data
	expiredHour := (time.Now().Hour() + 1) % 24
	currentHour := time.Now().Hour()
	countsToPersist.ExpireOldestCounts(expiredHour)

	// when this function returns, add any remaining counts back to the total to be retried for update
	defer c.requestCounts.Add(countsToPersist)

	var wg sync.WaitGroup
	for gvr := range countsToPersist.resourceToRequestCount {
		resourceCount := countsToPersist.Resource(gvr)
		wg.Add(1)
		go c.persistRequestCountForResource(ctx, &wg, currentHour, expiredHour, resourceCount)
	}
	wg.Wait()
}

func (c *controller) persistRequestCountForResource(ctx context.Context, wg *sync.WaitGroup, currentHour, expiredHour int, localResourceCount *resourceRequestCounts) {
	defer wg.Done()

	klog.V(2).Infof("updating top %v APIRequest counts", localResourceCount.resource)
	defer klog.V(2).Infof("finished updating top %v APIRequest counts", localResourceCount.resource)

	status, _, err := v1helpers.ApplyStatus(
		ctx,
		c.client,
		resourceToAPIName(localResourceCount.resource),
		SetRequestCountsForNode(c.nodeName, currentHour, expiredHour, localResourceCount),
	)
	if err != nil {
		runtime.HandleError(err)
		return
	}

	// on successful update, remove the counts we don't need.  This is every hour except the current hour
	// and every user recorded for the current hour on this node
	removePersistedRequestCounts(c.nodeName, currentHour, status, localResourceCount)
}

// removePersistedRequestCounts removes the counts we don't need to keep in memory.
// This is every hour except the current hour (those will no longer change) and every user recorded for the current hour on this node.
// Then it tracks the amount that needs to be kept out of the sum. This is logically the amount we're adding back in.
// Because we already counted all the users in the persisted sum, we need to exclude the amount we'll be placing back
// in memory.
func removePersistedRequestCounts(nodeName string, currentHour int, persistedStatus *apiv1.APIRequestCountStatus, localResourceCount *resourceRequestCounts) {
	for hourIndex := range localResourceCount.hourToRequestCount {
		if currentHour != hourIndex {
			localResourceCount.RemoveHour(hourIndex)
		}
	}
	for _, persistedNodeCount := range persistedStatus.CurrentHour.ByNode {
		if persistedNodeCount.NodeName != nodeName {
			continue
		}
		for _, peristedUserCount := range persistedNodeCount.ByUser {
			userKey := userKey{
				user:      peristedUserCount.UserName,
				userAgent: peristedUserCount.UserAgent,
			}
			localResourceCount.Hour(currentHour).RemoveUser(userKey)
		}
	}

	countToSuppress := int64(0)
	for _, userCounts := range localResourceCount.Hour(currentHour).usersToRequestCounts {
		for _, verbCount := range userCounts.verbsToRequestCounts {
			countToSuppress += verbCount.count
		}
	}

	localResourceCount.Hour(currentHour).countToSuppress = countToSuppress
}

func resourceToAPIName(resource schema.GroupVersionResource) string {
	apiName := resource.Resource + "." + resource.Version
	if len(resource.Group) > 0 {
		apiName += "." + resource.Group
	}
	return apiName
}
<commit_msg>UPSTREAM: <carry>: apirequestcount: smear out CR updates over interval<commit_after>package deprecatedapirequest

import (
	"context"
	"math/rand"
	"strings"
	"sync"
	"time"

	apiv1 "github.com/openshift/api/apiserver/v1"

	apiclientv1 "github.com/openshift/client-go/apiserver/clientset/versioned/typed/apiserver/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/klog/v2"
	"k8s.io/kubernetes/openshift-kube-apiserver/filters/deprecatedapirequest/v1helpers"
)

// NewController returns a controller
func NewController(client apiclientv1.APIRequestCountInterface, nodeName string) *controller {
	ret := &controller{
		client:       client,
		nodeName:     nodeName,
		updatePeriod: 5 * time.Minute,
	}
	ret.resetRequestCount()
	return ret
}

// APIRequestLogger support logging deprecated API requests.
type APIRequestLogger interface {
	IsDeprecated(resource, version, group string) bool
	LogRequest(resource schema.GroupVersionResource, timestamp time.Time, user, userAgent, verb string)
	Start(stop <-chan struct{})
}

type controller struct {
	client       apiclientv1.APIRequestCountInterface
	nodeName     string
	updatePeriod time.Duration

	requestCountLock sync.RWMutex
	requestCounts    *apiRequestCounts
}

// IsDeprecated return true if the resource is deprecated.
func (c *controller) IsDeprecated(resource, version, group string) bool {
	_, ok := deprecatedApiRemovedRelease[schema.GroupVersionResource{
		Group:    group,
		Version:  version,
		Resource: resource,
	}]
	return ok
}

// LogRequest queues an api request for logging
func (c *controller) LogRequest(resource schema.GroupVersionResource, timestamp time.Time, user, userAgent, verb string) {
	c.requestCountLock.RLock()
	defer c.requestCountLock.RUnlock()
	// we snip user agents to reduce cardinality and unique keys.  For well behaved agents, we see useragents about like
	// kube-controller-manager/v1.21.0 (linux/amd64) kubernetes/743bd58/kube-controller-manager
	// so we will snip at the first space.
	snippedUserAgent := userAgent
	if i := strings.Index(userAgent, " "); i > 0 {
		snippedUserAgent = userAgent[:i]
	}
	userKey := userKey{
		user:      user,
		userAgent: snippedUserAgent,
	}
	c.requestCounts.IncrementRequestCount(resource, timestamp.Hour(), userKey, verb, 1)
}

// resetCount returns the current count and creates a new requestCount instance var
func (c *controller) resetRequestCount() *apiRequestCounts {
	c.requestCountLock.Lock()
	defer c.requestCountLock.Unlock()
	existing := c.requestCounts
	c.requestCounts = newAPIRequestCounts(c.nodeName)
	return existing
}

// Start the controller
func (c *controller) Start(stop <-chan struct{}) {
	klog.Infof("Starting DeprecatedAPIRequest controller.")

	// create a context.Context needed for some API calls
	ctx, cancel := context.WithCancel(context.Background())
	go func() {
		<-stop
		klog.Infof("Shutting down DeprecatedAPIRequest controller.")
		cancel()
	}()

	// write out logs every c.updatePeriod
	go wait.NonSlidingUntilWithContext(ctx, c.persistRequestCountForAllResources, c.updatePeriod)
}

func (c *controller) persistRequestCountForAllResources(ctx context.Context) {
	klog.V(2).Infof("updating top APIRequest counts")
	defer klog.V(2).Infof("finished updating top APIRequest counts")

	// get the current count to persist, start a new in-memory count
	countsToPersist := c.resetRequestCount()

	// remove stale data
	expiredHour := (time.Now().Hour() + 1) % 24
	currentHour := time.Now().Hour()
	countsToPersist.ExpireOldestCounts(expiredHour)

	// when this function returns, add any remaining counts back to the total to be retried for update
	defer c.requestCounts.Add(countsToPersist)

	var wg sync.WaitGroup
	for gvr := range countsToPersist.resourceToRequestCount {
		resourceCount := countsToPersist.Resource(gvr)
		wg.Add(1)
		go func() {
			time.Sleep(time.Duration(rand.Int63n(int64(c.updatePeriod / 5 * 4)))) // smear out over the interval to avoid resource spikes
			c.persistRequestCountForResource(ctx, &wg, currentHour, expiredHour, resourceCount)
		}()
	}
	wg.Wait()
}

func (c *controller) persistRequestCountForResource(ctx context.Context, wg *sync.WaitGroup, currentHour, expiredHour int, localResourceCount *resourceRequestCounts) {
	defer wg.Done()

	klog.V(2).Infof("updating top %v APIRequest counts", localResourceCount.resource)
	defer klog.V(2).Infof("finished updating top %v APIRequest counts", localResourceCount.resource)

	status, _, err := v1helpers.ApplyStatus(
		ctx,
		c.client,
		resourceToAPIName(localResourceCount.resource),
		SetRequestCountsForNode(c.nodeName, currentHour, expiredHour, localResourceCount),
	)
	if err != nil {
		runtime.HandleError(err)
		return
	}

	// on successful update, remove the counts we don't need.  This is every hour except the current hour
	// and every user recorded for the current hour on this node
	removePersistedRequestCounts(c.nodeName, currentHour, status, localResourceCount)
}

// removePersistedRequestCounts removes the counts we don't need to keep in memory.
// This is every hour except the current hour (those will no longer change) and every user recorded for the current hour on this node.
// Then it tracks the amount that needs to be kept out of the sum. This is logically the amount we're adding back in.
// Because we already counted all the users in the persisted sum, we need to exclude the amount we'll be placing back
// in memory.
func removePersistedRequestCounts(nodeName string, currentHour int, persistedStatus *apiv1.APIRequestCountStatus, localResourceCount *resourceRequestCounts) {
	for hourIndex := range localResourceCount.hourToRequestCount {
		if currentHour != hourIndex {
			localResourceCount.RemoveHour(hourIndex)
		}
	}
	for _, persistedNodeCount := range persistedStatus.CurrentHour.ByNode {
		if persistedNodeCount.NodeName != nodeName {
			continue
		}
		for _, peristedUserCount := range persistedNodeCount.ByUser {
			userKey := userKey{
				user:      peristedUserCount.UserName,
				userAgent: peristedUserCount.UserAgent,
			}
			localResourceCount.Hour(currentHour).RemoveUser(userKey)
		}
	}

	countToSuppress := int64(0)
	for _, userCounts := range localResourceCount.Hour(currentHour).usersToRequestCounts {
		for _, verbCount := range userCounts.verbsToRequestCounts {
			countToSuppress += verbCount.count
		}
	}

	localResourceCount.Hour(currentHour).countToSuppress = countToSuppress
}

func resourceToAPIName(resource schema.GroupVersionResource) string {
	apiName := resource.Resource + "." + resource.Version
	if len(resource.Group) > 0 {
		apiName += "." + resource.Group
	}
	return apiName
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/ghodss/yaml"
	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/config-connector/tests/ccs-test/util"
	"github.com/spf13/cobra"
)

const (
	kubectlBinaryName          = "kubectl"
	testsDirPath               = "config-connector/tests"
	originalValuesFileName     = "original_values.yaml"
	requiredFieldsOnlyFileName = "required_fields_only.yaml"
	envFileRelativePath        = "testcases/environments.yaml"
	yamlFileSuffix             = ".yaml"
)

var (
	relativePath string

	// Regex of the env vars that require a randomized suffix.
	// It should be in the format of $ENV_VAR-$RANDOM_ID.
	re = regexp.MustCompile(`\$(?P<EnvName>[A-Z]+|[A-Z]+[A-Z_]*[A-Z]+)(-\$RANDOM_ID)`)

	rootCmd = &cobra.Command{
		Use:   "ccs-test",
		Short: "CLI to test Config Connector Solutions",
		Long:  `CLI to test Config Connector Solutions`,
	}

	runCmd = &cobra.Command{
		Use:   "run",
		Short: "Run a given test by the relative path --path",
		Long:  "Run a given test by the relative path --path",
		Run: func(cmd *cobra.Command, args []string) {
			// Check the required flag.
			if relativePath == "" {
				log.Fatal("\"--path\" must be specified to run test")
				return
			}

			log.Printf("======Testing solution %q...======\n", relativePath)

			// Calculate the path to testcase directory and solution directory.
			current, err := os.Getwd()
			if err != nil {
				log.Fatalf("error retrieving the current directory: %v", err)
				return
			}
			if !strings.HasSuffix(current, testsDirPath) {
				log.Fatalf("error running tests under directory: %s. Please "+
					"follow the instructions in the README.", current)
				return
			}
			testCasePath := filepath.Join(current, "testcases", relativePath)
			envFilePath := filepath.Join(current, envFileRelativePath)

			parent := filepath.Dir(current)
			solutionPath := filepath.Join(parent, "solutions", relativePath)

			// Clean up the left over resources if there are any.
			if err := deleteResources(solutionPath); err != nil {
				log.Fatalf("error cleaning up resources before running the "+
					"test. Please clean them up manually: %v", err)
				return
			}

			// Fetch the testcase values and run the test.
			envValues := make(map[string]string)
			if err := parseYamlToStringMap(envFilePath, envValues); err != nil {
				log.Fatalf("error retrieving envrionment variables: %v", err)
				return
			}

			originalValues := make(map[string]string)
			if err := parseYamlToStringMap(filepath.Join(testCasePath, originalValuesFileName), originalValues); err != nil {
				log.Fatalf("error retrieving orginal values: %v", err)
				return
			}

			testValues := make(map[string]string)
			if err := parseYamlToStringMap(filepath.Join(testCasePath, requiredFieldsOnlyFileName), testValues); err != nil {
				log.Fatalf("error retrieving test values: %v", err)
				return
			}

			// Generate the random IDs first.
			randomId, err := util.GenerateRandomizedSuffix()
			if err != nil {
				log.Fatalf("error generating the randomized suffix for resource names: %v", err)
			}

			// Then populate the values of the env var and the random id.
			realValues, err := finalizeValues(randomId, envValues, testValues)
			if err != nil {
				log.Fatalf("error finalizing test values: %v", err)
				return
			}

			if err := runKptTestcase(solutionPath, realValues, originalValues); err != nil {
				log.Fatalf("test failed for solution %q: %v", relativePath, err)
				return
			}

			log.Printf("======Successfully finished the test for solution %q======\n", relativePath)
		},
	}
)

func init() {
	runCmd.PersistentFlags().StringVarP(&relativePath, "path", "p", "", "[Required] The relative path to the folder of the solution's testcases, e.g. `iam/kpt/member-iam`.")
	rootCmd.AddCommand(runCmd)
}

// Execute adds all child commands to the root command and sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := rootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}

func parseYamlToStringMap(filePath string, result map[string]string) error {
	bytes, err := ioutil.ReadFile(filePath)
	if err != nil {
		return fmt.Errorf("error reading file '%s': %v", filePath, err)
	}
	err = yaml.Unmarshal(bytes, &result)
	if err != nil {
		return fmt.Errorf("error unmarshaling file '%s': %v", filePath, err)
	}
	return nil
}

func finalizeValues(randomId string, envValues map[string]string, testValues map[string]string) (map[string]string, error) {
	realValues := make(map[string]string)
	for key, value := range testValues {
		if !strings.HasPrefix(value, "$") {
			return nil, fmt.Errorf("test value for setter %q is %q, expect a reference, e.g. $ENV_VAR", key, value)
		}
		realValue := ""
		ok := false
		if re.MatchString(value) {
			submatch := re.FindStringSubmatch(value)
			if len(submatch) == 0 {
				return nil, fmt.Errorf("env var name is invalid in test value %q", value)
			}
			subexpNames := re.SubexpNames()
			for i, name := range subexpNames {
				if name == "EnvName" {
					prefix, ok := envValues[submatch[i]]
					if !ok {
						return nil, fmt.Errorf("couldn't find the env var %q", submatch[i])
					}
					realValue = fmt.Sprintf("%s-%s", prefix, randomId)
					break
				}
			}
		} else {
			realValue, ok = envValues[strings.TrimPrefix(value, "$")]
			if !ok {
				return nil, fmt.Errorf("couldn't find the env var %q", strings.TrimPrefix(value, "$"))
			}
		}
		realValues[key] = realValue
	}

	return realValues, nil
}

func runKptTestcase(solutionPath string, testValues map[string]string, originalValues map[string]string) error {
	// Set the kpt setters defined in the testcase.
	log.Println("======Setting the kpt setters...======")
	for key, value := range testValues {
		output, err := exec.Command("kpt", "cfg", "set", solutionPath, key,
			value, "--set-by", "test").CombinedOutput()
		if err != nil {
			errToReturn := fmt.Errorf("error setting setter '%s' with value "+
				"'%s': %v\nstdout: %s", key, value, err, string(output))

			// Clean up before exit with errors.
			if err := resetKptSetters(solutionPath, originalValues); err != nil {
				return concatErrors(
					"error resetting kpt setters before exiting",
					err, errToReturn)
			}

			return errToReturn
		}
		log.Printf("%s\n", string(output))
	}
	log.Println("======Successfully set the kpt setters======")

	// Apply all the resources.
	log.Println("======Creating the resources...======")
	output, err := exec.Command("kubectl", "create", "-f", solutionPath).CombinedOutput()
	if err != nil {
		errToReturn := fmt.Errorf("error creating resources: %v\nstdout: %s", err, string(output))

		// Clean up before exit with errors.
		if err := cleanUp(solutionPath, originalValues); err != nil {
			return concatErrors(
				"error cleanning up resources before exiting",
				err, errToReturn)
		}
		return errToReturn
	}
	log.Printf("%s\n", string(output))
	log.Println("======Successfully created the resources======")

	// Wait for all the resources to be ready.
	if err := verifyReadyCondition(solutionPath); err != nil {
		errToReturn := fmt.Errorf("error verifying the ready condition: %v", err)

		// Clean up before exit with errors.
		if err := cleanUp(solutionPath, originalValues); err != nil {
			return concatErrors(
				"error cleanning up resources before exiting",
				err, errToReturn)
		}
		return errToReturn
	}

	// Clean up.
	return cleanUp(solutionPath, originalValues)
}

func cleanUp(solutionPath string, originalValues map[string]string) error {
	resourceErr := deleteResources(solutionPath)
	setterErr := resetKptSetters(solutionPath, originalValues)
	if resourceErr != nil || setterErr != nil {
		return concatErrors(
			fmt.Sprintf("error cleanning up the test for solution %q. "+
				"Please manually delete the Config Connector resources and "+
				"reset the kpt setters", solutionPath),
			resourceErr, setterErr)
	}
	return nil
}

func verifyReadyCondition(solutionPath string) error {
	log.Println("======Verifying that all the Config Connector resources are ready...======")

	files, err := ioutil.ReadDir(solutionPath)
	if err != nil {
		return fmt.Errorf("error reading solution directory %q: %v", solutionPath, err)
	}

	for _, file := range files {
		// We should only verify the YAML config files for Config Connector
		// resources.
		fileName := file.Name()
		if !strings.HasSuffix(fileName, yamlFileSuffix) || strings.Contains(fileName, "namespace") {
			continue
		}

		output, err := exec.Command("kubectl", "wait", "--for=condition=ready",
			"-f", filepath.Join(solutionPath, fileName), "--timeout=60s").CombinedOutput()
		if err != nil {
			return fmt.Errorf("resource in file %q is not ready in 60 seconds: %v\nstdout: %s", fileName, err, string(output))
		}
		log.Printf("%s\n", string(output))

	}

	log.Println("======All the Config Connector resrouces are ready======")
	return nil
}

func deleteResources(solutionPath string) error {
	log.Println("======Deleting the resources...======")
	output, err := exec.Command("kubectl", "delete", "-f", solutionPath, "--wait").CombinedOutput()
	if err != nil {
		err = fmt.Errorf("error deleting resources: %v\nstdout: %s", err, string(output))
		if isNotFoundErrorOnly(err) {
			log.Println(err)
			log.Println("======Finished deleting the resources======")
			return nil
		}
		return fmt.Errorf("error deleting resources: %v\nstdout: %s", err, string(output))
	}
	log.Printf("%s\n", output)
	log.Println("======Successfully deleted the resources======")
	return nil
}

func isNotFoundErrorOnly(err error) bool {
	numErrors := strings.Count(err.Error(), "Error from server")
	numNotFoundErrors := strings.Count(err.Error(), "Error from server (NotFound)")
	return numErrors == numNotFoundErrors
}

func resetKptSetters(solutionPath string, originalValues map[string]string) error {
	log.Println("======Resetting the kpt setters...======")
	for key, value := range originalValues {
		output, err := exec.Command("kpt", "cfg", "set", solutionPath, key, value, "--set-by", "PLACEHOLDER").CombinedOutput()
		if err != nil {
			return fmt.Errorf("error setting setter '%s' back to the original value '%s': %v\nstdout: %s", key, value, err, string(output))
		}
		log.Printf("%s\n", string(output))
	}
	log.Println("======Successfully reset the kpt setters======")
	return nil
}

func concatErrors(msg string, errs ...error) error {
	errToReturn := errors.New(msg)
	for _, err := range errs {
		if err == nil {
			continue
		}
		errToReturn = fmt.Errorf("%v:%v", errToReturn, err)
	}
	return errToReturn
}
<commit_msg>[KCC] Print out resource status if they are not ready after the timeout (#667)<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/ghodss/yaml"
	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/config-connector/tests/ccs-test/util"
	"github.com/spf13/cobra"
)

const (
	kubectlBinaryName          = "kubectl"
	testsDirPath               = "config-connector/tests"
	originalValuesFileName     = "original_values.yaml"
	requiredFieldsOnlyFileName = "required_fields_only.yaml"
	envFileRelativePath        = "testcases/environments.yaml"
	yamlFileSuffix             = ".yaml"
)

var (
	relativePath string

	// Regex of the env vars that require a randomized suffix.
	// It should be in the format of $ENV_VAR-$RANDOM_ID.
	re = regexp.MustCompile(`\$(?P<EnvName>[A-Z]+|[A-Z]+[A-Z_]*[A-Z]+)(-\$RANDOM_ID)`)

	rootCmd = &cobra.Command{
		Use:   "ccs-test",
		Short: "CLI to test Config Connector Solutions",
		Long:  `CLI to test Config Connector Solutions`,
	}

	runCmd = &cobra.Command{
		Use:   "run",
		Short: "Run a given test by the relative path --path",
		Long:  "Run a given test by the relative path --path",
		Run: func(cmd *cobra.Command, args []string) {
			// Check the required flag.
			if relativePath == "" {
				log.Fatal("\"--path\" must be specified to run test")
				return
			}

			log.Printf("======Testing solution %q...======\n", relativePath)

			// Calculate the path to testcase directory and solution directory.
			current, err := os.Getwd()
			if err != nil {
				log.Fatalf("error retrieving the current directory: %v", err)
				return
			}
			if !strings.HasSuffix(current, testsDirPath) {
				log.Fatalf("error running tests under directory: %s. Please "+
					"follow the instructions in the README.", current)
				return
			}
			testCasePath := filepath.Join(current, "testcases", relativePath)
			envFilePath := filepath.Join(current, envFileRelativePath)

			parent := filepath.Dir(current)
			solutionPath := filepath.Join(parent, "solutions", relativePath)

			// Clean up the left over resources if there are any.
			if err := deleteResources(solutionPath); err != nil {
				log.Fatalf("error cleaning up resources before running the "+
					"test. Please clean them up manually: %v", err)
				return
			}

			// Fetch the testcase values and run the test.
			envValues := make(map[string]string)
			if err := parseYamlToStringMap(envFilePath, envValues); err != nil {
				log.Fatalf("error retrieving envrionment variables: %v", err)
				return
			}

			originalValues := make(map[string]string)
			if err := parseYamlToStringMap(filepath.Join(testCasePath, originalValuesFileName), originalValues); err != nil {
				log.Fatalf("error retrieving orginal values: %v", err)
				return
			}

			testValues := make(map[string]string)
			if err := parseYamlToStringMap(filepath.Join(testCasePath, requiredFieldsOnlyFileName), testValues); err != nil {
				log.Fatalf("error retrieving test values: %v", err)
				return
			}

			// Generate the random IDs first.
			randomId, err := util.GenerateRandomizedSuffix()
			if err != nil {
				log.Fatalf("error generating the randomized suffix for resource names: %v", err)
			}

			// Then populate the values of the env var and the random id.
			realValues, err := finalizeValues(randomId, envValues, testValues)
			if err != nil {
				log.Fatalf("error finalizing test values: %v", err)
				return
			}

			if err := runKptTestcase(solutionPath, realValues, originalValues); err != nil {
				log.Fatalf("test failed for solution %q: %v", relativePath, err)
				return
			}

			log.Printf("======Successfully finished the test for solution %q======\n", relativePath)
		},
	}
)

func init() {
	runCmd.PersistentFlags().StringVarP(&relativePath, "path", "p", "", "[Required] The relative path to the folder of the solution's testcases, e.g. `iam/kpt/member-iam`.")
	rootCmd.AddCommand(runCmd)
}

// Execute adds all child commands to the root command and sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := rootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}

func parseYamlToStringMap(filePath string, result map[string]string) error {
	bytes, err := ioutil.ReadFile(filePath)
	if err != nil {
		return fmt.Errorf("error reading file '%s': %v", filePath, err)
	}
	err = yaml.Unmarshal(bytes, &result)
	if err != nil {
		return fmt.Errorf("error unmarshaling file '%s': %v", filePath, err)
	}
	return nil
}

func finalizeValues(randomId string, envValues map[string]string, testValues map[string]string) (map[string]string, error) {
	realValues := make(map[string]string)
	for key, value := range testValues {
		if !strings.HasPrefix(value, "$") {
			return nil, fmt.Errorf("test value for setter %q is %q, expect a reference, e.g. $ENV_VAR", key, value)
		}
		realValue := ""
		ok := false
		if re.MatchString(value) {
			submatch := re.FindStringSubmatch(value)
			if len(submatch) == 0 {
				return nil, fmt.Errorf("env var name is invalid in test value %q", value)
			}
			subexpNames := re.SubexpNames()
			for i, name := range subexpNames {
				if name == "EnvName" {
					prefix, ok := envValues[submatch[i]]
					if !ok {
						return nil, fmt.Errorf("couldn't find the env var %q", submatch[i])
					}
					realValue = fmt.Sprintf("%s-%s", prefix, randomId)
					break
				}
			}
		} else {
			realValue, ok = envValues[strings.TrimPrefix(value, "$")]
			if !ok {
				return nil, fmt.Errorf("couldn't find the env var %q", strings.TrimPrefix(value, "$"))
			}
		}
		realValues[key] = realValue
	}

	return realValues, nil
}

func runKptTestcase(solutionPath string, testValues map[string]string, originalValues map[string]string) error {
	// Set the kpt setters defined in the testcase.
	log.Println("======Setting the kpt setters...======")
	for key, value := range testValues {
		output, err := exec.Command("kpt", "cfg", "set", solutionPath, key,
			value, "--set-by", "test").CombinedOutput()
		if err != nil {
			errToReturn := fmt.Errorf("error setting setter '%s' with value "+
				"'%s': %v\nstdout: %s", key, value, err, string(output))

			// Clean up before exit with errors.
			if err := resetKptSetters(solutionPath, originalValues); err != nil {
				return concatErrors(
					"error resetting kpt setters before exiting",
					err, errToReturn)
			}

			return errToReturn
		}
		log.Printf("%s\n", string(output))
	}
	log.Println("======Successfully set the kpt setters======")

	// Apply all the resources.
	log.Println("======Creating the resources...======")
	output, err := exec.Command("kubectl", "create", "-f", solutionPath).CombinedOutput()
	if err != nil {
		errToReturn := fmt.Errorf("error creating resources: %v\nstdout: %s", err, string(output))

		// Clean up before exit with errors.
		if err := cleanUp(solutionPath, originalValues); err != nil {
			return concatErrors(
				"error cleanning up resources before exiting",
				err, errToReturn)
		}
		return errToReturn
	}
	log.Printf("%s\n", string(output))
	log.Println("======Successfully created the resources======")

	// Wait for all the resources to be ready.
	if err := verifyReadyCondition(solutionPath); err != nil {
		errToReturn := fmt.Errorf("error verifying the ready condition: %v", err)

		// Clean up before exit with errors.
		if err := cleanUp(solutionPath, originalValues); err != nil {
			return concatErrors(
				"error cleanning up resources before exiting",
				err, errToReturn)
		}
		return errToReturn
	}

	// Clean up.
	return cleanUp(solutionPath, originalValues)
}

func cleanUp(solutionPath string, originalValues map[string]string) error {
	resourceErr := deleteResources(solutionPath)
	setterErr := resetKptSetters(solutionPath, originalValues)
	if resourceErr != nil || setterErr != nil {
		return concatErrors(
			fmt.Sprintf("error cleanning up the test for solution %q. "+
				"Please manually delete the Config Connector resources and "+
				"reset the kpt setters", solutionPath),
			resourceErr, setterErr)
	}
	return nil
}

func verifyReadyCondition(solutionPath string) error {
	log.Println("======Verifying that all the Config Connector resources are ready...======")

	files, err := ioutil.ReadDir(solutionPath)
	if err != nil {
		return fmt.Errorf("error reading solution directory %q: %v", solutionPath, err)
	}

	for _, file := range files {
		// We should only verify the YAML config files for Config Connector
		// resources.
		fileName := file.Name()
		if !strings.HasSuffix(fileName, yamlFileSuffix) || strings.Contains(fileName, "namespace") {
			continue
		}

		resourceFilePath := filepath.Join(solutionPath, fileName)
		output, err := exec.Command("kubectl", "wait", "--for=condition=ready",
			"-f", resourceFilePath, "--timeout=60s").CombinedOutput()
		if err != nil {
			errToReturn := fmt.Errorf("resource in file %q is not ready in 60 seconds: %v\nstdout: %s", fileName, err, string(output))
			status, err := getResourceStatus(resourceFilePath)
			if err != nil {
				return concatErrors("error printing resource status", err, errToReturn)
			}

			return fmt.Errorf("%v\nResource status:\n%s", errToReturn, status)
		}
		log.Printf("%s\n", string(output))

	}

	log.Println("======All the Config Connector resrouces are ready======")
	return nil
}

func getResourceStatus(resourceFilePath string) (string, error) {
	output, err := exec.Command("kubectl", "get", "-f", resourceFilePath,
		"-o=custom-columns=NAME:.metadata.name,KIND:.kind,CONDITION.REASON:.status.conditions[0].reason,CONDITION.MESSAGE:.status.conditions[0].message").
		CombinedOutput()

	if err != nil {
		return "", fmt.Errorf("error getting the resource status: %v\nstdout: %s", err, string(output))
	}

	return string(output), nil
}

func deleteResources(solutionPath string) error {
	log.Println("======Deleting the resources...======")
	output, err := exec.Command("kubectl", "delete", "-f", solutionPath, "--wait").CombinedOutput()
	if err != nil {
		err = fmt.Errorf("error deleting resources: %v\nstdout: %s", err, string(output))
		if isNotFoundErrorOnly(err) {
			log.Println(err)
			log.Println("======Finished deleting the resources======")
			return nil
		}
		return fmt.Errorf("error deleting resources: %v\nstdout: %s", err, string(output))
	}
	log.Printf("%s\n", output)
	log.Println("======Successfully deleted the resources======")
	return nil
}

func isNotFoundErrorOnly(err error) bool {
	numErrors := strings.Count(err.Error(), "Error from server")
	numNotFoundErrors := strings.Count(err.Error(), "Error from server (NotFound)")
	return numErrors == numNotFoundErrors
}

func resetKptSetters(solutionPath string, originalValues map[string]string) error {
	log.Println("======Resetting the kpt setters...======")
	for key, value := range originalValues {
		output, err := exec.Command("kpt", "cfg", "set", solutionPath, key, value, "--set-by", "PLACEHOLDER").CombinedOutput()
		if err != nil {
			return fmt.Errorf("error setting setter '%s' back to the original value '%s': %v\nstdout: %s", key, value, err, string(output))
		}
		log.Printf("%s\n", string(output))
	}
	log.Println("======Successfully reset the kpt setters======")
	return nil
}

func concatErrors(msg string, errs ...error) error {
	errToReturn := errors.New(msg)
	for _, err := range errs {
		if err == nil {
			continue
		}
		errToReturn = fmt.Errorf("%v:%v", errToReturn, err)
	}
	return errToReturn
}
<|endoftext|>


<commit_before>package torrent

import (
	"crypto/rand"
	"errors"
	"fmt"
	"io"
	"net"
	"net/url"
	"sync"

	"github.com/cenkalti/rain/bitfield"
	"github.com/cenkalti/rain/logger"
	"github.com/cenkalti/rain/metainfo"
	"github.com/cenkalti/rain/mse"
	"github.com/cenkalti/rain/torrentdata"
	"github.com/cenkalti/rain/tracker"
	"github.com/cenkalti/rain/tracker/httptracker"
	"github.com/cenkalti/rain/tracker/udptracker"
	"github.com/hashicorp/go-multierror"
)

// TODO move these to config
const (
	// Do not connect more than maxPeerPerTorrent peers.
	maxPeerPerTorrent = 60

	// Maximum simultaneous uploads.
	uploadSlotsPerTorrent = 4

	// Request pieces in blocks of this size.
	blockSize = 16 * 1024
)

var (
	// http://www.bittorrent.org/beps/bep_0020.html
	peerIDPrefix = []byte("-RN" + Version + "-")

	// Version of client. Set during build.
	Version = "0000" // zero means development version

	// ErrClosed is returned when doing and operation on an already closed torrent.
	ErrClosed = errors.New("torrent is closed")
)

// Torrent connect to peers and downloads files from swarm.
type Torrent struct {
	peerID        [20]byte             // unique id per torrent
	metainfo      *metainfo.MetaInfo   // parsed torrent file
	data          *torrentdata.Data    // provides access to files on disk
	dest          string               // path of files on disk
	port          int                  // listen for peer connections
	listener      *net.TCPListener     // listens port for peer connections
	tracker       tracker.Tracker      // tracker to announce
	peers         map[[20]byte]*peer   // connected peers
	sKeyHash      [20]byte             // for encryption, hash of the info hash
	bitfield      *bitfield.Bitfield   // keeps track of the pieces we have
	peerLimiter   chan struct{}        // semaphore for limiting number of peers
	completed     chan struct{}        // closed when all pieces are downloaded
	onceCompleted sync.Once            // for closing completed channel only once
	stopC         chan struct{}        // all goroutines stop when closed
	stopWG        sync.WaitGroup       // for waiting running goroutines
	peerAddrs     []*peerAddr          // contains peers not connected yet, sorted by oldest first
	peerAddrsMap  map[string]*peerAddr // contains peers not connected yet, keyed by addr string
	m             sync.Mutex           // protects all state in this torrent and it's peers
	running       bool                 // true after Start() is called
	closed        bool                 // true after Close() is called
	log           logger.Logger
}

// New returns a new torrent by reading a metainfo file.
//
// Files are read from disk. If there are existing files, hash check will be done.
//
// Returned torrent is in stopped state.
//
// Close must be called before discarding the torrent.
func New(r io.Reader, dest string, port int) (*Torrent, error) {
	m, err := metainfo.New(r)
	if err != nil {
		return nil, err
	}
	logName := m.Info.Name
	if len(logName) > 8 {
		logName = logName[:8]
	}
	var peerID [20]byte
	copy(peerID[:], peerIDPrefix)
	_, err = rand.Read(peerID[len(peerIDPrefix):]) // nolint: gosec
	if err != nil {
		return nil, err
	}
	trk, err := newTracker(m.Announce)
	if err != nil {
		return nil, err
	}
	data, err := torrentdata.New(m.Info, dest)
	if err != nil {
		return nil, err
	}
	bf, err := data.Verify()
	if err != nil {
		return nil, err
	}
	t := &Torrent{
		peerID:       peerID,
		metainfo:     m,
		data:         data,
		dest:         dest,
		port:         port,
		tracker:      trk,
		peers:        make(map[[20]byte]*peer),
		sKeyHash:     mse.HashSKey(m.Info.Hash[:]),
		bitfield:     bf,
		peerLimiter:  make(chan struct{}, maxPeerPerTorrent),
		completed:    make(chan struct{}),
		peerAddrsMap: make(map[string]*peerAddr),
		log:          logger.New("download " + logName),
	}
	t.checkCompletion()
	return t, nil
}

func (t *Torrent) checkCompletion() {
	if t.bitfield.All() {
		t.onceCompleted.Do(func() {
			close(t.completed)
			t.log.Notice("Download completed")
		})
	}
}

func newTracker(trackerURL string) (tracker.Tracker, error) {
	u, err := url.Parse(trackerURL)
	if err != nil {
		return nil, err
	}
	l := logger.New("tracker " + trackerURL)
	switch u.Scheme {
	case "http", "https":
		return httptracker.New(u, l), nil
	case "udp":
		return udptracker.New(u, l), nil
	default:
		return nil, fmt.Errorf("unsupported tracker scheme: %s", u.Scheme)
	}
}

// Start listening peer port, accepting incoming peer connections and download missing pieces.
//
// Seeding continues after all files are donwloaded.
func (t *Torrent) Start() error {
	t.m.Lock()
	defer t.m.Unlock()
	if t.closed {
		return ErrClosed
	}
	if t.running {
		return nil
	}
	var err error
	t.listener, err = net.ListenTCP("tcp4", &net.TCPAddr{Port: t.port})
	if err != nil {
		return err
	}
	t.log.Notice("Listening peers on tcp://" + t.listener.Addr().String())
	t.stopC = make(chan struct{})
	t.stopWG.Add(5)
	go t.announcer()
	go t.accepter()
	go t.dialer()
	go t.downloader()
	go t.uploader()
	return nil
}

// Stop downloading and uploading, disconnect all peers and close peer port.
func (t *Torrent) Stop() error {
	t.m.Lock()
	defer t.m.Unlock()
	if t.closed {
		return ErrClosed
	}
	if !t.running {
		return nil
	}
	err := t.listener.Close()
	if err != nil {
		return err
	}
	close(t.stopC)
	t.stopWG.Wait()
	// TODO remove temporary structures
	return nil
}

// Close this torrent and release all resources.
func (t *Torrent) Close() error {
	t.m.Lock()
	defer t.m.Unlock()
	if t.closed {
		return nil
	}
	if t.running {
		err := t.Stop()
		if err != nil {
			return err
		}
	}
	var result error
	err := t.listener.Close()
	if err != nil {
		result = multierror.Append(result, err)
	}
	for _, p := range t.peers {
		err = p.Close()
		if err != nil {
			result = multierror.Append(result, err)
		}
	}
	err = t.data.Close()
	if err != nil {
		result = multierror.Append(result, err)
	}
	return result
}

// Port returns the port number that the client is listening.
// If the client does not listen any port, returns 0.
func (t *Torrent) Port() int {
	if t.listener != nil {
		return t.listener.Addr().(*net.TCPAddr).Port
	}
	return 0
}

// BytesCompleted returns the number of bytes downlaoded and passed hash check.
func (t *Torrent) BytesCompleted() int64 {
	t.m.Lock()
	defer t.m.Unlock()
	sum := int64(t.bitfield.Count() * t.metainfo.Info.PieceLength)

	// Last piece usually not in full size.
	lastPiece := t.metainfo.Info.NumPieces - 1
	if t.bitfield.Test(lastPiece) {
		sum -= int64(t.metainfo.Info.PieceLength)
		sum += int64(t.data.Piece(int(lastPiece)).Length)
	}
	return sum
}

// PeerID is unique per torrent.
func (t *Torrent) PeerID() [20]byte { return t.peerID }

// InfoHash identifies the torrent file that is being downloaded.
func (t *Torrent) InfoHash() [20]byte { return t.metainfo.Info.Hash }

// CompleteNotify returns a channel that is closed once all pieces are downloaded successfully.
func (t *Torrent) CompleteNotify() chan struct{} { return t.completed }

// BytesDownloaded is the number of bytes downloaded from swarm.
//
// Because some pieces may be downloaded more than once, this number may be greater than BytesCompleted returns.
func (t *Torrent) BytesDownloaded() int64 { return t.BytesCompleted() } // TODO not the same thing

// BytesUploaded is the number of bytes uploaded to the swarm.
func (t *Torrent) BytesUploaded() int64 { return 0 } // TODO count uploaded bytes

// BytesLeft is the number of bytes that is needed to complete all missing pieces.
func (t *Torrent) BytesLeft() int64 { return t.BytesTotal() - t.BytesCompleted() }

// BytesTotal is the number of total bytes of files in torrent.
func (t *Torrent) BytesTotal() int64 { return t.metainfo.Info.TotalLength }

func closeConn(conn net.Conn, log logger.Logger) {
	err := conn.Close()
	if err != nil {
		log.Errorln("cannot close conn:", err)
	}
}
<commit_msg>comment<commit_after>package torrent

import (
	"crypto/rand"
	"errors"
	"fmt"
	"io"
	"net"
	"net/url"
	"sync"

	"github.com/cenkalti/rain/bitfield"
	"github.com/cenkalti/rain/logger"
	"github.com/cenkalti/rain/metainfo"
	"github.com/cenkalti/rain/mse"
	"github.com/cenkalti/rain/torrentdata"
	"github.com/cenkalti/rain/tracker"
	"github.com/cenkalti/rain/tracker/httptracker"
	"github.com/cenkalti/rain/tracker/udptracker"
	"github.com/hashicorp/go-multierror"
)

// TODO move these to config
const (
	// Do not connect more than maxPeerPerTorrent peers.
	maxPeerPerTorrent = 60

	// Maximum simultaneous uploads.
	uploadSlotsPerTorrent = 4

	// Request pieces in blocks of this size.
	blockSize = 16 * 1024
)

var (
	// http://www.bittorrent.org/beps/bep_0020.html
	peerIDPrefix = []byte("-RN" + Version + "-")

	// Version of client. Set during build.
	Version = "0000" // zero means development version

	// ErrClosed is returned when doing and operation on an already closed torrent.
	ErrClosed = errors.New("torrent is closed")
)

// Torrent connect to peers and downloads files from swarm.
type Torrent struct {
	peerID        [20]byte             // unique id per torrent
	metainfo      *metainfo.MetaInfo   // parsed torrent file
	data          *torrentdata.Data    // provides access to files on disk
	dest          string               // path of files on disk
	port          int                  // listen for peer connections
	listener      *net.TCPListener     // listens port for peer connections
	tracker       tracker.Tracker      // tracker to announce
	peers         map[[20]byte]*peer   // connected peers
	sKeyHash      [20]byte             // for encryption, hash of the info hash
	bitfield      *bitfield.Bitfield   // keeps track of the pieces we have
	peerLimiter   chan struct{}        // semaphore for limiting number of peers
	completed     chan struct{}        // closed when all pieces are downloaded
	onceCompleted sync.Once            // for closing completed channel only once
	stopC         chan struct{}        // all goroutines stop when closed
	stopWG        sync.WaitGroup       // for waiting running goroutines
	peerAddrs     []*peerAddr          // contains peers not connected yet, sorted by oldest first
	peerAddrsMap  map[string]*peerAddr // contains peers not connected yet, keyed by addr string
	m             sync.Mutex           // protects all state in this torrent and it's peers
	running       bool                 // true after Start() is called
	closed        bool                 // true after Close() is called
	log           logger.Logger
}

// New returns a new torrent by reading a metainfo file.
//
// Files are read from disk. If there are existing files, hash check will be done.
//
// Returned torrent is in stopped state.
//
// Close must be called before discarding the torrent.
func New(r io.Reader, dest string, port int) (*Torrent, error) {
	m, err := metainfo.New(r)
	if err != nil {
		return nil, err
	}
	logName := m.Info.Name
	if len(logName) > 8 {
		logName = logName[:8]
	}
	var peerID [20]byte
	copy(peerID[:], peerIDPrefix)
	_, err = rand.Read(peerID[len(peerIDPrefix):]) // nolint: gosec
	if err != nil {
		return nil, err
	}
	trk, err := newTracker(m.Announce)
	if err != nil {
		return nil, err
	}
	data, err := torrentdata.New(m.Info, dest)
	if err != nil {
		return nil, err
	}
	bf, err := data.Verify()
	if err != nil {
		return nil, err
	}
	t := &Torrent{
		peerID:       peerID,
		metainfo:     m,
		data:         data,
		dest:         dest,
		port:         port,
		tracker:      trk,
		peers:        make(map[[20]byte]*peer),
		sKeyHash:     mse.HashSKey(m.Info.Hash[:]),
		bitfield:     bf,
		peerLimiter:  make(chan struct{}, maxPeerPerTorrent),
		completed:    make(chan struct{}),
		peerAddrsMap: make(map[string]*peerAddr),
		log:          logger.New("download " + logName),
	}
	t.checkCompletion()
	return t, nil
}

func (t *Torrent) checkCompletion() {
	if t.bitfield.All() {
		t.onceCompleted.Do(func() {
			close(t.completed)
			t.log.Notice("Download completed")
		})
	}
}

func newTracker(trackerURL string) (tracker.Tracker, error) {
	u, err := url.Parse(trackerURL)
	if err != nil {
		return nil, err
	}
	l := logger.New("tracker " + trackerURL)
	switch u.Scheme {
	case "http", "https":
		return httptracker.New(u, l), nil
	case "udp":
		return udptracker.New(u, l), nil
	default:
		return nil, fmt.Errorf("unsupported tracker scheme: %s", u.Scheme)
	}
}

// Start listening peer port, accepting incoming peer connections and download missing pieces.
//
// Seeding continues after all files are donwloaded.
func (t *Torrent) Start() error {
	t.m.Lock()
	defer t.m.Unlock()
	if t.closed {
		return ErrClosed
	}
	if t.running {
		return nil
	}
	var err error
	t.listener, err = net.ListenTCP("tcp4", &net.TCPAddr{Port: t.port})
	if err != nil {
		return err
	}
	t.log.Notice("Listening peers on tcp://" + t.listener.Addr().String())
	t.stopC = make(chan struct{})
	t.stopWG.Add(5)
	go t.announcer()  // get peers from tracker
	go t.accepter()   // accept incoming peer connections
	go t.dialer()     // connect to new peers
	go t.downloader() // request missing pieces from peers
	go t.uploader()   // send requested blocks
	return nil
}

// Stop downloading and uploading, disconnect all peers and close peer port.
func (t *Torrent) Stop() error {
	t.m.Lock()
	defer t.m.Unlock()
	if t.closed {
		return ErrClosed
	}
	if !t.running {
		return nil
	}
	err := t.listener.Close()
	if err != nil {
		return err
	}
	close(t.stopC)
	t.stopWG.Wait()
	// TODO remove temporary structures
	return nil
}

// Close this torrent and release all resources.
func (t *Torrent) Close() error {
	t.m.Lock()
	defer t.m.Unlock()
	if t.closed {
		return nil
	}
	if t.running {
		err := t.Stop()
		if err != nil {
			return err
		}
	}
	var result error
	err := t.listener.Close()
	if err != nil {
		result = multierror.Append(result, err)
	}
	for _, p := range t.peers {
		err = p.Close()
		if err != nil {
			result = multierror.Append(result, err)
		}
	}
	err = t.data.Close()
	if err != nil {
		result = multierror.Append(result, err)
	}
	return result
}

// Port returns the port number that the client is listening.
// If the client does not listen any port, returns 0.
func (t *Torrent) Port() int {
	if t.listener != nil {
		return t.listener.Addr().(*net.TCPAddr).Port
	}
	return 0
}

// BytesCompleted returns the number of bytes downlaoded and passed hash check.
func (t *Torrent) BytesCompleted() int64 {
	t.m.Lock()
	defer t.m.Unlock()
	sum := int64(t.bitfield.Count() * t.metainfo.Info.PieceLength)

	// Last piece usually not in full size.
	lastPiece := t.metainfo.Info.NumPieces - 1
	if t.bitfield.Test(lastPiece) {
		sum -= int64(t.metainfo.Info.PieceLength)
		sum += int64(t.data.Piece(int(lastPiece)).Length)
	}
	return sum
}

// PeerID is unique per torrent.
func (t *Torrent) PeerID() [20]byte { return t.peerID }

// InfoHash identifies the torrent file that is being downloaded.
func (t *Torrent) InfoHash() [20]byte { return t.metainfo.Info.Hash }

// CompleteNotify returns a channel that is closed once all pieces are downloaded successfully.
func (t *Torrent) CompleteNotify() chan struct{} { return t.completed }

// BytesDownloaded is the number of bytes downloaded from swarm.
//
// Because some pieces may be downloaded more than once, this number may be greater than BytesCompleted returns.
func (t *Torrent) BytesDownloaded() int64 { return t.BytesCompleted() } // TODO not the same thing

// BytesUploaded is the number of bytes uploaded to the swarm.
func (t *Torrent) BytesUploaded() int64 { return 0 } // TODO count uploaded bytes

// BytesLeft is the number of bytes that is needed to complete all missing pieces.
func (t *Torrent) BytesLeft() int64 { return t.BytesTotal() - t.BytesCompleted() }

// BytesTotal is the number of total bytes of files in torrent.
func (t *Torrent) BytesTotal() int64 { return t.metainfo.Info.TotalLength }

func closeConn(conn net.Conn, log logger.Logger) {
	err := conn.Close()
	if err != nil {
		log.Errorln("cannot close conn:", err)
	}
}
<|endoftext|>


<commit_before>package transfer

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os/exec"
	"regexp"
	"strings"

	"github.com/github/git-lfs/localstorage"

	"github.com/github/git-lfs/api"
	"github.com/github/git-lfs/subprocess"
	"github.com/rubyist/tracerx"

	"github.com/github/git-lfs/config"
)

// Adapter for custom transfer via external process
type customAdapter struct {
	*adapterBase
	path                string
	args                string
	concurrent          bool
	originalConcurrency int
}

type customAdapterWorkerContext struct {
	cmd         *exec.Cmd
	stdout      io.ReadCloser
	bufferedOut *bufio.Reader
	stdin       io.WriteCloser
}

type customAdapterInitRequest struct {
	Operation           string `json:"operation"`
	Concurrent          bool   `json:"concurrent"`
	ConcurrentTransfers int    `json:"concurrenttransfers"`
}
type customAdapterInitResponse struct {
	Error *api.ObjectError `json:"error,omitempty"`
}
type customAdapterTransferRequest struct { // common between upload/download
	Oid    string            `json:"oid"`
	Size   int64             `json:"size"`
	Path   string            `json:"path,omitempty"`
	Action *api.LinkRelation `json:"action"`
}
type customAdapterTransferResponse struct { // common between upload/download
	Oid   string           `json:"oid"`
	Path  string           `json:"path,omitempty"` // always blank for upload
	Error *api.ObjectError `json:"error,omitempty"`
}
type customAdapterTerminateRequest struct {
	Complete bool `json:"complete"`
}
type customAdapterProgressResponse struct {
	Oid            string `json:"oid"`
	BytesSoFar     int64  `json:"bytesSoFar"`
	BytesSinceLast int    `json:"bytesSinceLast"`
}

func (a *customAdapter) Begin(maxConcurrency int, cb TransferProgressCallback, completion chan TransferResult) error {
	// If config says not to launch multiple processes, downgrade incoming value
	useConcurrency := maxConcurrency
	if !a.concurrent {
		useConcurrency = 1
	}
	a.originalConcurrency = maxConcurrency

	tracerx.Printf("xfer: Custom transfer adapter %q using concurrency %d", a.name, useConcurrency)

	// Use common workers impl, but downgrade workers to number of processes
	return a.adapterBase.Begin(useConcurrency, cb, completion)
}

func (a *customAdapter) ClearTempStorage() error {
	// no action requred
	return nil
}

func (a *customAdapter) WorkerStarting(workerNum int) (interface{}, error) {

	// Start a process per worker
	// If concurrent = false we have already dialled back workers to 1
	tracerx.Printf("xfer: starting up custom transfer process %q for worker %d", a.name, workerNum)
	cmd := subprocess.ExecCommand(a.path, a.args)
	outp, err := cmd.StdoutPipe()
	if err != nil {
		return nil, fmt.Errorf("Failed to get stdout for custom transfer command %q remote: %v", a.path, err)
	}
	inp, err := cmd.StdinPipe()
	if err != nil {
		return nil, fmt.Errorf("Failed to get stdin for custom transfer command %q remote: %v", a.path, err)
	}
	err = cmd.Start()
	if err != nil {
		return nil, fmt.Errorf("Failed to start custom transfer command %q remote: %v", a.path, err)
	}
	// Set up buffered reader/writer since we operate on lines
	ctx := &customAdapterWorkerContext{cmd, outp, bufio.NewReader(outp), inp}

	// send initiate message
	initReq := &customAdapterInitRequest{a.getOperationName(), a.concurrent, a.originalConcurrency}
	var initResp customAdapterInitResponse
	err = a.exchangeMessage(ctx, initReq, &initResp)
	if err != nil {
		a.abortWorkerProcess(ctx)
		return nil, err
	}

	tracerx.Printf("xfer: %q for worker %d started OK", a.name, workerNum)

	// Save this process context and use in future callbacks
	return ctx, nil
}

func (a *customAdapter) getOperationName() string {
	if a.direction == Download {
		return "download"
	}
	return "upload"
}

// sendMessage sends a JSON message to the custom adapter process
func (a *customAdapter) sendMessage(ctx *customAdapterWorkerContext, req interface{}) error {
	b, err := json.Marshal(req)
	if err != nil {
		return err
	}
	// Line oriented JSON
	b = append(b, '\n')
	_, err = ctx.stdin.Write(b)
	if err != nil {
		return err
	}
	return nil
}

// readResponse reads one of 1..N possible responses and populates the first one which
// was unmarshalled correctly. This allows us to listen for one of possibly many responses
// Returns the index of the item which was populated
func (a *customAdapter) readResponse(ctx *customAdapterWorkerContext, possResps []interface{}) (int, error) {
	line, err := ctx.bufferedOut.ReadString('\n')
	if err != nil {
		return 0, err
	}
	for i, resp := range possResps {
		if json.Unmarshal([]byte(line), resp) == nil {
			return i, nil
		}
	}
	return 0, fmt.Errorf("Response %q did not match any of possible responses %v", string(line), possResps)

}

// exchangeMessage sends a message to a process and reads a response if resp != nil
// Only fatal errors to communicate return an error, errors may be embedded in reply
func (a *customAdapter) exchangeMessage(ctx *customAdapterWorkerContext, req, resp interface{}) error {

	err := a.sendMessage(ctx, req)
	if err != nil {
		return err
	}
	// Read response if needed
	if resp != nil {
		_, err = a.readResponse(ctx, []interface{}{resp})
		return err
	}
	return nil
}

// shutdownWorkerProcess terminates gracefully a custom adapter process
// returns an error if it couldn't shut down gracefully (caller may abortWorkerProcess)
func (a *customAdapter) shutdownWorkerProcess(ctx *customAdapterWorkerContext) error {
	termReq := &customAdapterTerminateRequest{true}
	err := a.exchangeMessage(ctx, termReq, nil)
	if err != nil {
		return err
	}
	ctx.stdin.Close()
	ctx.stdout.Close()
	return ctx.cmd.Wait()
}

// abortWorkerProcess terminates & aborts untidily, most probably breakdown of comms or internal error
func (a *customAdapter) abortWorkerProcess(ctx *customAdapterWorkerContext) {
	ctx.stdin.Close()
	ctx.stdout.Close()
	ctx.cmd.Process.Kill()
}
func (a *customAdapter) WorkerEnding(workerNum int, ctx interface{}) {
	customCtx, ok := ctx.(*customAdapterWorkerContext)
	if !ok {
		tracerx.Printf("Context object for custom transfer %q was of the wrong type", a.name)
		return
	}

	err := a.shutdownWorkerProcess(customCtx)
	if err != nil {
		tracerx.Printf("xfer: error finishing up custom transfer process %q, aborting: %v", a.name, err)
		a.abortWorkerProcess(customCtx)
	}
}

func (a *customAdapter) DoTransfer(ctx interface{}, t *Transfer, cb TransferProgressCallback, authOkFunc func()) error {
	if ctx == nil {
		return fmt.Errorf("Custom transfer %q was not properly initialized, see previous errors", a.name)
	}

	customCtx, ok := ctx.(*customAdapterWorkerContext)
	if !ok {
		return fmt.Errorf("Context object for custom transfer %q was of the wrong type", a.name)
	}
	var authCalled bool

	rel, ok := t.Object.Rel(a.getOperationName())
	if !ok {
		return errors.New("Object not found on the server.")
	}
	req := &customAdapterTransferRequest{t.Object.Oid, t.Object.Size, "", rel}
	if a.direction == Upload {
		req.Path = localstorage.Objects().ObjectPath(t.Object.Oid)
	}
	err := a.sendMessage(customCtx, req)
	if err != nil {
		return err
	}

	// 1..N replies (including progress & one of download / upload)
	possResps := []interface{}{&customAdapterProgressResponse{}, &customAdapterTransferResponse{}}
	var complete bool
	for !complete {
		respIdx, err := a.readResponse(customCtx, possResps)
		if err != nil {
			return err
		}
		var wasAuthOk bool
		switch respIdx {
		case 0:
			// Progress
			prog := possResps[0].(customAdapterProgressResponse)
			if prog.Oid != t.Object.Oid {
				return fmt.Errorf("Unexpected oid %q in response, expecting %q", prog.Oid, t.Object.Oid)
			}
			if cb != nil {
				cb(t.Name, t.Object.Size, prog.BytesSoFar, prog.BytesSinceLast)
			}
			wasAuthOk = prog.BytesSoFar > 0
		case 1:
			// Download/Upload complete
			comp := possResps[0].(customAdapterTransferResponse)
			if comp.Oid != t.Object.Oid {
				return fmt.Errorf("Unexpected oid %q in response, expecting %q", comp.Oid, t.Object.Oid)
			}
			if comp.Error != nil {
				return fmt.Errorf("Error transferring %q: %v", t.Object.Oid, comp.Error.Error())
			}
			wasAuthOk = true
			complete = true
		}
		// Call auth on first progress or success
		if wasAuthOk && authOkFunc != nil && !authCalled {
			authOkFunc()
			authCalled = true
		}
	}

	// Send verify if successful upload
	if a.direction == Upload {
		return api.VerifyUpload(t.Object)
	}
	return nil
}

func newCustomAdapter(name string, dir Direction, path, args string, concurrent bool) *customAdapter {
	c := &customAdapter{newAdapterBase(name, dir, nil), path, args, concurrent, 3}
	// self implements impl
	c.transferImpl = c
	return c
}

// Initialise custom adapters based on current config
func ConfigureCustomAdapters() {
	pathRegex := regexp.MustCompile(`lfs.customtransfer.([^.]+).path`)
	for k, v := range config.Config.AllGitConfig() {
		if match := pathRegex.FindStringSubmatch(k); match != nil {
			name := match[1]
			path := v
			var args string
			var concurrent bool
			var direction string
			// retrieve other values
			args, _ = config.Config.GitConfig(fmt.Sprintf("lfs.customtransfer.%s.args", name))
			concurrent = config.Config.GitConfigBool(fmt.Sprintf("lfs.customtransfer.%s.concurrent", name), true)
			direction, _ = config.Config.GitConfig(fmt.Sprintf("lfs.customtransfer.%s.direction", name))
			if len(direction) == 0 {
				direction = "both"
			} else {
				direction = strings.ToLower(direction)
			}

			// Separate closure for each since we need to capture vars above
			newfunc := func(name string, dir Direction) TransferAdapter {
				return newCustomAdapter(name, dir, path, args, concurrent)
			}

			if direction == "download" || direction == "both" {
				RegisterNewTransferAdapterFunc(name, Download, newfunc)
			}
			if direction == "upload" || direction == "both" {
				RegisterNewTransferAdapterFunc(name, Upload, newfunc)
			}

		}
	}

}

func init() {
	ConfigureCustomAdapters()
}
<commit_msg>Fix indexes when reading responses<commit_after>package transfer

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os/exec"
	"regexp"
	"strings"

	"github.com/github/git-lfs/localstorage"

	"github.com/github/git-lfs/api"
	"github.com/github/git-lfs/subprocess"
	"github.com/rubyist/tracerx"

	"github.com/github/git-lfs/config"
)

// Adapter for custom transfer via external process
type customAdapter struct {
	*adapterBase
	path                string
	args                string
	concurrent          bool
	originalConcurrency int
}

type customAdapterWorkerContext struct {
	cmd         *exec.Cmd
	stdout      io.ReadCloser
	bufferedOut *bufio.Reader
	stdin       io.WriteCloser
}

type customAdapterInitRequest struct {
	Operation           string `json:"operation"`
	Concurrent          bool   `json:"concurrent"`
	ConcurrentTransfers int    `json:"concurrenttransfers"`
}
type customAdapterInitResponse struct {
	Error *api.ObjectError `json:"error,omitempty"`
}
type customAdapterTransferRequest struct { // common between upload/download
	Oid    string            `json:"oid"`
	Size   int64             `json:"size"`
	Path   string            `json:"path,omitempty"`
	Action *api.LinkRelation `json:"action"`
}
type customAdapterTransferResponse struct { // common between upload/download
	Oid   string           `json:"oid"`
	Path  string           `json:"path,omitempty"` // always blank for upload
	Error *api.ObjectError `json:"error,omitempty"`
}
type customAdapterTerminateRequest struct {
	Complete bool `json:"complete"`
}
type customAdapterProgressResponse struct {
	Oid            string `json:"oid"`
	BytesSoFar     int64  `json:"bytesSoFar"`
	BytesSinceLast int    `json:"bytesSinceLast"`
}

func (a *customAdapter) Begin(maxConcurrency int, cb TransferProgressCallback, completion chan TransferResult) error {
	// If config says not to launch multiple processes, downgrade incoming value
	useConcurrency := maxConcurrency
	if !a.concurrent {
		useConcurrency = 1
	}
	a.originalConcurrency = maxConcurrency

	tracerx.Printf("xfer: Custom transfer adapter %q using concurrency %d", a.name, useConcurrency)

	// Use common workers impl, but downgrade workers to number of processes
	return a.adapterBase.Begin(useConcurrency, cb, completion)
}

func (a *customAdapter) ClearTempStorage() error {
	// no action requred
	return nil
}

func (a *customAdapter) WorkerStarting(workerNum int) (interface{}, error) {

	// Start a process per worker
	// If concurrent = false we have already dialled back workers to 1
	tracerx.Printf("xfer: starting up custom transfer process %q for worker %d", a.name, workerNum)
	cmd := subprocess.ExecCommand(a.path, a.args)
	outp, err := cmd.StdoutPipe()
	if err != nil {
		return nil, fmt.Errorf("Failed to get stdout for custom transfer command %q remote: %v", a.path, err)
	}
	inp, err := cmd.StdinPipe()
	if err != nil {
		return nil, fmt.Errorf("Failed to get stdin for custom transfer command %q remote: %v", a.path, err)
	}
	err = cmd.Start()
	if err != nil {
		return nil, fmt.Errorf("Failed to start custom transfer command %q remote: %v", a.path, err)
	}
	// Set up buffered reader/writer since we operate on lines
	ctx := &customAdapterWorkerContext{cmd, outp, bufio.NewReader(outp), inp}

	// send initiate message
	initReq := &customAdapterInitRequest{a.getOperationName(), a.concurrent, a.originalConcurrency}
	var initResp customAdapterInitResponse
	err = a.exchangeMessage(ctx, initReq, &initResp)
	if err != nil {
		a.abortWorkerProcess(ctx)
		return nil, err
	}

	tracerx.Printf("xfer: %q for worker %d started OK", a.name, workerNum)

	// Save this process context and use in future callbacks
	return ctx, nil
}

func (a *customAdapter) getOperationName() string {
	if a.direction == Download {
		return "download"
	}
	return "upload"
}

// sendMessage sends a JSON message to the custom adapter process
func (a *customAdapter) sendMessage(ctx *customAdapterWorkerContext, req interface{}) error {
	b, err := json.Marshal(req)
	if err != nil {
		return err
	}
	// Line oriented JSON
	b = append(b, '\n')
	_, err = ctx.stdin.Write(b)
	if err != nil {
		return err
	}
	return nil
}

// readResponse reads one of 1..N possible responses and populates the first one which
// was unmarshalled correctly. This allows us to listen for one of possibly many responses
// Returns the index of the item which was populated
func (a *customAdapter) readResponse(ctx *customAdapterWorkerContext, possResps []interface{}) (int, error) {
	line, err := ctx.bufferedOut.ReadString('\n')
	if err != nil {
		return 0, err
	}
	for i, resp := range possResps {
		if json.Unmarshal([]byte(line), resp) == nil {
			return i, nil
		}
	}
	return 0, fmt.Errorf("Response %q did not match any of possible responses %v", string(line), possResps)

}

// exchangeMessage sends a message to a process and reads a response if resp != nil
// Only fatal errors to communicate return an error, errors may be embedded in reply
func (a *customAdapter) exchangeMessage(ctx *customAdapterWorkerContext, req, resp interface{}) error {

	err := a.sendMessage(ctx, req)
	if err != nil {
		return err
	}
	// Read response if needed
	if resp != nil {
		_, err = a.readResponse(ctx, []interface{}{resp})
		return err
	}
	return nil
}

// shutdownWorkerProcess terminates gracefully a custom adapter process
// returns an error if it couldn't shut down gracefully (caller may abortWorkerProcess)
func (a *customAdapter) shutdownWorkerProcess(ctx *customAdapterWorkerContext) error {
	termReq := &customAdapterTerminateRequest{true}
	err := a.exchangeMessage(ctx, termReq, nil)
	if err != nil {
		return err
	}
	ctx.stdin.Close()
	ctx.stdout.Close()
	return ctx.cmd.Wait()
}

// abortWorkerProcess terminates & aborts untidily, most probably breakdown of comms or internal error
func (a *customAdapter) abortWorkerProcess(ctx *customAdapterWorkerContext) {
	ctx.stdin.Close()
	ctx.stdout.Close()
	ctx.cmd.Process.Kill()
}
func (a *customAdapter) WorkerEnding(workerNum int, ctx interface{}) {
	customCtx, ok := ctx.(*customAdapterWorkerContext)
	if !ok {
		tracerx.Printf("Context object for custom transfer %q was of the wrong type", a.name)
		return
	}

	err := a.shutdownWorkerProcess(customCtx)
	if err != nil {
		tracerx.Printf("xfer: error finishing up custom transfer process %q, aborting: %v", a.name, err)
		a.abortWorkerProcess(customCtx)
	}
}

func (a *customAdapter) DoTransfer(ctx interface{}, t *Transfer, cb TransferProgressCallback, authOkFunc func()) error {
	if ctx == nil {
		return fmt.Errorf("Custom transfer %q was not properly initialized, see previous errors", a.name)
	}

	customCtx, ok := ctx.(*customAdapterWorkerContext)
	if !ok {
		return fmt.Errorf("Context object for custom transfer %q was of the wrong type", a.name)
	}
	var authCalled bool

	rel, ok := t.Object.Rel(a.getOperationName())
	if !ok {
		return errors.New("Object not found on the server.")
	}
	req := &customAdapterTransferRequest{t.Object.Oid, t.Object.Size, "", rel}
	if a.direction == Upload {
		req.Path = localstorage.Objects().ObjectPath(t.Object.Oid)
	}
	err := a.sendMessage(customCtx, req)
	if err != nil {
		return err
	}

	// 1..N replies (including progress & one of download / upload)
	possResps := []interface{}{&customAdapterProgressResponse{}, &customAdapterTransferResponse{}}
	var complete bool
	for !complete {
		respIdx, err := a.readResponse(customCtx, possResps)
		if err != nil {
			return err
		}
		var wasAuthOk bool
		switch respIdx {
		case 0:
			// Progress
			prog := possResps[respIdx].(customAdapterProgressResponse)
			if prog.Oid != t.Object.Oid {
				return fmt.Errorf("Unexpected oid %q in response, expecting %q", prog.Oid, t.Object.Oid)
			}
			if cb != nil {
				cb(t.Name, t.Object.Size, prog.BytesSoFar, prog.BytesSinceLast)
			}
			wasAuthOk = prog.BytesSoFar > 0
		case 1:
			// Download/Upload complete
			comp := possResps[respIdx].(customAdapterTransferResponse)
			if comp.Oid != t.Object.Oid {
				return fmt.Errorf("Unexpected oid %q in response, expecting %q", comp.Oid, t.Object.Oid)
			}
			if comp.Error != nil {
				return fmt.Errorf("Error transferring %q: %v", t.Object.Oid, comp.Error.Error())
			}
			wasAuthOk = true
			complete = true
		}
		// Call auth on first progress or success
		if wasAuthOk && authOkFunc != nil && !authCalled {
			authOkFunc()
			authCalled = true
		}
	}

	// Send verify if successful upload
	if a.direction == Upload {
		return api.VerifyUpload(t.Object)
	}
	return nil
}

func newCustomAdapter(name string, dir Direction, path, args string, concurrent bool) *customAdapter {
	c := &customAdapter{newAdapterBase(name, dir, nil), path, args, concurrent, 3}
	// self implements impl
	c.transferImpl = c
	return c
}

// Initialise custom adapters based on current config
func ConfigureCustomAdapters() {
	pathRegex := regexp.MustCompile(`lfs.customtransfer.([^.]+).path`)
	for k, v := range config.Config.AllGitConfig() {
		if match := pathRegex.FindStringSubmatch(k); match != nil {
			name := match[1]
			path := v
			var args string
			var concurrent bool
			var direction string
			// retrieve other values
			args, _ = config.Config.GitConfig(fmt.Sprintf("lfs.customtransfer.%s.args", name))
			concurrent = config.Config.GitConfigBool(fmt.Sprintf("lfs.customtransfer.%s.concurrent", name), true)
			direction, _ = config.Config.GitConfig(fmt.Sprintf("lfs.customtransfer.%s.direction", name))
			if len(direction) == 0 {
				direction = "both"
			} else {
				direction = strings.ToLower(direction)
			}

			// Separate closure for each since we need to capture vars above
			newfunc := func(name string, dir Direction) TransferAdapter {
				return newCustomAdapter(name, dir, path, args, concurrent)
			}

			if direction == "download" || direction == "both" {
				RegisterNewTransferAdapterFunc(name, Download, newfunc)
			}
			if direction == "upload" || direction == "both" {
				RegisterNewTransferAdapterFunc(name, Upload, newfunc)
			}

		}
	}

}

func init() {
	ConfigureCustomAdapters()
}
<|endoftext|>


<commit_before>package ts3sqlib

import (
	"fmt"
	"strconv"
	"strings"
)

var (
	MsgEndError       = NewError(0, "ok", "")
	ClosedError       = NewError(-1, "connection closed", "")
	InvalidLoginError = NewError(520, "invalid\\sloginname\\sor\\spassword", "") //TODO: remove the \\s after implementing escape/unescape
)

//Error contains additional error information.
type Error struct {
	Id       int
	Msg      string
	ExtraMsg string
}

//Error returns the error in a string representation.
func (err Error) Error() string {
	s := fmt.Sprintf("error id=%d msg=%s", err.Id, err.Msg)
	if err.ExtraMsg != "" {
		s += fmt.Sprintf(" extra_msg=%s", err.ExtraMsg)
	}
	return s
}

//NewError creates a new Error from an id, message and an extra_message.
func NewError(id int, msg, extramsg string) Error {
	return Error{id, msg, extramsg}
}

func (e Error) equals(err error) bool {
	if e2, ok := err.(Error); ok {
		return e.Id == e2.Id
	} else {
		return false
	}
}

//isError tests if a given string is a ts3 server query error
func isError(line string) bool {
	return strings.HasPrefix(line, "error") &&
		strings.Contains(line, "id=") && strings.Contains(line, "msg=")
}

//toError converts a given string into a ts3 server query error.
func toError(line string) (err Error) {
	if !isError(line) {
		err = NewError(666, "line is not an error!", "")
		return
	}

	parts := strings.Split(line, " ")
	for i := range parts {
		if strings.Contains(parts[i], "=") {
			key, value, err2 := splitAtEqual(parts[i])
			if err2 {
				continue
			}

			switch {
			case strings.Contains(key, "id"):
				var err3 error
				err.Id, err3 = strconv.Atoi(value)
				if err3 != nil {
					err.Id = 999
				}
			case strings.Contains(key, "extra_msg"):
				err.ExtraMsg = value
			case strings.Contains(key, "msg"):
				err.Msg = value
			}
		}
	}
	return
}

func splitAtEqual(s string) (key string, value string, err bool) {
	if strings.Contains(s, "=") {
		tmp := strings.Split(s, "=")
		//fmt.Println(" -> ", s, " => ", tmp)

		key = tmp[0]
		value = tmp[1]

		err = false
	} else {
		err = true
	}
	return
}
<commit_msg>unescape error messages<commit_after>package ts3sqlib

import (
	"fmt"
	"strconv"
	"strings"
)

var (
	MsgEndError       = NewError(0, "ok", "")
	ClosedError       = NewError(-1, "connection closed", "")
	InvalidLoginError = NewError(520, "invalid loginname or password", "")
)

//Error contains additional error information.
type Error struct {
	Id       int
	Msg      string
	ExtraMsg string
}

//Error returns the error in a string representation.
func (err Error) Error() string {
	s := fmt.Sprintf("error id=%d msg=%s", err.Id, err.Msg)
	if err.ExtraMsg != "" {
		s += fmt.Sprintf(" extra_msg=%s", err.ExtraMsg)
	}
	return s
}

//NewError creates a new Error from an id, message and an extra_message.
func NewError(id int, msg, extramsg string) Error {
	return Error{id, msg, extramsg}
}

func (e Error) equals(err error) bool {
	if e2, ok := err.(Error); ok {
		return e.Id == e2.Id
	} else {
		return false
	}
}

//isError tests if a given string is a ts3 server query error
func isError(line string) bool {
	return strings.HasPrefix(line, "error") &&
		strings.Contains(line, "id=") && strings.Contains(line, "msg=")
}

//toError converts a given string into a ts3 server query error.
func toError(line string) (err Error) {
	if !isError(line) {
		err = NewError(666, "line is not an error!", "")
		return
	}

	parts := strings.Split(line, " ")
	for i := range parts {
		if strings.Contains(parts[i], "=") {
			key, value, err2 := splitAtEqual(parts[i])
			if err2 {
				continue
			}

			switch {
			case strings.Contains(key, "id"):
				var err3 error
				err.Id, err3 = strconv.Atoi(value)
				if err3 != nil {
					err.Id = 999
				}
			case strings.Contains(key, "extra_msg"):
				err.ExtraMsg = Unescape(value)
			case strings.Contains(key, "msg"):
				err.Msg = Unescape(value)
			}
		}
	}
	return
}

func splitAtEqual(s string) (key string, value string, err bool) {
	if strings.Contains(s, "=") {
		tmp := strings.Split(s, "=")
		//fmt.Println(" -> ", s, " => ", tmp)

		key = tmp[0]
		value = tmp[1]

		err = false
	} else {
		err = true
	}
	return
}
<|endoftext|>


<commit_before>package xerrors

import (
	"context"
	"errors"
	"fmt"
	"github.com/stretchr/testify/assert"
	"net/http"
	"net/http/httptest"
	"net/url"
	"testing"
	"time"
)

type subError struct {
	err error
	str string
}

func (err subError) Error() string {
	return fmt.Sprintf("%s(%s)", err.str, err.err.Error())
}

type subErrorPtr struct {
	err *error
	str string
}

func (err subErrorPtr) Error() string {
	return fmt.Sprintf("%s(%s)", err.str, (*err.err).Error())
}

func TestGetError_Int(t *testing.T) {
	assert := assert.New(t)

	assert.Equal(nil, getError(5))

}
func TestGetError_String(t *testing.T) {
	assert := assert.New(t)

	assert.Equal(nil, getError("hi"))
}
func TestGetError_Nil(t *testing.T) {
	assert := assert.New(t)

	assert.Equal(nil, getError(nil))
}

func TestGetError_Error(t *testing.T) {
	assert := assert.New(t)

	expected := errors.New("hi")
	assert.Equal(expected, getError(expected))
}
func TestGetError_ErrorPTR(t *testing.T) {
	assert := assert.New(t)

	expected := errors.New("hi")
	assert.Equal(expected, getError(&expected))
}

func TestGetError_ErrorComplex(t *testing.T) {
	assert := assert.New(t)
	type testA struct {
		error
		str string
	}
	expected := testA{errors.New("hi"), "bye"}
	assert.Equal(expected, getError(expected))
}

func TestGetError_ErrorComplexWithPointer(t *testing.T) {
	assert := assert.New(t)
	type testA struct {
		error
		str string
	}
	expected := testA{errors.New("hi"), "bye"}
	assert.Equal(expected, getError(&expected))
}

func TestGetError_NoError(t *testing.T) {
	assert := assert.New(t)
	type testA struct {
		str string
	}
	expected := testA{"bye"}
	assert.Nil(getError(&expected))
}

func TestFirstCause_CustomSubError(t *testing.T) {
	assert := assert.New(t)
	type testA struct {
		error
		str string
	}
	exectedErr := errors.New("testA")
	test := testA{exectedErr, "cool"}
	assert.Equal(exectedErr, FirstCause(subError{test, "neat"}))
}

func TestFirstCause_Nil(t *testing.T) {
	assert := assert.New(t)

	assert.Nil(FirstCause(nil))
}

func TestFirstCause_ChainSubError(t *testing.T) {
	assert := assert.New(t)

	exectedErr := errors.New("expected error")
	test := subError{
		subError{
			subError{
				subErrorPtr{&exectedErr, "cool"},
				"c",
			},
			"b",
		},
		"a",
	}
	assert.Equal(exectedErr, FirstCause(subError{test, "root"}))
}

func TestGetError_SubError(t *testing.T) {
	assert := assert.New(t)

	expected := subError{errors.New("hi"), "bye"}
	assert.Equal(expected, getError(&expected))
}

func TestFirstCause_Basic(t *testing.T) {
	assert := assert.New(t)

	err := errors.New("my bad")
	assert.Equal(err, FirstCause(err))
}

func testFirstCauseHTTPHandler(t *testing.T, serverSleep time.Duration, contextDeadline time.Duration, timeout time.Duration, useDefer bool) {
	assert := assert.New(t)

	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		time.Sleep(serverSleep)
		fmt.Fprintln(w, "Hello, client")
	}))
	defer ts.Close()

	client := http.Client{Timeout: timeout}
	req, err := http.NewRequest(http.MethodGet, ts.URL, nil)
	assert.NoError(err)

	ctx, cancel := context.WithTimeout(context.Background(), contextDeadline)
	if useDefer {
		defer cancel()
	} else {
		cancel() // cancel() is a hook to cancel the deadline
	}

	reqWithDeadline := req.WithContext(ctx)

	_, clientErr := client.Do(reqWithDeadline)

	if !assert.Error(clientErr) {
		assert.FailNow("clientErr can not be nil to continue")
	}

	xerr := FirstCause(clientErr)
	assert.Equal(clientErr.(*url.Error).Err, xerr)
}

func TestFirstCause_HTTP(t *testing.T) {

	testData := []struct {
		name            string
		serverSleep     time.Duration
		contextDeadline time.Duration
		timeout         time.Duration
		useDefer        bool
	}{
		{"client-timeout", time.Second, 5 * time.Millisecond, time.Millisecond, true},
		{"context-cancel", time.Nanosecond, 5 * time.Millisecond, time.Millisecond, false},
		{"context-timeout", time.Second, time.Millisecond, 5 * time.Millisecond, true},
	}

	for _, record := range testData {
		t.Run(fmt.Sprintf("handle/%s", record.name), func(t *testing.T) {
			testFirstCauseHTTPHandler(t, record.serverSleep, record.contextDeadline, record.timeout, record.useDefer)
		})
	}
}
<commit_msg>removing _ in test names<commit_after>package xerrors

import (
	"context"
	"errors"
	"fmt"
	"github.com/stretchr/testify/assert"
	"net/http"
	"net/http/httptest"
	"net/url"
	"testing"
	"time"
)

type subError struct {
	err error
	str string
}

func (err subError) Error() string {
	return fmt.Sprintf("%s(%s)", err.str, err.err.Error())
}

type subErrorPtr struct {
	err *error
	str string
}

func (err subErrorPtr) Error() string {
	return fmt.Sprintf("%s(%s)", err.str, (*err.err).Error())
}

func TestGetErrorInt(t *testing.T) {
	assert := assert.New(t)

	assert.Equal(nil, getError(5))

}
func TestGetErrorString(t *testing.T) {
	assert := assert.New(t)

	assert.Equal(nil, getError("hi"))
}
func TestGetErrorNil(t *testing.T) {
	assert := assert.New(t)

	assert.Equal(nil, getError(nil))
}

func TestGetErrorError(t *testing.T) {
	assert := assert.New(t)

	expected := errors.New("hi")
	assert.Equal(expected, getError(expected))
}
func TestGetErrorErrorPTR(t *testing.T) {
	assert := assert.New(t)

	expected := errors.New("hi")
	assert.Equal(expected, getError(&expected))
}

func TestGetErrorErrorComplex(t *testing.T) {
	assert := assert.New(t)
	type testA struct {
		error
		str string
	}
	expected := testA{errors.New("hi"), "bye"}
	assert.Equal(expected, getError(expected))
}

func TestGetErrorErrorComplexWithPointer(t *testing.T) {
	assert := assert.New(t)
	type testA struct {
		error
		str string
	}
	expected := testA{errors.New("hi"), "bye"}
	assert.Equal(expected, getError(&expected))
}

func TestGetErrorNoError(t *testing.T) {
	assert := assert.New(t)
	type testA struct {
		str string
	}
	expected := testA{"bye"}
	assert.Nil(getError(&expected))
}

func TestFirstCauseCustomSubError(t *testing.T) {
	assert := assert.New(t)
	type testA struct {
		error
		str string
	}
	exectedErr := errors.New("testA")
	test := testA{exectedErr, "cool"}
	assert.Equal(exectedErr, FirstCause(subError{test, "neat"}))
}

func TestFirstCauseNil(t *testing.T) {
	assert := assert.New(t)

	assert.Nil(FirstCause(nil))
}

func TestFirstCauseChainSubError(t *testing.T) {
	assert := assert.New(t)

	exectedErr := errors.New("expected error")
	test := subError{
		subError{
			subError{
				subErrorPtr{&exectedErr, "cool"},
				"c",
			},
			"b",
		},
		"a",
	}
	assert.Equal(exectedErr, FirstCause(subError{test, "root"}))
}

func TestGetErrorSubError(t *testing.T) {
	assert := assert.New(t)

	expected := subError{errors.New("hi"), "bye"}
	assert.Equal(expected, getError(&expected))
}

func TestFirstCauseBasic(t *testing.T) {
	assert := assert.New(t)

	err := errors.New("my bad")
	assert.Equal(err, FirstCause(err))
}

func testFirstCauseHTTPHandler(t *testing.T, serverSleep time.Duration, contextDeadline time.Duration, timeout time.Duration, useDefer bool) {
	assert := assert.New(t)

	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		time.Sleep(serverSleep)
		fmt.Fprintln(w, "Hello, client")
	}))
	defer ts.Close()

	client := http.Client{Timeout: timeout}
	req, err := http.NewRequest(http.MethodGet, ts.URL, nil)
	assert.NoError(err)

	ctx, cancel := context.WithTimeout(context.Background(), contextDeadline)
	if useDefer {
		defer cancel()
	} else {
		cancel() // cancel() is a hook to cancel the deadline
	}

	reqWithDeadline := req.WithContext(ctx)

	_, clientErr := client.Do(reqWithDeadline)

	if !assert.Error(clientErr) {
		assert.FailNow("clientErr can not be nil to continue")
	}

	xerr := FirstCause(clientErr)
	assert.Equal(clientErr.(*url.Error).Err, xerr)
}

func TestFirstCauseHTTP(t *testing.T) {

	testData := []struct {
		name            string
		serverSleep     time.Duration
		contextDeadline time.Duration
		timeout         time.Duration
		useDefer        bool
	}{
		{"client-timeout", time.Second, 5 * time.Millisecond, time.Millisecond, true},
		{"context-cancel", time.Nanosecond, 5 * time.Millisecond, time.Millisecond, false},
		{"context-timeout", time.Second, time.Millisecond, 5 * time.Millisecond, true},
	}

	for _, record := range testData {
		t.Run(fmt.Sprintf("handle/%s", record.name), func(t *testing.T) {
			testFirstCauseHTTPHandler(t, record.serverSleep, record.contextDeadline, record.timeout, record.useDefer)
		})
	}
}
<|endoftext|>


<commit_before><commit_msg>comparer_test: add case to Test_writeToFileMissingTags<commit_after><|endoftext|>