<commit_before>package integration_tests

import (
	. "gcp-service-broker/brokerapi/brokers"

	"golang.org/x/net/context"

	"fmt"
	"gcp-service-broker/brokerapi/brokers"
	"gcp-service-broker/brokerapi/brokers/models"
	"gcp-service-broker/brokerapi/brokers/name_generator"
	"gcp-service-broker/db_service"
	"gcp-service-broker/fakes"
	"hash/crc32"
	"net/http"
	"os"

	googlepubsub "cloud.google.com/go/pubsub"

	"encoding/json"

	googlebigtable "cloud.google.com/go/bigtable"
	googlestorage "cloud.google.com/go/storage"
	"code.cloudfoundry.org/lager"
	"github.com/jinzhu/gorm"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	googlebigquery "google.golang.org/api/bigquery/v2"
	iam "google.golang.org/api/iam/v1"
	"google.golang.org/api/option"
)

const timeout = 60

type genericService struct {
	serviceId              string
	planId                 string
	bindingId              string
	rawBindingParams       map[string]interface{}
	instanceId             string
	serviceExistsFn        func(bool) bool
	cleanupFn              func()
	serviceMetadataSavedFn func(string) bool
}

type iamService struct {
	bindingId string
	serviceId string
	planId    string
}

func getAndUnmarshalInstanceDetails(instanceId string) map[string]string {
	var instanceRecord models.ServiceInstanceDetails
	db_service.DbConnection.Find(&instanceRecord).Where("id = ?", instanceId)
	var instanceDetails map[string]string
	json.Unmarshal([]byte(instanceRecord.OtherDetails), &instanceDetails)
	return instanceDetails
}

func testGenericService(gcpBroker *GCPAsyncServiceBroker, params *genericService) {
	// If the service already exists (eg, failed previous test), clean it up before the run
	if params.serviceExistsFn != nil && params.serviceExistsFn(false) {
		params.cleanupFn()
	}
	//
	// Provision
	//
	provisionDetails := models.ProvisionDetails{
		ServiceID: params.serviceId,
		PlanID:    params.planId,
	}

	_, err := gcpBroker.Provision(params.instanceId, provisionDetails, true)
	Expect(err).ToNot(HaveOccurred())

	// Provision is registered in the database
	var count int
	db_service.DbConnection.Model(&models.ServiceInstanceDetails{}).Where("id = ?", params.instanceId).Count(&count)
	Expect(count).To(Equal(1))

	if params.serviceExistsFn != nil {
		Expect(params.serviceExistsFn(true)).To(BeTrue())
	}
	Expect(params.serviceMetadataSavedFn(params.instanceId)).To(BeTrue())

	//
	// Bind
	//
	bindDetails := models.BindDetails{
		ServiceID:  params.serviceId,
		PlanID:     params.planId,
		Parameters: params.rawBindingParams,
	}
	creds, err := gcpBroker.Bind(params.instanceId, params.bindingId, bindDetails)
	Expect(err).ToNot(HaveOccurred())

	db_service.DbConnection.Model(&models.ServiceBindingCredentials{}).Where("binding_id = ?", params.bindingId).Count(&count)
	Expect(count).To(Equal(1))

	iamService, err := iam.New(gcpBroker.GCPClient)
	Expect(err).ToNot(HaveOccurred())
	saService := iam.NewProjectsServiceAccountsService(iamService)
	resourceName := "projects/" + gcpBroker.RootGCPCredentials.ProjectId + "/serviceAccounts/" + creds.Credentials.(map[string]string)["UniqueId"]
	_, err = saService.Get(resourceName).Do()
	Expect(err).ToNot(HaveOccurred())

	//
	// Unbind
	//
	unbindDetails := models.UnbindDetails{
		ServiceID: params.serviceId,
		PlanID:    params.planId,
	}
	err = gcpBroker.Unbind(params.instanceId, params.bindingId, unbindDetails)
	Expect(err).ToNot(HaveOccurred())

	binding := models.ServiceBindingCredentials{}
	if err := db_service.DbConnection.Unscoped().Where("binding_id = ?", params.bindingId).First(&binding).Error; err != nil {
		panic("error checking for binding details: " + err.Error())
	}
	Expect(binding.DeletedAt).NotTo(Equal(nil))

	_, err = saService.Get(resourceName).Do()
	Expect(err).To(HaveOccurred())

	//
	// Deprovision

	deprovisionDetails := models.DeprovisionDetails{
		ServiceID: params.serviceId,
		PlanID:    params.planId,
	}
	_, err = gcpBroker.Deprovision(params.instanceId, deprovisionDetails, true)
	Expect(err).ToNot(HaveOccurred())
	instance := models.ServiceInstanceDetails{}
	if err := db_service.DbConnection.Unscoped().Where("ID = ?", params.instanceId).First(&instance).Error; err != nil {
		panic("error checking for service instance details: " + err.Error())
	}
	Expect(instance.DeletedAt).NotTo(Equal(nil))

	if params.serviceExistsFn != nil {
		Expect(params.serviceExistsFn(false)).To(BeFalse())
	}
}

// For services that only create a service account and bind those credentials.
func testIamBasedService(gcpBroker *GCPAsyncServiceBroker, params *iamService) {
	genericServiceParams := &genericService{
		serviceId:        params.serviceId,
		planId:           params.planId,
		instanceId:       "iam-instance",
		bindingId:        "iam-instance",
		rawBindingParams: map[string]interface{}{},
		serviceMetadataSavedFn: func(instanceId string) bool {
			// Metadata should be empty, there is no additional information required
			instanceDetails := getAndUnmarshalInstanceDetails(instanceId)
			return len(instanceDetails) == 0
		},
	}

	testGenericService(gcpBroker, genericServiceParams)
}

// Instance Name is used to name every instance created in GCP (eg, a storage bucket)
// The name should be consistent between runs to ensure there's bounds to the resources it creates
// and to have some insurance that they are properly destroyed.
//
// Why:
// - If we allow it to generate a random instance name every time the test will
//   not fail if the resource existed before hand.
// - If we always use a static one, globally named resources (eg, a storage bucket)
//   would fail to create when two different projects run these tests.
func generateInstanceName(projectId string, sep string) string {
	hashed := crc32.ChecksumIEEE([]byte(projectId))
	if sep == "" {
		sep = "_"
	}
	return fmt.Sprintf("pcf%ssb%s1%s%d", sep, sep, sep, hashed)
}

var _ = Describe("LiveIntegrationTests", func() {
	var (
		gcpBroker           *GCPAsyncServiceBroker
		err                 error
		logger              lager.Logger
		serviceNameToId     map[string]string = make(map[string]string)
		serviceNameToPlanId map[string]string = make(map[string]string)
		instance_name       string
	)

	BeforeEach(func() {
		logger = lager.NewLogger("brokers_test")
		logger.RegisterSink(lager.NewWriterSink(GinkgoWriter, lager.DEBUG))

		testDb, _ := gorm.Open("sqlite3", "test.db")
		testDb.CreateTable(models.ServiceInstanceDetails{})
		testDb.CreateTable(models.ServiceBindingCredentials{})
		testDb.CreateTable(models.PlanDetails{})
		testDb.CreateTable(models.ProvisionRequestDetails{})

		db_service.DbConnection = testDb

		os.Setenv("SECURITY_USER_NAME", "username")
		os.Setenv("SECURITY_USER_PASSWORD", "password")
		os.Setenv("SERVICES", fakes.Services)
		os.Setenv("PRECONFIGURED_PLANS", fakes.PreconfiguredPlans)

		os.Setenv("CLOUDSQL_CUSTOM_PLANS", fakes.TestCloudSQLPlan)
		os.Setenv("BIGTABLE_CUSTOM_PLANS", fakes.TestBigtablePlan)
		os.Setenv("SPANNER_CUSTOM_PLANS", fakes.TestSpannerPlan)

		var creds models.GCPCredentials
		creds, err = brokers.GetCredentialsFromEnv()
		if err != nil {
			logger.Error("error", err)
		}
		instance_name = generateInstanceName(creds.ProjectId, "")
		name_generator.Basic = &fakes.StaticNameGenerator{Val: instance_name}

		gcpBroker, err = brokers.New(logger)
		if err != nil {
			logger.Error("error", err)
		}

		for _, service := range *gcpBroker.Catalog {
			serviceNameToId[service.Name] = service.ID
			serviceNameToPlanId[service.Name] = service.Plans[0].ID
		}
	})

	Describe("Broker init", func() {
		It("should have 9 services in sevices map", func() {
			Expect(len(gcpBroker.ServiceBrokerMap)).To(Equal(9))
		})

		It("should have a default client", func() {
			Expect(gcpBroker.GCPClient).NotTo(Equal(&http.Client{}))
		})

		It("should have loaded credentials correctly and have a project id", func() {
			Expect(gcpBroker.RootGCPCredentials.ProjectId).ToNot(BeEmpty())
		})
	})

	Describe("getting broker catalog", func() {
		It("should have 9 services available", func() {
			Expect(len(gcpBroker.Services())).To(Equal(9))
		})

		It("should have 3 storage plans available", func() {
			serviceList := gcpBroker.Services()
			for _, s := range serviceList {
				if s.ID == serviceNameToId[models.StorageName] {
					Expect(len(s.Plans)).To(Equal(3))
				}
			}

		})
	})

	Describe("bigquery", func() {
		It("can provision/bind/unbind/deprovision", func() {
			service, err := googlebigquery.New(gcpBroker.GCPClient)
			Expect(err).NotTo(HaveOccurred())

			params := &genericService{
				serviceId:  serviceNameToId[models.BigqueryName],
				planId:     serviceNameToPlanId[models.BigqueryName],
				bindingId:  "integration_test_bind",
				instanceId: "integration_test_dataset",
				rawBindingParams: map[string]interface{}{
					"role": "bigquery.admin",
				},
				serviceExistsFn: func(expected bool) bool {
					_, err = service.Datasets.Get(gcpBroker.RootGCPCredentials.ProjectId, instance_name).Do()

					return err == nil
				},
				serviceMetadataSavedFn: func(instanceId string) bool {
					instanceDetails := getAndUnmarshalInstanceDetails(instanceId)
					return instanceDetails["dataset_id"] != ""
				},
				cleanupFn: func() {
					err := service.Datasets.Delete(gcpBroker.RootGCPCredentials.ProjectId, instance_name).Do()
					Expect(err).NotTo(HaveOccurred())
				},
			}
			testGenericService(gcpBroker, params)
		}, timeout)
	})

	Describe("bigtable", func() {
		var bigtableInstanceName string
		BeforeEach(func() {
			bigtableInstanceName = generateInstanceName(gcpBroker.RootGCPCredentials.ProjectId, "-")
			name_generator.Basic = &fakes.StaticNameGenerator{Val: bigtableInstanceName}
		})

		AfterEach(func() {
			name_generator.Basic = &fakes.StaticNameGenerator{Val: instance_name}
		})

		It("can provision/bind/unbind/deprovision", func() {

			ctx := context.Background()
			service, err := googlebigtable.NewInstanceAdminClient(ctx, gcpBroker.RootGCPCredentials.ProjectId)
			Expect(err).NotTo(HaveOccurred())

			params := &genericService{
				serviceId:  serviceNameToId[models.BigtableName],
				planId:     serviceNameToPlanId[models.BigtableName],
				bindingId:  "integration_test_bind",
				instanceId: "integration_test_instance",
				rawBindingParams: map[string]interface{}{
					"role": "editor",
				},
				serviceExistsFn: func(expected bool) bool {
					instances, err := service.Instances(ctx)

					return err == nil && len(instances) == 1 && instances[0].Name == bigtableInstanceName
				},
				serviceMetadataSavedFn: func(instanceId string) bool {
					instanceDetails := getAndUnmarshalInstanceDetails(instanceId)
					return instanceDetails["instance_id"] != ""
				},
				cleanupFn: func() {
					err := service.DeleteInstance(ctx, bigtableInstanceName)
					Expect(err).NotTo(HaveOccurred())
				},
			}
			testGenericService(gcpBroker, params)
		}, timeout)
	})

	Describe("cloud storage", func() {
		It("can provision/bind/unbind/deprovision", func() {
			service, err := googlestorage.NewClient(context.Background(), option.WithUserAgent(models.CustomUserAgent))
			Expect(err).NotTo(HaveOccurred())

			params := &genericService{
				serviceId:  serviceNameToId[models.StorageName],
				planId:     serviceNameToPlanId[models.StorageName],
				instanceId: "integration_test_bucket",
				bindingId:  "integration_test_bucket_binding",
				rawBindingParams: map[string]interface{}{
					"role": "storage.admin",
				},
				serviceExistsFn: func(bool) bool {
					bucket := service.Bucket(instance_name)
					_, err = bucket.Attrs(context.Background())

					return err == nil
				},
				serviceMetadataSavedFn: func(instanceId string) bool {
					instanceDetails := getAndUnmarshalInstanceDetails(instanceId)
					return instanceDetails["bucket_name"] != ""
				},
				cleanupFn: func() {
					bucket := service.Bucket(instance_name)
					bucket.Delete(context.Background())
				},
			}

			testGenericService(gcpBroker, params)
		}, timeout)
	})

	Describe("pub sub", func() {
		It("can provision/bind/unbind/deprovision", func() {
			service, err := googlepubsub.NewClient(context.Background(), gcpBroker.RootGCPCredentials.ProjectId, option.WithUserAgent(models.CustomUserAgent))
			Expect(err).NotTo(HaveOccurred())

			topic := service.Topic(instance_name)

			params := &genericService{
				serviceId:  serviceNameToId[models.PubsubName],
				planId:     serviceNameToPlanId[models.PubsubName],
				instanceId: "integration_test_topic",
				bindingId:  "integration_test_topic_bindingId",
				rawBindingParams: map[string]interface{}{
					"role": "pubsub.admin",
				},
				serviceExistsFn: func(bool) bool {
					exists, err := topic.Exists(context.Background())
					return exists && err == nil
				},
				serviceMetadataSavedFn: func(instanceId string) bool {
					instanceDetails := getAndUnmarshalInstanceDetails(instanceId)
					return instanceDetails["topic_name"] != ""
				},
				cleanupFn: func() {
					err := topic.Delete(context.Background())
					Expect(err).NotTo(HaveOccurred())
				},
			}

			testGenericService(gcpBroker, params)
		}, timeout)
	})

	Describe("stadkdriver debugger", func() {
		It("can provision/bind/unbind/deprovision", func() {
			params := &iamService{
				serviceId: serviceNameToId[models.StackdriverDebuggerName],
				planId:    serviceNameToPlanId[models.StackdriverDebuggerName],
			}
			testIamBasedService(gcpBroker, params)
		}, timeout)
	})

	AfterEach(func() {
		os.Remove(models.AppCredsFileName)
		os.Remove("test.db")
	})
})
<commit_msg>integration test exercise for trace<commit_after>package integration_tests

import (
	. "gcp-service-broker/brokerapi/brokers"

	"golang.org/x/net/context"

	"fmt"
	"gcp-service-broker/brokerapi/brokers"
	"gcp-service-broker/brokerapi/brokers/models"
	"gcp-service-broker/brokerapi/brokers/name_generator"
	"gcp-service-broker/db_service"
	"gcp-service-broker/fakes"
	"hash/crc32"
	"net/http"
	"os"

	googlepubsub "cloud.google.com/go/pubsub"

	"encoding/json"

	googlebigtable "cloud.google.com/go/bigtable"
	googlestorage "cloud.google.com/go/storage"
	"code.cloudfoundry.org/lager"
	"github.com/jinzhu/gorm"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	googlebigquery "google.golang.org/api/bigquery/v2"
	iam "google.golang.org/api/iam/v1"
	"google.golang.org/api/option"
)

const timeout = 60

type genericService struct {
	serviceId              string
	planId                 string
	bindingId              string
	rawBindingParams       map[string]interface{}
	instanceId             string
	serviceExistsFn        func(bool) bool
	cleanupFn              func()
	serviceMetadataSavedFn func(string) bool
}

type iamService struct {
	bindingId string
	serviceId string
	planId    string
}

func getAndUnmarshalInstanceDetails(instanceId string) map[string]string {
	var instanceRecord models.ServiceInstanceDetails
	db_service.DbConnection.Find(&instanceRecord).Where("id = ?", instanceId)
	var instanceDetails map[string]string
	json.Unmarshal([]byte(instanceRecord.OtherDetails), &instanceDetails)
	return instanceDetails
}

func testGenericService(gcpBroker *GCPAsyncServiceBroker, params *genericService) {
	// If the service already exists (eg, failed previous test), clean it up before the run
	if params.serviceExistsFn != nil && params.serviceExistsFn(false) {
		params.cleanupFn()
	}
	//
	// Provision
	//
	provisionDetails := models.ProvisionDetails{
		ServiceID: params.serviceId,
		PlanID:    params.planId,
	}

	_, err := gcpBroker.Provision(params.instanceId, provisionDetails, true)
	Expect(err).ToNot(HaveOccurred())

	// Provision is registered in the database
	var count int
	db_service.DbConnection.Model(&models.ServiceInstanceDetails{}).Where("id = ?", params.instanceId).Count(&count)
	Expect(count).To(Equal(1))

	if params.serviceExistsFn != nil {
		Expect(params.serviceExistsFn(true)).To(BeTrue())
	}
	Expect(params.serviceMetadataSavedFn(params.instanceId)).To(BeTrue())

	//
	// Bind
	//
	bindDetails := models.BindDetails{
		ServiceID:  params.serviceId,
		PlanID:     params.planId,
		Parameters: params.rawBindingParams,
	}
	creds, err := gcpBroker.Bind(params.instanceId, params.bindingId, bindDetails)
	Expect(err).ToNot(HaveOccurred())

	db_service.DbConnection.Model(&models.ServiceBindingCredentials{}).Where("binding_id = ?", params.bindingId).Count(&count)
	Expect(count).To(Equal(1))

	iamService, err := iam.New(gcpBroker.GCPClient)
	Expect(err).ToNot(HaveOccurred())
	saService := iam.NewProjectsServiceAccountsService(iamService)
	resourceName := "projects/" + gcpBroker.RootGCPCredentials.ProjectId + "/serviceAccounts/" + creds.Credentials.(map[string]string)["UniqueId"]
	_, err = saService.Get(resourceName).Do()
	Expect(err).ToNot(HaveOccurred())

	//
	// Unbind
	//
	unbindDetails := models.UnbindDetails{
		ServiceID: params.serviceId,
		PlanID:    params.planId,
	}
	err = gcpBroker.Unbind(params.instanceId, params.bindingId, unbindDetails)
	Expect(err).ToNot(HaveOccurred())

	binding := models.ServiceBindingCredentials{}
	if err := db_service.DbConnection.Unscoped().Where("binding_id = ?", params.bindingId).First(&binding).Error; err != nil {
		panic("error checking for binding details: " + err.Error())
	}
	Expect(binding.DeletedAt).NotTo(Equal(nil))

	_, err = saService.Get(resourceName).Do()
	Expect(err).To(HaveOccurred())

	//
	// Deprovision

	deprovisionDetails := models.DeprovisionDetails{
		ServiceID: params.serviceId,
		PlanID:    params.planId,
	}
	_, err = gcpBroker.Deprovision(params.instanceId, deprovisionDetails, true)
	Expect(err).ToNot(HaveOccurred())
	instance := models.ServiceInstanceDetails{}
	if err := db_service.DbConnection.Unscoped().Where("ID = ?", params.instanceId).First(&instance).Error; err != nil {
		panic("error checking for service instance details: " + err.Error())
	}
	Expect(instance.DeletedAt).NotTo(Equal(nil))

	if params.serviceExistsFn != nil {
		Expect(params.serviceExistsFn(false)).To(BeFalse())
	}
}

// For services that only create a service account and bind those credentials.
func testIamBasedService(gcpBroker *GCPAsyncServiceBroker, params *iamService) {
	genericServiceParams := &genericService{
		serviceId:        params.serviceId,
		planId:           params.planId,
		instanceId:       "iam-instance",
		bindingId:        "iam-instance",
		rawBindingParams: map[string]interface{}{},
		serviceMetadataSavedFn: func(instanceId string) bool {
			// Metadata should be empty, there is no additional information required
			instanceDetails := getAndUnmarshalInstanceDetails(instanceId)
			return len(instanceDetails) == 0
		},
	}

	testGenericService(gcpBroker, genericServiceParams)
}

// Instance Name is used to name every instance created in GCP (eg, a storage bucket)
// The name should be consistent between runs to ensure there's bounds to the resources it creates
// and to have some insurance that they are properly destroyed.
//
// Why:
// - If we allow it to generate a random instance name every time the test will
//   not fail if the resource existed before hand.
// - If we always use a static one, globally named resources (eg, a storage bucket)
//   would fail to create when two different projects run these tests.
func generateInstanceName(projectId string, sep string) string {
	hashed := crc32.ChecksumIEEE([]byte(projectId))
	if sep == "" {
		sep = "_"
	}
	return fmt.Sprintf("pcf%ssb%s1%s%d", sep, sep, sep, hashed)
}

var _ = Describe("LiveIntegrationTests", func() {
	var (
		gcpBroker           *GCPAsyncServiceBroker
		err                 error
		logger              lager.Logger
		serviceNameToId     map[string]string = make(map[string]string)
		serviceNameToPlanId map[string]string = make(map[string]string)
		instance_name       string
	)

	BeforeEach(func() {
		logger = lager.NewLogger("brokers_test")
		logger.RegisterSink(lager.NewWriterSink(GinkgoWriter, lager.DEBUG))

		testDb, _ := gorm.Open("sqlite3", "test.db")
		testDb.CreateTable(models.ServiceInstanceDetails{})
		testDb.CreateTable(models.ServiceBindingCredentials{})
		testDb.CreateTable(models.PlanDetails{})
		testDb.CreateTable(models.ProvisionRequestDetails{})

		db_service.DbConnection = testDb

		os.Setenv("SECURITY_USER_NAME", "username")
		os.Setenv("SECURITY_USER_PASSWORD", "password")
		os.Setenv("SERVICES", fakes.Services)
		os.Setenv("PRECONFIGURED_PLANS", fakes.PreconfiguredPlans)

		os.Setenv("CLOUDSQL_CUSTOM_PLANS", fakes.TestCloudSQLPlan)
		os.Setenv("BIGTABLE_CUSTOM_PLANS", fakes.TestBigtablePlan)
		os.Setenv("SPANNER_CUSTOM_PLANS", fakes.TestSpannerPlan)

		var creds models.GCPCredentials
		creds, err = brokers.GetCredentialsFromEnv()
		if err != nil {
			logger.Error("error", err)
		}
		instance_name = generateInstanceName(creds.ProjectId, "")
		name_generator.Basic = &fakes.StaticNameGenerator{Val: instance_name}

		gcpBroker, err = brokers.New(logger)
		if err != nil {
			logger.Error("error", err)
		}

		for _, service := range *gcpBroker.Catalog {
			serviceNameToId[service.Name] = service.ID
			serviceNameToPlanId[service.Name] = service.Plans[0].ID
		}
	})

	Describe("Broker init", func() {
		It("should have 9 services in sevices map", func() {
			Expect(len(gcpBroker.ServiceBrokerMap)).To(Equal(9))
		})

		It("should have a default client", func() {
			Expect(gcpBroker.GCPClient).NotTo(Equal(&http.Client{}))
		})

		It("should have loaded credentials correctly and have a project id", func() {
			Expect(gcpBroker.RootGCPCredentials.ProjectId).ToNot(BeEmpty())
		})
	})

	Describe("getting broker catalog", func() {
		It("should have 9 services available", func() {
			Expect(len(gcpBroker.Services())).To(Equal(9))
		})

		It("should have 3 storage plans available", func() {
			serviceList := gcpBroker.Services()
			for _, s := range serviceList {
				if s.ID == serviceNameToId[models.StorageName] {
					Expect(len(s.Plans)).To(Equal(3))
				}
			}

		})
	})

	Describe("bigquery", func() {
		It("can provision/bind/unbind/deprovision", func() {
			service, err := googlebigquery.New(gcpBroker.GCPClient)
			Expect(err).NotTo(HaveOccurred())

			params := &genericService{
				serviceId:  serviceNameToId[models.BigqueryName],
				planId:     serviceNameToPlanId[models.BigqueryName],
				bindingId:  "integration_test_bind",
				instanceId: "integration_test_dataset",
				rawBindingParams: map[string]interface{}{
					"role": "bigquery.admin",
				},
				serviceExistsFn: func(expected bool) bool {
					_, err = service.Datasets.Get(gcpBroker.RootGCPCredentials.ProjectId, instance_name).Do()

					return err == nil
				},
				serviceMetadataSavedFn: func(instanceId string) bool {
					instanceDetails := getAndUnmarshalInstanceDetails(instanceId)
					return instanceDetails["dataset_id"] != ""
				},
				cleanupFn: func() {
					err := service.Datasets.Delete(gcpBroker.RootGCPCredentials.ProjectId, instance_name).Do()
					Expect(err).NotTo(HaveOccurred())
				},
			}
			testGenericService(gcpBroker, params)
		}, timeout)
	})

	Describe("bigtable", func() {
		var bigtableInstanceName string
		BeforeEach(func() {
			bigtableInstanceName = generateInstanceName(gcpBroker.RootGCPCredentials.ProjectId, "-")
			name_generator.Basic = &fakes.StaticNameGenerator{Val: bigtableInstanceName}
		})

		AfterEach(func() {
			name_generator.Basic = &fakes.StaticNameGenerator{Val: instance_name}
		})

		It("can provision/bind/unbind/deprovision", func() {

			ctx := context.Background()
			service, err := googlebigtable.NewInstanceAdminClient(ctx, gcpBroker.RootGCPCredentials.ProjectId)
			Expect(err).NotTo(HaveOccurred())

			params := &genericService{
				serviceId:  serviceNameToId[models.BigtableName],
				planId:     serviceNameToPlanId[models.BigtableName],
				bindingId:  "integration_test_bind",
				instanceId: "integration_test_instance",
				rawBindingParams: map[string]interface{}{
					"role": "editor",
				},
				serviceExistsFn: func(expected bool) bool {
					instances, err := service.Instances(ctx)

					return err == nil && len(instances) == 1 && instances[0].Name == bigtableInstanceName
				},
				serviceMetadataSavedFn: func(instanceId string) bool {
					instanceDetails := getAndUnmarshalInstanceDetails(instanceId)
					return instanceDetails["instance_id"] != ""
				},
				cleanupFn: func() {
					err := service.DeleteInstance(ctx, bigtableInstanceName)
					Expect(err).NotTo(HaveOccurred())
				},
			}
			testGenericService(gcpBroker, params)
		}, timeout)
	})

	Describe("cloud storage", func() {
		It("can provision/bind/unbind/deprovision", func() {
			service, err := googlestorage.NewClient(context.Background(), option.WithUserAgent(models.CustomUserAgent))
			Expect(err).NotTo(HaveOccurred())

			params := &genericService{
				serviceId:  serviceNameToId[models.StorageName],
				planId:     serviceNameToPlanId[models.StorageName],
				instanceId: "integration_test_bucket",
				bindingId:  "integration_test_bucket_binding",
				rawBindingParams: map[string]interface{}{
					"role": "storage.admin",
				},
				serviceExistsFn: func(bool) bool {
					bucket := service.Bucket(instance_name)
					_, err = bucket.Attrs(context.Background())

					return err == nil
				},
				serviceMetadataSavedFn: func(instanceId string) bool {
					instanceDetails := getAndUnmarshalInstanceDetails(instanceId)
					return instanceDetails["bucket_name"] != ""
				},
				cleanupFn: func() {
					bucket := service.Bucket(instance_name)
					bucket.Delete(context.Background())
				},
			}

			testGenericService(gcpBroker, params)
		}, timeout)
	})

	Describe("pub sub", func() {
		It("can provision/bind/unbind/deprovision", func() {
			service, err := googlepubsub.NewClient(context.Background(), gcpBroker.RootGCPCredentials.ProjectId, option.WithUserAgent(models.CustomUserAgent))
			Expect(err).NotTo(HaveOccurred())

			topic := service.Topic(instance_name)

			params := &genericService{
				serviceId:  serviceNameToId[models.PubsubName],
				planId:     serviceNameToPlanId[models.PubsubName],
				instanceId: "integration_test_topic",
				bindingId:  "integration_test_topic_bindingId",
				rawBindingParams: map[string]interface{}{
					"role": "pubsub.admin",
				},
				serviceExistsFn: func(bool) bool {
					exists, err := topic.Exists(context.Background())
					return exists && err == nil
				},
				serviceMetadataSavedFn: func(instanceId string) bool {
					instanceDetails := getAndUnmarshalInstanceDetails(instanceId)
					return instanceDetails["topic_name"] != ""
				},
				cleanupFn: func() {
					err := topic.Delete(context.Background())
					Expect(err).NotTo(HaveOccurred())
				},
			}

			testGenericService(gcpBroker, params)
		}, timeout)
	})

	Describe("stackdriver debugger", func() {
		It("can provision/bind/unbind/deprovision", func() {
			params := &iamService{
				serviceId: serviceNameToId[models.StackdriverDebuggerName],
				planId:    serviceNameToPlanId[models.StackdriverDebuggerName],
			}
			testIamBasedService(gcpBroker, params)
		}, timeout)
	})

	Describe("stackdriver trace", func() {
		It("can provision/bind/unbind/deprovision", func() {
			params := &iamService{
				serviceId: serviceNameToId[models.StackdriverTraceName],
				planId:    serviceNameToPlanId[models.StackdriverTraceName],
			}
			testIamBasedService(gcpBroker, params)
		}, timeout)
	})

	AfterEach(func() {
		os.Remove(models.AppCredsFileName)
		os.Remove("test.db")
	})
})
<|endoftext|>


<commit_before>package quicproxy

import (
	"net"
	"sync"
	"sync/atomic"
	"time"

	"github.com/lucas-clemente/quic-go/internal/protocol"
)

// Connection is a UDP connection
type connection struct {
	ClientAddr *net.UDPAddr // Address of the client
	ServerConn *net.UDPConn // UDP connection to server

	incomingPacketCounter uint64
	outgoingPacketCounter uint64
}

// Direction is the direction a packet is sent.
type Direction int

const (
	// DirectionIncoming is the direction from the client to the server.
	DirectionIncoming Direction = iota
	// DirectionOutgoing is the direction from the server to the client.
	DirectionOutgoing
	// DirectionBoth is both incoming and outgoing
	DirectionBoth
)

func (d Direction) String() string {
	switch d {
	case DirectionIncoming:
		return "incoming"
	case DirectionOutgoing:
		return "outgoing"
	case DirectionBoth:
		return "both"
	default:
		panic("unknown direction")
	}
}

func (d Direction) Is(dir Direction) bool {
	if d == DirectionBoth || dir == DirectionBoth {
		return true
	}
	return d == dir
}

// DropCallback is a callback that determines which packet gets dropped.
type DropCallback func(dir Direction, packetCount uint64) bool

// NoDropper doesn't drop packets.
var NoDropper DropCallback = func(Direction, uint64) bool {
	return false
}

// DelayCallback is a callback that determines how much delay to apply to a packet.
type DelayCallback func(dir Direction, packetCount uint64) time.Duration

// NoDelay doesn't apply a delay.
var NoDelay DelayCallback = func(Direction, uint64) time.Duration {
	return 0
}

// Opts are proxy options.
type Opts struct {
	// The address this proxy proxies packets to.
	RemoteAddr string
	// DropPacket determines whether a packet gets dropped.
	DropPacket DropCallback
	// DelayPacket determines how long a packet gets delayed. This allows
	// simulating a connection with non-zero RTTs.
	// Note that the RTT is the sum of the delay for the incoming and the outgoing packet.
	DelayPacket DelayCallback
}

// QuicProxy is a QUIC proxy that can drop and delay packets.
type QuicProxy struct {
	mutex sync.Mutex

	version protocol.VersionNumber

	conn       *net.UDPConn
	serverAddr *net.UDPAddr

	dropPacket  DropCallback
	delayPacket DelayCallback

	// Mapping from client addresses (as host:port) to connection
	clientDict map[string]*connection
}

// NewQuicProxy creates a new UDP proxy
func NewQuicProxy(local string, version protocol.VersionNumber, opts *Opts) (*QuicProxy, error) {
	if opts == nil {
		opts = &Opts{}
	}
	laddr, err := net.ResolveUDPAddr("udp", local)
	if err != nil {
		return nil, err
	}
	conn, err := net.ListenUDP("udp", laddr)
	if err != nil {
		return nil, err
	}
	raddr, err := net.ResolveUDPAddr("udp", opts.RemoteAddr)
	if err != nil {
		return nil, err
	}

	packetDropper := NoDropper
	if opts.DropPacket != nil {
		packetDropper = opts.DropPacket
	}

	packetDelayer := NoDelay
	if opts.DelayPacket != nil {
		packetDelayer = opts.DelayPacket
	}

	p := QuicProxy{
		clientDict:  make(map[string]*connection),
		conn:        conn,
		serverAddr:  raddr,
		dropPacket:  packetDropper,
		delayPacket: packetDelayer,
		version:     version,
	}

	go p.runProxy()
	return &p, nil
}

// Close stops the UDP Proxy
func (p *QuicProxy) Close() error {
	return p.conn.Close()
}

// LocalAddr is the address the proxy is listening on.
func (p *QuicProxy) LocalAddr() net.Addr {
	return p.conn.LocalAddr()
}

// LocalPort is the UDP port number the proxy is listening on.
func (p *QuicProxy) LocalPort() int {
	return p.conn.LocalAddr().(*net.UDPAddr).Port
}

func (p *QuicProxy) newConnection(cliAddr *net.UDPAddr) (*connection, error) {
	srvudp, err := net.DialUDP("udp", nil, p.serverAddr)
	if err != nil {
		return nil, err
	}
	return &connection{
		ClientAddr: cliAddr,
		ServerConn: srvudp,
	}, nil
}

// runProxy listens on the proxy address and handles incoming packets.
func (p *QuicProxy) runProxy() error {
	for {
		buffer := make([]byte, protocol.MaxReceivePacketSize)
		n, cliaddr, err := p.conn.ReadFromUDP(buffer)
		if err != nil {
			return err
		}
		raw := buffer[0:n]

		saddr := cliaddr.String()
		p.mutex.Lock()
		conn, ok := p.clientDict[saddr]

		if !ok {
			conn, err = p.newConnection(cliaddr)
			if err != nil {
				p.mutex.Unlock()
				return err
			}
			p.clientDict[saddr] = conn
			go p.runConnection(conn)
		}
		p.mutex.Unlock()

		packetCount := atomic.AddUint64(&conn.incomingPacketCounter, 1)

		if p.dropPacket(DirectionIncoming, packetCount) {
			continue
		}

		// Send the packet to the server
		delay := p.delayPacket(DirectionIncoming, packetCount)
		if delay != 0 {
			time.AfterFunc(delay, func() {
				// TODO: handle error
				_, _ = conn.ServerConn.Write(raw)
			})
		} else {
			_, err := conn.ServerConn.Write(raw)
			if err != nil {
				return err
			}
		}
	}
}

// runConnection handles packets from server to a single client
func (p *QuicProxy) runConnection(conn *connection) error {
	for {
		buffer := make([]byte, protocol.MaxReceivePacketSize)
		n, err := conn.ServerConn.Read(buffer)
		if err != nil {
			return err
		}
		raw := buffer[0:n]

		packetCount := atomic.AddUint64(&conn.outgoingPacketCounter, 1)

		if p.dropPacket(DirectionOutgoing, packetCount) {
			continue
		}

		delay := p.delayPacket(DirectionOutgoing, packetCount)
		if delay != 0 {
			time.AfterFunc(delay, func() {
				// TODO: handle error
				_, _ = p.conn.WriteToUDP(raw, conn.ClientAddr)
			})
		} else {
			_, err := p.conn.WriteToUDP(raw, conn.ClientAddr)
			if err != nil {
				return err
			}
		}
	}
}
<commit_msg>add some debug logging for the proxy<commit_after>package quicproxy

import (
	"net"
	"sync"
	"sync/atomic"
	"time"

	"github.com/lucas-clemente/quic-go/internal/protocol"
	"github.com/lucas-clemente/quic-go/internal/utils"
)

// Connection is a UDP connection
type connection struct {
	ClientAddr *net.UDPAddr // Address of the client
	ServerConn *net.UDPConn // UDP connection to server

	incomingPacketCounter uint64
	outgoingPacketCounter uint64
}

// Direction is the direction a packet is sent.
type Direction int

const (
	// DirectionIncoming is the direction from the client to the server.
	DirectionIncoming Direction = iota
	// DirectionOutgoing is the direction from the server to the client.
	DirectionOutgoing
	// DirectionBoth is both incoming and outgoing
	DirectionBoth
)

func (d Direction) String() string {
	switch d {
	case DirectionIncoming:
		return "incoming"
	case DirectionOutgoing:
		return "outgoing"
	case DirectionBoth:
		return "both"
	default:
		panic("unknown direction")
	}
}

func (d Direction) Is(dir Direction) bool {
	if d == DirectionBoth || dir == DirectionBoth {
		return true
	}
	return d == dir
}

// DropCallback is a callback that determines which packet gets dropped.
type DropCallback func(dir Direction, packetCount uint64) bool

// NoDropper doesn't drop packets.
var NoDropper DropCallback = func(Direction, uint64) bool {
	return false
}

// DelayCallback is a callback that determines how much delay to apply to a packet.
type DelayCallback func(dir Direction, packetCount uint64) time.Duration

// NoDelay doesn't apply a delay.
var NoDelay DelayCallback = func(Direction, uint64) time.Duration {
	return 0
}

// Opts are proxy options.
type Opts struct {
	// The address this proxy proxies packets to.
	RemoteAddr string
	// DropPacket determines whether a packet gets dropped.
	DropPacket DropCallback
	// DelayPacket determines how long a packet gets delayed. This allows
	// simulating a connection with non-zero RTTs.
	// Note that the RTT is the sum of the delay for the incoming and the outgoing packet.
	DelayPacket DelayCallback
}

// QuicProxy is a QUIC proxy that can drop and delay packets.
type QuicProxy struct {
	mutex sync.Mutex

	version protocol.VersionNumber

	conn       *net.UDPConn
	serverAddr *net.UDPAddr

	dropPacket  DropCallback
	delayPacket DelayCallback

	// Mapping from client addresses (as host:port) to connection
	clientDict map[string]*connection
}

// NewQuicProxy creates a new UDP proxy
func NewQuicProxy(local string, version protocol.VersionNumber, opts *Opts) (*QuicProxy, error) {
	if opts == nil {
		opts = &Opts{}
	}
	laddr, err := net.ResolveUDPAddr("udp", local)
	if err != nil {
		return nil, err
	}
	conn, err := net.ListenUDP("udp", laddr)
	if err != nil {
		return nil, err
	}
	raddr, err := net.ResolveUDPAddr("udp", opts.RemoteAddr)
	if err != nil {
		return nil, err
	}

	packetDropper := NoDropper
	if opts.DropPacket != nil {
		packetDropper = opts.DropPacket
	}

	packetDelayer := NoDelay
	if opts.DelayPacket != nil {
		packetDelayer = opts.DelayPacket
	}

	p := QuicProxy{
		clientDict:  make(map[string]*connection),
		conn:        conn,
		serverAddr:  raddr,
		dropPacket:  packetDropper,
		delayPacket: packetDelayer,
		version:     version,
	}

	utils.Debugf("Starting UDP Proxy %s <-> %s", conn.LocalAddr(), raddr)
	go p.runProxy()
	return &p, nil
}

// Close stops the UDP Proxy
func (p *QuicProxy) Close() error {
	return p.conn.Close()
}

// LocalAddr is the address the proxy is listening on.
func (p *QuicProxy) LocalAddr() net.Addr {
	return p.conn.LocalAddr()
}

// LocalPort is the UDP port number the proxy is listening on.
func (p *QuicProxy) LocalPort() int {
	return p.conn.LocalAddr().(*net.UDPAddr).Port
}

func (p *QuicProxy) newConnection(cliAddr *net.UDPAddr) (*connection, error) {
	srvudp, err := net.DialUDP("udp", nil, p.serverAddr)
	if err != nil {
		return nil, err
	}
	return &connection{
		ClientAddr: cliAddr,
		ServerConn: srvudp,
	}, nil
}

// runProxy listens on the proxy address and handles incoming packets.
func (p *QuicProxy) runProxy() error {
	for {
		buffer := make([]byte, protocol.MaxReceivePacketSize)
		n, cliaddr, err := p.conn.ReadFromUDP(buffer)
		if err != nil {
			return err
		}
		raw := buffer[0:n]

		saddr := cliaddr.String()
		p.mutex.Lock()
		conn, ok := p.clientDict[saddr]

		if !ok {
			conn, err = p.newConnection(cliaddr)
			if err != nil {
				p.mutex.Unlock()
				return err
			}
			p.clientDict[saddr] = conn
			go p.runConnection(conn)
		}
		p.mutex.Unlock()

		packetCount := atomic.AddUint64(&conn.incomingPacketCounter, 1)

		if p.dropPacket(DirectionIncoming, packetCount) {
			if utils.Debug() {
				utils.Debugf("dropping incoming packet %d (%d bytes)", packetCount, n)
			}
			continue
		}

		// Send the packet to the server
		delay := p.delayPacket(DirectionIncoming, packetCount)
		if delay != 0 {
			if utils.Debug() {
				utils.Debugf("delaying incoming packet %d (%d bytes) to %s by %s", packetCount, n, conn.ServerConn.RemoteAddr(), delay)
			}
			time.AfterFunc(delay, func() {
				// TODO: handle error
				_, _ = conn.ServerConn.Write(raw)
			})
		} else {
			if utils.Debug() {
				utils.Debugf("forwarding incoming packet %d (%d bytes) to %s", packetCount, n, conn.ServerConn.RemoteAddr())
			}
			if _, err := conn.ServerConn.Write(raw); err != nil {
				return err
			}
		}
	}
}

// runConnection handles packets from server to a single client
func (p *QuicProxy) runConnection(conn *connection) error {
	for {
		buffer := make([]byte, protocol.MaxReceivePacketSize)
		n, err := conn.ServerConn.Read(buffer)
		if err != nil {
			return err
		}
		raw := buffer[0:n]

		packetCount := atomic.AddUint64(&conn.outgoingPacketCounter, 1)

		if p.dropPacket(DirectionOutgoing, packetCount) {
			if utils.Debug() {
				utils.Debugf("dropping outgoing packet %d (%d bytes)", packetCount, n)
			}
			continue
		}
		utils.Debugf("forwarding outgoing packet %d (%d bytes) to %s", packetCount, n, conn.ClientAddr)

		delay := p.delayPacket(DirectionOutgoing, packetCount)
		if delay != 0 {
			if utils.Debug() {
				utils.Debugf("delaying outgoing packet %d (%d bytes) to %s by %s", packetCount, n, conn.ClientAddr, delay)
			}
			time.AfterFunc(delay, func() {
				// TODO: handle error
				_, _ = p.conn.WriteToUDP(raw, conn.ClientAddr)
			})
		} else {
			if utils.Debug() {
				utils.Debugf("forwarding outgoing packet %d (%d bytes) to %s", packetCount, n, conn.ClientAddr)
			}
			if _, err := p.conn.WriteToUDP(raw, conn.ClientAddr); err != nil {
				return err
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2020 Marc-Antoine Ruel. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

package internaltest

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"
)

// PanicwebOutput returns the output of panicweb with inlining disabled.
//
// The function panics if any internal error occurs.
func PanicwebOutput() []byte {
	panicwebOnce.Do(func() {
		p := build("panicweb", false)
		if p == "" {
			panic("building panicweb failed")
		}
		defer func() {
			if err := os.Remove(p); err != nil {
				panic(err)
			}
		}()
		panicwebOutput = execRun(p)
	})
	out := make([]byte, len(panicwebOutput))
	copy(out, panicwebOutput)
	return out
}

// PanicOutputs returns a map of the output of every subcommands.
//
// panic is built with inlining disabled.
//
// The subcommand "race" is built with the race detector. Others are built
// without. In particular "asleep" doesn't work with the race detector.
//
// The function panics if any internal error occurs.
func PanicOutputs() map[string][]byte {
	panicOutputsOnce.Do(func() {
		// Extracts the subcommands, then run each of them individually.
		pplain := build("panic", false)
		if pplain == "" {
			// The odd of this failing is close to nil.
			panic("building panic failed")
		}
		defer func() {
			if err := os.Remove(pplain); err != nil {
				panic(err)
			}
		}()

		prace := build("panic", true)
		if prace == "" {
			// Race detector is not supported on this platform.
		} else {
			defer func() {
				if err := os.Remove(prace); err != nil {
					panic(err)
				}
			}()
		}

		// Collect the subcommands.
		cmds := strings.Split(strings.TrimSpace(string(execRun(pplain, "dump_commands"))), "\n")
		if len(cmds) == 0 {
			panic("no command retrieved")
		}

		// Collect the output of each subcommand.
		panicOutputs = map[string][]byte{}
		for _, cmd := range cmds {
			cmd = strings.TrimSpace(cmd)
			p := pplain
			if cmd == "race" {
				if prace == "" {
					// Race detector is not supported.
					continue
				}
				p = prace
			}
			if panicOutputs[cmd] = execRun(p, cmd); len(panicOutputs[cmd]) == 0 {
				panic(fmt.Sprintf("no output for %s", cmd))
			}
		}
	})
	out := make(map[string][]byte, len(panicOutputs))
	for k, v := range panicOutputs {
		w := make([]byte, len(v))
		copy(w, v)
		out[k] = w
	}
	return out
}

// StaticPanicwebOutput returns a constant version of panicweb output for use
// in benchmarks.
func StaticPanicwebOutput() []byte {
	return []byte(staticPanicweb)
}

// StaticPanicRaceOutput returns a constant version of 'panic race' output.
func StaticPanicRaceOutput() []byte {
	return []byte(staticPanicRace)
}

// IsUsingModules is best guess to know if go module are enabled.
//
// Panics if an internal error occurs.
//
// It reads the current value of GO111MODULES.
func IsUsingModules() bool {
	// Calculate the default. We assume developer builds are recent (go1.14 and
	// later).
	ver := GetGoMinorVersion()
	if ver > 0 && ver < 11 {
		// go1.9.7+ and go1.10.3+ were fixed to tolerate semantic versioning import
		// but they do not support the environment variable.
		return false
	}
	def := (ver == 0 || ver >= 14)
	s := os.Getenv("GO111MODULE")
	return (def && (s == "auto" || s == "")) || s == "on"
}

//

var (
	panicwebOnce     sync.Once
	panicwebOutput   []byte
	panicOutputsOnce sync.Once
	panicOutputs     map[string][]byte
)

// GetGoMinorVersion returns the Go1 minor version.
//
// Returns 0 for a developer build, panics if can't parse the version.
//
// Ignores the revision (go1.<minor>.<revision>).
func GetGoMinorVersion() int {
	ver := runtime.Version()
	if strings.HasPrefix(ver, "devel +") {
		return 0
	}
	if !strings.HasPrefix(ver, "go1.") {
		// This will break on go2. Please submit a PR to fix this once Go2 is
		// released.
		panic(fmt.Sprintf("unexpected go version %q", ver))
	}
	v := ver[4:]
	if i := strings.IndexByte(v, '.'); i != -1 {
		v = v[:i]
	} else if i := strings.Index(v, "beta"); i != -1 {
		v = v[:i]
	} else if i := strings.Index(v, "rc"); i != -1 {
		v = v[:i]
	}

	m, err := strconv.Atoi(v)
	if err != nil {
		panic(fmt.Sprintf("failed to parse %q: %v", ver, err))
	}
	return m
}

// build creates a temporary file and returns the path to it.
func build(tool string, race bool) string {
	p := filepath.Join(os.TempDir(), tool)
	if race {
		p += "_race"
	}
	// Starting with go1.11, ioutil.TempFile() supports specifying a suffix. This
	// is necessary to set the ".exe" suffix on Windows. Until we drop support
	// for go1.10 and earlier, do the equivalent ourselves in an lousy way.
	p += fmt.Sprintf("_%d", os.Getpid())
	if runtime.GOOS == "windows" {
		p += ".exe"
	}
	path := "github.com/maruel/panicparse/cmd/"
	if IsUsingModules() {
		path = "github.com/maruel/panicparse/v2/cmd/"
	}
	if err := Compile(path+tool, p, "", true, race); err != nil {
		_, _ = os.Stderr.WriteString(err.Error())
		return ""
	}
	return p
}

var errNoRace = errors.New("platform does not support -race")

// Compile compiles sources into an executable.
func Compile(in, exe, cwd string, disableInlining, race bool) error {
	// Disable inlining otherwise the inlining varies between local execution and
	// remote execution. This can be observed as Elided being true without any
	// argument.
	args := []string{"build", "-o", exe}
	if disableInlining {
		args = append(args, "-gcflags", "-l")
	}
	if race {
		args = append(args, "-race")
	}
	c := exec.Command("go", append(args, in)...)
	c.Dir = cwd
	if out, err := c.CombinedOutput(); err != nil {
		if race && strings.HasPrefix(string(out), "go test: -race is only supported on ") {
			return errNoRace
		}
		return fmt.Errorf("compile failure: "+wrap+"\n%s", err, out)
	}
	return nil
}

// execRun runs a command and returns the combined output.
//
// It ignores the exit code, since it's meant to run panic, which crashes by
// design.
func execRun(cmd ...string) []byte {
	c := exec.Command(cmd[0], cmd[1:]...)
	c.Env = append(os.Environ(), "GOTRACEBACK=all")
	out, _ := c.CombinedOutput()
	return out
}
<commit_msg>go1.17beta1 doesn't have to + after devel<commit_after>// Copyright 2020 Marc-Antoine Ruel. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

package internaltest

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"
)

// PanicwebOutput returns the output of panicweb with inlining disabled.
//
// The function panics if any internal error occurs.
func PanicwebOutput() []byte {
	panicwebOnce.Do(func() {
		p := build("panicweb", false)
		if p == "" {
			panic("building panicweb failed")
		}
		defer func() {
			if err := os.Remove(p); err != nil {
				panic(err)
			}
		}()
		panicwebOutput = execRun(p)
	})
	out := make([]byte, len(panicwebOutput))
	copy(out, panicwebOutput)
	return out
}

// PanicOutputs returns a map of the output of every subcommands.
//
// panic is built with inlining disabled.
//
// The subcommand "race" is built with the race detector. Others are built
// without. In particular "asleep" doesn't work with the race detector.
//
// The function panics if any internal error occurs.
func PanicOutputs() map[string][]byte {
	panicOutputsOnce.Do(func() {
		// Extracts the subcommands, then run each of them individually.
		pplain := build("panic", false)
		if pplain == "" {
			// The odd of this failing is close to nil.
			panic("building panic failed")
		}
		defer func() {
			if err := os.Remove(pplain); err != nil {
				panic(err)
			}
		}()

		prace := build("panic", true)
		if prace == "" {
			// Race detector is not supported on this platform.
		} else {
			defer func() {
				if err := os.Remove(prace); err != nil {
					panic(err)
				}
			}()
		}

		// Collect the subcommands.
		cmds := strings.Split(strings.TrimSpace(string(execRun(pplain, "dump_commands"))), "\n")
		if len(cmds) == 0 {
			panic("no command retrieved")
		}

		// Collect the output of each subcommand.
		panicOutputs = map[string][]byte{}
		for _, cmd := range cmds {
			cmd = strings.TrimSpace(cmd)
			p := pplain
			if cmd == "race" {
				if prace == "" {
					// Race detector is not supported.
					continue
				}
				p = prace
			}
			if panicOutputs[cmd] = execRun(p, cmd); len(panicOutputs[cmd]) == 0 {
				panic(fmt.Sprintf("no output for %s", cmd))
			}
		}
	})
	out := make(map[string][]byte, len(panicOutputs))
	for k, v := range panicOutputs {
		w := make([]byte, len(v))
		copy(w, v)
		out[k] = w
	}
	return out
}

// StaticPanicwebOutput returns a constant version of panicweb output for use
// in benchmarks.
func StaticPanicwebOutput() []byte {
	return []byte(staticPanicweb)
}

// StaticPanicRaceOutput returns a constant version of 'panic race' output.
func StaticPanicRaceOutput() []byte {
	return []byte(staticPanicRace)
}

// IsUsingModules is best guess to know if go module are enabled.
//
// Panics if an internal error occurs.
//
// It reads the current value of GO111MODULES.
func IsUsingModules() bool {
	// Calculate the default. We assume developer builds are recent (go1.14 and
	// later).
	ver := GetGoMinorVersion()
	if ver > 0 && ver < 11 {
		// go1.9.7+ and go1.10.3+ were fixed to tolerate semantic versioning import
		// but they do not support the environment variable.
		return false
	}
	def := (ver == 0 || ver >= 14)
	s := os.Getenv("GO111MODULE")
	return (def && (s == "auto" || s == "")) || s == "on"
}

//

var (
	panicwebOnce     sync.Once
	panicwebOutput   []byte
	panicOutputsOnce sync.Once
	panicOutputs     map[string][]byte
)

// GetGoMinorVersion returns the Go1 minor version.
//
// Returns 0 for a developer build, panics if can't parse the version.
//
// Ignores the revision (go1.<minor>.<revision>).
func GetGoMinorVersion() int {
	ver := runtime.Version()
	if strings.HasPrefix(ver, "devel ") {
		return 0
	}
	if !strings.HasPrefix(ver, "go1.") {
		// This will break on go2. Please submit a PR to fix this once Go2 is
		// released.
		panic(fmt.Sprintf("unexpected go version %q", ver))
	}
	v := ver[4:]
	if i := strings.IndexByte(v, '.'); i != -1 {
		v = v[:i]
	} else if i := strings.Index(v, "beta"); i != -1 {
		v = v[:i]
	} else if i := strings.Index(v, "rc"); i != -1 {
		v = v[:i]
	}

	m, err := strconv.Atoi(v)
	if err != nil {
		panic(fmt.Sprintf("failed to parse %q: %v", ver, err))
	}
	return m
}

// build creates a temporary file and returns the path to it.
func build(tool string, race bool) string {
	p := filepath.Join(os.TempDir(), tool)
	if race {
		p += "_race"
	}
	// Starting with go1.11, ioutil.TempFile() supports specifying a suffix. This
	// is necessary to set the ".exe" suffix on Windows. Until we drop support
	// for go1.10 and earlier, do the equivalent ourselves in an lousy way.
	p += fmt.Sprintf("_%d", os.Getpid())
	if runtime.GOOS == "windows" {
		p += ".exe"
	}
	path := "github.com/maruel/panicparse/cmd/"
	if IsUsingModules() {
		path = "github.com/maruel/panicparse/v2/cmd/"
	}
	if err := Compile(path+tool, p, "", true, race); err != nil {
		_, _ = os.Stderr.WriteString(err.Error())
		return ""
	}
	return p
}

var errNoRace = errors.New("platform does not support -race")

// Compile compiles sources into an executable.
func Compile(in, exe, cwd string, disableInlining, race bool) error {
	// Disable inlining otherwise the inlining varies between local execution and
	// remote execution. This can be observed as Elided being true without any
	// argument.
	args := []string{"build", "-o", exe}
	if disableInlining {
		args = append(args, "-gcflags", "-l")
	}
	if race {
		args = append(args, "-race")
	}
	c := exec.Command("go", append(args, in)...)
	c.Dir = cwd
	if out, err := c.CombinedOutput(); err != nil {
		if race && strings.HasPrefix(string(out), "go test: -race is only supported on ") {
			return errNoRace
		}
		return fmt.Errorf("compile failure: "+wrap+"\n%s", err, out)
	}
	return nil
}

// execRun runs a command and returns the combined output.
//
// It ignores the exit code, since it's meant to run panic, which crashes by
// design.
func execRun(cmd ...string) []byte {
	c := exec.Command(cmd[0], cmd[1:]...)
	c.Env = append(os.Environ(), "GOTRACEBACK=all")
	out, _ := c.CombinedOutput()
	return out
}
<|endoftext|>


<commit_before>package cache

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"time"

	"github.com/pkg/errors"
	"github.com/restic/restic/internal/debug"
	"github.com/restic/restic/internal/fs"
	"github.com/restic/restic/internal/restic"
)

// Cache manages a local cache.
type Cache struct {
	Path             string
	Base             string
	Created          bool
	PerformReadahead func(restic.Handle) bool
}

const dirMode = 0700
const fileMode = 0644

func readVersion(dir string) (v uint, err error) {
	buf, err := ioutil.ReadFile(filepath.Join(dir, "version"))
	if os.IsNotExist(err) {
		return 0, nil
	}

	if err != nil {
		return 0, errors.Wrap(err, "ReadFile")
	}

	ver, err := strconv.ParseUint(string(buf), 10, 32)
	if err != nil {
		return 0, errors.Wrap(err, "ParseUint")
	}

	return uint(ver), nil
}

const cacheVersion = 1

// ensure Cache implements restic.Cache
var _ restic.Cache = &Cache{}

var cacheLayoutPaths = map[restic.FileType]string{
	restic.PackFile:     "data",
	restic.SnapshotFile: "snapshots",
	restic.IndexFile:    "index",
}

const cachedirTagSignature = "Signature: 8a477f597d28d172789f06886806bc55\n"

func writeCachedirTag(dir string) error {
	if err := fs.MkdirAll(dir, dirMode); err != nil {
		return err
	}

	tagfile := filepath.Join(dir, "CACHEDIR.TAG")
	_, err := fs.Lstat(tagfile)
	if err != nil && !os.IsNotExist(err) {
		return errors.Wrap(err, "Lstat")
	}

	f, err := fs.OpenFile(tagfile, os.O_CREATE|os.O_EXCL|os.O_WRONLY, fileMode)
	if err != nil {
		if os.IsExist(errors.Cause(err)) {
			return nil
		}

		return errors.Wrap(err, "OpenFile")
	}

	debug.Log("Create CACHEDIR.TAG at %v", dir)
	if _, err := f.Write([]byte(cachedirTagSignature)); err != nil {
		_ = f.Close()
		return errors.Wrap(err, "Write")
	}

	return f.Close()
}

// New returns a new cache for the repo ID at basedir. If basedir is the empty
// string, the default cache location (according to the XDG standard) is used.
//
// For partial files, the complete file is loaded and stored in the cache when
// performReadahead returns true.
func New(id string, basedir string) (c *Cache, err error) {
	if basedir == "" {
		basedir, err = DefaultDir()
		if err != nil {
			return nil, err
		}
	}

	created, err := mkdirCacheDir(basedir)
	if err != nil {
		return nil, err
	}

	// create base dir and tag it as a cache directory
	if err = writeCachedirTag(basedir); err != nil {
		return nil, err
	}

	cachedir := filepath.Join(basedir, id)
	debug.Log("using cache dir %v", cachedir)

	v, err := readVersion(cachedir)
	if err != nil {
		return nil, err
	}

	if v > cacheVersion {
		return nil, errors.New("cache version is newer")
	}

	// create the repo cache dir if it does not exist yet
	_, err = fs.Lstat(cachedir)
	if os.IsNotExist(err) {
		err = fs.MkdirAll(cachedir, dirMode)
		if err != nil {
			return nil, err
		}
		created = true
	}

	// update the timestamp so that we can detect old cache dirs
	err = updateTimestamp(cachedir)
	if err != nil {
		return nil, err
	}

	if v < cacheVersion {
		err = ioutil.WriteFile(filepath.Join(cachedir, "version"), []byte(fmt.Sprintf("%d", cacheVersion)), fileMode)
		if err != nil {
			return nil, errors.Wrap(err, "WriteFile")
		}
	}

	for _, p := range cacheLayoutPaths {
		if err = fs.MkdirAll(filepath.Join(cachedir, p), dirMode); err != nil {
			return nil, err
		}
	}

	c = &Cache{
		Path:    cachedir,
		Base:    basedir,
		Created: created,
		PerformReadahead: func(restic.Handle) bool {
			// do not perform readahead by default
			return false
		},
	}

	return c, nil
}

// updateTimestamp sets the modification timestamp (mtime and atime) for the
// directory d to the current time.
func updateTimestamp(d string) error {
	t := time.Now()
	return fs.Chtimes(d, t, t)
}

// MaxCacheAge is the default age (30 days) after which cache directories are considered old.
const MaxCacheAge = 30 * 24 * time.Hour

func validCacheDirName(s string) bool {
	r := regexp.MustCompile(`^[a-fA-F0-9]{64}$`)
	return r.MatchString(s)
}

// listCacheDirs returns the list of cache directories.
func listCacheDirs(basedir string) ([]os.FileInfo, error) {
	f, err := fs.Open(basedir)
	if err != nil && os.IsNotExist(errors.Cause(err)) {
		return nil, nil
	}

	if err != nil {
		return nil, err
	}

	entries, err := f.Readdir(-1)
	if err != nil {
		return nil, err
	}

	err = f.Close()
	if err != nil {
		return nil, err
	}

	result := make([]os.FileInfo, 0, len(entries))
	for _, entry := range entries {
		if !entry.IsDir() {
			continue
		}

		if !validCacheDirName(entry.Name()) {
			continue
		}

		result = append(result, entry)
	}

	return result, nil
}

// All returns a list of cache directories.
func All(basedir string) (dirs []os.FileInfo, err error) {
	return listCacheDirs(basedir)
}

// OlderThan returns the list of cache directories older than max.
func OlderThan(basedir string, max time.Duration) ([]os.FileInfo, error) {
	entries, err := listCacheDirs(basedir)
	if err != nil {
		return nil, err
	}

	var oldCacheDirs []os.FileInfo
	for _, fi := range entries {
		if !IsOld(fi.ModTime(), max) {
			continue
		}

		oldCacheDirs = append(oldCacheDirs, fi)
	}

	debug.Log("%d old cache dirs found", len(oldCacheDirs))

	return oldCacheDirs, nil
}

// Old returns a list of cache directories with a modification time of more
// than 30 days ago.
func Old(basedir string) ([]os.FileInfo, error) {
	return OlderThan(basedir, MaxCacheAge)
}

// IsOld returns true if the timestamp is considered old.
func IsOld(t time.Time, maxAge time.Duration) bool {
	oldest := time.Now().Add(-maxAge)
	return t.Before(oldest)
}

// errNoSuchFile is returned when a file is not cached.
type errNoSuchFile struct {
	Type string
	Name string
}

func (e errNoSuchFile) Error() string {
	return fmt.Sprintf("file %v (%v) is not cached", e.Name, e.Type)
}

// IsNotExist returns true if the error was caused by a non-existing file.
func (c *Cache) IsNotExist(err error) bool {
	_, ok := errors.Cause(err).(errNoSuchFile)
	return ok
}

// Wrap returns a backend with a cache.
func (c *Cache) Wrap(be restic.Backend) restic.Backend {
	return newBackend(be, c)
}

// BaseDir returns the base directory.
func (c *Cache) BaseDir() string {
	return c.Base
}
<commit_msg>Remove unused error type from internal/cache<commit_after>package cache

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"time"

	"github.com/pkg/errors"
	"github.com/restic/restic/internal/debug"
	"github.com/restic/restic/internal/fs"
	"github.com/restic/restic/internal/restic"
)

// Cache manages a local cache.
type Cache struct {
	Path             string
	Base             string
	Created          bool
	PerformReadahead func(restic.Handle) bool
}

const dirMode = 0700
const fileMode = 0644

func readVersion(dir string) (v uint, err error) {
	buf, err := ioutil.ReadFile(filepath.Join(dir, "version"))
	if os.IsNotExist(err) {
		return 0, nil
	}

	if err != nil {
		return 0, errors.Wrap(err, "ReadFile")
	}

	ver, err := strconv.ParseUint(string(buf), 10, 32)
	if err != nil {
		return 0, errors.Wrap(err, "ParseUint")
	}

	return uint(ver), nil
}

const cacheVersion = 1

// ensure Cache implements restic.Cache
var _ restic.Cache = &Cache{}

var cacheLayoutPaths = map[restic.FileType]string{
	restic.PackFile:     "data",
	restic.SnapshotFile: "snapshots",
	restic.IndexFile:    "index",
}

const cachedirTagSignature = "Signature: 8a477f597d28d172789f06886806bc55\n"

func writeCachedirTag(dir string) error {
	if err := fs.MkdirAll(dir, dirMode); err != nil {
		return err
	}

	tagfile := filepath.Join(dir, "CACHEDIR.TAG")
	_, err := fs.Lstat(tagfile)
	if err != nil && !os.IsNotExist(err) {
		return errors.Wrap(err, "Lstat")
	}

	f, err := fs.OpenFile(tagfile, os.O_CREATE|os.O_EXCL|os.O_WRONLY, fileMode)
	if err != nil {
		if os.IsExist(errors.Cause(err)) {
			return nil
		}

		return errors.Wrap(err, "OpenFile")
	}

	debug.Log("Create CACHEDIR.TAG at %v", dir)
	if _, err := f.Write([]byte(cachedirTagSignature)); err != nil {
		_ = f.Close()
		return errors.Wrap(err, "Write")
	}

	return f.Close()
}

// New returns a new cache for the repo ID at basedir. If basedir is the empty
// string, the default cache location (according to the XDG standard) is used.
//
// For partial files, the complete file is loaded and stored in the cache when
// performReadahead returns true.
func New(id string, basedir string) (c *Cache, err error) {
	if basedir == "" {
		basedir, err = DefaultDir()
		if err != nil {
			return nil, err
		}
	}

	created, err := mkdirCacheDir(basedir)
	if err != nil {
		return nil, err
	}

	// create base dir and tag it as a cache directory
	if err = writeCachedirTag(basedir); err != nil {
		return nil, err
	}

	cachedir := filepath.Join(basedir, id)
	debug.Log("using cache dir %v", cachedir)

	v, err := readVersion(cachedir)
	if err != nil {
		return nil, err
	}

	if v > cacheVersion {
		return nil, errors.New("cache version is newer")
	}

	// create the repo cache dir if it does not exist yet
	_, err = fs.Lstat(cachedir)
	if os.IsNotExist(err) {
		err = fs.MkdirAll(cachedir, dirMode)
		if err != nil {
			return nil, err
		}
		created = true
	}

	// update the timestamp so that we can detect old cache dirs
	err = updateTimestamp(cachedir)
	if err != nil {
		return nil, err
	}

	if v < cacheVersion {
		err = ioutil.WriteFile(filepath.Join(cachedir, "version"), []byte(fmt.Sprintf("%d", cacheVersion)), fileMode)
		if err != nil {
			return nil, errors.Wrap(err, "WriteFile")
		}
	}

	for _, p := range cacheLayoutPaths {
		if err = fs.MkdirAll(filepath.Join(cachedir, p), dirMode); err != nil {
			return nil, err
		}
	}

	c = &Cache{
		Path:    cachedir,
		Base:    basedir,
		Created: created,
		PerformReadahead: func(restic.Handle) bool {
			// do not perform readahead by default
			return false
		},
	}

	return c, nil
}

// updateTimestamp sets the modification timestamp (mtime and atime) for the
// directory d to the current time.
func updateTimestamp(d string) error {
	t := time.Now()
	return fs.Chtimes(d, t, t)
}

// MaxCacheAge is the default age (30 days) after which cache directories are considered old.
const MaxCacheAge = 30 * 24 * time.Hour

func validCacheDirName(s string) bool {
	r := regexp.MustCompile(`^[a-fA-F0-9]{64}$`)
	return r.MatchString(s)
}

// listCacheDirs returns the list of cache directories.
func listCacheDirs(basedir string) ([]os.FileInfo, error) {
	f, err := fs.Open(basedir)
	if err != nil && os.IsNotExist(errors.Cause(err)) {
		return nil, nil
	}

	if err != nil {
		return nil, err
	}

	entries, err := f.Readdir(-1)
	if err != nil {
		return nil, err
	}

	err = f.Close()
	if err != nil {
		return nil, err
	}

	result := make([]os.FileInfo, 0, len(entries))
	for _, entry := range entries {
		if !entry.IsDir() {
			continue
		}

		if !validCacheDirName(entry.Name()) {
			continue
		}

		result = append(result, entry)
	}

	return result, nil
}

// All returns a list of cache directories.
func All(basedir string) (dirs []os.FileInfo, err error) {
	return listCacheDirs(basedir)
}

// OlderThan returns the list of cache directories older than max.
func OlderThan(basedir string, max time.Duration) ([]os.FileInfo, error) {
	entries, err := listCacheDirs(basedir)
	if err != nil {
		return nil, err
	}

	var oldCacheDirs []os.FileInfo
	for _, fi := range entries {
		if !IsOld(fi.ModTime(), max) {
			continue
		}

		oldCacheDirs = append(oldCacheDirs, fi)
	}

	debug.Log("%d old cache dirs found", len(oldCacheDirs))

	return oldCacheDirs, nil
}

// Old returns a list of cache directories with a modification time of more
// than 30 days ago.
func Old(basedir string) ([]os.FileInfo, error) {
	return OlderThan(basedir, MaxCacheAge)
}

// IsOld returns true if the timestamp is considered old.
func IsOld(t time.Time, maxAge time.Duration) bool {
	oldest := time.Now().Add(-maxAge)
	return t.Before(oldest)
}

// Wrap returns a backend with a cache.
func (c *Cache) Wrap(be restic.Backend) restic.Backend {
	return newBackend(be, c)
}

// BaseDir returns the base directory.
func (c *Cache) BaseDir() string {
	return c.Base
}
<|endoftext|>


<commit_before><commit_msg>Make marking of parents separate member function<commit_after><|endoftext|>


<commit_before>// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package setup

import (
	"flag"

	"github.com/matrix-org/dendrite/internal/config"

	"github.com/sirupsen/logrus"
)

var configPath = flag.String("config", "dendrite.yaml", "The path to the config file. For more information, see the config file in this repository.")

// ParseFlags parses the commandline flags and uses them to create a config.
func ParseFlags(monolith bool) *config.Dendrite {
	flag.Parse()

	if *configPath == "" {
		logrus.Fatal("--config must be supplied")
	}

	cfg, err := config.Load(*configPath, monolith)

	if err != nil {
		logrus.Fatalf("Invalid config file: %s", err)
	}

	return cfg
}
<commit_msg>Add -version flag (#1581)<commit_after>// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package setup

import (
	"flag"
	"fmt"
	"os"

	"github.com/matrix-org/dendrite/internal"
	"github.com/matrix-org/dendrite/internal/config"
	"github.com/sirupsen/logrus"
)

var (
	configPath = flag.String("config", "dendrite.yaml", "The path to the config file. For more information, see the config file in this repository.")
	version    = flag.Bool("version", false, "Shows the current version and exits immediately.")
)

// ParseFlags parses the commandline flags and uses them to create a config.
func ParseFlags(monolith bool) *config.Dendrite {
	flag.Parse()

	if *version {
		fmt.Println(internal.VersionString())
		os.Exit(0)
	}

	if *configPath == "" {
		logrus.Fatal("--config must be supplied")
	}

	cfg, err := config.Load(*configPath, monolith)

	if err != nil {
		logrus.Fatalf("Invalid config file: %s", err)
	}

	return cfg
}
<|endoftext|>


<commit_before>package tracker

// http://bittorrent.org/beps/bep_0015.html
// http://xbtt.sourceforge.net/udp_tracker_protocol.html
// http://www.rasterbar.com/products/libtorrent/udp_tracker_protocol.html

import (
	"bytes"
	"encoding/binary"
	"errors"
	"io"
	"math/rand"
	"net"
	"sync"
	"time"

	"github.com/cenkalti/backoff"

	"github.com/cenkalti/rain/internal/protocol"
)

const connectionIDMagic = 0x41727101980
const connectionIDInterval = time.Minute

type action int32

// UDP tracker Actions
const (
	connect action = iota
	announce
	scrape
	errorAction
)

type transaction struct {
	request  udpReqeust
	response []byte
	err      error
	done     chan struct{}
}

func newTransaction(req udpReqeust) *transaction {
	req.SetTransactionID(rand.Int31())
	return &transaction{
		request: req,
		done:    make(chan struct{}),
	}
}

func (t *transaction) ID() int32 { return t.request.GetTransactionID() }
func (t *transaction) Done()     { close(t.done) }

type udpTracker struct {
	*trackerBase
	conn          *net.UDPConn
	dialMutex     sync.Mutex
	connected     bool
	transactions  map[int32]*transaction
	transactionsM sync.Mutex
	writeC        chan *transaction
}

func newUDPTracker(b *trackerBase) *udpTracker {
	return &udpTracker{
		trackerBase:  b,
		transactions: make(map[int32]*transaction),
		writeC:       make(chan *transaction),
	}
}

func (t *udpTracker) dial() error {
	serverAddr, err := net.ResolveUDPAddr("udp", t.url.Host)
	if err != nil {
		return err
	}
	t.conn, err = net.DialUDP("udp", nil, serverAddr)
	if err != nil {
		return err
	}
	go t.readLoop()
	go t.writeLoop()
	return nil
}

// Close the tracker connection.
// TODO end all goroutines.
func (t *udpTracker) Close() error {
	return t.conn.Close()
}

// readLoop reads datagrams from connection, finds the transaction and
// sends the bytes to the transaction's response channel.
func (t *udpTracker) readLoop() {
	// Read buffer must be big enough to hold a UDP packet of maximum expected size.
	// Current value is: 320 = 20 + 50*6 (AnnounceResponse with 50 peers)
	buf := make([]byte, 320)
	for {
		n, err := t.conn.Read(buf)
		if err != nil {
			t.log.Error(err)
			if nerr, ok := err.(net.Error); ok && !nerr.Temporary() {
				t.log.Debug("End of tracker read loop")
				return
			}
			continue
		}
		t.log.Debug("Read ", n, " bytes")

		var header udpMessageHeader
		if n < binary.Size(header) {
			t.log.Error("response is too small")
			continue
		}

		err = binary.Read(bytes.NewReader(buf), binary.BigEndian, &header)
		if err != nil {
			t.log.Error(err)
			continue
		}

		t.transactionsM.Lock()
		trx, ok := t.transactions[header.TransactionID]
		delete(t.transactions, header.TransactionID)
		t.transactionsM.Unlock()
		if !ok {
			t.log.Errorln("unexpected transaction_id:", header.TransactionID)
			continue
		}

		// Tracker has sent and error.
		if header.Action == errorAction {
			// The part after the header is the error message.
			trx.err = Error(buf[binary.Size(header):])
			trx.Done()
			continue
		}

		// Copy data into a new slice because buf will be overwritten at next read.
		trx.response = make([]byte, n)
		copy(trx.response, buf)
		trx.Done()
	}
}

// writeLoop receives a request from t.transactionC, sets a random TransactionID
// and sends it to the tracker.
func (t *udpTracker) writeLoop() {
	var connectionID int64
	var connectionIDtime time.Time

	for trx := range t.writeC {
		if time.Since(connectionIDtime) > connectionIDInterval {
			connectionID = t.connect()
			connectionIDtime = time.Now()
		}
		trx.request.SetConnectionID(connectionID)

		t.writeTrx(trx)
	}
}

func (t *udpTracker) writeTrx(trx *transaction) {
	t.log.Debugln("Writing transaction. ID:", trx.ID())
	_, err := trx.request.WriteTo(t.conn)
	if err != nil {
		t.log.Error(err)
	}
}

// connect sends a connectRequest and returns a ConnectionID given by the tracker.
// On error, it backs off with the algorithm described in BEP15 and retries.
// It does not return until tracker sends a ConnectionID.
func (t *udpTracker) connect() int64 {
	req := new(connectRequest)
	req.SetAction(connect)
	req.SetConnectionID(connectionIDMagic)

	trx := newTransaction(req)

	for {
		data, err := t.retryTransaction(t.writeTrx, trx, nil) // Does not return until transaction is completed.
		if err != nil {
			t.log.Error(err)
			continue
		}

		var response connectResponse
		err = binary.Read(bytes.NewReader(data), binary.BigEndian, &response)
		if err != nil {
			t.log.Error(err)
			continue
		}

		if response.Action != connect {
			t.log.Error("invalid action in connect response")
			continue
		}

		t.log.Debugf("connect Response: %#v\n", response)
		return response.ConnectionID
	}
}

func (t *udpTracker) retryTransaction(f func(*transaction), trx *transaction, cancel <-chan struct{}) ([]byte, error) {
	t.transactionsM.Lock()
	t.transactions[trx.ID()] = trx
	t.transactionsM.Unlock()

	ticker := backoff.NewTicker(UDPBackOff())
	defer ticker.Stop()
	for {
		select {
		case <-ticker.C:
			f(trx)
		case <-trx.done:
			// transaction is deleted in readLoop()
			return trx.response, trx.err
		case <-cancel:
			t.transactionsM.Lock()
			delete(t.transactions, trx.ID())
			t.transactionsM.Unlock()
			return nil, errors.New("transaction cancelled")
		}
	}
}

func (t *udpTracker) sendTransaction(trx *transaction, cancel <-chan struct{}) ([]byte, error) {
	f := func(trx *transaction) { t.writeC <- trx }
	return t.retryTransaction(f, trx, cancel)
}

func (t *udpTracker) Announce(transfer Transfer, e Event) (*AnnounceResponse, error) {
	t.dialMutex.Lock()
	if !t.connected {
		err := t.dial()
		if err != nil {
			t.dialMutex.Unlock()
			return nil, err
		}
		t.connected = true
	}
	t.dialMutex.Unlock()

	request := &announceRequest{
		InfoHash:   transfer.InfoHash(),
		PeerID:     t.peerID,
		Event:      e,
		IP:         0, // Tracker uses sender of this UDP packet.
		Key:        0, // TODO set it
		NumWant:    NumWant,
		Port:       t.port,
		Extensions: 0,
	}
	request.SetAction(announce)
	request2 := &transferAnnounceRequest{transfer: transfer, announceRequest: request}

	// t.request may block, that's why we pass cancel as argument.
	trx := newTransaction(request2)
	reply, err := t.sendTransaction(trx, nil) // TODO pass cancel instead of nil
	if err != nil {
		return nil, err
	}

	response, peers, err := t.parseAnnounceResponse(reply)
	if err != nil {
		return nil, err
	}
	t.log.Debugf("Announce response: %#v", response)

	return &AnnounceResponse{
		Error:    nil, // TODO handler error
		Interval: time.Duration(response.Interval) * time.Second,
		Leechers: response.Leechers,
		Seeders:  response.Seeders,
		Peers:    peers,
	}, nil
}

func (t *udpTracker) parseAnnounceResponse(data []byte) (*announceResponse, []Peer, error) {
	response := new(announceResponse)
	if len(data) < binary.Size(response) {
		return nil, nil, errors.New("response is too small")
	}

	reader := bytes.NewReader(data)

	err := binary.Read(reader, binary.BigEndian, response)
	if err != nil {
		return nil, nil, err
	}
	t.log.Debugf("annouceResponse: %#v", response)

	if response.Action != announce {
		return nil, nil, errors.New("invalid action")
	}

	peers, err := t.parsePeers(reader)
	if err != nil {
		return nil, nil, err
	}

	return response, peers, nil
}

func (t *udpTracker) Scrape(transfers []Transfer) (*ScrapeResponse, error) { return nil, nil }

type udpBackOff int

func (b *udpBackOff) NextBackOff() time.Duration {
	defer func() { *b++ }()
	if *b > 8 {
		*b = 8
	}
	return time.Duration(15*(2^*b)) * time.Second
}

func (b *udpBackOff) Reset() { *b = 0 }

var UDPBackOff = func() backoff.BackOff { return new(udpBackOff) }

type udpMessage interface {
	GetAction() action
	SetAction(action)
	GetTransactionID() int32
	SetTransactionID(int32)
}

type udpReqeust interface {
	udpMessage
	GetConnectionID() int64
	SetConnectionID(int64)
	io.WriterTo
}

// udpMessageHeader implements udpMessage.
type udpMessageHeader struct {
	Action        action
	TransactionID int32
}

func (h *udpMessageHeader) GetAction() action         { return h.Action }
func (h *udpMessageHeader) SetAction(a action)        { h.Action = a }
func (h *udpMessageHeader) GetTransactionID() int32   { return h.TransactionID }
func (h *udpMessageHeader) SetTransactionID(id int32) { h.TransactionID = id }

// udpRequestHeader implements udpMessage and udpReqeust.
type udpRequestHeader struct {
	ConnectionID int64
	udpMessageHeader
}

func (h *udpRequestHeader) GetConnectionID() int64   { return h.ConnectionID }
func (h *udpRequestHeader) SetConnectionID(id int64) { h.ConnectionID = id }

type connectRequest struct {
	udpRequestHeader
}

func (r *connectRequest) WriteTo(w io.Writer) (int64, error) {
	return 0, binary.Write(w, binary.BigEndian, r)
}

type connectResponse struct {
	udpMessageHeader
	ConnectionID int64
}

type announceRequest struct {
	udpRequestHeader
	InfoHash   protocol.InfoHash
	PeerID     protocol.PeerID
	Downloaded int64
	Left       int64
	Uploaded   int64
	Event      Event
	IP         uint32
	Key        uint32
	NumWant    int32
	Port       uint16
	Extensions uint16
}

type transferAnnounceRequest struct {
	transfer Transfer
	*announceRequest
}

func (r *transferAnnounceRequest) WriteTo(w io.Writer) (int64, error) {
	r.Downloaded = r.transfer.Downloaded()
	r.Uploaded = r.transfer.Uploaded()
	r.Left = r.transfer.Left()
	return 0, binary.Write(w, binary.BigEndian, r.announceRequest)
}

type announceResponse struct {
	udpMessageHeader
	Interval int32
	Leechers int32
	Seeders  int32
}
<commit_msg>close udp tracker<commit_after>package tracker

// http://bittorrent.org/beps/bep_0015.html
// http://xbtt.sourceforge.net/udp_tracker_protocol.html
// http://www.rasterbar.com/products/libtorrent/udp_tracker_protocol.html

import (
	"bytes"
	"encoding/binary"
	"errors"
	"io"
	"math/rand"
	"net"
	"sync"
	"time"

	"github.com/cenkalti/backoff"

	"github.com/cenkalti/rain/internal/protocol"
)

const connectionIDMagic = 0x41727101980
const connectionIDInterval = time.Minute

type action int32

// UDP tracker Actions
const (
	connect action = iota
	announce
	scrape
	errorAction
)

type transaction struct {
	request  udpReqeust
	response []byte
	err      error
	done     chan struct{}
}

func newTransaction(req udpReqeust) *transaction {
	req.SetTransactionID(rand.Int31())
	return &transaction{
		request: req,
		done:    make(chan struct{}),
	}
}

func (t *transaction) ID() int32 { return t.request.GetTransactionID() }
func (t *transaction) Done()     { close(t.done) }

type udpTracker struct {
	*trackerBase
	conn          *net.UDPConn
	dialMutex     sync.Mutex
	connected     bool
	transactions  map[int32]*transaction
	transactionsM sync.Mutex
	writeC        chan *transaction
}

func newUDPTracker(b *trackerBase) *udpTracker {
	return &udpTracker{
		trackerBase:  b,
		transactions: make(map[int32]*transaction),
		writeC:       make(chan *transaction),
	}
}

func (t *udpTracker) dial() error {
	serverAddr, err := net.ResolveUDPAddr("udp", t.url.Host)
	if err != nil {
		return err
	}
	t.conn, err = net.DialUDP("udp", nil, serverAddr)
	if err != nil {
		return err
	}
	go t.readLoop()
	go t.writeLoop()
	return nil
}

// Close the tracker connection.
func (t *udpTracker) Close() error {
	close(t.writeC)
	return t.conn.Close()
}

// readLoop reads datagrams from connection, finds the transaction and
// sends the bytes to the transaction's response channel.
func (t *udpTracker) readLoop() {
	// Read buffer must be big enough to hold a UDP packet of maximum expected size.
	// Current value is: 320 = 20 + 50*6 (AnnounceResponse with 50 peers)
	buf := make([]byte, 320)
	for {
		n, err := t.conn.Read(buf)
		if err != nil {
			t.log.Error(err)
			if nerr, ok := err.(net.Error); ok && !nerr.Temporary() {
				t.log.Debug("End of tracker read loop")
				return
			}
			continue
		}
		t.log.Debug("Read ", n, " bytes")

		var header udpMessageHeader
		if n < binary.Size(header) {
			t.log.Error("response is too small")
			continue
		}

		err = binary.Read(bytes.NewReader(buf), binary.BigEndian, &header)
		if err != nil {
			t.log.Error(err)
			continue
		}

		t.transactionsM.Lock()
		trx, ok := t.transactions[header.TransactionID]
		delete(t.transactions, header.TransactionID)
		t.transactionsM.Unlock()
		if !ok {
			t.log.Errorln("unexpected transaction_id:", header.TransactionID)
			continue
		}

		// Tracker has sent and error.
		if header.Action == errorAction {
			// The part after the header is the error message.
			trx.err = Error(buf[binary.Size(header):])
			trx.Done()
			continue
		}

		// Copy data into a new slice because buf will be overwritten at next read.
		trx.response = make([]byte, n)
		copy(trx.response, buf)
		trx.Done()
	}
}

// writeLoop receives a request from t.transactionC, sets a random TransactionID
// and sends it to the tracker.
func (t *udpTracker) writeLoop() {
	var connectionID int64
	var connectionIDtime time.Time

	for trx := range t.writeC {
		if time.Since(connectionIDtime) > connectionIDInterval {
			connectionID = t.connect()
			connectionIDtime = time.Now()
		}
		trx.request.SetConnectionID(connectionID)

		t.writeTrx(trx)
	}
}

func (t *udpTracker) writeTrx(trx *transaction) {
	t.log.Debugln("Writing transaction. ID:", trx.ID())
	_, err := trx.request.WriteTo(t.conn)
	if err != nil {
		t.log.Error(err)
	}
}

// connect sends a connectRequest and returns a ConnectionID given by the tracker.
// On error, it backs off with the algorithm described in BEP15 and retries.
// It does not return until tracker sends a ConnectionID.
func (t *udpTracker) connect() int64 {
	req := new(connectRequest)
	req.SetAction(connect)
	req.SetConnectionID(connectionIDMagic)

	trx := newTransaction(req)

	for {
		data, err := t.retryTransaction(t.writeTrx, trx, nil) // Does not return until transaction is completed.
		if err != nil {
			t.log.Error(err)
			continue
		}

		var response connectResponse
		err = binary.Read(bytes.NewReader(data), binary.BigEndian, &response)
		if err != nil {
			t.log.Error(err)
			continue
		}

		if response.Action != connect {
			t.log.Error("invalid action in connect response")
			continue
		}

		t.log.Debugf("connect Response: %#v\n", response)
		return response.ConnectionID
	}
}

func (t *udpTracker) retryTransaction(f func(*transaction), trx *transaction, cancel <-chan struct{}) ([]byte, error) {
	t.transactionsM.Lock()
	t.transactions[trx.ID()] = trx
	t.transactionsM.Unlock()

	ticker := backoff.NewTicker(UDPBackOff())
	defer ticker.Stop()
	for {
		select {
		case <-ticker.C:
			f(trx)
		case <-trx.done:
			// transaction is deleted in readLoop()
			return trx.response, trx.err
		case <-cancel:
			t.transactionsM.Lock()
			delete(t.transactions, trx.ID())
			t.transactionsM.Unlock()
			return nil, errors.New("transaction cancelled")
		}
	}
}

func (t *udpTracker) sendTransaction(trx *transaction, cancel <-chan struct{}) ([]byte, error) {
	f := func(trx *transaction) { t.writeC <- trx }
	return t.retryTransaction(f, trx, cancel)
}

func (t *udpTracker) Announce(transfer Transfer, e Event) (*AnnounceResponse, error) {
	t.dialMutex.Lock()
	if !t.connected {
		err := t.dial()
		if err != nil {
			t.dialMutex.Unlock()
			return nil, err
		}
		t.connected = true
	}
	t.dialMutex.Unlock()

	request := &announceRequest{
		InfoHash:   transfer.InfoHash(),
		PeerID:     t.peerID,
		Event:      e,
		IP:         0, // Tracker uses sender of this UDP packet.
		Key:        0, // TODO set it
		NumWant:    NumWant,
		Port:       t.port,
		Extensions: 0,
	}
	request.SetAction(announce)
	request2 := &transferAnnounceRequest{transfer: transfer, announceRequest: request}

	// t.request may block, that's why we pass cancel as argument.
	trx := newTransaction(request2)
	reply, err := t.sendTransaction(trx, nil) // TODO pass cancel instead of nil
	if err != nil {
		return nil, err
	}

	response, peers, err := t.parseAnnounceResponse(reply)
	if err != nil {
		return nil, err
	}
	t.log.Debugf("Announce response: %#v", response)

	return &AnnounceResponse{
		Error:    nil, // TODO handler error
		Interval: time.Duration(response.Interval) * time.Second,
		Leechers: response.Leechers,
		Seeders:  response.Seeders,
		Peers:    peers,
	}, nil
}

func (t *udpTracker) parseAnnounceResponse(data []byte) (*announceResponse, []Peer, error) {
	response := new(announceResponse)
	if len(data) < binary.Size(response) {
		return nil, nil, errors.New("response is too small")
	}

	reader := bytes.NewReader(data)

	err := binary.Read(reader, binary.BigEndian, response)
	if err != nil {
		return nil, nil, err
	}
	t.log.Debugf("annouceResponse: %#v", response)

	if response.Action != announce {
		return nil, nil, errors.New("invalid action")
	}

	peers, err := t.parsePeers(reader)
	if err != nil {
		return nil, nil, err
	}

	return response, peers, nil
}

func (t *udpTracker) Scrape(transfers []Transfer) (*ScrapeResponse, error) { return nil, nil }

type udpBackOff int

func (b *udpBackOff) NextBackOff() time.Duration {
	defer func() { *b++ }()
	if *b > 8 {
		*b = 8
	}
	return time.Duration(15*(2^*b)) * time.Second
}

func (b *udpBackOff) Reset() { *b = 0 }

var UDPBackOff = func() backoff.BackOff { return new(udpBackOff) }

type udpMessage interface {
	GetAction() action
	SetAction(action)
	GetTransactionID() int32
	SetTransactionID(int32)
}

type udpReqeust interface {
	udpMessage
	GetConnectionID() int64
	SetConnectionID(int64)
	io.WriterTo
}

// udpMessageHeader implements udpMessage.
type udpMessageHeader struct {
	Action        action
	TransactionID int32
}

func (h *udpMessageHeader) GetAction() action         { return h.Action }
func (h *udpMessageHeader) SetAction(a action)        { h.Action = a }
func (h *udpMessageHeader) GetTransactionID() int32   { return h.TransactionID }
func (h *udpMessageHeader) SetTransactionID(id int32) { h.TransactionID = id }

// udpRequestHeader implements udpMessage and udpReqeust.
type udpRequestHeader struct {
	ConnectionID int64
	udpMessageHeader
}

func (h *udpRequestHeader) GetConnectionID() int64   { return h.ConnectionID }
func (h *udpRequestHeader) SetConnectionID(id int64) { h.ConnectionID = id }

type connectRequest struct {
	udpRequestHeader
}

func (r *connectRequest) WriteTo(w io.Writer) (int64, error) {
	return 0, binary.Write(w, binary.BigEndian, r)
}

type connectResponse struct {
	udpMessageHeader
	ConnectionID int64
}

type announceRequest struct {
	udpRequestHeader
	InfoHash   protocol.InfoHash
	PeerID     protocol.PeerID
	Downloaded int64
	Left       int64
	Uploaded   int64
	Event      Event
	IP         uint32
	Key        uint32
	NumWant    int32
	Port       uint16
	Extensions uint16
}

type transferAnnounceRequest struct {
	transfer Transfer
	*announceRequest
}

func (r *transferAnnounceRequest) WriteTo(w io.Writer) (int64, error) {
	r.Downloaded = r.transfer.Downloaded()
	r.Uploaded = r.transfer.Uploaded()
	r.Left = r.transfer.Left()
	return 0, binary.Write(w, binary.BigEndian, r.announceRequest)
}

type announceResponse struct {
	udpMessageHeader
	Interval int32
	Leechers int32
	Seeders  int32
}
<|endoftext|>


<commit_before>package utils

import (
	"fmt"
	"os"
	"strings"
)

// PathExists returns true if the path exists
func PathExists(path string) bool {
	_, err := os.Lstat(path)
	if err != nil && os.IsNotExist(err) {
		return false
	}

	return true
}

// StringInSlice returns true if the provided string is in the provided slice
func StringInSlice(key string, list []string) bool {
	for _, entry := range list {
		if entry == key {
			return true
		}
	}
	return false
}

// Int64InSlice returns true if the provided int64 is in the provided slice
func Int64InSlice(key int64, list []int64) bool {
	for _, entry := range list {
		if entry == key {
			return true
		}
	}
	return false
}

// ParseTags converts a serialized tags list to map[string]string
func ParseTags(in string) (map[string]string, error) {
	out := map[string]string{}

	for _, entry := range strings.Split(in, ",") {
		entry = strings.TrimSpace(entry)
		if entry == "" {
			continue
		}

		fields := strings.SplitN(entry, ":", 2)
		if len(fields) < 2 {
			return nil, fmt.Errorf("Invalid tag: %s", entry)
		}

		_, ok := out[fields[0]]
		if ok {
			return nil, fmt.Errorf("Duplicate tag: %s", entry)
		}

		out[fields[0]] = fields[1]
	}

	return out, nil
}

// PackTags converts map[string]string to a serialized tags list
func PackTags(in map[string]string) string {
	tags := []string{}

	for k, v := range in {
		tags = append(tags, fmt.Sprintf("%s:%s", k, v))
	}

	return strings.Join(tags, ",")
}
<commit_msg>Sort tags<commit_after>package utils

import (
	"fmt"
	"os"
	"sort"
	"strings"
)

// PathExists returns true if the path exists
func PathExists(path string) bool {
	_, err := os.Lstat(path)
	if err != nil && os.IsNotExist(err) {
		return false
	}

	return true
}

// StringInSlice returns true if the provided string is in the provided slice
func StringInSlice(key string, list []string) bool {
	for _, entry := range list {
		if entry == key {
			return true
		}
	}
	return false
}

// Int64InSlice returns true if the provided int64 is in the provided slice
func Int64InSlice(key int64, list []int64) bool {
	for _, entry := range list {
		if entry == key {
			return true
		}
	}
	return false
}

// ParseTags converts a serialized tags list to map[string]string
func ParseTags(in string) (map[string]string, error) {
	out := map[string]string{}

	for _, entry := range strings.Split(in, ",") {
		entry = strings.TrimSpace(entry)
		if entry == "" {
			continue
		}

		fields := strings.SplitN(entry, ":", 2)
		if len(fields) < 2 {
			return nil, fmt.Errorf("Invalid tag: %s", entry)
		}

		_, ok := out[fields[0]]
		if ok {
			return nil, fmt.Errorf("Duplicate tag: %s", entry)
		}

		out[fields[0]] = fields[1]
	}

	return out, nil
}

// PackTags converts map[string]string to a serialized tags list
func PackTags(in map[string]string) string {
	tags := []string{}

	for k, v := range in {
		tags = append(tags, fmt.Sprintf("%s:%s", k, v))
	}

	sort.Strings(tags)

	return strings.Join(tags, ",")
}
<|endoftext|>