<commit_before>package aws

import (
	"fmt"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/guardduty"
	"github.com/hashicorp/terraform-plugin-sdk/helper/resource"
	"github.com/hashicorp/terraform-plugin-sdk/terraform"
)

func testAccAwsGuardDutyDetector_basic(t *testing.T) {
	resourceName := "aws_guardduty_detector.test"

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAwsGuardDutyDetectorDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccGuardDutyDetectorConfig_basic1,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAwsGuardDutyDetectorExists(resourceName),
					resource.TestCheckResourceAttrSet(resourceName, "account_id"),
					resource.TestCheckResourceAttr(resourceName, "enable", "true"),
					resource.TestCheckResourceAttr(resourceName, "finding_publishing_frequency", "SIX_HOURS"),
				),
			},
			{
				Config: testAccGuardDutyDetectorConfig_basic2,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAwsGuardDutyDetectorExists(resourceName),
					resource.TestCheckResourceAttr(resourceName, "enable", "false"),
				),
			},
			{
				Config: testAccGuardDutyDetectorConfig_basic3,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAwsGuardDutyDetectorExists(resourceName),
					resource.TestCheckResourceAttr(resourceName, "enable", "true"),
				),
			},
			{
				Config: testAccGuardDutyDetectorConfig_basic4,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAwsGuardDutyDetectorExists(resourceName),
					resource.TestCheckResourceAttr(resourceName, "finding_publishing_frequency", "FIFTEEN_MINUTES"),
				),
			},
		},
	})
}

func testAccAwsGuardDutyDetector_import(t *testing.T) {
	resourceName := "aws_guardduty_detector.test"

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAwsGuardDutyDetectorDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccGuardDutyDetectorConfig_basic1,
			},

			{
				ResourceName:      resourceName,
				ImportState:       true,
				ImportStateVerify: true,
			},
		},
	})
}

func testAccCheckAwsGuardDutyDetectorDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).guarddutyconn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_guardduty_detector" {
			continue
		}

		input := &guardduty.GetDetectorInput{
			DetectorId: aws.String(rs.Primary.ID),
		}

		_, err := conn.GetDetector(input)
		if err != nil {
			if isAWSErr(err, guardduty.ErrCodeBadRequestException, "The request is rejected because the input detectorId is not owned by the current account.") {
				return nil
			}
			return err
		}

		return fmt.Errorf("Expected GuardDuty Detector to be destroyed, %s found", rs.Primary.ID)
	}

	return nil
}

func testAccCheckAwsGuardDutyDetectorExists(name string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		_, ok := s.RootModule().Resources[name]
		if !ok {
			return fmt.Errorf("Not found: %s", name)
		}

		return nil
	}
}

const testAccGuardDutyDetectorConfig_basic1 = `
resource "aws_guardduty_detector" "test" {}`

const testAccGuardDutyDetectorConfig_basic2 = `
resource "aws_guardduty_detector" "test" {
  enable = false
}`

const testAccGuardDutyDetectorConfig_basic3 = `
resource "aws_guardduty_detector" "test" {
  enable = true
}`

const testAccGuardDutyDetectorConfig_basic4 = `
resource "aws_guardduty_detector" "test" {
  finding_publishing_frequency = "FIFTEEN_MINUTES"
}`
<commit_msg>tests/resource/aws_guardduty_detector: Add sweeper (#11722)<commit_after>package aws

import (
	"fmt"
	"log"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/guardduty"
	"github.com/hashicorp/go-multierror"
	"github.com/hashicorp/terraform-plugin-sdk/helper/resource"
	"github.com/hashicorp/terraform-plugin-sdk/terraform"
)

func init() {
	resource.AddTestSweepers("aws_guardduty_detector", &resource.Sweeper{
		Name: "aws_guardduty_detector",
		F:    testSweepGuarddutyDetectors,
	})
}

func testSweepGuarddutyDetectors(region string) error {
	client, err := sharedClientForRegion(region)

	if err != nil {
		return fmt.Errorf("error getting client: %s", err)
	}

	conn := client.(*AWSClient).guarddutyconn
	input := &guardduty.ListDetectorsInput{}
	var sweeperErrs *multierror.Error

	err = conn.ListDetectorsPages(input, func(page *guardduty.ListDetectorsOutput, lastPage bool) bool {
		for _, detectorID := range page.DetectorIds {
			id := aws.StringValue(detectorID)
			input := &guardduty.DeleteDetectorInput{
				DetectorId: detectorID,
			}

			log.Printf("[INFO] Deleting GuardDuty Detector: %s", id)
			_, err := conn.DeleteDetector(input)

			if err != nil {
				sweeperErr := fmt.Errorf("error deleting GuardDuty Detector (%s): %w", id, err)
				log.Printf("[ERROR] %s", sweeperErr)
				sweeperErrs = multierror.Append(sweeperErrs, sweeperErr)
			}
		}

		return !lastPage
	})

	if testSweepSkipSweepError(err) {
		log.Printf("[WARN] Skipping GuardDuty Detector sweep for %s: %s", region, err)
		return nil
	}

	if err != nil {
		return fmt.Errorf("error retrieving GuardDuty Detectors: %s", err)
	}

	return sweeperErrs.ErrorOrNil()
}

func testAccAwsGuardDutyDetector_basic(t *testing.T) {
	resourceName := "aws_guardduty_detector.test"

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAwsGuardDutyDetectorDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccGuardDutyDetectorConfig_basic1,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAwsGuardDutyDetectorExists(resourceName),
					resource.TestCheckResourceAttrSet(resourceName, "account_id"),
					resource.TestCheckResourceAttr(resourceName, "enable", "true"),
					resource.TestCheckResourceAttr(resourceName, "finding_publishing_frequency", "SIX_HOURS"),
				),
			},
			{
				Config: testAccGuardDutyDetectorConfig_basic2,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAwsGuardDutyDetectorExists(resourceName),
					resource.TestCheckResourceAttr(resourceName, "enable", "false"),
				),
			},
			{
				Config: testAccGuardDutyDetectorConfig_basic3,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAwsGuardDutyDetectorExists(resourceName),
					resource.TestCheckResourceAttr(resourceName, "enable", "true"),
				),
			},
			{
				Config: testAccGuardDutyDetectorConfig_basic4,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAwsGuardDutyDetectorExists(resourceName),
					resource.TestCheckResourceAttr(resourceName, "finding_publishing_frequency", "FIFTEEN_MINUTES"),
				),
			},
		},
	})
}

func testAccAwsGuardDutyDetector_import(t *testing.T) {
	resourceName := "aws_guardduty_detector.test"

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAwsGuardDutyDetectorDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccGuardDutyDetectorConfig_basic1,
			},

			{
				ResourceName:      resourceName,
				ImportState:       true,
				ImportStateVerify: true,
			},
		},
	})
}

func testAccCheckAwsGuardDutyDetectorDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).guarddutyconn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_guardduty_detector" {
			continue
		}

		input := &guardduty.GetDetectorInput{
			DetectorId: aws.String(rs.Primary.ID),
		}

		_, err := conn.GetDetector(input)
		if err != nil {
			if isAWSErr(err, guardduty.ErrCodeBadRequestException, "The request is rejected because the input detectorId is not owned by the current account.") {
				return nil
			}
			return err
		}

		return fmt.Errorf("Expected GuardDuty Detector to be destroyed, %s found", rs.Primary.ID)
	}

	return nil
}

func testAccCheckAwsGuardDutyDetectorExists(name string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		_, ok := s.RootModule().Resources[name]
		if !ok {
			return fmt.Errorf("Not found: %s", name)
		}

		return nil
	}
}

const testAccGuardDutyDetectorConfig_basic1 = `
resource "aws_guardduty_detector" "test" {}`

const testAccGuardDutyDetectorConfig_basic2 = `
resource "aws_guardduty_detector" "test" {
  enable = false
}`

const testAccGuardDutyDetectorConfig_basic3 = `
resource "aws_guardduty_detector" "test" {
  enable = true
}`

const testAccGuardDutyDetectorConfig_basic4 = `
resource "aws_guardduty_detector" "test" {
  finding_publishing_frequency = "FIFTEEN_MINUTES"
}`
<|endoftext|>


<commit_before>package azurerm

import (
	"fmt"
	"net/http"
	"testing"

	"github.com/Azure/azure-sdk-for-go/arm/dns"
	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccAzureRMDnsPtrRecord_basic(t *testing.T) {
	ri := acctest.RandInt()
	config := testAccAzureRMDnsPtrRecord_basic(ri)

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testCheckAzureRMDnsPtrRecordDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: config,
				Check: resource.ComposeTestCheckFunc(
					testCheckAzureRMDnsPtrRecordExists("azurerm_dns_ptr_record.test"),
				),
			},
		},
	})
}

func TestAccAzureRMDnsPtrRecord_updateRecords(t *testing.T) {
	ri := acctest.RandInt()
	preConfig := testAccAzureRMDnsPtrRecord_basic(ri)
	postConfig := testAccAzureRMDnsPtrRecord_updateRecords(ri)

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testCheckAzureRMDnsPtrRecordDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: preConfig,
				Check: resource.ComposeTestCheckFunc(
					testCheckAzureRMDnsPtrRecordExists("azurerm_dns_ptr_record.test"),
					resource.TestCheckResourceAttr(
						"azurerm_dns_ptr_record.test", "records.#", "2"),
				),
			},

			resource.TestStep{
				Config: postConfig,
				Check: resource.ComposeTestCheckFunc(
					testCheckAzureRMDnsPtrRecordExists("azurerm_dns_ptr_record.test"),
					resource.TestCheckResourceAttr(
						"azurerm_dns_ptr_record.test", "records.#", "3"),
				),
			},
		},
	})
}

func TestAccAzureRMDnsPtrRecord_withTags(t *testing.T) {
	ri := acctest.RandInt()
	preConfig := testAccAzureRMDnsPtrRecord_withTags(ri)
	postConfig := testAccAzureRMDnsPtrRecord_withTagsUpdate(ri)

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testCheckAzureRMDnsPtrRecordDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: preConfig,
				Check: resource.ComposeTestCheckFunc(
					testCheckAzureRMDnsPtrRecordExists("azurerm_dns_ptr_record.test"),
					resource.TestCheckResourceAttr(
						"azurerm_dns_ptr_record.test", "tags.%", "2"),
				),
			},

			resource.TestStep{
				Config: postConfig,
				Check: resource.ComposeTestCheckFunc(
					testCheckAzureRMDnsPtrRecordExists("azurerm_dns_ptr_record.test"),
					resource.TestCheckResourceAttr(
						"azurerm_dns_ptr_record.test", "tags.%", "1"),
				),
			},
		},
	})
}

func testCheckAzureRMDnsPtrRecordExists(name string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		// Ensure we have enough information in state to look up in API
		rs, ok := s.RootModule().Resources[name]
		if !ok {
			return fmt.Errorf("Not found: %s", name)
		}

		ptrName := rs.Primary.Attributes["name"]
		zoneName := rs.Primary.Attributes["zone_name"]
		resourceGroup, hasResourceGroup := rs.Primary.Attributes["resource_group_name"]
		if !hasResourceGroup {
			return fmt.Errorf("Bad: no resource group found in state for DNS PTR record: %s", ptrName)
		}

		conn := testAccProvider.Meta().(*ArmClient).dnsClient
		resp, err := conn.Get(resourceGroup, zoneName, ptrName, dns.PTR)
		if err != nil {
			return fmt.Errorf("Bad: Get PTR RecordSet: %v", err)
		}

		if resp.StatusCode == http.StatusNotFound {
			return fmt.Errorf("Bad: DNS PTR record %s (resource group: %s) does not exist", ptrName, resourceGroup)
		}

		return nil
	}
}

func testCheckAzureRMDnsPtrRecordDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*ArmClient).dnsClient

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "azurerm_dns_ptr_record" {
			continue
		}

		ptrName := rs.Primary.Attributes["name"]
		zoneName := rs.Primary.Attributes["zone_name"]
		resourceGroup := rs.Primary.Attributes["resource_group_name"]

		resp, err := conn.Get(resourceGroup, zoneName, ptrName, dns.PTR)

		if err != nil {
			return nil
		}

		if resp.StatusCode != http.StatusNotFound {
			return fmt.Errorf("DNS PTR record still exists:\n%#v", resp.RecordSetProperties)
		}

	}

	return nil
}

func testAccAzureRMDnsPtrRecord_basic(rInt int) string {
	return fmt.Sprintf(`
resource "azurerm_resource_group" "test" {
    name = "acctestRG_%[1]d"
    location = "West US"
}
resource "azurerm_dns_zone" "test" {
    name = "acctestzone%[1]d.com"
    resource_group_name = "${azurerm_resource_group.test.name}"
}

resource "azurerm_dns_ptr_record" "test" {
    name = "testptrrecord%[1]d"
    resource_group_name = "${azurerm_resource_group.test.name}"
    zone_name = "${azurerm_dns_zone.test.name}"
    ttl = "300"
    records = ["hashicorp.com", "microsoft.com"]
}
`, rInt)
}

func testAccAzureRMDnsPtrRecord_updateRecords(rInt int) string {
	return fmt.Sprintf(`
resource "azurerm_resource_group" "test" {
    name = "acctestRG_%[1]d"
    location = "West US"
}
resource "azurerm_dns_zone" "test" {
    name = "acctestzone%[1]d.com"
    resource_group_name = "${azurerm_resource_group.test.name}"
}

resource "azurerm_dns_ptr_record" "test" {
    name = "testptrrecord%[1]d"
    resource_group_name = "${azurerm_resource_group.test.name}"
    zone_name = "${azurerm_dns_zone.test.name}"
    ttl = "300"
    records = ["hashicorp.com", "microsoft.com", "reddit.com"]
}
`, rInt)
}

func testAccAzureRMDnsPtrRecord_withTags(rInt int) string {
	return fmt.Sprintf(`
resource "azurerm_resource_group" "test" {
    name = "acctestRG_%[1]d"
    location = "West US"
}
resource "azurerm_dns_zone" "test" {
    name = "acctestzone%[1]d.com"
    resource_group_name = "${azurerm_resource_group.test.name}"
}

resource "azurerm_dns_ptr_record" "test" {
    name = "testptrrecord%[1]d"
    resource_group_name = "${azurerm_resource_group.test.name}"
    zone_name = "${azurerm_dns_zone.test.name}"
    ttl = "300"
    records = ["hashicorp.com", "microsoft.com"]

    tags {
		environment = "Dev"
		cost_center = "Ops"
    }
}
`, rInt)
}

func testAccAzureRMDnsPtrRecord_withTagsUpdate(rInt int) string {
	return fmt.Sprintf(`
resource "azurerm_resource_group" "test" {
    name = "acctestRG_%[1]d"
    location = "West US"
}
resource "azurerm_dns_zone" "test" {
    name = "acctestzone%[1]d.com"
    resource_group_name = "${azurerm_resource_group.test.name}"
}

resource "azurerm_dns_ptr_record" "test" {
    name = "testptrrecord%[1]d"
    resource_group_name = "${azurerm_resource_group.test.name}"
    zone_name = "${azurerm_dns_zone.test.name}"
    ttl = "300"
    records = ["hashicorp.com", "microsoft.com"]

    tags {
		environment = "Stage"
    }
}
`, rInt)
}
<commit_msg>Added tag validation to tests<commit_after>package azurerm

import (
	"fmt"
	"net/http"
	"testing"

	"github.com/Azure/azure-sdk-for-go/arm/dns"
	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccAzureRMDnsPtrRecord_basic(t *testing.T) {
	ri := acctest.RandInt()
	config := testAccAzureRMDnsPtrRecord_basic(ri)

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testCheckAzureRMDnsPtrRecordDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: config,
				Check: resource.ComposeTestCheckFunc(
					testCheckAzureRMDnsPtrRecordExists("azurerm_dns_ptr_record.test"),
				),
			},
		},
	})
}

func TestAccAzureRMDnsPtrRecord_updateRecords(t *testing.T) {
	ri := acctest.RandInt()
	preConfig := testAccAzureRMDnsPtrRecord_basic(ri)
	postConfig := testAccAzureRMDnsPtrRecord_updateRecords(ri)

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testCheckAzureRMDnsPtrRecordDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: preConfig,
				Check: resource.ComposeTestCheckFunc(
					testCheckAzureRMDnsPtrRecordExists("azurerm_dns_ptr_record.test"),
					resource.TestCheckResourceAttr(
						"azurerm_dns_ptr_record.test", "records.#", "2"),
				),
			},

			resource.TestStep{
				Config: postConfig,
				Check: resource.ComposeTestCheckFunc(
					testCheckAzureRMDnsPtrRecordExists("azurerm_dns_ptr_record.test"),
					resource.TestCheckResourceAttr(
						"azurerm_dns_ptr_record.test", "records.#", "3"),
				),
			},
		},
	})
}

func TestAccAzureRMDnsPtrRecord_withTags(t *testing.T) {
	ri := acctest.RandInt()
	preConfig := testAccAzureRMDnsPtrRecord_withTags(ri)
	postConfig := testAccAzureRMDnsPtrRecord_withTagsUpdate(ri)

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testCheckAzureRMDnsPtrRecordDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: preConfig,
				Check: resource.ComposeTestCheckFunc(
					testCheckAzureRMDnsPtrRecordExists("azurerm_dns_ptr_record.test"),
					resource.TestCheckResourceAttr(
						"azurerm_dns_ptr_record.test", "tags.%", "2"),
					resource.TestCheckResourceAttr("azurerm_dns_ptr_record.test", "tags.environment", "Dev"),
					resource.TestCheckResourceAttr("azurerm_dns_ptr_record.test", "tags.cost_center", "Ops"),
				),
			},

			resource.TestStep{
				Config: postConfig,
				Check: resource.ComposeTestCheckFunc(
					testCheckAzureRMDnsPtrRecordExists("azurerm_dns_ptr_record.test"),
					resource.TestCheckResourceAttr(
						"azurerm_dns_ptr_record.test", "tags.%", "1"),
				),
			},
		},
	})
}

func testCheckAzureRMDnsPtrRecordExists(name string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		// Ensure we have enough information in state to look up in API
		rs, ok := s.RootModule().Resources[name]
		if !ok {
			return fmt.Errorf("Not found: %s", name)
		}

		ptrName := rs.Primary.Attributes["name"]
		zoneName := rs.Primary.Attributes["zone_name"]
		resourceGroup, hasResourceGroup := rs.Primary.Attributes["resource_group_name"]
		if !hasResourceGroup {
			return fmt.Errorf("Bad: no resource group found in state for DNS PTR record: %s", ptrName)
		}

		conn := testAccProvider.Meta().(*ArmClient).dnsClient
		resp, err := conn.Get(resourceGroup, zoneName, ptrName, dns.PTR)
		if err != nil {
			return fmt.Errorf("Bad: Get PTR RecordSet: %v", err)
		}

		if resp.StatusCode == http.StatusNotFound {
			return fmt.Errorf("Bad: DNS PTR record %s (resource group: %s) does not exist", ptrName, resourceGroup)
		}

		return nil
	}
}

func testCheckAzureRMDnsPtrRecordDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*ArmClient).dnsClient

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "azurerm_dns_ptr_record" {
			continue
		}

		ptrName := rs.Primary.Attributes["name"]
		zoneName := rs.Primary.Attributes["zone_name"]
		resourceGroup := rs.Primary.Attributes["resource_group_name"]

		resp, err := conn.Get(resourceGroup, zoneName, ptrName, dns.PTR)

		if err != nil {
			return nil
		}

		if resp.StatusCode != http.StatusNotFound {
			return fmt.Errorf("DNS PTR record still exists:\n%#v", resp.RecordSetProperties)
		}

	}

	return nil
}

func testAccAzureRMDnsPtrRecord_basic(rInt int) string {
	return fmt.Sprintf(`
resource "azurerm_resource_group" "test" {
    name = "acctestRG_%[1]d"
    location = "West US"
}
resource "azurerm_dns_zone" "test" {
    name = "acctestzone%[1]d.com"
    resource_group_name = "${azurerm_resource_group.test.name}"
}

resource "azurerm_dns_ptr_record" "test" {
    name = "testptrrecord%[1]d"
    resource_group_name = "${azurerm_resource_group.test.name}"
    zone_name = "${azurerm_dns_zone.test.name}"
    ttl = "300"
    records = ["hashicorp.com", "microsoft.com"]
}
`, rInt)
}

func testAccAzureRMDnsPtrRecord_updateRecords(rInt int) string {
	return fmt.Sprintf(`
resource "azurerm_resource_group" "test" {
    name = "acctestRG_%[1]d"
    location = "West US"
}
resource "azurerm_dns_zone" "test" {
    name = "acctestzone%[1]d.com"
    resource_group_name = "${azurerm_resource_group.test.name}"
}

resource "azurerm_dns_ptr_record" "test" {
    name = "testptrrecord%[1]d"
    resource_group_name = "${azurerm_resource_group.test.name}"
    zone_name = "${azurerm_dns_zone.test.name}"
    ttl = "300"
    records = ["hashicorp.com", "microsoft.com", "reddit.com"]
}
`, rInt)
}

func testAccAzureRMDnsPtrRecord_withTags(rInt int) string {
	return fmt.Sprintf(`
resource "azurerm_resource_group" "test" {
    name = "acctestRG_%[1]d"
    location = "West US"
}
resource "azurerm_dns_zone" "test" {
    name = "acctestzone%[1]d.com"
    resource_group_name = "${azurerm_resource_group.test.name}"
}

resource "azurerm_dns_ptr_record" "test" {
    name = "testptrrecord%[1]d"
    resource_group_name = "${azurerm_resource_group.test.name}"
    zone_name = "${azurerm_dns_zone.test.name}"
    ttl = "300"
    records = ["hashicorp.com", "microsoft.com"]

    tags {
		environment = "Dev"
		cost_center = "Ops"
    }
}
`, rInt)
}

func testAccAzureRMDnsPtrRecord_withTagsUpdate(rInt int) string {
	return fmt.Sprintf(`
resource "azurerm_resource_group" "test" {
    name = "acctestRG_%[1]d"
    location = "West US"
}
resource "azurerm_dns_zone" "test" {
    name = "acctestzone%[1]d.com"
    resource_group_name = "${azurerm_resource_group.test.name}"
}

resource "azurerm_dns_ptr_record" "test" {
    name = "testptrrecord%[1]d"
    resource_group_name = "${azurerm_resource_group.test.name}"
    zone_name = "${azurerm_dns_zone.test.name}"
    ttl = "300"
    records = ["hashicorp.com", "microsoft.com"]

    tags {
		environment = "Stage"
    }
}
`, rInt)
}
<|endoftext|>


<commit_before>package azure

import (
	"context"
	"fmt"
	"sort"
	"strings"

	"github.com/hashicorp/terraform/backend"
	"github.com/hashicorp/terraform/states"
	"github.com/hashicorp/terraform/states/remote"
	"github.com/hashicorp/terraform/states/statemgr"
	"github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/blobs"
	"github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/containers"
)

const (
	// This will be used as directory name, the odd looking colon is simply to
	// reduce the chance of name conflicts with existing objects.
	keyEnvPrefix = "env:"
)

func (b *Backend) Workspaces() ([]string, error) {
	prefix := b.keyName + keyEnvPrefix
	params := containers.ListBlobsInput{
		Prefix: &prefix,
	}

	ctx := context.TODO()
	client, err := b.armClient.getContainersClient(ctx)
	if err != nil {
		return nil, err
	}
	resp, err := client.ListBlobs(ctx, b.armClient.storageAccountName, b.containerName, params)
	if err != nil {
		return nil, err
	}

	envs := map[string]struct{}{}
	for _, obj := range resp.Blobs.Blobs {
		key := obj.Name
		if strings.HasPrefix(key, prefix) {
			name := strings.TrimPrefix(key, prefix)
			// we store the state in a key, not a directory
			if strings.Contains(name, "/") {
				continue
			}

			envs[name] = struct{}{}
		}
	}

	result := []string{backend.DefaultStateName}
	for name := range envs {
		result = append(result, name)
	}
	sort.Strings(result[1:])
	return result, nil
}

func (b *Backend) DeleteWorkspace(name string) error {
	if name == backend.DefaultStateName || name == "" {
		return fmt.Errorf("can't delete default state")
	}

	ctx := context.TODO()
	client, err := b.armClient.getBlobClient(ctx)
	if err != nil {
		return err
	}

	if resp, err := client.Delete(ctx, b.armClient.storageAccountName, b.containerName, b.path(name), blobs.DeleteInput{}); err != nil {
		if resp.Response.StatusCode != 404 {
			return err
		}
	}

	return nil
}

func (b *Backend) StateMgr(name string) (statemgr.Full, error) {
	ctx := context.TODO()
	blobClient, err := b.armClient.getBlobClient(ctx)
	if err != nil {
		return nil, err
	}

	client := &RemoteClient{
		giovanniBlobClient: *blobClient,
		containerName:      b.containerName,
		keyName:            b.path(name),
		accountName:        b.accountName,
		snapshot:           b.snapshot,
	}

	stateMgr := &remote.State{Client: client}

	// Grab the value
	if err := stateMgr.RefreshState(); err != nil {
		return nil, err
	}
	//if this isn't the default state name, we need to create the object so
	//it's listed by States.
	if v := stateMgr.State(); v == nil {
		// take a lock on this state while we write it
		lockInfo := statemgr.NewLockInfo()
		lockInfo.Operation = "init"
		lockId, err := client.Lock(lockInfo)
		if err != nil {
			return nil, fmt.Errorf("failed to lock azure state: %s", err)
		}

		// Local helper function so we can call it multiple places
		lockUnlock := func(parent error) error {
			if err := stateMgr.Unlock(lockId); err != nil {
				return fmt.Errorf(strings.TrimSpace(errStateUnlock), lockId, err)
			}
			return parent
		}

		// If we have no state, we have to create an empty state
		if err := stateMgr.WriteState(states.NewState()); err != nil {
			err = lockUnlock(err)
			return nil, err
		}
		if err := stateMgr.PersistState(); err != nil {
			err = lockUnlock(err)
			return nil, err
		}

		// Unlock, the state should now be initialized
		if err := lockUnlock(nil); err != nil {
			return nil, err
		}
	}

	return stateMgr, nil
}

func (b *Backend) client() *RemoteClient {
	return &RemoteClient{}
}

func (b *Backend) path(name string) string {
	if name == backend.DefaultStateName {
		return b.keyName
	}

	return b.keyName + keyEnvPrefix + name
}

const errStateUnlock = `
Error unlocking Azure state. Lock ID: %s

Error: %s

You may have to force-unlock this state in order to use it again.
`
<commit_msg>double check the state wasn't created in the short time prior to grabbing the lock<commit_after>package azure

import (
	"context"
	"fmt"
	"sort"
	"strings"

	"github.com/hashicorp/terraform/backend"
	"github.com/hashicorp/terraform/states"
	"github.com/hashicorp/terraform/states/remote"
	"github.com/hashicorp/terraform/states/statemgr"
	"github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/blobs"
	"github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/containers"
)

const (
	// This will be used as directory name, the odd looking colon is simply to
	// reduce the chance of name conflicts with existing objects.
	keyEnvPrefix = "env:"
)

func (b *Backend) Workspaces() ([]string, error) {
	prefix := b.keyName + keyEnvPrefix
	params := containers.ListBlobsInput{
		Prefix: &prefix,
	}

	ctx := context.TODO()
	client, err := b.armClient.getContainersClient(ctx)
	if err != nil {
		return nil, err
	}
	resp, err := client.ListBlobs(ctx, b.armClient.storageAccountName, b.containerName, params)
	if err != nil {
		return nil, err
	}

	envs := map[string]struct{}{}
	for _, obj := range resp.Blobs.Blobs {
		key := obj.Name
		if strings.HasPrefix(key, prefix) {
			name := strings.TrimPrefix(key, prefix)
			// we store the state in a key, not a directory
			if strings.Contains(name, "/") {
				continue
			}

			envs[name] = struct{}{}
		}
	}

	result := []string{backend.DefaultStateName}
	for name := range envs {
		result = append(result, name)
	}
	sort.Strings(result[1:])
	return result, nil
}

func (b *Backend) DeleteWorkspace(name string) error {
	if name == backend.DefaultStateName || name == "" {
		return fmt.Errorf("can't delete default state")
	}

	ctx := context.TODO()
	client, err := b.armClient.getBlobClient(ctx)
	if err != nil {
		return err
	}

	if resp, err := client.Delete(ctx, b.armClient.storageAccountName, b.containerName, b.path(name), blobs.DeleteInput{}); err != nil {
		if resp.Response.StatusCode != 404 {
			return err
		}
	}

	return nil
}

func (b *Backend) StateMgr(name string) (statemgr.Full, error) {
	ctx := context.TODO()
	blobClient, err := b.armClient.getBlobClient(ctx)
	if err != nil {
		return nil, err
	}

	client := &RemoteClient{
		giovanniBlobClient: *blobClient,
		containerName:      b.containerName,
		keyName:            b.path(name),
		accountName:        b.accountName,
		snapshot:           b.snapshot,
	}

	stateMgr := &remote.State{Client: client}

	// Grab the value
	if err := stateMgr.RefreshState(); err != nil {
		return nil, err
	}
	//if this isn't the default state name, we need to create the object so
	//it's listed by States.
	if v := stateMgr.State(); v == nil {
		// take a lock on this state while we write it
		lockInfo := statemgr.NewLockInfo()
		lockInfo.Operation = "init"
		lockId, err := client.Lock(lockInfo)
		if err != nil {
			return nil, fmt.Errorf("failed to lock azure state: %s", err)
		}

		// Local helper function so we can call it multiple places
		lockUnlock := func(parent error) error {
			if err := stateMgr.Unlock(lockId); err != nil {
				return fmt.Errorf(strings.TrimSpace(errStateUnlock), lockId, err)
			}
			return parent
		}

		// Grab the value
		if err := stateMgr.RefreshState(); err != nil {
			err = lockUnlock(err)
			return nil, err
		}
		//if this isn't the default state name, we need to create the object so
		//it's listed by States.
		if v := stateMgr.State(); v == nil {
			// If we have no state, we have to create an empty state
			if err := stateMgr.WriteState(states.NewState()); err != nil {
				err = lockUnlock(err)
				return nil, err
			}
			if err := stateMgr.PersistState(); err != nil {
				err = lockUnlock(err)
				return nil, err
			}

			// Unlock, the state should now be initialized
			if err := lockUnlock(nil); err != nil {
				return nil, err
			}
		}
	}

	return stateMgr, nil
}

func (b *Backend) client() *RemoteClient {
	return &RemoteClient{}
}

func (b *Backend) path(name string) string {
	if name == backend.DefaultStateName {
		return b.keyName
	}

	return b.keyName + keyEnvPrefix + name
}

const errStateUnlock = `
Error unlocking Azure state. Lock ID: %s

Error: %s

You may have to force-unlock this state in order to use it again.
`
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controller

import (
	"fmt"
	"reflect"
	"strings"
	"time"

	"github.com/golang/glog"

	"github.com/munnerz/kubernetes-ingress/nginx-controller/nginx"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/apis/extensions"
	"k8s.io/kubernetes/pkg/client/cache"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/controller/framework"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/watch"
)

// LoadBalancerController watches Kubernetes API and
// reconfigures NGINX via NGINXController when needed
type LoadBalancerController struct {
	client         *client.Client
	ingController  *framework.Controller
	svcController  *framework.Controller
	endpController *framework.Controller
	ingLister      StoreToIngressLister
	svcLister      cache.StoreToServiceLister
	endpLister     cache.StoreToEndpointsLister
	ingQueue       *taskQueue
	stopCh         chan struct{}
	nginx          *nginx.NGINXController
}

var keyFunc = framework.DeletionHandlingMetaNamespaceKeyFunc

// NewLoadBalancerController creates a controller
func NewLoadBalancerController(kubeClient *client.Client, resyncPeriod time.Duration, namespace string, nginx *nginx.NGINXController) (*LoadBalancerController, error) {
	lbc := LoadBalancerController{
		client: kubeClient,
		stopCh: make(chan struct{}),
		nginx:  nginx,
	}

	lbc.ingQueue = NewTaskQueue(lbc.syncIng)

	ingHandlers := framework.ResourceEventHandlerFuncs{
		AddFunc: func(obj interface{}) {
			addIng := obj.(*extensions.Ingress)
			glog.Infof("Adding Ingress: %v", addIng.Name)
			lbc.ingQueue.enqueue(obj)
		},
		DeleteFunc: func(obj interface{}) {
			remIng := obj.(*extensions.Ingress)
			glog.Infof("Removing Ingress: %v", remIng.Name)
			lbc.ingQueue.enqueue(obj)
		},
		UpdateFunc: func(old, cur interface{}) {
			if !reflect.DeepEqual(old, cur) {
				glog.Infof("Ingress %v changed, syncing",
					cur.(*extensions.Ingress).Name)
				lbc.ingQueue.enqueue(cur)
			}
		},
	}
	lbc.ingLister.Store, lbc.ingController = framework.NewInformer(
		&cache.ListWatch{
			ListFunc:  ingressListFunc(kubeClient, namespace),
			WatchFunc: ingressWatchFunc(kubeClient, namespace),
		},
		&extensions.Ingress{}, resyncPeriod, ingHandlers)

	svcHandlers := framework.ResourceEventHandlerFuncs{
		AddFunc: func(obj interface{}) {
			addSvc := obj.(*api.Service)
			glog.Infof("Adding service: %v", addSvc.Name)
			lbc.enqueueIngressForService(obj)
		},
		DeleteFunc: func(obj interface{}) {
			remSvc := obj.(*api.Service)
			glog.Infof("Removing service: %v", remSvc.Name)
			lbc.enqueueIngressForService(obj)
		},
		UpdateFunc: func(old, cur interface{}) {
			if !reflect.DeepEqual(old, cur) {
				glog.Infof("Service %v changed, syncing",
					cur.(*api.Service).Name)
				lbc.enqueueIngressForService(cur)
			}
		},
	}
	lbc.svcLister.Store, lbc.svcController = framework.NewInformer(
		&cache.ListWatch{
			ListFunc:  serviceListFunc(kubeClient, namespace),
			WatchFunc: serviceWatchFunc(kubeClient, namespace),
		},
		&api.Service{}, resyncPeriod, svcHandlers)

	endpHandlers := framework.ResourceEventHandlerFuncs{
		AddFunc: func(obj interface{}) {
			addEndp := obj.(*api.Endpoints)
			glog.Infof("Adding endpoints: %v", addEndp.Name)
			lbc.enqueueIngressForEndpoints(obj)
		},
		DeleteFunc: func(obj interface{}) {
			remEndp := obj.(*api.Endpoints)
			glog.Infof("Removing endpoints: %v", remEndp.Name)
			lbc.enqueueIngressForEndpoints(obj)
		},
		UpdateFunc: func(old, cur interface{}) {
			if !reflect.DeepEqual(old, cur) {
				glog.Infof("Endpoints %v changed, syncing",
					cur.(*api.Endpoints).Name)
				lbc.enqueueIngressForEndpoints(cur)
			}
		},
	}
	lbc.endpLister.Store, lbc.endpController = framework.NewInformer(
		&cache.ListWatch{
			ListFunc:  endpointsListFunc(kubeClient, namespace),
			WatchFunc: endpointsWatchFunc(kubeClient, namespace),
		},
		&api.Endpoints{}, resyncPeriod, endpHandlers)

	return &lbc, nil
}

// Run starts the loadbalancer controller
func (lbc *LoadBalancerController) Run() {
	go lbc.ingController.Run(lbc.stopCh)
	go lbc.svcController.Run(lbc.stopCh)
	go lbc.endpController.Run(lbc.stopCh)
	go lbc.ingQueue.run(time.Second, lbc.stopCh)
	<-lbc.stopCh
}

func ingressListFunc(c *client.Client, ns string) func(api.ListOptions) (runtime.Object, error) {
	return func(opts api.ListOptions) (runtime.Object, error) {
		return c.Extensions().Ingress(ns).List(opts)
	}
}

func ingressWatchFunc(c *client.Client, ns string) func(options api.ListOptions) (watch.Interface, error) {
	return func(options api.ListOptions) (watch.Interface, error) {
		return c.Extensions().Ingress(ns).Watch(options)
	}
}

func serviceListFunc(c *client.Client, ns string) func(api.ListOptions) (runtime.Object, error) {
	return func(opts api.ListOptions) (runtime.Object, error) {
		return c.Services(ns).List(opts)
	}
}

func serviceWatchFunc(c *client.Client, ns string) func(options api.ListOptions) (watch.Interface, error) {
	return func(options api.ListOptions) (watch.Interface, error) {
		return c.Services(ns).Watch(options)
	}
}

func endpointsListFunc(c *client.Client, ns string) func(api.ListOptions) (runtime.Object, error) {
	return func(opts api.ListOptions) (runtime.Object, error) {
		return c.Endpoints(ns).List(opts)
	}
}

func endpointsWatchFunc(c *client.Client, ns string) func(options api.ListOptions) (watch.Interface, error) {
	return func(options api.ListOptions) (watch.Interface, error) {
		return c.Endpoints(ns).Watch(options)
	}
}

func (lbc *LoadBalancerController) syncIng(key string) {
	glog.Infof("Syncing %v", key)

	obj, ingExists, err := lbc.ingLister.Store.GetByKey(key)
	if err != nil {
		lbc.ingQueue.requeue(key, err)
		return
	}

	// defaut/some-ingress -> default-some-ingress
	name := strings.Replace(key, "/", "-", -1)

	if !ingExists {
		lbc.nginx.DeleteIngress(name)
	} else {
		ing := obj.(*extensions.Ingress)
		lbc.updateNGINX(name, ing)
	}

	lbc.nginx.Reload()
}

func (lbc *LoadBalancerController) enqueueIngressForService(obj interface{}) {
	svc := obj.(*api.Service)
	ings, err := lbc.ingLister.GetServiceIngress(svc)
	if err != nil {
		glog.Infof("ignoring service %v: %v", svc.Name, err)
		return
	}
	for _, ing := range ings {
		lbc.ingQueue.enqueue(&ing)
	}
}

func (lbc *LoadBalancerController) enqueueIngressForEndpoints(obj interface{}) {
	endp := obj.(*api.Endpoints)
	svcKey := endp.GetNamespace() + "/" + endp.GetName()
	svcObj, svcExists, err := lbc.svcLister.Store.GetByKey(svcKey)
	if err != nil {
		glog.Infof("error getting service %v from the cache: %v\n", svcKey, err)
	} else {
		if svcExists && svcObj.(*api.Service).Spec.ClusterIP == "None" {
			lbc.enqueueIngressForService(svcObj)
		}
	}
}

func (lbc *LoadBalancerController) updateNGINX(name string, ing *extensions.Ingress) {
	pems := make(map[string]string)

	for _, tls := range ing.Spec.TLS {
		secretName := tls.SecretName
		secret, err := lbc.client.Secrets(ing.Namespace).Get(secretName)
		if err != nil {
			glog.Warningf("Error retriveing secret %v for ing %v: %v", secretName, ing.Name, err)
			continue
		}
		cert, ok := secret.Data[api.TLSCertKey]
		if !ok {
			glog.Warningf("Secret %v has no private key", secretName)
			continue
		}
		key, ok := secret.Data[api.TLSPrivateKeyKey]
		if !ok {
			glog.Warningf("Secret %v has no cert", secretName)
			continue
		}

		pemFileName := lbc.nginx.AddOrUpdateCertAndKey(secretName, string(cert), string(key))

		for _, host := range tls.Hosts {
			pems[host] = pemFileName
		}
	}

	var upstreams []nginx.Upstream

	for _, rule := range ing.Spec.Rules {
		if rule.IngressRuleValue.HTTP == nil {
			continue
		}

		for _, path := range rule.HTTP.Paths {
			name := ing.Name + "-" + path.Backend.ServiceName
			ups := nginx.NewUpstreamWithDefaultServer(name)

			svcKey := ing.Namespace + "/" + path.Backend.ServiceName
			svcObj, svcExists, err := lbc.svcLister.Store.GetByKey(svcKey)
			if err != nil {
				glog.Infof("error getting service %v from the cache: %v", svcKey, err)
			} else {
				if svcExists {
					svc := svcObj.(*api.Service)
					if svc.Spec.ClusterIP != "None" && svc.Spec.ClusterIP != "" {
						upsServer := nginx.UpstreamServer{Address: svc.Spec.ClusterIP, Port: path.Backend.ServicePort.String()}
						ups.UpstreamServers = []nginx.UpstreamServer{upsServer}
					} else if svc.Spec.ClusterIP == "None" {
						endps, err := lbc.endpLister.GetServiceEndpoints(svc)
						if err != nil {
							glog.Infof("error getting endpoints for service %v from the cache: %v", svc, err)
						} else {
							upsServers := endpointsToUpstreamServers(endps, path.Backend.ServicePort.IntValue())
							if len(upsServers) > 0 {
								ups.UpstreamServers = upsServers
							}
						}
					}
				}
			}

			upstreams = append(upstreams, ups)
		}
	}

	var servers []nginx.Server
	for _, rule := range ing.Spec.Rules {
		server := nginx.Server{Name: rule.Host}

		if pemFile, ok := pems[rule.Host]; ok {
			server.SSL = true
			server.SSLCertificate = pemFile
			server.SSLCertificateKey = pemFile
		}

		var locations []nginx.Location

		for _, path := range rule.HTTP.Paths {
			loc := nginx.Location{Path: path.Path}
			upsName := ing.GetName() + "-" + path.Backend.ServiceName

			for _, ups := range upstreams {
				if upsName == ups.Name {
					loc.Upstream = ups
				}
			}
			locations = append(locations, loc)
		}

		server.Locations = locations
		servers = append(servers, server)
	}

	lbc.nginx.AddOrUpdateIngress(name, nginx.IngressNGINXConfig{Upstreams: upstreams, Servers: servers})
}

func endpointsToUpstreamServers(endps api.Endpoints, servicePort int) []nginx.UpstreamServer {
	var upsServers []nginx.UpstreamServer
	for _, subset := range endps.Subsets {
		for _, port := range subset.Ports {
			if port.Port == servicePort {
				for _, address := range subset.Addresses {
					ups := nginx.UpstreamServer{Address: address.IP, Port: fmt.Sprintf("%v", servicePort)}
					upsServers = append(upsServers, ups)
				}
				break
			}
		}
	}

	return upsServers
}
<commit_msg>Use a more unique name for upstreams<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controller

import (
	"fmt"
	"reflect"
	"strings"
	"time"

	"github.com/golang/glog"

	"github.com/munnerz/kubernetes-ingress/nginx-controller/nginx"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/apis/extensions"
	"k8s.io/kubernetes/pkg/client/cache"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/controller/framework"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/watch"
)

// LoadBalancerController watches Kubernetes API and
// reconfigures NGINX via NGINXController when needed
type LoadBalancerController struct {
	client         *client.Client
	ingController  *framework.Controller
	svcController  *framework.Controller
	endpController *framework.Controller
	ingLister      StoreToIngressLister
	svcLister      cache.StoreToServiceLister
	endpLister     cache.StoreToEndpointsLister
	ingQueue       *taskQueue
	stopCh         chan struct{}
	nginx          *nginx.NGINXController
}

var keyFunc = framework.DeletionHandlingMetaNamespaceKeyFunc

// NewLoadBalancerController creates a controller
func NewLoadBalancerController(kubeClient *client.Client, resyncPeriod time.Duration, namespace string, nginx *nginx.NGINXController) (*LoadBalancerController, error) {
	lbc := LoadBalancerController{
		client: kubeClient,
		stopCh: make(chan struct{}),
		nginx:  nginx,
	}

	lbc.ingQueue = NewTaskQueue(lbc.syncIng)

	ingHandlers := framework.ResourceEventHandlerFuncs{
		AddFunc: func(obj interface{}) {
			addIng := obj.(*extensions.Ingress)
			glog.Infof("Adding Ingress: %v", addIng.Name)
			lbc.ingQueue.enqueue(obj)
		},
		DeleteFunc: func(obj interface{}) {
			remIng := obj.(*extensions.Ingress)
			glog.Infof("Removing Ingress: %v", remIng.Name)
			lbc.ingQueue.enqueue(obj)
		},
		UpdateFunc: func(old, cur interface{}) {
			if !reflect.DeepEqual(old, cur) {
				glog.Infof("Ingress %v changed, syncing",
					cur.(*extensions.Ingress).Name)
				lbc.ingQueue.enqueue(cur)
			}
		},
	}
	lbc.ingLister.Store, lbc.ingController = framework.NewInformer(
		&cache.ListWatch{
			ListFunc:  ingressListFunc(kubeClient, namespace),
			WatchFunc: ingressWatchFunc(kubeClient, namespace),
		},
		&extensions.Ingress{}, resyncPeriod, ingHandlers)

	svcHandlers := framework.ResourceEventHandlerFuncs{
		AddFunc: func(obj interface{}) {
			addSvc := obj.(*api.Service)
			glog.Infof("Adding service: %v", addSvc.Name)
			lbc.enqueueIngressForService(obj)
		},
		DeleteFunc: func(obj interface{}) {
			remSvc := obj.(*api.Service)
			glog.Infof("Removing service: %v", remSvc.Name)
			lbc.enqueueIngressForService(obj)
		},
		UpdateFunc: func(old, cur interface{}) {
			if !reflect.DeepEqual(old, cur) {
				glog.Infof("Service %v changed, syncing",
					cur.(*api.Service).Name)
				lbc.enqueueIngressForService(cur)
			}
		},
	}
	lbc.svcLister.Store, lbc.svcController = framework.NewInformer(
		&cache.ListWatch{
			ListFunc:  serviceListFunc(kubeClient, namespace),
			WatchFunc: serviceWatchFunc(kubeClient, namespace),
		},
		&api.Service{}, resyncPeriod, svcHandlers)

	endpHandlers := framework.ResourceEventHandlerFuncs{
		AddFunc: func(obj interface{}) {
			addEndp := obj.(*api.Endpoints)
			glog.Infof("Adding endpoints: %v", addEndp.Name)
			lbc.enqueueIngressForEndpoints(obj)
		},
		DeleteFunc: func(obj interface{}) {
			remEndp := obj.(*api.Endpoints)
			glog.Infof("Removing endpoints: %v", remEndp.Name)
			lbc.enqueueIngressForEndpoints(obj)
		},
		UpdateFunc: func(old, cur interface{}) {
			if !reflect.DeepEqual(old, cur) {
				glog.Infof("Endpoints %v changed, syncing",
					cur.(*api.Endpoints).Name)
				lbc.enqueueIngressForEndpoints(cur)
			}
		},
	}
	lbc.endpLister.Store, lbc.endpController = framework.NewInformer(
		&cache.ListWatch{
			ListFunc:  endpointsListFunc(kubeClient, namespace),
			WatchFunc: endpointsWatchFunc(kubeClient, namespace),
		},
		&api.Endpoints{}, resyncPeriod, endpHandlers)

	return &lbc, nil
}

// Run starts the loadbalancer controller
func (lbc *LoadBalancerController) Run() {
	go lbc.ingController.Run(lbc.stopCh)
	go lbc.svcController.Run(lbc.stopCh)
	go lbc.endpController.Run(lbc.stopCh)
	go lbc.ingQueue.run(time.Second, lbc.stopCh)
	<-lbc.stopCh
}

func ingressListFunc(c *client.Client, ns string) func(api.ListOptions) (runtime.Object, error) {
	return func(opts api.ListOptions) (runtime.Object, error) {
		return c.Extensions().Ingress(ns).List(opts)
	}
}

func ingressWatchFunc(c *client.Client, ns string) func(options api.ListOptions) (watch.Interface, error) {
	return func(options api.ListOptions) (watch.Interface, error) {
		return c.Extensions().Ingress(ns).Watch(options)
	}
}

func serviceListFunc(c *client.Client, ns string) func(api.ListOptions) (runtime.Object, error) {
	return func(opts api.ListOptions) (runtime.Object, error) {
		return c.Services(ns).List(opts)
	}
}

func serviceWatchFunc(c *client.Client, ns string) func(options api.ListOptions) (watch.Interface, error) {
	return func(options api.ListOptions) (watch.Interface, error) {
		return c.Services(ns).Watch(options)
	}
}

func endpointsListFunc(c *client.Client, ns string) func(api.ListOptions) (runtime.Object, error) {
	return func(opts api.ListOptions) (runtime.Object, error) {
		return c.Endpoints(ns).List(opts)
	}
}

func endpointsWatchFunc(c *client.Client, ns string) func(options api.ListOptions) (watch.Interface, error) {
	return func(options api.ListOptions) (watch.Interface, error) {
		return c.Endpoints(ns).Watch(options)
	}
}

func (lbc *LoadBalancerController) syncIng(key string) {
	glog.Infof("Syncing %v", key)

	obj, ingExists, err := lbc.ingLister.Store.GetByKey(key)
	if err != nil {
		lbc.ingQueue.requeue(key, err)
		return
	}

	// defaut/some-ingress -> default-some-ingress
	name := strings.Replace(key, "/", "-", -1)

	if !ingExists {
		lbc.nginx.DeleteIngress(name)
	} else {
		ing := obj.(*extensions.Ingress)
		lbc.updateNGINX(name, ing)
	}

	lbc.nginx.Reload()
}

func (lbc *LoadBalancerController) enqueueIngressForService(obj interface{}) {
	svc := obj.(*api.Service)
	ings, err := lbc.ingLister.GetServiceIngress(svc)
	if err != nil {
		glog.Infof("ignoring service %v: %v", svc.Name, err)
		return
	}
	for _, ing := range ings {
		lbc.ingQueue.enqueue(&ing)
	}
}

func (lbc *LoadBalancerController) enqueueIngressForEndpoints(obj interface{}) {
	endp := obj.(*api.Endpoints)
	svcKey := endp.GetNamespace() + "/" + endp.GetName()
	svcObj, svcExists, err := lbc.svcLister.Store.GetByKey(svcKey)
	if err != nil {
		glog.Infof("error getting service %v from the cache: %v\n", svcKey, err)
	} else {
		if svcExists && svcObj.(*api.Service).Spec.ClusterIP == "None" {
			lbc.enqueueIngressForService(svcObj)
		}
	}
}

func (lbc *LoadBalancerController) updateNGINX(name string, ing *extensions.Ingress) {
	pems := make(map[string]string)

	for _, tls := range ing.Spec.TLS {
		secretName := tls.SecretName
		secret, err := lbc.client.Secrets(ing.Namespace).Get(secretName)
		if err != nil {
			glog.Warningf("Error retriveing secret %v for ing %v: %v", secretName, ing.Name, err)
			continue
		}
		cert, ok := secret.Data[api.TLSCertKey]
		if !ok {
			glog.Warningf("Secret %v has no private key", secretName)
			continue
		}
		key, ok := secret.Data[api.TLSPrivateKeyKey]
		if !ok {
			glog.Warningf("Secret %v has no cert", secretName)
			continue
		}

		pemFileName := lbc.nginx.AddOrUpdateCertAndKey(secretName, string(cert), string(key))

		for _, host := range tls.Hosts {
			pems[host] = pemFileName
		}
	}

	var upstreams []nginx.Upstream

	for _, rule := range ing.Spec.Rules {
		if rule.IngressRuleValue.HTTP == nil {
			continue
		}

		for _, path := range rule.HTTP.Paths {
			name := path.Backend.ServiceName + "-Ingress-" + rule.Host + "-" + strings.Replace(path.Path, "/", "-", -1)
			ups := nginx.NewUpstreamWithDefaultServer(name)

			svcKey := ing.Namespace + "/" + path.Backend.ServiceName
			svcObj, svcExists, err := lbc.svcLister.Store.GetByKey(svcKey)
			if err != nil {
				glog.Infof("error getting service %v from the cache: %v", svcKey, err)
			} else {
				if svcExists {
					svc := svcObj.(*api.Service)
					if svc.Spec.ClusterIP != "None" && svc.Spec.ClusterIP != "" {
						upsServer := nginx.UpstreamServer{Address: svc.Spec.ClusterIP, Port: path.Backend.ServicePort.String()}
						ups.UpstreamServers = []nginx.UpstreamServer{upsServer}
					} else if svc.Spec.ClusterIP == "None" {
						endps, err := lbc.endpLister.GetServiceEndpoints(svc)
						if err != nil {
							glog.Infof("error getting endpoints for service %v from the cache: %v", svc, err)
						} else {
							upsServers := endpointsToUpstreamServers(endps, path.Backend.ServicePort.IntValue())
							if len(upsServers) > 0 {
								ups.UpstreamServers = upsServers
							}
						}
					}
				}
			}

			upstreams = append(upstreams, ups)
		}
	}

	var servers []nginx.Server
	for _, rule := range ing.Spec.Rules {
		server := nginx.Server{Name: rule.Host}

		if pemFile, ok := pems[rule.Host]; ok {
			server.SSL = true
			server.SSLCertificate = pemFile
			server.SSLCertificateKey = pemFile
		}

		var locations []nginx.Location

		for _, path := range rule.HTTP.Paths {
			loc := nginx.Location{Path: path.Path}
			upsName := ing.GetName() + "-" + path.Backend.ServiceName

			for _, ups := range upstreams {
				if upsName == ups.Name {
					loc.Upstream = ups
				}
			}
			locations = append(locations, loc)
		}

		server.Locations = locations
		servers = append(servers, server)
	}

	lbc.nginx.AddOrUpdateIngress(name, nginx.IngressNGINXConfig{Upstreams: upstreams, Servers: servers})
}

func endpointsToUpstreamServers(endps api.Endpoints, servicePort int) []nginx.UpstreamServer {
	var upsServers []nginx.UpstreamServer
	for _, subset := range endps.Subsets {
		for _, port := range subset.Ports {
			if port.Port == servicePort {
				for _, address := range subset.Addresses {
					ups := nginx.UpstreamServer{Address: address.IP, Port: fmt.Sprintf("%v", servicePort)}
					upsServers = append(upsServers, ups)
				}
				break
			}
		}
	}

	return upsServers
}
<|endoftext|>


<commit_before>// The dt package describes the types of Lua API items.
package dt

import (
	"bytes"
	"encoding/json"
	"sort"
	"strings"
)

func marshal(v interface{}) (b []byte, err error) {
	var buf bytes.Buffer
	j := json.NewEncoder(&buf)
	j.SetEscapeHTML(false)
	if err = j.Encode(v); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

// Type describes an API type.
type Type interface {
	// String returns a readable representation of the type.
	String() string
	t()
}

// Prim is a Type that indicates the name of some defined type.
type Prim string

func (t Prim) t() {}
func (t Prim) String() string {
	return string(t)
}
func (t Prim) MarshalJSON() (b []byte, err error) {
	v := struct {
		Kind string
		Sig  string
		Type string
	}{"primitive", t.String(), string(t)}
	return marshal(v)
}

// Function is a Type that indicates the signature of a function type.
type Function struct {
	// Parameters are the values received by the function.
	Parameters Parameters `json:",omitempty"`
	// Returns are the values returned by the function.
	Returns Parameters `json:",omitempty"`
}

func (t Function) t() {}
func (t Function) String() string {
	var s strings.Builder
	s.WriteByte('(')
	for i, v := range t.Parameters {
		if i > 0 {
			s.WriteString(", ")
		}
		if v.Name != "" {
			s.WriteString(v.Name)
			s.WriteString(": ")
		}
		s.WriteString(v.Type.String())
	}
	for i, v := range t.Returns {
		if i > 0 {
			s.WriteString(", ")
		}
		if v.Name != "" {
			s.WriteString(v.Name)
			s.WriteString(": ")
		}
		s.WriteString(v.Type.String())
	}
	s.WriteByte(')')
	return s.String()
}
func (t Function) MarshalJSON() (b []byte, err error) {
	v := struct {
		Kind       string
		Sig        string
		Parameters Parameters `json:",omitempty"`
		Returns    Parameters `json:",omitempty"`
	}{"function", t.String(), t.Parameters, t.Returns}
	return marshal(v)
}

// Parameter describes a function parameter.
type Parameter struct {
	// Name is the name of the parameter.
	Name string `json:",omitempty"`
	// Type is the type of the parameter.
	Type Type
	// Default is the default value if the type is optional.
	Default string `json:",omitempty"`
	// Enum contains literal values that can be passed to the parameter.
	Enums Enums `json:",omitempty"`
}

// Parameters is a list of function parameters.
type Parameters = []Parameter

// Enums is a list of literal values that can be passed to a function parameter.
type Enums = []string

// Array is a Type that indicates an array of elements of some type.
type Array struct {
	T Type
}

func (t Array) t() {}
func (t Array) String() string {
	return "{" + t.T.String() + "}"
}
func (t Array) MarshalJSON() (b []byte, err error) {
	v := struct {
		Kind string
		Sig  string
		Type Type
	}{"array", t.String(), t.T}
	return marshal(v)
}

// Or is a Type that indicates a union of two or more types.
type Or []Type

func (t Or) t() {}
func (t Or) String() string {
	var s strings.Builder
	prim := true
	for _, v := range t {
		if _, ok := v.(Prim); !ok {
			prim = false
			break
		}
	}
	if prim {
		for i, v := range t {
			if i > 0 {
				s.WriteString(" | ")
			}
			s.WriteString(v.String())
		}
	} else {
		for i, v := range t {
			if i > 0 {
				s.WriteByte('|')
			}
			s.WriteString(v.String())
		}
	}
	return s.String()
}
func (t Or) MarshalJSON() (b []byte, err error) {
	v := struct {
		Kind  string
		Sig   string
		Types []Type
	}{"or", t.String(), t}
	return marshal(v)
}

// Optional is a Type that indicates a type of T or nil (shorthand for T | nil).
type Optional struct {
	T Type
}

func (t Optional) t() {}
func (t Optional) String() string {
	return t.T.String() + "?"
}
func (t Optional) MarshalJSON() (b []byte, err error) {
	v := struct {
		Kind string
		Sig  string
		Type Type
	}{"optional", t.String(), t.T}
	return marshal(v)
}

// Group is a Type that ensures the inner type is grouped unambiguously.
type Group struct {
	T Type
}

func (t Group) t() {}
func (t Group) String() string {
	return "(" + t.T.String() + ")"
}
func (t Group) MarshalJSON() (b []byte, err error) {
	v := struct {
		Kind string
		Sig  string
		Type Type
	}{"group", t.String(), t.T}
	return marshal(v)
}

// Struct is a Type that indicates a table with a number of named fields.
type Struct map[string]Type

func (t Struct) t() {}
func (t Struct) String() string {
	f := make([]string, 0, 16)
	for k := range t {
		f = append(f, k)
	}
	var s strings.Builder
	s.WriteByte('{')
	sort.Strings(f)
	for i, k := range f {
		if i > 0 {
			s.WriteString(", ")
		}
		v := t[k]
		s.WriteString(k)
		s.WriteString(": ")
		s.WriteString(v.String())
	}
	s.WriteByte('}')
	return s.String()
}
func (t Struct) MarshalJSON() (b []byte, err error) {
	v := struct {
		Kind   string
		Sig    string
		Fields map[string]Type
	}{"struct", t.String(), t}
	return marshal(v)
}

// Map is a Type that indicates a table where each element maps a key to a
// value.
type Map struct {
	K Type
	V Type
}

func (t Map) t() {}
func (t Map) String() string {
	return "{[" + t.K.String() + "]: " + t.V.String() + "}"
}
func (t Map) MarshalJSON() (b []byte, err error) {
	v := struct {
		Kind  string
		Sig   string
		Key   Type
		Value Type
	}{"map", t.String(), t.K, t.V}
	return marshal(v)
}

// Dictionary is a Type that indicates a table where each element maps a string
// to a value.
type Dictionary struct {
	V Type
}

func (t Dictionary) t() {}
func (t Dictionary) String() string {
	return "{[string]: " + t.V.String() + "}"
}
func (t Dictionary) MarshalJSON() (b []byte, err error) {
	v := struct {
		Kind  string
		Sig   string
		Value Type
	}{"dictionary", t.String(), t.V}
	return marshal(v)
}

// Table is a Type that indicates a table with both a map part and a struct
// part.
type Table struct {
	Map    Map
	Struct Struct
}

func (t Table) t() {}
func (t Table) String() string {
	f := make([]string, 0, 16)
	for k := range t.Struct {
		f = append(f, k)
	}
	sort.Strings(f)
	var s strings.Builder
	s.WriteString("{[")
	s.WriteString(t.Map.K.String())
	s.WriteString("]: ")
	s.WriteString(t.Map.V.String())
	for _, k := range f {
		s.WriteString(", ")
		v := t.Struct[k]
		s.WriteString(k)
		s.WriteString(": ")
		s.WriteString(v.String())
	}
	s.WriteByte('}')
	return s.String()
}
func (t Table) MarshalJSON() (b []byte, err error) {
	v := struct {
		Kind   string
		Sig    string
		Key    Type
		Value  Type
		Fields map[string]Type
	}{"table", t.String(), t.Map.K, t.Map.V, t.Struct}
	return marshal(v)
}

// MultiFunctionType is a Type that indicates a function with multiple
// signatures.
type MultiFunctionType struct{}

func (MultiFunctionType) t() {}
func (MultiFunctionType) String() string {
	return "(...) -> (...)"
}
func (t MultiFunctionType) MarshalJSON() (b []byte, err error) {
	v := struct {
		Kind string
		Sig  string
	}{"functions", t.String()}
	return marshal(v)
}
<commit_msg>Fix incorrect syntax of Function string.<commit_after>// The dt package describes the types of Lua API items.
package dt

import (
	"bytes"
	"encoding/json"
	"sort"
	"strings"
)

func marshal(v interface{}) (b []byte, err error) {
	var buf bytes.Buffer
	j := json.NewEncoder(&buf)
	j.SetEscapeHTML(false)
	if err = j.Encode(v); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

// Type describes an API type.
type Type interface {
	// String returns a readable representation of the type.
	String() string
	t()
}

// Prim is a Type that indicates the name of some defined type.
type Prim string

func (t Prim) t() {}
func (t Prim) String() string {
	return string(t)
}
func (t Prim) MarshalJSON() (b []byte, err error) {
	v := struct {
		Kind string
		Sig  string
		Type string
	}{"primitive", t.String(), string(t)}
	return marshal(v)
}

// Function is a Type that indicates the signature of a function type.
type Function struct {
	// Parameters are the values received by the function.
	Parameters Parameters `json:",omitempty"`
	// Returns are the values returned by the function.
	Returns Parameters `json:",omitempty"`
}

func (t Function) t() {}
func (t Function) String() string {
	var s strings.Builder
	s.WriteByte('(')
	for i, v := range t.Parameters {
		if i > 0 {
			s.WriteString(", ")
		}
		if v.Name != "" {
			s.WriteString(v.Name)
			s.WriteString(": ")
		}
		s.WriteString(v.Type.String())
	}
	s.WriteString(") -> (")
	for i, v := range t.Returns {
		if i > 0 {
			s.WriteString(", ")
		}
		if v.Name != "" {
			s.WriteString(v.Name)
			s.WriteString(": ")
		}
		s.WriteString(v.Type.String())
	}
	s.WriteByte(')')
	return s.String()
}
func (t Function) MarshalJSON() (b []byte, err error) {
	v := struct {
		Kind       string
		Sig        string
		Parameters Parameters `json:",omitempty"`
		Returns    Parameters `json:",omitempty"`
	}{"function", t.String(), t.Parameters, t.Returns}
	return marshal(v)
}

// Parameter describes a function parameter.
type Parameter struct {
	// Name is the name of the parameter.
	Name string `json:",omitempty"`
	// Type is the type of the parameter.
	Type Type
	// Default is the default value if the type is optional.
	Default string `json:",omitempty"`
	// Enum contains literal values that can be passed to the parameter.
	Enums Enums `json:",omitempty"`
}

// Parameters is a list of function parameters.
type Parameters = []Parameter

// Enums is a list of literal values that can be passed to a function parameter.
type Enums = []string

// Array is a Type that indicates an array of elements of some type.
type Array struct {
	T Type
}

func (t Array) t() {}
func (t Array) String() string {
	return "{" + t.T.String() + "}"
}
func (t Array) MarshalJSON() (b []byte, err error) {
	v := struct {
		Kind string
		Sig  string
		Type Type
	}{"array", t.String(), t.T}
	return marshal(v)
}

// Or is a Type that indicates a union of two or more types.
type Or []Type

func (t Or) t() {}
func (t Or) String() string {
	var s strings.Builder
	prim := true
	for _, v := range t {
		if _, ok := v.(Prim); !ok {
			prim = false
			break
		}
	}
	if prim {
		for i, v := range t {
			if i > 0 {
				s.WriteString(" | ")
			}
			s.WriteString(v.String())
		}
	} else {
		for i, v := range t {
			if i > 0 {
				s.WriteByte('|')
			}
			s.WriteString(v.String())
		}
	}
	return s.String()
}
func (t Or) MarshalJSON() (b []byte, err error) {
	v := struct {
		Kind  string
		Sig   string
		Types []Type
	}{"or", t.String(), t}
	return marshal(v)
}

// Optional is a Type that indicates a type of T or nil (shorthand for T | nil).
type Optional struct {
	T Type
}

func (t Optional) t() {}
func (t Optional) String() string {
	return t.T.String() + "?"
}
func (t Optional) MarshalJSON() (b []byte, err error) {
	v := struct {
		Kind string
		Sig  string
		Type Type
	}{"optional", t.String(), t.T}
	return marshal(v)
}

// Group is a Type that ensures the inner type is grouped unambiguously.
type Group struct {
	T Type
}

func (t Group) t() {}
func (t Group) String() string {
	return "(" + t.T.String() + ")"
}
func (t Group) MarshalJSON() (b []byte, err error) {
	v := struct {
		Kind string
		Sig  string
		Type Type
	}{"group", t.String(), t.T}
	return marshal(v)
}

// Struct is a Type that indicates a table with a number of named fields.
type Struct map[string]Type

func (t Struct) t() {}
func (t Struct) String() string {
	f := make([]string, 0, 16)
	for k := range t {
		f = append(f, k)
	}
	var s strings.Builder
	s.WriteByte('{')
	sort.Strings(f)
	for i, k := range f {
		if i > 0 {
			s.WriteString(", ")
		}
		v := t[k]
		s.WriteString(k)
		s.WriteString(": ")
		s.WriteString(v.String())
	}
	s.WriteByte('}')
	return s.String()
}
func (t Struct) MarshalJSON() (b []byte, err error) {
	v := struct {
		Kind   string
		Sig    string
		Fields map[string]Type
	}{"struct", t.String(), t}
	return marshal(v)
}

// Map is a Type that indicates a table where each element maps a key to a
// value.
type Map struct {
	K Type
	V Type
}

func (t Map) t() {}
func (t Map) String() string {
	return "{[" + t.K.String() + "]: " + t.V.String() + "}"
}
func (t Map) MarshalJSON() (b []byte, err error) {
	v := struct {
		Kind  string
		Sig   string
		Key   Type
		Value Type
	}{"map", t.String(), t.K, t.V}
	return marshal(v)
}

// Dictionary is a Type that indicates a table where each element maps a string
// to a value.
type Dictionary struct {
	V Type
}

func (t Dictionary) t() {}
func (t Dictionary) String() string {
	return "{[string]: " + t.V.String() + "}"
}
func (t Dictionary) MarshalJSON() (b []byte, err error) {
	v := struct {
		Kind  string
		Sig   string
		Value Type
	}{"dictionary", t.String(), t.V}
	return marshal(v)
}

// Table is a Type that indicates a table with both a map part and a struct
// part.
type Table struct {
	Map    Map
	Struct Struct
}

func (t Table) t() {}
func (t Table) String() string {
	f := make([]string, 0, 16)
	for k := range t.Struct {
		f = append(f, k)
	}
	sort.Strings(f)
	var s strings.Builder
	s.WriteString("{[")
	s.WriteString(t.Map.K.String())
	s.WriteString("]: ")
	s.WriteString(t.Map.V.String())
	for _, k := range f {
		s.WriteString(", ")
		v := t.Struct[k]
		s.WriteString(k)
		s.WriteString(": ")
		s.WriteString(v.String())
	}
	s.WriteByte('}')
	return s.String()
}
func (t Table) MarshalJSON() (b []byte, err error) {
	v := struct {
		Kind   string
		Sig    string
		Key    Type
		Value  Type
		Fields map[string]Type
	}{"table", t.String(), t.Map.K, t.Map.V, t.Struct}
	return marshal(v)
}

// MultiFunctionType is a Type that indicates a function with multiple
// signatures.
type MultiFunctionType struct{}

func (MultiFunctionType) t() {}
func (MultiFunctionType) String() string {
	return "(...) -> (...)"
}
func (t MultiFunctionType) MarshalJSON() (b []byte, err error) {
	v := struct {
		Kind string
		Sig  string
	}{"functions", t.String()}
	return marshal(v)
}
<|endoftext|>


<commit_before>/*
 * Copyright 2017 Dgraph Labs, Inc. and Contributors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package badger

import (
	"bufio"
	"bytes"
	"context"
	"encoding/binary"
	"io"

	"github.com/dgraph-io/badger/pb"
	"github.com/dgraph-io/badger/y"
)

// Backup is a wrapper function over Stream.Backup to generate full and incremental backups of the
// DB. For more control over how many goroutines are used to generate the backup, or if you wish to
// backup only a certain range of keys, use Stream.Backup directly.
func (db *DB) Backup(w io.Writer, since uint64) (uint64, error) {
	stream := db.NewStream()
	stream.LogPrefix = "DB.Backup"
	return stream.Backup(w, since)
}

// Backup dumps a protobuf-encoded list of all entries in the database into the
// given writer, that are newer than the specified version. It returns a
// timestamp indicating when the entries were dumped which can be passed into a
// later invocation to generate an incremental dump, of entries that have been
// added/modified since the last invocation of Stream.Backup().
//
// This can be used to backup the data in a database at a given point in time.
func (stream *Stream) Backup(w io.Writer, since uint64) (uint64, error) {
	stream.KeyToList = func(key []byte, itr *Iterator) (*pb.KVList, error) {
		list := &pb.KVList{}
		for ; itr.Valid(); itr.Next() {
			item := itr.Item()
			if !bytes.Equal(item.Key(), key) {
				return list, nil
			}
			if item.Version() < since {
				// Ignore versions less than given timestamp, or skip older
				// versions of the given key.
				return list, nil
			}

			var valCopy []byte
			if !item.IsDeletedOrExpired() {
				// No need to copy value, if item is deleted or expired.
				var err error
				valCopy, err = item.ValueCopy(nil)
				if err != nil {
					stream.db.opt.Errorf("Key [%x, %d]. Error while fetching value [%v]\n",
						item.Key(), item.Version(), err)
					return nil, err
				}
			}

			// clear txn bits
			meta := item.meta &^ (bitTxn | bitFinTxn)
			kv := &pb.KV{
				Key:       item.KeyCopy(nil),
				Value:     valCopy,
				UserMeta:  []byte{item.UserMeta()},
				Version:   item.Version(),
				ExpiresAt: item.ExpiresAt(),
				Meta:      []byte{meta},
			}
			list.Kv = append(list.Kv, kv)

			switch {
			case item.DiscardEarlierVersions():
				// If we need to discard earlier versions of this item, add a delete
				// marker just below the current version.
				list.Kv = append(list.Kv, &pb.KV{
					Key:     item.KeyCopy(nil),
					Version: item.Version() - 1,
					Meta:    []byte{bitDelete},
				})
				return list, nil

			case item.IsDeletedOrExpired():
				return list, nil
			}
		}
		return list, nil
	}

	var maxVersion uint64
	stream.Send = func(list *pb.KVList) error {
		for _, kv := range list.Kv {
			if maxVersion < kv.Version {
				maxVersion = kv.Version
			}
		}
		return writeTo(list, w)
	}

	if err := stream.Orchestrate(context.Background()); err != nil {
		return 0, err
	}
	return maxVersion, nil
}

func writeTo(list *pb.KVList, w io.Writer) error {
	if err := binary.Write(w, binary.LittleEndian, uint64(list.Size())); err != nil {
		return err
	}
	buf, err := list.Marshal()
	if err != nil {
		return err
	}
	_, err = w.Write(buf)
	return err
}

type loader struct {
	db       *DB
	throttle *y.Throttle
	entries  []*Entry
}

func (db *DB) newLoader(maxPendingWrites int) *loader {
	return &loader{
		db:       db,
		throttle: y.NewThrottle(maxPendingWrites),
	}
}

func (l *loader) set(kv *pb.KV) error {
	var userMeta, meta byte
	if len(kv.UserMeta) > 0 {
		userMeta = kv.UserMeta[0]
	}
	if len(kv.Meta) > 0 {
		meta = kv.Meta[0]
	}

	l.entries = append(l.entries, &Entry{
		Key:       y.KeyWithTs(kv.Key, kv.Version),
		Value:     kv.Value,
		UserMeta:  userMeta,
		ExpiresAt: kv.ExpiresAt,
		meta:      meta,
	})
	if len(l.entries) >= 1000 {
		return l.send()
	}
	return nil
}

func (l *loader) send() error {
	if err := l.throttle.Do(); err != nil {
		return err
	}
	if err := l.db.batchSetAsync(l.entries, func(err error) {
		l.throttle.Done(err)
	}); err != nil {
		return err
	}

	l.entries = make([]*Entry, 0, 1000)
	return nil
}

func (l *loader) finish() error {
	if len(l.entries) > 0 {
		if err := l.send(); err != nil {
			return err
		}
	}
	return l.throttle.Finish()
}

// Load reads a protobuf-encoded list of all entries from a reader and writes
// them to the database. This can be used to restore the database from a backup
// made by calling DB.Backup().
//
// DB.Load() should be called on a database that is not running any other
// concurrent transactions while it is running.
func (db *DB) Load(r io.Reader, maxPendingWrites int) error {
	br := bufio.NewReaderSize(r, 16<<10)
	unmarshalBuf := make([]byte, 1<<10)

	ldr := db.newLoader(maxPendingWrites)
	for {
		var sz uint64
		err := binary.Read(br, binary.LittleEndian, &sz)
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}

		if cap(unmarshalBuf) < int(sz) {
			unmarshalBuf = make([]byte, sz)
		}

		if _, err = io.ReadFull(br, unmarshalBuf[:sz]); err != nil {
			return err
		}

		list := &pb.KVList{}
		if err := list.Unmarshal(unmarshalBuf[:sz]); err != nil {
			return err
		}

		for _, kv := range list.Kv {
			if err := ldr.set(kv); err != nil {
				return err
			}

			// Update nextTxnTs, memtable stores this
			// timestamp in badger head when flushed.
			if kv.Version >= db.orc.nextTxnTs {
				db.orc.nextTxnTs = kv.Version + 1
			}
		}
	}

	if err := ldr.finish(); err != nil {
		return err
	}
	db.orc.txnMark.Done(db.orc.nextTxnTs - 1)
	return nil
}
<commit_msg>Export backup loader interface (#901)<commit_after>/*
 * Copyright 2017 Dgraph Labs, Inc. and Contributors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package badger

import (
	"bufio"
	"bytes"
	"context"
	"encoding/binary"
	"io"

	"github.com/dgraph-io/badger/pb"
	"github.com/dgraph-io/badger/y"
)

// Backup is a wrapper function over Stream.Backup to generate full and incremental backups of the
// DB. For more control over how many goroutines are used to generate the backup, or if you wish to
// backup only a certain range of keys, use Stream.Backup directly.
func (db *DB) Backup(w io.Writer, since uint64) (uint64, error) {
	stream := db.NewStream()
	stream.LogPrefix = "DB.Backup"
	return stream.Backup(w, since)
}

// Backup dumps a protobuf-encoded list of all entries in the database into the
// given writer, that are newer than the specified version. It returns a
// timestamp indicating when the entries were dumped which can be passed into a
// later invocation to generate an incremental dump, of entries that have been
// added/modified since the last invocation of Stream.Backup().
//
// This can be used to backup the data in a database at a given point in time.
func (stream *Stream) Backup(w io.Writer, since uint64) (uint64, error) {
	stream.KeyToList = func(key []byte, itr *Iterator) (*pb.KVList, error) {
		list := &pb.KVList{}
		for ; itr.Valid(); itr.Next() {
			item := itr.Item()
			if !bytes.Equal(item.Key(), key) {
				return list, nil
			}
			if item.Version() < since {
				// Ignore versions less than given timestamp, or skip older
				// versions of the given key.
				return list, nil
			}

			var valCopy []byte
			if !item.IsDeletedOrExpired() {
				// No need to copy value, if item is deleted or expired.
				var err error
				valCopy, err = item.ValueCopy(nil)
				if err != nil {
					stream.db.opt.Errorf("Key [%x, %d]. Error while fetching value [%v]\n",
						item.Key(), item.Version(), err)
					return nil, err
				}
			}

			// clear txn bits
			meta := item.meta &^ (bitTxn | bitFinTxn)
			kv := &pb.KV{
				Key:       item.KeyCopy(nil),
				Value:     valCopy,
				UserMeta:  []byte{item.UserMeta()},
				Version:   item.Version(),
				ExpiresAt: item.ExpiresAt(),
				Meta:      []byte{meta},
			}
			list.Kv = append(list.Kv, kv)

			switch {
			case item.DiscardEarlierVersions():
				// If we need to discard earlier versions of this item, add a delete
				// marker just below the current version.
				list.Kv = append(list.Kv, &pb.KV{
					Key:     item.KeyCopy(nil),
					Version: item.Version() - 1,
					Meta:    []byte{bitDelete},
				})
				return list, nil

			case item.IsDeletedOrExpired():
				return list, nil
			}
		}
		return list, nil
	}

	var maxVersion uint64
	stream.Send = func(list *pb.KVList) error {
		for _, kv := range list.Kv {
			if maxVersion < kv.Version {
				maxVersion = kv.Version
			}
		}
		return writeTo(list, w)
	}

	if err := stream.Orchestrate(context.Background()); err != nil {
		return 0, err
	}
	return maxVersion, nil
}

func writeTo(list *pb.KVList, w io.Writer) error {
	if err := binary.Write(w, binary.LittleEndian, uint64(list.Size())); err != nil {
		return err
	}
	buf, err := list.Marshal()
	if err != nil {
		return err
	}
	_, err = w.Write(buf)
	return err
}

// KVLoader is used to write KVList objects in to badger. It can be used to restore a backup.
type KVLoader struct {
	db       *DB
	throttle *y.Throttle
	entries  []*Entry
}

// NewKVLoader returns a new instance of KVLoader.
func (db *DB) NewKVLoader(maxPendingWrites int) *KVLoader {
	return &KVLoader{
		db:       db,
		throttle: y.NewThrottle(maxPendingWrites),
	}
}

// Set writes the key-value pair to the database.
func (l *KVLoader) Set(kv *pb.KV) error {
	var userMeta, meta byte
	if len(kv.UserMeta) > 0 {
		userMeta = kv.UserMeta[0]
	}
	if len(kv.Meta) > 0 {
		meta = kv.Meta[0]
	}

	l.entries = append(l.entries, &Entry{
		Key:       y.KeyWithTs(kv.Key, kv.Version),
		Value:     kv.Value,
		UserMeta:  userMeta,
		ExpiresAt: kv.ExpiresAt,
		meta:      meta,
	})
	if len(l.entries) >= 1000 {
		return l.send()
	}
	return nil
}

func (l *KVLoader) send() error {
	if err := l.throttle.Do(); err != nil {
		return err
	}
	if err := l.db.batchSetAsync(l.entries, func(err error) {
		l.throttle.Done(err)
	}); err != nil {
		return err
	}

	l.entries = make([]*Entry, 0, 1000)
	return nil
}

// Finish is meant to be called after all the key-value pairs have been loaded.
func (l *KVLoader) Finish() error {
	if len(l.entries) > 0 {
		if err := l.send(); err != nil {
			return err
		}
	}
	return l.throttle.Finish()
}

// Load reads a protobuf-encoded list of all entries from a reader and writes
// them to the database. This can be used to restore the database from a backup
// made by calling DB.Backup(). If more complex logic is needed to restore a badger
// backup, the KVLoader interface should be used instead.
//
// DB.Load() should be called on a database that is not running any other
// concurrent transactions while it is running.
func (db *DB) Load(r io.Reader, maxPendingWrites int) error {
	br := bufio.NewReaderSize(r, 16<<10)
	unmarshalBuf := make([]byte, 1<<10)

	ldr := db.NewKVLoader(maxPendingWrites)
	for {
		var sz uint64
		err := binary.Read(br, binary.LittleEndian, &sz)
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}

		if cap(unmarshalBuf) < int(sz) {
			unmarshalBuf = make([]byte, sz)
		}

		if _, err = io.ReadFull(br, unmarshalBuf[:sz]); err != nil {
			return err
		}

		list := &pb.KVList{}
		if err := list.Unmarshal(unmarshalBuf[:sz]); err != nil {
			return err
		}

		for _, kv := range list.Kv {
			if err := ldr.Set(kv); err != nil {
				return err
			}

			// Update nextTxnTs, memtable stores this
			// timestamp in badger head when flushed.
			if kv.Version >= db.orc.nextTxnTs {
				db.orc.nextTxnTs = kv.Version + 1
			}
		}
	}

	if err := ldr.Finish(); err != nil {
		return err
	}
	db.orc.txnMark.Done(db.orc.nextTxnTs - 1)
	return nil
}
<|endoftext|>


<commit_before>package manta

// A fieldpath, used to walk through the flattened table hierarchy
type fieldpath struct {
	hierarchy []*dt
	index     []int32
	tree      *HuffmanTree
	finished  bool
}

// Typedef for a field operation function
type FieldPathOpFcn func(*reader, *fieldpath)

// Initialize a fieldpath object
func fielpath_init(parentTbl *dt) *fieldpath {
	fp := &fieldpath{
		hierarchy: make([]*dt, 0),
		index:     make([]int32, 0),
	}

	fp.hierarchy = append(fp.hierarchy, parentTbl)
	fp.index = append(fp.index, -1) // Always start at -1
	fp.finished = false

	return fp
}

// Walk an encoded fieldpath based on a huffman tree
func (fp *fieldpath) fieldpath_walk(r *reader) []dt_field {
	fields := make([]dt_field, 0)

	// where is do-while when you need it -.-
	node := (*fp.tree).(HuffmanNode)
	for fp.finished == false {
		if r.readBits(1) == 1 {
			switch i := node.right.(type) {
			case HuffmanLeaf:
				i.value.(FieldPathOpFcn)(r, fp)
			case HuffmanNode:
				node = i
			}
		} else {
			switch i := node.left.(type) {
			case HuffmanLeaf:
				i.value.(FieldPathOpFcn)(r, fp)
			case HuffmanNode:
				node = i
			}
		}
	}

	return fields
}

// Returns a huffman tree based on the operation weights
func fieldpath_huffman() HuffmanTree {
	FieldPathOperations := make(map[int]interface{})

	FieldPathOperations[36271] = PlusOne
	FieldPathOperations[10334] = PlusTwo
	FieldPathOperations[1375] = PlusThree
	FieldPathOperations[646] = PlusFour
	FieldPathOperations[4128] = PlusN
	FieldPathOperations[35] = PushOneLeftDeltaZeroRightZero
	FieldPathOperations[3] = PushOneLeftDeltaZeroRightNonZero
	FieldPathOperations[521] = PushOneLeftDeltaOneRightZero
	FieldPathOperations[2942] = PushOneLeftDeltaOneRightNonZero
	FieldPathOperations[560] = PushOneLeftDeltaNRightZero
	FieldPathOperations[471] = PushOneLeftDeltaNRightNonZero
	FieldPathOperations[10530] = PushOneLeftDeltaNRightNonZeroPack6Bits
	FieldPathOperations[251] = PushOneLeftDeltaNRightNonZeroPack8Bits
	FieldPathOperations[0] = PushTwoLeftDeltaZero
	FieldPathOperations[0] = PushTwoLeftDeltaOne
	FieldPathOperations[0] = PushTwoLeftDeltaN
	FieldPathOperations[0] = PushTwoPack5LeftDeltaZero
	FieldPathOperations[0] = PushTwoPack5LeftDeltaOne
	FieldPathOperations[0] = PushTwoPack5LeftDeltaN
	FieldPathOperations[0] = PushThreeLeftDeltaZero
	FieldPathOperations[0] = PushThreeLeftDeltaOne
	FieldPathOperations[0] = PushThreeLeftDeltaN
	FieldPathOperations[0] = PushThreePack5LeftDeltaZero
	FieldPathOperations[0] = PushThreePack5LeftDeltaOne
	FieldPathOperations[0] = PushThreePack5LeftDeltaN
	FieldPathOperations[0] = PushN
	FieldPathOperations[310] = PushNAndNonTopological
	FieldPathOperations[2] = PopOnePlusOne
	FieldPathOperations[0] = PopOnePlusN
	FieldPathOperations[1837] = PopAllButOnePlusOne
	FieldPathOperations[149] = PopAllButOnePlusN
	FieldPathOperations[300] = PopAllButOnePlusNPack3Bits
	FieldPathOperations[634] = PopAllButOnePlusNPack6Bits
	FieldPathOperations[0] = PopNPlusOne
	FieldPathOperations[0] = PopNPlusN
	FieldPathOperations[1] = PopNAndNonTopographical
	FieldPathOperations[76] = NonTopoComplex
	FieldPathOperations[271] = NonTopoPenultimatePlusOne
	FieldPathOperations[99] = NonTopoComplexPack4Bits
	FieldPathOperations[25474] = FieldPathEncodeFinish

	return buildTree(FieldPathOperations)
}

func PlusOne(r *reader, fp *fieldpath) {
	fp.index[len(fp.index)-1] += 1
}

func PlusTwo(r *reader, fp *fieldpath) {
	fp.index[len(fp.index)-1] += 2
}

func PlusThree(r *reader, fp *fieldpath) {
	fp.index[len(fp.index)-1] += 3
}

func PlusFour(r *reader, fp *fieldpath) {
	fp.index[len(fp.index)-1] += 4
}

func PlusN(r *reader, fp *fieldpath) {

}

func PushOneLeftDeltaZeroRightZero(r *reader, fp *fieldpath) {

}

func PushOneLeftDeltaZeroRightNonZero(r *reader, fp *fieldpath) {

}

func PushOneLeftDeltaOneRightZero(r *reader, fp *fieldpath) {

}

func PushOneLeftDeltaOneRightNonZero(r *reader, fp *fieldpath) {

}

func PushOneLeftDeltaNRightZero(r *reader, fp *fieldpath) {

}

func PushOneLeftDeltaNRightNonZero(r *reader, fp *fieldpath) {

}

func PushOneLeftDeltaNRightNonZeroPack6Bits(r *reader, fp *fieldpath) {

}

func PushOneLeftDeltaNRightNonZeroPack8Bits(r *reader, fp *fieldpath) {

}

func PushTwoLeftDeltaZero(r *reader, fp *fieldpath) {

}

func PushTwoLeftDeltaOne(r *reader, fp *fieldpath) {

}

func PushTwoLeftDeltaN(r *reader, fp *fieldpath) {

}

func PushTwoPack5LeftDeltaZero(r *reader, fp *fieldpath) {

}

func PushTwoPack5LeftDeltaOne(r *reader, fp *fieldpath) {

}

func PushTwoPack5LeftDeltaN(r *reader, fp *fieldpath) {

}

func PushThreeLeftDeltaZero(r *reader, fp *fieldpath) {

}

func PushThreeLeftDeltaOne(r *reader, fp *fieldpath) {

}

func PushThreeLeftDeltaN(r *reader, fp *fieldpath) {

}

func PushThreePack5LeftDeltaZero(r *reader, fp *fieldpath) {

}

func PushThreePack5LeftDeltaOne(r *reader, fp *fieldpath) {

}

func PushThreePack5LeftDeltaN(r *reader, fp *fieldpath) {

}

func PushN(r *reader, fp *fieldpath) {

}

func PushNAndNonTopological(r *reader, fp *fieldpath) {

}

func PopOnePlusOne(r *reader, fp *fieldpath) {

}

func PopOnePlusN(r *reader, fp *fieldpath) {

}

func PopAllButOnePlusOne(r *reader, fp *fieldpath) {

}

func PopAllButOnePlusN(r *reader, fp *fieldpath) {

}

func PopAllButOnePlusNPackN(r *reader, fp *fieldpath) {

}

func PopAllButOnePlusNPack3Bits(r *reader, fp *fieldpath) {

}

func PopAllButOnePlusNPack6Bits(r *reader, fp *fieldpath) {

}

func PopNPlusOne(r *reader, fp *fieldpath) {

}

func PopNPlusN(r *reader, fp *fieldpath) {

}

func PopNAndNonTopographical(r *reader, fp *fieldpath) {

}

func NonTopoComplex(r *reader, fp *fieldpath) {

}

func NonTopoPenultimatePlusOne(r *reader, fp *fieldpath) {

}

func NonTopoComplexPack4Bits(r *reader, fp *fieldpath) {

}

func FieldPathEncodeFinish(r *reader, fp *fieldpath) {
	fp.finished = true
}
<commit_msg>Fixed node not being reset, added refactor todo<commit_after>package manta

// A fieldpath, used to walk through the flattened table hierarchy
type fieldpath struct {
	hierarchy []*dt
	index     []int32
	tree      *HuffmanTree
	finished  bool
}

// Typedef for a field operation function
type FieldPathOpFcn func(*reader, *fieldpath)

// Initialize a fieldpath object
func fielpath_init(parentTbl *dt) *fieldpath {
	fp := &fieldpath{
		hierarchy: make([]*dt, 0),
		index:     make([]int32, 0),
	}

	fp.hierarchy = append(fp.hierarchy, parentTbl)
	fp.index = append(fp.index, -1) // Always start at -1
	fp.finished = false

	return fp
}

// Walk an encoded fieldpath based on a huffman tree
func (fp *fieldpath) fieldpath_walk(r *reader) []dt_field {
	fields := make([]dt_field, 0)

	// where is do-while when you need it -.-
	// @todo: Refactor this using node.IsLeaf()
	node := (*fp.tree).(HuffmanNode)
	for fp.finished == false {
		if r.readBits(1) == 1 {
			switch i := node.right.(type) {
			case HuffmanLeaf:
				i.value.(FieldPathOpFcn)(r, fp)
				node = (*fp.tree).(HuffmanNode)
			case HuffmanNode:
				node = i
			}
		} else {
			switch i := node.left.(type) {
			case HuffmanLeaf:
				i.value.(FieldPathOpFcn)(r, fp)
				node = (*fp.tree).(HuffmanNode)
			case HuffmanNode:
				node = i
			}
		}
	}

	return fields
}

// Returns a huffman tree based on the operation weights
func fieldpath_huffman() HuffmanTree {
	FieldPathOperations := make(map[int]interface{})

	FieldPathOperations[36271] = PlusOne
	FieldPathOperations[10334] = PlusTwo
	FieldPathOperations[1375] = PlusThree
	FieldPathOperations[646] = PlusFour
	FieldPathOperations[4128] = PlusN
	FieldPathOperations[35] = PushOneLeftDeltaZeroRightZero
	FieldPathOperations[3] = PushOneLeftDeltaZeroRightNonZero
	FieldPathOperations[521] = PushOneLeftDeltaOneRightZero
	FieldPathOperations[2942] = PushOneLeftDeltaOneRightNonZero
	FieldPathOperations[560] = PushOneLeftDeltaNRightZero
	FieldPathOperations[471] = PushOneLeftDeltaNRightNonZero
	FieldPathOperations[10530] = PushOneLeftDeltaNRightNonZeroPack6Bits
	FieldPathOperations[251] = PushOneLeftDeltaNRightNonZeroPack8Bits
	FieldPathOperations[0] = PushTwoLeftDeltaZero
	FieldPathOperations[0] = PushTwoLeftDeltaOne
	FieldPathOperations[0] = PushTwoLeftDeltaN
	FieldPathOperations[0] = PushTwoPack5LeftDeltaZero
	FieldPathOperations[0] = PushTwoPack5LeftDeltaOne
	FieldPathOperations[0] = PushTwoPack5LeftDeltaN
	FieldPathOperations[0] = PushThreeLeftDeltaZero
	FieldPathOperations[0] = PushThreeLeftDeltaOne
	FieldPathOperations[0] = PushThreeLeftDeltaN
	FieldPathOperations[0] = PushThreePack5LeftDeltaZero
	FieldPathOperations[0] = PushThreePack5LeftDeltaOne
	FieldPathOperations[0] = PushThreePack5LeftDeltaN
	FieldPathOperations[0] = PushN
	FieldPathOperations[310] = PushNAndNonTopological
	FieldPathOperations[2] = PopOnePlusOne
	FieldPathOperations[0] = PopOnePlusN
	FieldPathOperations[1837] = PopAllButOnePlusOne
	FieldPathOperations[149] = PopAllButOnePlusN
	FieldPathOperations[300] = PopAllButOnePlusNPack3Bits
	FieldPathOperations[634] = PopAllButOnePlusNPack6Bits
	FieldPathOperations[0] = PopNPlusOne
	FieldPathOperations[0] = PopNPlusN
	FieldPathOperations[1] = PopNAndNonTopographical
	FieldPathOperations[76] = NonTopoComplex
	FieldPathOperations[271] = NonTopoPenultimatePlusOne
	FieldPathOperations[99] = NonTopoComplexPack4Bits
	FieldPathOperations[25474] = FieldPathEncodeFinish

	return buildTree(FieldPathOperations)
}

func PlusOne(r *reader, fp *fieldpath) {
	fp.index[len(fp.index)-1] += 1
}

func PlusTwo(r *reader, fp *fieldpath) {
	fp.index[len(fp.index)-1] += 2
}

func PlusThree(r *reader, fp *fieldpath) {
	fp.index[len(fp.index)-1] += 3
}

func PlusFour(r *reader, fp *fieldpath) {
	fp.index[len(fp.index)-1] += 4
}

func PlusN(r *reader, fp *fieldpath) {

}

func PushOneLeftDeltaZeroRightZero(r *reader, fp *fieldpath) {

}

func PushOneLeftDeltaZeroRightNonZero(r *reader, fp *fieldpath) {

}

func PushOneLeftDeltaOneRightZero(r *reader, fp *fieldpath) {

}

func PushOneLeftDeltaOneRightNonZero(r *reader, fp *fieldpath) {

}

func PushOneLeftDeltaNRightZero(r *reader, fp *fieldpath) {

}

func PushOneLeftDeltaNRightNonZero(r *reader, fp *fieldpath) {

}

func PushOneLeftDeltaNRightNonZeroPack6Bits(r *reader, fp *fieldpath) {

}

func PushOneLeftDeltaNRightNonZeroPack8Bits(r *reader, fp *fieldpath) {

}

func PushTwoLeftDeltaZero(r *reader, fp *fieldpath) {

}

func PushTwoLeftDeltaOne(r *reader, fp *fieldpath) {

}

func PushTwoLeftDeltaN(r *reader, fp *fieldpath) {

}

func PushTwoPack5LeftDeltaZero(r *reader, fp *fieldpath) {

}

func PushTwoPack5LeftDeltaOne(r *reader, fp *fieldpath) {

}

func PushTwoPack5LeftDeltaN(r *reader, fp *fieldpath) {

}

func PushThreeLeftDeltaZero(r *reader, fp *fieldpath) {

}

func PushThreeLeftDeltaOne(r *reader, fp *fieldpath) {

}

func PushThreeLeftDeltaN(r *reader, fp *fieldpath) {

}

func PushThreePack5LeftDeltaZero(r *reader, fp *fieldpath) {

}

func PushThreePack5LeftDeltaOne(r *reader, fp *fieldpath) {

}

func PushThreePack5LeftDeltaN(r *reader, fp *fieldpath) {

}

func PushN(r *reader, fp *fieldpath) {

}

func PushNAndNonTopological(r *reader, fp *fieldpath) {

}

func PopOnePlusOne(r *reader, fp *fieldpath) {

}

func PopOnePlusN(r *reader, fp *fieldpath) {

}

func PopAllButOnePlusOne(r *reader, fp *fieldpath) {

}

func PopAllButOnePlusN(r *reader, fp *fieldpath) {

}

func PopAllButOnePlusNPackN(r *reader, fp *fieldpath) {

}

func PopAllButOnePlusNPack3Bits(r *reader, fp *fieldpath) {

}

func PopAllButOnePlusNPack6Bits(r *reader, fp *fieldpath) {

}

func PopNPlusOne(r *reader, fp *fieldpath) {

}

func PopNPlusN(r *reader, fp *fieldpath) {

}

func PopNAndNonTopographical(r *reader, fp *fieldpath) {

}

func NonTopoComplex(r *reader, fp *fieldpath) {

}

func NonTopoPenultimatePlusOne(r *reader, fp *fieldpath) {

}

func NonTopoComplexPack4Bits(r *reader, fp *fieldpath) {

}

func FieldPathEncodeFinish(r *reader, fp *fieldpath) {
	fp.finished = true
}
<|endoftext|>


<commit_before>package api

import (
	"fmt"
	"net/http"
	"sync"

	"github.com/Dataman-Cloud/swan/types"
	log "github.com/Sirupsen/logrus"
)

func (r *Server) purge(w http.ResponseWriter, req *http.Request) {
	apps, err := r.db.ListApps()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	tokenBucket := make(chan struct{}, 10) // TODO(nmg): delete step, make it configurable

	go func() {
		var all sync.WaitGroup

		for _, app := range apps {
			var (
				hasError = false
				wg       sync.WaitGroup
			)

			tasks, err := r.db.ListTasks(app.ID)
			if err != nil {
				log.Errorf("list app tasks got error for purge: %v", err)
				continue
			}

			wg.Add(len(tasks))
			for _, task := range tasks {
				tokenBucket <- struct{}{}

				go func(task *types.Task, appId string) {
					all.Add(1)

					defer func() {
						wg.Done()
						all.Done()
						<-tokenBucket
					}()

					if err := r.driver.KillTask(task.ID, task.AgentId, false); err != nil {
						log.Errorf("Kill task %s got error: %v", task.ID, err)

						hasError = true

						task.OpStatus = fmt.Sprintf("kill task error: %v", err)
						if err = r.db.UpdateTask(appId, task); err != nil {
							log.Errorf("update task %s got error: %v", task.Name, err)
						}

						return
					}

					if err := r.db.DeleteTask(task.ID); err != nil {
						log.Errorf("Delete task %s got error: %v", task.ID, err)

						hasError = true

						task.OpStatus = fmt.Sprintf("delete task error: %v", err)
						if err = r.db.UpdateTask(appId, task); err != nil {
							log.Errorf("update task %s got error: %v", task.Name, err)
						}

						return
					}

				}(task, app.ID)
			}

			wg.Wait()

			if hasError {
				log.Errorf("Delete some tasks of app %s got error.", app.ID)
				return
			}

			if err := r.db.DeleteApp(app.ID); err != nil {
				log.Error("Delete app %s got error: %v", app.ID, err)
				return
			}

		}

		all.Wait()

		close(tokenBucket)
	}()

	writeJSON(w, http.StatusNoContent, "")
}
<commit_msg>rewrite api `purge` to delete task one by one<commit_after>package api

import (
	"fmt"
	"net/http"

	log "github.com/Sirupsen/logrus"
)

func (r *Server) purge(w http.ResponseWriter, req *http.Request) {
	apps, err := r.db.ListApps()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	go func() {
		for _, app := range apps {
			go func(appId string) {
				tasks, err := r.db.ListTasks(appId)
				if err != nil {
					log.Errorf("list app tasks got error for purge: %v", err)
					return
				}

				var (
					count   = len(tasks)
					succeed = 0
				)

				for _, task := range tasks {
					if err := r.driver.KillTask(task.ID, task.AgentId, true); err != nil {
						log.Errorf("Kill task %s got error: %v", task.ID, err)

						task.OpStatus = fmt.Sprintf("kill task error: %v", err)
						if err = r.db.UpdateTask(appId, task); err != nil {
							log.Errorf("update task %s got error: %v", task.Name, err)
						}

						continue
					}

					if err := r.db.DeleteTask(task.ID); err != nil {
						log.Errorf("Delete task %s got error: %v", task.ID, err)

						task.OpStatus = fmt.Sprintf("delete task error: %v", err)
						if err = r.db.UpdateTask(appId, task); err != nil {
							log.Errorf("update task %s got error: %v", task.Name, err)
						}

						continue
					}

					succeed++
				}

				if succeed == count {
					versions, err := r.db.ListVersions(app.ID)
					if err != nil {
						log.Errorf("list versions error for delete app. %v", err)
						return
					}

					for _, version := range versions {
						if err := r.db.DeleteVersion(appId, version.ID); err != nil {
							log.Errorf("Delete version %s for app %s got error: %v", version.ID, appId, err)
							return
						}
					}

					if err := r.db.DeleteApp(app.ID); err != nil {
						log.Errorf("Delete app %s got error: %v", appId, err)
					}
				}

			}(app.ID)
		}
	}()

	writeJSON(w, http.StatusNoContent, "")
}
<|endoftext|>