<commit_before>package aws

import (
	"fmt"
	"log"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/codeartifact"
	"github.com/hashicorp/go-multierror"
	"github.com/hashicorp/terraform-plugin-sdk/helper/acctest"
	"github.com/hashicorp/terraform-plugin-sdk/helper/resource"
	"github.com/hashicorp/terraform-plugin-sdk/terraform"
)

func init() {
	resource.AddTestSweepers("aws_codeartifact_repository", &resource.Sweeper{
		Name: "aws_codeartifact_repository",
		F:    testSweepCodeArtifactRepositories,
	})
}

func testSweepCodeArtifactRepositories(region string) error {
	client, err := sharedClientForRegion(region)
	if err != nil {
		return fmt.Errorf("error getting client: %w", err)
	}
	conn := client.(*AWSClient).codeartifactconn
	input := &codeartifact.ListRepositoriesInput{}
	var sweeperErrs *multierror.Error

	err = conn.ListRepositoriesPages(input, func(page *codeartifact.ListRepositoriesOutput, lastPage bool) bool {
		for _, repositoryPtr := range page.Repositories {
			if repositoryPtr == nil {
				continue
			}

			repository := aws.StringValue(repositoryPtr.Name)
			input := &codeartifact.DeleteRepositoryInput{
				Repository:  repositoryPtr.Name,
				Domain:      repositoryPtr.DomainName,
				DomainOwner: repositoryPtr.DomainOwner,
			}

			log.Printf("[INFO] Deleting CodeArtifact Repository: %s", repository)

			_, err := conn.DeleteRepository(input)

			if err != nil {
				sweeperErr := fmt.Errorf("error deleting CodeArtifact Repository (%s): %w", repository, err)
				log.Printf("[ERROR] %s", sweeperErr)
				sweeperErrs = multierror.Append(sweeperErrs, sweeperErr)
			}
		}

		return !lastPage
	})

	if testSweepSkipSweepError(err) {
		log.Printf("[WARN] Skipping CodeArtifact Repository sweep for %s: %s", region, err)
		return nil
	}

	if err != nil {
		return fmt.Errorf("error listing CodeArtifact Repositories: %w", err)
	}

	return sweeperErrs.ErrorOrNil()
}

func TestAccAWSCodeArtifactRepository_basic(t *testing.T) {
	rName := acctest.RandomWithPrefix("tf-acc-test")
	resourceName := "aws_codeartifact_repository.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSCodeArtifactRepositoryDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSCodeArtifactRepositoryBasicConfig(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCodeArtifactRepositoryExists(resourceName),
					testAccCheckResourceAttrRegionalARN(resourceName, "arn", "codeartifact", fmt.Sprintf("repository/%s/%s", rName, rName)),
					resource.TestCheckResourceAttr(resourceName, "repository", rName),
					resource.TestCheckResourceAttr(resourceName, "domain", rName),
					resource.TestCheckResourceAttrPair(resourceName, "domain_owner", "aws_codeartifact_domain", "owner"),
					resource.TestCheckResourceAttrPair(resourceName, "administrator_account", "aws_codeartifact_domain", "owner"),
					resource.TestCheckResourceAttr(resourceName, "description", ""),
					resource.TestCheckResourceAttr(resourceName, "upstream.#", "0"),
					resource.TestCheckResourceAttr(resourceName, "external_connections.#", "0"),
				),
			},
			{
				ResourceName:      resourceName,
				ImportState:       true,
				ImportStateVerify: true,
			},
		},
	})
}

func TestAccAWSCodeArtifactRepository_owner(t *testing.T) {
	rName := acctest.RandomWithPrefix("tf-acc-test")
	resourceName := "aws_codeartifact_repository.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSCodeArtifactRepositoryDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSCodeArtifactRepositoryOwnerConfig(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCodeArtifactRepositoryExists(resourceName),
					testAccCheckResourceAttrRegionalARN(resourceName, "arn", "codeartifact", fmt.Sprintf("repository/%s/%s", rName, rName)),
					resource.TestCheckResourceAttr(resourceName, "repository", rName),
					resource.TestCheckResourceAttr(resourceName, "domain", rName),
					resource.TestCheckResourceAttrPair(resourceName, "domain_owner", "aws_codeartifact_domain", "owner"),
					resource.TestCheckResourceAttr(resourceName, "description", ""),
					resource.TestCheckResourceAttr(resourceName, "upstream.#", "0"),
					resource.TestCheckResourceAttr(resourceName, "external_connections.#", "0"),
				),
			},
			{
				ResourceName:      resourceName,
				ImportState:       true,
				ImportStateVerify: true,
			},
		},
	})
}

func TestAccAWSCodeArtifactRepository_description(t *testing.T) {
	rName := acctest.RandomWithPrefix("tf-acc-test")
	resourceName := "aws_codeartifact_repository.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSCodeArtifactRepositoryDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSCodeArtifactRepositoryDescConfig(rName, "desc"),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCodeArtifactRepositoryExists(resourceName),
					resource.TestCheckResourceAttr(resourceName, "description", "desc"),
				),
			},
			{
				ResourceName:      resourceName,
				ImportState:       true,
				ImportStateVerify: true,
			},
			{
				Config: testAccAWSCodeArtifactRepositoryDescConfig(rName, "desc2"),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCodeArtifactRepositoryExists(resourceName),
					resource.TestCheckResourceAttr(resourceName, "description", "desc2"),
				),
			},
		},
	})
}

func TestAccAWSCodeArtifactRepository_upstreams(t *testing.T) {
	rName := acctest.RandomWithPrefix("tf-acc-test")
	resourceName := "aws_codeartifact_repository.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSCodeArtifactRepositoryDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSCodeArtifactRepositoryUpstreamsConfig1(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCodeArtifactRepositoryExists(resourceName),
					resource.TestCheckResourceAttr(resourceName, "upstream.#", "1"),
					resource.TestCheckResourceAttr(resourceName, "upstream.0.repository_name", fmt.Sprintf("%s-upstream1", rName)),
				),
			},
			{
				ResourceName:      resourceName,
				ImportState:       true,
				ImportStateVerify: true,
			},
			{
				Config: testAccAWSCodeArtifactRepositoryUpstreamsConfig2(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCodeArtifactRepositoryExists(resourceName),
					resource.TestCheckResourceAttr(resourceName, "upstream.#", "2"),
					resource.TestCheckResourceAttr(resourceName, "upstream.0.repository_name", fmt.Sprintf("%s-upstream1", rName)),
					resource.TestCheckResourceAttr(resourceName, "upstream.1.repository_name", fmt.Sprintf("%s-upstream2", rName)),
				),
			},
			{
				Config: testAccAWSCodeArtifactRepositoryUpstreamsConfig1(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCodeArtifactRepositoryExists(resourceName),
					resource.TestCheckResourceAttr(resourceName, "upstream.#", "1"),
					resource.TestCheckResourceAttr(resourceName, "upstream.0.repository_name", fmt.Sprintf("%s-upstream1", rName)),
				),
			},
		},
	})
}

func TestAccAWSCodeArtifactRepository_disappears(t *testing.T) {
	rName := acctest.RandomWithPrefix("tf-acc-test")
	resourceName := "aws_codeartifact_repository.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSCodeArtifactRepositoryDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSCodeArtifactRepositoryBasicConfig(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCodeArtifactRepositoryExists(resourceName),
					testAccCheckResourceDisappears(testAccProvider, resourceAwsCodeArtifactRepository(), resourceName),
				),
				ExpectNonEmptyPlan: true,
			},
		},
	})
}

func testAccCheckAWSCodeArtifactRepositoryExists(n string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("no CodeArtifact repository set")
		}

		conn := testAccProvider.Meta().(*AWSClient).codeartifactconn
		owner, domain, repo, err := decodeCodeArtifactRepositoryID(rs.Primary.ID)
		if err != nil {
			return err
		}
		_, err = conn.DescribeRepository(&codeartifact.DescribeRepositoryInput{
			Repository:  aws.String(repo),
			Domain:      aws.String(domain),
			DomainOwner: aws.String(owner),
		})

		return err
	}
}

func testAccCheckAWSCodeArtifactRepositoryDestroy(s *terraform.State) error {
	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_codeartifact_repository" {
			continue
		}

		owner, domain, repo, err := decodeCodeArtifactRepositoryID(rs.Primary.ID)
		if err != nil {
			return err
		}
		conn := testAccProvider.Meta().(*AWSClient).codeartifactconn
		resp, err := conn.DescribeRepository(&codeartifact.DescribeRepositoryInput{
			Repository:  aws.String(repo),
			Domain:      aws.String(domain),
			DomainOwner: aws.String(owner),
		})

		if err == nil {
			if aws.StringValue(resp.Repository.Name) == repo &&
				aws.StringValue(resp.Repository.DomainName) == domain &&
				aws.StringValue(resp.Repository.DomainOwner) == owner {
				return fmt.Errorf("CodeArtifact Repository %s in Domain %s still exists", repo, domain)
			}
		}

		if isAWSErr(err, codeartifact.ErrCodeResourceNotFoundException, "") {
			return nil
		}

		return err
	}

	return nil
}

func testAccAWSCodeArtifactRepositoryBasicConfig(rName string) string {
	return fmt.Sprintf(`
resource "aws_kms_key" "test" {
  description             = %[1]q
  deletion_window_in_days = 7
}

resource "aws_codeartifact_domain" "test" {
  domain         = %[1]q
  encryption_key = aws_kms_key.test.arn
}

resource "aws_codeartifact_repository" "test" {
  repository = %[1]q
  domain     = aws_codeartifact_domain.test.domain
}
`, rName)
}

func testAccAWSCodeArtifactRepositoryOwnerConfig(rName string) string {
	return fmt.Sprintf(`
resource "aws_kms_key" "test" {
  description             = %[1]q
  deletion_window_in_days = 7
}

resource "aws_codeartifact_domain" "test" {
  domain         = %[1]q
  encryption_key = aws_kms_key.test.arn
}

resource "aws_codeartifact_repository" "test" {
  repository   = %[1]q
  domain       = aws_codeartifact_domain.test.domain
  domain_owner = aws_codeartifact_domain.test.owner
}
`, rName)
}

func testAccAWSCodeArtifactRepositoryDescConfig(rName, desc string) string {
	return fmt.Sprintf(`
resource "aws_kms_key" "test" {
  description             = %[1]q
  deletion_window_in_days = 7
}

resource "aws_codeartifact_domain" "test" {
  domain         = %[1]q
  encryption_key = aws_kms_key.test.arn
}

resource "aws_codeartifact_repository" "test" {
  repository  = %[1]q
  domain      = aws_codeartifact_domain.test.domain
  description = %[2]q
}
`, rName, desc)
}

func testAccAWSCodeArtifactRepositoryUpstreamsConfig1(rName string) string {
	return fmt.Sprintf(`
resource "aws_kms_key" "test" {
  description             = %[1]q
  deletion_window_in_days = 7
}

resource "aws_codeartifact_domain" "test" {
  domain         = %[1]q
  encryption_key = aws_kms_key.test.arn
}

resource "aws_codeartifact_repository" "upstream1" {
  repository = "%[1]s-upstream1"
  domain     = aws_codeartifact_domain.test.domain
}

resource "aws_codeartifact_repository" "test" {
  repository = %[1]q
  domain     = aws_codeartifact_domain.test.domain

  upstream {
    repository_name = aws_codeartifact_repository.upstream1.repository
  }
}
`, rName)
}

func testAccAWSCodeArtifactRepositoryUpstreamsConfig2(rName string) string {
	return fmt.Sprintf(`
resource "aws_kms_key" "test" {
  description             = %[1]q
  deletion_window_in_days = 7
}

resource "aws_codeartifact_domain" "test" {
  domain         = %[1]q
  encryption_key = aws_kms_key.test.arn
}

resource "aws_codeartifact_repository" "upstream1" {
  repository = "%[1]s-upstream1"
  domain     = aws_codeartifact_domain.test.domain
}

resource "aws_codeartifact_repository" "upstream2" {
  repository = "%[1]s-upstream2"
  domain     = aws_codeartifact_domain.test.domain
}

resource "aws_codeartifact_repository" "test" {
  repository = %[1]q
  domain     = aws_codeartifact_domain.test.domain

  upstream {
    repository_name = aws_codeartifact_repository.upstream1.repository
  }

  upstream {
    repository_name = aws_codeartifact_repository.upstream2.repository
  }
}
`, rName)
}
<commit_msg>add owner test<commit_after>package aws

import (
	"fmt"
	"log"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/codeartifact"
	"github.com/hashicorp/go-multierror"
	"github.com/hashicorp/terraform-plugin-sdk/helper/acctest"
	"github.com/hashicorp/terraform-plugin-sdk/helper/resource"
	"github.com/hashicorp/terraform-plugin-sdk/terraform"
)

func init() {
	resource.AddTestSweepers("aws_codeartifact_repository", &resource.Sweeper{
		Name: "aws_codeartifact_repository",
		F:    testSweepCodeArtifactRepositories,
	})
}

func testSweepCodeArtifactRepositories(region string) error {
	client, err := sharedClientForRegion(region)
	if err != nil {
		return fmt.Errorf("error getting client: %w", err)
	}
	conn := client.(*AWSClient).codeartifactconn
	input := &codeartifact.ListRepositoriesInput{}
	var sweeperErrs *multierror.Error

	err = conn.ListRepositoriesPages(input, func(page *codeartifact.ListRepositoriesOutput, lastPage bool) bool {
		for _, repositoryPtr := range page.Repositories {
			if repositoryPtr == nil {
				continue
			}

			repository := aws.StringValue(repositoryPtr.Name)
			input := &codeartifact.DeleteRepositoryInput{
				Repository:  repositoryPtr.Name,
				Domain:      repositoryPtr.DomainName,
				DomainOwner: repositoryPtr.DomainOwner,
			}

			log.Printf("[INFO] Deleting CodeArtifact Repository: %s", repository)

			_, err := conn.DeleteRepository(input)

			if err != nil {
				sweeperErr := fmt.Errorf("error deleting CodeArtifact Repository (%s): %w", repository, err)
				log.Printf("[ERROR] %s", sweeperErr)
				sweeperErrs = multierror.Append(sweeperErrs, sweeperErr)
			}
		}

		return !lastPage
	})

	if testSweepSkipSweepError(err) {
		log.Printf("[WARN] Skipping CodeArtifact Repository sweep for %s: %s", region, err)
		return nil
	}

	if err != nil {
		return fmt.Errorf("error listing CodeArtifact Repositories: %w", err)
	}

	return sweeperErrs.ErrorOrNil()
}

func TestAccAWSCodeArtifactRepository_basic(t *testing.T) {
	rName := acctest.RandomWithPrefix("tf-acc-test")
	resourceName := "aws_codeartifact_repository.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSCodeArtifactRepositoryDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSCodeArtifactRepositoryBasicConfig(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCodeArtifactRepositoryExists(resourceName),
					testAccCheckResourceAttrRegionalARN(resourceName, "arn", "codeartifact", fmt.Sprintf("repository/%s/%s", rName, rName)),
					resource.TestCheckResourceAttr(resourceName, "repository", rName),
					resource.TestCheckResourceAttr(resourceName, "domain", rName),
					resource.TestCheckResourceAttrPair(resourceName, "domain_owner", "aws_codeartifact_domain.test", "owner"),
					resource.TestCheckResourceAttrPair(resourceName, "administrator_account", "aws_codeartifact_domain.test", "owner"),
					resource.TestCheckResourceAttr(resourceName, "description", ""),
					resource.TestCheckResourceAttr(resourceName, "upstream.#", "0"),
					resource.TestCheckResourceAttr(resourceName, "external_connections.#", "0"),
				),
			},
			{
				ResourceName:      resourceName,
				ImportState:       true,
				ImportStateVerify: true,
			},
		},
	})
}

func TestAccAWSCodeArtifactRepository_owner(t *testing.T) {
	rName := acctest.RandomWithPrefix("tf-acc-test")
	resourceName := "aws_codeartifact_repository.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSCodeArtifactRepositoryDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSCodeArtifactRepositoryOwnerConfig(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCodeArtifactRepositoryExists(resourceName),
					testAccCheckResourceAttrRegionalARN(resourceName, "arn", "codeartifact", fmt.Sprintf("repository/%s/%s", rName, rName)),
					resource.TestCheckResourceAttr(resourceName, "repository", rName),
					resource.TestCheckResourceAttr(resourceName, "domain", rName),
					resource.TestCheckResourceAttrPair(resourceName, "domain_owner", "aws_codeartifact_domain.test", "owner"),
					resource.TestCheckResourceAttrPair(resourceName, "administrator_account", "aws_codeartifact_domain.test", "owner"),
					resource.TestCheckResourceAttr(resourceName, "description", ""),
					resource.TestCheckResourceAttr(resourceName, "upstream.#", "0"),
					resource.TestCheckResourceAttr(resourceName, "external_connections.#", "0"),
				),
			},
			{
				ResourceName:      resourceName,
				ImportState:       true,
				ImportStateVerify: true,
			},
		},
	})
}

func TestAccAWSCodeArtifactRepository_description(t *testing.T) {
	rName := acctest.RandomWithPrefix("tf-acc-test")
	resourceName := "aws_codeartifact_repository.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSCodeArtifactRepositoryDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSCodeArtifactRepositoryDescConfig(rName, "desc"),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCodeArtifactRepositoryExists(resourceName),
					resource.TestCheckResourceAttr(resourceName, "description", "desc"),
				),
			},
			{
				ResourceName:      resourceName,
				ImportState:       true,
				ImportStateVerify: true,
			},
			{
				Config: testAccAWSCodeArtifactRepositoryDescConfig(rName, "desc2"),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCodeArtifactRepositoryExists(resourceName),
					resource.TestCheckResourceAttr(resourceName, "description", "desc2"),
				),
			},
		},
	})
}

func TestAccAWSCodeArtifactRepository_upstreams(t *testing.T) {
	rName := acctest.RandomWithPrefix("tf-acc-test")
	resourceName := "aws_codeartifact_repository.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSCodeArtifactRepositoryDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSCodeArtifactRepositoryUpstreamsConfig1(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCodeArtifactRepositoryExists(resourceName),
					resource.TestCheckResourceAttr(resourceName, "upstream.#", "1"),
					resource.TestCheckResourceAttr(resourceName, "upstream.0.repository_name", fmt.Sprintf("%s-upstream1", rName)),
				),
			},
			{
				ResourceName:      resourceName,
				ImportState:       true,
				ImportStateVerify: true,
			},
			{
				Config: testAccAWSCodeArtifactRepositoryUpstreamsConfig2(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCodeArtifactRepositoryExists(resourceName),
					resource.TestCheckResourceAttr(resourceName, "upstream.#", "2"),
					resource.TestCheckResourceAttr(resourceName, "upstream.0.repository_name", fmt.Sprintf("%s-upstream1", rName)),
					resource.TestCheckResourceAttr(resourceName, "upstream.1.repository_name", fmt.Sprintf("%s-upstream2", rName)),
				),
			},
			{
				Config: testAccAWSCodeArtifactRepositoryUpstreamsConfig1(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCodeArtifactRepositoryExists(resourceName),
					resource.TestCheckResourceAttr(resourceName, "upstream.#", "1"),
					resource.TestCheckResourceAttr(resourceName, "upstream.0.repository_name", fmt.Sprintf("%s-upstream1", rName)),
				),
			},
		},
	})
}

func TestAccAWSCodeArtifactRepository_disappears(t *testing.T) {
	rName := acctest.RandomWithPrefix("tf-acc-test")
	resourceName := "aws_codeartifact_repository.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSCodeArtifactRepositoryDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSCodeArtifactRepositoryBasicConfig(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCodeArtifactRepositoryExists(resourceName),
					testAccCheckResourceDisappears(testAccProvider, resourceAwsCodeArtifactRepository(), resourceName),
				),
				ExpectNonEmptyPlan: true,
			},
		},
	})
}

func testAccCheckAWSCodeArtifactRepositoryExists(n string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("no CodeArtifact repository set")
		}

		conn := testAccProvider.Meta().(*AWSClient).codeartifactconn
		owner, domain, repo, err := decodeCodeArtifactRepositoryID(rs.Primary.ID)
		if err != nil {
			return err
		}
		_, err = conn.DescribeRepository(&codeartifact.DescribeRepositoryInput{
			Repository:  aws.String(repo),
			Domain:      aws.String(domain),
			DomainOwner: aws.String(owner),
		})

		return err
	}
}

func testAccCheckAWSCodeArtifactRepositoryDestroy(s *terraform.State) error {
	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_codeartifact_repository" {
			continue
		}

		owner, domain, repo, err := decodeCodeArtifactRepositoryID(rs.Primary.ID)
		if err != nil {
			return err
		}
		conn := testAccProvider.Meta().(*AWSClient).codeartifactconn
		resp, err := conn.DescribeRepository(&codeartifact.DescribeRepositoryInput{
			Repository:  aws.String(repo),
			Domain:      aws.String(domain),
			DomainOwner: aws.String(owner),
		})

		if err == nil {
			if aws.StringValue(resp.Repository.Name) == repo &&
				aws.StringValue(resp.Repository.DomainName) == domain &&
				aws.StringValue(resp.Repository.DomainOwner) == owner {
				return fmt.Errorf("CodeArtifact Repository %s in Domain %s still exists", repo, domain)
			}
		}

		if isAWSErr(err, codeartifact.ErrCodeResourceNotFoundException, "") {
			return nil
		}

		return err
	}

	return nil
}

func testAccAWSCodeArtifactRepositoryBasicConfig(rName string) string {
	return fmt.Sprintf(`
resource "aws_kms_key" "test" {
  description             = %[1]q
  deletion_window_in_days = 7
}

resource "aws_codeartifact_domain" "test" {
  domain         = %[1]q
  encryption_key = aws_kms_key.test.arn
}

resource "aws_codeartifact_repository" "test" {
  repository = %[1]q
  domain     = aws_codeartifact_domain.test.domain
}
`, rName)
}

func testAccAWSCodeArtifactRepositoryOwnerConfig(rName string) string {
	return fmt.Sprintf(`
resource "aws_kms_key" "test" {
  description             = %[1]q
  deletion_window_in_days = 7
}

resource "aws_codeartifact_domain" "test" {
  domain         = %[1]q
  encryption_key = aws_kms_key.test.arn
}

resource "aws_codeartifact_repository" "test" {
  repository   = %[1]q
  domain       = aws_codeartifact_domain.test.domain
  domain_owner = aws_codeartifact_domain.test.owner
}
`, rName)
}

func testAccAWSCodeArtifactRepositoryDescConfig(rName, desc string) string {
	return fmt.Sprintf(`
resource "aws_kms_key" "test" {
  description             = %[1]q
  deletion_window_in_days = 7
}

resource "aws_codeartifact_domain" "test" {
  domain         = %[1]q
  encryption_key = aws_kms_key.test.arn
}

resource "aws_codeartifact_repository" "test" {
  repository  = %[1]q
  domain      = aws_codeartifact_domain.test.domain
  description = %[2]q
}
`, rName, desc)
}

func testAccAWSCodeArtifactRepositoryUpstreamsConfig1(rName string) string {
	return fmt.Sprintf(`
resource "aws_kms_key" "test" {
  description             = %[1]q
  deletion_window_in_days = 7
}

resource "aws_codeartifact_domain" "test" {
  domain         = %[1]q
  encryption_key = aws_kms_key.test.arn
}

resource "aws_codeartifact_repository" "upstream1" {
  repository = "%[1]s-upstream1"
  domain     = aws_codeartifact_domain.test.domain
}

resource "aws_codeartifact_repository" "test" {
  repository = %[1]q
  domain     = aws_codeartifact_domain.test.domain

  upstream {
    repository_name = aws_codeartifact_repository.upstream1.repository
  }
}
`, rName)
}

func testAccAWSCodeArtifactRepositoryUpstreamsConfig2(rName string) string {
	return fmt.Sprintf(`
resource "aws_kms_key" "test" {
  description             = %[1]q
  deletion_window_in_days = 7
}

resource "aws_codeartifact_domain" "test" {
  domain         = %[1]q
  encryption_key = aws_kms_key.test.arn
}

resource "aws_codeartifact_repository" "upstream1" {
  repository = "%[1]s-upstream1"
  domain     = aws_codeartifact_domain.test.domain
}

resource "aws_codeartifact_repository" "upstream2" {
  repository = "%[1]s-upstream2"
  domain     = aws_codeartifact_domain.test.domain
}

resource "aws_codeartifact_repository" "test" {
  repository = %[1]q
  domain     = aws_codeartifact_domain.test.domain

  upstream {
    repository_name = aws_codeartifact_repository.upstream1.repository
  }

  upstream {
    repository_name = aws_codeartifact_repository.upstream2.repository
  }
}
`, rName)
}
<|endoftext|>


<commit_before>// Copyright 2021 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package wrappers

import (
	"context"
	"errors"
	"net/http"
	"syscall"

	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/fuse/fuseutil"
	"google.golang.org/api/googleapi"
)

func errno(err error) error {
	if err == nil {
		return nil
	}

	// Use existing FS errno
	var errno syscall.Errno
	if errors.As(err, &errno) {
		return errno
	}

	// FS op is interrupted
	if errors.Is(err, context.Canceled) {
		return syscall.EINTR
	}

	// Translate API errors into an FS errno
	var apiErr *googleapi.Error
	if errors.As(err, &apiErr) {
		switch apiErr.Code {
		case http.StatusForbidden:
			return syscall.EACCES
		}
	}

	// Unknown errors
	return syscall.EIO
}

// WithErrorMapping wraps a FileSystem, processing the returned errors, and
// mapping them into syscall.Errno that can be understood by FUSE.
func WithErrorMapping(wrapped fuseutil.FileSystem) fuseutil.FileSystem {
	return &errorMapping{wrapped: wrapped}
}

type errorMapping struct {
	wrapped fuseutil.FileSystem
}

func (fs *errorMapping) Destroy() {
	fs.wrapped.Destroy()
}

func (fs *errorMapping) StatFS(
	ctx context.Context,
	op *fuseops.StatFSOp) error {
	err := fs.wrapped.StatFS(ctx, op)
	return errno(err)
}

func (fs *errorMapping) LookUpInode(
	ctx context.Context,
	op *fuseops.LookUpInodeOp) error {
	err := fs.wrapped.LookUpInode(ctx, op)
	return errno(err)
}

func (fs *errorMapping) GetInodeAttributes(
	ctx context.Context,
	op *fuseops.GetInodeAttributesOp) error {
	err := fs.wrapped.GetInodeAttributes(ctx, op)
	return errno(err)
}

func (fs *errorMapping) SetInodeAttributes(
	ctx context.Context,
	op *fuseops.SetInodeAttributesOp) error {
	err := fs.wrapped.SetInodeAttributes(ctx, op)
	return errno(err)
}

func (fs *errorMapping) ForgetInode(
	ctx context.Context,
	op *fuseops.ForgetInodeOp) error {
	err := fs.wrapped.ForgetInode(ctx, op)
	return errno(err)
}

func (fs *errorMapping) MkDir(
	ctx context.Context,
	op *fuseops.MkDirOp) error {
	err := fs.wrapped.MkDir(ctx, op)
	return errno(err)
}

func (fs *errorMapping) MkNode(
	ctx context.Context,
	op *fuseops.MkNodeOp) error {
	err := fs.wrapped.MkNode(ctx, op)
	return errno(err)
}

func (fs *errorMapping) CreateFile(
	ctx context.Context,
	op *fuseops.CreateFileOp) error {
	err := fs.wrapped.CreateFile(ctx, op)
	return errno(err)
}

func (fs *errorMapping) CreateLink(
	ctx context.Context,
	op *fuseops.CreateLinkOp) error {
	err := fs.wrapped.CreateLink(ctx, op)
	return errno(err)
}

func (fs *errorMapping) CreateSymlink(
	ctx context.Context,
	op *fuseops.CreateSymlinkOp) error {
	err := fs.wrapped.CreateSymlink(ctx, op)
	return errno(err)
}

func (fs *errorMapping) Rename(
	ctx context.Context,
	op *fuseops.RenameOp) error {
	err := fs.wrapped.Rename(ctx, op)
	return errno(err)
}

func (fs *errorMapping) RmDir(
	ctx context.Context,
	op *fuseops.RmDirOp) error {
	err := fs.wrapped.RmDir(ctx, op)
	return errno(err)
}

func (fs *errorMapping) Unlink(
	ctx context.Context,
	op *fuseops.UnlinkOp) error {
	err := fs.wrapped.Unlink(ctx, op)
	return errno(err)
}

func (fs *errorMapping) OpenDir(
	ctx context.Context,
	op *fuseops.OpenDirOp) error {
	err := fs.wrapped.OpenDir(ctx, op)
	return errno(err)
}

func (fs *errorMapping) ReadDir(
	ctx context.Context,
	op *fuseops.ReadDirOp) error {
	err := fs.wrapped.ReadDir(ctx, op)
	return errno(err)
}

func (fs *errorMapping) ReleaseDirHandle(
	ctx context.Context,
	op *fuseops.ReleaseDirHandleOp) error {
	err := fs.wrapped.ReleaseDirHandle(ctx, op)
	return errno(err)
}

func (fs *errorMapping) OpenFile(
	ctx context.Context,
	op *fuseops.OpenFileOp) error {
	err := fs.wrapped.OpenFile(ctx, op)
	return errno(err)
}

func (fs *errorMapping) ReadFile(
	ctx context.Context,
	op *fuseops.ReadFileOp) error {
	err := fs.wrapped.ReadFile(ctx, op)
	return errno(err)
}

func (fs *errorMapping) WriteFile(
	ctx context.Context,
	op *fuseops.WriteFileOp) error {
	err := fs.wrapped.WriteFile(ctx, op)
	return errno(err)
}

func (fs *errorMapping) SyncFile(
	ctx context.Context,
	op *fuseops.SyncFileOp) error {
	err := fs.wrapped.SyncFile(ctx, op)
	return errno(err)
}

func (fs *errorMapping) FlushFile(
	ctx context.Context,
	op *fuseops.FlushFileOp) error {
	err := fs.wrapped.FlushFile(ctx, op)
	return errno(err)
}

func (fs *errorMapping) ReleaseFileHandle(
	ctx context.Context,
	op *fuseops.ReleaseFileHandleOp) error {
	err := fs.wrapped.ReleaseFileHandle(ctx, op)
	return errno(err)
}

func (fs *errorMapping) ReadSymlink(
	ctx context.Context,
	op *fuseops.ReadSymlinkOp) error {
	err := fs.wrapped.ReadSymlink(ctx, op)
	return errno(err)
}

func (fs *errorMapping) RemoveXattr(
	ctx context.Context,
	op *fuseops.RemoveXattrOp) error {
	err := fs.wrapped.RemoveXattr(ctx, op)
	return errno(err)
}

func (fs *errorMapping) GetXattr(
	ctx context.Context,
	op *fuseops.GetXattrOp) error {
	err := fs.wrapped.GetXattr(ctx, op)
	return errno(err)
}

func (fs *errorMapping) ListXattr(
	ctx context.Context,
	op *fuseops.ListXattrOp) error {
	err := fs.wrapped.ListXattr(ctx, op)
	return errno(err)
}

func (fs *errorMapping) SetXattr(
	ctx context.Context,
	op *fuseops.SetXattrOp) error {
	err := fs.wrapped.SetXattr(ctx, op)
	return errno(err)
}

func (fs *errorMapping) Fallocate(
	ctx context.Context,
	op *fuseops.FallocateOp) error {
	err := fs.wrapped.Fallocate(ctx, op)
	return errno(err)
}
<commit_msg>Log file system op errors before they are converted into an errno<commit_after>// Copyright 2021 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package wrappers

import (
	"context"
	"errors"
	"log"
	"net/http"
	"syscall"

	"github.com/googlecloudplatform/gcsfuse/internal/logger"
	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/fuse/fuseutil"
	"google.golang.org/api/googleapi"
)

func errno(err error) error {
	if err == nil {
		return nil
	}

	// Use existing em errno
	var errno syscall.Errno
	if errors.As(err, &errno) {
		return errno
	}

	// em op is interrupted
	if errors.Is(err, context.Canceled) {
		return syscall.EINTR
	}

	// Translate API errors into an em errno
	var apiErr *googleapi.Error
	if errors.As(err, &apiErr) {
		switch apiErr.Code {
		case http.StatusForbidden:
			return syscall.EACCES
		}
	}

	// Unknown errors
	return syscall.EIO
}

// WithErrorMapping wraps a FileSystem, processing the returned errors, and
// mapping them into syscall.Errno that can be understood by FUSE.
func WithErrorMapping(wrapped fuseutil.FileSystem) fuseutil.FileSystem {
	return &errorMapping{
		wrapped: wrapped,
		logger:  logger.NewError(""),
	}
}

type errorMapping struct {
	wrapped fuseutil.FileSystem
	logger  *log.Logger
}

func (em *errorMapping) mapError(op string, err error) error {
	fsErr := errno(err)
	em.logger.Printf("%s: %v: %v", op, fsErr, err)
	return fsErr
}

func (em *errorMapping) Destroy() {
	em.wrapped.Destroy()
}

func (em *errorMapping) StatFS(
	ctx context.Context,
	op *fuseops.StatFSOp) error {
	err := em.wrapped.StatFS(ctx, op)
	return em.mapError("StatFS", err)
}

func (em *errorMapping) LookUpInode(
	ctx context.Context,
	op *fuseops.LookUpInodeOp) error {
	err := em.wrapped.LookUpInode(ctx, op)
	return em.mapError("LookUpInode", err)
}

func (em *errorMapping) GetInodeAttributes(
	ctx context.Context,
	op *fuseops.GetInodeAttributesOp) error {
	err := em.wrapped.GetInodeAttributes(ctx, op)
	return em.mapError("GetInodeAttributes", err)
}

func (em *errorMapping) SetInodeAttributes(
	ctx context.Context,
	op *fuseops.SetInodeAttributesOp) error {
	err := em.wrapped.SetInodeAttributes(ctx, op)
	return em.mapError("SetInodeAttributes", err)
}

func (em *errorMapping) ForgetInode(
	ctx context.Context,
	op *fuseops.ForgetInodeOp) error {
	err := em.wrapped.ForgetInode(ctx, op)
	return em.mapError("ForgetInode", err)
}

func (em *errorMapping) MkDir(
	ctx context.Context,
	op *fuseops.MkDirOp) error {
	err := em.wrapped.MkDir(ctx, op)
	return em.mapError("MkDir", err)
}

func (em *errorMapping) MkNode(
	ctx context.Context,
	op *fuseops.MkNodeOp) error {
	err := em.wrapped.MkNode(ctx, op)
	return em.mapError("MkNode", err)
}

func (em *errorMapping) CreateFile(
	ctx context.Context,
	op *fuseops.CreateFileOp) error {
	err := em.wrapped.CreateFile(ctx, op)
	return em.mapError("CreateFile", err)
}

func (em *errorMapping) CreateLink(
	ctx context.Context,
	op *fuseops.CreateLinkOp) error {
	err := em.wrapped.CreateLink(ctx, op)
	return em.mapError("CreateLink", err)
}

func (em *errorMapping) CreateSymlink(
	ctx context.Context,
	op *fuseops.CreateSymlinkOp) error {
	err := em.wrapped.CreateSymlink(ctx, op)
	return em.mapError("CreateSymlink", err)
}

func (em *errorMapping) Rename(
	ctx context.Context,
	op *fuseops.RenameOp) error {
	err := em.wrapped.Rename(ctx, op)
	return em.mapError("Rename", err)
}

func (em *errorMapping) RmDir(
	ctx context.Context,
	op *fuseops.RmDirOp) error {
	err := em.wrapped.RmDir(ctx, op)
	return em.mapError("RmDir", err)
}

func (em *errorMapping) Unlink(
	ctx context.Context,
	op *fuseops.UnlinkOp) error {
	err := em.wrapped.Unlink(ctx, op)
	return em.mapError("Unlink", err)
}

func (em *errorMapping) OpenDir(
	ctx context.Context,
	op *fuseops.OpenDirOp) error {
	err := em.wrapped.OpenDir(ctx, op)
	return em.mapError("OpenDir", err)
}

func (em *errorMapping) ReadDir(
	ctx context.Context,
	op *fuseops.ReadDirOp) error {
	err := em.wrapped.ReadDir(ctx, op)
	return em.mapError("ReadDir", err)
}

func (em *errorMapping) ReleaseDirHandle(
	ctx context.Context,
	op *fuseops.ReleaseDirHandleOp) error {
	err := em.wrapped.ReleaseDirHandle(ctx, op)
	return em.mapError("ReleaseDirHandle", err)
}

func (em *errorMapping) OpenFile(
	ctx context.Context,
	op *fuseops.OpenFileOp) error {
	err := em.wrapped.OpenFile(ctx, op)
	return em.mapError("OpenFile", err)
}

func (em *errorMapping) ReadFile(
	ctx context.Context,
	op *fuseops.ReadFileOp) error {
	err := em.wrapped.ReadFile(ctx, op)
	return em.mapError("ReadFile", err)
}

func (em *errorMapping) WriteFile(
	ctx context.Context,
	op *fuseops.WriteFileOp) error {
	err := em.wrapped.WriteFile(ctx, op)
	return em.mapError("WriteFile", err)
}

func (em *errorMapping) SyncFile(
	ctx context.Context,
	op *fuseops.SyncFileOp) error {
	err := em.wrapped.SyncFile(ctx, op)
	return em.mapError("SyncFile", err)
}

func (em *errorMapping) FlushFile(
	ctx context.Context,
	op *fuseops.FlushFileOp) error {
	err := em.wrapped.FlushFile(ctx, op)
	return em.mapError("FlushFile", err)
}

func (em *errorMapping) ReleaseFileHandle(
	ctx context.Context,
	op *fuseops.ReleaseFileHandleOp) error {
	err := em.wrapped.ReleaseFileHandle(ctx, op)
	return em.mapError("ReleaseFileHandle", err)
}

func (em *errorMapping) ReadSymlink(
	ctx context.Context,
	op *fuseops.ReadSymlinkOp) error {
	err := em.wrapped.ReadSymlink(ctx, op)
	return em.mapError("ReadSymlink", err)
}

func (em *errorMapping) RemoveXattr(
	ctx context.Context,
	op *fuseops.RemoveXattrOp) error {
	err := em.wrapped.RemoveXattr(ctx, op)
	return em.mapError("RemoveXattr", err)
}

func (em *errorMapping) GetXattr(
	ctx context.Context,
	op *fuseops.GetXattrOp) error {
	err := em.wrapped.GetXattr(ctx, op)
	return em.mapError("GetXattr", err)
}

func (em *errorMapping) ListXattr(
	ctx context.Context,
	op *fuseops.ListXattrOp) error {
	err := em.wrapped.ListXattr(ctx, op)
	return em.mapError("ListXattr", err)
}

func (em *errorMapping) SetXattr(
	ctx context.Context,
	op *fuseops.SetXattrOp) error {
	err := em.wrapped.SetXattr(ctx, op)
	return em.mapError("SetXattr", err)
}

func (em *errorMapping) Fallocate(
	ctx context.Context,
	op *fuseops.FallocateOp) error {
	err := em.wrapped.Fallocate(ctx, op)
	return em.mapError("Fallocate", err)
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ssl

import (
	"crypto/rand"
	"crypto/rsa"
	"crypto/sha1"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/hex"
	"encoding/pem"
	"errors"
	"fmt"
	"io/ioutil"
	"math/big"
	"os"
	"time"

	"github.com/golang/glog"

	"k8s.io/ingress/core/pkg/ingress"
)

// AddOrUpdateCertAndKey creates a .pem file wth the cert and the key with the specified name
func AddOrUpdateCertAndKey(name string, cert, key, ca []byte) (*ingress.SSLCert, error) {
	pemName := fmt.Sprintf("%v.pem", name)
	pemFileName := fmt.Sprintf("%v/%v", ingress.DefaultSSLDirectory, pemName)

	tempPemFile, err := ioutil.TempFile(ingress.DefaultSSLDirectory, pemName)

	glog.V(3).Infof("Creating temp file %v for Keypair: %v", tempPemFile.Name(), pemName)
	if err != nil {
		return nil, fmt.Errorf("could not create temp pem file %v: %v", pemFileName, err)
	}

	_, err = tempPemFile.Write(cert)
	if err != nil {
		return nil, fmt.Errorf("could not write to pem file %v: %v", tempPemFile.Name(), err)
	}
	_, err = tempPemFile.Write([]byte("\n"))
	if err != nil {
		return nil, fmt.Errorf("could not write to pem file %v: %v", tempPemFile.Name(), err)
	}
	_, err = tempPemFile.Write(key)
	if err != nil {
		return nil, fmt.Errorf("could not write to pem file %v: %v", tempPemFile.Name(), err)
	}

	err = tempPemFile.Close()
	if err != nil {
		return nil, fmt.Errorf("could not close temp pem file %v: %v", tempPemFile.Name(), err)
	}

	pemCerts, err := ioutil.ReadFile(tempPemFile.Name())
	if err != nil {
		return nil, err
	}

	pemBlock, _ := pem.Decode(pemCerts)
	if pemBlock == nil {
		return nil, fmt.Errorf("No valid PEM formatted block found")
	}

	// If the file does not start with 'BEGIN CERTIFICATE' it's invalid and must not be used.
	if pemBlock.Type != "CERTIFICATE" {
		_ = os.Remove(tempPemFile.Name())
		return nil, fmt.Errorf("Certificate %v contains invalid data, and must be created with 'kubectl create secret tls'", name)
	}

	pemCert, err := x509.ParseCertificate(pemBlock.Bytes)
	if err != nil {
		return nil, err
	}

	cn := []string{pemCert.Subject.CommonName}
	if len(pemCert.DNSNames) > 0 {
		cn = append(cn, pemCert.DNSNames...)
	}

	err = os.Rename(tempPemFile.Name(), pemFileName)
	if err != nil {
		return nil, fmt.Errorf("could not move temp pem file %v to destination %v: %v", tempPemFile.Name(), pemFileName, err)
	}

	if len(ca) > 0 {
		bundle := x509.NewCertPool()
		bundle.AppendCertsFromPEM(ca)
		opts := x509.VerifyOptions{
			Roots: bundle,
		}

		_, err := pemCert.Verify(opts)
		if err != nil {
			oe := fmt.Sprintf("failed to verify certificate chain: \n\t%s\n", err)
			return nil, errors.New(oe)
		}

		caFile, err := os.OpenFile(pemFileName, os.O_RDWR|os.O_APPEND, 0600)
		if err != nil {
			return nil, fmt.Errorf("Could not open file %v for writing additional CA chains: %v", pemFileName, err)
		}

		defer caFile.Close()
		_, err = caFile.Write([]byte("\n"))
		if err != nil {
			return nil, fmt.Errorf("could not append CA to cert file %v: %v", pemFileName, err)
		}
		caFile.Write(ca)
		caFile.Write([]byte("\n"))

		return &ingress.SSLCert{
			CAFileName:  pemFileName,
			PemFileName: pemFileName,
			PemSHA:      PemSHA1(pemFileName),
			CN:          cn,
		}, nil
	}

	return &ingress.SSLCert{
		PemFileName: pemFileName,
		PemSHA:      PemSHA1(pemFileName),
		CN:          cn,
	}, nil
}

// AddCertAuth creates a .pem file with the specified CAs to be used in Cert Authentication
// If it's already exists, it's clobbered.
func AddCertAuth(name string, ca []byte) (*ingress.SSLCert, error) {

	caName := fmt.Sprintf("ca-%v.pem", name)
	caFileName := fmt.Sprintf("%v/%v", ingress.DefaultSSLDirectory, caName)

	pemCABlock, _ := pem.Decode(ca)
	if pemCABlock == nil {
		return nil, fmt.Errorf("No valid PEM formatted block found")
	}
	// If the first certificate does not start with 'BEGIN CERTIFICATE' it's invalid and must not be used.
	if pemCABlock.Type != "CERTIFICATE" {
		return nil, fmt.Errorf("CA File %v contains invalid data, and must be created only with PEM formated certificates", name)
	}

	_, err := x509.ParseCertificate(pemCABlock.Bytes)
	if err != nil {
		return nil, err
	}

	err = ioutil.WriteFile(caFileName, ca, 0644)
	if err != nil {
		return nil, fmt.Errorf("could not write CA file %v: %v", caFileName, err)
	}

	glog.V(3).Infof("Created CA Certificate for authentication: %v", caFileName)
	return &ingress.SSLCert{
		CAFileName:  caFileName,
		PemFileName: caFileName,
		PemSHA:      PemSHA1(caFileName),
	}, nil
}

// SearchDHParamFile iterates all the secrets mounted inside the /etc/nginx-ssl directory
// in order to find a file with the name dhparam.pem. If such file exists it will
// returns the path. If not it just returns an empty string
func SearchDHParamFile(baseDir string) string {
	files, _ := ioutil.ReadDir(baseDir)
	for _, file := range files {
		if !file.IsDir() {
			continue
		}

		dhPath := fmt.Sprintf("%v/%v/dhparam.pem", baseDir, file.Name())
		if _, err := os.Stat(dhPath); err == nil {
			glog.Infof("using file '%v' for parameter ssl_dhparam", dhPath)
			return dhPath
		}
	}

	glog.Warning("no file dhparam.pem found in secrets")
	return ""
}

// PemSHA1 returns the SHA1 of a pem file. This is used to
// reload NGINX in case a secret with a SSL certificate changed.
func PemSHA1(filename string) string {
	hasher := sha1.New()
	s, err := ioutil.ReadFile(filename)
	if err != nil {
		return ""
	}

	hasher.Write(s)
	return hex.EncodeToString(hasher.Sum(nil))
}

// GetFakeSSLCert creates a Self Signed Certificate
// Based in the code https://golang.org/src/crypto/tls/generate_cert.go
func GetFakeSSLCert() ([]byte, []byte) {

	var priv interface{}
	var err error

	priv, err = rsa.GenerateKey(rand.Reader, 2048)

	if err != nil {
		glog.Fatalf("failed to generate fake private key: %s", err)
	}

	notBefore := time.Now()
	// This certificate is valid for 365 days
	notAfter := notBefore.Add(365 * 24 * time.Hour)

	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)

	if err != nil {
		glog.Fatalf("failed to generate fake serial number: %s", err)
	}

	template := x509.Certificate{
		SerialNumber: serialNumber,
		Subject: pkix.Name{
			Organization: []string{"Acme Co"},
			CommonName:   "Kubernetes Ingress Controller Fake Certificate",
		},
		NotBefore: notBefore,
		NotAfter:  notAfter,

		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
		BasicConstraintsValid: true,
		DNSNames:              []string{"ingress.local"},
	}
	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.(*rsa.PrivateKey).PublicKey, priv)
	if err != nil {
		glog.Fatalf("Failed to create fake certificate: %s", err)
	}

	cert := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes})

	key := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv.(*rsa.PrivateKey))})

	return cert, key
}
<commit_msg>Temporary PEM Files cleanup<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ssl

import (
	"crypto/rand"
	"crypto/rsa"
	"crypto/sha1"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/hex"
	"encoding/pem"
	"errors"
	"fmt"
	"io/ioutil"
	"math/big"
	"os"
	"time"

	"github.com/golang/glog"

	"k8s.io/ingress/core/pkg/ingress"
)

// AddOrUpdateCertAndKey creates a .pem file wth the cert and the key with the specified name
func AddOrUpdateCertAndKey(name string, cert, key, ca []byte) (*ingress.SSLCert, error) {
	pemName := fmt.Sprintf("%v.pem", name)
	pemFileName := fmt.Sprintf("%v/%v", ingress.DefaultSSLDirectory, pemName)

	tempPemFile, err := ioutil.TempFile(ingress.DefaultSSLDirectory, pemName)

	glog.V(3).Infof("Creating temp file %v for Keypair: %v", tempPemFile.Name(), pemName)
	if err != nil {
		return nil, fmt.Errorf("could not create temp pem file %v: %v", pemFileName, err)
	}

	_, err = tempPemFile.Write(cert)
	if err != nil {
		return nil, fmt.Errorf("could not write to pem file %v: %v", tempPemFile.Name(), err)
	}
	_, err = tempPemFile.Write([]byte("\n"))
	if err != nil {
		return nil, fmt.Errorf("could not write to pem file %v: %v", tempPemFile.Name(), err)
	}
	_, err = tempPemFile.Write(key)
	if err != nil {
		return nil, fmt.Errorf("could not write to pem file %v: %v", tempPemFile.Name(), err)
	}

	err = tempPemFile.Close()
	if err != nil {
		return nil, fmt.Errorf("could not close temp pem file %v: %v", tempPemFile.Name(), err)
	}

	pemCerts, err := ioutil.ReadFile(tempPemFile.Name())
	if err != nil {
		_ = os.Remove(tempPemFile.Name())
		return nil, err
	}

	pemBlock, _ := pem.Decode(pemCerts)
	if pemBlock == nil {
		_ = os.Remove(tempPemFile.Name())
		return nil, fmt.Errorf("No valid PEM formatted block found")
	}

	// If the file does not start with 'BEGIN CERTIFICATE' it's invalid and must not be used.
	if pemBlock.Type != "CERTIFICATE" {
		_ = os.Remove(tempPemFile.Name())
		return nil, fmt.Errorf("Certificate %v contains invalid data, and must be created with 'kubectl create secret tls'", name)
	}

	pemCert, err := x509.ParseCertificate(pemBlock.Bytes)
	if err != nil {
		_ = os.Remove(tempPemFile.Name())
		return nil, err
	}

	cn := []string{pemCert.Subject.CommonName}
	if len(pemCert.DNSNames) > 0 {
		cn = append(cn, pemCert.DNSNames...)
	}

	err = os.Rename(tempPemFile.Name(), pemFileName)
	if err != nil {
		return nil, fmt.Errorf("could not move temp pem file %v to destination %v: %v", tempPemFile.Name(), pemFileName, err)
	}

	if len(ca) > 0 {
		bundle := x509.NewCertPool()
		bundle.AppendCertsFromPEM(ca)
		opts := x509.VerifyOptions{
			Roots: bundle,
		}

		_, err := pemCert.Verify(opts)
		if err != nil {
			oe := fmt.Sprintf("failed to verify certificate chain: \n\t%s\n", err)
			return nil, errors.New(oe)
		}

		caFile, err := os.OpenFile(pemFileName, os.O_RDWR|os.O_APPEND, 0600)
		if err != nil {
			return nil, fmt.Errorf("Could not open file %v for writing additional CA chains: %v", pemFileName, err)
		}

		defer caFile.Close()
		_, err = caFile.Write([]byte("\n"))
		if err != nil {
			return nil, fmt.Errorf("could not append CA to cert file %v: %v", pemFileName, err)
		}
		caFile.Write(ca)
		caFile.Write([]byte("\n"))

		return &ingress.SSLCert{
			CAFileName:  pemFileName,
			PemFileName: pemFileName,
			PemSHA:      PemSHA1(pemFileName),
			CN:          cn,
		}, nil
	}

	return &ingress.SSLCert{
		PemFileName: pemFileName,
		PemSHA:      PemSHA1(pemFileName),
		CN:          cn,
	}, nil
}

// AddCertAuth creates a .pem file with the specified CAs to be used in Cert Authentication
// If it's already exists, it's clobbered.
func AddCertAuth(name string, ca []byte) (*ingress.SSLCert, error) {

	caName := fmt.Sprintf("ca-%v.pem", name)
	caFileName := fmt.Sprintf("%v/%v", ingress.DefaultSSLDirectory, caName)

	pemCABlock, _ := pem.Decode(ca)
	if pemCABlock == nil {
		return nil, fmt.Errorf("No valid PEM formatted block found")
	}
	// If the first certificate does not start with 'BEGIN CERTIFICATE' it's invalid and must not be used.
	if pemCABlock.Type != "CERTIFICATE" {
		return nil, fmt.Errorf("CA File %v contains invalid data, and must be created only with PEM formated certificates", name)
	}

	_, err := x509.ParseCertificate(pemCABlock.Bytes)
	if err != nil {
		return nil, err
	}

	err = ioutil.WriteFile(caFileName, ca, 0644)
	if err != nil {
		return nil, fmt.Errorf("could not write CA file %v: %v", caFileName, err)
	}

	glog.V(3).Infof("Created CA Certificate for authentication: %v", caFileName)
	return &ingress.SSLCert{
		CAFileName:  caFileName,
		PemFileName: caFileName,
		PemSHA:      PemSHA1(caFileName),
	}, nil
}

// SearchDHParamFile iterates all the secrets mounted inside the /etc/nginx-ssl directory
// in order to find a file with the name dhparam.pem. If such file exists it will
// returns the path. If not it just returns an empty string
func SearchDHParamFile(baseDir string) string {
	files, _ := ioutil.ReadDir(baseDir)
	for _, file := range files {
		if !file.IsDir() {
			continue
		}

		dhPath := fmt.Sprintf("%v/%v/dhparam.pem", baseDir, file.Name())
		if _, err := os.Stat(dhPath); err == nil {
			glog.Infof("using file '%v' for parameter ssl_dhparam", dhPath)
			return dhPath
		}
	}

	glog.Warning("no file dhparam.pem found in secrets")
	return ""
}

// PemSHA1 returns the SHA1 of a pem file. This is used to
// reload NGINX in case a secret with a SSL certificate changed.
func PemSHA1(filename string) string {
	hasher := sha1.New()
	s, err := ioutil.ReadFile(filename)
	if err != nil {
		return ""
	}

	hasher.Write(s)
	return hex.EncodeToString(hasher.Sum(nil))
}

// GetFakeSSLCert creates a Self Signed Certificate
// Based in the code https://golang.org/src/crypto/tls/generate_cert.go
func GetFakeSSLCert() ([]byte, []byte) {

	var priv interface{}
	var err error

	priv, err = rsa.GenerateKey(rand.Reader, 2048)

	if err != nil {
		glog.Fatalf("failed to generate fake private key: %s", err)
	}

	notBefore := time.Now()
	// This certificate is valid for 365 days
	notAfter := notBefore.Add(365 * 24 * time.Hour)

	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)

	if err != nil {
		glog.Fatalf("failed to generate fake serial number: %s", err)
	}

	template := x509.Certificate{
		SerialNumber: serialNumber,
		Subject: pkix.Name{
			Organization: []string{"Acme Co"},
			CommonName:   "Kubernetes Ingress Controller Fake Certificate",
		},
		NotBefore: notBefore,
		NotAfter:  notAfter,

		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
		BasicConstraintsValid: true,
		DNSNames:              []string{"ingress.local"},
	}
	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.(*rsa.PrivateKey).PublicKey, priv)
	if err != nil {
		glog.Fatalf("Failed to create fake certificate: %s", err)
	}

	cert := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes})

	key := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv.(*rsa.PrivateKey))})

	return cert, key
}
<|endoftext|>


<commit_before>package plist

import (
	"testing"
	"testing/quick"
)

func TestCFData(t *testing.T) {
	f := func(data []byte) []byte { return data }
	g := func(data []byte) []byte {
		cfData := convertBytesToCFData(data)
		defer cfRelease(cfTypeRef(cfData))
		return convertCFDataToBytes(cfData)
	}
	if err := quick.CheckEqual(f, g, nil); err != nil {
		t.Error(err)
	}
}
<commit_msg>Add a test for CFStrings<commit_after>package plist

import (
	"testing"
	"testing/quick"
)

func TestCFData(t *testing.T) {
	f := func(data []byte) []byte { return data }
	g := func(data []byte) []byte {
		cfData := convertBytesToCFData(data)
		if cfData == nil {
			t.Fatal("CFDataRef is NULL")
		}
		defer cfRelease(cfTypeRef(cfData))
		return convertCFDataToBytes(cfData)
	}
	if err := quick.CheckEqual(f, g, nil); err != nil {
		t.Error(err)
	}
}

func TestCFString(t *testing.T) {
	// because the generator for string produces invalid strings,
	// lets generate []runes instead and convert those to strings in the function
	f := func(runes []rune) string { return string(runes) }
	g := func(runes []rune) string {
		cfStr := convertStringToCFString(string(runes))
		if cfStr == nil {
			t.Fatal("CFStringRef is NULL (%#v)", runes)
		}
		defer cfRelease(cfTypeRef(cfStr))
		return convertCFStringToString(cfStr)
	}
	if err := quick.CheckEqual(f, g, nil); err != nil {
		t.Error(err)
	}
}
<|endoftext|>


<commit_before>package peerstore

import (
	"context"
	"sort"
	"sync"
	"time"

	peer "github.com/libp2p/go-libp2p-peer"
	addr "github.com/libp2p/go-libp2p-peerstore/addr"
	ma "github.com/multiformats/go-multiaddr"
)

const (

	// TempAddrTTL is the ttl used for a short lived address
	TempAddrTTL = time.Second * 10

	// ProviderAddrTTL is the TTL of an address we've received from a provider.
	// This is also a temporary address, but lasts longer. After this expires,
	// the records we return will require an extra lookup.
	ProviderAddrTTL = time.Minute * 10

	// RecentlyConnectedAddrTTL is used when we recently connected to a peer.
	// It means that we are reasonably certain of the peer's address.
	RecentlyConnectedAddrTTL = time.Minute * 10

	// OwnObservedAddrTTL is used for our own external addresses observed by peers.
	OwnObservedAddrTTL = time.Minute * 10

	// PermanentAddrTTL is the ttl for a "permanent address" (e.g. bootstrap nodes)
	// if we haven't shipped you an update to ipfs in 356 days
	// we probably arent running the same bootstrap nodes...
	PermanentAddrTTL = time.Hour * 24 * 356

	// ConnectedAddrTTL is the ttl used for the addresses of a peer to whom
	// we're connected directly. This is basically permanent, as we will
	// clear them + re-add under a TempAddrTTL after disconnecting.
	ConnectedAddrTTL = PermanentAddrTTL
)

type expiringAddr struct {
	Addr ma.Multiaddr
	TTL  time.Time
}

func (e *expiringAddr) ExpiredBy(t time.Time) bool {
	return t.After(e.TTL)
}

type addrSet map[string]expiringAddr

// AddrManager manages addresses.
// The zero-value is ready to be used.
type AddrManager struct {
	addrmu sync.Mutex // guards addrs
	addrs  map[peer.ID]addrSet

	addrSubs map[peer.ID][]*addrSub
}

// ensures the AddrManager is initialized.
// So we can use the zero value.
func (mgr *AddrManager) init() {
	if mgr.addrs == nil {
		mgr.addrs = make(map[peer.ID]addrSet)
	}
	if mgr.addrSubs == nil {
		mgr.addrSubs = make(map[peer.ID][]*addrSub)
	}
}

func (mgr *AddrManager) Peers() []peer.ID {
	mgr.addrmu.Lock()
	defer mgr.addrmu.Unlock()
	if mgr.addrs == nil {
		return nil
	}

	pids := make([]peer.ID, 0, len(mgr.addrs))
	for pid := range mgr.addrs {
		pids = append(pids, pid)
	}
	return pids
}

// AddAddr calls AddAddrs(p, []ma.Multiaddr{addr}, ttl)
func (mgr *AddrManager) AddAddr(p peer.ID, addr ma.Multiaddr, ttl time.Duration) {
	mgr.AddAddrs(p, []ma.Multiaddr{addr}, ttl)
}

// AddAddrs gives AddrManager addresses to use, with a given ttl
// (time-to-live), after which the address is no longer valid.
// If the manager has a longer TTL, the operation is a no-op for that address
func (mgr *AddrManager) AddAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duration) {
	mgr.addrmu.Lock()
	defer mgr.addrmu.Unlock()

	// if ttl is zero, exit. nothing to do.
	if ttl <= 0 {
		return
	}

	// so zero value can be used
	mgr.init()

	amap, found := mgr.addrs[p]
	if !found {
		amap = make(addrSet)
		mgr.addrs[p] = amap
	}

	subs := mgr.addrSubs[p]

	// only expand ttls
	exp := time.Now().Add(ttl)
	for _, addr := range addrs {
		if addr == nil {
			log.Warningf("was passed nil multiaddr for %s", p)
			continue
		}

		addrstr := string(addr.Bytes())
		a, found := amap[addrstr]
		if !found || exp.After(a.TTL) {
			amap[addrstr] = expiringAddr{Addr: addr, TTL: exp}

			for _, sub := range subs {
				sub.pubAddr(addr)
			}
		}
	}
}

// SetAddr calls mgr.SetAddrs(p, addr, ttl)
func (mgr *AddrManager) SetAddr(p peer.ID, addr ma.Multiaddr, ttl time.Duration) {
	mgr.SetAddrs(p, []ma.Multiaddr{addr}, ttl)
}

// SetAddrs sets the ttl on addresses. This clears any TTL there previously.
// This is used when we receive the best estimate of the validity of an address.
func (mgr *AddrManager) SetAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duration) {
	mgr.addrmu.Lock()
	defer mgr.addrmu.Unlock()

	// so zero value can be used
	mgr.init()

	amap, found := mgr.addrs[p]
	if !found {
		amap = make(addrSet)
		mgr.addrs[p] = amap
	}

	subs := mgr.addrSubs[p]

	exp := time.Now().Add(ttl)
	for _, addr := range addrs {
		if addr == nil {
			log.Warningf("was passed nil multiaddr for %s", p)
			continue
		}
		// re-set all of them for new ttl.
		addrs := string(addr.Bytes())

		if ttl > 0 {
			amap[addrs] = expiringAddr{Addr: addr, TTL: exp}

			for _, sub := range subs {
				sub.pubAddr(addr)
			}
		} else {
			delete(amap, addrs)
		}
	}
}

// Addresses returns all known (and valid) addresses for a given
func (mgr *AddrManager) Addrs(p peer.ID) []ma.Multiaddr {
	mgr.addrmu.Lock()
	defer mgr.addrmu.Unlock()

	// not initialized? nothing to give.
	if mgr.addrs == nil {
		return nil
	}

	maddrs, found := mgr.addrs[p]
	if !found {
		return nil
	}

	now := time.Now()
	good := make([]ma.Multiaddr, 0, len(maddrs))
	var expired []string
	for s, m := range maddrs {
		if m.ExpiredBy(now) {
			expired = append(expired, s)
		} else {
			good = append(good, m.Addr)
		}
	}

	// clean up the expired ones.
	for _, s := range expired {
		delete(maddrs, s)
	}
	if len(maddrs) == 0 {
		delete(mgr.addrs, p)
	}
	return good
}

// ClearAddresses removes all previously stored addresses
func (mgr *AddrManager) ClearAddrs(p peer.ID) {
	mgr.addrmu.Lock()
	defer mgr.addrmu.Unlock()
	mgr.init()

	delete(mgr.addrs, p)
}

func (mgr *AddrManager) removeSub(p peer.ID, s *addrSub) {
	mgr.addrmu.Lock()
	defer mgr.addrmu.Unlock()
	subs := mgr.addrSubs[p]
	if len(subs) == 1 {
		if subs[0] != s {
			return
		}
		delete(mgr.addrSubs, p)
		return
	}
	for i, v := range subs {
		if v == s {
			subs[i] = subs[len(subs)-1]
			subs[len(subs)-1] = nil
			mgr.addrSubs[p] = subs[:len(subs)-1]
			return
		}
	}
}

type addrSub struct {
	pubch  chan ma.Multiaddr
	lk     sync.Mutex
	buffer []ma.Multiaddr
	ctx    context.Context
}

func (s *addrSub) pubAddr(a ma.Multiaddr) {
	select {
	case s.pubch <- a:
	case <-s.ctx.Done():
	}
}

func (mgr *AddrManager) AddrStream(ctx context.Context, p peer.ID) <-chan ma.Multiaddr {
	mgr.addrmu.Lock()
	defer mgr.addrmu.Unlock()
	mgr.init()

	sub := &addrSub{pubch: make(chan ma.Multiaddr), ctx: ctx}

	out := make(chan ma.Multiaddr)

	mgr.addrSubs[p] = append(mgr.addrSubs[p], sub)

	baseaddrset := mgr.addrs[p]
	var initial []ma.Multiaddr
	for _, a := range baseaddrset {
		initial = append(initial, a.Addr)
	}

	sort.Sort(addr.AddrList(initial))

	go func(buffer []ma.Multiaddr) {
		defer close(out)

		sent := make(map[string]bool)
		var outch chan ma.Multiaddr

		for _, a := range buffer {
			sent[a.String()] = true
		}

		var next ma.Multiaddr
		if len(buffer) > 0 {
			next = buffer[0]
			buffer = buffer[1:]
			outch = out
		}

		for {
			select {
			case outch <- next:
				if len(buffer) > 0 {
					next = buffer[0]
					buffer = buffer[1:]
				} else {
					outch = nil
					next = nil
				}
			case naddr := <-sub.pubch:
				if sent[naddr.String()] {
					continue
				}

				sent[naddr.String()] = true
				if next == nil {
					next = naddr
					outch = out
				} else {
					buffer = append(buffer, naddr)
				}
			case <-ctx.Done():
				mgr.removeSub(p, sub)
				return
			}
		}

	}(initial)

	return out
}
<commit_msg>preallocate AddrStream buffer<commit_after>package peerstore

import (
	"context"
	"sort"
	"sync"
	"time"

	peer "github.com/libp2p/go-libp2p-peer"
	addr "github.com/libp2p/go-libp2p-peerstore/addr"
	ma "github.com/multiformats/go-multiaddr"
)

const (

	// TempAddrTTL is the ttl used for a short lived address
	TempAddrTTL = time.Second * 10

	// ProviderAddrTTL is the TTL of an address we've received from a provider.
	// This is also a temporary address, but lasts longer. After this expires,
	// the records we return will require an extra lookup.
	ProviderAddrTTL = time.Minute * 10

	// RecentlyConnectedAddrTTL is used when we recently connected to a peer.
	// It means that we are reasonably certain of the peer's address.
	RecentlyConnectedAddrTTL = time.Minute * 10

	// OwnObservedAddrTTL is used for our own external addresses observed by peers.
	OwnObservedAddrTTL = time.Minute * 10

	// PermanentAddrTTL is the ttl for a "permanent address" (e.g. bootstrap nodes)
	// if we haven't shipped you an update to ipfs in 356 days
	// we probably arent running the same bootstrap nodes...
	PermanentAddrTTL = time.Hour * 24 * 356

	// ConnectedAddrTTL is the ttl used for the addresses of a peer to whom
	// we're connected directly. This is basically permanent, as we will
	// clear them + re-add under a TempAddrTTL after disconnecting.
	ConnectedAddrTTL = PermanentAddrTTL
)

type expiringAddr struct {
	Addr ma.Multiaddr
	TTL  time.Time
}

func (e *expiringAddr) ExpiredBy(t time.Time) bool {
	return t.After(e.TTL)
}

type addrSet map[string]expiringAddr

// AddrManager manages addresses.
// The zero-value is ready to be used.
type AddrManager struct {
	addrmu sync.Mutex // guards addrs
	addrs  map[peer.ID]addrSet

	addrSubs map[peer.ID][]*addrSub
}

// ensures the AddrManager is initialized.
// So we can use the zero value.
func (mgr *AddrManager) init() {
	if mgr.addrs == nil {
		mgr.addrs = make(map[peer.ID]addrSet)
	}
	if mgr.addrSubs == nil {
		mgr.addrSubs = make(map[peer.ID][]*addrSub)
	}
}

func (mgr *AddrManager) Peers() []peer.ID {
	mgr.addrmu.Lock()
	defer mgr.addrmu.Unlock()
	if mgr.addrs == nil {
		return nil
	}

	pids := make([]peer.ID, 0, len(mgr.addrs))
	for pid := range mgr.addrs {
		pids = append(pids, pid)
	}
	return pids
}

// AddAddr calls AddAddrs(p, []ma.Multiaddr{addr}, ttl)
func (mgr *AddrManager) AddAddr(p peer.ID, addr ma.Multiaddr, ttl time.Duration) {
	mgr.AddAddrs(p, []ma.Multiaddr{addr}, ttl)
}

// AddAddrs gives AddrManager addresses to use, with a given ttl
// (time-to-live), after which the address is no longer valid.
// If the manager has a longer TTL, the operation is a no-op for that address
func (mgr *AddrManager) AddAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duration) {
	mgr.addrmu.Lock()
	defer mgr.addrmu.Unlock()

	// if ttl is zero, exit. nothing to do.
	if ttl <= 0 {
		return
	}

	// so zero value can be used
	mgr.init()

	amap, found := mgr.addrs[p]
	if !found {
		amap = make(addrSet)
		mgr.addrs[p] = amap
	}

	subs := mgr.addrSubs[p]

	// only expand ttls
	exp := time.Now().Add(ttl)
	for _, addr := range addrs {
		if addr == nil {
			log.Warningf("was passed nil multiaddr for %s", p)
			continue
		}

		addrstr := string(addr.Bytes())
		a, found := amap[addrstr]
		if !found || exp.After(a.TTL) {
			amap[addrstr] = expiringAddr{Addr: addr, TTL: exp}

			for _, sub := range subs {
				sub.pubAddr(addr)
			}
		}
	}
}

// SetAddr calls mgr.SetAddrs(p, addr, ttl)
func (mgr *AddrManager) SetAddr(p peer.ID, addr ma.Multiaddr, ttl time.Duration) {
	mgr.SetAddrs(p, []ma.Multiaddr{addr}, ttl)
}

// SetAddrs sets the ttl on addresses. This clears any TTL there previously.
// This is used when we receive the best estimate of the validity of an address.
func (mgr *AddrManager) SetAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duration) {
	mgr.addrmu.Lock()
	defer mgr.addrmu.Unlock()

	// so zero value can be used
	mgr.init()

	amap, found := mgr.addrs[p]
	if !found {
		amap = make(addrSet)
		mgr.addrs[p] = amap
	}

	subs := mgr.addrSubs[p]

	exp := time.Now().Add(ttl)
	for _, addr := range addrs {
		if addr == nil {
			log.Warningf("was passed nil multiaddr for %s", p)
			continue
		}
		// re-set all of them for new ttl.
		addrs := string(addr.Bytes())

		if ttl > 0 {
			amap[addrs] = expiringAddr{Addr: addr, TTL: exp}

			for _, sub := range subs {
				sub.pubAddr(addr)
			}
		} else {
			delete(amap, addrs)
		}
	}
}

// Addresses returns all known (and valid) addresses for a given
func (mgr *AddrManager) Addrs(p peer.ID) []ma.Multiaddr {
	mgr.addrmu.Lock()
	defer mgr.addrmu.Unlock()

	// not initialized? nothing to give.
	if mgr.addrs == nil {
		return nil
	}

	maddrs, found := mgr.addrs[p]
	if !found {
		return nil
	}

	now := time.Now()
	good := make([]ma.Multiaddr, 0, len(maddrs))
	var expired []string
	for s, m := range maddrs {
		if m.ExpiredBy(now) {
			expired = append(expired, s)
		} else {
			good = append(good, m.Addr)
		}
	}

	// clean up the expired ones.
	for _, s := range expired {
		delete(maddrs, s)
	}
	if len(maddrs) == 0 {
		delete(mgr.addrs, p)
	}
	return good
}

// ClearAddresses removes all previously stored addresses
func (mgr *AddrManager) ClearAddrs(p peer.ID) {
	mgr.addrmu.Lock()
	defer mgr.addrmu.Unlock()
	mgr.init()

	delete(mgr.addrs, p)
}

func (mgr *AddrManager) removeSub(p peer.ID, s *addrSub) {
	mgr.addrmu.Lock()
	defer mgr.addrmu.Unlock()
	subs := mgr.addrSubs[p]
	if len(subs) == 1 {
		if subs[0] != s {
			return
		}
		delete(mgr.addrSubs, p)
		return
	}
	for i, v := range subs {
		if v == s {
			subs[i] = subs[len(subs)-1]
			subs[len(subs)-1] = nil
			mgr.addrSubs[p] = subs[:len(subs)-1]
			return
		}
	}
}

type addrSub struct {
	pubch  chan ma.Multiaddr
	lk     sync.Mutex
	buffer []ma.Multiaddr
	ctx    context.Context
}

func (s *addrSub) pubAddr(a ma.Multiaddr) {
	select {
	case s.pubch <- a:
	case <-s.ctx.Done():
	}
}

func (mgr *AddrManager) AddrStream(ctx context.Context, p peer.ID) <-chan ma.Multiaddr {
	mgr.addrmu.Lock()
	defer mgr.addrmu.Unlock()
	mgr.init()

	sub := &addrSub{pubch: make(chan ma.Multiaddr), ctx: ctx}

	out := make(chan ma.Multiaddr)

	mgr.addrSubs[p] = append(mgr.addrSubs[p], sub)

	baseaddrset := mgr.addrs[p]
	initial := make([]ma.Multiaddr, 0, len(baseaddrset))
	for _, a := range baseaddrset {
		initial = append(initial, a.Addr)
	}

	sort.Sort(addr.AddrList(initial))

	go func(buffer []ma.Multiaddr) {
		defer close(out)

		sent := make(map[string]bool, len(buffer))
		var outch chan ma.Multiaddr

		for _, a := range buffer {
			sent[a.String()] = true
		}

		var next ma.Multiaddr
		if len(buffer) > 0 {
			next = buffer[0]
			buffer = buffer[1:]
			outch = out
		}

		for {
			select {
			case outch <- next:
				if len(buffer) > 0 {
					next = buffer[0]
					buffer = buffer[1:]
				} else {
					outch = nil
					next = nil
				}
			case naddr := <-sub.pubch:
				if sent[naddr.String()] {
					continue
				}

				sent[naddr.String()] = true
				if next == nil {
					next = naddr
					outch = out
				} else {
					buffer = append(buffer, naddr)
				}
			case <-ctx.Done():
				mgr.removeSub(p, sub)
				return
			}
		}

	}(initial)

	return out
}
<|endoftext|>


<commit_before>package scheduler

import (
	"encoding/json"
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/jhunt/go-log"

	"github.com/shieldproject/shield/db"
)

var next = 0

type Chore struct {
	ID         string
	TaskUUID   string
	Encryption string

	Do func(chore Chore)

	Stdout chan string
	Stderr chan string
	Exit   chan int
	Cancel chan bool
}

func NewChore(id string, do func(Chore)) Chore {
	next += 1
	return Chore{
		ID:       fmt.Sprintf("%s-%08d", time.Now().Format("20060102-150405"), next),
		TaskUUID: id,
		Do:       do,

		Stdout: make(chan string),
		Stderr: make(chan string),
		Exit:   make(chan int),
		Cancel: make(chan bool),
	}
}

func (chore Chore) String() string {
	return fmt.Sprintf("chore %s", chore.ID)
}

func (chore Chore) Infof(msg string, args ...interface{}) {
	log.Debugf(chore.String()+": stdout: "+msg, args...)
	chore.Stdout <- fmt.Sprintf(msg+"\n", args...)
}

func (chore Chore) Errorf(msg string, args ...interface{}) {
	log.Debugf(chore.String()+": stderr: "+msg, args...)
	chore.Stderr <- fmt.Sprintf(msg+"\n", args...)
}

func (chore Chore) UnixExit(rc int) {
	defer func() {
		recover()
	}()

	chore.Exit <- rc
	close(chore.Exit)
	log.Debugf("%s: exiting %d", chore, rc)
}

func (w *Worker) Execute(chore Chore) {
	defer func() {
		if err := recover(); err != nil {
			log.Errorf("%s: %s", chore, err)
			w.db.UpdateTaskLog(chore.TaskUUID, fmt.Sprintf("\n\nERROR: %s\n\n", err))

			log.Errorf("%s: FAILING task '%s' in database", chore, chore.TaskUUID)
			w.db.FailTask(chore.TaskUUID, time.Now())
		}
	}()

	var wait sync.WaitGroup

	w.Reserve(chore.TaskUUID)
	defer w.Release()

	log.Infof("%s: %s executing chore for task '%s'", chore, w, chore.TaskUUID)
	w.db.StartTask(chore.TaskUUID, time.Now())

	log.Debugf("%s: spinning up [stderr] goroutine to watch chore stderr and update the task log...", chore)
	wait.Add(1)
	go func() {
		for s := range chore.Stderr {
			w.db.UpdateTaskLog(chore.TaskUUID, s)
		}
		log.Debugf("%s: no more standard error; [stderr] goroutine shutting down...", chore)
		wait.Done()
	}()

	log.Debugf("%s: spinning up [stdout] goroutine to watch chore stdout and accumulate the output...", chore)
	output := ""
	wait.Add(1)
	go func() {
		for s := range chore.Stdout {
			output += s
		}
		log.Debugf("%s: no more standard output; [stdout] goroutine shutting down...", chore)
		wait.Done()
	}()

	log.Debugf("%s: spinning up [exit] goroutine to watch chore exit status and remember it...", chore)
	rc := 0
	wait.Add(1)
	go func() {
		rc = <-chore.Exit
		log.Debugf("%s: rc %d noted; [exit] goroutine shutting down...", chore, rc)
		wait.Done()
	}()

	log.Debugf("%s: spinning up [main] goroutine to execute chore `do' function...", chore)
	wait.Add(1)
	go func() {
		chore.Do(chore)
		log.Debugf("%s: chore execution complete; [main] goroutine shutting down...", chore)

		chore.UnixExit(0) /* catch-all */
		close(chore.Stderr)
		close(chore.Stdout)
		wait.Done()
	}()

	log.Debugf("%s: waiting for chore to complete...", chore)
	wait.Wait()
	w.db.UpdateTaskLog(chore.TaskUUID, "\n\n------\n")

	task, err := w.db.GetTask(chore.TaskUUID)
	if err != nil {
		panic(fmt.Errorf("failed to retrieve task '%s' from database: %s", chore.TaskUUID, err))
	}

	switch task.Op {
	case db.BackupOperation:
		output = strings.TrimSpace(output)
		w.db.UpdateTaskLog(task.UUID, fmt.Sprintf("BACKUP: `%s`\n", output))

		if rc != 0 {
			log.Debugf("%s: FAILING task '%s' in database", chore, chore.TaskUUID)
			w.db.FailTask(chore.TaskUUID, time.Now())
			return
		}

		log.Infof("%s: parsing output of %s operation, '%s'", chore, task.Op, output)
		var v struct {
			Key         string `json:"key"`
			Size        int64  `json:"archive_size"`
			Compression string `json:"compression"`
		}
		err := json.Unmarshal([]byte(output), &v)
		if err != nil {
			panic(fmt.Errorf("failed to unmarshal output [%s] from %s operation: %s", output, task.Op, err))
		}

		if v.Key == "" {
			panic(fmt.Errorf("%s: no restore key detected in %s operation output", chore, task.Op))
		}

		if v.Compression == "" {
			/* older shield-pipes will always bzip2; and if they aren't
			   reporting their compression type, it's gotta be bzip2 */
			v.Compression = "bzip2"
		}

		w.db.UpdateTaskLog(task.UUID, fmt.Sprintf("BACKUP: restore key  = %s\n", v.Key))
		w.db.UpdateTaskLog(task.UUID, fmt.Sprintf("BACKUP: archive size = %d bytes\n", v.Size))
		w.db.UpdateTaskLog(task.UUID, fmt.Sprintf("BACKUP: compression  = %s\n", v.Compression))

		log.Infof("%s: restore key for this %s operation is '%s'", chore, task.Op, v.Key)
		_, err = w.db.CreateTaskArchive(task.UUID, task.ArchiveUUID, v.Key, time.Now(),
			chore.Encryption, v.Compression, v.Size, task.TenantUUID)
		if err != nil {
			panic(fmt.Errorf("failed to create task archive database record '%s': %s", task.ArchiveUUID, err))
		}

		w.db.UpdateTaskLog(task.UUID, "\nBACKUP: recalculating cloud storage usage statistics...\n")
		store, err := w.db.GetStore(task.StoreUUID)
		if err != nil {
			log.Errorf("%s: failed to retrieve store from the database: %s", chore, err)
			w.db.UpdateTaskLog(task.UUID, "WARNING: store usage statistics were NOT updated...\n")

		} else {
			store.StorageUsed += v.Size
			store.ArchiveCount += 1
			store.DailyIncrease += v.Size
			err := w.db.UpdateStore(store)
			if err != nil {
				log.Errorf("%s: failed to update store in the database: %s", chore, err)
				w.db.UpdateTaskLog(task.UUID, "WARNING: store usage statistics were NOT updated...\n")
			} else {
				w.db.UpdateTaskLog(task.UUID, "    ... store usage statistics updated.\n")
			}
		}

		tenant, err := w.db.GetTenant(task.TenantUUID)
		if err != nil {
			log.Errorf("%s: failed to retrieve tenant from the database: %s", chore, err)
			w.db.UpdateTaskLog(task.UUID, "WARNING: tenant usage statistics were not updated...\n")

		} else {
			tenant.StorageUsed += v.Size
			tenant.ArchiveCount += 1
			tenant.DailyIncrease += v.Size
			_, err := w.db.UpdateTenant(tenant)
			if err != nil {
				log.Errorf("%s: failed to update tenant in the database: %s", chore, err)
				w.db.UpdateTaskLog(task.UUID, "WARNING: tenant usage statistics were not updated...\n")
			} else {
				w.db.UpdateTaskLog(task.UUID, "    ... tenant usage statistics updated.\n")
			}
		}
		w.db.UpdateTaskLog(task.UUID, "\n\n")

	case db.PurgeOperation:
		if rc != 0 {
			log.Debugf("%s: FAILING task '%s' in database", chore, chore.TaskUUID)
			w.db.UpdateTaskLog(task.UUID, "\nPURGE: operation failed; keeping archive metadata intact.\n")
			w.db.UpdateTaskLog(task.UUID, "PURGE: will try again later...\n")
			w.db.FailTask(chore.TaskUUID, time.Now())
			return
		}

		log.Infof("%s: purged archive '%s' from storage", chore, task.ArchiveUUID)
		err = w.db.PurgeArchive(task.ArchiveUUID)
		if err != nil {
			panic(fmt.Errorf("%s: failed to purge the archive record from the database: %s", chore, err))
		}

		w.db.UpdateTaskLog(task.UUID, "\nPURGE: recalculating cloud storage usage statistics...\n")
		archive, err := w.db.GetArchive(task.ArchiveUUID)
		if err != nil {
			panic(fmt.Errorf("%s: failed to retrieve archive record from the database: %s", chore, err))
		}

		store, err := w.db.GetStore(task.StoreUUID)
		if err != nil {
			log.Errorf("%s: failed to retrieve store from the database: %s", chore, err)
			w.db.UpdateTaskLog(task.UUID, "WARNING: store usage statistics were NOT updated...\n")

		} else {
			store.StorageUsed -= archive.Size
			store.ArchiveCount -= 1
			store.DailyIncrease -= archive.Size
			err := w.db.UpdateStore(store)
			if err != nil {
				log.Errorf("%s: failed to update store in the database: %s", chore, err)
				w.db.UpdateTaskLog(task.UUID, "WARNING: store usage statistics were NOT updated...\n")
			} else {
				w.db.UpdateTaskLog(task.UUID, "    ... store usage statistics updated.\n")
			}
		}

		tenant, err := w.db.GetTenant(task.TenantUUID)
		if err != nil {
			log.Errorf("%s: failed to retrieve tenant from the database: %s", chore, err)
			w.db.UpdateTaskLog(task.UUID, "WARNING: tenant usage statistics were not updated...\n")

		} else {
			tenant.StorageUsed -= archive.Size
			tenant.ArchiveCount -= 1
			tenant.DailyIncrease -= archive.Size
			_, err := w.db.UpdateTenant(tenant)
			if err != nil {
				log.Errorf("%s: failed to update tenant in the database: %s", chore, err)
				w.db.UpdateTaskLog(task.UUID, "WARNING: tenant usage statistics were not updated...\n")
			} else {
				w.db.UpdateTaskLog(task.UUID, "    ... tenant usage statistics updated.\n")
			}
		}
		w.db.UpdateTaskLog(task.UUID, "\n\n")

	case db.TestStoreOperation:
		var v struct {
			Healthy bool `json:"healthy"`
		}

		store, err := w.db.GetStore(task.StoreUUID)
		if err != nil {
			panic(fmt.Errorf("failed to retrieve store '%s' from database: %s", task.StoreUUID, err))
		}
		if store == nil {
			panic(fmt.Errorf("store '%s' not found in database", task.StoreUUID))
		}

		if rc == 0 {
			err = json.Unmarshal([]byte(output), &v)
			if err != nil {
				panic(fmt.Errorf("failed to unmarshal output [%s] from %s operation: %s", output, task.Op, err))
			}
			if v.Healthy {
				if store.Healthy != v.Healthy {
					w.db.UpdateTaskLog(task.UUID, "\nTEST-STORE: marking storage system as HEALTHY (recovery).\n")
				} else {
					w.db.UpdateTaskLog(task.UUID, "\nTEST-STORE: storage is still HEALTHY.\n")
				}
			} else {
				w.db.UpdateTaskLog(task.UUID, "\nTEST-STORE: marking storage system as UNHEALTHY.\n")
			}
			store.Healthy = v.Healthy

		} else {
			store.Healthy = false
			w.db.UpdateTaskLog(task.UUID, "\nTEST-STORE: marking storage system as UNHEALTHY.\n")
		}

		err = w.db.UpdateStore(store)
		if err != nil {
			panic(fmt.Errorf("failed to update store '%s' record in database: %s", task.StoreUUID, err))
		}

		if rc != 0 {
			log.Debugf("%s: FAILING task '%s' in database", chore, chore.TaskUUID)
			w.db.FailTask(chore.TaskUUID, time.Now())
			return
		}

	case db.AgentStatusOperation:
		if rc != 0 {
			log.Debugf("%s: FAILING task '%s' in database", chore, chore.TaskUUID)
			w.db.FailTask(chore.TaskUUID, time.Now())
			return
		}

		var v struct {
			Name    string `json:"name"`
			Version string `json:"version"`
			Health  string `json:"health"`
		}

		err = json.Unmarshal([]byte(output), &v)
		if err != nil {
			panic(fmt.Errorf("failed to unmarshal output [%s] from %s operation: %s", output, task.Op, err))
		}

		agents, err := w.db.GetAllAgents(&db.AgentFilter{Address: task.Agent})
		if err != nil {
			panic(fmt.Errorf("failed to retrieve agent '%s' from database: %s", task.Agent, err))
		}
		if len(agents) != 1 {
			panic(fmt.Errorf("found %d agent records for address '%s' (expected 1)", len(agents), task.Agent))
		}

		agent := agents[0]
		agent.Name = v.Name
		agent.Version = v.Version
		agent.Status = v.Health
		agent.RawMeta = output
		agent.LastCheckedAt = time.Now().Unix()
		err = w.db.UpdateAgent(agent)
		if err != nil {
			panic(fmt.Errorf("failed to update agent '%s' record in database: %s", task.Agent, err))
		}
	}

	log.Debugf("%s: completing task '%s' in database", chore, chore.TaskUUID)
	w.db.CompleteTask(chore.TaskUUID, time.Now())
}
<commit_msg>Update Agent Status on failed agent-status tasks<commit_after>package scheduler

import (
	"encoding/json"
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/jhunt/go-log"

	"github.com/shieldproject/shield/db"
)

var next = 0

type Chore struct {
	ID         string
	TaskUUID   string
	Encryption string

	Do func(chore Chore)

	Stdout chan string
	Stderr chan string
	Exit   chan int
	Cancel chan bool
}

func NewChore(id string, do func(Chore)) Chore {
	next += 1
	return Chore{
		ID:       fmt.Sprintf("%s-%08d", time.Now().Format("20060102-150405"), next),
		TaskUUID: id,
		Do:       do,

		Stdout: make(chan string),
		Stderr: make(chan string),
		Exit:   make(chan int),
		Cancel: make(chan bool),
	}
}

func (chore Chore) String() string {
	return fmt.Sprintf("chore %s", chore.ID)
}

func (chore Chore) Infof(msg string, args ...interface{}) {
	log.Debugf(chore.String()+": stdout: "+msg, args...)
	chore.Stdout <- fmt.Sprintf(msg+"\n", args...)
}

func (chore Chore) Errorf(msg string, args ...interface{}) {
	log.Debugf(chore.String()+": stderr: "+msg, args...)
	chore.Stderr <- fmt.Sprintf(msg+"\n", args...)
}

func (chore Chore) UnixExit(rc int) {
	defer func() {
		recover()
	}()

	chore.Exit <- rc
	close(chore.Exit)
	log.Debugf("%s: exiting %d", chore, rc)
}

func (w *Worker) Execute(chore Chore) {
	defer func() {
		if err := recover(); err != nil {
			log.Errorf("%s: %s", chore, err)
			w.db.UpdateTaskLog(chore.TaskUUID, fmt.Sprintf("\n\nERROR: %s\n\n", err))

			log.Errorf("%s: FAILING task '%s' in database", chore, chore.TaskUUID)
			w.db.FailTask(chore.TaskUUID, time.Now())
		}
	}()

	var wait sync.WaitGroup

	w.Reserve(chore.TaskUUID)
	defer w.Release()

	log.Infof("%s: %s executing chore for task '%s'", chore, w, chore.TaskUUID)
	w.db.StartTask(chore.TaskUUID, time.Now())

	log.Debugf("%s: spinning up [stderr] goroutine to watch chore stderr and update the task log...", chore)
	wait.Add(1)
	go func() {
		for s := range chore.Stderr {
			w.db.UpdateTaskLog(chore.TaskUUID, s)
		}
		log.Debugf("%s: no more standard error; [stderr] goroutine shutting down...", chore)
		wait.Done()
	}()

	log.Debugf("%s: spinning up [stdout] goroutine to watch chore stdout and accumulate the output...", chore)
	output := ""
	wait.Add(1)
	go func() {
		for s := range chore.Stdout {
			output += s
		}
		log.Debugf("%s: no more standard output; [stdout] goroutine shutting down...", chore)
		wait.Done()
	}()

	log.Debugf("%s: spinning up [exit] goroutine to watch chore exit status and remember it...", chore)
	rc := 0
	wait.Add(1)
	go func() {
		rc = <-chore.Exit
		log.Debugf("%s: rc %d noted; [exit] goroutine shutting down...", chore, rc)
		wait.Done()
	}()

	log.Debugf("%s: spinning up [main] goroutine to execute chore `do' function...", chore)
	wait.Add(1)
	go func() {
		chore.Do(chore)
		log.Debugf("%s: chore execution complete; [main] goroutine shutting down...", chore)

		chore.UnixExit(0) /* catch-all */
		close(chore.Stderr)
		close(chore.Stdout)
		wait.Done()
	}()

	log.Debugf("%s: waiting for chore to complete...", chore)
	wait.Wait()
	w.db.UpdateTaskLog(chore.TaskUUID, "\n\n------\n")

	task, err := w.db.GetTask(chore.TaskUUID)
	if err != nil {
		panic(fmt.Errorf("failed to retrieve task '%s' from database: %s", chore.TaskUUID, err))
	}

	switch task.Op {
	case db.BackupOperation:
		output = strings.TrimSpace(output)
		w.db.UpdateTaskLog(task.UUID, fmt.Sprintf("BACKUP: `%s`\n", output))

		if rc != 0 {
			log.Debugf("%s: FAILING task '%s' in database", chore, chore.TaskUUID)
			w.db.FailTask(chore.TaskUUID, time.Now())
			return
		}

		log.Infof("%s: parsing output of %s operation, '%s'", chore, task.Op, output)
		var v struct {
			Key         string `json:"key"`
			Size        int64  `json:"archive_size"`
			Compression string `json:"compression"`
		}
		err := json.Unmarshal([]byte(output), &v)
		if err != nil {
			panic(fmt.Errorf("failed to unmarshal output [%s] from %s operation: %s", output, task.Op, err))
		}

		if v.Key == "" {
			panic(fmt.Errorf("%s: no restore key detected in %s operation output", chore, task.Op))
		}

		if v.Compression == "" {
			/* older shield-pipes will always bzip2; and if they aren't
			   reporting their compression type, it's gotta be bzip2 */
			v.Compression = "bzip2"
		}

		w.db.UpdateTaskLog(task.UUID, fmt.Sprintf("BACKUP: restore key  = %s\n", v.Key))
		w.db.UpdateTaskLog(task.UUID, fmt.Sprintf("BACKUP: archive size = %d bytes\n", v.Size))
		w.db.UpdateTaskLog(task.UUID, fmt.Sprintf("BACKUP: compression  = %s\n", v.Compression))

		log.Infof("%s: restore key for this %s operation is '%s'", chore, task.Op, v.Key)
		_, err = w.db.CreateTaskArchive(task.UUID, task.ArchiveUUID, v.Key, time.Now(),
			chore.Encryption, v.Compression, v.Size, task.TenantUUID)
		if err != nil {
			panic(fmt.Errorf("failed to create task archive database record '%s': %s", task.ArchiveUUID, err))
		}

		w.db.UpdateTaskLog(task.UUID, "\nBACKUP: recalculating cloud storage usage statistics...\n")
		store, err := w.db.GetStore(task.StoreUUID)
		if err != nil {
			log.Errorf("%s: failed to retrieve store from the database: %s", chore, err)
			w.db.UpdateTaskLog(task.UUID, "WARNING: store usage statistics were NOT updated...\n")

		} else {
			store.StorageUsed += v.Size
			store.ArchiveCount += 1
			store.DailyIncrease += v.Size
			err := w.db.UpdateStore(store)
			if err != nil {
				log.Errorf("%s: failed to update store in the database: %s", chore, err)
				w.db.UpdateTaskLog(task.UUID, "WARNING: store usage statistics were NOT updated...\n")
			} else {
				w.db.UpdateTaskLog(task.UUID, "    ... store usage statistics updated.\n")
			}
		}

		tenant, err := w.db.GetTenant(task.TenantUUID)
		if err != nil {
			log.Errorf("%s: failed to retrieve tenant from the database: %s", chore, err)
			w.db.UpdateTaskLog(task.UUID, "WARNING: tenant usage statistics were not updated...\n")

		} else {
			tenant.StorageUsed += v.Size
			tenant.ArchiveCount += 1
			tenant.DailyIncrease += v.Size
			_, err := w.db.UpdateTenant(tenant)
			if err != nil {
				log.Errorf("%s: failed to update tenant in the database: %s", chore, err)
				w.db.UpdateTaskLog(task.UUID, "WARNING: tenant usage statistics were not updated...\n")
			} else {
				w.db.UpdateTaskLog(task.UUID, "    ... tenant usage statistics updated.\n")
			}
		}
		w.db.UpdateTaskLog(task.UUID, "\n\n")

	case db.PurgeOperation:
		if rc != 0 {
			log.Debugf("%s: FAILING task '%s' in database", chore, chore.TaskUUID)
			w.db.UpdateTaskLog(task.UUID, "\nPURGE: operation failed; keeping archive metadata intact.\n")
			w.db.UpdateTaskLog(task.UUID, "PURGE: will try again later...\n")
			w.db.FailTask(chore.TaskUUID, time.Now())
			return
		}

		log.Infof("%s: purged archive '%s' from storage", chore, task.ArchiveUUID)
		err = w.db.PurgeArchive(task.ArchiveUUID)
		if err != nil {
			panic(fmt.Errorf("%s: failed to purge the archive record from the database: %s", chore, err))
		}

		w.db.UpdateTaskLog(task.UUID, "\nPURGE: recalculating cloud storage usage statistics...\n")
		archive, err := w.db.GetArchive(task.ArchiveUUID)
		if err != nil {
			panic(fmt.Errorf("%s: failed to retrieve archive record from the database: %s", chore, err))
		}

		store, err := w.db.GetStore(task.StoreUUID)
		if err != nil {
			log.Errorf("%s: failed to retrieve store from the database: %s", chore, err)
			w.db.UpdateTaskLog(task.UUID, "WARNING: store usage statistics were NOT updated...\n")

		} else {
			store.StorageUsed -= archive.Size
			store.ArchiveCount -= 1
			store.DailyIncrease -= archive.Size
			err := w.db.UpdateStore(store)
			if err != nil {
				log.Errorf("%s: failed to update store in the database: %s", chore, err)
				w.db.UpdateTaskLog(task.UUID, "WARNING: store usage statistics were NOT updated...\n")
			} else {
				w.db.UpdateTaskLog(task.UUID, "    ... store usage statistics updated.\n")
			}
		}

		tenant, err := w.db.GetTenant(task.TenantUUID)
		if err != nil {
			log.Errorf("%s: failed to retrieve tenant from the database: %s", chore, err)
			w.db.UpdateTaskLog(task.UUID, "WARNING: tenant usage statistics were not updated...\n")

		} else {
			tenant.StorageUsed -= archive.Size
			tenant.ArchiveCount -= 1
			tenant.DailyIncrease -= archive.Size
			_, err := w.db.UpdateTenant(tenant)
			if err != nil {
				log.Errorf("%s: failed to update tenant in the database: %s", chore, err)
				w.db.UpdateTaskLog(task.UUID, "WARNING: tenant usage statistics were not updated...\n")
			} else {
				w.db.UpdateTaskLog(task.UUID, "    ... tenant usage statistics updated.\n")
			}
		}
		w.db.UpdateTaskLog(task.UUID, "\n\n")

	case db.TestStoreOperation:
		var v struct {
			Healthy bool `json:"healthy"`
		}

		store, err := w.db.GetStore(task.StoreUUID)
		if err != nil {
			panic(fmt.Errorf("failed to retrieve store '%s' from database: %s", task.StoreUUID, err))
		}
		if store == nil {
			panic(fmt.Errorf("store '%s' not found in database", task.StoreUUID))
		}

		if rc == 0 {
			err = json.Unmarshal([]byte(output), &v)
			if err != nil {
				panic(fmt.Errorf("failed to unmarshal output [%s] from %s operation: %s", output, task.Op, err))
			}
			if v.Healthy {
				if store.Healthy != v.Healthy {
					w.db.UpdateTaskLog(task.UUID, "\nTEST-STORE: marking storage system as HEALTHY (recovery).\n")
				} else {
					w.db.UpdateTaskLog(task.UUID, "\nTEST-STORE: storage is still HEALTHY.\n")
				}
			} else {
				w.db.UpdateTaskLog(task.UUID, "\nTEST-STORE: marking storage system as UNHEALTHY.\n")
			}
			store.Healthy = v.Healthy

		} else {
			store.Healthy = false
			w.db.UpdateTaskLog(task.UUID, "\nTEST-STORE: marking storage system as UNHEALTHY.\n")
		}

		err = w.db.UpdateStore(store)
		if err != nil {
			panic(fmt.Errorf("failed to update store '%s' record in database: %s", task.StoreUUID, err))
		}

		if rc != 0 {
			log.Debugf("%s: FAILING task '%s' in database", chore, chore.TaskUUID)
			w.db.FailTask(chore.TaskUUID, time.Now())
			return
		}

	case db.AgentStatusOperation:
		agent, err := w.db.GetAgentByAddress(task.Agent)
		if err != nil {
			log.Debugf("%s: FAILING task '%s' in database", chore, chore.TaskUUID)
			w.db.FailTask(chore.TaskUUID, time.Now())
			panic(fmt.Errorf("failed to retrieve agent '%s' from database: %s", task.Agent, err))
		}
		if agent == nil {
			log.Debugf("%s: FAILING task '%s' in database", chore, chore.TaskUUID)
			w.db.FailTask(chore.TaskUUID, time.Now())
			panic(fmt.Errorf("failed to retrieve agent '%s' from database: no such agent", task.Agent))
		}

		if rc == 0 {
			var v struct {
				Name    string `json:"name"`
				Version string `json:"version"`
				Health  string `json:"health"`
			}

			err = json.Unmarshal([]byte(output), &v)
			if err != nil {
				log.Debugf("%s: FAILING task '%s' in database", chore, chore.TaskUUID)
				w.db.FailTask(chore.TaskUUID, time.Now())
				panic(fmt.Errorf("failed to unmarshal output [%s] from %s operation: %s", output, task.Op, err))
			}

			agent.Name = v.Name
			agent.Version = v.Version
			agent.Status = v.Health
			agent.RawMeta = output
			agent.LastCheckedAt = time.Now().Unix()
		} else {
			agent.Status = "error"
			agent.LastCheckedAt = time.Now().Unix()
		}
		err = w.db.UpdateAgent(agent)
		if err != nil {
			log.Debugf("%s: FAILING task '%s' in database", chore, chore.TaskUUID)
			w.db.FailTask(chore.TaskUUID, time.Now())
			panic(fmt.Errorf("failed to update agent '%s' record in database: %s", task.Agent, err))
		}

		if rc != 0 {
			log.Debugf("%s: FAILING task '%s' in database", chore, chore.TaskUUID)
			w.db.FailTask(chore.TaskUUID, time.Now())
			return
		}
	}

	log.Debugf("%s: completing task '%s' in database", chore, chore.TaskUUID)
	w.db.CompleteTask(chore.TaskUUID, time.Now())
}
<|endoftext|>


<commit_before>package core

import (
	"errors"
	"io"

	"github.com/toorop/gopenstack/objectstorage/v1"
)

// DiskStore represents a physical disk store
type openstackStore struct {
	Region    string
	Container string
}

// newOpenstackStore check object storage and return a new openstackStore
func newOpenstackStore() (*openstackStore, error) {
	osPath := objectstorageV1.NewOsPathFromPath(Cfg.GetStoreSource())
	if !osPath.IsContainer() {
		return nil, errors.New("path " + Cfg.GetStoreDriver() + " is not a path to a valid openstack container")
	}
	// container exists ?
	container := &objectstorageV1.Container{
		Region: osPath.Region,
		Name:   osPath.Container,
	}
	err := container.Put(&objectstorageV1.ContainerRequestParameters{
		IfNoneMatch: true,
	})
	if err != nil {
		return nil, err
	}
	store := &openstackStore{
		Region:    osPath.Region,
		Container: osPath.Container,
	}
	return store, nil
}

// Put save key value in store
func (s *openstackStore) Put(key string, reader io.Reader) error {
	if key == "" {
		return errors.New("store.Put: key is empty")
	}
	object := objectstorageV1.Object{
		Name:      key,
		Region:    s.Region,
		Container: s.Container,
		RawData:   reader,
	}
	return object.Put(&objectstorageV1.ObjectRequestParameters{
		IfNoneMatch: true,
	})
}

// Get returns io.Reader corresponding to key
func (s *openstackStore) Get(key string) (io.Reader, error) {
	if key == "" {
		return nil, errors.New("store.Get: key is empty")
	}
	object := objectstorageV1.Object{
		Name:      key,
		Region:    s.Region,
		Container: s.Container,
	}
	err := object.Get(nil)
	if err != nil {
		return nil, err
	}
	return object.RawData, nil
}

// Del
func (s *openstackStore) Del(key string) error {
	if key == "" {
		return errors.New("store.Del: key is empty")
	}
	object := objectstorageV1.Object{
		Name:      key,
		Region:    s.Region,
		Container: s.Container,
	}
	return object.Delete(false)
}
<commit_msg>no error on store.Del if object is missing<commit_after>package core

import (
	"errors"
	"io"
	"strings"

	"github.com/toorop/gopenstack/objectstorage/v1"
)

// DiskStore represents a physical disk store
type openstackStore struct {
	Region    string
	Container string
}

// newOpenstackStore check object storage and return a new openstackStore
func newOpenstackStore() (*openstackStore, error) {
	osPath := objectstorageV1.NewOsPathFromPath(Cfg.GetStoreSource())
	if !osPath.IsContainer() {
		return nil, errors.New("path " + Cfg.GetStoreDriver() + " is not a path to a valid openstack container")
	}
	// container exists ?
	container := &objectstorageV1.Container{
		Region: osPath.Region,
		Name:   osPath.Container,
	}
	err := container.Put(&objectstorageV1.ContainerRequestParameters{
		IfNoneMatch: true,
	})
	if err != nil {
		return nil, err
	}
	store := &openstackStore{
		Region:    osPath.Region,
		Container: osPath.Container,
	}
	return store, nil
}

// Put save key value in store
func (s *openstackStore) Put(key string, reader io.Reader) error {
	if key == "" {
		return errors.New("store.Put: key is empty")
	}
	object := objectstorageV1.Object{
		Name:      key,
		Region:    s.Region,
		Container: s.Container,
		RawData:   reader,
	}
	return object.Put(&objectstorageV1.ObjectRequestParameters{
		IfNoneMatch: true,
	})
}

// Get returns io.Reader corresponding to key
func (s *openstackStore) Get(key string) (io.Reader, error) {
	if key == "" {
		return nil, errors.New("store.Get: key is empty")
	}
	object := objectstorageV1.Object{
		Name:      key,
		Region:    s.Region,
		Container: s.Container,
	}
	err := object.Get(nil)
	if err != nil {
		return nil, err
	}
	return object.RawData, nil
}

// Del
func (s *openstackStore) Del(key string) error {
	if key == "" {
		return errors.New("store.Del: key is empty")
	}
	object := objectstorageV1.Object{
		Name:      key,
		Region:    s.Region,
		Container: s.Container,
	}
	err := object.Delete(false)
	if err != nil && strings.HasPrefix("404 Not Found", err.Error()) {
		err = nil
	}
	return err
}
<|endoftext|>