<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"path"
	"regexp"
	"strings"

	dcl "github.com/GoogleCloudPlatform/declarative-resource-client-library/dcl"
	"github.com/golang/glog"
)

// Sample is the object containing sample data from DCL samples
type Sample struct {
	// Name of the file the sample was loaded from
	FileName string

	// Name is the name of a sample
	Name *string

	// Description is a short description of the sample
	Description *string

	// DependencyFileNames contains the filenames of every resource in the sample
	DependencyFileNames []string `yaml:"dependencies"`

	// PrimaryResource is the filename of the sample's primary resource
	PrimaryResource *string `yaml:"resource"`

	// Substitutions contains every substition in the sample
	Substitutions []Substitution `yaml:"substitutions"`

	IgnoreRead []string `yaml:"ignore_read"`

	// DependencyList is a list of objects containing metadata for each sample resource
	DependencyList []Dependency

	// The name of the test
	TestSlug string

	// The raw versions stated in the yaml
	Versions []string

	// A list of updates that the resource can transition between
	Updates []Update

	// HasGAEquivalent tells us if we should have `provider = google-beta`
	// in the testcase. (if the test doesn't have a ga version of the test)
	HasGAEquivalent bool

	// SamplesPath is the path to the directory where the original sample data is stored
	SamplesPath string

	// resourceReference is the resource the sample belongs to
	resourceReference *Resource

	// CustomCheck allows you to add a terraform check function to all tests
	CustomCheck []string `yaml:"check"`

	// CodeInject references reletive raw files that should be injected into the sample test
	CodeInject []string `yaml:"code_inject"`

	// DocHide specifies a list of samples to hide from docs
	DocHide []string `yaml:"doc_hide"`

	// Testhide specifies a list of samples to hide from tests
	Testhide []string `yaml:"test_hide"`

	// ExtraDependencies are the additional golang dependencies the injected code may require
	ExtraDependencies []string `yaml:"extra_dependencies"`

	// Type is the resource type.
	Type string `yaml:"type"`

	// Variables are the various attributes of the set of resources that need to be filled in.
	Variables []Variable `yaml:"variables"`
}

// Variable contains metadata about the types of variables in a sample.
type Variable struct {
	// Name is the variable name in the JSON.
	Name string `yaml:"name"`
	// Type is the variable type.
	Type string `yaml:"type"`
}

// Substitution contains metadata that varies for the sample context
type Substitution struct {
	// Substitution is the text to be substituted, e.g. topic_name
	Substitution *string

	// Value is the value of the substituted text
	Value *string `yaml:"value"`
}

// Dependency contains data that describes a single resource in a sample
type Dependency struct {
	// FileName is the name of the file as it appears in testcases.yaml
	FileName string

	// HCLLocalName is the local name of the HCL block, e.g. "basic" or "default"
	HCLLocalName string

	// DCLResourceType is the type represented in the DCL, e.g. "ComputeInstance"
	DCLResourceType string

	// TerraformResourceType is the type represented in Terraform, e.g. "google_compute_instance"
	TerraformResourceType string

	// HCLBlock is the snippet of HCL config that declares this resource
	HCLBlock string // Path to the directory where the sample data is stored
}

type Update struct {
	// The list of dependency resources to update.
	Dependencies []string `yaml:"dependencies"`

	// The resource to update.
	Resource string `yaml:"resource"`
}

// BuildDependency produces a Dependency using a file and filename
func BuildDependency(fileName, product, localname, version string, hasGAEquivalent bool, b []byte) (*Dependency, error) {
	var resourceName string
	fileParts := strings.Split(fileName, ".")
	if len(fileParts) == 4 {
		var err error
		// TODO(magic-modules-eng): Allow for resources which are split in terraform but not in DCL
		// (e.g. locational split) to find the right resource here.
		product, resourceName, err = DCLToTerraformSampleName(fileParts[1], fileParts[2])
		if err != nil {
			return nil, err
		}
	} else if len(fileParts) == 3 {
		resourceName = strings.Title(fileParts[1])
	} else {
		return nil, fmt.Errorf("Invalid sample dependency file name: %s", fileName)
	}

	if localname == "" {
		localname = fileParts[0]
	}
	dclResourceType := product + snakeToTitleCase(resourceName)
	terraformResourceType, err := DCLToTerraformReference(dclResourceType, version)
	if err != nil {
		return nil, fmt.Errorf("Error generating sample dependency %s: %s", fileName, err)
	}

	block, err := ConvertSampleJSONToHCL(dclResourceType, version, hasGAEquivalent, b)
	if err != nil {
		return nil, fmt.Errorf("Error generating sample dependency %s: %s", fileName, err)
	}

	re := regexp.MustCompile(`(resource "` + terraformResourceType + `" ")(\w*)`)
	block = re.ReplaceAllString(block, "${1}"+localname)

	d := &Dependency{
		FileName:              fileName,
		HCLLocalName:          localname,
		DCLResourceType:       dclResourceType,
		TerraformResourceType: terraformResourceType,
		HCLBlock:              block,
	}
	return d, nil
}

func (s *Sample) generateSampleDependency(fileName string) Dependency {
	return s.generateSampleDependencyWithName(fileName, "")
}

func (s *Sample) generateSampleDependencyWithName(fileName, localname string) Dependency {
	dFileNameParts := strings.Split(fileName, "samples/")
	fileName = dFileNameParts[len(dFileNameParts)-1]
	dependencyBytes, err := ioutil.ReadFile(path.Join(s.SamplesPath, fileName))
	version := s.resourceReference.versionMetadata.V
	product := s.resourceReference.productMetadata.ProductType()
	d, err := BuildDependency(fileName, product, localname, version, s.HasGAEquivalent, dependencyBytes)
	if err != nil {
		glog.Exit(err)
	}
	return *d
}

func (s *Sample) GetCodeToInject() []string {
	sampleAccessoryFolder := s.resourceReference.getSampleAccessoryFolder()
	var out []string
	for _, fileName := range s.CodeInject {
		filePath := path.Join(sampleAccessoryFolder, fileName)
		tc, err := ioutil.ReadFile(filePath)
		if err != nil {
			glog.Exit(err)
		}
		out = append(out, string(tc))
	}
	return out
}

// ReplaceReferences substitutes any reference tags for their HCL address
// This should only be called after every dependency for a sample is built
func (s Sample) ReplaceReferences(d *Dependency) error {
	re := regexp.MustCompile(`"?{{\s*ref:([a-z_]*\.[a-z_]*\.[a-z_]*(?:\.[a-z_]*)?):([a-zA-Z0-9_\.\[\]]*)\s*}}"?`)
	matches := re.FindAllStringSubmatch(d.HCLBlock, -1)

	for _, match := range matches {
		referenceFileName := match[1]
		idField := match[2]
		var tfReference string
		for _, dep := range s.DependencyList {
			if dep.FileName == referenceFileName {
				tfReference = dep.TerraformResourceType + "." + dep.HCLLocalName + "." + idField
				break
			}
		}
		if tfReference == "" {
			return fmt.Errorf("Could not find reference file name: %s", referenceFileName)
		}
		startsWithQuote := strings.HasPrefix(match[0], `"`)
		endsWithQuote := strings.HasSuffix(match[0], `"`)
		if !(startsWithQuote && endsWithQuote) {
			tfReference = fmt.Sprintf("${%s}", tfReference)
			if startsWithQuote {
				tfReference = `"` + tfReference
			}
			if endsWithQuote {
				tfReference += `"`
			}
		}
		d.HCLBlock = strings.Replace(d.HCLBlock, match[0], tfReference, 1)
	}
	return nil
}

func (s Sample) generateHCLTemplate() (string, error) {
	if len(s.DependencyList) == 0 {
		return "", fmt.Errorf("Could not generate HCL template for %s: there are no dependencies", *s.Name)
	}

	var hcl string
	for index := range s.DependencyList {
		err := s.ReplaceReferences(&s.DependencyList[index])
		if err != nil {
			return "", fmt.Errorf("Could not generate HCL template for %s: %s", *s.Name, err)
		}
		hcl = fmt.Sprintf("%s%s\n", hcl, s.DependencyList[index].HCLBlock)
	}

	return hcl, nil
}

// GenerateHCL generates sample HCL using docs substitution metadata
func (s Sample) GenerateHCL(isDocs bool) string {
	var hcl string
	var err error
	if !s.isNativeHCL() {
		hcl, err = s.generateHCLTemplate()
		if err != nil {
			glog.Exit(err)
		}
	} else {
		tc, err := ioutil.ReadFile(path.Join(s.SamplesPath, *s.PrimaryResource))
		if err != nil {
			glog.Exit(err)
		}
		hcl = string(tc)
	}
	for _, sub := range s.Substitutions {
		re := regexp.MustCompile(fmt.Sprintf(`{{%s}}`, *sub.Substitution))
		hcl = re.ReplaceAllString(hcl, sub.translateValue(isDocs))
	}
	return hcl
}

// isNativeHCL returns whether the resource file is terraform synatax
func (s Sample) isNativeHCL() bool {
	return strings.HasSuffix(*s.PrimaryResource, ".tf.tmpl")
}

// EnumerateWithUpdateSamples returns an array of new samples expanded with
// any subsequent samples
func (s *Sample) EnumerateWithUpdateSamples() []Sample {
	out := []Sample{*s}
	for i, update := range s.Updates {
		newSample := *s
		primaryResource := update.Resource
		// TODO(magic-modules-eng): Consume new dependency list.
		newSample.PrimaryResource = &primaryResource
		if !newSample.isNativeHCL() {
			var newDeps []Dependency
			newDeps = append(newDeps, newSample.generateSampleDependencyWithName(*newSample.PrimaryResource, "primary"))
			for _, newDepFilename := range update.Dependencies {
				newDepFilename = strings.TrimPrefix(newDepFilename, "samples/")
				newDeps = append(newDeps, newSample.generateSampleDependencyWithName(newDepFilename, basicResourceName(newDepFilename)))
			}
			newSample.DependencyList = newDeps
		}
		newSample.TestSlug = fmt.Sprintf("%sUpdate%v", newSample.TestSlug, i)
		newSample.Updates = nil
		out = append(out, newSample)
	}
	return out
}

func basicResourceName(depFilename string) string {
	re := regexp.MustCompile("^update(_\\d)?\\.")
	// update_1.resource.json -> basic.resource.json
	basicReplaced := re.ReplaceAllString(depFilename, "basic.")
	re = regexp.MustCompile("^update(_\\d)?_")
	// update_1_name.resource.json -> name.resource.json
	prefixTrimmed := re.ReplaceAllString(basicReplaced, "")
	return dcl.SnakeToJSONCase(strings.Split(prefixTrimmed, ".")[0])
}

// ExpandContext expands the context model used in the generated tests
func (s Sample) ExpandContext() map[string]string {
	out := map[string]string{}
	for _, sub := range s.Substitutions {
		translation, hasTranslation := translationMap[*sub.Value]
		if hasTranslation {
			out[translation.contextKey] = translation.contextValue
		}
	}
	return out
}

type translationIndex struct {
	docsValue    string
	contextKey   string
	contextValue string
}

var translationMap = map[string]translationIndex{
	":ORG_ID": {
		docsValue:    "123456789",
		contextKey:   "org_id",
		contextValue: "getTestOrgFromEnv(t)",
	},
	":ORG_DOMAIN": {
		docsValue:    "example.com",
		contextKey:   "org_domain",
		contextValue: "getTestOrgDomainFromEnv(t)",
	},
	":CREDENTIALS": {
		docsValue:    "my/credentials/filename.json",
		contextKey:   "credentials",
		contextValue: "getTestCredsFromEnv(t)",
	},
	":REGION": {
		docsValue:    "us-west1",
		contextKey:   "region",
		contextValue: "getTestRegionFromEnv()",
	},
	":ORG_TARGET": {
		docsValue:    "123456789",
		contextKey:   "org_target",
		contextValue: "getTestOrgTargetFromEnv(t)",
	},
	":BILLING_ACCT": {
		docsValue:    "000000-0000000-0000000-000000",
		contextKey:   "billing_acct",
		contextValue: "getTestBillingAccountFromEnv(t)",
	},
	":SERVICE_ACCT": {
		docsValue:    "emailAddress:my@service-account.com",
		contextKey:   "service_acct",
		contextValue: "getTestServiceAccountFromEnv(t)",
	},
	":PROJECT": {
		docsValue:    "my-project-name",
		contextKey:   "project_name",
		contextValue: "getTestProjectFromEnv()",
	},
	":PROJECT_NAME": {
		docsValue:    "my-project-name",
		contextKey:   "project_name",
		contextValue: "getTestProjectFromEnv()",
	},
	":FIRESTORE_PROJECT_NAME": {
		docsValue:    "my-project-name",
		contextKey:   "firestore_project_name",
		contextValue: "getTestFirestoreProjectFromEnv(t)",
	},
	":CUST_ID": {
		docsValue:    "A01b123xz",
		contextKey:   "cust_id",
		contextValue: "getTestCustIdFromEnv(t)",
	},
	":IDENTITY_USER": {
		docsValue:    "cloud_identity_user",
		contextKey:   "identity_user",
		contextValue: "getTestIdentityUserFromEnv(t)",
	},
}

// translateValue returns the value to embed in the hcl
func (sub *Substitution) translateValue(isDocs bool) string {
	value := *sub.Value
	translation, hasTranslation := translationMap[value]

	if isDocs {
		if hasTranslation {
			return translation.docsValue
		}
		return value
	}

	if hasTranslation {
		return fmt.Sprintf("%%{%s}", translation.contextKey)
	}

	if strings.Contains(value, "-") {
		value = fmt.Sprintf("tf-test-%s", value)
	} else if strings.Contains(value, "_") {
		value = fmt.Sprintf("tf_test_%s", value)
	}

	// Random suffix is 10 characters and standard name length <= 64
	if len(value) > 54 {
		value = value[:54]
	}
	return fmt.Sprintf("%s%%{random_suffix}", value)
}

func (s Sample) PrimaryResourceName() string {
	fileParts := strings.Split(*s.PrimaryResource, ".")
	return fileParts[0]
}
<commit_msg>Add deserializer for upcoming 'docs_value' field in sample. (#5350)<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"path"
	"regexp"
	"strings"

	dcl "github.com/GoogleCloudPlatform/declarative-resource-client-library/dcl"
	"github.com/golang/glog"
)

// Sample is the object containing sample data from DCL samples
type Sample struct {
	// Name of the file the sample was loaded from
	FileName string

	// Name is the name of a sample
	Name *string

	// Description is a short description of the sample
	Description *string

	// DependencyFileNames contains the filenames of every resource in the sample
	DependencyFileNames []string `yaml:"dependencies"`

	// PrimaryResource is the filename of the sample's primary resource
	PrimaryResource *string `yaml:"resource"`

	// Substitutions contains every substition in the sample
	Substitutions []Substitution `yaml:"substitutions"`

	IgnoreRead []string `yaml:"ignore_read"`

	// DependencyList is a list of objects containing metadata for each sample resource
	DependencyList []Dependency

	// The name of the test
	TestSlug string

	// The raw versions stated in the yaml
	Versions []string

	// A list of updates that the resource can transition between
	Updates []Update

	// HasGAEquivalent tells us if we should have `provider = google-beta`
	// in the testcase. (if the test doesn't have a ga version of the test)
	HasGAEquivalent bool

	// SamplesPath is the path to the directory where the original sample data is stored
	SamplesPath string

	// resourceReference is the resource the sample belongs to
	resourceReference *Resource

	// CustomCheck allows you to add a terraform check function to all tests
	CustomCheck []string `yaml:"check"`

	// CodeInject references reletive raw files that should be injected into the sample test
	CodeInject []string `yaml:"code_inject"`

	// DocHide specifies a list of samples to hide from docs
	DocHide []string `yaml:"doc_hide"`

	// Testhide specifies a list of samples to hide from tests
	Testhide []string `yaml:"test_hide"`

	// ExtraDependencies are the additional golang dependencies the injected code may require
	ExtraDependencies []string `yaml:"extra_dependencies"`

	// Type is the resource type.
	Type string `yaml:"type"`

	// Variables are the various attributes of the set of resources that need to be filled in.
	Variables []Variable `yaml:"variables"`
}

// Variable contains metadata about the types of variables in a sample.
type Variable struct {
	// Name is the variable name in the JSON.
	Name string `yaml:"name"`
	// Type is the variable type.
	Type string `yaml:"type"`
	// DocsValue is an optional value that should be substituted directly into
	// the documentation for this variable.  If not provided, tpgtools makes
	// its best guess about a suitable value.  Generally, this is only provided
	// if the "best guess" is a poor one.
	DocsValue string `yaml:"docs_value"`
}

// Substitution contains metadata that varies for the sample context
type Substitution struct {
	// Substitution is the text to be substituted, e.g. topic_name
	Substitution *string

	// Value is the value of the substituted text
	Value *string `yaml:"value"`
}

// Dependency contains data that describes a single resource in a sample
type Dependency struct {
	// FileName is the name of the file as it appears in testcases.yaml
	FileName string

	// HCLLocalName is the local name of the HCL block, e.g. "basic" or "default"
	HCLLocalName string

	// DCLResourceType is the type represented in the DCL, e.g. "ComputeInstance"
	DCLResourceType string

	// TerraformResourceType is the type represented in Terraform, e.g. "google_compute_instance"
	TerraformResourceType string

	// HCLBlock is the snippet of HCL config that declares this resource
	HCLBlock string // Path to the directory where the sample data is stored
}

type Update struct {
	// The list of dependency resources to update.
	Dependencies []string `yaml:"dependencies"`

	// The resource to update.
	Resource string `yaml:"resource"`
}

// BuildDependency produces a Dependency using a file and filename
func BuildDependency(fileName, product, localname, version string, hasGAEquivalent bool, b []byte) (*Dependency, error) {
	var resourceName string
	fileParts := strings.Split(fileName, ".")
	if len(fileParts) == 4 {
		var err error
		// TODO(magic-modules-eng): Allow for resources which are split in terraform but not in DCL
		// (e.g. locational split) to find the right resource here.
		product, resourceName, err = DCLToTerraformSampleName(fileParts[1], fileParts[2])
		if err != nil {
			return nil, err
		}
	} else if len(fileParts) == 3 {
		resourceName = strings.Title(fileParts[1])
	} else {
		return nil, fmt.Errorf("Invalid sample dependency file name: %s", fileName)
	}

	if localname == "" {
		localname = fileParts[0]
	}
	dclResourceType := product + snakeToTitleCase(resourceName)
	terraformResourceType, err := DCLToTerraformReference(dclResourceType, version)
	if err != nil {
		return nil, fmt.Errorf("Error generating sample dependency %s: %s", fileName, err)
	}

	block, err := ConvertSampleJSONToHCL(dclResourceType, version, hasGAEquivalent, b)
	if err != nil {
		return nil, fmt.Errorf("Error generating sample dependency %s: %s", fileName, err)
	}

	re := regexp.MustCompile(`(resource "` + terraformResourceType + `" ")(\w*)`)
	block = re.ReplaceAllString(block, "${1}"+localname)

	d := &Dependency{
		FileName:              fileName,
		HCLLocalName:          localname,
		DCLResourceType:       dclResourceType,
		TerraformResourceType: terraformResourceType,
		HCLBlock:              block,
	}
	return d, nil
}

func (s *Sample) generateSampleDependency(fileName string) Dependency {
	return s.generateSampleDependencyWithName(fileName, "")
}

func (s *Sample) generateSampleDependencyWithName(fileName, localname string) Dependency {
	dFileNameParts := strings.Split(fileName, "samples/")
	fileName = dFileNameParts[len(dFileNameParts)-1]
	dependencyBytes, err := ioutil.ReadFile(path.Join(s.SamplesPath, fileName))
	version := s.resourceReference.versionMetadata.V
	product := s.resourceReference.productMetadata.ProductType()
	d, err := BuildDependency(fileName, product, localname, version, s.HasGAEquivalent, dependencyBytes)
	if err != nil {
		glog.Exit(err)
	}
	return *d
}

func (s *Sample) GetCodeToInject() []string {
	sampleAccessoryFolder := s.resourceReference.getSampleAccessoryFolder()
	var out []string
	for _, fileName := range s.CodeInject {
		filePath := path.Join(sampleAccessoryFolder, fileName)
		tc, err := ioutil.ReadFile(filePath)
		if err != nil {
			glog.Exit(err)
		}
		out = append(out, string(tc))
	}
	return out
}

// ReplaceReferences substitutes any reference tags for their HCL address
// This should only be called after every dependency for a sample is built
func (s Sample) ReplaceReferences(d *Dependency) error {
	re := regexp.MustCompile(`"?{{\s*ref:([a-z_]*\.[a-z_]*\.[a-z_]*(?:\.[a-z_]*)?):([a-zA-Z0-9_\.\[\]]*)\s*}}"?`)
	matches := re.FindAllStringSubmatch(d.HCLBlock, -1)

	for _, match := range matches {
		referenceFileName := match[1]
		idField := match[2]
		var tfReference string
		for _, dep := range s.DependencyList {
			if dep.FileName == referenceFileName {
				tfReference = dep.TerraformResourceType + "." + dep.HCLLocalName + "." + idField
				break
			}
		}
		if tfReference == "" {
			return fmt.Errorf("Could not find reference file name: %s", referenceFileName)
		}
		startsWithQuote := strings.HasPrefix(match[0], `"`)
		endsWithQuote := strings.HasSuffix(match[0], `"`)
		if !(startsWithQuote && endsWithQuote) {
			tfReference = fmt.Sprintf("${%s}", tfReference)
			if startsWithQuote {
				tfReference = `"` + tfReference
			}
			if endsWithQuote {
				tfReference += `"`
			}
		}
		d.HCLBlock = strings.Replace(d.HCLBlock, match[0], tfReference, 1)
	}
	return nil
}

func (s Sample) generateHCLTemplate() (string, error) {
	if len(s.DependencyList) == 0 {
		return "", fmt.Errorf("Could not generate HCL template for %s: there are no dependencies", *s.Name)
	}

	var hcl string
	for index := range s.DependencyList {
		err := s.ReplaceReferences(&s.DependencyList[index])
		if err != nil {
			return "", fmt.Errorf("Could not generate HCL template for %s: %s", *s.Name, err)
		}
		hcl = fmt.Sprintf("%s%s\n", hcl, s.DependencyList[index].HCLBlock)
	}

	return hcl, nil
}

// GenerateHCL generates sample HCL using docs substitution metadata
func (s Sample) GenerateHCL(isDocs bool) string {
	var hcl string
	var err error
	if !s.isNativeHCL() {
		hcl, err = s.generateHCLTemplate()
		if err != nil {
			glog.Exit(err)
		}
	} else {
		tc, err := ioutil.ReadFile(path.Join(s.SamplesPath, *s.PrimaryResource))
		if err != nil {
			glog.Exit(err)
		}
		hcl = string(tc)
	}
	for _, sub := range s.Substitutions {
		re := regexp.MustCompile(fmt.Sprintf(`{{%s}}`, *sub.Substitution))
		hcl = re.ReplaceAllString(hcl, sub.translateValue(isDocs))
	}
	return hcl
}

// isNativeHCL returns whether the resource file is terraform synatax
func (s Sample) isNativeHCL() bool {
	return strings.HasSuffix(*s.PrimaryResource, ".tf.tmpl")
}

// EnumerateWithUpdateSamples returns an array of new samples expanded with
// any subsequent samples
func (s *Sample) EnumerateWithUpdateSamples() []Sample {
	out := []Sample{*s}
	for i, update := range s.Updates {
		newSample := *s
		primaryResource := update.Resource
		// TODO(magic-modules-eng): Consume new dependency list.
		newSample.PrimaryResource = &primaryResource
		if !newSample.isNativeHCL() {
			var newDeps []Dependency
			newDeps = append(newDeps, newSample.generateSampleDependencyWithName(*newSample.PrimaryResource, "primary"))
			for _, newDepFilename := range update.Dependencies {
				newDepFilename = strings.TrimPrefix(newDepFilename, "samples/")
				newDeps = append(newDeps, newSample.generateSampleDependencyWithName(newDepFilename, basicResourceName(newDepFilename)))
			}
			newSample.DependencyList = newDeps
		}
		newSample.TestSlug = fmt.Sprintf("%sUpdate%v", newSample.TestSlug, i)
		newSample.Updates = nil
		out = append(out, newSample)
	}
	return out
}

func basicResourceName(depFilename string) string {
	re := regexp.MustCompile("^update(_\\d)?\\.")
	// update_1.resource.json -> basic.resource.json
	basicReplaced := re.ReplaceAllString(depFilename, "basic.")
	re = regexp.MustCompile("^update(_\\d)?_")
	// update_1_name.resource.json -> name.resource.json
	prefixTrimmed := re.ReplaceAllString(basicReplaced, "")
	return dcl.SnakeToJSONCase(strings.Split(prefixTrimmed, ".")[0])
}

// ExpandContext expands the context model used in the generated tests
func (s Sample) ExpandContext() map[string]string {
	out := map[string]string{}
	for _, sub := range s.Substitutions {
		translation, hasTranslation := translationMap[*sub.Value]
		if hasTranslation {
			out[translation.contextKey] = translation.contextValue
		}
	}
	return out
}

type translationIndex struct {
	docsValue    string
	contextKey   string
	contextValue string
}

var translationMap = map[string]translationIndex{
	":ORG_ID": {
		docsValue:    "123456789",
		contextKey:   "org_id",
		contextValue: "getTestOrgFromEnv(t)",
	},
	":ORG_DOMAIN": {
		docsValue:    "example.com",
		contextKey:   "org_domain",
		contextValue: "getTestOrgDomainFromEnv(t)",
	},
	":CREDENTIALS": {
		docsValue:    "my/credentials/filename.json",
		contextKey:   "credentials",
		contextValue: "getTestCredsFromEnv(t)",
	},
	":REGION": {
		docsValue:    "us-west1",
		contextKey:   "region",
		contextValue: "getTestRegionFromEnv()",
	},
	":ORG_TARGET": {
		docsValue:    "123456789",
		contextKey:   "org_target",
		contextValue: "getTestOrgTargetFromEnv(t)",
	},
	":BILLING_ACCT": {
		docsValue:    "000000-0000000-0000000-000000",
		contextKey:   "billing_acct",
		contextValue: "getTestBillingAccountFromEnv(t)",
	},
	":SERVICE_ACCT": {
		docsValue:    "emailAddress:my@service-account.com",
		contextKey:   "service_acct",
		contextValue: "getTestServiceAccountFromEnv(t)",
	},
	":PROJECT": {
		docsValue:    "my-project-name",
		contextKey:   "project_name",
		contextValue: "getTestProjectFromEnv()",
	},
	":PROJECT_NAME": {
		docsValue:    "my-project-name",
		contextKey:   "project_name",
		contextValue: "getTestProjectFromEnv()",
	},
	":FIRESTORE_PROJECT_NAME": {
		docsValue:    "my-project-name",
		contextKey:   "firestore_project_name",
		contextValue: "getTestFirestoreProjectFromEnv(t)",
	},
	":CUST_ID": {
		docsValue:    "A01b123xz",
		contextKey:   "cust_id",
		contextValue: "getTestCustIdFromEnv(t)",
	},
	":IDENTITY_USER": {
		docsValue:    "cloud_identity_user",
		contextKey:   "identity_user",
		contextValue: "getTestIdentityUserFromEnv(t)",
	},
}

// translateValue returns the value to embed in the hcl
func (sub *Substitution) translateValue(isDocs bool) string {
	value := *sub.Value
	translation, hasTranslation := translationMap[value]

	if isDocs {
		if hasTranslation {
			return translation.docsValue
		}
		return value
	}

	if hasTranslation {
		return fmt.Sprintf("%%{%s}", translation.contextKey)
	}

	if strings.Contains(value, "-") {
		value = fmt.Sprintf("tf-test-%s", value)
	} else if strings.Contains(value, "_") {
		value = fmt.Sprintf("tf_test_%s", value)
	}

	// Random suffix is 10 characters and standard name length <= 64
	if len(value) > 54 {
		value = value[:54]
	}
	return fmt.Sprintf("%s%%{random_suffix}", value)
}

func (s Sample) PrimaryResourceName() string {
	fileParts := strings.Split(*s.PrimaryResource, ".")
	return fileParts[0]
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file was auto-generated by the vanadium vdl tool.
// Source: chat.vdl

package vdl

import (
	// VDL system imports
	"v.io/v23"
	"v.io/v23/context"
	"v.io/v23/rpc"
)

func __VDLEnsureNativeBuilt_chat() {
}

// ChatClientMethods is the client interface
// containing Chat methods.
type ChatClientMethods interface {
	// SendMessage sends a message to a user.
	SendMessage(_ *context.T, text string, _ ...rpc.CallOpt) error
}

// ChatClientStub adds universal methods to ChatClientMethods.
type ChatClientStub interface {
	ChatClientMethods
	rpc.UniversalServiceMethods
}

// ChatClient returns a client stub for Chat.
func ChatClient(name string) ChatClientStub {
	return implChatClientStub{name}
}

type implChatClientStub struct {
	name string
}

func (c implChatClientStub) SendMessage(ctx *context.T, i0 string, opts ...rpc.CallOpt) (err error) {
	err = v23.GetClient(ctx).Call(ctx, c.name, "SendMessage", []interface{}{i0}, nil, opts...)
	return
}

// ChatServerMethods is the interface a server writer
// implements for Chat.
type ChatServerMethods interface {
	// SendMessage sends a message to a user.
	SendMessage(_ *context.T, _ rpc.ServerCall, text string) error
}

// ChatServerStubMethods is the server interface containing
// Chat methods, as expected by rpc.Server.
// There is no difference between this interface and ChatServerMethods
// since there are no streaming methods.
type ChatServerStubMethods ChatServerMethods

// ChatServerStub adds universal methods to ChatServerStubMethods.
type ChatServerStub interface {
	ChatServerStubMethods
	// Describe the Chat interfaces.
	Describe__() []rpc.InterfaceDesc
}

// ChatServer returns a server stub for Chat.
// It converts an implementation of ChatServerMethods into
// an object that may be used by rpc.Server.
func ChatServer(impl ChatServerMethods) ChatServerStub {
	stub := implChatServerStub{
		impl: impl,
	}
	// Initialize GlobState; always check the stub itself first, to handle the
	// case where the user has the Glob method defined in their VDL source.
	if gs := rpc.NewGlobState(stub); gs != nil {
		stub.gs = gs
	} else if gs := rpc.NewGlobState(impl); gs != nil {
		stub.gs = gs
	}
	return stub
}

type implChatServerStub struct {
	impl ChatServerMethods
	gs   *rpc.GlobState
}

func (s implChatServerStub) SendMessage(ctx *context.T, call rpc.ServerCall, i0 string) error {
	return s.impl.SendMessage(ctx, call, i0)
}

func (s implChatServerStub) Globber() *rpc.GlobState {
	return s.gs
}

func (s implChatServerStub) Describe__() []rpc.InterfaceDesc {
	return []rpc.InterfaceDesc{ChatDesc}
}

// ChatDesc describes the Chat interface.
var ChatDesc rpc.InterfaceDesc = descChat

// descChat hides the desc to keep godoc clean.
var descChat = rpc.InterfaceDesc{
	Name:    "Chat",
	PkgPath: "v.io/x/chat/vdl",
	Methods: []rpc.MethodDesc{
		{
			Name: "SendMessage",
			Doc:  "// SendMessage sends a message to a user.",
			InArgs: []rpc.ArgDesc{
				{"text", ``}, // string
			},
		},
	},
}
<commit_msg>chat: Update vdl for 2 pass gen<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file was auto-generated by the vanadium vdl tool.
// Source: chat.vdl

package vdl

import (
	"v.io/v23"
	"v.io/v23/context"
	"v.io/v23/rpc"
)

func __VDLEnsureNativeBuilt_chat() {
}

// ChatClientMethods is the client interface
// containing Chat methods.
type ChatClientMethods interface {
	// SendMessage sends a message to a user.
	SendMessage(_ *context.T, text string, _ ...rpc.CallOpt) error
}

// ChatClientStub adds universal methods to ChatClientMethods.
type ChatClientStub interface {
	ChatClientMethods
	rpc.UniversalServiceMethods
}

// ChatClient returns a client stub for Chat.
func ChatClient(name string) ChatClientStub {
	return implChatClientStub{name}
}

type implChatClientStub struct {
	name string
}

func (c implChatClientStub) SendMessage(ctx *context.T, i0 string, opts ...rpc.CallOpt) (err error) {
	err = v23.GetClient(ctx).Call(ctx, c.name, "SendMessage", []interface{}{i0}, nil, opts...)
	return
}

// ChatServerMethods is the interface a server writer
// implements for Chat.
type ChatServerMethods interface {
	// SendMessage sends a message to a user.
	SendMessage(_ *context.T, _ rpc.ServerCall, text string) error
}

// ChatServerStubMethods is the server interface containing
// Chat methods, as expected by rpc.Server.
// There is no difference between this interface and ChatServerMethods
// since there are no streaming methods.
type ChatServerStubMethods ChatServerMethods

// ChatServerStub adds universal methods to ChatServerStubMethods.
type ChatServerStub interface {
	ChatServerStubMethods
	// Describe the Chat interfaces.
	Describe__() []rpc.InterfaceDesc
}

// ChatServer returns a server stub for Chat.
// It converts an implementation of ChatServerMethods into
// an object that may be used by rpc.Server.
func ChatServer(impl ChatServerMethods) ChatServerStub {
	stub := implChatServerStub{
		impl: impl,
	}
	// Initialize GlobState; always check the stub itself first, to handle the
	// case where the user has the Glob method defined in their VDL source.
	if gs := rpc.NewGlobState(stub); gs != nil {
		stub.gs = gs
	} else if gs := rpc.NewGlobState(impl); gs != nil {
		stub.gs = gs
	}
	return stub
}

type implChatServerStub struct {
	impl ChatServerMethods
	gs   *rpc.GlobState
}

func (s implChatServerStub) SendMessage(ctx *context.T, call rpc.ServerCall, i0 string) error {
	return s.impl.SendMessage(ctx, call, i0)
}

func (s implChatServerStub) Globber() *rpc.GlobState {
	return s.gs
}

func (s implChatServerStub) Describe__() []rpc.InterfaceDesc {
	return []rpc.InterfaceDesc{ChatDesc}
}

// ChatDesc describes the Chat interface.
var ChatDesc rpc.InterfaceDesc = descChat

// descChat hides the desc to keep godoc clean.
var descChat = rpc.InterfaceDesc{
	Name:    "Chat",
	PkgPath: "v.io/x/chat/vdl",
	Methods: []rpc.MethodDesc{
		{
			Name: "SendMessage",
			Doc:  "// SendMessage sends a message to a user.",
			InArgs: []rpc.ArgDesc{
				{"text", ``}, // string
			},
		},
	},
}
<|endoftext|>


<commit_before>package main

import (
	//"encoding/binary"
	"encoding/xml"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"sort"
	"strings"
)

/**
 * uavObjectDefinitions storage
 */
var uavObjectDefinitions []*UAVObjectDefinition

/**
 * Utils
 */
type FieldTypeInfo struct {
	index int
	name  string
	size  int
}

type TypeIndex []*FieldTypeInfo

var typeInfos = TypeIndex{
	&FieldTypeInfo{0, "int8", 1},
	&FieldTypeInfo{1, "int16", 2},
	&FieldTypeInfo{2, "int32", 4},
	&FieldTypeInfo{3, "uint8", 1},
	&FieldTypeInfo{4, "uint16", 2},
	&FieldTypeInfo{5, "uint32", 4},
	&FieldTypeInfo{6, "float", 4},
	&FieldTypeInfo{7, "enum", 1},
}

func (t TypeIndex) fieldTypeForString(ts string) (*FieldTypeInfo, error) {
	for _, fieldTypeInfo := range typeInfos {
		if fieldTypeInfo.name == ts {
			return fieldTypeInfo, nil
		}
	}
	return nil, errors.New(fmt.Sprintf("Not found field type: %s", ts))
}

// sorted slice of fields
type FieldSlice []*UAVObjectFieldDefinition

func (fields FieldSlice) fieldForName(name string) (*UAVObjectFieldDefinition, error) {
	for _, field := range fields {
		if field.Name == name {
			return field, nil
		}
	}
	return nil, errors.New(fmt.Sprintf("Not found field name: %s", name))
}

func (fields FieldSlice) Len() int {
	return len(fields)
}

func (fields FieldSlice) Less(i, j int) bool {
	return fields[i].fieldTypeInfo.size > fields[j].fieldTypeInfo.size
}

func (fields FieldSlice) Swap(i, j int) {
	fields[i], fields[j] = fields[j], fields[i]
}

// uavObjectDefinitions models
type UAVObjectFieldDefinition struct {
	Name  string `xml:"name,attr" json:"name"`
	Units string `xml:"units,attr" json:"units"`
	Type  string `xml:"type,attr" json:"type"`

	fieldTypeInfo *FieldTypeInfo

	Elements         int      `xml:"elements,attr" json:"elements"`
	ElementNamesAttr string   `xml:"elementnames,attr" json:"-"`
	ElementNames     []string `xml:"elementnames>elementname" json:"elementsName"`
	OptionsAttr      string   `xml:"options,attr" json:"-"`
	Options          []string `xml:"options>option" json:"options"`
	DefaultValue     string   `xml:"defaultvalue,attr" json:"defaultValue"`

	CloneOf string `xml:"cloneof,attr" json:"cloneOf"`
}

type UAVObjectDefinition struct {
	Name           string `xml:"name,attr" json:"name"`
	Description    string `xml:"description" json:"description"`
	SingleInstance bool   `xml:"singleinstance,attr" json:"singleInstance"`
	Settings       bool   `xml:"settings,attr" json:"settings"`
	Category       string `xml:"category,attr" json:"category"`

	ObjectID uint32 `json:"id"`

	Access struct {
		Gcs    string `xml:"gcs,attr" json:"gcs"`
		Flight string `xml:"flight,attr" json:"flight"`
	} `xml:"access" json:"access"`

	TelemetryGcs struct {
		Acked      string `xml:"acked,attr" json:"acked"` // TODO shouldn't it be boolean ?
		UpdateMode string `xml:"updatemode,attr" json:"updateMode"`
		Period     string `xml:"period,attr" json:"period"`
	} `xml:"telemetrygcs" json:"telemetryGcs"`

	TelemetryFlight struct {
		Acked      string `xml:"acked,attr" json:"acked"`
		UpdateMode string `xml:"updatemode,attr" json:"updateMode"`
		Period     string `xml:"period,attr" json:"period"`
	} `xml:"telemetryflight" json:"telemetryFlight"`

	Logging struct {
		UpdateMode string `xml:"updatemode,attr" json:"updateMode"`
		Period     string `xml:"period,attr" json:"period"`
	} `xml:"logging" json:"logging"`

	Fields FieldSlice `xml:"field" json:"fields"`
}

func newUAVObjectDefinition(filePath string) (*UAVObjectDefinition, error) {
	file, err := os.Open(filePath)
	if err != nil {
		return nil, err
	}

	decoder := xml.NewDecoder(file)

	var content = &struct {
		UAVObject *UAVObjectDefinition `xml:"object"`
	}{}
	decoder.Decode(content)

	uavObject := content.UAVObject

	// fields post process
	for _, field := range uavObject.Fields {
		if len(field.CloneOf) != 0 {
			continue
		}

		if field.Elements == 0 {
			field.Elements = 1
		}

		if len(field.ElementNamesAttr) > 0 {
			field.ElementNames = strings.Split(field.ElementNamesAttr, ",")
			field.Elements = len(field.ElementNames)
		} else if len(field.ElementNames) > 0 {
			field.Elements = len(field.ElementNames)
		}

		if len(field.OptionsAttr) > 0 {
			field.Options = strings.Split(field.OptionsAttr, ",")
		}

		field.fieldTypeInfo, err = typeInfos.fieldTypeForString(field.Type)
		if err != nil {
			return nil, err
		}
	}

	// create clones
	for _, field := range uavObject.Fields {
		if len(field.CloneOf) != 0 {
			clonedField, err := uavObject.Fields.fieldForName(field.CloneOf)
			if err != nil {
				return nil, err
			}
			name, cloneOf := field.Name, field.CloneOf
			*field = *clonedField
			field.Name, field.CloneOf = name, cloneOf
		}
	}

	sort.Stable(uavObject.Fields)

	uavObject.calculateId()

	return uavObject, nil
}

// exported functions
func getUAVObjectDefinitionForObjectID(objectID uint32) (*UAVObjectDefinition, error) {
	for _, uavdef := range uavObjectDefinitions {
		if uavdef.ObjectID == objectID {
			return uavdef, nil
		}
	}
	return nil, errors.New(fmt.Sprint(objectID, " Not found"))
}

// TODO: refac
func isUniqueInstanceForObjectID(objectID uint32) (bool, error) {
	uavdef, err := getUAVObjectDefinitionForObjectID(objectID)
	if err != nil {
		return true, err
	}
	return uavdef.SingleInstance, nil
}

func loadUAVObjectDefinitions(dir string) error {
	fileInfos, err := ioutil.ReadDir(dir)
	if err != nil {
		return err
	}

	for _, fileInfo := range fileInfos {
		filePath := fmt.Sprintf("%s%s", dir, fileInfo.Name())
		uavdef, err := newUAVObjectDefinition(filePath)
		if err != nil {
			log.Fatal(err)
		}
		uavObjectDefinitions = append(uavObjectDefinitions, uavdef)
	}
	return nil
}
<commit_msg>Fix 'Acked' being parsed as a string instead of bool<commit_after>package main

import (
	//"encoding/binary"
	"encoding/xml"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"sort"
	"strings"
)

/**
 * uavObjectDefinitions storage
 */
var uavObjectDefinitions []*UAVObjectDefinition

/**
 * Utils
 */
type FieldTypeInfo struct {
	index int
	name  string
	size  int
}

type TypeIndex []*FieldTypeInfo

var typeInfos = TypeIndex{
	&FieldTypeInfo{0, "int8", 1},
	&FieldTypeInfo{1, "int16", 2},
	&FieldTypeInfo{2, "int32", 4},
	&FieldTypeInfo{3, "uint8", 1},
	&FieldTypeInfo{4, "uint16", 2},
	&FieldTypeInfo{5, "uint32", 4},
	&FieldTypeInfo{6, "float", 4},
	&FieldTypeInfo{7, "enum", 1},
}

func (t TypeIndex) fieldTypeForString(ts string) (*FieldTypeInfo, error) {
	for _, fieldTypeInfo := range typeInfos {
		if fieldTypeInfo.name == ts {
			return fieldTypeInfo, nil
		}
	}
	return nil, errors.New(fmt.Sprintf("Not found field type: %s", ts))
}

// sorted slice of fields
type FieldSlice []*UAVObjectFieldDefinition

func (fields FieldSlice) fieldForName(name string) (*UAVObjectFieldDefinition, error) {
	for _, field := range fields {
		if field.Name == name {
			return field, nil
		}
	}
	return nil, errors.New(fmt.Sprintf("Not found field name: %s", name))
}

func (fields FieldSlice) Len() int {
	return len(fields)
}

func (fields FieldSlice) Less(i, j int) bool {
	return fields[i].fieldTypeInfo.size > fields[j].fieldTypeInfo.size
}

func (fields FieldSlice) Swap(i, j int) {
	fields[i], fields[j] = fields[j], fields[i]
}

// uavObjectDefinitions models
type UAVObjectFieldDefinition struct {
	Name  string `xml:"name,attr" json:"name"`
	Units string `xml:"units,attr" json:"units"`
	Type  string `xml:"type,attr" json:"type"`

	fieldTypeInfo *FieldTypeInfo

	Elements         int      `xml:"elements,attr" json:"elements"`
	ElementNamesAttr string   `xml:"elementnames,attr" json:"-"`
	ElementNames     []string `xml:"elementnames>elementname" json:"elementsName"`
	OptionsAttr      string   `xml:"options,attr" json:"-"`
	Options          []string `xml:"options>option" json:"options"`
	DefaultValue     string   `xml:"defaultvalue,attr" json:"defaultValue"`

	CloneOf string `xml:"cloneof,attr" json:"cloneOf"`
}

type UAVObjectDefinition struct {
	Name           string `xml:"name,attr" json:"name"`
	Description    string `xml:"description" json:"description"`
	SingleInstance bool   `xml:"singleinstance,attr" json:"singleInstance"`
	Settings       bool   `xml:"settings,attr" json:"settings"`
	Category       string `xml:"category,attr" json:"category"`

	ObjectID uint32 `json:"id"`

	Access struct {
		Gcs    string `xml:"gcs,attr" json:"gcs"`
		Flight string `xml:"flight,attr" json:"flight"`
	} `xml:"access" json:"access"`

	TelemetryGcs struct {
		Acked      bool   `xml:"acked,attr" json:"acked"` // TODO shouldn't it be boolean ?
		UpdateMode string `xml:"updatemode,attr" json:"updateMode"`
		Period     string `xml:"period,attr" json:"period"`
	} `xml:"telemetrygcs" json:"telemetryGcs"`

	TelemetryFlight struct {
		Acked      bool   `xml:"acked,attr" json:"acked"`
		UpdateMode string `xml:"updatemode,attr" json:"updateMode"`
		Period     string `xml:"period,attr" json:"period"`
	} `xml:"telemetryflight" json:"telemetryFlight"`

	Logging struct {
		UpdateMode string `xml:"updatemode,attr" json:"updateMode"`
		Period     string `xml:"period,attr" json:"period"`
	} `xml:"logging" json:"logging"`

	Fields FieldSlice `xml:"field" json:"fields"`
}

func newUAVObjectDefinition(filePath string) (*UAVObjectDefinition, error) {
	file, err := os.Open(filePath)
	if err != nil {
		return nil, err
	}

	decoder := xml.NewDecoder(file)

	var content = &struct {
		UAVObject *UAVObjectDefinition `xml:"object"`
	}{}
	decoder.Decode(content)

	uavObject := content.UAVObject

	// fields post process
	for _, field := range uavObject.Fields {
		if len(field.CloneOf) != 0 {
			continue
		}

		if field.Elements == 0 {
			field.Elements = 1
		}

		if len(field.ElementNamesAttr) > 0 {
			field.ElementNames = strings.Split(field.ElementNamesAttr, ",")
			field.Elements = len(field.ElementNames)
		} else if len(field.ElementNames) > 0 {
			field.Elements = len(field.ElementNames)
		}

		if len(field.OptionsAttr) > 0 {
			field.Options = strings.Split(field.OptionsAttr, ",")
		}

		field.fieldTypeInfo, err = typeInfos.fieldTypeForString(field.Type)
		if err != nil {
			return nil, err
		}
	}

	// create clones
	for _, field := range uavObject.Fields {
		if len(field.CloneOf) != 0 {
			clonedField, err := uavObject.Fields.fieldForName(field.CloneOf)
			if err != nil {
				return nil, err
			}
			name, cloneOf := field.Name, field.CloneOf
			*field = *clonedField
			field.Name, field.CloneOf = name, cloneOf
		}
	}

	sort.Stable(uavObject.Fields)

	uavObject.calculateId()

	return uavObject, nil
}

// exported functions
func getUAVObjectDefinitionForObjectID(objectID uint32) (*UAVObjectDefinition, error) {
	for _, uavdef := range uavObjectDefinitions {
		if uavdef.ObjectID == objectID {
			return uavdef, nil
		}
	}
	return nil, errors.New(fmt.Sprint(objectID, " Not found"))
}

// TODO: refac
func isUniqueInstanceForObjectID(objectID uint32) (bool, error) {
	uavdef, err := getUAVObjectDefinitionForObjectID(objectID)
	if err != nil {
		return true, err
	}
	return uavdef.SingleInstance, nil
}

func loadUAVObjectDefinitions(dir string) error {
	fileInfos, err := ioutil.ReadDir(dir)
	if err != nil {
		return err
	}

	for _, fileInfo := range fileInfos {
		filePath := fmt.Sprintf("%s%s", dir, fileInfo.Name())
		uavdef, err := newUAVObjectDefinition(filePath)
		if err != nil {
			log.Fatal(err)
		}
		uavObjectDefinitions = append(uavObjectDefinitions, uavdef)
	}
	return nil
}
<|endoftext|>


<commit_before>// +build !windows

/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mount

import (
	"fmt"
	"os"
	"strconv"
	"strings"
	"syscall"

	utilio "k8s.io/utils/io"
)

const (
	// At least number of fields per line in /proc/<pid>/mountinfo.
	expectedAtLeastNumFieldsPerMountInfo = 10
	// How many times to retry for a consistent read of /proc/mounts.
	maxListTries = 3
)

// IsCorruptedMnt return true if err is about corrupted mount point
func IsCorruptedMnt(err error) bool {
	if err == nil {
		return false
	}
	var underlyingError error
	switch pe := err.(type) {
	case nil:
		return false
	case *os.PathError:
		underlyingError = pe.Err
	case *os.LinkError:
		underlyingError = pe.Err
	case *os.SyscallError:
		underlyingError = pe.Err
	}

	return underlyingError == syscall.ENOTCONN || underlyingError == syscall.ESTALE || underlyingError == syscall.EIO || underlyingError == syscall.EACCES || underlyingError == syscall.EHOSTDOWN
}

// MountInfo represents a single line in /proc/<pid>/mountinfo.
type MountInfo struct { // nolint: golint
	// Unique ID for the mount (maybe reused after umount).
	ID int
	// The ID of the parent mount (or of self for the root of this mount namespace's mount tree).
	ParentID int
	// Major indicates one half of the device ID which identifies the device class
	// (parsed from `st_dev` for files on this filesystem).
	Major int
	// Minor indicates one half of the device ID which identifies a specific
	// instance of device (parsed from `st_dev` for files on this filesystem).
	Minor int
	// The pathname of the directory in the filesystem which forms the root of this mount.
	Root string
	// Mount source, filesystem-specific information. e.g. device, tmpfs name.
	Source string
	// Mount point, the pathname of the mount point.
	MountPoint string
	// Optional fieds, zero or more fields of the form "tag[:value]".
	OptionalFields []string
	// The filesystem type in the form "type[.subtype]".
	FsType string
	// Per-mount options.
	MountOptions []string
	// Per-superblock options.
	SuperOptions []string
}

// ParseMountInfo parses /proc/xxx/mountinfo.
func ParseMountInfo(filename string) ([]MountInfo, error) {
	content, err := utilio.ConsistentRead(filename, maxListTries)
	if err != nil {
		return []MountInfo{}, err
	}
	contentStr := string(content)
	infos := []MountInfo{}

	for _, line := range strings.Split(contentStr, "\n") {
		if line == "" {
			// the last split() item is empty string following the last \n
			continue
		}
		// See `man proc` for authoritative description of format of the file.
		fields := strings.Fields(line)
		if len(fields) < expectedAtLeastNumFieldsPerMountInfo {
			return nil, fmt.Errorf("wrong number of fields in (expected at least %d, got %d): %s", expectedAtLeastNumFieldsPerMountInfo, len(fields), line)
		}
		id, err := strconv.Atoi(fields[0])
		if err != nil {
			return nil, err
		}
		parentID, err := strconv.Atoi(fields[1])
		if err != nil {
			return nil, err
		}
		mm := strings.Split(fields[2], ":")
		if len(mm) != 2 {
			return nil, fmt.Errorf("parsing '%s' failed: unexpected minor:major pair %s", line, mm)
		}
		major, err := strconv.Atoi(mm[0])
		if err != nil {
			return nil, fmt.Errorf("parsing '%s' failed: unable to parse major device id, err:%v", mm[0], err)
		}
		minor, err := strconv.Atoi(mm[1])
		if err != nil {
			return nil, fmt.Errorf("parsing '%s' failed: unable to parse minor device id, err:%v", mm[1], err)
		}

		info := MountInfo{
			ID:           id,
			ParentID:     parentID,
			Major:        major,
			Minor:        minor,
			Root:         fields[3],
			MountPoint:   fields[4],
			MountOptions: strings.Split(fields[5], ","),
		}
		// All fields until "-" are "optional fields".
		i := 6
		for ; i < len(fields) && fields[i] != "-"; i++ {
			info.OptionalFields = append(info.OptionalFields, fields[i])
		}
		// Parse the rest 3 fields.
		i++
		if len(fields)-i < 3 {
			return nil, fmt.Errorf("expect 3 fields in %s, got %d", line, len(fields)-i)
		}
		info.FsType = fields[i]
		info.Source = fields[i+1]
		info.SuperOptions = strings.Split(fields[i+2], ",")
		infos = append(infos, info)
	}
	return infos, nil
}

// isMountPointMatch returns true if the path in mp is the same as dir.
// Handles case where mountpoint dir has been renamed due to stale NFS mount.
func isMountPointMatch(mp MountPoint, dir string) bool {
	deletedDir := fmt.Sprintf("%s\\040(deleted)", dir)
	return ((mp.Path == dir) || (mp.Path == deletedDir))
}
<commit_msg>ConsistentRead tries 10 times<commit_after>// +build !windows

/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mount

import (
	"fmt"
	"os"
	"strconv"
	"strings"
	"syscall"

	utilio "k8s.io/utils/io"
)

const (
	// At least number of fields per line in /proc/<pid>/mountinfo.
	expectedAtLeastNumFieldsPerMountInfo = 10
	// How many times to retry for a consistent read of /proc/mounts.
	maxListTries = 10
)

// IsCorruptedMnt return true if err is about corrupted mount point
func IsCorruptedMnt(err error) bool {
	if err == nil {
		return false
	}
	var underlyingError error
	switch pe := err.(type) {
	case nil:
		return false
	case *os.PathError:
		underlyingError = pe.Err
	case *os.LinkError:
		underlyingError = pe.Err
	case *os.SyscallError:
		underlyingError = pe.Err
	}

	return underlyingError == syscall.ENOTCONN || underlyingError == syscall.ESTALE || underlyingError == syscall.EIO || underlyingError == syscall.EACCES || underlyingError == syscall.EHOSTDOWN
}

// MountInfo represents a single line in /proc/<pid>/mountinfo.
type MountInfo struct { // nolint: golint
	// Unique ID for the mount (maybe reused after umount).
	ID int
	// The ID of the parent mount (or of self for the root of this mount namespace's mount tree).
	ParentID int
	// Major indicates one half of the device ID which identifies the device class
	// (parsed from `st_dev` for files on this filesystem).
	Major int
	// Minor indicates one half of the device ID which identifies a specific
	// instance of device (parsed from `st_dev` for files on this filesystem).
	Minor int
	// The pathname of the directory in the filesystem which forms the root of this mount.
	Root string
	// Mount source, filesystem-specific information. e.g. device, tmpfs name.
	Source string
	// Mount point, the pathname of the mount point.
	MountPoint string
	// Optional fieds, zero or more fields of the form "tag[:value]".
	OptionalFields []string
	// The filesystem type in the form "type[.subtype]".
	FsType string
	// Per-mount options.
	MountOptions []string
	// Per-superblock options.
	SuperOptions []string
}

// ParseMountInfo parses /proc/xxx/mountinfo.
func ParseMountInfo(filename string) ([]MountInfo, error) {
	content, err := utilio.ConsistentRead(filename, maxListTries)
	if err != nil {
		return []MountInfo{}, err
	}
	contentStr := string(content)
	infos := []MountInfo{}

	for _, line := range strings.Split(contentStr, "\n") {
		if line == "" {
			// the last split() item is empty string following the last \n
			continue
		}
		// See `man proc` for authoritative description of format of the file.
		fields := strings.Fields(line)
		if len(fields) < expectedAtLeastNumFieldsPerMountInfo {
			return nil, fmt.Errorf("wrong number of fields in (expected at least %d, got %d): %s", expectedAtLeastNumFieldsPerMountInfo, len(fields), line)
		}
		id, err := strconv.Atoi(fields[0])
		if err != nil {
			return nil, err
		}
		parentID, err := strconv.Atoi(fields[1])
		if err != nil {
			return nil, err
		}
		mm := strings.Split(fields[2], ":")
		if len(mm) != 2 {
			return nil, fmt.Errorf("parsing '%s' failed: unexpected minor:major pair %s", line, mm)
		}
		major, err := strconv.Atoi(mm[0])
		if err != nil {
			return nil, fmt.Errorf("parsing '%s' failed: unable to parse major device id, err:%v", mm[0], err)
		}
		minor, err := strconv.Atoi(mm[1])
		if err != nil {
			return nil, fmt.Errorf("parsing '%s' failed: unable to parse minor device id, err:%v", mm[1], err)
		}

		info := MountInfo{
			ID:           id,
			ParentID:     parentID,
			Major:        major,
			Minor:        minor,
			Root:         fields[3],
			MountPoint:   fields[4],
			MountOptions: strings.Split(fields[5], ","),
		}
		// All fields until "-" are "optional fields".
		i := 6
		for ; i < len(fields) && fields[i] != "-"; i++ {
			info.OptionalFields = append(info.OptionalFields, fields[i])
		}
		// Parse the rest 3 fields.
		i++
		if len(fields)-i < 3 {
			return nil, fmt.Errorf("expect 3 fields in %s, got %d", line, len(fields)-i)
		}
		info.FsType = fields[i]
		info.Source = fields[i+1]
		info.SuperOptions = strings.Split(fields[i+2], ",")
		infos = append(infos, info)
	}
	return infos, nil
}

// isMountPointMatch returns true if the path in mp is the same as dir.
// Handles case where mountpoint dir has been renamed due to stale NFS mount.
func isMountPointMatch(mp MountPoint, dir string) bool {
	deletedDir := fmt.Sprintf("%s\\040(deleted)", dir)
	return ((mp.Path == dir) || (mp.Path == deletedDir))
}
<|endoftext|>


<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package invocations

import (
	"bytes"
	"context"
	"fmt"
	"sync"
	"time"

	"cloud.google.com/go/spanner"
	"github.com/gomodule/redigo/redis"
	"golang.org/x/sync/errgroup"
	"golang.org/x/sync/semaphore"

	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/trace"
	"go.chromium.org/luci/server/redisconn"

	"go.chromium.org/luci/resultdb/internal/spanutil"
)

// MaxNodes is the maximum number of invocation nodes that ResultDB
// can operate on at a time.
const MaxNodes = 10000

// reachCacheExpiration is expiration duration of ReachCache.
// It is more important to have *some* expiration; the value itself matters less
// because Redis evicts LRU keys only with *some* expiration set,
// see volatile-lru policy: https://redis.io/topics/lru-cache
const reachCacheExpiration = 30 * 24 * time.Hour // 30 days

// InclusionKey returns a spanner key for an Inclusion row.
func InclusionKey(including, included ID) spanner.Key {
	return spanner.Key{including.RowID(), included.RowID()}
}

// ReadIncluded reads ids of included invocations.
func ReadIncluded(ctx context.Context, txn spanutil.Txn, id ID) (IDSet, error) {
	var ret IDSet
	var b spanutil.Buffer
	err := txn.Read(ctx, "IncludedInvocations", id.Key().AsPrefix(), []string{"IncludedInvocationId"}).Do(func(row *spanner.Row) error {
		var included ID
		if err := b.FromSpanner(row, &included); err != nil {
			return err
		}
		if ret == nil {
			ret = make(IDSet)
		}
		ret.Add(included)
		return nil
	})
	if err != nil {
		return nil, err
	}
	return ret, nil
}

// TooManyTag set in an error indicates that too many invocations
// matched a condition.
var TooManyTag = errors.BoolTag{
	Key: errors.NewTagKey("too many matching invocations matched the condition"),
}

// Reachable returns all invocations reachable from roots along the inclusion
// edges.
// May return an appstatus-annotated error.
func Reachable(ctx context.Context, txn spanutil.Txn, roots IDSet) (reachable IDSet, err error) {
	ctx, ts := trace.StartSpan(ctx, "resultdb.readReachableInvocations")
	defer func() { ts.End(err) }()

	eg, ctx := errgroup.WithContext(ctx)
	defer eg.Wait()

	tooMany := func() error {
		return errors.Reason("more than %d invocations match", MaxNodes).Tag(TooManyTag).Err()
	}

	reachable = make(IDSet, len(roots))
	var mu sync.Mutex
	var visit func(id ID) error

	spanSem := semaphore.NewWeighted(64) // limits Spanner RPC concurrency.

	visit = func(id ID) error {
		mu.Lock()
		defer mu.Unlock()

		// Check if we already started/finished fetching this invocation.
		if reachable.Has(id) {
			return nil
		}

		// Consider fetching a new invocation.
		if len(reachable) == MaxNodes {
			return tooMany()
		}

		// Mark the invocation as being processed.
		reachable.Add(id)

		// Concurrently fetch the inclusions without a lock.
		eg.Go(func() error {
			if err := ctx.Err(); err != nil {
				return err
			}

			// First check the cache.
			switch ids, err := ReachCache(id).Read(ctx); {
			case err == redisconn.ErrNotConfigured || err == ErrUnknownReach:
				// Ignore this error.
			case err != nil:
				logging.Warningf(ctx, "ReachCache: failed to read %s: %s", id, err)
			default:
				// Cache hit. Copy the results to `reachable` and exit without
				// recursion.
				mu.Lock()
				defer mu.Unlock()
				reachable.Union(ids)
				if len(reachable) > MaxNodes {
					return tooMany()
				}
				return nil
			}

			// Cache miss. => Read from Spanner.
			if err := spanSem.Acquire(ctx, 1); err != nil {
				return err
			}
			included, err := ReadIncluded(ctx, txn, id)
			spanSem.Release(1)
			if err != nil {
				return errors.Annotate(err, "failed to read inclusions of %s", id.Name()).Err()
			}

			for id := range included {
				if err := visit(id); err != nil {
					return err
				}
			}
			return nil
		})
		return nil
	}

	// Trigger fetching by requesting all roots.
	for id := range roots {
		if err := visit(id); err != nil {
			return nil, err
		}
	}

	// Wait for the entire graph to be fetched.
	if err := eg.Wait(); err != nil {
		return nil, err
	}
	logging.Debugf(ctx, "%d invocations are reachable from %s", len(reachable), roots.Names())
	return reachable, nil
}

// ReachCache is a cache of all invocations reachable from the given
// invocation, stored in Redis. The cached set is either correct or absent.
//
// The cache must be written only after the set of reachable invocations
// becomes immutable, i.e. when the including invocation is finalized.
// This is important to be able to tolerate transient Redis failures
// and avoid a situation where we failed to update the currently stored set,
// ignored the failure and then, after Redis came back online, read the
// stale set.
type ReachCache ID

// key returns the Redis key.
func (c ReachCache) key() string {
	return fmt.Sprintf("reach:%s", c)
}

// Write writes the new value.
// The value does not have to include c, this is implied.
func (c ReachCache) Write(ctx context.Context, value IDSet) (err error) {
	ctx, ts := trace.StartSpan(ctx, "resultdb.reachCache.write")
	ts.Attribute("id", string(c))
	defer func() { ts.End(err) }()

	conn, err := redisconn.Get(ctx)
	if err != nil {
		return err
	}
	defer conn.Close()

	key := c.key()

	marshaled := &bytes.Buffer{}
	if len(value) == 0 {
		// Redis does not support empty values. Write just "\n".
		fmt.Fprintln(marshaled)
	} else {
		for id := range value {
			if id != ID(c) {
				fmt.Fprintln(marshaled, id)
			}
		}
	}
	ts.Attribute("size", marshaled.Len())

	conn.Send("SET", key, marshaled.Bytes())
	conn.Send("EXPIRE", key, int(reachCacheExpiration.Seconds()))
	_, err = conn.Do("")
	return err
}

// TryWrite tries to write the new value. On failure, logs it.
func (c ReachCache) TryWrite(ctx context.Context, value IDSet) {
	switch err := c.Write(ctx, value); {
	case err == redisconn.ErrNotConfigured:

	case err != nil:
		logging.Warningf(ctx, "ReachCache: failed to write %s: %s", c, err)
	}
}

// ErrUnknownReach is returned by ReachCache.Read if the cached value is absent.
var ErrUnknownReach = fmt.Errorf("the reachable set is unknown")

var memberSep = []byte("\n")

// Read reads the current value.
// Returns ErrUnknownReach if the value is absent.
//
// If err is nil, ids includes c, even if it was not passed in Write().
func (c ReachCache) Read(ctx context.Context) (ids IDSet, err error) {
	ctx, ts := trace.StartSpan(ctx, "resultdb.reachCache.read")
	ts.Attribute("id", string(c))
	defer func() { ts.End(err) }()

	conn, err := redisconn.Get(ctx)
	if err != nil {
		return nil, err
	}
	defer conn.Close()

	members, err := redis.Bytes(conn.Do("GET", c.key()))
	switch {
	case err == redis.ErrNil:
		return nil, ErrUnknownReach
	case err != nil:
		return nil, err
	}
	ts.Attribute("size", len(members))

	split := bytes.Split(members, memberSep)
	ids = make(IDSet, len(split)+1)
	ids.Add(ID(c))
	for _, id := range split {
		if len(id) > 0 {
			ids.Add(ID(id))
		}
	}
	return ids, nil
}
<commit_msg>[resultdb] Added logging in invocations.Reachable<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package invocations

import (
	"bytes"
	"context"
	"fmt"
	"sync"
	"time"

	"cloud.google.com/go/spanner"
	"github.com/gomodule/redigo/redis"
	"golang.org/x/sync/errgroup"
	"golang.org/x/sync/semaphore"

	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/trace"
	"go.chromium.org/luci/server/redisconn"

	"go.chromium.org/luci/resultdb/internal/spanutil"
)

// MaxNodes is the maximum number of invocation nodes that ResultDB
// can operate on at a time.
const MaxNodes = 10000

// reachCacheExpiration is expiration duration of ReachCache.
// It is more important to have *some* expiration; the value itself matters less
// because Redis evicts LRU keys only with *some* expiration set,
// see volatile-lru policy: https://redis.io/topics/lru-cache
const reachCacheExpiration = 30 * 24 * time.Hour // 30 days

// InclusionKey returns a spanner key for an Inclusion row.
func InclusionKey(including, included ID) spanner.Key {
	return spanner.Key{including.RowID(), included.RowID()}
}

// ReadIncluded reads ids of included invocations.
func ReadIncluded(ctx context.Context, txn spanutil.Txn, id ID) (IDSet, error) {
	var ret IDSet
	var b spanutil.Buffer
	err := txn.Read(ctx, "IncludedInvocations", id.Key().AsPrefix(), []string{"IncludedInvocationId"}).Do(func(row *spanner.Row) error {
		var included ID
		if err := b.FromSpanner(row, &included); err != nil {
			return err
		}
		if ret == nil {
			ret = make(IDSet)
		}
		ret.Add(included)
		return nil
	})
	if err != nil {
		return nil, err
	}
	return ret, nil
}

// TooManyTag set in an error indicates that too many invocations
// matched a condition.
var TooManyTag = errors.BoolTag{
	Key: errors.NewTagKey("too many matching invocations matched the condition"),
}

// Reachable returns all invocations reachable from roots along the inclusion
// edges.
// May return an appstatus-annotated error.
func Reachable(ctx context.Context, txn spanutil.Txn, roots IDSet) (reachable IDSet, err error) {
	ctx, ts := trace.StartSpan(ctx, "resultdb.readReachableInvocations")
	defer func() { ts.End(err) }()

	eg, ctx := errgroup.WithContext(ctx)
	defer eg.Wait()

	tooMany := func() error {
		return errors.Reason("more than %d invocations match", MaxNodes).Tag(TooManyTag).Err()
	}

	reachable = make(IDSet, len(roots))
	var mu sync.Mutex
	var visit func(id ID) error

	spanSem := semaphore.NewWeighted(64) // limits Spanner RPC concurrency.

	visit = func(id ID) error {
		mu.Lock()
		defer mu.Unlock()

		// Check if we already started/finished fetching this invocation.
		if reachable.Has(id) {
			return nil
		}

		// Consider fetching a new invocation.
		if len(reachable) == MaxNodes {
			return tooMany()
		}

		// Mark the invocation as being processed.
		reachable.Add(id)

		// Concurrently fetch the inclusions without a lock.
		eg.Go(func() error {
			if err := ctx.Err(); err != nil {
				return err
			}

			// First check the cache.
			switch ids, err := ReachCache(id).Read(ctx); {
			case err == redisconn.ErrNotConfigured || err == ErrUnknownReach:
				// Ignore this error.
			case err != nil:
				logging.Warningf(ctx, "ReachCache: failed to read %s: %s", id, err)
			default:
				// Cache hit. Copy the results to `reachable` and exit without
				// recursion.
				mu.Lock()
				defer mu.Unlock()
				reachable.Union(ids)
				if len(reachable) > MaxNodes {
					return tooMany()
				}
				return nil
			}

			// Cache miss. => Read from Spanner.
			if err := spanSem.Acquire(ctx, 1); err != nil {
				return err
			}
			included, err := ReadIncluded(ctx, txn, id)
			spanSem.Release(1)
			if err != nil {
				logging.Debugf(ctx, "ReadIncluded errored: %s", err)
				return errors.Annotate(err, "failed to read inclusions of %s", id.Name()).Err()
			}

			for id := range included {
				if err := visit(id); err != nil {
					return err
				}
			}
			return nil
		})
		return nil
	}

	// Trigger fetching by requesting all roots.
	for id := range roots {
		if err := visit(id); err != nil {
			return nil, err
		}
	}

	// Wait for the entire graph to be fetched.
	if err := eg.Wait(); err != nil {
		return nil, err
	}
	logging.Debugf(ctx, "%d invocations are reachable from %s", len(reachable), roots.Names())
	return reachable, nil
}

// ReachCache is a cache of all invocations reachable from the given
// invocation, stored in Redis. The cached set is either correct or absent.
//
// The cache must be written only after the set of reachable invocations
// becomes immutable, i.e. when the including invocation is finalized.
// This is important to be able to tolerate transient Redis failures
// and avoid a situation where we failed to update the currently stored set,
// ignored the failure and then, after Redis came back online, read the
// stale set.
type ReachCache ID

// key returns the Redis key.
func (c ReachCache) key() string {
	return fmt.Sprintf("reach:%s", c)
}

// Write writes the new value.
// The value does not have to include c, this is implied.
func (c ReachCache) Write(ctx context.Context, value IDSet) (err error) {
	ctx, ts := trace.StartSpan(ctx, "resultdb.reachCache.write")
	ts.Attribute("id", string(c))
	defer func() { ts.End(err) }()

	conn, err := redisconn.Get(ctx)
	if err != nil {
		return err
	}
	defer conn.Close()

	key := c.key()

	marshaled := &bytes.Buffer{}
	if len(value) == 0 {
		// Redis does not support empty values. Write just "\n".
		fmt.Fprintln(marshaled)
	} else {
		for id := range value {
			if id != ID(c) {
				fmt.Fprintln(marshaled, id)
			}
		}
	}
	ts.Attribute("size", marshaled.Len())

	conn.Send("SET", key, marshaled.Bytes())
	conn.Send("EXPIRE", key, int(reachCacheExpiration.Seconds()))
	_, err = conn.Do("")
	return err
}

// TryWrite tries to write the new value. On failure, logs it.
func (c ReachCache) TryWrite(ctx context.Context, value IDSet) {
	switch err := c.Write(ctx, value); {
	case err == redisconn.ErrNotConfigured:

	case err != nil:
		logging.Warningf(ctx, "ReachCache: failed to write %s: %s", c, err)
	}
}

// ErrUnknownReach is returned by ReachCache.Read if the cached value is absent.
var ErrUnknownReach = fmt.Errorf("the reachable set is unknown")

var memberSep = []byte("\n")

// Read reads the current value.
// Returns ErrUnknownReach if the value is absent.
//
// If err is nil, ids includes c, even if it was not passed in Write().
func (c ReachCache) Read(ctx context.Context) (ids IDSet, err error) {
	ctx, ts := trace.StartSpan(ctx, "resultdb.reachCache.read")
	ts.Attribute("id", string(c))
	defer func() { ts.End(err) }()

	conn, err := redisconn.Get(ctx)
	if err != nil {
		return nil, err
	}
	defer conn.Close()

	members, err := redis.Bytes(conn.Do("GET", c.key()))
	switch {
	case err == redis.ErrNil:
		return nil, ErrUnknownReach
	case err != nil:
		return nil, err
	}
	ts.Attribute("size", len(members))

	split := bytes.Split(members, memberSep)
	ids = make(IDSet, len(split)+1)
	ids.Add(ID(c))
	for _, id := range split {
		if len(id) > 0 {
			ids.Add(ID(id))
		}
	}
	return ids, nil
}
<|endoftext|>


<commit_before>// Copyright 2013 The go-toggl AUTHORS. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package toggl

import (
	"errors"
	"fmt"
	"time"
)

// ProjectUsersService handles communication with the project_users related
// methods of the Toggl API.
//
// Toggl API docs: https://github.com/toggl/toggl_api_docs/blob/master/chapters/project_users.md
type ProjectUsersService struct {
	client *Client
}

// ProjectUser represents association between project and user.
type ProjectUser struct {
	ID          int        `json:"id,omitempty"`
	ProjectID   int        `json:"pid,omitempty"`
	UserID      int        `json:"uid,omitempty"`
	WorkspaceID int        `json:"wid,omitempty"`
	Manager     bool       `json:"manager,omitempty"`
	Rate        float64    `json:"rate,omitempty"`
	At          *time.Time `json:"time,omitempty"`
}

// ProjectUserMultipleUserID represents a project user where UID is a string which can hold
// multiple IDs separated by comma.
type ProjectUserMultipleUserID struct {
	ID          int        `json:"id,omitempty"`
	ProjectID   int        `json:"pid,omitempty"`
	UserID      string     `json:"uid,omitempty"`
	WorkspaceID int        `json:"wid,omitempty"`
	Manager     bool       `json:"manager,omitempty"`
	Rate        float64    `json:"rate,omitempty"`
	At          *time.Time `json:"time,omitempty"`
}

// ProjectUserResponse acts as a response wrapper where response returns
// in format of "data": ProjectUser's object.
type ProjectUserResponse struct {
	Data *ProjectUser `json:"data,omitempty"`
}

// ProjectUserMassResponse acts as a response wrapper where response returns
// in format of "data": [ ... ].
type ProjectUserMassResponse struct {
	Data []ProjectUser `json:"data,omitempty"`
}

// ProjectUserCreate represents posted data to be sent to project users endpoint.
type ProjectUserCreate struct {
	ProjectUser *ProjectUser `json:"project_user,omitempty"`
}

// ProjectUserMassCreate represents posted data to create multipe project users
// for single project.
type ProjectUserMassCreate struct {
	ProjectUser *ProjectUserMultipleUserID `json:"project_user,omitempty"`
}

// Create a project user.
//
// Toggl API docs: https://github.com/toggl/toggl_api_docs/blob/master/chapters/project_users.md#create-a-project-user
func (s *ProjectUsersService) Create(pu *ProjectUser) (*ProjectUser, error) {
	u := "project_users"
	puc := &ProjectUserCreate{ProjectUser: pu}
	req, err := s.client.NewRequest("POST", u, puc)
	if err != nil {
		return nil, err
	}

	data := new(ProjectUserResponse)
	_, err = s.client.Do(req, data)

	return data.Data, err
}

// MassCreate creates multiple project users for single project.
//
// Toggl API docs: https://github.com/toggl/toggl_api_docs/blob/master/chapters/project_users.md#create-multiple-project-users-for-single-project
func (s *ProjectUsersService) MassCreate(pu *ProjectUserMultipleUserID) ([]ProjectUser, error) {
	u := "project_users"
	puc := &ProjectUserMassCreate{ProjectUser: pu}
	req, err := s.client.NewRequest("POST", u, puc)
	if err != nil {
		return nil, err
	}

	data := new(ProjectUserMassResponse)
	_, err = s.client.Do(req, data)

	return data.Data, err
}

// Update a project user.
//
// Toggl API docs: https://github.com/toggl/toggl_api_docs/blob/master/chapters/project_users.md#update-a-project-user
func (s *ProjectUsersService) Update(pu *ProjectUser) (*ProjectUser, error) {
	if pu == nil {
		return nil, errors.New("ProjectUser cannot be nil")
	}
	if pu.ID <= 0 {
		return nil, errors.New("Invalid ProjectUser.ID")
	}

	u := fmt.Sprintf("project_users/%v", pu.ID)

	puc := &ProjectUserCreate{pu}
	req, err := s.client.NewRequest("PUT", u, puc)
	if err != nil {
		return nil, err
	}

	data := new(ProjectUserResponse)
	_, err = s.client.Do(req, data)

	return data.Data, err
}

// MassUpdate mass update project users.
//
// Toggl API docs: https://github.com/toggl/toggl_api_docs/blob/master/chapters/project_users.md#mass-update-for-project-users
func (s *ProjectUsersService) MassUpdate(pids string, pu *ProjectUser) ([]ProjectUser, error) {
	u := fmt.Sprintf("project_users/%v", pids)

	puc := &ProjectUserCreate{pu}
	req, err := s.client.NewRequest("PUT", u, puc)
	if err != nil {
		return nil, err
	}

	data := new(ProjectUserMassResponse)
	_, err = s.client.Do(req, data)

	return data.Data, err
}

// Delete a project user.
//
// Toggl API docs: https://github.com/toggl/toggl_api_docs/blob/master/chapters/project_users.md#delete-a-project-user
func (s *ProjectUsersService) Delete(id int) error {
	u := fmt.Sprintf("project_users/%v", id)
	req, err := s.client.NewRequest("DELETE", u, nil)
	if err != nil {
		return err
	}

	_, err = s.client.Do(req, nil)
	return err
}

// MassDelete a project user.
//
// Toggl API docs: https://github.com/toggl/toggl_api_docs/blob/master/chapters/project_users.md#delete-multiple-project-users
func (s *ProjectUsersService) MassDelete(pids string) error {
	u := fmt.Sprintf("project_users/%v", pids)
	req, err := s.client.NewRequest("DELETE", u, nil)
	if err != nil {
		return err
	}

	_, err = s.client.Do(req, nil)
	return err
}
<commit_msg>Fixed invalid 'at' JSON key's name.<commit_after>// Copyright 2013 The go-toggl AUTHORS. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package toggl

import (
	"errors"
	"fmt"
	"time"
)

// ProjectUsersService handles communication with the project_users related
// methods of the Toggl API.
//
// Toggl API docs: https://github.com/toggl/toggl_api_docs/blob/master/chapters/project_users.md
type ProjectUsersService struct {
	client *Client
}

// ProjectUser represents association between project and user.
type ProjectUser struct {
	ID          int        `json:"id,omitempty"`
	ProjectID   int        `json:"pid,omitempty"`
	UserID      int        `json:"uid,omitempty"`
	WorkspaceID int        `json:"wid,omitempty"`
	Manager     bool       `json:"manager,omitempty"`
	Rate        float64    `json:"rate,omitempty"`
	At          *time.Time `json:"at,omitempty"`
}

// ProjectUserMultipleUserID represents a project user where UID is a string which can hold
// multiple IDs separated by comma.
type ProjectUserMultipleUserID struct {
	ID          int        `json:"id,omitempty"`
	ProjectID   int        `json:"pid,omitempty"`
	UserID      string     `json:"uid,omitempty"`
	WorkspaceID int        `json:"wid,omitempty"`
	Manager     bool       `json:"manager,omitempty"`
	Rate        float64    `json:"rate,omitempty"`
	At          *time.Time `json:"at,omitempty"`
}

// ProjectUserResponse acts as a response wrapper where response returns
// in format of "data": ProjectUser's object.
type ProjectUserResponse struct {
	Data *ProjectUser `json:"data,omitempty"`
}

// ProjectUserMassResponse acts as a response wrapper where response returns
// in format of "data": [ ... ].
type ProjectUserMassResponse struct {
	Data []ProjectUser `json:"data,omitempty"`
}

// ProjectUserCreate represents posted data to be sent to project users endpoint.
type ProjectUserCreate struct {
	ProjectUser *ProjectUser `json:"project_user,omitempty"`
}

// ProjectUserMassCreate represents posted data to create multipe project users
// for single project.
type ProjectUserMassCreate struct {
	ProjectUser *ProjectUserMultipleUserID `json:"project_user,omitempty"`
}

// Create a project user.
//
// Toggl API docs: https://github.com/toggl/toggl_api_docs/blob/master/chapters/project_users.md#create-a-project-user
func (s *ProjectUsersService) Create(pu *ProjectUser) (*ProjectUser, error) {
	u := "project_users"
	puc := &ProjectUserCreate{ProjectUser: pu}
	req, err := s.client.NewRequest("POST", u, puc)
	if err != nil {
		return nil, err
	}

	data := new(ProjectUserResponse)
	_, err = s.client.Do(req, data)

	return data.Data, err
}

// MassCreate creates multiple project users for single project.
//
// Toggl API docs: https://github.com/toggl/toggl_api_docs/blob/master/chapters/project_users.md#create-multiple-project-users-for-single-project
func (s *ProjectUsersService) MassCreate(pu *ProjectUserMultipleUserID) ([]ProjectUser, error) {
	u := "project_users"
	puc := &ProjectUserMassCreate{ProjectUser: pu}
	req, err := s.client.NewRequest("POST", u, puc)
	if err != nil {
		return nil, err
	}

	data := new(ProjectUserMassResponse)
	_, err = s.client.Do(req, data)

	return data.Data, err
}

// Update a project user.
//
// Toggl API docs: https://github.com/toggl/toggl_api_docs/blob/master/chapters/project_users.md#update-a-project-user
func (s *ProjectUsersService) Update(pu *ProjectUser) (*ProjectUser, error) {
	if pu == nil {
		return nil, errors.New("ProjectUser cannot be nil")
	}
	if pu.ID <= 0 {
		return nil, errors.New("Invalid ProjectUser.ID")
	}

	u := fmt.Sprintf("project_users/%v", pu.ID)

	puc := &ProjectUserCreate{pu}
	req, err := s.client.NewRequest("PUT", u, puc)
	if err != nil {
		return nil, err
	}

	data := new(ProjectUserResponse)
	_, err = s.client.Do(req, data)

	return data.Data, err
}

// MassUpdate mass update project users.
//
// Toggl API docs: https://github.com/toggl/toggl_api_docs/blob/master/chapters/project_users.md#mass-update-for-project-users
func (s *ProjectUsersService) MassUpdate(pids string, pu *ProjectUser) ([]ProjectUser, error) {
	u := fmt.Sprintf("project_users/%v", pids)

	puc := &ProjectUserCreate{pu}
	req, err := s.client.NewRequest("PUT", u, puc)
	if err != nil {
		return nil, err
	}

	data := new(ProjectUserMassResponse)
	_, err = s.client.Do(req, data)

	return data.Data, err
}

// Delete a project user.
//
// Toggl API docs: https://github.com/toggl/toggl_api_docs/blob/master/chapters/project_users.md#delete-a-project-user
func (s *ProjectUsersService) Delete(id int) error {
	u := fmt.Sprintf("project_users/%v", id)
	req, err := s.client.NewRequest("DELETE", u, nil)
	if err != nil {
		return err
	}

	_, err = s.client.Do(req, nil)
	return err
}

// MassDelete a project user.
//
// Toggl API docs: https://github.com/toggl/toggl_api_docs/blob/master/chapters/project_users.md#delete-multiple-project-users
func (s *ProjectUsersService) MassDelete(pids string) error {
	u := fmt.Sprintf("project_users/%v", pids)
	req, err := s.client.NewRequest("DELETE", u, nil)
	if err != nil {
		return err
	}

	_, err = s.client.Do(req, nil)
	return err
}
<|endoftext|>