<commit_before>package job

import (
	"encoding/json"
	"errors"
	"fmt"
	"strings"

	"github.com/coreos/fleet/unit"
)

// Fleet specific unit file requirement keys.
// "X-" prefix only appears in unit file and dropped
// in code before value is used.
const (
	// Require the unit be scheduled to a specific machine defined by given boot ID.
	FleetXConditionMachineBootID = "ConditionMachineBootID"
	// Limit eligible machines to the one that hosts a specific unit.
	FleetXConditionMachineOf = "ConditionMachineOf"
	// Prevent a unit from being collocated with other units using glob-matching on the other unit names.
	FleetXConflicts = "Conflicts"
)

func SupportedUnitTypes() []string {
	return []string{"service", "socket"}
}

type JobPayload struct {
	Name string
	Unit unit.SystemdUnitFile
}

func NewJobPayload(name string, uFile unit.SystemdUnitFile) *JobPayload {
	return &JobPayload{name, uFile}
}

func (jp *JobPayload) Type() (string, error) {
	for _, ut := range SupportedUnitTypes() {
		if strings.HasSuffix(jp.Name, fmt.Sprintf(".%s", ut)) {
			return ut, nil
		}
	}

	return "", errors.New(fmt.Sprintf("Unrecognized systemd unit %s", jp.Name))
}

// Peers returns a list of Payload names that must be scheduled to the same
// machine as this Payload. If no peers were explicitly defined for certain unit
// types, a default list of peers will be returned. This behavior only applies
// to the socket unit type. For example, the default peer of foo.socket would
// be foo.service.
func (jp *JobPayload) Peers() []string {
	if peers, ok := jp.Requirements()[FleetXConditionMachineOf]; ok {
		return peers
	}

	peers := make([]string, 0)

	jpType, err := jp.Type()
	if err != nil {
		return peers
	}

	if jpType != "socket" {
		return peers
	}

	baseName := strings.TrimSuffix(jp.Name, fmt.Sprintf(".%s", jpType))
	serviceName := fmt.Sprintf("%s.%s", baseName, "service")
	peers = append(peers, serviceName)

	return peers
}

func (jp *JobPayload) Conflicts() []string {
	conflicts, ok := jp.Requirements()["Conflicts"]
	if ok {
		return conflicts
	} else {
		return make([]string, 0)
	}
}

// Requirements returns all relevant options from the [X-Fleet] section
// of a unit file. Relevant options are identified with a `X-` prefix in
// the unit. This prefix is stripped from relevant options before
// being returned.
func (jp *JobPayload) Requirements() map[string][]string {
	requirements := make(map[string][]string)
	for key, value := range jp.Unit.Contents["X-Fleet"] {
		if !strings.HasPrefix(key, "X-") {
			continue
		}

		// Strip off leading X-
		key = key[2:]

		if _, ok := requirements[key]; !ok {
			requirements[key] = make([]string, 0)
		}

		requirements[key] = value
	}

	return requirements
}

func (jp *JobPayload) UnmarshalJSON(data []byte) error {
	var jpm jobPayloadModel
	err := json.Unmarshal(data, &jpm)
	if err != nil {
		return errors.New(fmt.Sprintf("Unable to JSON-deserialize object: %s", err))
	}

	if len(jpm.Unit.Raw) > 0 {
		jp.Unit = *unit.NewSystemdUnitFile(jpm.Unit.Raw)
	} else {
		jp.Unit = *unit.NewSystemdUnitFileFromLegacyContents(jpm.Unit.Contents)
	}

	jp.Name = jpm.Name
	return nil
}

func (jp *JobPayload) MarshalJSON() ([]byte, error) {
	ufm := unitFileModel{
		Contents: jp.Unit.LegacyContents(),
		Raw:      jp.Unit.String(),
	}
	jpm := jobPayloadModel{Name: jp.Name, Unit: ufm}
	return json.Marshal(jpm)
}

// unitFileModel is just used for serialization
type unitFileModel struct {
	// Contents is now a legacy field, only read by older instances of fleet
	Contents map[string]map[string]string
	Raw      string
}

// jobPayloadModel is just used for serialization
type jobPayloadModel struct {
	Name string
	Unit unitFileModel
}
<commit_msg>fix(payload): Use FleetXConflicts constant<commit_after>package job

import (
	"encoding/json"
	"errors"
	"fmt"
	"strings"

	"github.com/coreos/fleet/unit"
)

// Fleet specific unit file requirement keys.
// "X-" prefix only appears in unit file and dropped
// in code before value is used.
const (
	// Require the unit be scheduled to a specific machine defined by given boot ID.
	FleetXConditionMachineBootID = "ConditionMachineBootID"
	// Limit eligible machines to the one that hosts a specific unit.
	FleetXConditionMachineOf = "ConditionMachineOf"
	// Prevent a unit from being collocated with other units using glob-matching on the other unit names.
	FleetXConflicts = "Conflicts"
)

func SupportedUnitTypes() []string {
	return []string{"service", "socket"}
}

type JobPayload struct {
	Name string
	Unit unit.SystemdUnitFile
}

func NewJobPayload(name string, uFile unit.SystemdUnitFile) *JobPayload {
	return &JobPayload{name, uFile}
}

func (jp *JobPayload) Type() (string, error) {
	for _, ut := range SupportedUnitTypes() {
		if strings.HasSuffix(jp.Name, fmt.Sprintf(".%s", ut)) {
			return ut, nil
		}
	}

	return "", errors.New(fmt.Sprintf("Unrecognized systemd unit %s", jp.Name))
}

// Peers returns a list of Payload names that must be scheduled to the same
// machine as this Payload. If no peers were explicitly defined for certain unit
// types, a default list of peers will be returned. This behavior only applies
// to the socket unit type. For example, the default peer of foo.socket would
// be foo.service.
func (jp *JobPayload) Peers() []string {
	if peers, ok := jp.Requirements()[FleetXConditionMachineOf]; ok {
		return peers
	}

	peers := make([]string, 0)

	jpType, err := jp.Type()
	if err != nil {
		return peers
	}

	if jpType != "socket" {
		return peers
	}

	baseName := strings.TrimSuffix(jp.Name, fmt.Sprintf(".%s", jpType))
	serviceName := fmt.Sprintf("%s.%s", baseName, "service")
	peers = append(peers, serviceName)

	return peers
}

func (jp *JobPayload) Conflicts() []string {
	conflicts, ok := jp.Requirements()[FleetXConflicts]
	if ok {
		return conflicts
	} else {
		return make([]string, 0)
	}
}

// Requirements returns all relevant options from the [X-Fleet] section
// of a unit file. Relevant options are identified with a `X-` prefix in
// the unit. This prefix is stripped from relevant options before
// being returned.
func (jp *JobPayload) Requirements() map[string][]string {
	requirements := make(map[string][]string)
	for key, value := range jp.Unit.Contents["X-Fleet"] {
		if !strings.HasPrefix(key, "X-") {
			continue
		}

		// Strip off leading X-
		key = key[2:]

		if _, ok := requirements[key]; !ok {
			requirements[key] = make([]string, 0)
		}

		requirements[key] = value
	}

	return requirements
}

func (jp *JobPayload) UnmarshalJSON(data []byte) error {
	var jpm jobPayloadModel
	err := json.Unmarshal(data, &jpm)
	if err != nil {
		return errors.New(fmt.Sprintf("Unable to JSON-deserialize object: %s", err))
	}

	if len(jpm.Unit.Raw) > 0 {
		jp.Unit = *unit.NewSystemdUnitFile(jpm.Unit.Raw)
	} else {
		jp.Unit = *unit.NewSystemdUnitFileFromLegacyContents(jpm.Unit.Contents)
	}

	jp.Name = jpm.Name
	return nil
}

func (jp *JobPayload) MarshalJSON() ([]byte, error) {
	ufm := unitFileModel{
		Contents: jp.Unit.LegacyContents(),
		Raw:      jp.Unit.String(),
	}
	jpm := jobPayloadModel{Name: jp.Name, Unit: ufm}
	return json.Marshal(jpm)
}

// unitFileModel is just used for serialization
type unitFileModel struct {
	// Contents is now a legacy field, only read by older instances of fleet
	Contents map[string]map[string]string
	Raw      string
}

// jobPayloadModel is just used for serialization
type jobPayloadModel struct {
	Name string
	Unit unitFileModel
}
<|endoftext|>


<commit_before>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This binary reads the raw conversions from a file, splits the conversions into secret DPF keys and encrypts the shares for different helpers.
// The pipeline can be executed in two ways:
//
// 1. Directly on local
// /path/to/dpf_generate_partial_report \
// --conversion_file=/path/to/conversion_data.csv \
// --partial_report_file1=/path/to/partial_report_1.txt \
// --partial_report_file2=/path/to/partial_report_2.txt \
// --public_key_dir1=/path/to/public_key_dir1 \
// --public_key_dir2=/path/to/public_key_dir2 \
// --runner=direct
//
// 2. Dataflow on cloud
// /path/to/dpf_generate_partial_report \
// --conversion_file=gs://<browser bucket>/conversion_data.csv \
// --partial_report_file1=gs://<helper bucket>/partial_report_1.txt \
// --partial_report_file2=gs://<helper bucket>/partial_report_2.txt \
// --public_key_dir1=/path/to/public_key_dir1 \
// --public_key_dir2=/path/to/public_key_dir2 \
// --runner=dataflow \
// --project=<GCP project> \
// --temp_location=gs://<dataflow temp dir> \
// --staging_location=gs://<dataflow temp dir> \
// --worker_binary=/path/to/dpf_generate_partial_report

package main

import (
	"context"
	"flag"
	"path"

	"github.com/apache/beam/sdks/go/pkg/beam"
	"github.com/apache/beam/sdks/go/pkg/beam/log"
	"github.com/apache/beam/sdks/go/pkg/beam/x/beamx"
	"github.com/google/privacy-sandbox-aggregation-service/pipeline/cryptoio"
	"github.com/google/privacy-sandbox-aggregation-service/pipeline/dpfbrowsersimulator"
)

var (
	conversionFile     = flag.String("conversion_file", "", "Input raw conversion data.")
	partialReportFile1 = flag.String("partial_report_file1", "", "Output partial report for helper 1.")
	partialReportFile2 = flag.String("partial_report_file2", "", "Output partial report for helper 2.")

	publicKeyDir1 = flag.String("public_key_dir1", "", "Directory for public keys from helper 1.")
	publicKeyDir2 = flag.String("public_key_dir2", "", "Directory for public keys from helper 2.")
	keyBitSize    = flag.Int("key_bit_size", 32, "Bit size of the conversion keys.")

	fileShards = flag.Int64("file_shards", 1, "The number of shards for the output file.")
)

func main() {
	flag.Parse()

	beam.Init()

	ctx := context.Background()
	helperPubKey1, err := cryptoio.ReadStandardPublicKey(path.Join(*publicKeyDir1, cryptoio.DefaultStandardPublicKey))
	if err != nil {
		log.Exit(ctx, err)
	}
	helperPubKey2, err := cryptoio.ReadStandardPublicKey(path.Join(*publicKeyDir2, cryptoio.DefaultStandardPublicKey))
	if err != nil {
		log.Exit(ctx, err)
	}

	pipeline := beam.NewPipeline()
	scope := pipeline.Root()

	dpfbrowsersimulator.GeneratePartialReport(scope, &dpfbrowsersimulator.GeneratePartialReportParams{
		ConversionFile:     *conversionFile,
		PartialReportFile1: *partialReportFile1,
		PartialReportFile2: *partialReportFile2,
		KeyBitSize:         int32(*keyBitSize),
		PublicKey1:         helperPubKey1,
		PublicKey2:         helperPubKey2,
		Shards:             *fileShards,
	})
	if err := beamx.Run(ctx, pipeline); err != nil {
		log.Exitf(ctx, "Failed to execute job: %s", err)
	}
}
<commit_msg>Use a specific public key instead of specifying the public key directory, since there may be multiple public keys.<commit_after>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This binary reads the raw conversions from a file, splits the conversions into secret DPF keys and encrypts the shares for different helpers.
// The pipeline can be executed in two ways:
//
// 1. Directly on local
// /path/to/dpf_generate_partial_report \
// --conversion_file=/path/to/conversion_data.csv \
// --partial_report_file1=/path/to/partial_report_1.txt \
// --partial_report_file2=/path/to/partial_report_2.txt \
// --public_key_file1=/path/to/public_key_file1 \
// --public_key_file2=/path/to/public_key_file2 \
// --runner=direct
//
// 2. Dataflow on cloud
// /path/to/dpf_generate_partial_report \
// --conversion_file=gs://<browser bucket>/conversion_data.csv \
// --partial_report_file1=gs://<helper bucket>/partial_report_1.txt \
// --partial_report_file2=gs://<helper bucket>/partial_report_2.txt \
// --public_key_file1=/path/to/public_key_file1 \
// --public_key_file2=/path/to/public_key_file2 \
// --runner=dataflow \
// --project=<GCP project> \
// --temp_location=gs://<dataflow temp dir> \
// --staging_location=gs://<dataflow temp dir> \
// --worker_binary=/path/to/dpf_generate_partial_report

package main

import (
	"context"
	"flag"

	"github.com/apache/beam/sdks/go/pkg/beam"
	"github.com/apache/beam/sdks/go/pkg/beam/log"
	"github.com/apache/beam/sdks/go/pkg/beam/x/beamx"
	"github.com/google/privacy-sandbox-aggregation-service/pipeline/cryptoio"
	"github.com/google/privacy-sandbox-aggregation-service/pipeline/dpfbrowsersimulator"
)

var (
	conversionFile     = flag.String("conversion_file", "", "Input raw conversion data.")
	partialReportFile1 = flag.String("partial_report_file1", "", "Output partial report for helper 1.")
	partialReportFile2 = flag.String("partial_report_file2", "", "Output partial report for helper 2.")

	publicKeyFile1 = flag.String("public_key_file1", "", "Input file containing the public key from helper 1.")
	publicKeyFile2 = flag.String("public_key_file2", "", "Input file containing the public key from helper 2.")
	keyBitSize     = flag.Int("key_bit_size", 32, "Bit size of the conversion keys.")

	fileShards = flag.Int64("file_shards", 1, "The number of shards for the output file.")
)

func main() {
	flag.Parse()

	beam.Init()

	ctx := context.Background()
	helperPubKey1, err := cryptoio.ReadStandardPublicKey(*publicKeyFile1)
	if err != nil {
		log.Exit(ctx, err)
	}
	helperPubKey2, err := cryptoio.ReadStandardPublicKey(*publicKeyFile2)
	if err != nil {
		log.Exit(ctx, err)
	}

	pipeline := beam.NewPipeline()
	scope := pipeline.Root()

	dpfbrowsersimulator.GeneratePartialReport(scope, &dpfbrowsersimulator.GeneratePartialReportParams{
		ConversionFile:     *conversionFile,
		PartialReportFile1: *partialReportFile1,
		PartialReportFile2: *partialReportFile2,
		KeyBitSize:         int32(*keyBitSize),
		PublicKey1:         helperPubKey1,
		PublicKey2:         helperPubKey2,
		Shards:             *fileShards,
	})
	if err := beamx.Run(ctx, pipeline); err != nil {
		log.Exitf(ctx, "Failed to execute job: %s", err)
	}
}
<|endoftext|>


<commit_before>package v20170831

import "encoding/json"

// The validate tag is used for validation
// Reference to gopkg.in/go-playground/validator.v9

// ResourcePurchasePlan defines resource plan as required by ARM
// for billing purposes.
type ResourcePurchasePlan struct {
	Name          string `json:"name,omitempty"`
	Product       string `json:"product,omitempty"`
	PromotionCode string `json:"promotionCode,omitempty"`
	Publisher     string `json:"publisher,omitempty"`
}

// HostedMaster complies with the ARM model of
// resource definition in a JSON template.
type HostedMaster struct {
	ID       string                `json:"id,omitempty"`
	Location string                `json:"location,omitempty" validate:"required"`
	Name     string                `json:"name,omitempty"`
	Plan     *ResourcePurchasePlan `json:"plan,omitempty"`
	Tags     map[string]string     `json:"tags,omitempty"`
	Type     string                `json:"type,omitempty"`

	Properties *Properties `json:"properties"`
}

// Properties represents the ACS cluster definition
type Properties struct {
	ProvisioningState ProvisioningState   `json:"provisioningState,omitempty"`
	Version           OrchestratorVersion `json:"version"`
	DNSPrefix         string              `json:"dnsPrefix" validate:"required"`
	// Master LB public endpoint/FQDN with port
	// The format will be FQDN:2376
	// Not used during PUT, returned as part of GET
	FQDN                    string                   `json:"fqdn,omitempty"`
	AgentPoolProfiles       []*AgentPoolProfile      `json:"agentPoolProfiles,omitempty" validate:"dive,required"`
	LinuxProfile            *LinuxProfile            `json:"linuxProfile,omitempty" validate:"required"`
	WindowsProfile          *WindowsProfile          `json:"windowsProfile,omitempty"`
	ServicePrincipalProfile *ServicePrincipalProfile `json:"servicePrincipalProfile,omitempty"`
	NetworkProfile          *NetworkProfile          `json:"networkProfile,omitempty"`
	AccessProfiles          []*AccessProfile         `json:"accessProfiles,omitempty"`
	JumpboxProfile          *JumpboxProfile          `json:"jumpboxProfile,omitempty"`
}

// ServicePrincipalProfile contains the client and secret used by the cluster for Azure Resource CRUD
// The 'Secret' parameter could be either a plain text, or referenced to a secret in a keyvault.
// In the latter case, the format of the parameter's value should be
// "/subscriptions/<SUB_ID>/resourceGroups/<RG_NAME>/providers/Microsoft.KeyVault/vaults/<KV_NAME>/secrets/<NAME>[/<VERSION>]"
// where:
//    <SUB_ID> is the subscription ID of the keyvault
//    <RG_NAME> is the resource group of the keyvault
//    <KV_NAME> is the name of the keyvault
//    <NAME> is the name of the secret.
//    <VERSION> (optional) is the version of the secret (default: the latest version)
type ServicePrincipalProfile struct {
	ClientID string `json:"clientId,omitempty" validate:"required"`
	Secret   string `json:"secret,omitempty" validate:"required"`
}

// LinuxProfile represents the Linux configuration passed to the cluster
type LinuxProfile struct {
	AdminUsername string `json:"adminUsername" validate:"required"`

	SSH struct {
		PublicKeys []PublicKey `json:"publicKeys" validate:"required,len=1"`
	} `json:"ssh" validate:"required"`
}

// PublicKey represents an SSH key for LinuxProfile
type PublicKey struct {
	KeyData string `json:"keyData"`
}

// WindowsProfile represents the Windows configuration passed to the cluster
type WindowsProfile struct {
	AdminUsername string `json:"adminUsername,omitempty" validate:"required"`
	AdminPassword string `json:"adminPassword,omitempty" validate:"required"`
}

// ProvisioningState represents the current state of container service resource.
type ProvisioningState string

const (
	// Creating means ContainerService resource is being created.
	Creating ProvisioningState = "Creating"
	// Updating means an existing ContainerService resource is being updated
	Updating ProvisioningState = "Updating"
	// Failed means resource is in failed state
	Failed ProvisioningState = "Failed"
	// Succeeded means resource created succeeded during last create/update
	Succeeded ProvisioningState = "Succeeded"
	// Deleting means resource is in the process of being deleted
	Deleting ProvisioningState = "Deleting"
	// Migrating means resource is being migrated from one subscription or
	// resource group to another
	Migrating ProvisioningState = "Migrating"
)

// NetworkProfile represents the definition of network profile for the cluster
type NetworkProfile struct {
	PodCidr          string `json:"podCidr,omitempty"`
	ServiceCidr      string `json:"serviceCidr,omitempty"`
	KubeDNSServiceIP string `json:"kubeDNSServiceIP,omitempty"`
}

// JumpboxProfile represents the definition of bastion machine to access the agent nodes
type JumpboxProfile struct {
	PublicIPAddressID string `json:"publicIPAddressID,omitempty"`
	PublicIPAddress   string `json:"publicIPAddress,omitempty"`
}

// AgentPoolProfile represents configuration of VMs running agent
// daemons that register with the master and offer resources to
// host applications in containers.
type AgentPoolProfile struct {
	Name           string `json:"name" validate:"required"`
	Count          int    `json:"count" validate:"required,min=1,max=100"`
	VMSize         string `json:"vmSize" validate:"required"`
	OSDiskSizeGB   int    `json:"osDiskSizeGB,omitempty" validate:"min=0,max=1023"`
	StorageProfile string `json:"storageProfile" validate:"eq=StorageAccount|eq=ManagedDisks|len=0"`
	VnetSubnetID   string `json:"vnetSubnetID,omitempty"`

	// OSType is the operating system type for agents
	// Set as nullable to support backward compat because
	// this property was added later.
	// If the value is null or not set, it defaulted to Linux.
	OSType OSType `json:"osType,omitempty"`

	// subnet is internal
	subnet string
}

// AccessProfile represents role name and kubeconfig
type AccessProfile struct {
	RoleName   string `json:"roleName"`
	KubeConfig string `json:"kubeConfig"`
}

// UnmarshalJSON unmarshal json using the default behavior
// And do fields manipulation, such as populating default value
func (a *AgentPoolProfile) UnmarshalJSON(b []byte) error {
	// Need to have a alias type to avoid circular unmarshal
	type aliasAgentPoolProfile AgentPoolProfile
	aa := aliasAgentPoolProfile{}
	if e := json.Unmarshal(b, &aa); e != nil {
		return e
	}
	*a = AgentPoolProfile(aa)
	if a.Count == 0 {
		// if AgentPoolProfile.Count is missing or 0, set it to default 1
		a.Count = 1
	}

	if a.StorageProfile == "" {
		// if StorageProfile is missing, set to default StorageAccount
		a.StorageProfile = StorageAccount
	}

	if string(a.OSType) == "" {
		// OSType is the operating system type for agents
		// Set as nullable to support backward compat because
		// this property was added later.
		// If the value is null or not set, it defaulted to Linux.
		a.OSType = Linux
	}

	// OSDiskSizeGB is an override value. vm sizes have default OS disk sizes.
	// If it is not set. The user should get the default for the vm size
	return nil
}

// OrchestratorVersion defines the version for orchestratorType
type OrchestratorVersion string

// OSType represents OS types of agents
type OSType string

// HasWindows returns true if the cluster contains windows
func (a *Properties) HasWindows() bool {
	for _, agentPoolProfile := range a.AgentPoolProfiles {
		if agentPoolProfile.OSType == Windows {
			return true
		}
	}
	return false
}

// IsCustomVNET returns true if the customer brought their own VNET
func (a *AgentPoolProfile) IsCustomVNET() bool {
	return len(a.VnetSubnetID) > 0
}

// IsWindows returns true if the agent pool is windows
func (a *AgentPoolProfile) IsWindows() bool {
	return a.OSType == Windows
}

// IsLinux returns true if the agent pool is linux
func (a *AgentPoolProfile) IsLinux() bool {
	return a.OSType == Linux
}

// GetSubnet returns the read-only subnet for the agent pool
func (a *AgentPoolProfile) GetSubnet() string {
	return a.subnet
}

// SetSubnet sets the read-only subnet for the agent pool
func (a *AgentPoolProfile) SetSubnet(subnet string) {
	a.subnet = subnet
}

// IsManagedDisks returns true if the customer specified managed disks
func (a *AgentPoolProfile) IsManagedDisks() bool {
	return a.StorageProfile == ManagedDisks
}

// IsStorageAccount returns true if the customer specified storage account
func (a *AgentPoolProfile) IsStorageAccount() bool {
	return a.StorageProfile == StorageAccount
}
<commit_msg>Add new fields to agentpoolonly API model (#1211)<commit_after>package v20170831

import "encoding/json"

// The validate tag is used for validation
// Reference to gopkg.in/go-playground/validator.v9

// ResourcePurchasePlan defines resource plan as required by ARM
// for billing purposes.
type ResourcePurchasePlan struct {
	Name          string `json:"name,omitempty"`
	Product       string `json:"product,omitempty"`
	PromotionCode string `json:"promotionCode,omitempty"`
	Publisher     string `json:"publisher,omitempty"`
}

// HostedMaster complies with the ARM model of
// resource definition in a JSON template.
type HostedMaster struct {
	ID       string                `json:"id,omitempty"`
	Location string                `json:"location,omitempty" validate:"required"`
	Name     string                `json:"name,omitempty"`
	Plan     *ResourcePurchasePlan `json:"plan,omitempty"`
	Tags     map[string]string     `json:"tags,omitempty"`
	Type     string                `json:"type,omitempty"`

	Properties *Properties `json:"properties"`
}

// Properties represents the ACS cluster definition
type Properties struct {
	ProvisioningState ProvisioningState   `json:"provisioningState,omitempty"`
	Version           OrchestratorVersion `json:"version"`
	DNSPrefix         string              `json:"dnsPrefix" validate:"required"`
	// Master LB public endpoint/FQDN with port
	// The format will be FQDN:2376
	// Not used during PUT, returned as part of GET
	FQDN                    string                   `json:"fqdn,omitempty"`
	AgentPoolProfiles       []*AgentPoolProfile      `json:"agentPoolProfiles,omitempty" validate:"dive,required"`
	LinuxProfile            *LinuxProfile            `json:"linuxProfile,omitempty" validate:"required"`
	WindowsProfile          *WindowsProfile          `json:"windowsProfile,omitempty"`
	ServicePrincipalProfile *ServicePrincipalProfile `json:"servicePrincipalProfile,omitempty"`
	NetworkProfile          *NetworkProfile          `json:"networkProfile,omitempty"`
	AccessProfiles          []*AccessProfile         `json:"accessProfiles,omitempty"`
	JumpboxProfile          *JumpboxProfile          `json:"jumpboxProfile,omitempty"`
}

// ServicePrincipalProfile contains the client and secret used by the cluster for Azure Resource CRUD
// The 'Secret' parameter could be either a plain text, or referenced to a secret in a keyvault.
// In the latter case, the format of the parameter's value should be
// "/subscriptions/<SUB_ID>/resourceGroups/<RG_NAME>/providers/Microsoft.KeyVault/vaults/<KV_NAME>/secrets/<NAME>[/<VERSION>]"
// where:
//    <SUB_ID> is the subscription ID of the keyvault
//    <RG_NAME> is the resource group of the keyvault
//    <KV_NAME> is the name of the keyvault
//    <NAME> is the name of the secret.
//    <VERSION> (optional) is the version of the secret (default: the latest version)
type ServicePrincipalProfile struct {
	ClientID string `json:"clientId,omitempty" validate:"required"`
	Secret   string `json:"secret,omitempty" validate:"required"`
}

// LinuxProfile represents the Linux configuration passed to the cluster
type LinuxProfile struct {
	AdminUsername string `json:"adminUsername" validate:"required"`

	SSH struct {
		PublicKeys []PublicKey `json:"publicKeys" validate:"required,len=1"`
	} `json:"ssh" validate:"required"`
}

// PublicKey represents an SSH key for LinuxProfile
type PublicKey struct {
	KeyData string `json:"keyData"`
}

// WindowsProfile represents the Windows configuration passed to the cluster
type WindowsProfile struct {
	AdminUsername string `json:"adminUsername,omitempty" validate:"required"`
	AdminPassword string `json:"adminPassword,omitempty" validate:"required"`
}

// ProvisioningState represents the current state of container service resource.
type ProvisioningState string

const (
	// Creating means ContainerService resource is being created.
	Creating ProvisioningState = "Creating"
	// Updating means an existing ContainerService resource is being updated
	Updating ProvisioningState = "Updating"
	// Failed means resource is in failed state
	Failed ProvisioningState = "Failed"
	// Succeeded means resource created succeeded during last create/update
	Succeeded ProvisioningState = "Succeeded"
	// Deleting means resource is in the process of being deleted
	Deleting ProvisioningState = "Deleting"
	// Migrating means resource is being migrated from one subscription or
	// resource group to another
	Migrating ProvisioningState = "Migrating"
)

// NetworkProfile represents the definition of network profile for the cluster
type NetworkProfile struct {
	// PodCidr defines the range of addresses that will be used for Pods.
	// Allocation of subnets to agents will be handed by kube-controller-manager
	PodCidr string `json:"podCidr,omitempty"`
	// ServiceCidr defines the range for Kubernetes services
	ServiceCidr string `json:"serviceCidr,omitempty"`
	// AgentCidr defines the range for Agent VMs in the subnet
	AgentCidr        string `json:"agentCidr,omitempty"`
	KubeDNSServiceIP string `json:"kubeDNSServiceIP,omitempty"`
}

// JumpboxProfile represents the definition of bastion machine to access the agent nodes
type JumpboxProfile struct {
	PublicIPAddressID string `json:"publicIPAddressID,omitempty"`
	PublicIPAddress   string `json:"publicIPAddress,omitempty"`
	InternalIPAddress string `json:"internalIPAddress,omitempty"`
}

// AgentPoolProfile represents configuration of VMs running agent
// daemons that register with the master and offer resources to
// host applications in containers.
type AgentPoolProfile struct {
	Name           string `json:"name" validate:"required"`
	Count          int    `json:"count" validate:"required,min=1,max=100"`
	VMSize         string `json:"vmSize" validate:"required"`
	OSDiskSizeGB   int    `json:"osDiskSizeGB,omitempty" validate:"min=0,max=1023"`
	StorageProfile string `json:"storageProfile" validate:"eq=StorageAccount|eq=ManagedDisks|len=0"`
	VnetSubnetID   string `json:"vnetSubnetID,omitempty"`

	// OSType is the operating system type for agents
	// Set as nullable to support backward compat because
	// this property was added later.
	// If the value is null or not set, it defaulted to Linux.
	OSType OSType `json:"osType,omitempty"`

	// subnet is internal
	subnet string
}

// AccessProfile represents role name and kubeconfig
type AccessProfile struct {
	RoleName   string `json:"roleName"`
	KubeConfig string `json:"kubeConfig"`
}

// UnmarshalJSON unmarshal json using the default behavior
// And do fields manipulation, such as populating default value
func (a *AgentPoolProfile) UnmarshalJSON(b []byte) error {
	// Need to have a alias type to avoid circular unmarshal
	type aliasAgentPoolProfile AgentPoolProfile
	aa := aliasAgentPoolProfile{}
	if e := json.Unmarshal(b, &aa); e != nil {
		return e
	}
	*a = AgentPoolProfile(aa)
	if a.Count == 0 {
		// if AgentPoolProfile.Count is missing or 0, set it to default 1
		a.Count = 1
	}

	if a.StorageProfile == "" {
		// if StorageProfile is missing, set to default StorageAccount
		a.StorageProfile = StorageAccount
	}

	if string(a.OSType) == "" {
		// OSType is the operating system type for agents
		// Set as nullable to support backward compat because
		// this property was added later.
		// If the value is null or not set, it defaulted to Linux.
		a.OSType = Linux
	}

	// OSDiskSizeGB is an override value. vm sizes have default OS disk sizes.
	// If it is not set. The user should get the default for the vm size
	return nil
}

// OrchestratorVersion defines the version for orchestratorType
type OrchestratorVersion string

// OSType represents OS types of agents
type OSType string

// HasWindows returns true if the cluster contains windows
func (a *Properties) HasWindows() bool {
	for _, agentPoolProfile := range a.AgentPoolProfiles {
		if agentPoolProfile.OSType == Windows {
			return true
		}
	}
	return false
}

// IsCustomVNET returns true if the customer brought their own VNET
func (a *AgentPoolProfile) IsCustomVNET() bool {
	return len(a.VnetSubnetID) > 0
}

// IsWindows returns true if the agent pool is windows
func (a *AgentPoolProfile) IsWindows() bool {
	return a.OSType == Windows
}

// IsLinux returns true if the agent pool is linux
func (a *AgentPoolProfile) IsLinux() bool {
	return a.OSType == Linux
}

// GetSubnet returns the read-only subnet for the agent pool
func (a *AgentPoolProfile) GetSubnet() string {
	return a.subnet
}

// SetSubnet sets the read-only subnet for the agent pool
func (a *AgentPoolProfile) SetSubnet(subnet string) {
	a.subnet = subnet
}

// IsManagedDisks returns true if the customer specified managed disks
func (a *AgentPoolProfile) IsManagedDisks() bool {
	return a.StorageProfile == ManagedDisks
}

// IsStorageAccount returns true if the customer specified storage account
func (a *AgentPoolProfile) IsStorageAccount() bool {
	return a.StorageProfile == StorageAccount
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package aws

import (
	"fmt"

	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/golang/glog"
	"k8s.io/apimachinery/pkg/util/wait"
)

// TagNameKubernetesClusterPrefix is the tag name we use to differentiate multiple
// logically independent clusters running in the same AZ.
// The tag key = TagNameKubernetesClusterPrefix + clusterID
// The tag value is an ownership value
const TagNameKubernetesClusterPrefix = "kubernetes.io/cluster/"

// TagNameKubernetesClusterLegacy is the legacy tag name we use to differentiate multiple
// logically independent clusters running in the same AZ.  The problem with it was that it
// did not allow shared resources.
const TagNameKubernetesClusterLegacy = "KubernetesCluster"

type ResourceLifecycle string

const (
	// ResourceLifecycleOwned is the value we use when tagging resources to indicate
	// that the resource is considered owned and managed by the cluster,
	// and in particular that the lifecycle is tied to the lifecycle of the cluster.
	ResourceLifecycleOwned = "owned"
	// ResourceLifecycleShared is the value we use when tagging resources to indicate
	// that the resource is shared between multiple clusters, and should not be destroyed
	// if the cluster is destroyed.
	ResourceLifecycleShared = "shared"
)

type awsTagging struct {
	// ClusterID is our cluster identifier: we tag AWS resources with this value,
	// and thus we can run two independent clusters in the same VPC or subnets.
	// This gives us similar functionality to GCE projects.
	ClusterID string

	// usesLegacyTags is true if we are using the legacy TagNameKubernetesClusterLegacy tags
	usesLegacyTags bool
}

func (t *awsTagging) init(legacyClusterID string, clusterID string) error {
	if legacyClusterID != "" {
		if clusterID != "" && legacyClusterID != clusterID {
			return fmt.Errorf("ClusterID tags did not match: %q vs %q", clusterID, legacyClusterID)
		}
		t.usesLegacyTags = true
		clusterID = legacyClusterID
	}

	t.ClusterID = clusterID

	if clusterID != "" {
		glog.Infof("AWS cloud filtering on ClusterID: %v", clusterID)
	} else {
		return fmt.Errorf("AWS cloud failed to find ClusterID")
	}

	return nil
}

// Extracts a clusterID from the given tags, if one is present
// If no clusterID is found, returns "", nil
// If multiple (different) clusterIDs are found, returns an error
func (t *awsTagging) initFromTags(tags []*ec2.Tag) error {
	legacyClusterID, newClusterID, err := findClusterIDs(tags)
	if err != nil {
		return err
	}

	if legacyClusterID == "" && newClusterID == "" {
		glog.Errorf("Tag %q nor %q not found; Kubernetes may behave unexpectedly.", TagNameKubernetesClusterLegacy, TagNameKubernetesClusterPrefix+"...")
	}

	return t.init(legacyClusterID, newClusterID)
}

// Extracts the legacy & new cluster ids from the given tags, if they are present
// If duplicate tags are found, returns an error
func findClusterIDs(tags []*ec2.Tag) (string, string, error) {
	legacyClusterID := ""
	newClusterID := ""

	for _, tag := range tags {
		tagKey := aws.StringValue(tag.Key)
		if strings.HasPrefix(tagKey, TagNameKubernetesClusterPrefix) {
			id := strings.TrimPrefix(tagKey, TagNameKubernetesClusterPrefix)
			if newClusterID != "" {
				return "", "", fmt.Errorf("Found multiple cluster tags with prefix %s (%q and %q)", TagNameKubernetesClusterPrefix, newClusterID, id)
			}
			newClusterID = id
		}

		if tagKey == TagNameKubernetesClusterLegacy {
			id := aws.StringValue(tag.Value)
			if legacyClusterID != "" {
				return "", "", fmt.Errorf("Found multiple %s tags (%q and %q)", TagNameKubernetesClusterLegacy, legacyClusterID, id)
			}
			legacyClusterID = id
		}
	}

	return legacyClusterID, newClusterID, nil
}

func (t *awsTagging) clusterTagKey() string {
	return TagNameKubernetesClusterPrefix + t.ClusterID
}

func (t *awsTagging) hasClusterTag(tags []*ec2.Tag) bool {
	// if the clusterID is not configured -- we consider all instances.
	if len(t.ClusterID) == 0 {
		return true
	}
	clusterTagKey := t.clusterTagKey()
	for _, tag := range tags {
		tagKey := aws.StringValue(tag.Key)
		// For 1.6, we continue to recognize the legacy tags, for the 1.5 -> 1.6 upgrade
		if tagKey == TagNameKubernetesClusterLegacy {
			return aws.StringValue(tag.Value) == t.ClusterID
		}

		if tagKey == clusterTagKey {
			return true
		}
	}
	return false
}

// Ensure that a resource has the correct tags
// If it has no tags, we assume that this was a problem caused by an error in between creation and tagging,
// and we add the tags.  If it has a different cluster's tags, that is an error.
func (c *awsTagging) readRepairClusterTags(client EC2, resourceID string, lifecycle ResourceLifecycle, additionalTags map[string]string, observedTags []*ec2.Tag) error {
	actualTagMap := make(map[string]string)
	for _, tag := range observedTags {
		actualTagMap[aws.StringValue(tag.Key)] = aws.StringValue(tag.Value)
	}

	expectedTags := c.buildTags(lifecycle, additionalTags)

	addTags := make(map[string]string)
	for k, expected := range expectedTags {
		actual := actualTagMap[k]
		if actual == expected {
			continue
		}
		if actual == "" {
			glog.Warningf("Resource %q was missing expected cluster tag %q.  Will add (with value %q)", resourceID, k, expected)
			addTags[k] = expected
		} else {
			return fmt.Errorf("resource %q has tag belonging to another cluster: %q=%q (expected %q)", resourceID, k, actual, expected)
		}
	}

	if len(addTags) == 0 {
		return nil
	}

	if err := c.createTags(client, resourceID, lifecycle, addTags); err != nil {
		return fmt.Errorf("error adding missing tags to resource %q: %q", resourceID, err)
	}

	return nil
}

// createTags calls EC2 CreateTags, but adds retry-on-failure logic
// We retry mainly because if we create an object, we cannot tag it until it is "fully created" (eventual consistency)
// The error code varies though (depending on what we are tagging), so we simply retry on all errors
func (t *awsTagging) createTags(client EC2, resourceID string, lifecycle ResourceLifecycle, additionalTags map[string]string) error {
	tags := t.buildTags(lifecycle, additionalTags)

	if tags == nil || len(tags) == 0 {
		return nil
	}

	var awsTags []*ec2.Tag
	for k, v := range tags {
		tag := &ec2.Tag{
			Key:   aws.String(k),
			Value: aws.String(v),
		}
		awsTags = append(awsTags, tag)
	}

	backoff := wait.Backoff{
		Duration: createTagInitialDelay,
		Factor:   createTagFactor,
		Steps:    createTagSteps,
	}
	request := &ec2.CreateTagsInput{}
	request.Resources = []*string{&resourceID}
	request.Tags = awsTags

	var lastErr error
	err := wait.ExponentialBackoff(backoff, func() (bool, error) {
		_, err := client.CreateTags(request)
		if err == nil {
			return true, nil
		}

		// We could check that the error is retryable, but the error code changes based on what we are tagging
		// SecurityGroup: InvalidGroup.NotFound
		glog.V(2).Infof("Failed to create tags; will retry.  Error was %q", err)
		lastErr = err
		return false, nil
	})
	if err == wait.ErrWaitTimeout {
		// return real CreateTags error instead of timeout
		err = lastErr
	}
	return err
}

// Add additional filters, to match on our tags
// This lets us run multiple k8s clusters in a single EC2 AZ
func (t *awsTagging) addFilters(filters []*ec2.Filter) []*ec2.Filter {
	// if there are no clusterID configured - no filtering by special tag names
	// should be applied to revert to legacy behaviour.
	if len(t.ClusterID) == 0 {
		if len(filters) == 0 {
			// We can't pass a zero-length Filters to AWS (it's an error)
			// So if we end up with no filters; just return nil
			return nil
		}
		return filters
	}
	// For 1.6, we always recognize the legacy tag, for the 1.5 -> 1.6 upgrade
	// There are no "or" filters by key, so we look for both the legacy and new key, and then we have to post-filter
	f := newEc2Filter("tag-key", TagNameKubernetesClusterLegacy, t.clusterTagKey())

	// We can't pass a zero-length Filters to AWS (it's an error)
	// So if we end up with no filters; we need to return nil
	filters = append(filters, f)
	return filters
}

func (t *awsTagging) buildTags(lifecycle ResourceLifecycle, additionalTags map[string]string) map[string]string {
	tags := make(map[string]string)
	for k, v := range additionalTags {
		tags[k] = v
	}

	// no clusterID is a sign of misconfigured cluster, but we can't be tagging the resources with empty
	// strings
	if len(t.ClusterID) == 0 {
		return tags
	}

	// We only create legacy tags if we are using legacy tags, i.e. if we have seen a legacy tag on our instance
	if t.usesLegacyTags {
		tags[TagNameKubernetesClusterLegacy] = t.ClusterID
	}
	tags[t.clusterTagKey()] = string(lifecycle)

	return tags
}

func (t *awsTagging) clusterID() string {
	return t.ClusterID
}
<commit_msg>Update function hasClusterTag to fix issue #64230<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package aws

import (
	"fmt"

	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/golang/glog"
	"k8s.io/apimachinery/pkg/util/wait"
)

// TagNameKubernetesClusterPrefix is the tag name we use to differentiate multiple
// logically independent clusters running in the same AZ.
// The tag key = TagNameKubernetesClusterPrefix + clusterID
// The tag value is an ownership value
const TagNameKubernetesClusterPrefix = "kubernetes.io/cluster/"

// TagNameKubernetesClusterLegacy is the legacy tag name we use to differentiate multiple
// logically independent clusters running in the same AZ.  The problem with it was that it
// did not allow shared resources.
const TagNameKubernetesClusterLegacy = "KubernetesCluster"

type ResourceLifecycle string

const (
	// ResourceLifecycleOwned is the value we use when tagging resources to indicate
	// that the resource is considered owned and managed by the cluster,
	// and in particular that the lifecycle is tied to the lifecycle of the cluster.
	ResourceLifecycleOwned = "owned"
	// ResourceLifecycleShared is the value we use when tagging resources to indicate
	// that the resource is shared between multiple clusters, and should not be destroyed
	// if the cluster is destroyed.
	ResourceLifecycleShared = "shared"
)

type awsTagging struct {
	// ClusterID is our cluster identifier: we tag AWS resources with this value,
	// and thus we can run two independent clusters in the same VPC or subnets.
	// This gives us similar functionality to GCE projects.
	ClusterID string

	// usesLegacyTags is true if we are using the legacy TagNameKubernetesClusterLegacy tags
	usesLegacyTags bool
}

func (t *awsTagging) init(legacyClusterID string, clusterID string) error {
	if legacyClusterID != "" {
		if clusterID != "" && legacyClusterID != clusterID {
			return fmt.Errorf("ClusterID tags did not match: %q vs %q", clusterID, legacyClusterID)
		}
		t.usesLegacyTags = true
		clusterID = legacyClusterID
	}

	t.ClusterID = clusterID

	if clusterID != "" {
		glog.Infof("AWS cloud filtering on ClusterID: %v", clusterID)
	} else {
		return fmt.Errorf("AWS cloud failed to find ClusterID")
	}

	return nil
}

// Extracts a clusterID from the given tags, if one is present
// If no clusterID is found, returns "", nil
// If multiple (different) clusterIDs are found, returns an error
func (t *awsTagging) initFromTags(tags []*ec2.Tag) error {
	legacyClusterID, newClusterID, err := findClusterIDs(tags)
	if err != nil {
		return err
	}

	if legacyClusterID == "" && newClusterID == "" {
		glog.Errorf("Tag %q nor %q not found; Kubernetes may behave unexpectedly.", TagNameKubernetesClusterLegacy, TagNameKubernetesClusterPrefix+"...")
	}

	return t.init(legacyClusterID, newClusterID)
}

// Extracts the legacy & new cluster ids from the given tags, if they are present
// If duplicate tags are found, returns an error
func findClusterIDs(tags []*ec2.Tag) (string, string, error) {
	legacyClusterID := ""
	newClusterID := ""

	for _, tag := range tags {
		tagKey := aws.StringValue(tag.Key)
		if strings.HasPrefix(tagKey, TagNameKubernetesClusterPrefix) {
			id := strings.TrimPrefix(tagKey, TagNameKubernetesClusterPrefix)
			if newClusterID != "" {
				return "", "", fmt.Errorf("Found multiple cluster tags with prefix %s (%q and %q)", TagNameKubernetesClusterPrefix, newClusterID, id)
			}
			newClusterID = id
		}

		if tagKey == TagNameKubernetesClusterLegacy {
			id := aws.StringValue(tag.Value)
			if legacyClusterID != "" {
				return "", "", fmt.Errorf("Found multiple %s tags (%q and %q)", TagNameKubernetesClusterLegacy, legacyClusterID, id)
			}
			legacyClusterID = id
		}
	}

	return legacyClusterID, newClusterID, nil
}

func (t *awsTagging) clusterTagKey() string {
	return TagNameKubernetesClusterPrefix + t.ClusterID
}

func (t *awsTagging) hasClusterTag(tags []*ec2.Tag) bool {
	// if the clusterID is not configured -- we consider all instances.
	if len(t.ClusterID) == 0 {
		return true
	}
	clusterTagKey := t.clusterTagKey()
	for _, tag := range tags {
		tagKey := aws.StringValue(tag.Key)
                // Check if this is a newer-style cluster tag before checking if legacy tag value matches ClusterID
		if tagKey == clusterTagKey {
			return true
		}
		// For 1.6, we continue to recognize the legacy tags, for the 1.5 -> 1.6 upgrade
		if tagKey == TagNameKubernetesClusterLegacy {
			return aws.StringValue(tag.Value) == t.ClusterID
		}
	}
	return false
}

// Ensure that a resource has the correct tags
// If it has no tags, we assume that this was a problem caused by an error in between creation and tagging,
// and we add the tags.  If it has a different cluster's tags, that is an error.
func (c *awsTagging) readRepairClusterTags(client EC2, resourceID string, lifecycle ResourceLifecycle, additionalTags map[string]string, observedTags []*ec2.Tag) error {
	actualTagMap := make(map[string]string)
	for _, tag := range observedTags {
		actualTagMap[aws.StringValue(tag.Key)] = aws.StringValue(tag.Value)
	}

	expectedTags := c.buildTags(lifecycle, additionalTags)

	addTags := make(map[string]string)
	for k, expected := range expectedTags {
		actual := actualTagMap[k]
		if actual == expected {
			continue
		}
		if actual == "" {
			glog.Warningf("Resource %q was missing expected cluster tag %q.  Will add (with value %q)", resourceID, k, expected)
			addTags[k] = expected
		} else {
			return fmt.Errorf("resource %q has tag belonging to another cluster: %q=%q (expected %q)", resourceID, k, actual, expected)
		}
	}

	if len(addTags) == 0 {
		return nil
	}

	if err := c.createTags(client, resourceID, lifecycle, addTags); err != nil {
		return fmt.Errorf("error adding missing tags to resource %q: %q", resourceID, err)
	}

	return nil
}

// createTags calls EC2 CreateTags, but adds retry-on-failure logic
// We retry mainly because if we create an object, we cannot tag it until it is "fully created" (eventual consistency)
// The error code varies though (depending on what we are tagging), so we simply retry on all errors
func (t *awsTagging) createTags(client EC2, resourceID string, lifecycle ResourceLifecycle, additionalTags map[string]string) error {
	tags := t.buildTags(lifecycle, additionalTags)

	if tags == nil || len(tags) == 0 {
		return nil
	}

	var awsTags []*ec2.Tag
	for k, v := range tags {
		tag := &ec2.Tag{
			Key:   aws.String(k),
			Value: aws.String(v),
		}
		awsTags = append(awsTags, tag)
	}

	backoff := wait.Backoff{
		Duration: createTagInitialDelay,
		Factor:   createTagFactor,
		Steps:    createTagSteps,
	}
	request := &ec2.CreateTagsInput{}
	request.Resources = []*string{&resourceID}
	request.Tags = awsTags

	var lastErr error
	err := wait.ExponentialBackoff(backoff, func() (bool, error) {
		_, err := client.CreateTags(request)
		if err == nil {
			return true, nil
		}

		// We could check that the error is retryable, but the error code changes based on what we are tagging
		// SecurityGroup: InvalidGroup.NotFound
		glog.V(2).Infof("Failed to create tags; will retry.  Error was %q", err)
		lastErr = err
		return false, nil
	})
	if err == wait.ErrWaitTimeout {
		// return real CreateTags error instead of timeout
		err = lastErr
	}
	return err
}

// Add additional filters, to match on our tags
// This lets us run multiple k8s clusters in a single EC2 AZ
func (t *awsTagging) addFilters(filters []*ec2.Filter) []*ec2.Filter {
	// if there are no clusterID configured - no filtering by special tag names
	// should be applied to revert to legacy behaviour.
	if len(t.ClusterID) == 0 {
		if len(filters) == 0 {
			// We can't pass a zero-length Filters to AWS (it's an error)
			// So if we end up with no filters; just return nil
			return nil
		}
		return filters
	}
	// For 1.6, we always recognize the legacy tag, for the 1.5 -> 1.6 upgrade
	// There are no "or" filters by key, so we look for both the legacy and new key, and then we have to post-filter
	f := newEc2Filter("tag-key", TagNameKubernetesClusterLegacy, t.clusterTagKey())

	// We can't pass a zero-length Filters to AWS (it's an error)
	// So if we end up with no filters; we need to return nil
	filters = append(filters, f)
	return filters
}

func (t *awsTagging) buildTags(lifecycle ResourceLifecycle, additionalTags map[string]string) map[string]string {
	tags := make(map[string]string)
	for k, v := range additionalTags {
		tags[k] = v
	}

	// no clusterID is a sign of misconfigured cluster, but we can't be tagging the resources with empty
	// strings
	if len(t.ClusterID) == 0 {
		return tags
	}

	// We only create legacy tags if we are using legacy tags, i.e. if we have seen a legacy tag on our instance
	if t.usesLegacyTags {
		tags[TagNameKubernetesClusterLegacy] = t.ClusterID
	}
	tags[t.clusterTagKey()] = string(lifecycle)

	return tags
}

func (t *awsTagging) clusterID() string {
	return t.ClusterID
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package osd for the Ceph OSDs.
package osd

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/rook/rook/pkg/operator/ceph/config"
	"github.com/rook/rook/pkg/operator/k8sutil"
	"github.com/rook/rook/pkg/util"
	v1 "k8s.io/api/core/v1"
	kerrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/watch"
)

const (
	OrchestrationStatusStarting      = "starting"
	OrchestrationStatusComputingDiff = "computingDiff"
	OrchestrationStatusOrchestrating = "orchestrating"
	OrchestrationStatusCompleted     = "completed"
	OrchestrationStatusFailed        = "failed"
	orchestrationStatusMapName       = "rook-ceph-osd-%s-status"
	orchestrationStatusKey           = "status"
	provisioningLabelKey             = "provisioning"
	nodeLabelKey                     = "node"
	completeProvisionTimeout         = 20
	completeProvisionSkipOSDTimeout  = 5
)

type provisionConfig struct {
	errorMessages []string
	DataPathMap   *config.DataPathMap // location to store data in container
}

func (c *Cluster) newProvisionConfig() *provisionConfig {
	return &provisionConfig{
		DataPathMap: config.NewDatalessDaemonDataPathMap(c.Namespace, c.dataDirHostPath),
	}
}

func (c *provisionConfig) addError(message string, args ...interface{}) {
	logger.Errorf(message, args...)
	c.errorMessages = append(c.errorMessages, fmt.Sprintf(message, args...))
}

func (c *Cluster) updateOSDStatus(node string, status OrchestrationStatus) {
	UpdateNodeStatus(c.kv, node, status)
}

func UpdateNodeStatus(kv *k8sutil.ConfigMapKVStore, node string, status OrchestrationStatus) {
	labels := map[string]string{
		k8sutil.AppAttr:        AppName,
		orchestrationStatusKey: provisioningLabelKey,
		nodeLabelKey:           node,
	}

	// update the status map with the given status now
	s, _ := json.Marshal(status)
	if err := kv.SetValueWithLabels(
		k8sutil.TruncateNodeName(orchestrationStatusMapName, node),
		orchestrationStatusKey,
		string(s),
		labels,
	); err != nil {
		// log the error, but allow the orchestration to continue even if the status update failed
		logger.Errorf("failed to set node %q status to %q for osd orchestration. %s", node, status.Status, status.Message)
	}
}

func (c *Cluster) handleOrchestrationFailure(config *provisionConfig, nodeName, message string) {
	config.addError(message)
	status := OrchestrationStatus{Status: OrchestrationStatusFailed, Message: message}
	UpdateNodeStatus(c.kv, nodeName, status)
}

func isStatusCompleted(status OrchestrationStatus) bool {
	return status.Status == OrchestrationStatusCompleted || status.Status == OrchestrationStatusFailed
}

func parseOrchestrationStatus(data map[string]string) *OrchestrationStatus {
	if data == nil {
		return nil
	}

	statusRaw, ok := data[orchestrationStatusKey]
	if !ok {
		return nil
	}

	// we have status for this node, unmarshal it
	var status OrchestrationStatus
	if err := json.Unmarshal([]byte(statusRaw), &status); err != nil {
		logger.Warningf("failed to unmarshal orchestration status. status: %s. %v", statusRaw, err)
		return nil
	}

	return &status
}

func (c *Cluster) completeProvision(config *provisionConfig) bool {
	return c.completeOSDsForAllNodes(config, true, completeProvisionTimeout)
}

func (c *Cluster) completeProvisionSkipOSDStart(config *provisionConfig) bool {
	return c.completeOSDsForAllNodes(config, false, completeProvisionSkipOSDTimeout)
}

func (c *Cluster) checkNodesCompleted(selector string, config *provisionConfig, configOSDs bool) (int, *util.Set, bool, *v1.ConfigMapList, error) {
	opts := metav1.ListOptions{
		LabelSelector: selector,
		Watch:         false,
	}
	remainingNodes := util.NewSet()
	// check the status map to see if the node is already completed before we start watching
	statuses, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).List(opts)
	if err != nil {
		if !kerrors.IsNotFound(err) {
			config.addError("failed to get config status. %v", err)
			return 0, remainingNodes, false, statuses, err
		}
		// the status map doesn't exist yet, watching below is still an OK thing to do
	}

	originalNodes := len(statuses.Items)
	// check the nodes to see which ones are already completed
	for _, configMap := range statuses.Items {
		node, ok := configMap.Labels[nodeLabelKey]
		if !ok {
			logger.Warningf("missing node label on configmap %s", configMap.Name)
			continue
		}

		completed := c.handleStatusConfigMapStatus(node, config, &configMap, configOSDs)
		if !completed {
			remainingNodes.Add(node)
		}
	}
	if remainingNodes.Count() == 0 {
		return originalNodes, remainingNodes, true, statuses, nil
	}
	return originalNodes, remainingNodes, false, statuses, nil
}

func (c *Cluster) completeOSDsForAllNodes(config *provisionConfig, configOSDs bool, timeoutMinutes int) bool {
	selector := fmt.Sprintf("%s=%s,%s=%s",
		k8sutil.AppAttr, AppName,
		orchestrationStatusKey, provisioningLabelKey,
	)

	originalNodes, remainingNodes, completed, statuses, err := c.checkNodesCompleted(selector, config, configOSDs)
	if err == nil && completed {
		return true
	}

	currentTimeoutMinutes := 0
	for {
		opts := metav1.ListOptions{
			LabelSelector: selector,
			Watch:         true,
			// start watching for changes on the orchestration status map
			ResourceVersion: statuses.ResourceVersion,
		}
		logger.Infof("%d/%d node(s) completed osd provisioning, resource version %v", (originalNodes - remainingNodes.Count()), originalNodes, opts.ResourceVersion)

		w, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Watch(opts)
		if err != nil {
			logger.Warningf("failed to start watch on osd status, trying again. %v", err)
			time.Sleep(5 * time.Second)
			continue
		}
		defer w.Stop()

	ResultLoop:
		for {
			select {
			case e, ok := <-w.ResultChan():
				if !ok {
					logger.Infof("orchestration status config map result channel closed, will restart watch.")
					w.Stop()
					<-time.After(5 * time.Second)
					leftNodes, leftRemainingNodes, completed, _, err := c.checkNodesCompleted(selector, config, configOSDs)
					if err == nil {
						if completed {
							logger.Infof("additional %d/%d node(s) completed osd provisioning", leftNodes, originalNodes)
							return true
						}
						remainingNodes = leftRemainingNodes
					} else {
						logger.Warningf("failed to list orchestration configmap, status: %v", err)
					}
					break ResultLoop
				}
				if e.Type == watch.Modified {
					configMap, ok := e.Object.(*v1.ConfigMap)
					if !ok {
						logger.Errorf("expected type ConfigMap but found %T", configMap)
						continue
					}
					node, ok := configMap.Labels[nodeLabelKey]
					if !ok {
						logger.Infof("missing node label on configmap %s", configMap.Name)
						continue
					}
					if !remainingNodes.Contains(node) {
						logger.Infof("skipping event from node %s status update since it is already completed", node)
						continue
					}
					completed := c.handleStatusConfigMapStatus(node, config, configMap, configOSDs)
					if completed {
						remainingNodes.Remove(node)
						if remainingNodes.Count() == 0 {
							logger.Infof("%d/%d node(s) completed osd provisioning", originalNodes, originalNodes)
							return true
						}
					}
				}

			case <-time.After(time.Minute):
				// log every so often while we are waiting
				currentTimeoutMinutes++
				if currentTimeoutMinutes == timeoutMinutes {
					config.addError("timed out waiting for %d nodes: %+v", remainingNodes.Count(), remainingNodes)
					//start to remove remainingNodes waiting timeout.
					for remainingNode := range remainingNodes.Iter() {
						clearNodeName := k8sutil.TruncateNodeName(orchestrationStatusMapName, remainingNode)
						if err := c.kv.ClearStore(clearNodeName); err != nil {
							config.addError("failed to clear node %q status with name %q. %v", remainingNode, clearNodeName, err)
						}
					}
					return false
				}
				logger.Infof("waiting on orchestration status update from %d remaining nodes", remainingNodes.Count())
			}
		}
	}
}

func (c *Cluster) handleStatusConfigMapStatus(nodeName string, config *provisionConfig, configMap *v1.ConfigMap, configOSDs bool) bool {

	status := parseOrchestrationStatus(configMap.Data)
	if status == nil {
		return false
	}

	logger.Infof("osd orchestration status for node %s is %s", nodeName, status.Status)
	if status.Status == OrchestrationStatusCompleted {
		if configOSDs {
			if status.PvcBackedOSD {
				c.startOSDDaemonsOnPVC(nodeName, config, configMap, status)
			} else {
				c.startOSDDaemonsOnNode(nodeName, config, configMap, status)
			}
			// remove the status configmap that indicated the progress
			if err := c.kv.ClearStore(fmt.Sprintf(orchestrationStatusMapName, nodeName)); err != nil {
				logger.Errorf("failed to remove the status configmap. %v", err)
			}
		}

		return true
	}

	if status.Status == OrchestrationStatusFailed {
		config.addError("orchestration for node %s failed: %+v", nodeName, status)
		return true
	}
	return false
}
<commit_msg>ceph: remove dead osd code<commit_after>/*
Copyright 2018 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package osd for the Ceph OSDs.
package osd

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/rook/rook/pkg/operator/ceph/config"
	"github.com/rook/rook/pkg/operator/k8sutil"
	"github.com/rook/rook/pkg/util"
	v1 "k8s.io/api/core/v1"
	kerrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/watch"
)

const (
	OrchestrationStatusStarting      = "starting"
	OrchestrationStatusComputingDiff = "computingDiff"
	OrchestrationStatusOrchestrating = "orchestrating"
	OrchestrationStatusCompleted     = "completed"
	OrchestrationStatusFailed        = "failed"
	orchestrationStatusMapName       = "rook-ceph-osd-%s-status"
	orchestrationStatusKey           = "status"
	provisioningLabelKey             = "provisioning"
	nodeLabelKey                     = "node"
	completeProvisionTimeout         = 20
	completeProvisionSkipOSDTimeout  = 5
)

type provisionConfig struct {
	errorMessages []string
	DataPathMap   *config.DataPathMap // location to store data in container
}

func (c *Cluster) newProvisionConfig() *provisionConfig {
	return &provisionConfig{
		DataPathMap: config.NewDatalessDaemonDataPathMap(c.Namespace, c.dataDirHostPath),
	}
}

func (c *provisionConfig) addError(message string, args ...interface{}) {
	logger.Errorf(message, args...)
	c.errorMessages = append(c.errorMessages, fmt.Sprintf(message, args...))
}

func (c *Cluster) updateOSDStatus(node string, status OrchestrationStatus) {
	UpdateNodeStatus(c.kv, node, status)
}

func UpdateNodeStatus(kv *k8sutil.ConfigMapKVStore, node string, status OrchestrationStatus) {
	labels := map[string]string{
		k8sutil.AppAttr:        AppName,
		orchestrationStatusKey: provisioningLabelKey,
		nodeLabelKey:           node,
	}

	// update the status map with the given status now
	s, _ := json.Marshal(status)
	if err := kv.SetValueWithLabels(
		k8sutil.TruncateNodeName(orchestrationStatusMapName, node),
		orchestrationStatusKey,
		string(s),
		labels,
	); err != nil {
		// log the error, but allow the orchestration to continue even if the status update failed
		logger.Errorf("failed to set node %q status to %q for osd orchestration. %s", node, status.Status, status.Message)
	}
}

func (c *Cluster) handleOrchestrationFailure(config *provisionConfig, nodeName, message string) {
	config.addError(message)
	status := OrchestrationStatus{Status: OrchestrationStatusFailed, Message: message}
	UpdateNodeStatus(c.kv, nodeName, status)
}

func isStatusCompleted(status OrchestrationStatus) bool {
	return status.Status == OrchestrationStatusCompleted || status.Status == OrchestrationStatusFailed
}

func parseOrchestrationStatus(data map[string]string) *OrchestrationStatus {
	if data == nil {
		return nil
	}

	statusRaw, ok := data[orchestrationStatusKey]
	if !ok {
		return nil
	}

	// we have status for this node, unmarshal it
	var status OrchestrationStatus
	if err := json.Unmarshal([]byte(statusRaw), &status); err != nil {
		logger.Warningf("failed to unmarshal orchestration status. status: %s. %v", statusRaw, err)
		return nil
	}

	return &status
}

func (c *Cluster) completeProvision(config *provisionConfig) bool {
	return c.completeOSDsForAllNodes(config, true, completeProvisionTimeout)
}

func (c *Cluster) checkNodesCompleted(selector string, config *provisionConfig, configOSDs bool) (int, *util.Set, bool, *v1.ConfigMapList, error) {
	opts := metav1.ListOptions{
		LabelSelector: selector,
		Watch:         false,
	}
	remainingNodes := util.NewSet()
	// check the status map to see if the node is already completed before we start watching
	statuses, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).List(opts)
	if err != nil {
		if !kerrors.IsNotFound(err) {
			config.addError("failed to get config status. %v", err)
			return 0, remainingNodes, false, statuses, err
		}
		// the status map doesn't exist yet, watching below is still an OK thing to do
	}

	originalNodes := len(statuses.Items)
	// check the nodes to see which ones are already completed
	for _, configMap := range statuses.Items {
		node, ok := configMap.Labels[nodeLabelKey]
		if !ok {
			logger.Warningf("missing node label on configmap %s", configMap.Name)
			continue
		}

		completed := c.handleStatusConfigMapStatus(node, config, &configMap, configOSDs)
		if !completed {
			remainingNodes.Add(node)
		}
	}
	if remainingNodes.Count() == 0 {
		return originalNodes, remainingNodes, true, statuses, nil
	}
	return originalNodes, remainingNodes, false, statuses, nil
}

func (c *Cluster) completeOSDsForAllNodes(config *provisionConfig, configOSDs bool, timeoutMinutes int) bool {
	selector := fmt.Sprintf("%s=%s,%s=%s",
		k8sutil.AppAttr, AppName,
		orchestrationStatusKey, provisioningLabelKey,
	)

	originalNodes, remainingNodes, completed, statuses, err := c.checkNodesCompleted(selector, config, configOSDs)
	if err == nil && completed {
		return true
	}

	currentTimeoutMinutes := 0
	for {
		opts := metav1.ListOptions{
			LabelSelector: selector,
			Watch:         true,
			// start watching for changes on the orchestration status map
			ResourceVersion: statuses.ResourceVersion,
		}
		logger.Infof("%d/%d node(s) completed osd provisioning, resource version %v", (originalNodes - remainingNodes.Count()), originalNodes, opts.ResourceVersion)

		w, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Watch(opts)
		if err != nil {
			logger.Warningf("failed to start watch on osd status, trying again. %v", err)
			time.Sleep(5 * time.Second)
			continue
		}
		defer w.Stop()

	ResultLoop:
		for {
			select {
			case e, ok := <-w.ResultChan():
				if !ok {
					logger.Infof("orchestration status config map result channel closed, will restart watch.")
					w.Stop()
					<-time.After(5 * time.Second)
					leftNodes, leftRemainingNodes, completed, _, err := c.checkNodesCompleted(selector, config, configOSDs)
					if err == nil {
						if completed {
							logger.Infof("additional %d/%d node(s) completed osd provisioning", leftNodes, originalNodes)
							return true
						}
						remainingNodes = leftRemainingNodes
					} else {
						logger.Warningf("failed to list orchestration configmap, status: %v", err)
					}
					break ResultLoop
				}
				if e.Type == watch.Modified {
					configMap, ok := e.Object.(*v1.ConfigMap)
					if !ok {
						logger.Errorf("expected type ConfigMap but found %T", configMap)
						continue
					}
					node, ok := configMap.Labels[nodeLabelKey]
					if !ok {
						logger.Infof("missing node label on configmap %s", configMap.Name)
						continue
					}
					if !remainingNodes.Contains(node) {
						logger.Infof("skipping event from node %s status update since it is already completed", node)
						continue
					}
					completed := c.handleStatusConfigMapStatus(node, config, configMap, configOSDs)
					if completed {
						remainingNodes.Remove(node)
						if remainingNodes.Count() == 0 {
							logger.Infof("%d/%d node(s) completed osd provisioning", originalNodes, originalNodes)
							return true
						}
					}
				}

			case <-time.After(time.Minute):
				// log every so often while we are waiting
				currentTimeoutMinutes++
				if currentTimeoutMinutes == timeoutMinutes {
					config.addError("timed out waiting for %d nodes: %+v", remainingNodes.Count(), remainingNodes)
					//start to remove remainingNodes waiting timeout.
					for remainingNode := range remainingNodes.Iter() {
						clearNodeName := k8sutil.TruncateNodeName(orchestrationStatusMapName, remainingNode)
						if err := c.kv.ClearStore(clearNodeName); err != nil {
							config.addError("failed to clear node %q status with name %q. %v", remainingNode, clearNodeName, err)
						}
					}
					return false
				}
				logger.Infof("waiting on orchestration status update from %d remaining nodes", remainingNodes.Count())
			}
		}
	}
}

func (c *Cluster) handleStatusConfigMapStatus(nodeName string, config *provisionConfig, configMap *v1.ConfigMap, configOSDs bool) bool {

	status := parseOrchestrationStatus(configMap.Data)
	if status == nil {
		return false
	}

	logger.Infof("osd orchestration status for node %s is %s", nodeName, status.Status)
	if status.Status == OrchestrationStatusCompleted {
		if configOSDs {
			if status.PvcBackedOSD {
				c.startOSDDaemonsOnPVC(nodeName, config, configMap, status)
			} else {
				c.startOSDDaemonsOnNode(nodeName, config, configMap, status)
			}
			// remove the status configmap that indicated the progress
			if err := c.kv.ClearStore(fmt.Sprintf(orchestrationStatusMapName, nodeName)); err != nil {
				logger.Errorf("failed to remove the status configmap. %v", err)
			}
		}

		return true
	}

	if status.Status == OrchestrationStatusFailed {
		config.addError("orchestration for node %s failed: %+v", nodeName, status)
		return true
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright e-Xpert Solutions SA. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sys

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"

	"github.com/e-XpertSolutions/f5-rest-client/f5"
)

// FileIFileConfigList holds a list of FileIFile configuration.
type FileIFileConfigList struct {
	Items    []FileIFileConfig `json:"items"`
	Kind     string            `json:"kind"`
	SelfLink string            `json:"selflink"`
}

// FileIFileConfig holds the configuration of a single FileIFile.
type FileIFileConfig struct {
	Kind           string `json:"kind,omitempty"`
	Name           string `json:"name,omitempty"`
	Partition      string `json:"partition,omitempty"`
	FullPath       string `json:"fullPath,omitempty"`
	Generation     int    `json:"generation,omitempty"`
	SelfLink       string `json:"selfLink,omitempty"`
	Checksum       string `json:"checksum,omitempty"`
	CreateTime     string `json:"createTime,omitempty"`
	CreatedBy      string `json:"createdBy,omitempty"`
	LastUpdateTime string `json:"lastUpdateTime,omitempty"`
	Mode           int    `json:"mode,omitempty"`
	Revision       int    `json:"revision,omitempty"`
	Size           int    `json:"size,omitempty"`
	UpdatedBy      string `json:"updatedBy,omitempty"`
}

// FileIFileEndpoint represents the REST resource for managing FileIFile.
const FileIFileEndpoint = "/file/ifile"

// FileIFileResource provides an API to manage FileIFile configurations.
type FileIFileResource struct {
	c *f5.Client
}

// ListAll  lists all the FileIFile configurations.
func (r *FileIFileResource) ListAll() (*FileIFileConfigList, error) {
	var list FileIFileConfigList
	if err := r.c.ReadQuery(BasePath+FileIFileEndpoint, &list); err != nil {
		return nil, err
	}
	return &list, nil
}

// Get a single FileIFile configuration identified by id.
func (r *FileIFileResource) Get(id string) (*FileIFileConfig, error) {
	var item FileIFileConfig
	if err := r.c.ReadQuery(BasePath+FileIFileEndpoint, &item); err != nil {
		return nil, err
	}
	return &item, nil
}

// Create a new FileIFile configuration.
func (r *FileIFileResource) Create(name, path string) error {
	info, err := os.Stat(path)
	if err != nil {
		return fmt.Errorf("failed to gather information about '%s': %v", path, err)
	}
	f, err := os.Open(path)
	if err != nil {
		return fmt.Errorf("failed to read file from path: %v", err)
	}
	defer f.Close()

	req, err := r.c.MakeUploadRequest(f5.UploadRESTPath+"/"+filepath.Base(path), f, info.Size())
	if err != nil {
		return fmt.Errorf("failed to create upload request: %v", err)
	}
	resp, err := r.c.Do(req)
	if err != nil {
		return fmt.Errorf("failed to upload file '%s': %v", path, err)
	}
	defer resp.Body.Close()

	bs, _ := ioutil.ReadAll(resp.Body)
	log.Print("DEBUG resp=", string(bs))

	data := map[string]string{
		"name":        name,
		"source-path": "file://localhost/var/config/rest/downloads/" + filepath.Base(path),
	}
	if err := r.c.ModQuery("POST", BasePath+FileIFileEndpoint, data); err != nil {
		return fmt.Errorf("failed to create ifile configuration: %v", err)
	}

	return nil
}

// Edit a FileIFile configuration identified by id.
func (r *FileIFileResource) Edit(id, path string) error {
	info, err := os.Stat(path)
	if err != nil {
		return fmt.Errorf("failed to gather information about '%s': %v", path, err)
	}
	f, err := os.Open(path)
	if err != nil {
		return fmt.Errorf("failed to read file from path: %v", err)
	}
	defer f.Close()

	req, err := r.c.MakeUploadRequest(f5.UploadRESTPath+"/"+filepath.Base(path), f, info.Size())
	if err != nil {
		return fmt.Errorf("failed to create upload request: %v", err)
	}
	resp, err := r.c.Do(req)
	if err != nil {
		return fmt.Errorf("failed to upload file '%s': %v", path, err)
	}
	defer resp.Body.Close()

	bs, _ := ioutil.ReadAll(resp.Body)
	log.Print("DEBUG resp=", string(bs))

	data := map[string]string{
		"source-path": "file://localhost/var/config/rest/downloads/" + filepath.Base(path),
	}
	if err := r.c.ModQuery("PUT", BasePath+FileIFileEndpoint+"/"+id, data); err != nil {
		return fmt.Errorf("failed to create ifile configuration: %v", err)
	}

	return nil
}

// Delete a single FileIFile configuration identified by id.
func (r *FileIFileResource) Delete(id string) error {
	if err := r.c.ModQuery("DELETE", BasePath+FileIFileEndpoint+"/"+id, nil); err != nil {
		return err
	}
	return nil
}
<commit_msg>f5/sys: fix functions to upload a new iFile<commit_after>// Copyright e-Xpert Solutions SA. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sys

import (
	"fmt"
	"io"

	"github.com/e-XpertSolutions/f5-rest-client/f5"
)

// FileIFileConfigList holds a list of FileIFile configuration.
type FileIFileConfigList struct {
	Items    []FileIFileConfig `json:"items"`
	Kind     string            `json:"kind"`
	SelfLink string            `json:"selflink"`
}

// FileIFileConfig holds the configuration of a single FileIFile.
type FileIFileConfig struct {
	Kind           string `json:"kind,omitempty"`
	Name           string `json:"name,omitempty"`
	Partition      string `json:"partition,omitempty"`
	FullPath       string `json:"fullPath,omitempty"`
	Generation     int    `json:"generation,omitempty"`
	SelfLink       string `json:"selfLink,omitempty"`
	Checksum       string `json:"checksum,omitempty"`
	CreateTime     string `json:"createTime,omitempty"`
	CreatedBy      string `json:"createdBy,omitempty"`
	LastUpdateTime string `json:"lastUpdateTime,omitempty"`
	Mode           int    `json:"mode,omitempty"`
	Revision       int    `json:"revision,omitempty"`
	Size           int    `json:"size,omitempty"`
	UpdatedBy      string `json:"updatedBy,omitempty"`
}

// FileIFileEndpoint represents the REST resource for managing FileIFile.
const FileIFileEndpoint = "/file/ifile"

// FileIFileResource provides an API to manage FileIFile configurations.
type FileIFileResource struct {
	c *f5.Client
}

// ListAll  lists all the FileIFile configurations.
func (r *FileIFileResource) ListAll() (*FileIFileConfigList, error) {
	var list FileIFileConfigList
	if err := r.c.ReadQuery(BasePath+FileIFileEndpoint, &list); err != nil {
		return nil, err
	}
	return &list, nil
}

// Get a single FileIFile configuration identified by id.
func (r *FileIFileResource) Get(id string) (*FileIFileConfig, error) {
	var item FileIFileConfig
	if err := r.c.ReadQuery(BasePath+FileIFileEndpoint, &item); err != nil {
		return nil, err
	}
	return &item, nil
}

// CreateFromFile uploads an iFile and create a new entry in the
// iFiles list.
func (r *FileIFileResource) CreateFromFile(filename string, file io.Reader, filesize int64) error {
	uploadResp, err := r.c.UploadFile(file, filename, filesize)
	if err != nil {
		return fmt.Errorf("failed to create upload request: %v", err)
	}
	data := map[string]string{
		"name":        filename,
		"source-path": "file:" + uploadResp.LocalFilePath,
	}
	if err := r.c.ModQuery("POST", BasePath+FileIFileEndpoint, data); err != nil {
		return fmt.Errorf("failed to import ifile: %v", err)
	}
	return nil
}

// EditFromFile uploads an iFile and update an existing entry in the
// iFiles list.
func (r *FileIFileResource) EditFromFile(filename string, file io.Reader, filesize int64) error {
	uploadResp, err := r.c.UploadFile(file, filename, filesize)
	if err != nil {
		return fmt.Errorf("failed to create upload request: %v", err)
	}
	data := map[string]string{
		"name":        filename,
		"source-path": "file:" + uploadResp.LocalFilePath,
	}
	if err := r.c.ModQuery("PUT", BasePath+FileIFileEndpoint+"/"+filename, data); err != nil {
		return fmt.Errorf("failed to update ifile: %v", err)
	}
	return nil
}

// Delete a single FileIFile configuration identified by id.
func (r *FileIFileResource) Delete(id string) error {
	if err := r.c.ModQuery("DELETE", BasePath+FileIFileEndpoint+"/"+id, nil); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package filehost

import (
	"context"
	"html/template"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"mime"

	"github.com/pressly/chi"
	"github.com/sirupsen/logrus"
)

var log logrus.FieldLogger = logrus.StandardLogger()
var cfg *Config
var templ *template.Template

// Database is some sort of database that stores FileInfo
type Database interface {
	// GetFileInfo returns the FileInfo associated with the given
	// File Name, it returns nil if the file was not found
	GetFileInfo(string) *FileInfo
	SaveFileInfo(*FileInfo)
}

// Config contains the needed information to call New
type Config struct {
	AllowedMimeTypes []string
	BasePath         string
	BaseURL          string
	UploadPage       bool
	ExposedPassword  string
	NewFileName      func() string
	DB               Database
	Authenticate     func(*http.Request) bool
	AllowFileName    func(*http.Request) bool
	Logger           logrus.FieldLogger
}

// FileInfo contains additional information about the File
type FileInfo struct {
	Name     string // filename without extension
	Path     string
	MimeType string
	Uploader interface{} // information about the person who uploaded it
	Time     time.Time
	Expire   time.Duration
	Clicks   int
}

// New initializes a http Handler and returns it
func New(conf *Config) http.Handler {
	go ratelimit()
	cfg = conf
	log = cfg.Logger
	err := os.MkdirAll(cfg.BasePath, 644)
	if err != nil {
		log.Fatal(err)
	}
	r := chi.NewRouter()
	r.Use(func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			r = r.WithContext(context.WithValue(r.Context(), "logger",
				log.WithFields(logrus.Fields{
					"ip":         r.RemoteAddr,
					"user-agent": r.UserAgent(),
				})))
			next.ServeHTTP(w, r)
		})
	})
	r.Post("/upload", upload)
	r.Get("/:file", serveFile)
	if cfg.UploadPage {
		templ = template.Must(template.ParseFiles("upload.html"))
		r.Get("/", uploadPage)
	} else {
		r.Get("/", http.NotFound)
	}

	return r
}

func uploadPage(w http.ResponseWriter, r *http.Request) {
	data := struct {
		Key     string
		BaseURL string
	}{
		Key:     cfg.ExposedPassword,
		BaseURL: cfg.BaseURL,
	}
	if err := templ.Execute(w, data); err != nil {
		r.Context().Value("logger").(logrus.FieldLogger).
			WithError(err).Error("cannot execute template")
	}
}

func upload(w http.ResponseWriter, r *http.Request) {
	l := r.Context().Value("logger").(logrus.FieldLogger)
	defer r.Body.Close()
	if !cfg.Authenticate(r) {
		http.Error(w, "Not Authenticated", http.StatusUnauthorized)
		return
	}
	if ratelimited(r.RemoteAddr) {
		l.Warning("ratelimited")
		http.Error(w, "Rate Limit Exceeded", 429)
		return
	}
	l.Debug(r.Header)
	err := r.ParseMultipartForm(1024 * 1024 * 64)
	if err != nil {
		l.WithError(err).Error("cannot read multi part form")
		http.Error(w, "Bad Request", http.StatusBadRequest)
		return
	}
	if r.MultipartForm == nil {
		http.Error(w, "Bad Request", http.StatusBadRequest)
		return
	}
	files := r.MultipartForm.File
	l.Debug(files)
	if len(files) < 1 {
		http.Error(w, "Bad Request", http.StatusBadRequest)
		return
	}
	for _, headers := range files {
		for _, h := range headers {
			file, err := h.Open()
			if err != nil {
				l.Error(err)
			}
			name := h.Filename
			if !cfg.AllowFileName(r) || name == "" {
				if cfg.NewFileName != nil {
					name = cfg.NewFileName()
				} else {
					name = RandString(5)
				}
			}
			l = l.WithField("file", name)
			l.Info("uploading...")
			l.Debug(r.Header)
			l.Debug(h.Header)
			mimeType := h.Header.Get("Content-Type")
			if !whiteListed(cfg.AllowedMimeTypes, mimeType) {
				spl := strings.Split(h.Filename, ".")
				if len(spl) > 1 {
					ext := spl[len(spl)-1]
					mimeType = mime.TypeByExtension("." + ext)
					l.WithField("mime-type", mimeType).Debug("type from ext")
				}
			}
			if mimeType == octetStream || mimeType == "" {
				mimeType = "text/plain"
			}
			l = l.WithField("mime-type", mimeType)
			if !whiteListed(cfg.AllowedMimeTypes, mimeType) {
				l.Warning("mime type not allowed")
				http.Error(w, "Unsupported Media Type", 415)
				return
			}
			extension := ExtensionFromMime(mimeType)
			if extension != "" {
				extension = "." + extension
			}

			fullName := name + extension
			dstPath := filepath.Join(cfg.BasePath, fullName)
			// TODO: check if file exists
			dst, err := os.Create(dstPath)
			if err != nil {
				l.WithError(err).Error("cannot create file")
				http.Error(w, "Internal Server Error", 500)
				return
			}
			_, err = io.Copy(dst, file)
			if err != nil {
				l.WithError(err).Error("failed to save file")
				http.Error(w, "Internal Server Error", 500)
				return
			}
			w.Write([]byte(cfg.BaseURL + fullName))
			l.Info("uploaded to ", cfg.BaseURL+fullName)
			if cfg.DB != nil {
				info := &FileInfo{
					Name: name,
					Path: dstPath,
					Uploader: map[string]interface{}{
						"ip":         r.RemoteAddr,
						"user-agent": r.UserAgent(),
					},
					Time:     time.Now(),
					MimeType: mimeType,
				}
				cfg.DB.SaveFileInfo(info)
			}
		}
	}
}

const octetStream = "application/octet-stream"

func serveFile(w http.ResponseWriter, r *http.Request) {
	l := r.Context().Value("logger").(logrus.FieldLogger)
	name := chi.URLParam(r, "file")
	l = l.WithField("file", name)
	if ratelimited(r.RemoteAddr) {
		l.Warning("ratelimited")
		http.Error(w, "Rate Limit Exceeded", 429)
		return
	}
	file, err := os.Open(filepath.Join(cfg.BasePath, name))
	if err != nil {
		l.WithError(err).Warning("not found")
		http.Error(w, "404 Not Found", 404)
		return
	}
	defer file.Close()
	rateAdd(r.RemoteAddr)
	spl := strings.Split(name, ".")
	id := spl[0]
	extension := ""
	if len(spl) > 1 {
		extension = spl[len(spl)-1]
	}
	mimeType := ""
	if cfg.DB != nil {
		info := cfg.DB.GetFileInfo(id)
		if info != nil {
			if info.Expire != 0 {
				if time.Since(time.Now().Add(info.Expire)) > info.Expire {
					l.Info("expired")
					http.Error(w, "404 Not Found", 404)
					return
				}
			}
			mimeType = info.MimeType
			info.Clicks++
		}
	}

	if mimeType == "" {
		mimeType = mime.TypeByExtension("." + extension)
		if mimeType == "" {
			sniffData := make([]byte, 512)
			n, err := file.Read(sniffData)
			if err != nil {
				l.WithError(err).Error("cannot read from file")
				http.Error(w, "Internal Server Error", 500)
				return
			}
			sniffData = sniffData[:n]
			mimeType = http.DetectContentType(sniffData)
			_, err = file.Seek(0, 0)
			if err != nil {
				l.WithError(err).Error("cannot seek file")
				http.Error(w, "Internal Server Error", 500)
				return
			}
		}
	}
	if mimeType == octetStream {
		switch extension {
		case "mp3":
			mimeType = "audio/mpeg"
		case "wav":
			mimeType = "audio/wav"
		default:
			mimeType = "text/plain"
		}
	}
	if dl, _ := strconv.ParseBool(r.URL.Query().Get("download")); dl {
		mimeType = octetStream
	}
	w.Header().Set("X-Content-Type-Options", "nosniff")
	w.Header().Set("Content-Type", mimeType)

	http.ServeContent(w, r, "", time.Time{}, file)
}
<commit_msg>better safe than sorry :thinking:<commit_after>package filehost

import (
	"context"
	"html/template"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"mime"

	"github.com/pressly/chi"
	"github.com/sirupsen/logrus"
)

var log logrus.FieldLogger = logrus.StandardLogger()
var cfg *Config
var templ *template.Template

// Database is some sort of database that stores FileInfo
type Database interface {
	// GetFileInfo returns the FileInfo associated with the given
	// File Name, it returns nil if the file was not found
	GetFileInfo(string) *FileInfo
	SaveFileInfo(*FileInfo)
}

// Config contains the needed information to call New
type Config struct {
	AllowedMimeTypes []string
	BasePath         string
	BaseURL          string
	UploadPage       bool
	ExposedPassword  string
	NewFileName      func() string
	DB               Database
	Authenticate     func(*http.Request) bool
	AllowFileName    func(*http.Request) bool
	Logger           logrus.FieldLogger
}

// FileInfo contains additional information about the File
type FileInfo struct {
	Name     string // filename without extension
	Path     string
	MimeType string
	Uploader interface{} // information about the person who uploaded it
	Time     time.Time
	Expire   time.Duration
	Clicks   int
}

// New initializes a http Handler and returns it
func New(conf *Config) http.Handler {
	go ratelimit()
	cfg = conf
	log = cfg.Logger
	err := os.MkdirAll(cfg.BasePath, 644)
	if err != nil {
		log.Fatal(err)
	}
	r := chi.NewRouter()
	r.Use(func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			r = r.WithContext(context.WithValue(r.Context(), "logger",
				log.WithFields(logrus.Fields{
					"ip":         r.RemoteAddr,
					"user-agent": r.UserAgent(),
				})))
			next.ServeHTTP(w, r)
		})
	})
	r.Post("/upload", upload)
	r.Get("/:file", serveFile)
	if cfg.UploadPage {
		templ = template.Must(template.ParseFiles("upload.html"))
		r.Get("/", uploadPage)
	} else {
		r.Get("/", http.NotFound)
	}

	return r
}

func uploadPage(w http.ResponseWriter, r *http.Request) {
	data := struct {
		Key     string
		BaseURL string
	}{
		Key:     cfg.ExposedPassword,
		BaseURL: cfg.BaseURL,
	}
	if err := templ.Execute(w, data); err != nil {
		r.Context().Value("logger").(logrus.FieldLogger).
			WithError(err).Error("cannot execute template")
	}
}

func upload(w http.ResponseWriter, r *http.Request) {
	l := r.Context().Value("logger").(logrus.FieldLogger)
	defer r.Body.Close()
	if !cfg.Authenticate(r) {
		http.Error(w, "Not Authenticated", http.StatusUnauthorized)
		return
	}
	if ratelimited(r.RemoteAddr) {
		l.Warning("ratelimited")
		http.Error(w, "Rate Limit Exceeded", 429)
		return
	}
	l.Debug(r.Header)
	err := r.ParseMultipartForm(1024 * 1024 * 64)
	if err != nil {
		l.WithError(err).Error("cannot read multi part form")
		http.Error(w, "Bad Request", http.StatusBadRequest)
		return
	}
	if r.MultipartForm == nil {
		http.Error(w, "Bad Request", http.StatusBadRequest)
		return
	}
	files := r.MultipartForm.File
	l.Debug(files)
	if len(files) < 1 {
		http.Error(w, "Bad Request", http.StatusBadRequest)
		return
	}
	for _, headers := range files {
		for _, h := range headers {
			file, err := h.Open()
			if err != nil {
				l.Error(err)
			}
			name := h.Filename
			if !cfg.AllowFileName(r) || name == "" {
				if cfg.NewFileName != nil {
					name = cfg.NewFileName()
				} else {
					name = RandString(5)
				}
			}
			l = l.WithField("file", name)
			l.Info("uploading...")
			l.Debug(r.Header)
			l.Debug(h.Header)
			mimeType := h.Header.Get("Content-Type")
			if !whiteListed(cfg.AllowedMimeTypes, mimeType) {
				spl := strings.Split(h.Filename, ".")
				if len(spl) > 1 {
					ext := spl[len(spl)-1]
					mimeType = mime.TypeByExtension("." + ext)
					l.WithField("mime-type", mimeType).Debug("type from ext")
				}
			}
			if mimeType == octetStream || mimeType == "" {
				mimeType = "text/plain"
			}
			l = l.WithField("mime-type", mimeType)
			if !whiteListed(cfg.AllowedMimeTypes, mimeType) {
				l.Warning("mime type not allowed")
				http.Error(w, "Unsupported Media Type", 415)
				return
			}
			extension := ExtensionFromMime(mimeType)
			if extension != "" {
				extension = "." + extension
			}

			fullName := name + extension
			dstPath := filepath.Join(cfg.BasePath, fullName)
			// TODO: check if file exists
			dst, err := os.Create(dstPath)
			if err != nil {
				l.WithError(err).Error("cannot create file")
				http.Error(w, "Internal Server Error", 500)
				return
			}
			_, err = io.Copy(dst, file)
			if err != nil {
				l.WithError(err).Error("failed to save file")
				http.Error(w, "Internal Server Error", 500)
				return
			}
			w.Write([]byte(cfg.BaseURL + fullName))
			l.Info("uploaded to ", cfg.BaseURL+fullName)
			if cfg.DB != nil {
				info := &FileInfo{
					Name: name,
					Path: dstPath,
					Uploader: map[string]interface{}{
						"ip":         r.RemoteAddr,
						"user-agent": r.UserAgent(),
					},
					Time:     time.Now(),
					MimeType: mimeType,
				}
				cfg.DB.SaveFileInfo(info)
			}
		}
	}
}

const octetStream = "application/octet-stream"

func serveFile(w http.ResponseWriter, r *http.Request) {
	l := r.Context().Value("logger").(logrus.FieldLogger)
	name := chi.URLParam(r, "file")
	name = filepath.Base(name)
	l = l.WithField("file", name)
	if ratelimited(r.RemoteAddr) {
		l.Warning("ratelimited")
		http.Error(w, "Rate Limit Exceeded", 429)
		return
	}
	file, err := os.Open(filepath.Join(cfg.BasePath, name))
	if err != nil {
		l.WithError(err).Warning("not found")
		http.Error(w, "404 Not Found", 404)
		return
	}
	defer file.Close()
	rateAdd(r.RemoteAddr)
	spl := strings.Split(name, ".")
	id := spl[0]
	extension := ""
	if len(spl) > 1 {
		extension = spl[len(spl)-1]
	}
	mimeType := ""
	if cfg.DB != nil {
		info := cfg.DB.GetFileInfo(id)
		if info != nil {
			if info.Expire != 0 {
				if time.Since(time.Now().Add(info.Expire)) > info.Expire {
					l.Info("expired")
					http.Error(w, "404 Not Found", 404)
					return
				}
			}
			mimeType = info.MimeType
			info.Clicks++
		}
	}

	if mimeType == "" {
		mimeType = mime.TypeByExtension("." + extension)
		if mimeType == "" {
			sniffData := make([]byte, 512)
			n, err := file.Read(sniffData)
			if err != nil {
				l.WithError(err).Error("cannot read from file")
				http.Error(w, "Internal Server Error", 500)
				return
			}
			sniffData = sniffData[:n]
			mimeType = http.DetectContentType(sniffData)
			_, err = file.Seek(0, 0)
			if err != nil {
				l.WithError(err).Error("cannot seek file")
				http.Error(w, "Internal Server Error", 500)
				return
			}
		}
	}
	if mimeType == octetStream {
		switch extension {
		case "mp3":
			mimeType = "audio/mpeg"
		case "wav":
			mimeType = "audio/wav"
		default:
			mimeType = "text/plain"
		}
	}
	if dl, _ := strconv.ParseBool(r.URL.Query().Get("download")); dl {
		mimeType = octetStream
	}
	w.Header().Set("X-Content-Type-Options", "nosniff")
	w.Header().Set("Content-Type", mimeType)

	http.ServeContent(w, r, "", time.Time{}, file)
}
<|endoftext|>