<commit_before>package snmpgo

var StripHexPrefix = stripHexPrefix
var ToHexStr = toHexStr
var Retry = retry
var NewNotInTimeWindowError = func() error { return &notInTimeWindowError{&MessageError{}} }

// For snmpgo testing
var NewSNMPEngine = newSNMPEngine

func ArgsValidate(args *SNMPArguments) error { return args.validate() }
func CheckPdu(engine *snmpEngine, pdu Pdu, args *SNMPArguments) error {
	return engine.checkPdu(pdu, args)
}

// For message testing
var NewMessage = newMessage
var UnmarshalMessage = unmarshalMessage
var NewMessageWithPdu = newMessageWithPdu
var NewMessageProcessing = newMessageProcessing

func ToMessageV1(msg message) *messageV1 { return msg.(*messageV1) }
func ToMessageV3(msg message) *messageV3 { return msg.(*messageV3) }
func ToUsm(sec security) *usm            { return sec.(*usm) }

// For security testing
var NewSecurity = newSecurity
var PasswordToKey = passwordToKey
var EncryptDES = encryptDES
var EncryptAES = encryptAES
var DecryptDES = decryptDES
var DecryptAES = decryptAES
var NewSecurityMap = newSecurityMap

func NewCommunity() *community { return &community{} }
func NewUsm() *usm             { return &usm{} }

// For server
func ListeningUDPAddress(s *TrapServer) string {
	return s.transport.(*packetTransport).conn.LocalAddr().String()
}
<commit_msg>avoid a test fail in go 1.6<commit_after>package snmpgo

import (
	"time"
)

var StripHexPrefix = stripHexPrefix
var ToHexStr = toHexStr
var Retry = retry
var NewNotInTimeWindowError = func() error { return &notInTimeWindowError{&MessageError{}} }

// For snmpgo testing
var NewSNMPEngine = newSNMPEngine

func ArgsValidate(args *SNMPArguments) error { return args.validate() }
func CheckPdu(engine *snmpEngine, pdu Pdu, args *SNMPArguments) error {
	return engine.checkPdu(pdu, args)
}

// For message testing
var NewMessage = newMessage
var UnmarshalMessage = unmarshalMessage
var NewMessageWithPdu = newMessageWithPdu
var NewMessageProcessing = newMessageProcessing

func ToMessageV1(msg message) *messageV1 { return msg.(*messageV1) }
func ToMessageV3(msg message) *messageV3 { return msg.(*messageV3) }
func ToUsm(sec security) *usm            { return sec.(*usm) }

// For security testing
var NewSecurity = newSecurity
var PasswordToKey = passwordToKey
var EncryptDES = encryptDES
var EncryptAES = encryptAES
var DecryptDES = decryptDES
var DecryptAES = decryptAES
var NewSecurityMap = newSecurityMap

func NewCommunity() *community { return &community{} }
func NewUsm() *usm             { return &usm{} }

// For server
func ListeningUDPAddress(s *TrapServer) string {
	for i := 0; i < 12; i++ {
		if conn := s.transport.(*packetTransport).conn; conn != nil {
			return conn.LocalAddr().String()
		}
		// XXX Wait until a connection is available, but this code is a kludge
		time.Sleep(time.Millisecond * time.Duration(1<<uint(i)))
	}
	return ""
}
<|endoftext|>


<commit_before>// Package crawler provides helper methods and defines an interface for lauching
// source repository crawlers that retrieve files from a source and forwards
// to a channel for indexing and retrieval.
package crawler

import (
	"context"
	"fmt"
	"log"
	"os"
	"sync"

	"sigs.k8s.io/kustomize/api/internal/crawl/index"

	_ "github.com/gomodule/redigo/redis"

	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
)

var (
	logger = log.New(os.Stdout, "Crawler: ", log.LstdFlags|log.LUTC|log.Llongfile)
)

// Crawler forwards documents from source repositories to index and store them
// for searching. Each crawler is responsible for querying it's source of
// information, and forwarding files that have not been seen before or that need
// updating.
type Crawler interface {
	// Crawl returns when it is done processing. This method does not take
	// ownership of the channel. The channel is write only, and it
	// designates where the crawler should forward the documents.
	Crawl(ctx context.Context, output chan<- CrawledDocument) error

	// Get the document data given the FilePath, Repo, and Ref/Tag/Branch.
	FetchDocument(context.Context, *doc.Document) error
	// Write to the document what the created time is.
	SetCreated(context.Context, *doc.Document) error

	Match(*doc.Document) bool
}

type CrawledDocument interface {
	ID() string
	GetDocument() *doc.Document
	// Get all the Documents directly referred in a Document.
	GetResources() ([]*doc.Document, error)
	WasCached() bool
}

type CrawlSeed []*doc.Document

type IndexFunc func(CrawledDocument, index.Mode) error
type Converter func(*doc.Document) (CrawledDocument, error)

func logIfErr(err error) {
	if err == nil {
		return
	}
	logger.Println("error: ", err)
}

func findMatch(d *doc.Document, crawlers []Crawler) Crawler {
	for _, crawl := range crawlers {
		if crawl.Match(d) {
			return crawl
		}
	}
	return nil
}

func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc,
	seen map[string]struct{}, stack *CrawlSeed) {

	seen[cdoc.ID()] = struct{}{}

	// Insert into index
	if err := indx(cdoc, index.InsertOrUpdate); err != nil {
		logger.Printf("Failed to insert or update %s %s: %v",
			cdoc.GetDocument().RepositoryURL, cdoc.GetDocument().FilePath, err)
		return
	}

	deps, err := cdoc.GetResources()
	if err != nil {
		logger.Println(err)
		return
	}

	for _, dep := range deps {
		if _, ok := seen[dep.ID()]; ok {
			continue
		}
		*stack = append(*stack, dep)
	}
}

func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv Converter, indx IndexFunc,
	seen map[string]struct{}, stack *CrawlSeed) {

	UpdatedDocCount := 0
	seenDocCount := 0
	cachedDocCount := 0
	findMatchErrCount := 0
	FetchDocumentErrCount := 0
	SetCreatedErrCount := 0
	convErrCount := 0
	deleteDocCount := 0

	// During the execution of the for loop, more Documents may be added into (*docsPtr).
	for len(*docsPtr) > 0 {
		// get the last Document in (*docPtr), which will be crawled in this iteration.
		tail := (*docsPtr)[len(*docsPtr)-1]

		// remove the last Document in (*docPtr)
		*docsPtr = (*docsPtr)[:(len(*docsPtr) - 1)]

		if _, ok := seen[tail.ID()]; ok {
			seenDocCount++
			continue
		}

		if tail.WasCached() {
			logger.Printf("%s %s is cached already", tail.RepositoryURL, tail.FilePath)
			cachedDocCount++
			continue
		}

		match := findMatch(tail, crawlers)
		if match == nil {
			logIfErr(fmt.Errorf("%v could not match any crawler", tail))
			findMatchErrCount++
			continue
		}

		logger.Println("Crawling ", tail.RepositoryURL, tail.FilePath)
		if err := match.FetchDocument(ctx, tail); err != nil {
			logger.Printf("FetchDocument failed on %s %s: %v",
				tail.RepositoryURL, tail.FilePath, err)
			FetchDocumentErrCount++
			// delete the document from the index
			cdoc := &doc.KustomizationDocument{
				Document: *tail,
			}
			seen[cdoc.ID()] = struct{}{}
			if err := indx(cdoc, index.Delete); err != nil {
				logger.Printf("Failed to delete %s %s: %v",
					cdoc.RepositoryURL, cdoc.FilePath, err)
			}
			deleteDocCount++
			continue
		}

		if err := match.SetCreated(ctx, tail); err != nil {
			logger.Printf("SetCreated failed on %s %s: %v",
				tail.RepositoryURL, tail.FilePath, err)
			SetCreatedErrCount++
		}

		cdoc, err := conv(tail)
		// If conv returns an error, cdoc can still be added into the index so that
		// cdoc.Document can be searched.
		if err != nil {
			logger.Printf("conv failed on %s %s: %v",
				tail.RepositoryURL, tail.FilePath, err)
			convErrCount++
		}

		UpdatedDocCount++
		addBranches(cdoc, match, indx, seen, stack)
	}
	logger.Printf("Summary of doCrawl:\n")
	logger.Printf("\t%d documents were updated\n", UpdatedDocCount)
	logger.Printf("\t%d documents were seen by the crawler already and skipped\n", seenDocCount)
	logger.Printf("\t%d documents were cached already and skipped\n", cachedDocCount)
	logger.Printf("\t%d documents didn't have a matching crawler and skipped\n", findMatchErrCount)
	logger.Printf("\t%d documents cannot be fetched, %d out of them are deleted\n",
		FetchDocumentErrCount, deleteDocCount)
	logger.Printf("\t%d documents cannot update its creation time but still were inserted or updated in the index\n", SetCreatedErrCount)
	logger.Printf("\t%d documents cannot be converted but still were inserted or updated in the index\n", convErrCount)
}

// CrawlFromSeed updates all the documents in seed, and crawls all the new
// documents referred in the seed.
func CrawlFromSeed(ctx context.Context, seed CrawlSeed, crawlers []Crawler,
	conv Converter, indx IndexFunc, seen map[string]struct{}) {

	// stack tracks the documents directly referred in other documents.
	stack := make(CrawlSeed, 0)

	// Exploit seed to update bulk of corpus.
	logger.Printf("updating %d documents from seed\n", len(seed))
	// each unique document in seed will be crawled once.
	doCrawl(ctx, &seed, crawlers, conv, indx, seen, &stack)

	// Traverse any new documents added while updating corpus.
	logger.Printf("crawling %d new documents found in the seed\n", len(stack))
	// While crawling each document in stack, the documents directly referred in the document
	// will be added into stack.
	// After this statement is done, stack will become empty.
	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack)
}

// CrawlGithubRunner is a blocking function and only returns once all of the
// crawlers are finished with execution.
//
// This function uses the output channel to forward kustomization documents
// from a list of crawlers. The output is to be consumed by a database/search
// indexer for later retrieval.
//
// The return value is an array of errors in which each index represents the
// index of the crawler that emitted the error. Although the errors themselves
// can be nil, the array will always be exactly the size of the crawlers array.
//
// CrawlGithubRunner takes in a seed, which represents the documents stored in an
// index somewhere. The document data is not required to be populated. If there
// are many documents, this is preferable. The order of iteration over the seed
// is not guaranteed, but the CrawlGithub does guarantee that every element
// from the seed will be processed before any other documents from the
// crawlers.
func CrawlGithubRunner(ctx context.Context, output chan<- CrawledDocument,
	crawlers []Crawler) []error {

	errs := make([]error, len(crawlers))
	wg := sync.WaitGroup{}

	for i, crawler := range crawlers {
		// Crawler implementations get their own channels to prevent a
		// crawler from closing the main output channel.
		docs := make(chan CrawledDocument)
		wg.Add(2)

		// Forward all of the documents from this crawler's channel to
		// the main output channel.
		go func(docs <-chan CrawledDocument) {
			defer wg.Done()
			for d := range docs {
				output <- d
			}
		}(docs)

		// Run this crawler and capture its returned error.
		go func(idx int, crawler Crawler,
			docs chan<- CrawledDocument) {

			defer func() {
				wg.Done()
				if r := recover(); r != nil {
					errs[idx] = fmt.Errorf(
						"%+v panicked: %v, additional error %v",
						crawler, r, errs[idx],
					)
				}
			}()
			defer close(docs)
			errs[idx] = crawler.Crawl(ctx, docs)
		}(i, crawler, docs) // Copies the index and the crawler
	}

	wg.Wait()
	return errs
}

// CrawlGithub crawls all the kustomization files on Github.
func CrawlGithub(ctx context.Context, crawlers []Crawler, conv Converter,
	indx IndexFunc, seen map[string]struct{}) {
	// stack tracks the documents directly referred in other documents.
	stack := make(CrawlSeed, 0)

	// ch is channel where all the crawlers sends the crawled documents to.
	ch := make(chan CrawledDocument, 1<<10)

	wg := sync.WaitGroup{}

	wg.Add(1)
	go func() {
		defer wg.Done()
		for cdoc := range ch {
			if _, ok := seen[cdoc.ID()]; ok {
				continue
			}
			match := findMatch(cdoc.GetDocument(), crawlers)
			if match == nil {
				logIfErr(fmt.Errorf(
					"%v could not match any crawler", cdoc))
				continue
			}
			addBranches(cdoc, match, indx, seen, &stack)
		}
	}()

	logger.Println("processing the documents found from crawling github")
	if errs := CrawlGithubRunner(ctx, ch, crawlers); errs != nil {
		for _, err := range errs {
			logIfErr(err)
		}
	}
	close(ch)
	wg.Wait()

	// Handle deps of newly discovered documents.
	logger.Printf("crawling the %d new documents referred by other documents",
		len(stack))
	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack)
}
<commit_msg>Track the crawling process<commit_after>// Package crawler provides helper methods and defines an interface for lauching
// source repository crawlers that retrieve files from a source and forwards
// to a channel for indexing and retrieval.
package crawler

import (
	"context"
	"fmt"
	"log"
	"os"
	"sync"

	"sigs.k8s.io/kustomize/api/internal/crawl/index"

	_ "github.com/gomodule/redigo/redis"

	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
)

var (
	logger = log.New(os.Stdout, "Crawler: ", log.LstdFlags|log.LUTC|log.Llongfile)
)

// Crawler forwards documents from source repositories to index and store them
// for searching. Each crawler is responsible for querying it's source of
// information, and forwarding files that have not been seen before or that need
// updating.
type Crawler interface {
	// Crawl returns when it is done processing. This method does not take
	// ownership of the channel. The channel is write only, and it
	// designates where the crawler should forward the documents.
	Crawl(ctx context.Context, output chan<- CrawledDocument) error

	// Get the document data given the FilePath, Repo, and Ref/Tag/Branch.
	FetchDocument(context.Context, *doc.Document) error
	// Write to the document what the created time is.
	SetCreated(context.Context, *doc.Document) error

	Match(*doc.Document) bool
}

type CrawledDocument interface {
	ID() string
	GetDocument() *doc.Document
	// Get all the Documents directly referred in a Document.
	GetResources() ([]*doc.Document, error)
	WasCached() bool
}

type CrawlSeed []*doc.Document

type IndexFunc func(CrawledDocument, index.Mode) error
type Converter func(*doc.Document) (CrawledDocument, error)

func logIfErr(err error) {
	if err == nil {
		return
	}
	logger.Println("error: ", err)
}

func findMatch(d *doc.Document, crawlers []Crawler) Crawler {
	for _, crawl := range crawlers {
		if crawl.Match(d) {
			return crawl
		}
	}
	return nil
}

func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc,
	seen map[string]struct{}, stack *CrawlSeed) {

	seen[cdoc.ID()] = struct{}{}

	// Insert into index
	if err := indx(cdoc, index.InsertOrUpdate); err != nil {
		logger.Printf("Failed to insert or update %s %s: %v",
			cdoc.GetDocument().RepositoryURL, cdoc.GetDocument().FilePath, err)
		return
	}

	deps, err := cdoc.GetResources()
	if err != nil {
		logger.Println(err)
		return
	}

	for _, dep := range deps {
		if _, ok := seen[dep.ID()]; ok {
			continue
		}
		*stack = append(*stack, dep)
	}
}

func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv Converter, indx IndexFunc,
	seen map[string]struct{}, stack *CrawlSeed) {

	UpdatedDocCount := 0
	seenDocCount := 0
	cachedDocCount := 0
	findMatchErrCount := 0
	FetchDocumentErrCount := 0
	SetCreatedErrCount := 0
	convErrCount := 0
	deleteDocCount := 0
	crawledDocCount := 0

	// During the execution of the for loop, more Documents may be added into (*docsPtr).
	for len(*docsPtr) > 0 {
		// get the last Document in (*docPtr), which will be crawled in this iteration.
		tail := (*docsPtr)[len(*docsPtr)-1]

		// remove the last Document in (*docPtr)
		*docsPtr = (*docsPtr)[:(len(*docsPtr) - 1)]

		crawledDocCount++
		logger.Printf("Crawling doc %d: %s %s", crawledDocCount, tail.RepositoryURL, tail.FilePath)

		if _, ok := seen[tail.ID()]; ok {
			logger.Printf("this doc has been seen before")
			seenDocCount++
			continue
		}

		if tail.WasCached() {
			logger.Printf("%s %s is cached already", tail.RepositoryURL, tail.FilePath)
			cachedDocCount++
			continue
		}

		match := findMatch(tail, crawlers)
		if match == nil {
			logIfErr(fmt.Errorf("%v could not match any crawler", tail))
			findMatchErrCount++
			continue
		}

		if err := match.FetchDocument(ctx, tail); err != nil {
			logger.Printf("FetchDocument failed on %s %s: %v",
				tail.RepositoryURL, tail.FilePath, err)
			FetchDocumentErrCount++
			// delete the document from the index
			cdoc := &doc.KustomizationDocument{
				Document: *tail,
			}
			seen[cdoc.ID()] = struct{}{}
			if err := indx(cdoc, index.Delete); err != nil {
				logger.Printf("Failed to delete %s %s: %v",
					cdoc.RepositoryURL, cdoc.FilePath, err)
			}
			deleteDocCount++
			continue
		}

		if err := match.SetCreated(ctx, tail); err != nil {
			logger.Printf("SetCreated failed on %s %s: %v",
				tail.RepositoryURL, tail.FilePath, err)
			SetCreatedErrCount++
		}

		cdoc, err := conv(tail)
		// If conv returns an error, cdoc can still be added into the index so that
		// cdoc.Document can be searched.
		if err != nil {
			logger.Printf("conv failed on %s %s: %v",
				tail.RepositoryURL, tail.FilePath, err)
			convErrCount++
		}

		UpdatedDocCount++
		addBranches(cdoc, match, indx, seen, stack)
	}
	logger.Printf("Summary of doCrawl:\n")
	logger.Printf("\t%d documents were updated\n", UpdatedDocCount)
	logger.Printf("\t%d documents were seen by the crawler already and skipped\n", seenDocCount)
	logger.Printf("\t%d documents were cached already and skipped\n", cachedDocCount)
	logger.Printf("\t%d documents didn't have a matching crawler and skipped\n", findMatchErrCount)
	logger.Printf("\t%d documents cannot be fetched, %d out of them are deleted\n",
		FetchDocumentErrCount, deleteDocCount)
	logger.Printf("\t%d documents cannot update its creation time but still were inserted or updated in the index\n", SetCreatedErrCount)
	logger.Printf("\t%d documents cannot be converted but still were inserted or updated in the index\n", convErrCount)
}

// CrawlFromSeed updates all the documents in seed, and crawls all the new
// documents referred in the seed.
func CrawlFromSeed(ctx context.Context, seed CrawlSeed, crawlers []Crawler,
	conv Converter, indx IndexFunc, seen map[string]struct{}) {

	// stack tracks the documents directly referred in other documents.
	stack := make(CrawlSeed, 0)

	// Exploit seed to update bulk of corpus.
	logger.Printf("updating %d documents from seed\n", len(seed))
	// each unique document in seed will be crawled once.
	doCrawl(ctx, &seed, crawlers, conv, indx, seen, &stack)

	// Traverse any new documents added while updating corpus.
	logger.Printf("crawling %d new documents found in the seed\n", len(stack))
	// While crawling each document in stack, the documents directly referred in the document
	// will be added into stack.
	// After this statement is done, stack will become empty.
	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack)
}

// CrawlGithubRunner is a blocking function and only returns once all of the
// crawlers are finished with execution.
//
// This function uses the output channel to forward kustomization documents
// from a list of crawlers. The output is to be consumed by a database/search
// indexer for later retrieval.
//
// The return value is an array of errors in which each index represents the
// index of the crawler that emitted the error. Although the errors themselves
// can be nil, the array will always be exactly the size of the crawlers array.
//
// CrawlGithubRunner takes in a seed, which represents the documents stored in an
// index somewhere. The document data is not required to be populated. If there
// are many documents, this is preferable. The order of iteration over the seed
// is not guaranteed, but the CrawlGithub does guarantee that every element
// from the seed will be processed before any other documents from the
// crawlers.
func CrawlGithubRunner(ctx context.Context, output chan<- CrawledDocument,
	crawlers []Crawler) []error {

	errs := make([]error, len(crawlers))
	wg := sync.WaitGroup{}

	for i, crawler := range crawlers {
		// Crawler implementations get their own channels to prevent a
		// crawler from closing the main output channel.
		docs := make(chan CrawledDocument)
		wg.Add(2)

		// Forward all of the documents from this crawler's channel to
		// the main output channel.
		go func(docs <-chan CrawledDocument) {
			defer wg.Done()
			for d := range docs {
				output <- d
			}
		}(docs)

		// Run this crawler and capture its returned error.
		go func(idx int, crawler Crawler,
			docs chan<- CrawledDocument) {

			defer func() {
				wg.Done()
				if r := recover(); r != nil {
					errs[idx] = fmt.Errorf(
						"%+v panicked: %v, additional error %v",
						crawler, r, errs[idx],
					)
				}
			}()
			defer close(docs)
			errs[idx] = crawler.Crawl(ctx, docs)
		}(i, crawler, docs) // Copies the index and the crawler
	}

	wg.Wait()
	return errs
}

// CrawlGithub crawls all the kustomization files on Github.
func CrawlGithub(ctx context.Context, crawlers []Crawler, conv Converter,
	indx IndexFunc, seen map[string]struct{}) {
	// stack tracks the documents directly referred in other documents.
	stack := make(CrawlSeed, 0)

	// ch is channel where all the crawlers sends the crawled documents to.
	ch := make(chan CrawledDocument, 1<<10)

	wg := sync.WaitGroup{}

	wg.Add(1)
	go func() {
		defer wg.Done()
		docCount := 0
		for cdoc := range ch {
			docCount++
			logger.Printf("Processing doc %d found on Github", docCount)
			if _, ok := seen[cdoc.ID()]; ok {
				logger.Printf("the doc has been seen before")
				continue
			}
			match := findMatch(cdoc.GetDocument(), crawlers)
			if match == nil {
				logIfErr(fmt.Errorf(
					"%v could not match any crawler", cdoc))
				continue
			}
			addBranches(cdoc, match, indx, seen, &stack)
		}
	}()

	logger.Println("processing the documents found from crawling github")
	if errs := CrawlGithubRunner(ctx, ch, crawlers); errs != nil {
		for _, err := range errs {
			logIfErr(err)
		}
	}
	close(ch)
	wg.Wait()

	// Handle deps of newly discovered documents.
	logger.Printf("crawling the %d new documents referred by other documents",
		len(stack))
	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack)
}
<|endoftext|>


<commit_before>/*
Copyright 2022 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package service

import (
	"context"
	"testing"

	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	clientset "k8s.io/client-go/kubernetes"
	restclient "k8s.io/client-go/rest"
	"k8s.io/kubernetes/test/integration/framework"
)

// Test_ExternalNameServiceStopsDefaultingInternalTrafficPolicy tests that Services no longer default
// the internalTrafficPolicy field when Type is ExternalName. This test exists due to historic reasons where
// the internalTrafficPolicy field was being defaulted in older versions. New versions stop defauting the
// field and drop on read, but for compatibility reasons we still accept the field.
func Test_ExternalNameServiceStopsDefaultingInternalTrafficPolicy(t *testing.T) {
	controlPlaneConfig := framework.NewIntegrationTestControlPlaneConfig()
	_, server, closeFn := framework.RunAnAPIServer(controlPlaneConfig)
	defer closeFn()

	config := restclient.Config{Host: server.URL}
	client, err := clientset.NewForConfig(&config)
	if err != nil {
		t.Fatalf("Error creating clientset: %v", err)
	}

	ns := framework.CreateTestingNamespace("test-external-name-drops-internal-traffic-policy", server, t)
	defer framework.DeleteTestingNamespace(ns, server, t)

	service := &corev1.Service{
		ObjectMeta: metav1.ObjectMeta{
			Name: "test-123",
		},
		Spec: corev1.ServiceSpec{
			Type:         corev1.ServiceTypeExternalName,
			ExternalName: "foo.bar.com",
		},
	}

	service, err = client.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{})
	if err != nil {
		t.Fatalf("Error creating test service: %v", err)
	}

	if service.Spec.InternalTrafficPolicy != nil {
		t.Errorf("service internalTrafficPolicy should be droppped but is set: %v", service.Spec.InternalTrafficPolicy)
	}

	service, err = client.CoreV1().Services(ns.Name).Get(context.TODO(), service.Name, metav1.GetOptions{})
	if err != nil {
		t.Fatalf("error getting service: %v", err)
	}

	if service.Spec.InternalTrafficPolicy != nil {
		t.Errorf("service internalTrafficPolicy should be droppped but is set: %v", service.Spec.InternalTrafficPolicy)
	}
}
<commit_msg>add integration test to validate that ExternalName Services can set internalTrafficPolicy, but the field is dropped on read<commit_after>/*
Copyright 2022 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package service

import (
	"context"
	"testing"

	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	clientset "k8s.io/client-go/kubernetes"
	restclient "k8s.io/client-go/rest"
	"k8s.io/kubernetes/test/integration/framework"
)

// Test_ExternalNameServiceStopsDefaultingInternalTrafficPolicy tests that Services no longer default
// the internalTrafficPolicy field when Type is ExternalName. This test exists due to historic reasons where
// the internalTrafficPolicy field was being defaulted in older versions. New versions stop defauting the
// field and drop on read, but for compatibility reasons we still accept the field.
func Test_ExternalNameServiceStopsDefaultingInternalTrafficPolicy(t *testing.T) {
	controlPlaneConfig := framework.NewIntegrationTestControlPlaneConfig()
	_, server, closeFn := framework.RunAnAPIServer(controlPlaneConfig)
	defer closeFn()

	config := restclient.Config{Host: server.URL}
	client, err := clientset.NewForConfig(&config)
	if err != nil {
		t.Fatalf("Error creating clientset: %v", err)
	}

	ns := framework.CreateTestingNamespace("test-external-name-drops-internal-traffic-policy", server, t)
	defer framework.DeleteTestingNamespace(ns, server, t)

	service := &corev1.Service{
		ObjectMeta: metav1.ObjectMeta{
			Name: "test-123",
		},
		Spec: corev1.ServiceSpec{
			Type:         corev1.ServiceTypeExternalName,
			ExternalName: "foo.bar.com",
		},
	}

	service, err = client.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{})
	if err != nil {
		t.Fatalf("Error creating test service: %v", err)
	}

	if service.Spec.InternalTrafficPolicy != nil {
		t.Errorf("service internalTrafficPolicy should be droppped but is set: %v", service.Spec.InternalTrafficPolicy)
	}

	service, err = client.CoreV1().Services(ns.Name).Get(context.TODO(), service.Name, metav1.GetOptions{})
	if err != nil {
		t.Fatalf("error getting service: %v", err)
	}

	if service.Spec.InternalTrafficPolicy != nil {
		t.Errorf("service internalTrafficPolicy should be droppped but is set: %v", service.Spec.InternalTrafficPolicy)
	}
}

// Test_ExternalNameServiceDropsInternalTrafficPolicy tests that Services accepts the internalTrafficPolicy field on Create,
// but drops the field on read. This test exists due to historic reasons where the internalTrafficPolicy field was being defaulted
// in older versions. New versions stop defauting the field and drop on read, but for compatibility reasons we still accept the field.
func Test_ExternalNameServiceDropsInternalTrafficPolicy(t *testing.T) {
	controlPlaneConfig := framework.NewIntegrationTestControlPlaneConfig()
	_, server, closeFn := framework.RunAnAPIServer(controlPlaneConfig)
	defer closeFn()

	config := restclient.Config{Host: server.URL}
	client, err := clientset.NewForConfig(&config)
	if err != nil {
		t.Fatalf("Error creating clientset: %v", err)
	}

	ns := framework.CreateTestingNamespace("test-external-name-drops-internal-traffic-policy", server, t)
	defer framework.DeleteTestingNamespace(ns, server, t)

	internalTrafficPolicy := corev1.ServiceInternalTrafficPolicyCluster
	service := &corev1.Service{
		ObjectMeta: metav1.ObjectMeta{
			Name: "test-123",
		},
		Spec: corev1.ServiceSpec{
			Type:                  corev1.ServiceTypeExternalName,
			ExternalName:          "foo.bar.com",
			InternalTrafficPolicy: &internalTrafficPolicy,
		},
	}

	service, err = client.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{})
	if err != nil {
		t.Fatalf("Error creating test service: %v", err)
	}

	if service.Spec.InternalTrafficPolicy != nil {
		t.Errorf("service internalTrafficPolicy should be droppped but is set: %v", service.Spec.InternalTrafficPolicy)
	}

	service, err = client.CoreV1().Services(ns.Name).Get(context.TODO(), service.Name, metav1.GetOptions{})
	if err != nil {
		t.Fatalf("error getting service: %v", err)
	}

	if service.Spec.InternalTrafficPolicy != nil {
		t.Errorf("service internalTrafficPolicy should be droppped but is set: %v", service.Spec.InternalTrafficPolicy)
	}
}
<|endoftext|>


<commit_before>package admin

import (
	"errors"

	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/app"
	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/app/args"
	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/app/with"
	"github.com/BytemarkHosting/bytemark-client/lib"
	"github.com/BytemarkHosting/bytemark-client/lib/billing"
	"github.com/BytemarkHosting/bytemark-client/lib/brain"
	billingMethods "github.com/BytemarkHosting/bytemark-client/lib/requests/billing"
	"github.com/BytemarkHosting/bytemark-client/util/log"
	"github.com/urfave/cli"
)

func init() {
	readUpdateFlags := func(c *app.Context) (usageStrategy *string, overcommitRatio *int, label *string) {
		if c.Context.IsSet("usage-strategy") {
			v := c.String("usage-strategy")
			usageStrategy = &v
		}

		if c.Context.IsSet("overcommit-ratio") {
			v := c.Int("overcommit-ratio")
			overcommitRatio = &v
		}

		if c.Context.IsSet("label") {
			v := c.String("label")
			label = &v
		}

		return
	}

	Commands = append(Commands, cli.Command{
		Name:   "update",
		Action: cli.ShowSubcommandHelp,
		Subcommands: []cli.Command{
			{
				Name:      "billing-definition",
				Usage:     "update a bmbilling definition",
				UsageText: "bytemark --admin update billing-definition [flags] [name] [value]",
				Flags: []cli.Flag{
					cli.StringFlag{
						Name:  "name",
						Usage: "the name of the definition to set",
					},
					cli.StringFlag{
						Name:  "value",
						Usage: "the value of the definition to set",
					},
					cli.StringFlag{
						Name:  "group",
						Usage: "the group a user must be in to update the definition",
					},
				},
				Action: app.Action(args.Optional("name", "value"), with.RequiredFlags("name", "value"), with.Auth, func(ctx *app.Context) error {
					def := billing.Definition{
						Name:           ctx.String("name"),
						Value:          ctx.String("value"),
						UpdateGroupReq: ctx.String("group"),
					}
					if _, err := billingMethods.GetDefinition(ctx.Client(), def.Name); err != nil {
						if _, ok := err.(lib.NotFoundError); ok {
							ctx.LogErr("Couldn't find a definition called %s - aborting.", def.Name)
							return nil
						}
						return err
					}
					err := billingMethods.UpdateDefinition(ctx.Client(), def)
					if err == nil {
						ctx.LogErr("Updated %s to %s", def.Name, def.Value)
					}
					return err

				}),
			}, {
				Name:      "head",
				Usage:     "update the settings of a head",
				UsageText: "bytemark --admin update head <head> [--usage-strategy] [--overcommit-ratio] [--label]",
				Flags: []cli.Flag{
					cli.StringFlag{
						Name:  "head",
						Usage: "the ID or label of the head to be updated",
					},
					cli.StringFlag{
						Name:  "usage-strategy",
						Usage: "the usage strategy of the head",
					},
					cli.IntFlag{
						Name:  "overcommit-ratio",
						Usage: "the overcommit ratio of the head",
					},
					cli.StringFlag{
						Name:  "label",
						Usage: "the label of the head",
					},
				},
				Action: app.Action(args.Optional("head", "usage-strategy", "overcommit-ratio", "label"), with.RequiredFlags("head"), with.Auth, func(c *app.Context) error {
					usageStrategy, overcommitRatio, label := readUpdateFlags(c)

					options := lib.UpdateHead{
						UsageStrategy:   usageStrategy,
						OvercommitRatio: overcommitRatio,
						Label:           label,
					}

					if err := c.Client().UpdateHead(c.String("head"), options); err != nil {
						return err
					}

					log.Outputf("Head %s updated\n", c.String("head"))

					return nil
				}),
			},
			{
				Name:      "tail",
				Usage:     "update the settings of a tail",
				UsageText: "bytemark --admin update tail <tail> [--usage-strategy] [--overcommit-ratio] [--label]",
				Flags: []cli.Flag{
					cli.StringFlag{
						Name:  "tail",
						Usage: "the ID or label of the tail to be updated",
					},
					cli.StringFlag{
						Name:  "usage-strategy",
						Usage: "the usage strategy of the tail",
					},
					cli.IntFlag{
						Name:  "overcommit-ratio",
						Usage: "the overcommit ratio of the tail",
					},
					cli.StringFlag{
						Name:  "label",
						Usage: "the label of the tail",
					},
				},
				Action: app.Action(args.Optional("tail", "usage-strategy", "overcommit-ratio", "label"), with.RequiredFlags("tail"), with.Auth, func(c *app.Context) error {
					usageStrategy, overcommitRatio, label := readUpdateFlags(c)

					options := lib.UpdateTail{
						UsageStrategy:   usageStrategy,
						OvercommitRatio: overcommitRatio,
						Label:           label,
					}

					if err := c.Client().UpdateTail(c.String("tail"), options); err != nil {
						return err
					}

					log.Outputf("Tail %s updated\n", c.String("tail"))

					return nil
				}),
			},
			{
				Name:      "storage pool",
				Usage:     "update the settings of a storage pool",
				UsageText: "bytemark --admin update storage pool [--usage-strategy new-strategy] [--overcommit-ratio new-ratio] [--label new-label] [--migration-concurrency new-limit] <storage pool>",
				Flags: []cli.Flag{
					cli.StringFlag{
						Name:  "storage-pool",
						Usage: "the ID or label of the storage pool to be updated",
					},
					cli.StringFlag{
						Name:  "usage-strategy",
						Usage: "the usage strategy of the storage pool",
					},
					cli.IntFlag{
						Name:  "overcommit-ratio",
						Usage: "the overcommit ratio of the storage pool",
					},
					cli.StringFlag{
						Name:  "label",
						Usage: "the label of the storage pool",
					},
					cli.IntFlag{
						Name:  "migration-concurrency",
						Usage: "the number of concurrent migrations the storage pool can handle",
					},
				},
				Action: app.Action(args.Optional("storage-pool", "usage-strategy", "overcommit-ratio"), with.RequiredFlags("storage-pool"), with.Auth, func(c *app.Context) error {

					options := brain.StoragePool{
						UsageStrategy:        c.String("usage-strategy"),
						OvercommitRatio:      c.Int("overcommit-ratio"),
						Label:                c.String("label"),
						MigrationConcurrency: c.Int("migration-concurrency"),
					}

					if err := c.Client().UpdateStoragePool(c.String("storage-pool"), options); err != nil {
						return err
					}

					log.Outputf("Storage pool %s updated\n", c.String("storage-pool"))

					return nil
				}),
			},
			{
				Name:    "server",
				Aliases: []string{"vm"},
				Action:  cli.ShowSubcommandHelp,
				Subcommands: []cli.Command{
					{
						Name:        "migration",
						Usage:       "update the settings of an in-progress migration",
						UsageText:   "bytemark --admin update server migration <name> [--migrate-speed] [--migrate-downtime]",
						Description: `This command migrates a server to a new head. If a new head isn't supplied, a new one is picked automatically.`,
						Flags: []cli.Flag{
							cli.GenericFlag{
								Name:  "server",
								Usage: "the server to migrate",
								Value: new(app.VirtualMachineNameFlag),
							},
							cli.Int64Flag{
								Name:  "migrate-speed",
								Usage: "the max speed to migrate the server at",
							},
							cli.IntFlag{
								Name:  "migrate-downtime",
								Usage: "the max allowed downtime",
							},
						},
						Action: app.Action(args.Optional("server", "migrate-speed", "migrate-downtime"), with.RequiredFlags("server"), with.Auth, func(c *app.Context) error {
							vm := c.VirtualMachineName("server")

							var speed *int64
							var downtime *int

							if c.Context.IsSet("migrate-speed") {
								s := c.Int64("migrate-speed")
								speed = &s
							}
							if c.Context.IsSet("migrate-downtime") {
								d := c.Int("migrate-downtime")
								downtime = &d
							}

							if speed == nil && downtime == nil {
								return errors.New("Nothing to update")
							}

							if err := c.Client().UpdateVMMigration(vm, speed, downtime); err != nil {
								return err
							}

							log.Outputf("Migration for server %s updated\n", vm.String())

							return nil
						}),
					},
				},
			},
		},
	})
}
<commit_msg>added command line commands for updating a migrating job<commit_after>package admin

import (
	"errors"
	"fmt"

	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/app"
	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/app/args"
	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/app/with"
	"github.com/BytemarkHosting/bytemark-client/lib"
	"github.com/BytemarkHosting/bytemark-client/lib/billing"
	"github.com/BytemarkHosting/bytemark-client/lib/brain"
	billingMethods "github.com/BytemarkHosting/bytemark-client/lib/requests/billing"
	brainMethods "github.com/BytemarkHosting/bytemark-client/lib/requests/brain"
	"github.com/BytemarkHosting/bytemark-client/util/log"
	"github.com/urfave/cli"
)

func init() {
	readUpdateFlags := func(c *app.Context) (usageStrategy *string, overcommitRatio *int, label *string) {
		if c.Context.IsSet("usage-strategy") {
			v := c.String("usage-strategy")
			usageStrategy = &v
		}

		if c.Context.IsSet("overcommit-ratio") {
			v := c.Int("overcommit-ratio")
			overcommitRatio = &v
		}

		if c.Context.IsSet("label") {
			v := c.String("label")
			label = &v
		}

		return
	}

	Commands = append(Commands, cli.Command{
		Name:   "update",
		Action: cli.ShowSubcommandHelp,
		Subcommands: []cli.Command{
			{
				Name:      "billing-definition",
				Usage:     "update a bmbilling definition",
				UsageText: "bytemark --admin update billing-definition [flags] [name] [value]",
				Flags: []cli.Flag{
					cli.StringFlag{
						Name:  "name",
						Usage: "the name of the definition to set",
					},
					cli.StringFlag{
						Name:  "value",
						Usage: "the value of the definition to set",
					},
					cli.StringFlag{
						Name:  "group",
						Usage: "the group a user must be in to update the definition",
					},
				},
				Action: app.Action(args.Optional("name", "value"), with.RequiredFlags("name", "value"), with.Auth, func(ctx *app.Context) error {
					def := billing.Definition{
						Name:           ctx.String("name"),
						Value:          ctx.String("value"),
						UpdateGroupReq: ctx.String("group"),
					}
					if _, err := billingMethods.GetDefinition(ctx.Client(), def.Name); err != nil {
						if _, ok := err.(lib.NotFoundError); ok {
							ctx.LogErr("Couldn't find a definition called %s - aborting.", def.Name)
							return nil
						}
						return err
					}
					err := billingMethods.UpdateDefinition(ctx.Client(), def)
					if err == nil {
						ctx.LogErr("Updated %s to %s", def.Name, def.Value)
					}
					return err

				}),
			}, {
				Name:      "head",
				Usage:     "update the settings of a head",
				UsageText: "bytemark --admin update head <head> [--usage-strategy] [--overcommit-ratio] [--label]",
				Flags: []cli.Flag{
					cli.StringFlag{
						Name:  "head",
						Usage: "the ID or label of the head to be updated",
					},
					cli.StringFlag{
						Name:  "usage-strategy",
						Usage: "the usage strategy of the head",
					},
					cli.IntFlag{
						Name:  "overcommit-ratio",
						Usage: "the overcommit ratio of the head",
					},
					cli.StringFlag{
						Name:  "label",
						Usage: "the label of the head",
					},
				},
				Action: app.Action(args.Optional("head", "usage-strategy", "overcommit-ratio", "label"), with.RequiredFlags("head"), with.Auth, func(c *app.Context) error {
					usageStrategy, overcommitRatio, label := readUpdateFlags(c)

					options := lib.UpdateHead{
						UsageStrategy:   usageStrategy,
						OvercommitRatio: overcommitRatio,
						Label:           label,
					}

					if err := c.Client().UpdateHead(c.String("head"), options); err != nil {
						return err
					}

					log.Outputf("Head %s updated\n", c.String("head"))

					return nil
				}),
			},
			{
				Name:      "tail",
				Usage:     "update the settings of a tail",
				UsageText: "bytemark --admin update tail <tail> [--usage-strategy] [--overcommit-ratio] [--label]",
				Flags: []cli.Flag{
					cli.StringFlag{
						Name:  "tail",
						Usage: "the ID or label of the tail to be updated",
					},
					cli.StringFlag{
						Name:  "usage-strategy",
						Usage: "the usage strategy of the tail",
					},
					cli.IntFlag{
						Name:  "overcommit-ratio",
						Usage: "the overcommit ratio of the tail",
					},
					cli.StringFlag{
						Name:  "label",
						Usage: "the label of the tail",
					},
				},
				Action: app.Action(args.Optional("tail", "usage-strategy", "overcommit-ratio", "label"), with.RequiredFlags("tail"), with.Auth, func(c *app.Context) error {
					usageStrategy, overcommitRatio, label := readUpdateFlags(c)

					options := lib.UpdateTail{
						UsageStrategy:   usageStrategy,
						OvercommitRatio: overcommitRatio,
						Label:           label,
					}

					if err := c.Client().UpdateTail(c.String("tail"), options); err != nil {
						return err
					}

					log.Outputf("Tail %s updated\n", c.String("tail"))

					return nil
				}),
			},
			{
				Name:      "storage pool",
				Usage:     "update the settings of a storage pool",
				UsageText: "bytemark --admin update storage pool [--usage-strategy new-strategy] [--overcommit-ratio new-ratio] [--label new-label] [--migration-concurrency new-limit] <storage pool>",
				Flags: []cli.Flag{
					cli.StringFlag{
						Name:  "storage-pool",
						Usage: "the ID or label of the storage pool to be updated",
					},
					cli.StringFlag{
						Name:  "usage-strategy",
						Usage: "the usage strategy of the storage pool",
					},
					cli.IntFlag{
						Name:  "overcommit-ratio",
						Usage: "the overcommit ratio of the storage pool",
					},
					cli.StringFlag{
						Name:  "label",
						Usage: "the label of the storage pool",
					},
					cli.IntFlag{
						Name:  "migration-concurrency",
						Usage: "the number of concurrent migrations the storage pool can handle",
					},
				},
				Action: app.Action(args.Optional("storage-pool", "usage-strategy", "overcommit-ratio"), with.RequiredFlags("storage-pool"), with.Auth, func(c *app.Context) error {

					options := brain.StoragePool{
						UsageStrategy:        c.String("usage-strategy"),
						OvercommitRatio:      c.Int("overcommit-ratio"),
						Label:                c.String("label"),
						MigrationConcurrency: c.Int("migration-concurrency"),
					}

					if err := c.Client().UpdateStoragePool(c.String("storage-pool"), options); err != nil {
						return err
					}

					log.Outputf("Storage pool %s updated\n", c.String("storage-pool"))

					return nil
				}),
			},
			{
				Name:    "server",
				Aliases: []string{"vm"},
				Action:  cli.ShowSubcommandHelp,
				Subcommands: []cli.Command{
					{
						Name:        "migration",
						Usage:       "update the settings of an in-progress migration",
						UsageText:   "bytemark --admin update server migration <name> [--migrate-speed] [--migrate-downtime]",
						Description: `This command migrates a server to a new head. If a new head isn't supplied, a new one is picked automatically.`,
						Flags: []cli.Flag{
							cli.GenericFlag{
								Name:  "server",
								Usage: "the server to migrate",
								Value: new(app.VirtualMachineNameFlag),
							},
							cli.Int64Flag{
								Name:  "migrate-speed",
								Usage: "the max speed to migrate the server at",
							},
							cli.IntFlag{
								Name:  "migrate-downtime",
								Usage: "the max allowed downtime",
							},
						},
						Action: app.Action(args.Optional("server", "migrate-speed", "migrate-downtime"), with.RequiredFlags("server"), with.Auth, func(c *app.Context) error {
							vm := c.VirtualMachineName("server")

							var speed *int64
							var downtime *int

							if c.Context.IsSet("migrate-speed") {
								s := c.Int64("migrate-speed")
								speed = &s
							}
							if c.Context.IsSet("migrate-downtime") {
								d := c.Int("migrate-downtime")
								downtime = &d
							}

							if speed == nil && downtime == nil {
								return errors.New("Nothing to update")
							}

							if err := c.Client().UpdateVMMigration(vm, speed, downtime); err != nil {
								return err
							}

							log.Outputf("Migration for server %s updated\n", vm.String())

							return nil
						}),
					},
				},
			},
			{
				Name:        "migration",
				Usage:       "update a migration",
				UsageText:   "bbytemark --admin update migration --id 123 --priority 10 --cancel-disc disc1 --cancel-pool pool2 --cancel-tail tail3 | --cancel-all",
				Description: `This command allows you to update an ongoing migration job by altering its priority, cancelling migrating discs, pools, tails, or canceling everything for the current job`,
				Flags: []cli.Flag{
					cli.IntFlag{
						Name:  "id",
						Usage: "the id of the migration job",
					},
					cli.IntFlag{
						Name:  "priority",
						Usage: "the priority of the current job",
					},
					cli.StringSliceFlag{
						Name:  "cancel-disc",
						Usage: "the disc(s) to cancel migration of",
					},
					cli.StringSliceFlag{
						Name:  "cancel-pool",
						Usage: "the pool(s) to cancel migration of",
					},
					cli.StringSliceFlag{
						Name:  "cancel-tail",
						Usage: "the tail(s) to cancel migration of",
					},
					cli.BoolFlag{
						Name:  "cancel-all",
						Usage: "cancel the all migrations of the job",
					},
				},
				Action: app.Action(with.RequiredFlags("id"), with.Auth, func(c *app.Context) error {
					// read all flags?
					discs := c.Context.StringSlice("cancel-disc")
					pools := c.Context.StringSlice("cancel-pool")
					tails := c.Context.StringSlice("cancel-tail")

					totalLength := append(discs, pools...)
					totalLength = append(totalLength, tails...)

					// check the presence of cancel all first
					if c.Context.IsSet("cancel-all") {
						if len(totalLength) > 0 {

							return fmt.Errorf("You have set additional flags as well as --cancel-all. Nothing else can be specified when --cancel-all has been set.")
						}
						if err := brainMethods.CancelMigrationJob(c.Client(), c.Context.Int("id")); err != nil {
							return err
						}

						// make the call to the cancel all migration.
					}

					modifications := brain.MigrationJobModification{
						Cancel: brain.MigrationJobLocations{
							Discs: stringsToJsonNumbers(discs),
							Pools: stringsToJsonNumbers(pools),
							Tails: stringsToJsonNumbers(tails),
						},
						Options: brain.MigrationJobOptions{
							Priority: c.Context.Int("priority"),
						},
					}

					err := brainMethods.EditMigrationJob(c.Client(), c.Context.Int("id"), modifications)
					if err != nil {
						return err
					}

					// if we have set a new priority, show a message.
					c.LogErr("Priority updated for Job %d", c.Context.Int("id"))

					// not really sure if we should be showing a confirmation message for all individual things that have been cancelled.
					return err
				}),
			},
		},
	})
}
<|endoftext|>


<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func TestElideNestedPaths(t *testing.T) {
	doElision := func(deps []string) []string {
		// Ignore OS-dependent path sep
		return elideNestedPaths(deps, "/")
	}

	Convey(`Mixed`, t, func() {
		deps := []string{
			"ab/foo",
			"ab/",
			"foo",
			"b/c/",
			"b/a",
			"b/c/a",
			"ab/cd/",
		}
		So(doElision(deps), ShouldResemble, []string{"ab/", "b/a", "b/c/", "foo"})
	})

	Convey(`All files`, t, func() {
		deps := []string{
			"ab/foo",
			"ab/cd/foo",
			"foo",
			"ab/bar",
		}
		So(doElision(deps), ShouldResemble, []string{"ab/bar", "ab/cd/foo", "ab/foo", "foo"})
	})

	Convey(`Cousin paths`, t, func() {
		deps := []string{
			"ab/foo", // This is a file
			"ab/cd/",
			"ab/ef/",
			"ab/bar",
		}
		So(doElision(deps), ShouldResemble, []string{"ab/bar", "ab/cd/", "ab/ef/", "ab/foo"})
	})

	Convey(`Interesting dirs`, t, func() {
		deps := []string{
			"a/b/",
			"a/b/c/",
			"a/bc/",
			"a/bc/d/",
			"a/bcd/",
			"a/c/",
		}
		// Make sure:
		// 1. "a/b/" elides "a/b/c/", but not "a/bc/"
		// 2. "a/bc/" elides "a/bc/d/", but not "a/bcd/"
		So(doElision(deps), ShouldResemble, []string{"a/b/", "a/bc/", "a/bcd/", "a/c/"})
	})

	Convey(`Interesting files`, t, func() {
		deps := []string{
			"a/b",
			"a/bc",
			"a/bcd",
			"a/c",
		}
		// Make sure "a/b" elides neither "a/bc" nor "a/bcd"
		So(doElision(deps), ShouldResemble, []string{"a/b", "a/bc", "a/bcd", "a/c"})
	})
}
<commit_msg>[isolate] Parallelize tests in common_test.go<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func TestElideNestedPaths(t *testing.T) {
	t.Parallel()
	doElision := func(deps []string) []string {
		// Ignore OS-dependent path sep
		return elideNestedPaths(deps, "/")
	}

	Convey(`Mixed`, t, func() {
		deps := []string{
			"ab/foo",
			"ab/",
			"foo",
			"b/c/",
			"b/a",
			"b/c/a",
			"ab/cd/",
		}
		So(doElision(deps), ShouldResemble, []string{"ab/", "b/a", "b/c/", "foo"})
	})

	Convey(`All files`, t, func() {
		deps := []string{
			"ab/foo",
			"ab/cd/foo",
			"foo",
			"ab/bar",
		}
		So(doElision(deps), ShouldResemble, []string{"ab/bar", "ab/cd/foo", "ab/foo", "foo"})
	})

	Convey(`Cousin paths`, t, func() {
		deps := []string{
			"ab/foo", // This is a file
			"ab/cd/",
			"ab/ef/",
			"ab/bar",
		}
		So(doElision(deps), ShouldResemble, []string{"ab/bar", "ab/cd/", "ab/ef/", "ab/foo"})
	})

	Convey(`Interesting dirs`, t, func() {
		deps := []string{
			"a/b/",
			"a/b/c/",
			"a/bc/",
			"a/bc/d/",
			"a/bcd/",
			"a/c/",
		}
		// Make sure:
		// 1. "a/b/" elides "a/b/c/", but not "a/bc/"
		// 2. "a/bc/" elides "a/bc/d/", but not "a/bcd/"
		So(doElision(deps), ShouldResemble, []string{"a/b/", "a/bc/", "a/bcd/", "a/c/"})
	})

	Convey(`Interesting files`, t, func() {
		deps := []string{
			"a/b",
			"a/bc",
			"a/bcd",
			"a/c",
		}
		// Make sure "a/b" elides neither "a/bc" nor "a/bcd"
		So(doElision(deps), ShouldResemble, []string{"a/b", "a/bc", "a/bcd", "a/c"})
	})
}
<|endoftext|>


<commit_before>package protocol

import (
	"github.com/AutogrowSystems/faye-go/utils"
	"reflect"
	"sync"
	"time"
)

type MsgStore interface {
	EnqueueMessages([]Message)
	GetAndClearMessages() []Message
}

// Connect requests starts a session
type Session struct {
	conn     Connection
	timeout  int
	response Message
	client   *Client
	started  time.Time
	logger   utils.Logger
}

func NewSession(client *Client, conn Connection, timeout int, response Message, logger utils.Logger) *Session {
	session := Session{conn, timeout, response, client, time.Now(), logger}
	if timeout > 0 {
		go func() {
			time.Sleep(time.Duration(timeout) * time.Millisecond)
			session.End()
		}()
	}
	return &session
}

func (s Session) End() {
	// c.mutex.Lock()
	// defer c.mutex.Unlock()

	if s.conn.IsConnected() {
		s.conn.Send([]Message{s.response})
	} else {
		s.logger.Debugf("No longer connected %s", s.client.clientId)
	}
}

type Client struct {
	clientId    string
	connection  Connection
	msgStore    MsgStore
	isConnected bool
	responseMsg Message
	mutex       sync.Mutex
	lastSession *Session
	created     time.Time
	logger      utils.Logger
}

func NewClient(clientId string, msgStore MsgStore, logger utils.Logger) Client {
	client := Client{
		clientId:    clientId,
		msgStore:    msgStore,
		isConnected: false,
		created:     time.Now(),
		logger:      logger,
	}

	return client
}

func (c Client) Id() string {
	return c.clientId
}

func (c *Client) Connect(timeout int, interval int, responseMsg Message, connection Connection) {
	c.mutex.Lock()
	defer c.mutex.Unlock()

	c.lastSession = NewSession(c, connection, timeout, responseMsg, c.logger)
	c.responseMsg = responseMsg

	c.flushMsgs()
}

func (c *Client) SetConnection(connection Connection) {
	c.mutex.Lock()
	defer c.mutex.Unlock()

	if c.connection == nil || connection.Priority() > c.connection.Priority() {
		c.connection = connection
		c.isConnected = true
	}
}

func (c Client) Queue(msg Message) {
	c.mutex.Lock()
	defer c.mutex.Unlock()
	msg.SetClientId(c.clientId)
	c.msgStore.EnqueueMessages([]Message{msg})
	c.flushMsgs()
}

func (c Client) QueueMany(msgs []Message) {
	c.mutex.Lock()
	defer c.mutex.Unlock()

	c.msgStore.EnqueueMessages(msgs)
	c.flushMsgs()
}

func (c Client) IsExpired() bool {
	c.mutex.Lock()
	defer c.mutex.Unlock()

	if time.Now().Sub(c.created) > time.Duration(1*time.Minute) {
		if c.lastSession != nil &&
			time.Now().Sub(c.lastSession.started) > time.Duration(2*time.Hour) {
			return true
		}
	}
	return false
}

func (c Client) IsConnected() bool {
	c.logger.Debugf("Client %s connected?: %t, %t", c.clientId, c.isConnected, c.connection.IsConnected())
	return c.isConnected && c.connection != nil && c.connection.IsConnected()
}

func (c Client) flushMsgs() {
	if c.IsConnected() {

		msgs := c.msgStore.GetAndClearMessages()
		if len(msgs) > 0 {

			var msgsToSend []Message

			if c.responseMsg != nil {
				msgsToSend = append(msgsToSend, c.responseMsg)
				msgsToSend = append(msgsToSend, msgs...)
			} else {
				msgsToSend = msgs
			}

			c.logger.Debugf("Sending %d msgs to %s on %s", len(msgsToSend), c.clientId, reflect.TypeOf(c.connection))
			err := c.connection.Send(msgsToSend)

			// failed, so requeue
			if err != nil {
				c.logger.Errorf("ERROR: %+v", err)
				c.logger.Debugf("Was unable to send to %s, requeued %d messages", c.clientId, len(msgs))
				c.msgStore.EnqueueMessages(msgs)
			} else {
				c.responseMsg = nil
				c.isConnected = false
			}
		}
	} else {
		c.logger.Debugf("Not connected to %s", c.clientId)
	}
}
<commit_msg>set connection and clientID on response in client Connect()<commit_after>package protocol

import (
	"github.com/AutogrowSystems/faye-go/utils"
	"reflect"
	"sync"
	"time"
)

type MsgStore interface {
	EnqueueMessages([]Message)
	GetAndClearMessages() []Message
}

// Connect requests starts a session
type Session struct {
	conn     Connection
	timeout  int
	response Message
	client   *Client
	started  time.Time
	logger   utils.Logger
}

func NewSession(client *Client, conn Connection, timeout int, response Message, logger utils.Logger) *Session {
	session := Session{conn, timeout, response, client, time.Now(), logger}
	if timeout > 0 {
		go func() {
			time.Sleep(time.Duration(timeout) * time.Millisecond)
			session.End()
		}()
	}
	return &session
}

func (s Session) End() {
	// c.mutex.Lock()
	// defer c.mutex.Unlock()

	if s.conn.IsConnected() {
		s.conn.Send([]Message{s.response})
	} else {
		s.logger.Debugf("No longer connected %s", s.client.clientId)
	}
}

type Client struct {
	clientId    string
	connection  Connection
	msgStore    MsgStore
	isConnected bool
	responseMsg Message
	mutex       sync.Mutex
	lastSession *Session
	created     time.Time
	logger      utils.Logger
}

func NewClient(clientId string, msgStore MsgStore, logger utils.Logger) Client {
	client := Client{
		clientId:    clientId,
		msgStore:    msgStore,
		isConnected: false,
		created:     time.Now(),
		logger:      logger,
	}

	return client
}

func (c Client) Id() string {
	return c.clientId
}

func (c *Client) Connect(timeout int, interval int, responseMsg Message, connection Connection) {
	c.mutex.Lock()
	defer c.mutex.Unlock()

	responseMsg["clientId"] = c.Id()

	c.connection = connection
	c.lastSession = NewSession(c, connection, timeout, responseMsg, c.logger)
	c.responseMsg = responseMsg

	c.flushMsgs()
}

func (c *Client) SetConnection(connection Connection) {
	c.mutex.Lock()
	defer c.mutex.Unlock()

	if c.connection == nil || connection.Priority() > c.connection.Priority() {
		c.connection = connection
		c.isConnected = true
	}
}

func (c Client) Queue(msg Message) {
	c.mutex.Lock()
	defer c.mutex.Unlock()
	msg.SetClientId(c.clientId)
	c.msgStore.EnqueueMessages([]Message{msg})
	c.flushMsgs()
}

func (c Client) QueueMany(msgs []Message) {
	c.mutex.Lock()
	defer c.mutex.Unlock()

	c.msgStore.EnqueueMessages(msgs)
	c.flushMsgs()
}

func (c Client) IsExpired() bool {
	c.mutex.Lock()
	defer c.mutex.Unlock()

	if time.Now().Sub(c.created) > time.Duration(1*time.Minute) {
		if c.lastSession != nil &&
			time.Now().Sub(c.lastSession.started) > time.Duration(2*time.Hour) {
			return true
		}
	}
	return false
}

func (c Client) IsConnected() bool {
	c.logger.Debugf("Client %s connected?: %t, %t", c.clientId, c.isConnected, c.connection.IsConnected())
	return c.isConnected && c.connection != nil && c.connection.IsConnected()
}

func (c Client) flushMsgs() {
	if c.IsConnected() {

		msgs := c.msgStore.GetAndClearMessages()
		if len(msgs) > 0 {

			var msgsToSend []Message

			if c.responseMsg != nil {
				msgsToSend = append(msgsToSend, c.responseMsg)
				msgsToSend = append(msgsToSend, msgs...)
			} else {
				msgsToSend = msgs
			}

			c.logger.Debugf("Sending %d msgs to %s on %s", len(msgsToSend), c.clientId, reflect.TypeOf(c.connection))
			err := c.connection.Send(msgsToSend)

			// failed, so requeue
			if err != nil {
				c.logger.Errorf("ERROR: %+v", err)
				c.logger.Debugf("Was unable to send to %s, requeued %d messages", c.clientId, len(msgs))
				c.msgStore.EnqueueMessages(msgs)
			} else {
				c.responseMsg = nil
				c.isConnected = false
			}
		}
	} else {
		c.logger.Debugf("Not connected to %s", c.clientId)
	}
}
<|endoftext|>


<commit_before>package vix

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path/filepath"
	"strings"
	"time"

	govix "github.com/c4milo/govix"
	"github.com/dustin/go-humanize"
)

// Virtual machine configuration
type VM struct {
	// Which VMware VIX service provider to use. ie: fusion, workstation, server, etc
	Provider string
	// Whether to verify SSL or not for remote connections in ESXi
	VerifySSL bool
	// Name of the virtual machine
	Name string
	// Description for the virtual machine, it is created as an annotation in
	// VMware.
	Description string
	// Image to use during the creation of this virtual machine
	Image Image
	// Number of virtual cpus
	CPUs uint
	// Memory size in megabytes.
	Memory string
	// Switches to where this machine is going to be attach to
	VSwitches []string
	// Whether to upgrade the VM virtual hardware
	UpgradeVHardware bool
	// The timeout to wait for VMware Tools to be initialized inside the VM
	ToolsInitTimeout time.Duration
	// Whether to launch the VM with graphical environment
	LaunchGUI bool
	// Whether to enable or disable shared folders for this VM
	SharedFolders bool
	// Network adapters
	VNetworkAdapters []*govix.NetworkAdapter
	// VM IP address as reported by VIX
	IPAddress string
}

// Creates VIX instance with VMware
func (v *VM) client() (*govix.Host, error) {
	var p govix.Provider

	switch strings.ToLower(v.Provider) {
	case "fusion", "workstation":
		p = govix.VMWARE_WORKSTATION
	case "serverv1":
		p = govix.VMWARE_SERVER
	case "serverv2":
		p = govix.VMWARE_VI_SERVER
	case "player":
		p = govix.VMWARE_PLAYER
	case "workstation_shared":
		p = govix.VMWARE_WORKSTATION_SHARED
	default:
		p = govix.VMWARE_WORKSTATION
	}

	var options govix.HostOption
	if v.VerifySSL {
		options = govix.VERIFY_SSL_CERT
	}

	host, err := govix.Connect(govix.ConnectConfig{
		Provider: p,
		Options:  options,
	})

	if err != nil {
		return nil, err
	}

	log.Printf("[INFO] VIX client configured for product: VMware %s. SSL: %t", v.Provider, v.VerifySSL)

	return host, nil
}

// Sets default values for VM attributes
func (v *VM) SetDefaults() {
	if v.CPUs <= 0 {
		v.CPUs = 1
	}

	if v.Memory == "" {
		v.Memory = "512mib"
	}

	if v.Description == "" {
		v.Description = "Machine was created using Terraform VIX provider"
	}

	if v.ToolsInitTimeout.Seconds() <= 0 {
		v.ToolsInitTimeout = time.Duration(30) * time.Second
	}
}

// Downloads, extracts and opens Gold virtual machine, then it creates a clone
// out of it.
func (v *VM) Create() (string, error) {
	log.Printf("[DEBUG] Creating VM resource...")

	usr, err := user.Current()
	if err != nil {
		return "", err
	}

	image := v.Image

	imgPath := filepath.Join(usr.HomeDir, ".terraform/vix/images", image.Checksum)
	if err = image.Download(imgPath); err != nil {
		return "", err
	}
	defer image.file.Close()

	goldPath := filepath.Join(usr.HomeDir, filepath.Join(".terraform/vix/gold", image.Checksum))
	_, err = os.Stat(goldPath)
	goldPathExist := err == nil || err != os.ErrNotExist

	// There is no need to get the error as the slice will be empty anyway
	finfo, _ := ioutil.ReadDir(goldPath)
	goldPathEmpty := len(finfo) == 0

	if !goldPathExist || goldPathEmpty {
		log.Printf("[DEBUG] Gold virtual machine does not exist or is empty: %s", goldPath)
		// TODO(c4milo): Make sure the file is a tgz file before attempting
		// to unpack it.
		_, err = image.Unpack(goldPath)
		if err != nil {
			return "", err
		}
	}

	pattern := filepath.Join(goldPath, "/**/*.vmx")

	log.Printf("[DEBUG] Finding Gold virtual machine vmx file in %s", pattern)
	files, _ := filepath.Glob(pattern)

	if len(files) == 0 {
		return "", fmt.Errorf("[ERROR] vmx file was not found: %s", pattern)
	}

	vmxFile := files[0]
	log.Printf("[DEBUG] Gold virtual machine vmx file found %v", vmxFile)

	// Gets VIX instance
	client, err := v.client()
	if err != nil {
		return "", err
	}
	defer client.Disconnect()

	log.Printf("[INFO] Opening Gold virtual machine from %s", vmxFile)

	vm, err := client.OpenVm(vmxFile, v.Image.Password)
	if err != nil {
		return "", err
	}

	newvmx := filepath.Join(usr.HomeDir, ".terraform/vix/vms",
		image.Checksum, v.Name, v.Name+".vmx")

	if _, err = os.Stat(newvmx); err != os.ErrExist {
		log.Printf("[INFO] Cloning gold vm into %s...", newvmx)
		_, err := vm.Clone(govix.CLONETYPE_LINKED, newvmx)
		// If there is an error and the error is other than "The snapshot already exists"
		// then return the error
		if err != nil && err.(*govix.VixError).Code != 13004 {
			return "", err
		}

		// Makes sure the first time the VM is created it has no virtual network adapters
		// log.Printf("[DEBUG] Removing all virtual network adapters from cloned VM...")
		// err = clonedVM.RemoveAllNetworkAdapters()
		// if err != nil {
		// 	return "", err
		// }
	} else {
		log.Printf("[INFO] VM Clone %s already exist, moving on.", newvmx)
	}

	if err = v.Update(newvmx); err != nil {
		return "", err
	}

	return newvmx, nil
}

// Opens and updates virtual machine resource
func (v *VM) Update(vmxFile string) error {
	// Sets default values if some attributes were not set or have
	// invalid values
	v.SetDefaults()

	// Gets VIX instance
	client, err := v.client()
	if err != nil {
		return err
	}
	defer client.Disconnect()

	if client.Provider == govix.VMWARE_VI_SERVER ||
		client.Provider == govix.VMWARE_SERVER {
		log.Printf("[INFO] Registering VM in host's inventory...")
		err = client.RegisterVm(vmxFile)
		if err != nil {
			return err
		}
	}

	log.Printf("[INFO] Opening virtual machine from %s", vmxFile)

	vm, err := client.OpenVm(vmxFile, v.Image.Password)
	if err != nil {
		return err
	}

	running, err := vm.IsRunning()
	if err != nil {
		return err
	}

	if running {
		log.Printf("[INFO] Virtual machine seems to be running, we need to " +
			"power it off in order to make changes.")
		err = v.powerOff(vm)
		if err != nil {
			return err
		}
	}

	memoryInMb, err := humanize.ParseBytes(v.Memory)
	if err != nil {
		log.Printf("[WARN] Unable to set memory size, defaulting to 512mib: %s", err)
		memoryInMb = 512
	} else {
		memoryInMb = (memoryInMb / 1024) / 1024
	}

	log.Printf("[DEBUG] Setting memory size to %d megabytes", memoryInMb)
	vm.SetMemorySize(uint(memoryInMb))

	log.Printf("[DEBUG] Setting vcpus to %d", v.CPUs)
	vm.SetNumberVcpus(uint8(v.CPUs))

	log.Printf("[DEBUG] Setting name to %s", v.Name)
	vm.SetDisplayName(v.Name)

	log.Printf("[DEBUG] Setting description to %s", v.Description)
	vm.SetAnnotation(v.Description)

	if v.UpgradeVHardware &&
		client.Provider != govix.VMWARE_PLAYER {

		log.Println("[INFO] Upgrading virtual hardware...")
		err = vm.UpgradeVHardware()
		if err != nil {
			return err
		}
	}

	// log.Printf("[DEBUG] Loading all network adapters in memory...")
	// curAdapters, err := vm.NetworkAdapters()
	// if err != nil {
	// 	return err
	// }

	log.Printf("[DEBUG] Removing all network adapters from vmx file...")
	err = vm.RemoveAllNetworkAdapters()
	if err != nil {
		return err
	}

	log.Println("[DEBUG] Attaching virtual network adapters...")
	for _, adapter := range v.VNetworkAdapters {
		adapter.StartConnected = true
		if adapter.ConnType == govix.NETWORK_BRIDGED {
			adapter.LinkStatePropagation = true
		}

		log.Printf("[DEBUG] Adapter: %+v", adapter)
		err := vm.AddNetworkAdapter(adapter)
		if err != nil {
			return err
		}
	}

	log.Println("[INFO] Powering virtual machine on...")
	var options govix.VMPowerOption

	if v.LaunchGUI {
		log.Println("[INFO] Preparing to launch GUI...")
		options |= govix.VMPOWEROP_LAUNCH_GUI
	}

	options |= govix.VMPOWEROP_NORMAL

	err = vm.PowerOn(options)
	if err != nil {
		return err
	}

	log.Println("[INFO] Waiting for VMware Tools to initialize...")
	err = vm.WaitForToolsInGuest(v.ToolsInitTimeout)
	if err != nil {
		log.Println("[WARN] VMware Tools took too long to initialize or is not " +
			"installed.")

		if v.SharedFolders {
			log.Println("[WARN] Enabling shared folders is not possible.")
		}
		return nil
	}

	if v.SharedFolders {
		log.Println("[DEBUG] Enabling shared folders...")

		err = vm.EnableSharedFolders(v.SharedFolders)
		if err != nil {
			return err
		}
	}
	return nil
}

// Powers off a virtual machine attempting a graceful shutdown.
func (v *VM) powerOff(vm *govix.VM) error {
	tstate, err := vm.ToolsState()
	if err != nil {
		return err
	}

	var powerOpts govix.VMPowerOption
	log.Printf("Tools state %d", tstate)

	if (tstate & govix.TOOLSSTATE_RUNNING) != 0 {
		log.Printf("[INFO] VMware Tools is running, attempting a graceful shutdown...")
		// if VMware Tools is running, attempt a graceful shutdown.
		powerOpts |= govix.VMPOWEROP_FROM_GUEST
	} else {
		log.Printf("[INFO] VMware Tools is NOT running, shutting down the " +
			"machine abruptly...")
		powerOpts |= govix.VMPOWEROP_NORMAL
	}

	err = vm.PowerOff(powerOpts)
	if err != nil {
		return err
	}
	log.Printf("[DEBUG] Virtual machine is off.")

	return nil
}

// Destroys a virtual machine resource
func (v *VM) Destroy(vmxFile string) error {
	log.Printf("[DEBUG] Destroying VM resource %s...", vmxFile)

	client, err := v.client()
	if err != nil {
		return err
	}
	defer client.Disconnect()

	vm, err := client.OpenVm(vmxFile, v.Image.Password)
	if err != nil {
		return err
	}

	running, err := vm.IsRunning()
	if err != nil {
		return err
	}

	if running {
		if err = v.powerOff(vm); err != nil {
			return err
		}
	}

	if client.Provider == govix.VMWARE_VI_SERVER ||
		client.Provider == govix.VMWARE_SERVER {
		log.Printf("[INFO] Unregistering VM from host's inventory...")

		err := client.UnregisterVm(vmxFile)
		if err != nil {
			return err
		}
	}

	return vm.Delete(govix.VMDELETE_KEEP_FILES | govix.VMDELETE_FORCE)
}

// Refreshes state with VMware
func (v *VM) Refresh(vmxFile string) (bool, error) {
	log.Printf("[DEBUG] Syncing VM resource %s...", vmxFile)

	client, err := v.client()
	if err != nil {
		return false, err
	}
	defer client.Disconnect()

	vm, err := client.OpenVm(vmxFile, v.Image.Password)
	if err != nil {
		return false, err
	}

	running, err := vm.IsRunning()
	if !running {
		return running, err
	}

	vcpus, err := vm.Vcpus()
	if err != nil {
		return running, err
	}

	memory, err := vm.MemorySize()
	if err != nil {
		return running, err
	}

	// We need to convert memory value to megabytes so humanize can interpret it
	// properly.
	memory = (memory * 1024) * 1024
	v.Memory = strings.ToLower(humanize.IBytes(uint64(memory)))
	v.CPUs = uint(vcpus)
	v.Name, err = vm.DisplayName()
	v.Description, err = vm.Annotation()
	v.VNetworkAdapters, err = vm.NetworkAdapters()
	v.IPAddress, err = vm.IPAddress()

	return running, err
}
<commit_msg>Linked clones does not allow resizing disks.<commit_after>package vix

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path/filepath"
	"strings"
	"time"

	govix "github.com/c4milo/govix"
	"github.com/dustin/go-humanize"
)

// Virtual machine configuration
type VM struct {
	// Which VMware VIX service provider to use. ie: fusion, workstation, server, etc
	Provider string
	// Whether to verify SSL or not for remote connections in ESXi
	VerifySSL bool
	// Name of the virtual machine
	Name string
	// Description for the virtual machine, it is created as an annotation in
	// VMware.
	Description string
	// Image to use during the creation of this virtual machine
	Image Image
	// Number of virtual cpus
	CPUs uint
	// Memory size in megabytes.
	Memory string
	// Switches to where this machine is going to be attach to
	VSwitches []string
	// Whether to upgrade the VM virtual hardware
	UpgradeVHardware bool
	// The timeout to wait for VMware Tools to be initialized inside the VM
	ToolsInitTimeout time.Duration
	// Whether to launch the VM with graphical environment
	LaunchGUI bool
	// Whether to enable or disable shared folders for this VM
	SharedFolders bool
	// Network adapters
	VNetworkAdapters []*govix.NetworkAdapter
	// VM IP address as reported by VIX
	IPAddress string
}

// Creates VIX instance with VMware
func (v *VM) client() (*govix.Host, error) {
	var p govix.Provider

	switch strings.ToLower(v.Provider) {
	case "fusion", "workstation":
		p = govix.VMWARE_WORKSTATION
	case "serverv1":
		p = govix.VMWARE_SERVER
	case "serverv2":
		p = govix.VMWARE_VI_SERVER
	case "player":
		p = govix.VMWARE_PLAYER
	case "workstation_shared":
		p = govix.VMWARE_WORKSTATION_SHARED
	default:
		p = govix.VMWARE_WORKSTATION
	}

	var options govix.HostOption
	if v.VerifySSL {
		options = govix.VERIFY_SSL_CERT
	}

	host, err := govix.Connect(govix.ConnectConfig{
		Provider: p,
		Options:  options,
	})

	if err != nil {
		return nil, err
	}

	log.Printf("[INFO] VIX client configured for product: VMware %s. SSL: %t", v.Provider, v.VerifySSL)

	return host, nil
}

// Sets default values for VM attributes
func (v *VM) SetDefaults() {
	if v.CPUs <= 0 {
		v.CPUs = 1
	}

	if v.Memory == "" {
		v.Memory = "512mib"
	}

	if v.Description == "" {
		v.Description = "Machine was created using Terraform VIX provider"
	}

	if v.ToolsInitTimeout.Seconds() <= 0 {
		v.ToolsInitTimeout = time.Duration(30) * time.Second
	}
}

// Downloads, extracts and opens Gold virtual machine, then it creates a clone
// out of it.
func (v *VM) Create() (string, error) {
	log.Printf("[DEBUG] Creating VM resource...")

	usr, err := user.Current()
	if err != nil {
		return "", err
	}

	image := v.Image

	imgPath := filepath.Join(usr.HomeDir, ".terraform/vix/images", image.Checksum)
	if err = image.Download(imgPath); err != nil {
		return "", err
	}
	defer image.file.Close()

	goldPath := filepath.Join(usr.HomeDir, filepath.Join(".terraform/vix/gold", image.Checksum))
	_, err = os.Stat(goldPath)
	goldPathExist := err == nil || err != os.ErrNotExist

	// There is no need to get the error as the slice will be empty anyway
	finfo, _ := ioutil.ReadDir(goldPath)
	goldPathEmpty := len(finfo) == 0

	if !goldPathExist || goldPathEmpty {
		log.Printf("[DEBUG] Gold virtual machine does not exist or is empty: %s", goldPath)
		// TODO(c4milo): Make sure the file is a tgz file before attempting
		// to unpack it.
		_, err = image.Unpack(goldPath)
		if err != nil {
			return "", err
		}
	}

	pattern := filepath.Join(goldPath, "/**/*.vmx")

	log.Printf("[DEBUG] Finding Gold virtual machine vmx file in %s", pattern)
	files, _ := filepath.Glob(pattern)

	if len(files) == 0 {
		return "", fmt.Errorf("[ERROR] vmx file was not found: %s", pattern)
	}

	vmxFile := files[0]
	log.Printf("[DEBUG] Gold virtual machine vmx file found %v", vmxFile)

	// Gets VIX instance
	client, err := v.client()
	if err != nil {
		return "", err
	}
	defer client.Disconnect()

	log.Printf("[INFO] Opening Gold virtual machine from %s", vmxFile)

	vm, err := client.OpenVm(vmxFile, v.Image.Password)
	if err != nil {
		return "", err
	}

	newvmx := filepath.Join(usr.HomeDir, ".terraform/vix/vms",
		image.Checksum, v.Name, v.Name+".vmx")

	if _, err = os.Stat(newvmx); err != os.ErrExist {
		log.Printf("[INFO] Cloning gold vm into %s...", newvmx)
		_, err := vm.Clone(govix.CLONETYPE_FULL, newvmx)
		// If there is an error and the error is other than "The snapshot already exists"
		// then return the error
		if err != nil && err.(*govix.VixError).Code != 13004 {
			return "", err
		}

		// Makes sure the first time the VM is created it has no virtual network adapters
		// log.Printf("[DEBUG] Removing all virtual network adapters from cloned VM...")
		// err = clonedVM.RemoveAllNetworkAdapters()
		// if err != nil {
		// 	return "", err
		// }
	} else {
		log.Printf("[INFO] VM Clone %s already exist, moving on.", newvmx)
	}

	if err = v.Update(newvmx); err != nil {
		return "", err
	}

	return newvmx, nil
}

// Opens and updates virtual machine resource
func (v *VM) Update(vmxFile string) error {
	// Sets default values if some attributes were not set or have
	// invalid values
	v.SetDefaults()

	// Gets VIX instance
	client, err := v.client()
	if err != nil {
		return err
	}
	defer client.Disconnect()

	if client.Provider == govix.VMWARE_VI_SERVER ||
		client.Provider == govix.VMWARE_SERVER {
		log.Printf("[INFO] Registering VM in host's inventory...")
		err = client.RegisterVm(vmxFile)
		if err != nil {
			return err
		}
	}

	log.Printf("[INFO] Opening virtual machine from %s", vmxFile)

	vm, err := client.OpenVm(vmxFile, v.Image.Password)
	if err != nil {
		return err
	}

	running, err := vm.IsRunning()
	if err != nil {
		return err
	}

	if running {
		log.Printf("[INFO] Virtual machine seems to be running, we need to " +
			"power it off in order to make changes.")
		err = v.powerOff(vm)
		if err != nil {
			return err
		}
	}

	memoryInMb, err := humanize.ParseBytes(v.Memory)
	if err != nil {
		log.Printf("[WARN] Unable to set memory size, defaulting to 512mib: %s", err)
		memoryInMb = 512
	} else {
		memoryInMb = (memoryInMb / 1024) / 1024
	}

	log.Printf("[DEBUG] Setting memory size to %d megabytes", memoryInMb)
	vm.SetMemorySize(uint(memoryInMb))

	log.Printf("[DEBUG] Setting vcpus to %d", v.CPUs)
	vm.SetNumberVcpus(uint8(v.CPUs))

	log.Printf("[DEBUG] Setting name to %s", v.Name)
	vm.SetDisplayName(v.Name)

	log.Printf("[DEBUG] Setting description to %s", v.Description)
	vm.SetAnnotation(v.Description)

	if v.UpgradeVHardware &&
		client.Provider != govix.VMWARE_PLAYER {

		log.Println("[INFO] Upgrading virtual hardware...")
		err = vm.UpgradeVHardware()
		if err != nil {
			return err
		}
	}

	// log.Printf("[DEBUG] Loading all network adapters in memory...")
	// curAdapters, err := vm.NetworkAdapters()
	// if err != nil {
	// 	return err
	// }

	log.Printf("[DEBUG] Removing all network adapters from vmx file...")
	err = vm.RemoveAllNetworkAdapters()
	if err != nil {
		return err
	}

	log.Println("[DEBUG] Attaching virtual network adapters...")
	for _, adapter := range v.VNetworkAdapters {
		adapter.StartConnected = true
		if adapter.ConnType == govix.NETWORK_BRIDGED {
			adapter.LinkStatePropagation = true
		}

		log.Printf("[DEBUG] Adapter: %+v", adapter)
		err := vm.AddNetworkAdapter(adapter)
		if err != nil {
			return err
		}
	}

	log.Println("[INFO] Powering virtual machine on...")
	var options govix.VMPowerOption

	if v.LaunchGUI {
		log.Println("[INFO] Preparing to launch GUI...")
		options |= govix.VMPOWEROP_LAUNCH_GUI
	}

	options |= govix.VMPOWEROP_NORMAL

	err = vm.PowerOn(options)
	if err != nil {
		return err
	}

	log.Println("[INFO] Waiting for VMware Tools to initialize...")
	err = vm.WaitForToolsInGuest(v.ToolsInitTimeout)
	if err != nil {
		log.Println("[WARN] VMware Tools took too long to initialize or is not " +
			"installed.")

		if v.SharedFolders {
			log.Println("[WARN] Enabling shared folders is not possible.")
		}
		return nil
	}

	if v.SharedFolders {
		log.Println("[DEBUG] Enabling shared folders...")

		err = vm.EnableSharedFolders(v.SharedFolders)
		if err != nil {
			return err
		}
	}
	return nil
}

// Powers off a virtual machine attempting a graceful shutdown.
func (v *VM) powerOff(vm *govix.VM) error {
	tstate, err := vm.ToolsState()
	if err != nil {
		return err
	}

	var powerOpts govix.VMPowerOption
	log.Printf("Tools state %d", tstate)

	if (tstate & govix.TOOLSSTATE_RUNNING) != 0 {
		log.Printf("[INFO] VMware Tools is running, attempting a graceful shutdown...")
		// if VMware Tools is running, attempt a graceful shutdown.
		powerOpts |= govix.VMPOWEROP_FROM_GUEST
	} else {
		log.Printf("[INFO] VMware Tools is NOT running, shutting down the " +
			"machine abruptly...")
		powerOpts |= govix.VMPOWEROP_NORMAL
	}

	err = vm.PowerOff(powerOpts)
	if err != nil {
		return err
	}
	log.Printf("[DEBUG] Virtual machine is off.")

	return nil
}

// Destroys a virtual machine resource
func (v *VM) Destroy(vmxFile string) error {
	log.Printf("[DEBUG] Destroying VM resource %s...", vmxFile)

	client, err := v.client()
	if err != nil {
		return err
	}
	defer client.Disconnect()

	vm, err := client.OpenVm(vmxFile, v.Image.Password)
	if err != nil {
		return err
	}

	running, err := vm.IsRunning()
	if err != nil {
		return err
	}

	if running {
		if err = v.powerOff(vm); err != nil {
			return err
		}
	}

	if client.Provider == govix.VMWARE_VI_SERVER ||
		client.Provider == govix.VMWARE_SERVER {
		log.Printf("[INFO] Unregistering VM from host's inventory...")

		err := client.UnregisterVm(vmxFile)
		if err != nil {
			return err
		}
	}

	return vm.Delete(govix.VMDELETE_KEEP_FILES | govix.VMDELETE_FORCE)
}

// Refreshes state with VMware
func (v *VM) Refresh(vmxFile string) (bool, error) {
	log.Printf("[DEBUG] Syncing VM resource %s...", vmxFile)

	client, err := v.client()
	if err != nil {
		return false, err
	}
	defer client.Disconnect()

	vm, err := client.OpenVm(vmxFile, v.Image.Password)
	if err != nil {
		return false, err
	}

	running, err := vm.IsRunning()
	if !running {
		return running, err
	}

	vcpus, err := vm.Vcpus()
	if err != nil {
		return running, err
	}

	memory, err := vm.MemorySize()
	if err != nil {
		return running, err
	}

	// We need to convert memory value to megabytes so humanize can interpret it
	// properly.
	memory = (memory * 1024) * 1024
	v.Memory = strings.ToLower(humanize.IBytes(uint64(memory)))
	v.CPUs = uint(vcpus)
	v.Name, err = vm.DisplayName()
	v.Description, err = vm.Annotation()
	v.VNetworkAdapters, err = vm.NetworkAdapters()
	v.IPAddress, err = vm.IPAddress()

	return running, err
}
<|endoftext|>


<commit_before>package main_test

import (
	"fmt"
	"os"
	"os/exec"

	Bbs "github.com/cloudfoundry-incubator/runtime-schema/bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/cb"
	"github.com/cloudfoundry/storeadapter"
	"github.com/cloudfoundry/storeadapter/storerunner/etcdstorerunner"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"
	"github.com/pivotal-golang/lager/lagertest"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"

	"testing"
	"time"
)

var auctioneer ifrit.Process

var auctioneerPath string

var dotNetStack = "dot-net"
var lucidStack = "lucid64"
var dotNetCell, lucidCell *FakeCell

var etcdPort int
var auctioneerServerPort int
var auctioneerAddress string

var runner *ginkgomon.Runner
var etcdRunner *etcdstorerunner.ETCDClusterRunner
var etcdClient storeadapter.StoreAdapter

var auctioneerClient cb.AuctioneerClient

var bbs *Bbs.BBS
var logger lager.Logger

func TestAuctioneer(t *testing.T) {
	// these integration tests can take a bit, especially under load;
	// 1 second is too harsh
	SetDefaultEventuallyTimeout(10 * time.Second)

	RegisterFailHandler(Fail)
	RunSpecs(t, "Auctioneer Cmd Suite")
}

var _ = SynchronizedBeforeSuite(func() []byte {
	compiledAuctioneerPath, err := gexec.Build("github.com/cloudfoundry-incubator/auctioneer/cmd/auctioneer", "-race")
	Ω(err).ShouldNot(HaveOccurred())
	return []byte(compiledAuctioneerPath)
}, func(compiledAuctioneerPath []byte) {
	auctioneerPath = string(compiledAuctioneerPath)

	auctioneerServerPort = 1800 + GinkgoParallelNode()
	auctioneerAddress = fmt.Sprintf("http://127.0.0.1:%d", auctioneerServerPort)

	etcdPort = 5001 + GinkgoParallelNode()
	etcdRunner = etcdstorerunner.NewETCDClusterRunner(etcdPort, 1)
	etcdClient = etcdRunner.Adapter()

	auctioneerClient = cb.NewAuctioneerClient()

	logger = lagertest.NewTestLogger("test")

	bbs = Bbs.NewBBS(etcdClient, clock.NewClock(), logger)
})

var _ = BeforeEach(func() {
	runner = ginkgomon.New(ginkgomon.Config{
		Name: "auctioneer",
		Command: exec.Command(
			auctioneerPath,
			"-etcdCluster", fmt.Sprintf("http://127.0.0.1:%d", etcdPort),
			"-heartbeatInterval", "1s",
			"-listenAddr", fmt.Sprintf("0.0.0.0:%d", auctioneerServerPort),
		),
		StartCheck: "auctioneer.started",
	})

	etcdRunner.Start()

	dotNetCell = SpinUpFakeCell("dot-net-cell", dotNetStack)
	lucidCell = SpinUpFakeCell("lucid-cell", lucidStack)
})

var _ = AfterEach(func() {
	auctioneer.Signal(os.Kill)
	Eventually(auctioneer.Wait()).Should(Receive())

	etcdRunner.Stop()

	dotNetCell.Stop()
	lucidCell.Stop()
})

var _ = SynchronizedAfterSuite(func() {
	if etcdRunner != nil {
		etcdRunner.Stop()
	}
}, func() {
	gexec.CleanupBuildArtifacts()
})
<commit_msg>Set GinkgoWriter as etcd client logger for diagnostics<commit_after>package main_test

import (
	"fmt"
	"log"
	"os"
	"os/exec"

	Bbs "github.com/cloudfoundry-incubator/runtime-schema/bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/cb"
	"github.com/cloudfoundry/storeadapter"
	"github.com/cloudfoundry/storeadapter/storerunner/etcdstorerunner"
	"github.com/coreos/go-etcd/etcd"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"
	"github.com/pivotal-golang/lager/lagertest"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"

	"testing"
	"time"
)

var auctioneer ifrit.Process

var auctioneerPath string

var dotNetStack = "dot-net"
var lucidStack = "lucid64"
var dotNetCell, lucidCell *FakeCell

var etcdPort int
var auctioneerServerPort int
var auctioneerAddress string

var runner *ginkgomon.Runner
var etcdRunner *etcdstorerunner.ETCDClusterRunner
var etcdClient storeadapter.StoreAdapter

var auctioneerClient cb.AuctioneerClient

var bbs *Bbs.BBS
var logger lager.Logger

func TestAuctioneer(t *testing.T) {
	// these integration tests can take a bit, especially under load;
	// 1 second is too harsh
	SetDefaultEventuallyTimeout(10 * time.Second)

	RegisterFailHandler(Fail)
	RunSpecs(t, "Auctioneer Cmd Suite")
}

var _ = SynchronizedBeforeSuite(func() []byte {
	compiledAuctioneerPath, err := gexec.Build("github.com/cloudfoundry-incubator/auctioneer/cmd/auctioneer", "-race")
	Ω(err).ShouldNot(HaveOccurred())
	return []byte(compiledAuctioneerPath)
}, func(compiledAuctioneerPath []byte) {
	auctioneerPath = string(compiledAuctioneerPath)

	auctioneerServerPort = 1800 + GinkgoParallelNode()
	auctioneerAddress = fmt.Sprintf("http://127.0.0.1:%d", auctioneerServerPort)

	etcdPort = 5001 + GinkgoParallelNode()
	etcdRunner = etcdstorerunner.NewETCDClusterRunner(etcdPort, 1)
	etcdClient = etcdRunner.Adapter()

	auctioneerClient = cb.NewAuctioneerClient()

	logger = lagertest.NewTestLogger("test")

	bbs = Bbs.NewBBS(etcdClient, clock.NewClock(), logger)
})

var _ = BeforeEach(func() {
	runner = ginkgomon.New(ginkgomon.Config{
		Name: "auctioneer",
		Command: exec.Command(
			auctioneerPath,
			"-etcdCluster", fmt.Sprintf("http://127.0.0.1:%d", etcdPort),
			"-heartbeatInterval", "1s",
			"-listenAddr", fmt.Sprintf("0.0.0.0:%d", auctioneerServerPort),
		),
		StartCheck: "auctioneer.started",
	})
	etcd.SetLogger(log.New(GinkgoWriter, "go-etcd", log.LstdFlags))

	etcdRunner.Start()

	dotNetCell = SpinUpFakeCell("dot-net-cell", dotNetStack)
	lucidCell = SpinUpFakeCell("lucid-cell", lucidStack)
})

var _ = AfterEach(func() {
	auctioneer.Signal(os.Kill)
	Eventually(auctioneer.Wait()).Should(Receive())

	etcdRunner.Stop()

	dotNetCell.Stop()
	lucidCell.Stop()
})

var _ = SynchronizedAfterSuite(func() {
	if etcdRunner != nil {
		etcdRunner.Stop()
	}
}, func() {
	gexec.CleanupBuildArtifacts()
})
<|endoftext|>


<commit_before>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package images

import (
	"errors"
	"fmt"
	"io"
	"os"

	"github.com/containerd/containerd/cmd/ctr/commands"
	"github.com/containerd/containerd/images/archive"
	"github.com/containerd/containerd/platforms"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/urfave/cli"
)

var exportCommand = cli.Command{
	Name:      "export",
	Usage:     "export images",
	ArgsUsage: "[flags] <out> <image> ...",
	Description: `Export images to an OCI tar archive.

Tar output is formatted as an OCI archive, a Docker manifest is provided for the platform.
Use '--skip-manifest-json' to avoid including the Docker manifest.json file.
Use '--platform' to define the output platform.
When '--all-platforms' is given all images in a manifest list must be available.
`,
	Flags: []cli.Flag{
		cli.BoolFlag{
			Name:  "skip-manifest-json",
			Usage: "do not add Docker compatible manifest.json to archive",
		},
		cli.BoolFlag{
			Name:  "skip-non-distributable",
			Usage: "do not add non-distributable blobs such as Windows layers to archive",
		},
		cli.StringSliceFlag{
			Name:  "platform",
			Usage: "Pull content from a specific platform",
			Value: &cli.StringSlice{},
		},
		cli.BoolFlag{
			Name:  "all-platforms",
			Usage: "exports content from all platforms",
		},
	},
	Action: func(context *cli.Context) error {
		var (
			out        = context.Args().First()
			images     = context.Args().Tail()
			exportOpts = []archive.ExportOpt{}
		)
		if out == "" || len(images) == 0 {
			return errors.New("please provide both an output filename and an image reference to export")
		}

		if pss := context.StringSlice("platform"); len(pss) > 0 {
			var all []ocispec.Platform
			for _, ps := range pss {
				p, err := platforms.Parse(ps)
				if err != nil {
					return fmt.Errorf("invalid platform %q: %w", ps, err)
				}
				all = append(all, p)
			}
			exportOpts = append(exportOpts, archive.WithPlatform(platforms.Ordered(all...)))
		} else {
			exportOpts = append(exportOpts, archive.WithPlatform(platforms.Default()))
		}

		if context.Bool("all-platforms") {
			exportOpts = append(exportOpts, archive.WithAllPlatforms())
		}

		if context.Bool("skip-manifest-json") {
			exportOpts = append(exportOpts, archive.WithSkipDockerManifest())
		}

		if context.Bool("skip-non-distributable") {
			exportOpts = append(exportOpts, archive.WithSkipNonDistributableBlobs())
		}

		client, ctx, cancel, err := commands.NewClient(context)
		if err != nil {
			return err
		}
		defer cancel()

		is := client.ImageService()
		for _, img := range images {
			exportOpts = append(exportOpts, archive.WithImage(is, img))
		}

		var w io.WriteCloser
		if out == "-" {
			w = os.Stdout
		} else {
			w, err = os.Create(out)
			if err != nil {
				return err
			}
		}
		defer w.Close()

		return client.Export(ctx, w, exportOpts...)
	},
}
<commit_msg>ctr export strictly match default platform<commit_after>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package images

import (
	"errors"
	"fmt"
	"io"
	"os"

	"github.com/containerd/containerd/cmd/ctr/commands"
	"github.com/containerd/containerd/images/archive"
	"github.com/containerd/containerd/platforms"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/urfave/cli"
)

var exportCommand = cli.Command{
	Name:      "export",
	Usage:     "export images",
	ArgsUsage: "[flags] <out> <image> ...",
	Description: `Export images to an OCI tar archive.

Tar output is formatted as an OCI archive, a Docker manifest is provided for the platform.
Use '--skip-manifest-json' to avoid including the Docker manifest.json file.
Use '--platform' to define the output platform.
When '--all-platforms' is given all images in a manifest list must be available.
`,
	Flags: []cli.Flag{
		cli.BoolFlag{
			Name:  "skip-manifest-json",
			Usage: "do not add Docker compatible manifest.json to archive",
		},
		cli.BoolFlag{
			Name:  "skip-non-distributable",
			Usage: "do not add non-distributable blobs such as Windows layers to archive",
		},
		cli.StringSliceFlag{
			Name:  "platform",
			Usage: "Pull content from a specific platform",
			Value: &cli.StringSlice{},
		},
		cli.BoolFlag{
			Name:  "all-platforms",
			Usage: "exports content from all platforms",
		},
	},
	Action: func(context *cli.Context) error {
		var (
			out        = context.Args().First()
			images     = context.Args().Tail()
			exportOpts = []archive.ExportOpt{}
		)
		if out == "" || len(images) == 0 {
			return errors.New("please provide both an output filename and an image reference to export")
		}

		if pss := context.StringSlice("platform"); len(pss) > 0 {
			var all []ocispec.Platform
			for _, ps := range pss {
				p, err := platforms.Parse(ps)
				if err != nil {
					return fmt.Errorf("invalid platform %q: %w", ps, err)
				}
				all = append(all, p)
			}
			exportOpts = append(exportOpts, archive.WithPlatform(platforms.Ordered(all...)))
		} else {
			exportOpts = append(exportOpts, archive.WithPlatform(platforms.DefaultStrict()))
		}

		if context.Bool("all-platforms") {
			exportOpts = append(exportOpts, archive.WithAllPlatforms())
		}

		if context.Bool("skip-manifest-json") {
			exportOpts = append(exportOpts, archive.WithSkipDockerManifest())
		}

		if context.Bool("skip-non-distributable") {
			exportOpts = append(exportOpts, archive.WithSkipNonDistributableBlobs())
		}

		client, ctx, cancel, err := commands.NewClient(context)
		if err != nil {
			return err
		}
		defer cancel()

		is := client.ImageService()
		for _, img := range images {
			exportOpts = append(exportOpts, archive.WithImage(is, img))
		}

		var w io.WriteCloser
		if out == "-" {
			w = os.Stdout
		} else {
			w, err = os.Create(out)
			if err != nil {
				return err
			}
		}
		defer w.Close()

		return client.Export(ctx, w, exportOpts...)
	},
}
<|endoftext|>