<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build windows

package winfsnotify

import (
	"os"
	"testing"
	"time"
)

func expect(t *testing.T, eventstream <-chan *Event, name string, mask uint32) {
	t.Logf(`expected: "%s": 0x%x`, name, mask)
	select {
	case event := <-eventstream:
		if event == nil {
			t.Fatal("nil event received")
		}
		t.Logf("received: %s", event)
		if event.Name != name || event.Mask != mask {
			t.Fatal("did not receive expected event")
		}
	case <-time.After(1 * time.Second):
		t.Fatal("timed out waiting for event")
	}
}

func TestNotifyEvents(t *testing.T) {
	watcher, err := NewWatcher()
	if err != nil {
		t.Fatalf("NewWatcher() failed: %s", err)
	}

	testDir := "TestNotifyEvents.testdirectory"
	testFile := testDir + "/TestNotifyEvents.testfile"
	testFile2 := testFile + ".new"
	const mask = FS_ALL_EVENTS & ^(FS_ATTRIB|FS_CLOSE) | FS_IGNORED

	// Add a watch for testDir
	os.RemoveAll(testDir)
	if err = os.Mkdir(testDir, 0777); err != nil {
		t.Fatalf("Failed to create test directory: %s", err)
	}
	defer os.RemoveAll(testDir)
	err = watcher.AddWatch(testDir, mask)
	if err != nil {
		t.Fatalf("Watcher.Watch() failed: %s", err)
	}

	// Receive errors on the error channel on a separate goroutine
	go func() {
		for err := range watcher.Error {
			t.Fatalf("error received: %s", err)
		}
	}()

	// Create a file
	file, err := os.Create(testFile)
	if err != nil {
		t.Fatalf("creating test file failed: %s", err)
	}
	expect(t, watcher.Event, testFile, FS_CREATE)

	err = watcher.AddWatch(testFile, mask)
	if err != nil {
		t.Fatalf("Watcher.Watch() failed: %s", err)
	}

	if _, err = file.WriteString("hello, world"); err != nil {
		t.Fatalf("failed to write to test file: %s", err)
	}
	if err = file.Close(); err != nil {
		t.Fatalf("failed to close test file: %s", err)
	}
	expect(t, watcher.Event, testFile, FS_MODIFY)
	expect(t, watcher.Event, testFile, FS_MODIFY)

	if err = os.Rename(testFile, testFile2); err != nil {
		t.Fatalf("failed to rename test file: %s", err)
	}
	expect(t, watcher.Event, testFile, FS_MOVED_FROM)
	expect(t, watcher.Event, testFile2, FS_MOVED_TO)
	expect(t, watcher.Event, testFile, FS_MOVE_SELF)

	if err = os.RemoveAll(testDir); err != nil {
		t.Fatalf("failed to remove test directory: %s", err)
	}
	expect(t, watcher.Event, testFile2, FS_DELETE_SELF)
	expect(t, watcher.Event, testFile2, FS_IGNORED)
	expect(t, watcher.Event, testFile2, FS_DELETE)
	expect(t, watcher.Event, testDir, FS_DELETE_SELF)
	expect(t, watcher.Event, testDir, FS_IGNORED)

	t.Log("calling Close()")
	if err = watcher.Close(); err != nil {
		t.Fatalf("failed to close watcher: %s", err)
	}
}

func TestNotifyClose(t *testing.T) {
	watcher, _ := NewWatcher()
	watcher.Close()

	done := false
	go func() {
		watcher.Close()
		done = true
	}()

	time.Sleep(50 * time.Millisecond)
	if !done {
		t.Fatal("double Close() test failed: second Close() call didn't return")
	}

	err := watcher.Watch("_test")
	if err == nil {
		t.Fatal("expected error on Watch() after Close(), got nil")
	}
}
<commit_msg>exp/winfsnotify: remove reference to _test Updates issue 2573.<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build windows

package winfsnotify

import (
	"os"
	"testing"
	"time"
)

func expect(t *testing.T, eventstream <-chan *Event, name string, mask uint32) {
	t.Logf(`expected: "%s": 0x%x`, name, mask)
	select {
	case event := <-eventstream:
		if event == nil {
			t.Fatal("nil event received")
		}
		t.Logf("received: %s", event)
		if event.Name != name || event.Mask != mask {
			t.Fatal("did not receive expected event")
		}
	case <-time.After(1 * time.Second):
		t.Fatal("timed out waiting for event")
	}
}

func TestNotifyEvents(t *testing.T) {
	watcher, err := NewWatcher()
	if err != nil {
		t.Fatalf("NewWatcher() failed: %s", err)
	}

	testDir := "TestNotifyEvents.testdirectory"
	testFile := testDir + "/TestNotifyEvents.testfile"
	testFile2 := testFile + ".new"
	const mask = FS_ALL_EVENTS & ^(FS_ATTRIB|FS_CLOSE) | FS_IGNORED

	// Add a watch for testDir
	os.RemoveAll(testDir)
	if err = os.Mkdir(testDir, 0777); err != nil {
		t.Fatalf("Failed to create test directory: %s", err)
	}
	defer os.RemoveAll(testDir)
	err = watcher.AddWatch(testDir, mask)
	if err != nil {
		t.Fatalf("Watcher.Watch() failed: %s", err)
	}

	// Receive errors on the error channel on a separate goroutine
	go func() {
		for err := range watcher.Error {
			t.Fatalf("error received: %s", err)
		}
	}()

	// Create a file
	file, err := os.Create(testFile)
	if err != nil {
		t.Fatalf("creating test file failed: %s", err)
	}
	expect(t, watcher.Event, testFile, FS_CREATE)

	err = watcher.AddWatch(testFile, mask)
	if err != nil {
		t.Fatalf("Watcher.Watch() failed: %s", err)
	}

	if _, err = file.WriteString("hello, world"); err != nil {
		t.Fatalf("failed to write to test file: %s", err)
	}
	if err = file.Close(); err != nil {
		t.Fatalf("failed to close test file: %s", err)
	}
	expect(t, watcher.Event, testFile, FS_MODIFY)
	expect(t, watcher.Event, testFile, FS_MODIFY)

	if err = os.Rename(testFile, testFile2); err != nil {
		t.Fatalf("failed to rename test file: %s", err)
	}
	expect(t, watcher.Event, testFile, FS_MOVED_FROM)
	expect(t, watcher.Event, testFile2, FS_MOVED_TO)
	expect(t, watcher.Event, testFile, FS_MOVE_SELF)

	if err = os.RemoveAll(testDir); err != nil {
		t.Fatalf("failed to remove test directory: %s", err)
	}
	expect(t, watcher.Event, testFile2, FS_DELETE_SELF)
	expect(t, watcher.Event, testFile2, FS_IGNORED)
	expect(t, watcher.Event, testFile2, FS_DELETE)
	expect(t, watcher.Event, testDir, FS_DELETE_SELF)
	expect(t, watcher.Event, testDir, FS_IGNORED)

	t.Log("calling Close()")
	if err = watcher.Close(); err != nil {
		t.Fatalf("failed to close watcher: %s", err)
	}
}

func TestNotifyClose(t *testing.T) {
	watcher, _ := NewWatcher()
	watcher.Close()

	done := false
	go func() {
		watcher.Close()
		done = true
	}()

	time.Sleep(50 * time.Millisecond)
	if !done {
		t.Fatal("double Close() test failed: second Close() call didn't return")
	}

	dir, err := ioutil.TempDir("", "wininotify")
	if err != nil {
		t.Fatalf("TempDir failed: %s", err)
	}
	defer os.RemoveAll(dir)

	err = watcher.Watch(dir)
	if err == nil {
		t.Fatal("expected error on Watch() after Close(), got nil")
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package filters

import (
	"context"
	"fmt"
	"net/http"
	"sync/atomic"

	flowcontrol "k8s.io/api/flowcontrol/v1beta1"
	apitypes "k8s.io/apimachinery/pkg/types"
	epmetrics "k8s.io/apiserver/pkg/endpoints/metrics"
	apirequest "k8s.io/apiserver/pkg/endpoints/request"
	utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
	fcmetrics "k8s.io/apiserver/pkg/util/flowcontrol/metrics"
	"k8s.io/klog/v2"
)

type priorityAndFairnessKeyType int

const priorityAndFairnessKey priorityAndFairnessKeyType = iota

// PriorityAndFairnessClassification identifies the results of
// classification for API Priority and Fairness
type PriorityAndFairnessClassification struct {
	FlowSchemaName    string
	FlowSchemaUID     apitypes.UID
	PriorityLevelName string
	PriorityLevelUID  apitypes.UID
}

// GetClassification returns the classification associated with the
// given context, if any, otherwise nil
func GetClassification(ctx context.Context) *PriorityAndFairnessClassification {
	return ctx.Value(priorityAndFairnessKey).(*PriorityAndFairnessClassification)
}

// waitingMark tracks requests waiting rather than being executed
var waitingMark = &requestWatermark{
	phase:            epmetrics.WaitingPhase,
	readOnlyObserver: fcmetrics.ReadWriteConcurrencyObserverPairGenerator.Generate(1, 1, []string{epmetrics.ReadOnlyKind}).RequestsWaiting,
	mutatingObserver: fcmetrics.ReadWriteConcurrencyObserverPairGenerator.Generate(1, 1, []string{epmetrics.MutatingKind}).RequestsWaiting,
}

var atomicMutatingExecuting, atomicReadOnlyExecuting int32
var atomicMutatingWaiting, atomicReadOnlyWaiting int32

// WithPriorityAndFairness limits the number of in-flight
// requests in a fine-grained way.
func WithPriorityAndFairness(
	handler http.Handler,
	longRunningRequestCheck apirequest.LongRunningRequestCheck,
	fcIfc utilflowcontrol.Interface,
) http.Handler {
	if fcIfc == nil {
		klog.Warningf("priority and fairness support not found, skipping")
		return handler
	}
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		ctx := r.Context()
		requestInfo, ok := apirequest.RequestInfoFrom(ctx)
		if !ok {
			handleError(w, r, fmt.Errorf("no RequestInfo found in context"))
			return
		}
		user, ok := apirequest.UserFrom(ctx)
		if !ok {
			handleError(w, r, fmt.Errorf("no User found in context"))
			return
		}

		// Skip tracking long running requests.
		if longRunningRequestCheck != nil && longRunningRequestCheck(r, requestInfo) {
			klog.V(6).Infof("Serving RequestInfo=%#+v, user.Info=%#+v as longrunning\n", requestInfo, user)
			handler.ServeHTTP(w, r)
			return
		}

		var classification *PriorityAndFairnessClassification
		note := func(fs *flowcontrol.FlowSchema, pl *flowcontrol.PriorityLevelConfiguration) {
			classification = &PriorityAndFairnessClassification{
				FlowSchemaName:    fs.Name,
				FlowSchemaUID:     fs.UID,
				PriorityLevelName: pl.Name,
				PriorityLevelUID:  pl.UID}
		}

		var served bool
		isMutatingRequest := !nonMutatingRequestVerbs.Has(requestInfo.Verb)
		noteExecutingDelta := func(delta int32) {
			if isMutatingRequest {
				watermark.recordMutating(int(atomic.AddInt32(&atomicMutatingExecuting, delta)))
			} else {
				watermark.recordReadOnly(int(atomic.AddInt32(&atomicReadOnlyExecuting, delta)))
			}
		}
		noteWaitingDelta := func(delta int32) {
			if isMutatingRequest {
				waitingMark.recordMutating(int(atomic.AddInt32(&atomicMutatingWaiting, delta)))
			} else {
				waitingMark.recordReadOnly(int(atomic.AddInt32(&atomicReadOnlyWaiting, delta)))
			}
		}
		execute := func() {
			noteExecutingDelta(1)
			defer noteExecutingDelta(-1)
			served = true
			innerCtx := context.WithValue(ctx, priorityAndFairnessKey, classification)
			innerReq := r.Clone(innerCtx)

			// We intentionally set the UID of the flow-schema and priority-level instead of name. This is so that
			// the names that cluster-admins choose for categorization and priority levels are not exposed, also
			// the names might make it obvious to the users that they are rejected due to classification with low priority.
			w.Header().Set(flowcontrol.ResponseHeaderMatchedPriorityLevelConfigurationUID, string(classification.PriorityLevelUID))
			w.Header().Set(flowcontrol.ResponseHeaderMatchedFlowSchemaUID, string(classification.FlowSchemaUID))

			handler.ServeHTTP(w, innerReq)
		}
		digest := utilflowcontrol.RequestDigest{RequestInfo: requestInfo, User: user}
		fcIfc.Handle(ctx, digest, note, func(inQueue bool) {
			if inQueue {
				noteWaitingDelta(1)
			} else {
				noteWaitingDelta(-1)
			}
		}, execute)
		if !served {
			if isMutatingRequest {
				epmetrics.DroppedRequests.WithContext(ctx).WithLabelValues(epmetrics.MutatingKind).Inc()
			} else {
				epmetrics.DroppedRequests.WithContext(ctx).WithLabelValues(epmetrics.ReadOnlyKind).Inc()
			}
			epmetrics.RecordRequestTermination(r, requestInfo, epmetrics.APIServerComponent, http.StatusTooManyRequests)
			tooManyRequests(r, w)
		}

	})
}

// StartPriorityAndFairnessWatermarkMaintenance starts the goroutines to observe and maintain watermarks for
// priority-and-fairness requests.
func StartPriorityAndFairnessWatermarkMaintenance(stopCh <-chan struct{}) {
	startWatermarkMaintenance(watermark, stopCh)
	startWatermarkMaintenance(waitingMark, stopCh)
}
<commit_msg>apf: set response headers for rejected requests<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package filters

import (
	"context"
	"fmt"
	"net/http"
	"sync/atomic"

	flowcontrol "k8s.io/api/flowcontrol/v1beta1"
	apitypes "k8s.io/apimachinery/pkg/types"
	epmetrics "k8s.io/apiserver/pkg/endpoints/metrics"
	apirequest "k8s.io/apiserver/pkg/endpoints/request"
	utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
	fcmetrics "k8s.io/apiserver/pkg/util/flowcontrol/metrics"
	"k8s.io/klog/v2"
)

type priorityAndFairnessKeyType int

const priorityAndFairnessKey priorityAndFairnessKeyType = iota

// PriorityAndFairnessClassification identifies the results of
// classification for API Priority and Fairness
type PriorityAndFairnessClassification struct {
	FlowSchemaName    string
	FlowSchemaUID     apitypes.UID
	PriorityLevelName string
	PriorityLevelUID  apitypes.UID
}

// GetClassification returns the classification associated with the
// given context, if any, otherwise nil
func GetClassification(ctx context.Context) *PriorityAndFairnessClassification {
	return ctx.Value(priorityAndFairnessKey).(*PriorityAndFairnessClassification)
}

// waitingMark tracks requests waiting rather than being executed
var waitingMark = &requestWatermark{
	phase:            epmetrics.WaitingPhase,
	readOnlyObserver: fcmetrics.ReadWriteConcurrencyObserverPairGenerator.Generate(1, 1, []string{epmetrics.ReadOnlyKind}).RequestsWaiting,
	mutatingObserver: fcmetrics.ReadWriteConcurrencyObserverPairGenerator.Generate(1, 1, []string{epmetrics.MutatingKind}).RequestsWaiting,
}

var atomicMutatingExecuting, atomicReadOnlyExecuting int32
var atomicMutatingWaiting, atomicReadOnlyWaiting int32

// WithPriorityAndFairness limits the number of in-flight
// requests in a fine-grained way.
func WithPriorityAndFairness(
	handler http.Handler,
	longRunningRequestCheck apirequest.LongRunningRequestCheck,
	fcIfc utilflowcontrol.Interface,
) http.Handler {
	if fcIfc == nil {
		klog.Warningf("priority and fairness support not found, skipping")
		return handler
	}
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		ctx := r.Context()
		requestInfo, ok := apirequest.RequestInfoFrom(ctx)
		if !ok {
			handleError(w, r, fmt.Errorf("no RequestInfo found in context"))
			return
		}
		user, ok := apirequest.UserFrom(ctx)
		if !ok {
			handleError(w, r, fmt.Errorf("no User found in context"))
			return
		}

		// Skip tracking long running requests.
		if longRunningRequestCheck != nil && longRunningRequestCheck(r, requestInfo) {
			klog.V(6).Infof("Serving RequestInfo=%#+v, user.Info=%#+v as longrunning\n", requestInfo, user)
			handler.ServeHTTP(w, r)
			return
		}

		var classification *PriorityAndFairnessClassification
		note := func(fs *flowcontrol.FlowSchema, pl *flowcontrol.PriorityLevelConfiguration) {
			classification = &PriorityAndFairnessClassification{
				FlowSchemaName:    fs.Name,
				FlowSchemaUID:     fs.UID,
				PriorityLevelName: pl.Name,
				PriorityLevelUID:  pl.UID}
		}

		var served bool
		isMutatingRequest := !nonMutatingRequestVerbs.Has(requestInfo.Verb)
		noteExecutingDelta := func(delta int32) {
			if isMutatingRequest {
				watermark.recordMutating(int(atomic.AddInt32(&atomicMutatingExecuting, delta)))
			} else {
				watermark.recordReadOnly(int(atomic.AddInt32(&atomicReadOnlyExecuting, delta)))
			}
		}
		noteWaitingDelta := func(delta int32) {
			if isMutatingRequest {
				waitingMark.recordMutating(int(atomic.AddInt32(&atomicMutatingWaiting, delta)))
			} else {
				waitingMark.recordReadOnly(int(atomic.AddInt32(&atomicReadOnlyWaiting, delta)))
			}
		}
		execute := func() {
			noteExecutingDelta(1)
			defer noteExecutingDelta(-1)
			served = true
			innerCtx := context.WithValue(ctx, priorityAndFairnessKey, classification)
			innerReq := r.Clone(innerCtx)
			setResponseHeaders(classification, w)

			handler.ServeHTTP(w, innerReq)
		}
		digest := utilflowcontrol.RequestDigest{RequestInfo: requestInfo, User: user}
		fcIfc.Handle(ctx, digest, note, func(inQueue bool) {
			if inQueue {
				noteWaitingDelta(1)
			} else {
				noteWaitingDelta(-1)
			}
		}, execute)
		if !served {
			setResponseHeaders(classification, w)

			if isMutatingRequest {
				epmetrics.DroppedRequests.WithContext(ctx).WithLabelValues(epmetrics.MutatingKind).Inc()
			} else {
				epmetrics.DroppedRequests.WithContext(ctx).WithLabelValues(epmetrics.ReadOnlyKind).Inc()
			}
			epmetrics.RecordRequestTermination(r, requestInfo, epmetrics.APIServerComponent, http.StatusTooManyRequests)
			tooManyRequests(r, w)
		}

	})
}

// StartPriorityAndFairnessWatermarkMaintenance starts the goroutines to observe and maintain watermarks for
// priority-and-fairness requests.
func StartPriorityAndFairnessWatermarkMaintenance(stopCh <-chan struct{}) {
	startWatermarkMaintenance(watermark, stopCh)
	startWatermarkMaintenance(waitingMark, stopCh)
}

func setResponseHeaders(classification *PriorityAndFairnessClassification, w http.ResponseWriter) {
	if classification == nil {
		return
	}

	// We intentionally set the UID of the flow-schema and priority-level instead of name. This is so that
	// the names that cluster-admins choose for categorization and priority levels are not exposed, also
	// the names might make it obvious to the users that they are rejected due to classification with low priority.
	w.Header().Set(flowcontrol.ResponseHeaderMatchedPriorityLevelConfigurationUID, string(classification.PriorityLevelUID))
	w.Header().Set(flowcontrol.ResponseHeaderMatchedFlowSchemaUID, string(classification.FlowSchemaUID))
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package factory

import (
	"context"
	"fmt"
	"sync/atomic"
	"time"

	"github.com/coreos/etcd/clientv3"
	"github.com/coreos/etcd/pkg/transport"

	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/apiserver/pkg/storage"
	"k8s.io/apiserver/pkg/storage/etcd3"
	"k8s.io/apiserver/pkg/storage/storagebackend"
	"k8s.io/apiserver/pkg/storage/value"
)

var (
	// The short keepalive timeout and interval have been chosen to aggressively
	// detect a failed etcd server without introducing much overhead.
	keepaliveTime    = 30 * time.Second
	keepaliveTimeout = 10 * time.Second
	// dialTimeout is the timeout for failing to establish a connection.
	dialTimeout = 10 * time.Second
)

func newETCD3HealthCheck(c storagebackend.Config) (func() error, error) {
	// constructing the etcd v3 client blocks and times out if etcd is not available.
	// retry in a loop in the background until we successfully create the client, storing the client or error encountered

	clientValue := &atomic.Value{}

	clientErrMsg := &atomic.Value{}
	clientErrMsg.Store("etcd client connection not yet established")

	go wait.PollUntil(time.Second, func() (bool, error) {
		client, err := newETCD3Client(c)
		if err != nil {
			clientErrMsg.Store(err.Error())
			return false, nil
		}
		clientValue.Store(client)
		clientErrMsg.Store("")
		return true, nil
	}, wait.NeverStop)

	return func() error {
		if errMsg := clientErrMsg.Load().(string); len(errMsg) > 0 {
			return fmt.Errorf(errMsg)
		}
		client := clientValue.Load().(*clientv3.Client)
		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
		defer cancel()
		if _, err := client.Cluster.MemberList(ctx); err != nil {
			return fmt.Errorf("error listing etcd members: %v", err)
		}
		return nil
	}, nil
}

func newETCD3Client(c storagebackend.Config) (*clientv3.Client, error) {
	tlsInfo := transport.TLSInfo{
		CertFile: c.CertFile,
		KeyFile:  c.KeyFile,
		CAFile:   c.CAFile,
	}
	tlsConfig, err := tlsInfo.ClientConfig()
	if err != nil {
		return nil, err
	}
	// NOTE: Client relies on nil tlsConfig
	// for non-secure connections, update the implicit variable
	if len(c.CertFile) == 0 && len(c.KeyFile) == 0 && len(c.CAFile) == 0 {
		tlsConfig = nil
	}
	cfg := clientv3.Config{
		DialTimeout:          dialTimeout,
		DialKeepAliveTime:    keepaliveTime,
		DialKeepAliveTimeout: keepaliveTimeout,
		Endpoints:            c.ServerList,
		TLS:                  tlsConfig,
	}
	client, err := clientv3.New(cfg)
	return client, err
}

func newETCD3Storage(c storagebackend.Config) (storage.Interface, DestroyFunc, error) {
	client, err := newETCD3Client(c)
	if err != nil {
		return nil, nil, err
	}
	ctx, cancel := context.WithCancel(context.Background())
	etcd3.StartCompactor(ctx, client, c.CompactionInterval)
	destroyFunc := func() {
		cancel()
		client.Close()
	}
	transformer := c.Transformer
	if transformer == nil {
		transformer = value.IdentityTransformer
	}
	if c.Quorum {
		return etcd3.New(client, c.Codec, c.Prefix, transformer, c.Paging), destroyFunc, nil
	}
	return etcd3.NewWithNoQuorumRead(client, c.Codec, c.Prefix, transformer, c.Paging), destroyFunc, nil
}
<commit_msg>enable etcdv3 client prometheus metics<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package factory

import (
	"context"
	"fmt"
	"sync/atomic"
	"time"

	"github.com/coreos/etcd/clientv3"
	"github.com/coreos/etcd/pkg/transport"
	grpcprom "github.com/grpc-ecosystem/go-grpc-prometheus"
	"google.golang.org/grpc"

	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/apiserver/pkg/storage"
	"k8s.io/apiserver/pkg/storage/etcd3"
	"k8s.io/apiserver/pkg/storage/storagebackend"
	"k8s.io/apiserver/pkg/storage/value"
)

var (
	// The short keepalive timeout and interval have been chosen to aggressively
	// detect a failed etcd server without introducing much overhead.
	keepaliveTime    = 30 * time.Second
	keepaliveTimeout = 10 * time.Second
	// dialTimeout is the timeout for failing to establish a connection.
	dialTimeout = 10 * time.Second
)

func newETCD3HealthCheck(c storagebackend.Config) (func() error, error) {
	// constructing the etcd v3 client blocks and times out if etcd is not available.
	// retry in a loop in the background until we successfully create the client, storing the client or error encountered

	clientValue := &atomic.Value{}

	clientErrMsg := &atomic.Value{}
	clientErrMsg.Store("etcd client connection not yet established")

	go wait.PollUntil(time.Second, func() (bool, error) {
		client, err := newETCD3Client(c)
		if err != nil {
			clientErrMsg.Store(err.Error())
			return false, nil
		}
		clientValue.Store(client)
		clientErrMsg.Store("")
		return true, nil
	}, wait.NeverStop)

	return func() error {
		if errMsg := clientErrMsg.Load().(string); len(errMsg) > 0 {
			return fmt.Errorf(errMsg)
		}
		client := clientValue.Load().(*clientv3.Client)
		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
		defer cancel()
		if _, err := client.Cluster.MemberList(ctx); err != nil {
			return fmt.Errorf("error listing etcd members: %v", err)
		}
		return nil
	}, nil
}

func newETCD3Client(c storagebackend.Config) (*clientv3.Client, error) {
	tlsInfo := transport.TLSInfo{
		CertFile: c.CertFile,
		KeyFile:  c.KeyFile,
		CAFile:   c.CAFile,
	}
	tlsConfig, err := tlsInfo.ClientConfig()
	if err != nil {
		return nil, err
	}
	// NOTE: Client relies on nil tlsConfig
	// for non-secure connections, update the implicit variable
	if len(c.CertFile) == 0 && len(c.KeyFile) == 0 && len(c.CAFile) == 0 {
		tlsConfig = nil
	}
	cfg := clientv3.Config{
		DialTimeout:          dialTimeout,
		DialKeepAliveTime:    keepaliveTime,
		DialKeepAliveTimeout: keepaliveTimeout,
		DialOptions: []grpc.DialOption{
			grpc.WithUnaryInterceptor(grpcprom.UnaryClientInterceptor),
			grpc.WithStreamInterceptor(grpcprom.StreamClientInterceptor),
		},
		Endpoints: c.ServerList,
		TLS:       tlsConfig,
	}
	client, err := clientv3.New(cfg)
	return client, err
}

func newETCD3Storage(c storagebackend.Config) (storage.Interface, DestroyFunc, error) {
	client, err := newETCD3Client(c)
	if err != nil {
		return nil, nil, err
	}
	ctx, cancel := context.WithCancel(context.Background())
	etcd3.StartCompactor(ctx, client, c.CompactionInterval)
	destroyFunc := func() {
		cancel()
		client.Close()
	}
	transformer := c.Transformer
	if transformer == nil {
		transformer = value.IdentityTransformer
	}
	if c.Quorum {
		return etcd3.New(client, c.Codec, c.Prefix, transformer, c.Paging), destroyFunc, nil
	}
	return etcd3.NewWithNoQuorumRead(client, c.Codec, c.Prefix, transformer, c.Paging), destroyFunc, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"regexp"
	"testing"
	"time"
)

// Test that useful common cache-related parameters are sent to the
// client by this CDN provider.

// Should set an Age header, when origin doesn't provide one, representing
// how long the object has been in edge's cache.
func TestRespHeaderAgeFromEdge(t *testing.T) {
	ResetBackends(backendsByPriority)

	const secondsToWaitBetweenRequests = 5
	expectedHeaderVals := []string{
		"0",
		fmt.Sprintf("%d", secondsToWaitBetweenRequests),
	}

	req := NewUniqueEdgeGET(t)

	for requestCount, expectedHeaderVal := range expectedHeaderVals {
		requestCount = requestCount + 1
		switch requestCount {
		case 1:
			originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
				w.Header().Set("Cache-Control", "max-age=1800, public")
				w.Write([]byte("cacheable request"))
			})
		case 2:
			originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
				t.Error("Origin received request and it shouldn't have")
			})

			// Wait for Age to increment.
			time.Sleep(time.Duration(secondsToWaitBetweenRequests) * time.Second)
		}

		resp := RoundTripCheckError(t, req)
		defer resp.Body.Close()

		if resp.StatusCode != http.StatusOK {
			t.Fatalf("Request %d received incorrect status %q", requestCount, resp.Status)
		}

		if val := resp.Header.Get("Age"); val != expectedHeaderVal {
			t.Errorf(
				"Request %d received incorrect Age header. Got %q, expected %q",
				requestCount,
				val,
				expectedHeaderVal,
			)
		}
	}
}

// Should propagate an Age header from origin and then increment it for the
// time it is in edge's cache. This assumes no request/response delay:
// http://tools.ietf.org/html/rfc7234#section-4.2.3
func TestRespHeaderAgeFromOrigin(t *testing.T) {
	ResetBackends(backendsByPriority)

	const originAgeInSeconds = 100
	const secondsToWaitBetweenRequests = 5
	expectedHeaderVals := []string{
		fmt.Sprintf("%d", originAgeInSeconds),
		fmt.Sprintf("%d", originAgeInSeconds + secondsToWaitBetweenRequests),
	}

	req := NewUniqueEdgeGET(t)

	for requestCount, expectedHeaderVal := range expectedHeaderVals {
		requestCount = requestCount + 1
		switch requestCount {
		case 1:
			originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
				w.Header().Set("Age", expectedHeaderVal)
				w.Header().Set("Cache-Control", "max-age=1800, public")
				w.Write([]byte("cacheable request"))
			})
		case 2:
			originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
				t.Error("Origin received request and it shouldn't have")
			})

			// Wait for Age to increment.
			time.Sleep(time.Duration(secondsToWaitBetweenRequests) * time.Second)
		}

		resp := RoundTripCheckError(t, req)
		defer resp.Body.Close()

		if resp.StatusCode != http.StatusOK {
			t.Fatalf("Request %d received incorrect status %q", requestCount, resp.Status)
		}

		if val := resp.Header.Get("Age"); val != expectedHeaderVal {
			t.Errorf(
				"Request %d received incorrect Age header. Got %q, expected %q",
				requestCount,
				val,
				expectedHeaderVal,
			)
		}
	}
}

// Should set an X-Cache header containing HIT/MISS from 'origin, itself'
func TestRespHeaderXCacheAppend(t *testing.T) {
	ResetBackends(backendsByPriority)

	if vendorCloudflare {
		t.Skip(notSupportedByVendor)
	}

	const originXCache = "HIT"

	var (
		xCache         string
		expectedXCache string
	)

	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("X-Cache", originXCache)
	})

	// Get first request, will come from origin, cannot be cached - hence cache MISS
	req := NewUniqueEdgeGET(t)
	resp := RoundTripCheckError(t, req)
	defer resp.Body.Close()

	xCache = resp.Header.Get("X-Cache")
	expectedXCache = fmt.Sprintf("%s, MISS", originXCache)
	if xCache != expectedXCache {
		t.Errorf(
			"X-Cache on initial hit is wrong: expected %q, got %q",
			expectedXCache,
			xCache,
		)
	}

}

// Should set a header containing 'HIT' or 'MISS' depending on whether request is cached
func TestRespHeaderCacheHitMiss(t *testing.T) {
	ResetBackends(backendsByPriority)

	var (
		headerName  string
		headerValue string
	)

	switch {
	case vendorCloudflare:
		headerName = "CF-Cache-Status"
	case vendorFastly:
		headerName = "X-Cache"
	default:
		t.Fatal(notImplementedForVendor)
	}

	expectedHeaderValues := []string{"MISS", "HIT"}
	const cacheDuration = time.Second

	if vendorCloudflare {
		cloudFlareStatuses := []string{"EXPIRED", "HIT"}
		expectedHeaderValues = append(expectedHeaderValues, cloudFlareStatuses...)
	}

	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		cacheControlValue := fmt.Sprintf("max-age=%.0f", cacheDuration.Seconds())
		w.Header().Set("Cache-Control", cacheControlValue)
	})

	req := NewUniqueEdgeGET(t)

	for count, expectedValue := range expectedHeaderValues {

		if expectedValue == "EXPIRED" {
			// sleep long enough for object to have expired
			sleepDuration := cacheDuration + time.Second
			time.Sleep(sleepDuration)
		}

		resp := RoundTripCheckError(t, req)
		defer resp.Body.Close()

		headerValue = resp.Header.Get(headerName)

		if headerValue != expectedValue {
			t.Errorf(
				"%s on request %d is wrong: expected %q, got %q",
				headerName,
				count+1,
				expectedValue,
				headerValue,
			)
		}
	}
}

// Should set an 'Served-By' header giving information on the edge node and location served from.
func TestRespHeaderServedBy(t *testing.T) {
	ResetBackends(backendsByPriority)

	var expectedServedByRegexp *regexp.Regexp
	var headerName string

	switch {
	case vendorCloudflare:
		headerName = "CF-RAY"
		expectedServedByRegexp = regexp.MustCompile("^[a-z0-9]{16}-[A-Z]{3}$")
	case vendorFastly:
		headerName = "X-Served-By"
		expectedServedByRegexp = regexp.MustCompile("^cache-[a-z0-9]+-[A-Z]{3}$")
	default:
		t.Fatal(notImplementedForVendor)
	}

	req := NewUniqueEdgeGET(t)
	resp := RoundTripCheckError(t, req)
	defer resp.Body.Close()

	actualHeader := resp.Header.Get(headerName)

	if actualHeader == "" {
		t.Error(headerName + " header has not been set by Edge")
	}

	if expectedServedByRegexp.FindString(actualHeader) != actualHeader {
		t.Errorf("%s is not as expected: got %q", headerName, actualHeader)
	}

}

// Should set an X-Cache-Hits header containing hit count for this object,
// from the Edge AND the Origin, assuming Origin sets one.
// This is in the format "{origin-hit-count}, {edge-hit-count}"
func TestRespHeaderXCacheHitsAppend(t *testing.T) {
	ResetBackends(backendsByPriority)

	if vendorCloudflare {
		t.Skip(notSupportedByVendor)
	}

	const originXCacheHits = "53"

	var (
		xCacheHits         string
		expectedXCacheHits string
	)

	uuid := NewUUID()

	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "GET" && r.URL.Path == fmt.Sprintf("/%s", uuid) {
			w.Header().Set("X-Cache-Hits", originXCacheHits)
		}
	})

	sourceURL := fmt.Sprintf("https://%s/%s", *edgeHost, uuid)

	// Get first request, will come from origin. Edge Hit Count 0
	req, _ := http.NewRequest("GET", sourceURL, nil)
	resp := RoundTripCheckError(t, req)
	defer resp.Body.Close()

	xCacheHits = resp.Header.Get("X-Cache-Hits")
	expectedXCacheHits = fmt.Sprintf("%s, 0", originXCacheHits)
	if xCacheHits != expectedXCacheHits {
		t.Errorf(
			"X-Cache-Hits on initial hit is wrong: expected %q, got %q",
			expectedXCacheHits,
			xCacheHits,
		)
	}

	// Get request again. Should come from Edge now, hit count 1
	resp = RoundTripCheckError(t, req)
	defer resp.Body.Close()

	xCacheHits = resp.Header.Get("X-Cache-Hits")
	expectedXCacheHits = fmt.Sprintf("%s, 1", originXCacheHits)
	if xCacheHits != expectedXCacheHits {
		t.Errorf(
			"X-Cache-Hits on second hit is wrong: expected %q, got %q",
			expectedXCacheHits,
			xCacheHits,
		)
	}
}
<commit_msg>Run go fmt on cdn_resp_headers_test.go<commit_after>package main

import (
	"fmt"
	"net/http"
	"regexp"
	"testing"
	"time"
)

// Test that useful common cache-related parameters are sent to the
// client by this CDN provider.

// Should set an Age header, when origin doesn't provide one, representing
// how long the object has been in edge's cache.
func TestRespHeaderAgeFromEdge(t *testing.T) {
	ResetBackends(backendsByPriority)

	const secondsToWaitBetweenRequests = 5
	expectedHeaderVals := []string{
		"0",
		fmt.Sprintf("%d", secondsToWaitBetweenRequests),
	}

	req := NewUniqueEdgeGET(t)

	for requestCount, expectedHeaderVal := range expectedHeaderVals {
		requestCount = requestCount + 1
		switch requestCount {
		case 1:
			originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
				w.Header().Set("Cache-Control", "max-age=1800, public")
				w.Write([]byte("cacheable request"))
			})
		case 2:
			originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
				t.Error("Origin received request and it shouldn't have")
			})

			// Wait for Age to increment.
			time.Sleep(time.Duration(secondsToWaitBetweenRequests) * time.Second)
		}

		resp := RoundTripCheckError(t, req)
		defer resp.Body.Close()

		if resp.StatusCode != http.StatusOK {
			t.Fatalf("Request %d received incorrect status %q", requestCount, resp.Status)
		}

		if val := resp.Header.Get("Age"); val != expectedHeaderVal {
			t.Errorf(
				"Request %d received incorrect Age header. Got %q, expected %q",
				requestCount,
				val,
				expectedHeaderVal,
			)
		}
	}
}

// Should propagate an Age header from origin and then increment it for the
// time it is in edge's cache. This assumes no request/response delay:
// http://tools.ietf.org/html/rfc7234#section-4.2.3
func TestRespHeaderAgeFromOrigin(t *testing.T) {
	ResetBackends(backendsByPriority)

	const originAgeInSeconds = 100
	const secondsToWaitBetweenRequests = 5
	expectedHeaderVals := []string{
		fmt.Sprintf("%d", originAgeInSeconds),
		fmt.Sprintf("%d", originAgeInSeconds+secondsToWaitBetweenRequests),
	}

	req := NewUniqueEdgeGET(t)

	for requestCount, expectedHeaderVal := range expectedHeaderVals {
		requestCount = requestCount + 1
		switch requestCount {
		case 1:
			originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
				w.Header().Set("Age", expectedHeaderVal)
				w.Header().Set("Cache-Control", "max-age=1800, public")
				w.Write([]byte("cacheable request"))
			})
		case 2:
			originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
				t.Error("Origin received request and it shouldn't have")
			})

			// Wait for Age to increment.
			time.Sleep(time.Duration(secondsToWaitBetweenRequests) * time.Second)
		}

		resp := RoundTripCheckError(t, req)
		defer resp.Body.Close()

		if resp.StatusCode != http.StatusOK {
			t.Fatalf("Request %d received incorrect status %q", requestCount, resp.Status)
		}

		if val := resp.Header.Get("Age"); val != expectedHeaderVal {
			t.Errorf(
				"Request %d received incorrect Age header. Got %q, expected %q",
				requestCount,
				val,
				expectedHeaderVal,
			)
		}
	}
}

// Should set an X-Cache header containing HIT/MISS from 'origin, itself'
func TestRespHeaderXCacheAppend(t *testing.T) {
	ResetBackends(backendsByPriority)

	if vendorCloudflare {
		t.Skip(notSupportedByVendor)
	}

	const originXCache = "HIT"

	var (
		xCache         string
		expectedXCache string
	)

	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("X-Cache", originXCache)
	})

	// Get first request, will come from origin, cannot be cached - hence cache MISS
	req := NewUniqueEdgeGET(t)
	resp := RoundTripCheckError(t, req)
	defer resp.Body.Close()

	xCache = resp.Header.Get("X-Cache")
	expectedXCache = fmt.Sprintf("%s, MISS", originXCache)
	if xCache != expectedXCache {
		t.Errorf(
			"X-Cache on initial hit is wrong: expected %q, got %q",
			expectedXCache,
			xCache,
		)
	}

}

// Should set a header containing 'HIT' or 'MISS' depending on whether request is cached
func TestRespHeaderCacheHitMiss(t *testing.T) {
	ResetBackends(backendsByPriority)

	var (
		headerName  string
		headerValue string
	)

	switch {
	case vendorCloudflare:
		headerName = "CF-Cache-Status"
	case vendorFastly:
		headerName = "X-Cache"
	default:
		t.Fatal(notImplementedForVendor)
	}

	expectedHeaderValues := []string{"MISS", "HIT"}
	const cacheDuration = time.Second

	if vendorCloudflare {
		cloudFlareStatuses := []string{"EXPIRED", "HIT"}
		expectedHeaderValues = append(expectedHeaderValues, cloudFlareStatuses...)
	}

	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		cacheControlValue := fmt.Sprintf("max-age=%.0f", cacheDuration.Seconds())
		w.Header().Set("Cache-Control", cacheControlValue)
	})

	req := NewUniqueEdgeGET(t)

	for count, expectedValue := range expectedHeaderValues {

		if expectedValue == "EXPIRED" {
			// sleep long enough for object to have expired
			sleepDuration := cacheDuration + time.Second
			time.Sleep(sleepDuration)
		}

		resp := RoundTripCheckError(t, req)
		defer resp.Body.Close()

		headerValue = resp.Header.Get(headerName)

		if headerValue != expectedValue {
			t.Errorf(
				"%s on request %d is wrong: expected %q, got %q",
				headerName,
				count+1,
				expectedValue,
				headerValue,
			)
		}
	}
}

// Should set an 'Served-By' header giving information on the edge node and location served from.
func TestRespHeaderServedBy(t *testing.T) {
	ResetBackends(backendsByPriority)

	var expectedServedByRegexp *regexp.Regexp
	var headerName string

	switch {
	case vendorCloudflare:
		headerName = "CF-RAY"
		expectedServedByRegexp = regexp.MustCompile("^[a-z0-9]{16}-[A-Z]{3}$")
	case vendorFastly:
		headerName = "X-Served-By"
		expectedServedByRegexp = regexp.MustCompile("^cache-[a-z0-9]+-[A-Z]{3}$")
	default:
		t.Fatal(notImplementedForVendor)
	}

	req := NewUniqueEdgeGET(t)
	resp := RoundTripCheckError(t, req)
	defer resp.Body.Close()

	actualHeader := resp.Header.Get(headerName)

	if actualHeader == "" {
		t.Error(headerName + " header has not been set by Edge")
	}

	if expectedServedByRegexp.FindString(actualHeader) != actualHeader {
		t.Errorf("%s is not as expected: got %q", headerName, actualHeader)
	}

}

// Should set an X-Cache-Hits header containing hit count for this object,
// from the Edge AND the Origin, assuming Origin sets one.
// This is in the format "{origin-hit-count}, {edge-hit-count}"
func TestRespHeaderXCacheHitsAppend(t *testing.T) {
	ResetBackends(backendsByPriority)

	if vendorCloudflare {
		t.Skip(notSupportedByVendor)
	}

	const originXCacheHits = "53"

	var (
		xCacheHits         string
		expectedXCacheHits string
	)

	uuid := NewUUID()

	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "GET" && r.URL.Path == fmt.Sprintf("/%s", uuid) {
			w.Header().Set("X-Cache-Hits", originXCacheHits)
		}
	})

	sourceURL := fmt.Sprintf("https://%s/%s", *edgeHost, uuid)

	// Get first request, will come from origin. Edge Hit Count 0
	req, _ := http.NewRequest("GET", sourceURL, nil)
	resp := RoundTripCheckError(t, req)
	defer resp.Body.Close()

	xCacheHits = resp.Header.Get("X-Cache-Hits")
	expectedXCacheHits = fmt.Sprintf("%s, 0", originXCacheHits)
	if xCacheHits != expectedXCacheHits {
		t.Errorf(
			"X-Cache-Hits on initial hit is wrong: expected %q, got %q",
			expectedXCacheHits,
			xCacheHits,
		)
	}

	// Get request again. Should come from Edge now, hit count 1
	resp = RoundTripCheckError(t, req)
	defer resp.Body.Close()

	xCacheHits = resp.Header.Get("X-Cache-Hits")
	expectedXCacheHits = fmt.Sprintf("%s, 1", originXCacheHits)
	if xCacheHits != expectedXCacheHits {
		t.Errorf(
			"X-Cache-Hits on second hit is wrong: expected %q, got %q",
			expectedXCacheHits,
			xCacheHits,
		)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"ims-release/endpoints"
	"ims-release/models"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
)

func getArg(argIdx int) (string, error) {
	if len(os.Args) > argIdx {
		return os.Args[argIdx], nil
	} else {
		return "", errors.New("Argument missing")
	}
}

func main() {
	const usage = "Usage: import_project <apiRoute> <authToken> <projectFolder>\n"
	const missingApiRoute = "apiRoute missing.\n"
	const missingAuthToken = "authToken missing.\n"
	const missingProjectFolder = "projectFolder missing.\n"

	apiRoute, err := getArg(1)
	if err != nil {
		log.Print(usage)
		log.Fatal(missingApiRoute)
	}

	authToken, err := getArg(2)
	if err != nil {
		log.Print(usage)
		log.Fatal(missingAuthToken)
	}

	projectFolder, err := getArg(3)
	if err != nil {
		log.Print(usage)
		log.Fatal(missingProjectFolder)
	}

	importProject(apiRoute, authToken, projectFolder)
}

type pageAddRequest struct {
	Name string `json:"name"`
	Data string `json:"data"`
}

func importProject(apiRoute, authToken, projectsFolder string) error {
	// imports projects from the imangascans reader
	// assumes the following path: <projects folder>/<prjoect folder>/<chapter folder>/<images>
	projectDirs, err := ioutil.ReadDir(projectsFolder)
	if err != nil {
		return err
	}

	str, err := getProjects(apiRoute)
	if err != nil {
		return err
	}

	decoder := json.NewDecoder(strings.NewReader(str))
	var projectsResponse endpoints.ProjectResponse
	err = decoder.Decode(&projectsResponse)
	if err != nil {
		return err
	}

	usedIdentifiers := map[string]string{} // a map of projectId-identifier to path

	for _, d := range projectDirs {
		if !d.IsDir() {
			continue
		}
		projectFolder := path.Join(projectsFolder, d.Name())
		releaseDirs, err := ioutil.ReadDir(projectFolder)
		if err != nil {
			return err
		}

		shorhtand := path.Base(projectFolder)
		projectId, err := addProject(apiRoute, authToken, shorhtand, projectsResponse)

		if err != nil {
			return err
		}

		str, err = getReleases(apiRoute, projectId)
		if err != nil {
			return err
		}

		decoder = json.NewDecoder(strings.NewReader(str))
		var releasesResponse endpoints.ReleaseResponse
		err = decoder.Decode(&releasesResponse)
		if err != nil {
			return err
		}

		for _, d := range releaseDirs {
			if !d.IsDir() {
				continue
			}
			tokens := strings.Split(d.Name(), "-")
			identifier := strings.Trim(tokens[0], " ")
			if len(identifier) == 0 {
				continue
			}
			if identifier[0] >= '0' && identifier[0] <= '9' {
				identifier = fmt.Sprintf("Ch%s", identifier)
			}
			identifier = strings.Replace(identifier, "Volume", "v", 1)
			identifier = strings.Replace(identifier, "(", "", 1)
			identifier = strings.Replace(identifier, ")", "", 1)
			identifier = strings.Replace(identifier, "Extra", "e", 1)
			identifier = strings.Replace(identifier, "Prologue part", "p", 1)
			if len(identifier) > 10 {
				log.Println("skipping release (identifier too long):", projectId, path.Join(projectFolder, d.Name()))
				continue
			}
			identifierKey := fmt.Sprintf("%d-%s", projectId, identifier)
			existingPath := usedIdentifiers[identifierKey]
			if existingPath != "" {
				log.Printf("skipping release (identifier already used by %s): %d %s\n", existingPath, projectId, path.Join(projectFolder, d.Name()))
				continue
			}

			usedIdentifiers[identifierKey] = path.Join(projectFolder, d.Name())

			releaseId, err := addRelease(apiRoute, authToken, identifier, releasesResponse, projectId)
			if err != nil {
				return err
			}
			imageFolder := path.Join(projectFolder, d.Name())
			err = addPages(apiRoute, authToken, imageFolder, projectId, releaseId)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func makeRequest(req *http.Request) (string, error) {
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	return string(data), err
}

func addProject(apiRoute, authToken, shorthand string, prResp endpoints.ProjectResponse) (uint32, error) {
	if len(shorthand) > 30 {
		shorthand = shorthand[0:30]
	}

	for _, pr := range prResp.Result {
		if pr.Shorthand == shorthand {
			return pr.Id, nil
		}
	}

	project := models.Project{Shorthand: shorthand, Name: shorthand, Status: "active"}
	buffer := bytes.NewBuffer([]byte{})
	encoder := json.NewEncoder(buffer)
	err := encoder.Encode(project)
	if err != nil {
		return 0, err
	}
	httpReq, err := http.NewRequest("POST", apiRoute+"/projects", buffer)
	if err != nil {
		return 0, err
	}
	httpReq.Header.Add("Auth-Token", authToken)
	str, err := makeRequest(httpReq)
	if err != nil {
		return 0, err
	}
	decoder := json.NewDecoder(strings.NewReader(str))
	prResp = endpoints.ProjectResponse{}
	err = decoder.Decode(&prResp)
	if err != nil {
		return 0, err
	}

	if prResp.Error != nil {
		return 0, errors.New(*prResp.Error)
	}

	if 1 != len(prResp.Result) {
		return 0, errors.New("no result found")
	}

	return prResp.Result[0].Id, nil
}

func addRelease(apiRoute, authToken, identifier string, resp endpoints.ReleaseResponse, projectId uint32) (uint32, error) {
	uri := fmt.Sprintf("%s/projects/%d/releases", apiRoute, projectId)

	for _, rel := range resp.Result {
		if rel.Identifier == identifier && rel.Version == uint32(0) {
			return rel.Id, nil
		}
	}

	release := models.Release{Identifier: identifier, Version: uint32(0)}
	buffer := bytes.NewBuffer([]byte{})
	encoder := json.NewEncoder(buffer)
	err := encoder.Encode(release)
	if err != nil {
		return 0, err
	}
	httpReq, err := http.NewRequest("POST", uri, buffer)
	if err != nil {
		return 0, err
	}
	httpReq.Header.Add("Auth-Token", authToken)
	str, err := makeRequest(httpReq)
	if err != nil {
		return 0, err
	}
	decoder := json.NewDecoder(strings.NewReader(str))
	resp = endpoints.ReleaseResponse{}
	err = decoder.Decode(&resp)
	if err != nil {
		return 0, err
	}

	if resp.Error != nil {
		return 0, errors.New(*resp.Error)
	}

	if 1 != len(resp.Result) {
		return 0, errors.New("no result found")
	}

	return resp.Result[0].Id, nil
}

func getReleases(apiRoute string, projectId uint32) (string, error) {
	uri := fmt.Sprintf("%s/projects/%d/releases", apiRoute, projectId)
	httpReq, err := http.NewRequest("GET", uri, nil)
	if err != nil {
		log.Println(err)
		return "", err
	}

	return makeRequest(httpReq)
}

func getPages(apiRoute string, projectId, releaseId uint32) (string, error) {
	uri := fmt.Sprintf("%s/projects/%d/releases/%d/pages", apiRoute, projectId, releaseId)
	httpReq, err := http.NewRequest("GET", uri, nil)
	if err != nil {
		log.Println(err)
		return "", err
	}

	return makeRequest(httpReq)
}

func getProjects(apiRoute string) (string, error) {
	httpReq, err := http.NewRequest("GET", apiRoute+"/projects", nil)
	if err != nil {
		log.Println(err)
		return "", err
	}

	return makeRequest(httpReq)
}

func addPages(apiRoute, authToken, imageFolder string, projectId, releaseId uint32) error {
	uri := fmt.Sprintf("%s/projects/%d/releases/%d/pages", apiRoute, projectId, releaseId)
	files, err := ioutil.ReadDir(imageFolder)
	if err != nil {
		return err
	}

	pagesStr, err := getPages(apiRoute, projectId, releaseId)
	if err != nil {
		return err
	}

	decoder := json.NewDecoder(strings.NewReader(pagesStr))
	var pagesResponse endpoints.PageResponse
	err = decoder.Decode(&pagesResponse)
	if err != nil {
		return err
	}

	if pagesResponse.Error != nil {
		return errors.New(*pagesResponse.Error)
	}

	addRequests := make([]pageAddRequest, 0, len(files))

	for _, file := range files {
		fn := file.Name()
		name := fn
		if file.IsDir() {
			continue
		}
		ext := path.Ext(fn)
		if strings.EqualFold(ext, ".png") {
			name = fn[0:len(fn)-len(ext)] + ".png"
		} else if strings.EqualFold(ext, ".jpg") || strings.EqualFold(ext, ".jpeg") {
			name = fn[0:len(fn)-len(ext)] + ".jpg"
		} else {
			continue
		}

		pageExists := false
		for _, pg := range pagesResponse.Result {
			if pg.Name == name {
				pageExists = true
				break
			}
		}

		if pageExists {
			continue
		}

		fn = filepath.Join(imageFolder, fn)
		f, err := ioutil.ReadFile(fn)
		if err != nil {
			return err
		}
		data := base64.StdEncoding.EncodeToString(f)
		addRequests = append(addRequests, pageAddRequest{Name: name, Data: data})
	}

	for _, req := range addRequests {
		buffer := bytes.NewBuffer([]byte{})
		encoder := json.NewEncoder(buffer)
		err = encoder.Encode(req)
		if err != nil {
			return err
		}
		httpReq, err := http.NewRequest("POST", uri, buffer)
		if err != nil {
			return err
		}
		httpReq.Header.Add("Auth-Token", authToken)
		log.Printf("Adding page '%s' (projectId %d, releaseId %d)", req.Name, projectId, releaseId)
		client := &http.Client{}
		resp, err := client.Do(httpReq)
		if err != nil {
			return err
		}
		defer resp.Body.Close()
		if resp.StatusCode != http.StatusOK {
			log.Println("response Status:", resp.Status)
			data, _ := ioutil.ReadAll(resp.Body)
			log.Println(string(data))
		}
	}

	return nil
}
<commit_msg>add import of oneshots, clean-up<commit_after>package main

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"ims-release/endpoints"
	"ims-release/models"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
)

func getArg(argIdx int) (string, error) {
	if len(os.Args) > argIdx {
		return os.Args[argIdx], nil
	} else {
		return "", errors.New("Argument missing")
	}
}

func main() {
	const usage = "Usage: import_project <apiRoute> <authToken> <projectFolder>\n"
	const missingApiRoute = "apiRoute missing.\n"
	const missingAuthToken = "authToken missing.\n"
	const missingProjectFolder = "projectFolder missing.\n"

	apiRoute, err := getArg(1)
	if err != nil {
		log.Print(usage)
		log.Fatal(missingApiRoute)
	}

	authToken, err := getArg(2)
	if err != nil {
		log.Print(usage)
		log.Fatal(missingAuthToken)
	}

	projectFolder, err := getArg(3)
	if err != nil {
		log.Print(usage)
		log.Fatal(missingProjectFolder)
	}

	importProject(apiRoute, authToken, projectFolder)
}

type pageAddRequest struct {
	Name string `json:"name"`
	Data string `json:"data"`
}

func getOneshotDirs(projectsFolder string, projectDirs []os.FileInfo) (map[string]bool, error) {
	oneshotDirs := map[string]bool{} // key is folder path, true if it's a oneshot

	for _, d := range projectDirs {
		if !d.IsDir() {
			continue
		}
		projectFolder := path.Join(projectsFolder, d.Name())
		releaseDirs, err := ioutil.ReadDir(projectFolder)
		if err != nil {
			return oneshotDirs, err
		}
		isOneshotFolder := true
		isOneshotSuperfolder := (d.Name() == "Oneshot")
		for _, d := range releaseDirs {
			if !d.IsDir() {
				continue
			}

			if isOneshotSuperfolder {
				oneshotDir := path.Join(projectFolder, d.Name())
				oneshotDirs[oneshotDir] = true
			}
			isOneshotFolder = false
		}
		if isOneshotFolder {
			oneshotDirs[projectFolder] = true
		}
	}

	return oneshotDirs, nil
}

func importRelease(apiRoute, authToken, releaseFolder string, projectId uint32, releasesResponse endpoints.ReleaseResponse, oneshotDirs map[string]bool, usedIdentifiers map[string]string) (map[string]string, error) {
	d, err := os.Stat(releaseFolder)
	if err != nil {
		return usedIdentifiers, nil
	}
	if !d.IsDir() {
		return usedIdentifiers, nil
	}
	tokens := strings.Split(d.Name(), "-")
	identifier := strings.Trim(tokens[0], " ")
	if oneshotDirs[releaseFolder] {
		identifier = "" // use empty identifier for oneshots
	} else {
		if len(identifier) == 0 {
			return usedIdentifiers, nil
		}
		if identifier[0] >= '0' && identifier[0] <= '9' {
			identifier = fmt.Sprintf("Ch%s", identifier)
		}
		identifier = strings.Replace(identifier, "Volume", "v", 1)
		identifier = strings.Replace(identifier, "(", "", 1)
		identifier = strings.Replace(identifier, ")", "", 1)
		identifier = strings.Replace(identifier, "Extra", "e", 1)
		identifier = strings.Replace(identifier, "Prologue part", "p", 1)

		if len(identifier) > 10 {
			log.Println("skipping release (identifier too long):", projectId, releaseFolder)
			return usedIdentifiers, nil
		}
	}

	identifierKey := fmt.Sprintf("%d-%s", projectId, identifier)
	existingPath := usedIdentifiers[identifierKey]
	if existingPath != "" {
		log.Printf("skipping release (identifier already used by %s): %d %s\n", existingPath, projectId, releaseFolder)
		return usedIdentifiers, nil
	}

	usedIdentifiers[identifierKey] = releaseFolder

	releaseId, err := addRelease(apiRoute, authToken, identifier, releasesResponse, projectId)
	if err != nil {
		return usedIdentifiers, err
	}
	err = addPages(apiRoute, authToken, releaseFolder, projectId, releaseId)
	if err != nil {
		return usedIdentifiers, err
	}
	return usedIdentifiers, nil
}

func importProject(apiRoute, authToken, projectsFolder string) error {
	// imports projects from the imangascans reader
	// assumes the following path: <projects folder>/<prjoect folder>/<chapter folder>/<images>
	projectDirs, err := ioutil.ReadDir(projectsFolder)
	if err != nil {
		return err
	}

	oneshotDirs, err := getOneshotDirs(projectsFolder, projectDirs)
	if err != nil {
		return err
	}

	str, err := getProjects(apiRoute)
	if err != nil {
		return err
	}

	projectsResponse, err := parseProjectResponse(str)
	if err != nil {
		return err
	}

	usedIdentifiers := map[string]string{} // a map of projectId-identifier to path

	// first add the oneshots
	for oneshotDir, _ := range oneshotDirs {
		usedIdentifiers, err = addOneshot(apiRoute, authToken, oneshotDir, projectsResponse, oneshotDirs, usedIdentifiers)
		if err != nil {
			return err
		}
	}

	for _, d := range projectDirs {
		if !d.IsDir() {
			continue
		}

		if d.Name() == "Oneshot" {
			continue
		}

		projectFolder := path.Join(projectsFolder, d.Name())
		if oneshotDirs[projectFolder] {
			continue
		}

		releaseDirs, err := ioutil.ReadDir(projectFolder)
		if err != nil {
			return err
		}

		name := path.Base(projectFolder)
		projectId, err := addProject(apiRoute, authToken, name, projectsResponse)

		if err != nil {
			return err
		}

		str, err = getReleases(apiRoute, projectId)
		if err != nil {
			return err
		}

		releasesResponse, err := parseReleaseResponse(str)
		if err != nil {
			return err
		}

		for _, d := range releaseDirs {
			releaseFolder := path.Join(projectFolder, d.Name())
			if oneshotDirs[releaseFolder] {
				continue
			}
			usedIdentifiers, err = importRelease(apiRoute, authToken,
				releaseFolder, projectId, releasesResponse,
				oneshotDirs, usedIdentifiers)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func addOneshot(apiRoute, authToken, oneshotDir string, projectsResponse endpoints.ProjectResponse, oneshotDirs map[string]bool, usedIdentifiers map[string]string) (map[string]string, error) {
	name := path.Base(oneshotDir)
	projectId, err := addProject(apiRoute, authToken, name, projectsResponse)

	str, err := getReleases(apiRoute, projectId)
	if err != nil {
		return usedIdentifiers, err
	}

	releasesResponse, err := parseReleaseResponse(str)
	if err != nil {
		return usedIdentifiers, err
	}

	return importRelease(apiRoute, authToken,
		oneshotDir, projectId, releasesResponse,
		oneshotDirs, usedIdentifiers)
}

func makeRequest(req *http.Request) (string, error) {
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	return string(data), err
}

func addProject(apiRoute, authToken, name string, prResp endpoints.ProjectResponse) (uint32, error) {
	shorthand := name
	if len(shorthand) > 30 {
		shorthand = shorthand[0:30]
	}

	for _, pr := range prResp.Result {
		if pr.Shorthand == shorthand {
			if pr.Name != name {
				pr.Name = name
				uri := fmt.Sprintf("%s/projects/%d", apiRoute, pr.Id)
				str, err := makeJsonRequest("PUT", uri, authToken, pr)
				if err != nil {
					log.Println("failed to update project", pr.Id, err)
				} else {
					resp, err := parseProjectResponse(str)
					if err != nil {
						log.Println("failed to update project", pr.Id, err)
					}
					if resp.Error != nil {
						log.Println("failed to update project", pr.Id, *resp.Error)
					}
				}
			}
			return pr.Id, nil
		}
	}

	project := models.Project{Shorthand: shorthand, Name: name, Status: "active"}
	uri := fmt.Sprintf("%s/projects", apiRoute)
	str, err := makeJsonRequest("POST", uri, authToken, project)
	if err != nil {
		return 0, err
	}
	resp, err := parseProjectResponse(str)
	if err != nil {
		return 0, err
	}

	if resp.Error != nil {
		return 0, errors.New(*resp.Error)
	}

	if 1 != len(resp.Result) {
		return 0, errors.New("no result found")
	}

	return resp.Result[0].Id, nil
}

func makeJsonRequest(method, uri, authToken string, entity interface{}) (string, error) {
	buffer := bytes.NewBuffer([]byte{})
	encoder := json.NewEncoder(buffer)
	err := encoder.Encode(entity)
	if err != nil {
		return "", err
	}
	httpReq, err := http.NewRequest(method, uri, buffer)
	if err != nil {
		return "", err
	}
	httpReq.Header.Add("Auth-Token", authToken)
	return makeRequest(httpReq)
}

func addRelease(apiRoute, authToken, identifier string, resp endpoints.ReleaseResponse, projectId uint32) (uint32, error) {
	uri := fmt.Sprintf("%s/projects/%d/releases", apiRoute, projectId)

	for _, rel := range resp.Result {
		if rel.Identifier == identifier && rel.Version == uint32(0) {
			return rel.Id, nil
		}
	}

	release := models.Release{Identifier: identifier, Version: uint32(0)}
	str, err := makeJsonRequest("POST", uri, authToken, release)
	if err != nil {
		return 0, err
	}
	resp, err = parseReleaseResponse(str)
	if err != nil {
		return 0, err
	}

	if resp.Error != nil {
		return 0, errors.New(*resp.Error)
	}

	if 1 != len(resp.Result) {
		return 0, errors.New("no result found")
	}

	return resp.Result[0].Id, nil
}

func getReleases(apiRoute string, projectId uint32) (string, error) {
	uri := fmt.Sprintf("%s/projects/%d/releases", apiRoute, projectId)
	httpReq, err := http.NewRequest("GET", uri, nil)
	if err != nil {
		log.Println(err)
		return "", err
	}

	return makeRequest(httpReq)
}

func parseReleaseResponse(str string) (endpoints.ReleaseResponse, error) {
	var resp endpoints.ReleaseResponse
	r, err := parseResponse(str, resp)
	return r.(endpoints.ReleaseResponse), err
}

func getPages(apiRoute string, projectId, releaseId uint32) (string, error) {
	uri := fmt.Sprintf("%s/projects/%d/releases/%d/pages", apiRoute, projectId, releaseId)
	httpReq, err := http.NewRequest("GET", uri, nil)
	if err != nil {
		log.Println(err)
		return "", err
	}

	return makeRequest(httpReq)
}

func parsePageResponse(str string) (endpoints.PageResponse, error) {
	var resp endpoints.PageResponse
	r, err := parseResponse(str, resp)
	return r.(endpoints.PageResponse), err
}

func getProjects(apiRoute string) (string, error) {
	uri := fmt.Sprintf("%s/projects", apiRoute)
	httpReq, err := http.NewRequest("GET", uri, nil)
	if err != nil {
		log.Println(err)
		return "", err
	}

	return makeRequest(httpReq)
}

func parseProjectResponse(str string) (endpoints.ProjectResponse, error) {
	var resp endpoints.ProjectResponse
	r, err := parseResponse(str, resp)
	return r.(endpoints.ProjectResponse), err
}

func parseResponse(str string, resp interface{}) (interface{}, error) {
	decoder := json.NewDecoder(strings.NewReader(str))
	err := decoder.Decode(&resp)
	if err != nil {
		return resp, err
	}
	return resp, nil
}

func addPages(apiRoute, authToken, imageFolder string, projectId, releaseId uint32) error {
	uri := fmt.Sprintf("%s/projects/%d/releases/%d/pages", apiRoute, projectId, releaseId)
	files, err := ioutil.ReadDir(imageFolder)
	if err != nil {
		return err
	}

	pagesStr, err := getPages(apiRoute, projectId, releaseId)
	if err != nil {
		return err
	}

	pagesResponse, err := parsePageResponse(pagesStr)
	if err != nil {
		return err
	}

	if pagesResponse.Error != nil {
		return errors.New(*pagesResponse.Error)
	}

	addRequests := make([]pageAddRequest, 0, len(files))

	for _, file := range files {
		fn := file.Name()
		name := fn
		if file.IsDir() {
			continue
		}
		ext := path.Ext(fn)
		if strings.EqualFold(ext, ".png") {
			name = fn[0:len(fn)-len(ext)] + ".png"
		} else if strings.EqualFold(ext, ".jpg") || strings.EqualFold(ext, ".jpeg") {
			name = fn[0:len(fn)-len(ext)] + ".jpg"
		} else {
			continue
		}

		pageExists := false
		for _, pg := range pagesResponse.Result {
			if pg.Name == name {
				pageExists = true
				break
			}
		}

		if pageExists {
			continue
		}

		fn = filepath.Join(imageFolder, fn)
		f, err := ioutil.ReadFile(fn)
		if err != nil {
			return err
		}
		data := base64.StdEncoding.EncodeToString(f)
		addRequests = append(addRequests, pageAddRequest{Name: name, Data: data})
	}

	for _, req := range addRequests {
		log.Printf("Adding page '%s' (projectId %d, releaseId %d)", req.Name, projectId, releaseId)
		str, err := makeJsonRequest("POST", uri, authToken, req)
		if err != nil {
			return err
		}
		resp, err := parsePageResponse(str)
		if err != nil {
			return err
		}
		if resp.Error != nil {
			log.Println("error:", *resp.Error)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"
	"regexp"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/efs"
	"github.com/aws/aws-sdk-go/service/sagemaker"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
	"github.com/hashicorp/terraform-plugin-sdk/v2/terraform"
	"github.com/terraform-providers/terraform-provider-aws/aws/internal/service/sagemaker/finder"
)

func init() {
	resource.AddTestSweepers("aws_sagemaker_domain", &resource.Sweeper{
		Name: "aws_sagemaker_domain",
		F:    testSweepSagemakerDomains,
		Dependencies: []string{
			"aws_efs_mount_target",
			"aws_efs_file_system",
		},
	})
}

func testSweepSagemakerDomains(region string) error {
	client, err := sharedClientForRegion(region)
	if err != nil {
		return fmt.Errorf("error getting client: %s", err)
	}
	conn := client.(*AWSClient).sagemakerconn

	err = conn.ListDomainsPages(&sagemaker.ListDomainsInput{}, func(page *sagemaker.ListDomainsOutput, lastPage bool) bool {
		for _, instance := range page.Domains {
			domainArn := aws.StringValue(instance.DomainArn)
			domainID, err := decodeSagemakerDomainID(domainArn)
			if err != nil {
				log.Printf("[ERROR] Error parsing sagemaker domain arn (%s): %s", domainArn, err)
			}
			input := &sagemaker.DeleteDomainInput{
				DomainId: aws.String(domainID),
			}

			log.Printf("[INFO] Deleting SageMaker domain: %s", domainArn)
			if _, err := conn.DeleteDomain(input); err != nil {
				log.Printf("[ERROR] Error deleting SageMaker domain (%s): %s", domainArn, err)
				continue
			}
		}

		return !lastPage
	})

	if testSweepSkipSweepError(err) {
		log.Printf("[WARN] Skipping SageMaker domain sweep for %s: %s", region, err)
		return nil
	}

	if err != nil {
		return fmt.Errorf("Error retrieving SageMaker domains: %w", err)
	}

	return nil
}

func TestAccAWSSagemakerDomain_basic(t *testing.T) {
	var notebook sagemaker.DescribeDomainOutput
	rName := acctest.RandomWithPrefix("tf-acc-test")
	resourceName := "aws_sagemaker_domain.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSSagemakerDomainDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSSagemakerDomainBasicConfig(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSSagemakerDomainExists(resourceName, &notebook),
					resource.TestCheckResourceAttr(resourceName, "domain_name", rName),
					resource.TestCheckResourceAttr(resourceName, "auth_mode", "IAM"),
					resource.TestCheckResourceAttr(resourceName, "app_network_access_type", "PublicInternetOnly"),
					resource.TestCheckResourceAttr(resourceName, "subnet_ids.#", "1"),
					resource.TestCheckResourceAttr(resourceName, "default_user_settings.#", "1"),
					resource.TestCheckResourceAttrPair(resourceName, "default_user_settings.0.execution_role", "aws_iam_role.test", "arn"),
					testAccMatchResourceAttrRegionalARN(resourceName, "arn", "sagemaker", regexp.MustCompile(`domain/.+`)),
					resource.TestCheckResourceAttr(resourceName, "tags.%", "0"),
					resource.TestCheckResourceAttrPair(resourceName, "vpc_id", "aws_vpc.test", "id"),
					resource.TestCheckResourceAttrSet(resourceName, "url"),
					testAccCheckAWSSagemakerDomainDeleteImplicitResources(resourceName),
				),
			},
			{
				ResourceName:      resourceName,
				ImportState:       true,
				ImportStateVerify: true,
			},
		},
	})
}

// func TestAccAWSSagemakerDomain_gitConfig_branch(t *testing.T) {
// 	var notebook sagemaker.DescribeDomainOutput
// 	rName := acctest.RandomWithPrefix("tf-acc-test")
// 	resourceName := "aws_sagemaker_domain.test"

// 	resource.ParallelTest(t, resource.TestCase{
// 		PreCheck:     func() { testAccPreCheck(t) },
// 		Providers:    testAccProviders,
// 		CheckDestroy: testAccCheckAWSSagemakerDomainDestroy,
// 		Steps: []resource.TestStep{
// 			{
// 				Config: testAccAWSSagemakerDomainGitConfigBranchConfig(rName),
// 				Check: resource.ComposeTestCheckFunc(
// 					testAccCheckAWSSagemakerDomainExists(resourceName, &notebook),
// 					resource.TestCheckResourceAttr(resourceName, "domain_name", rName),
// 					testAccCheckResourceAttrRegionalARN(resourceName0......, "arn", "sagemaker", fmt.Sprintf("code-repository/%s", rName)),
// 					resource.TestCheckResourceAttr(resourceName, "git_config.#", "1"),
// 					resource.TestCheckResourceAttr(resourceName, "git_config.0.repository_url", "https://github.com/terraform-providers/terraform-provider-aws.git"),
// 					resource.TestCheckResourceAttr(resourceName, "git_config.0.branch", "master"),
// 				),
// 			},
// 			{
// 				ResourceName:      resourceName,
// 				ImportState:       true,
// 				ImportStateVerify: true,
// 			},
// 		},
// 	})
// }

// func TestAccAWSSagemakerDomain_gitConfig_secret(t *testing.T) {
// 	var notebook sagemaker.DescribeDomainOutput
// 	rName := acctest.RandomWithPrefix("tf-acc-test")
// 	resourceName := "aws_sagemaker_domain.test"

// 	resource.ParallelTest(t, resource.TestCase{
// 		PreCheck:     func() { testAccPreCheck(t) },
// 		Providers:    testAccProviders,
// 		CheckDestroy: testAccCheckAWSSagemakerDomainDestroy,
// 		Steps: []resource.TestStep{
// 			{
// 				Config: testAccAWSSagemakerDomainGitConfigSecretConfig(rName),
// 				Check: resource.ComposeTestCheckFunc(
// 					testAccCheckAWSSagemakerDomainExists(resourceName, &notebook),
// 					resource.TestCheckResourceAttr(resourceName, "domain_name", rName),
// 					testAccCheckResourceAttrRegionalARN(resourceName, "arn", "sagemaker", fmt.Sprintf("code-repository/%s", rName)),
// 					resource.TestCheckResourceAttr(resourceName, "git_config.#", "1"),
// 					resource.TestCheckResourceAttr(resourceName, "git_config.0.repository_url", "https://github.com/terraform-providers/terraform-provider-aws.git"),
// 					resource.TestCheckResourceAttrPair(resourceName, "git_config.0.secret_arn", "aws_secretsmanager_secret.test", "arn"),
// 				),
// 			},
// 			{
// 				ResourceName:      resourceName,
// 				ImportState:       true,
// 				ImportStateVerify: true,
// 			},
// 			{
// 				Config: testAccAWSSagemakerDomainGitConfigSecretUpdatedConfig(rName),
// 				Check: resource.ComposeTestCheckFunc(
// 					testAccCheckAWSSagemakerDomainExists(resourceName, &notebook),
// 					resource.TestCheckResourceAttr(resourceName, "domain_name", rName),
// 					testAccCheckResourceAttrRegionalARN(resourceName, "arn", "sagemaker", fmt.Sprintf("code-repository/%s", rName)),
// 					resource.TestCheckResourceAttr(resourceName, "git_config.#", "1"),
// 					resource.TestCheckResourceAttr(resourceName, "git_config.0.repository_url", "https://github.com/terraform-providers/terraform-provider-aws.git"),
// 					resource.TestCheckResourceAttrPair(resourceName, "git_config.0.secret_arn", "aws_secretsmanager_secret.test2", "arn"),
// 				),
// 			},
// 		},
// 	})
// }

func TestAccAWSSagemakerDomain_disappears(t *testing.T) {
	var notebook sagemaker.DescribeDomainOutput
	rName := acctest.RandomWithPrefix("tf-acc-test")
	resourceName := "aws_sagemaker_domain.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSSagemakerDomainDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSSagemakerDomainBasicConfig(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSSagemakerDomainExists(resourceName, &notebook),
					testAccCheckResourceDisappears(testAccProvider, resourceAwsSagemakerDomain(), resourceName),
				),
				ExpectNonEmptyPlan: true,
			},
		},
	})
}

func testAccCheckAWSSagemakerDomainDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).sagemakerconn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_sagemaker_domain" {
			continue
		}

		domain, err := finder.DomainByName(conn, rs.Primary.ID)
		if err != nil {
			return nil
		}

		domainArn := aws.StringValue(domain.DomainArn)
		domainID, err := decodeSagemakerDomainID(domainArn)
		if err != nil {
			return err
		}

		if domainID == rs.Primary.ID {
			return fmt.Errorf("sagemaker domain %q still exists", rs.Primary.ID)
		}
	}

	return nil
}

func testAccCheckAWSSagemakerDomainExists(n string, codeRepo *sagemaker.DescribeDomainOutput) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No sagmaker domain ID is set")
		}

		conn := testAccProvider.Meta().(*AWSClient).sagemakerconn
		resp, err := finder.DomainByName(conn, rs.Primary.ID)
		if err != nil {
			return err
		}

		*codeRepo = *resp

		return nil
	}
}

func testAccCheckAWSSagemakerDomainDeleteImplicitResources(n string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Sagemaker domain not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("Sagemaker domain name not set")
		}

		conn := testAccProvider.Meta().(*AWSClient).efsconn
		efsFsID := rs.Primary.Attributes["home_efs_file_system_id"]
		vpcID := rs.Primary.Attributes["vpc_id"]

		resp, err := conn.DescribeMountTargets(&efs.DescribeMountTargetsInput{
			FileSystemId: aws.String(efsFsID),
		})

		if err != nil {
			return fmt.Errorf("Sagemaker domain EFS mount targets not found: %w", err)
		}

		//reusing EFS mount target delete for wait logic
		mountTargets := resp.MountTargets
		for _, mt := range mountTargets {
			r := resourceAwsEfsMountTarget()
			d := r.Data(nil)
			mtId := aws.StringValue(mt.MountTargetId)
			d.SetId(mtId)
			err := r.Delete(d, testAccProvider.Meta())
			if err != nil {
				return fmt.Errorf("Sagemaker domain EFS mount target (%s) failed to delete: %w", mtId, err)
			}
		}

		r := resourceAwsEfsFileSystem()
		d := r.Data(nil)
		d.SetId(efsFsID)
		err = r.Delete(d, testAccProvider.Meta())
		if err != nil {
			return fmt.Errorf("Sagemaker domain EFS file system (%s) failed to delete: %w", efsFsID, err)
		}

		return nil
	}
}

func testAccAWSSagemakerDomainConfigBase(rName string) string {
	return fmt.Sprintf(`
resource "aws_vpc" "test" {

  cidr_block = "10.0.0.0/16"

  tags = {
    Name = %[1]q
  }
}

resource "aws_subnet" "test" {
  vpc_id     = aws_vpc.test.id
  cidr_block = "10.0.1.0/24"

  tags = {
    Name = %[1]q
  }
}

resource "aws_iam_role" "test" {
  name               = %[1]q
  path               = "/"
  assume_role_policy = data.aws_iam_policy_document.test.json
}

data "aws_iam_policy_document" "test" {
  statement {
    actions = ["sts:AssumeRole"]

    principals {
      type        = "Service"
      identifiers = ["sagemaker.amazonaws.com"]
    }
  }
}
`, rName)
}

func testAccAWSSagemakerDomainBasicConfig(rName string) string {
	return testAccAWSSagemakerDomainConfigBase(rName) + fmt.Sprintf(`
resource "aws_sagemaker_domain" "test" {
  domain_name = %[1]q
  auth_mode   = "IAM"
  vpc_id      = aws_vpc.test.id
  subnet_ids  = [aws_subnet.test.id]

  default_user_settings {
    execution_role = aws_iam_role.test.arn
  }
}
`, rName)
}

// func testAccAWSSagemakerDomainGitConfigBranchConfig(rName string) string {
// 	return fmt.Sprintf(`
// resource "aws_sagemaker_domain" "test" {
//   domain_name = %[1]q

//   git_config {
//     repository_url = "https://github.com/terraform-providers/terraform-provider-aws.git"
//     branch         = "master"
//   }
// }
// `, rName)
// }

// func testAccAWSSagemakerDomainGitConfigSecretConfig(rName string) string {
// 	return fmt.Sprintf(`
// resource "aws_secretsmanager_secret" "test" {
//   name = %[1]q
// }

// resource "aws_secretsmanager_secret_version" "test" {
//   secret_id     = aws_secretsmanager_secret.test.id
//   secret_string = jsonencode({ username = "example", passowrd = "example" })
// }

// resource "aws_sagemaker_domain" "test" {
//   domain_name = %[1]q

//   git_config {
//     repository_url = "https://github.com/terraform-providers/terraform-provider-aws.git"
//     secret_arn     = aws_secretsmanager_secret.test.arn
//   }

//   depends_on = [aws_secretsmanager_secret_version.test]
// }
// `, rName)
// }

// func testAccAWSSagemakerDomainGitConfigSecretUpdatedConfig(rName string) string {
// 	return fmt.Sprintf(`
// resource "aws_secretsmanager_secret" "test2" {
//   name = "%[1]s-2"
// }

// resource "aws_secretsmanager_secret_version" "test2" {
//   secret_id     = aws_secretsmanager_secret.test2.id
//   secret_string = jsonencode({ username = "example", passowrd = "example" })
// }

// resource "aws_sagemaker_domain" "test" {
//   domain_name = %[1]q

//   git_config {
//     repository_url = "https://github.com/terraform-providers/terraform-provider-aws.git"
//     secret_arn     = aws_secretsmanager_secret.test2.arn
//   }

//   depends_on = [aws_secretsmanager_secret_version.test2]
// }
// `, rName)
// }
<commit_msg>add tags test<commit_after>package aws

import (
	"fmt"
	"log"
	"regexp"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/efs"
	"github.com/aws/aws-sdk-go/service/sagemaker"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
	"github.com/hashicorp/terraform-plugin-sdk/v2/terraform"
	"github.com/terraform-providers/terraform-provider-aws/aws/internal/service/sagemaker/finder"
)

func init() {
	resource.AddTestSweepers("aws_sagemaker_domain", &resource.Sweeper{
		Name: "aws_sagemaker_domain",
		F:    testSweepSagemakerDomains,
		Dependencies: []string{
			"aws_efs_mount_target",
			"aws_efs_file_system",
		},
	})
}

func testSweepSagemakerDomains(region string) error {
	client, err := sharedClientForRegion(region)
	if err != nil {
		return fmt.Errorf("error getting client: %s", err)
	}
	conn := client.(*AWSClient).sagemakerconn

	err = conn.ListDomainsPages(&sagemaker.ListDomainsInput{}, func(page *sagemaker.ListDomainsOutput, lastPage bool) bool {
		for _, instance := range page.Domains {
			domainArn := aws.StringValue(instance.DomainArn)
			domainID, err := decodeSagemakerDomainID(domainArn)
			if err != nil {
				log.Printf("[ERROR] Error parsing sagemaker domain arn (%s): %s", domainArn, err)
			}
			input := &sagemaker.DeleteDomainInput{
				DomainId: aws.String(domainID),
			}

			log.Printf("[INFO] Deleting SageMaker domain: %s", domainArn)
			if _, err := conn.DeleteDomain(input); err != nil {
				log.Printf("[ERROR] Error deleting SageMaker domain (%s): %s", domainArn, err)
				continue
			}
		}

		return !lastPage
	})

	if testSweepSkipSweepError(err) {
		log.Printf("[WARN] Skipping SageMaker domain sweep for %s: %s", region, err)
		return nil
	}

	if err != nil {
		return fmt.Errorf("Error retrieving SageMaker domains: %w", err)
	}

	return nil
}

func TestAccAWSSagemakerDomain_basic(t *testing.T) {
	var notebook sagemaker.DescribeDomainOutput
	rName := acctest.RandomWithPrefix("tf-acc-test")
	resourceName := "aws_sagemaker_domain.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSSagemakerDomainDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSSagemakerDomainBasicConfig(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSSagemakerDomainExists(resourceName, &notebook),
					resource.TestCheckResourceAttr(resourceName, "domain_name", rName),
					resource.TestCheckResourceAttr(resourceName, "auth_mode", "IAM"),
					resource.TestCheckResourceAttr(resourceName, "app_network_access_type", "PublicInternetOnly"),
					resource.TestCheckResourceAttr(resourceName, "subnet_ids.#", "1"),
					resource.TestCheckResourceAttr(resourceName, "default_user_settings.#", "1"),
					resource.TestCheckResourceAttrPair(resourceName, "default_user_settings.0.execution_role", "aws_iam_role.test", "arn"),
					testAccMatchResourceAttrRegionalARN(resourceName, "arn", "sagemaker", regexp.MustCompile(`domain/.+`)),
					resource.TestCheckResourceAttr(resourceName, "tags.%", "0"),
					resource.TestCheckResourceAttrPair(resourceName, "vpc_id", "aws_vpc.test", "id"),
					resource.TestCheckResourceAttrSet(resourceName, "url"),
					resource.TestCheckResourceAttrSet(resourceName, "home_efs_file_system_id"),
					testAccCheckAWSSagemakerDomainDeleteImplicitResources(resourceName),
				),
			},
			{
				ResourceName:      resourceName,
				ImportState:       true,
				ImportStateVerify: true,
			},
		},
	})
}

func TestAccAWSSagemakerDomain_tags(t *testing.T) {
	var notebook sagemaker.DescribeDomainOutput
	rName := acctest.RandomWithPrefix("tf-acc-test")
	resourceName := "aws_sagemaker_domain.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSSagemakerDomainDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSSagemakerDomainBasicConfigTags1(rName, "key1", "value1"),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSSagemakerDomainExists(resourceName, &notebook),
					resource.TestCheckResourceAttr(resourceName, "tags.%", "1"),
					resource.TestCheckResourceAttr(resourceName, "tags.key1", "value1"),
				),
			},
			{
				ResourceName:      resourceName,
				ImportState:       true,
				ImportStateVerify: true,
			},
			{
				Config: testAccAWSSagemakerDomainBasicConfigTags2(rName, "key1", "value1updated", "key2", "value2"),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSSagemakerDomainExists(resourceName, &notebook),
					resource.TestCheckResourceAttr(resourceName, "tags.%", "2"),
					resource.TestCheckResourceAttr(resourceName, "tags.key1", "value1updated"),
					resource.TestCheckResourceAttr(resourceName, "tags.key2", "value2"),
				),
			},
			{
				Config: testAccAWSSagemakerDomainBasicConfigTags1(rName, "key2", "value2"),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSSagemakerDomainExists(resourceName, &notebook),
					resource.TestCheckResourceAttr(resourceName, "tags.%", "1"),
					resource.TestCheckResourceAttr(resourceName, "tags.key2", "value2"),
				),
			},
		},
	})
}

func TestAccAWSSagemakerDomain_disappears(t *testing.T) {
	var notebook sagemaker.DescribeDomainOutput
	rName := acctest.RandomWithPrefix("tf-acc-test")
	resourceName := "aws_sagemaker_domain.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSSagemakerDomainDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSSagemakerDomainBasicConfig(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSSagemakerDomainExists(resourceName, &notebook),
					testAccCheckAWSSagemakerDomainDeleteImplicitResources(resourceName),
					testAccCheckResourceDisappears(testAccProvider, resourceAwsSagemakerDomain(), resourceName),
				),
				ExpectNonEmptyPlan: true,
			},
		},
	})
}

func testAccCheckAWSSagemakerDomainDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).sagemakerconn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_sagemaker_domain" {
			continue
		}

		domain, err := finder.DomainByName(conn, rs.Primary.ID)
		if err != nil {
			return nil
		}

		domainArn := aws.StringValue(domain.DomainArn)
		domainID, err := decodeSagemakerDomainID(domainArn)
		if err != nil {
			return err
		}

		if domainID == rs.Primary.ID {
			return fmt.Errorf("sagemaker domain %q still exists", rs.Primary.ID)
		}
	}

	return nil
}

func testAccCheckAWSSagemakerDomainExists(n string, codeRepo *sagemaker.DescribeDomainOutput) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No sagmaker domain ID is set")
		}

		conn := testAccProvider.Meta().(*AWSClient).sagemakerconn
		resp, err := finder.DomainByName(conn, rs.Primary.ID)
		if err != nil {
			return err
		}

		*codeRepo = *resp

		return nil
	}
}

func testAccCheckAWSSagemakerDomainDeleteImplicitResources(n string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Sagemaker domain not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("Sagemaker domain name not set")
		}

		conn := testAccProvider.Meta().(*AWSClient).efsconn
		efsFsID := rs.Primary.Attributes["home_efs_file_system_id"]

		resp, err := conn.DescribeMountTargets(&efs.DescribeMountTargetsInput{
			FileSystemId: aws.String(efsFsID),
		})

		if err != nil {
			return fmt.Errorf("Sagemaker domain EFS mount targets not found: %w", err)
		}

		//reusing EFS mount target delete for wait logic
		mountTargets := resp.MountTargets
		for _, mt := range mountTargets {
			r := resourceAwsEfsMountTarget()
			d := r.Data(nil)
			mtID := aws.StringValue(mt.MountTargetId)
			d.SetId(mtID)
			err := r.Delete(d, testAccProvider.Meta())
			if err != nil {
				return fmt.Errorf("Sagemaker domain EFS mount target (%s) failed to delete: %w", mtID, err)
			}
		}

		r := resourceAwsEfsFileSystem()
		d := r.Data(nil)
		d.SetId(efsFsID)
		err = r.Delete(d, testAccProvider.Meta())
		if err != nil {
			return fmt.Errorf("Sagemaker domain EFS file system (%s) failed to delete: %w", efsFsID, err)
		}

		return nil
	}
}

func testAccAWSSagemakerDomainConfigBase(rName string) string {
	return fmt.Sprintf(`
resource "aws_vpc" "test" {

  cidr_block = "10.0.0.0/16"

  tags = {
    Name = %[1]q
  }
}

resource "aws_subnet" "test" {
  vpc_id     = aws_vpc.test.id
  cidr_block = "10.0.1.0/24"

  tags = {
    Name = %[1]q
  }
}

resource "aws_iam_role" "test" {
  name               = %[1]q
  path               = "/"
  assume_role_policy = data.aws_iam_policy_document.test.json
}

data "aws_iam_policy_document" "test" {
  statement {
    actions = ["sts:AssumeRole"]

    principals {
      type        = "Service"
      identifiers = ["sagemaker.amazonaws.com"]
    }
  }
}
`, rName)
}

func testAccAWSSagemakerDomainBasicConfig(rName string) string {
	return testAccAWSSagemakerDomainConfigBase(rName) + fmt.Sprintf(`
resource "aws_sagemaker_domain" "test" {
  domain_name = %[1]q
  auth_mode   = "IAM"
  vpc_id      = aws_vpc.test.id
  subnet_ids  = [aws_subnet.test.id]

  default_user_settings {
    execution_role = aws_iam_role.test.arn
  }
}
`, rName)
}

func testAccAWSSagemakerDomainBasicConfigTags1(rName, tagKey1, tagValue1 string) string {
	return testAccAWSSagemakerDomainConfigBase(rName) + fmt.Sprintf(`
resource "aws_sagemaker_domain" "test" {
  domain_name = %[1]q
  auth_mode   = "IAM"
  vpc_id      = aws_vpc.test.id
  subnet_ids  = [aws_subnet.test.id]

  default_user_settings {
    execution_role = aws_iam_role.test.arn
  }

  tags = {
    %[2]q = %[3]q
  }
}
`, rName, tagKey1, tagValue1)
}

func testAccAWSSagemakerDomainBasicConfigTags2(rName, tagKey1, tagValue1, tagKey2, tagValue2 string) string {
	return testAccAWSSagemakerDomainConfigBase(rName) + fmt.Sprintf(`
resource "aws_sagemaker_domain" "test" {
  domain_name = %[1]q
  auth_mode   = "IAM"
  vpc_id      = aws_vpc.test.id
  subnet_ids  = [aws_subnet.test.id]

  default_user_settings {
    execution_role = aws_iam_role.test.arn
  }

  tags = {
	%[2]q = %[3]q
	%[4]q = %[5]q
  }
}
`, rName, tagKey1, tagValue1, tagKey2, tagValue2)
}
<|endoftext|>


<commit_before>package waterutil

import (
	"net"
)

func IPv4DSCP(packet []byte) byte {
	return packet[1] >> 2
}

func IPv4ECN(packet []byte) byte {
	return packet[1] & 0x03
}

func IPv4Identification(packet []byte) [2]byte {
	return [2]byte{packet[4], packet[5]}
}

func IPv4TTL(packet []byte) byte {
	return packet[8]
}

func IPv4Protocol(packet []byte) IPProtocol {
	return IPProtocol(packet[9])
}

func IPv4Source(packet []byte) net.IP {
	return net.IPv4(packet[12], packet[13], packet[14], packet[15])
}

func SetIPv4Source(packet []byte, source net.IP) {
	copy(packet[12:16], source)
}

func IPv4Destination(packet []byte) net.IP {
	return net.IPv4(packet[16], packet[17], packet[18], packet[19])
}

func SetIPv4Destination(packet []byte, dest net.IP) {
	copy(packet[16:20], dest)
}

func IPv4Payload(packet []byte) []byte {
	ihl := packet[0] & 0x0F
	return packet[ihl*4:]
}

// For TCP/UDP
func IPv4SourcePort(packet []byte) uint16 {
	payload := IPv4Payload(packet)
	return (uint16(payload[0]) << 8) | uint16(payload[1])
}

func IPv4DestinationPort(packet []byte) uint16 {
	payload := IPv4Payload(packet)
	return (uint16(payload[2]) << 8) | uint16(payload[3])
}

func SetIPv4SourcePort(packet []byte, port uint16) {
	payload := IPv4Payload(packet)
	payload[0] = byte(port >> 8)
	payload[1] = byte(port & 0xFF)
}

func SetIPv4DestinationPort(packet []byte, port uint16) {
	payload := IPv4Payload(packet)
	payload[2] = byte(port >> 8)
	payload[3] = byte(port & 0xFF)
}
<commit_msg>Fix to use To4()<commit_after>package waterutil

import (
	"net"
)

func IPv4DSCP(packet []byte) byte {
	return packet[1] >> 2
}

func IPv4ECN(packet []byte) byte {
	return packet[1] & 0x03
}

func IPv4Identification(packet []byte) [2]byte {
	return [2]byte{packet[4], packet[5]}
}

func IPv4TTL(packet []byte) byte {
	return packet[8]
}

func IPv4Protocol(packet []byte) IPProtocol {
	return IPProtocol(packet[9])
}

func IPv4Source(packet []byte) net.IP {
	return net.IPv4(packet[12], packet[13], packet[14], packet[15])
}

func SetIPv4Source(packet []byte, source net.IP) {
	copy(packet[12:16], source.To4())
}

func IPv4Destination(packet []byte) net.IP {
	return net.IPv4(packet[16], packet[17], packet[18], packet[19])
}

func SetIPv4Destination(packet []byte, dest net.IP) {
	copy(packet[16:20], dest.To4())
}

func IPv4Payload(packet []byte) []byte {
	ihl := packet[0] & 0x0F
	return packet[ihl*4:]
}

// For TCP/UDP
func IPv4SourcePort(packet []byte) uint16 {
	payload := IPv4Payload(packet)
	return (uint16(payload[0]) << 8) | uint16(payload[1])
}

func IPv4DestinationPort(packet []byte) uint16 {
	payload := IPv4Payload(packet)
	return (uint16(payload[2]) << 8) | uint16(payload[3])
}

func SetIPv4SourcePort(packet []byte, port uint16) {
	payload := IPv4Payload(packet)
	payload[0] = byte(port >> 8)
	payload[1] = byte(port & 0xFF)
}

func SetIPv4DestinationPort(packet []byte, port uint16) {
	payload := IPv4Payload(packet)
	payload[2] = byte(port >> 8)
	payload[3] = byte(port & 0xFF)
}
<|endoftext|>


<commit_before>package telegram_bot

import (
	"github.com/strongo/bots-api-telegram"
	"github.com/strongo/bots-framework/core"
)

type TelegramWebhookCallbackQuery struct {
	updateID      int
	callbackQuery *tgbotapi.CallbackQuery
	message       bots.WebhookMessage
}

var _ bots.WebhookCallbackQuery = (*TelegramWebhookCallbackQuery)(nil)

func NewTelegramWebhookCallbackQuery(updateID int, callbackQuery *tgbotapi.CallbackQuery) TelegramWebhookCallbackQuery {
	if updateID == 0 {
		panic("updateID == 0")
	}
	if callbackQuery == nil {
		panic("callbackQuery == nil")
	}
	q := TelegramWebhookCallbackQuery{updateID: updateID, callbackQuery: callbackQuery}
	if callbackQuery.Message != nil && callbackQuery.Message.MessageID != 0 {
		q.message = NewTelegramWebhookMessage(updateID, callbackQuery.Message)
	}
	return q
}

func (iq TelegramWebhookCallbackQuery) GetID() interface{} {
	return iq.updateID
}

func (iq TelegramWebhookCallbackQuery) Chat() bots.WebhookChat {
	if iq.callbackQuery != nil && iq.callbackQuery.Message != nil {
		return TelegramWebhookChat{
			chat: iq.callbackQuery.Message.Chat,
		}
	}
	return nil
}

func (iq TelegramWebhookCallbackQuery) Sequence() int {
	return iq.updateID
}

func (q TelegramWebhookCallbackQuery) GetMessage() bots.WebhookMessage {
	return q.message
}

func (iq TelegramWebhookCallbackQuery) GetFrom() bots.WebhookSender {
	return TelegramSender{tgUser: iq.callbackQuery.From}
}

func (iq TelegramWebhookCallbackQuery) GetData() string {
	return iq.callbackQuery.Data
}

func (iq TelegramWebhookCallbackQuery) GetInlineMessageID() string {
	return iq.callbackQuery.InlineMessageID
}

func EditMessageOnCallbackQuery(whcbq bots.WebhookCallbackQuery, parseMode, text string) tgbotapi.EditMessageTextConfig {
	twhcbq := whcbq.(TelegramWebhookCallbackQuery)
	callbackQuery := twhcbq.callbackQuery

	emc := tgbotapi.EditMessageTextConfig{
		Text: text,
		ParseMode: parseMode,
		BaseEdit: tgbotapi.BaseEdit{
			InlineMessageID: callbackQuery.InlineMessageID,
		},
	}
	if emc.InlineMessageID == "" {
		emc.ChatID = callbackQuery.Message.Chat.ID
		emc.MessageID = callbackQuery.Message.MessageID
	}
	return emc
}<commit_msg>Step by step to the stars<commit_after>package telegram_bot

import (
	"github.com/strongo/bots-api-telegram"
	"github.com/strongo/bots-framework/core"
)

type TelegramWebhookCallbackQuery struct {
	updateID      int
	callbackQuery *tgbotapi.CallbackQuery
	message       bots.WebhookMessage
}

var _ bots.WebhookCallbackQuery = (*TelegramWebhookCallbackQuery)(nil)

func NewTelegramWebhookCallbackQuery(updateID int, callbackQuery *tgbotapi.CallbackQuery) TelegramWebhookCallbackQuery {
	if updateID == 0 {
		panic("updateID == 0")
	}
	if callbackQuery == nil {
		panic("callbackQuery == nil")
	}
	q := TelegramWebhookCallbackQuery{updateID: updateID, callbackQuery: callbackQuery}
	if callbackQuery.Message != nil && callbackQuery.Message.MessageID != 0 {
		q.message = NewTelegramWebhookMessage(updateID, callbackQuery.Message)
	}
	return q
}

func (iq TelegramWebhookCallbackQuery) GetID() interface{} {
	return iq.updateID
}

func (iq TelegramWebhookCallbackQuery) Chat() bots.WebhookChat {
	if iq.callbackQuery != nil && iq.callbackQuery.Message != nil {
		return TelegramWebhookChat{
			chat: iq.callbackQuery.Message.Chat,
		}
	}
	return nil
}

func (iq TelegramWebhookCallbackQuery) Sequence() int {
	return iq.updateID
}

func (q TelegramWebhookCallbackQuery) GetMessage() bots.WebhookMessage {
	return q.message
}

func (iq TelegramWebhookCallbackQuery) GetFrom() bots.WebhookSender {
	return TelegramSender{tgUser: iq.callbackQuery.From}
}

func (iq TelegramWebhookCallbackQuery) GetData() string {
	return iq.callbackQuery.Data
}

func (iq TelegramWebhookCallbackQuery) GetInlineMessageID() string {
	return iq.callbackQuery.InlineMessageID
}

func EditMessageOnCallbackQuery(whcbq bots.WebhookCallbackQuery, parseMode, text string) *tgbotapi.EditMessageTextConfig {
	twhcbq := whcbq.(TelegramWebhookCallbackQuery)
	callbackQuery := twhcbq.callbackQuery

	emc := tgbotapi.EditMessageTextConfig{
		Text: text,
		ParseMode: parseMode,
		BaseEdit: tgbotapi.BaseEdit{
			InlineMessageID: callbackQuery.InlineMessageID,
		},
	}
	if emc.InlineMessageID == "" {
		emc.ChatID = callbackQuery.Message.Chat.ID
		emc.MessageID = callbackQuery.Message.MessageID
	}
	return &emc
}<|endoftext|>


<commit_before>// Code generated by protoc-gen-gogo.
// source: identify.proto
// DO NOT EDIT!

/*
Package identify_pb is a generated protocol buffer package.

It is generated from these files:
	identify.proto

It has these top-level messages:
	Identify
*/
package identify_pb

import proto "code.google.com/p/gogoprotobuf/proto"
import json "encoding/json"
import math "math"

// Reference proto, json, and math imports to suppress error if they are not otherwise used.
var _ = proto.Marshal
var _ = &json.SyntaxError{}
var _ = math.Inf

type Identify struct {
	// protocolVersion determines compatibility between peers
	ProtocolVersion *string `protobuf:"bytes,5,opt,name=protocolVersion" json:"protocolVersion,omitempty"`
	// agentVersion is like a UserAgent string in browsers, or client version in bittorrent
	// includes the client name and client.
	AgentVersion *string `protobuf:"bytes,6,opt,name=agentVersion" json:"agentVersion,omitempty"`
	// publicKey is this node's public key (which also gives its node.ID)
	// - may not need to be sent, as secure channel implies it has been sent.
	// - then again, if we change / disable secure channel, may still want it.
	PublicKey []byte `protobuf:"bytes,1,opt,name=publicKey" json:"publicKey,omitempty"`
	// listenAddrs are the multiaddrs the sender node listens for open connections on
	ListenAddrs [][]byte `protobuf:"bytes,2,rep,name=listenAddrs" json:"listenAddrs,omitempty"`
	// oservedAddr is the multiaddr of the remote endpoint that the sender node perceives
	// this is useful information to convey to the other side, as it helps the remote endpoint
	// determine whether its connection to the local peer goes through NAT.
	ObservedAddr []byte `protobuf:"bytes,4,opt,name=observedAddr" json:"observedAddr,omitempty"`
	// protocols are the services this node is running
	Protocols        []string `protobuf:"bytes,3,rep,name=protocols" json:"protocols,omitempty"`
	XXX_unrecognized []byte   `json:"-"`
}

func (m *Identify) Reset()         { *m = Identify{} }
func (m *Identify) String() string { return proto.CompactTextString(m) }
func (*Identify) ProtoMessage()    {}

func (m *Identify) GetProtocolVersion() string {
	if m != nil && m.ProtocolVersion != nil {
		return *m.ProtocolVersion
	}
	return ""
}

func (m *Identify) GetAgentVersion() string {
	if m != nil && m.AgentVersion != nil {
		return *m.AgentVersion
	}
	return ""
}

func (m *Identify) GetPublicKey() []byte {
	if m != nil {
		return m.PublicKey
	}
	return nil
}

func (m *Identify) GetListenAddrs() [][]byte {
	if m != nil {
		return m.ListenAddrs
	}
	return nil
}

func (m *Identify) GetObservedAddr() []byte {
	if m != nil {
		return m.ObservedAddr
	}
	return nil
}

func (m *Identify) GetProtocols() []string {
	if m != nil {
		return m.Protocols
	}
	return nil
}

func init() {
}
<commit_msg>vendor identify pb<commit_after>// Code generated by protoc-gen-gogo.
// source: identify.proto
// DO NOT EDIT!

/*
Package identify_pb is a generated protocol buffer package.

It is generated from these files:
	identify.proto

It has these top-level messages:
	Identify
*/
package identify_pb

import proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/gogoprotobuf/proto"
import json "encoding/json"
import math "math"

// Reference proto, json, and math imports to suppress error if they are not otherwise used.
var _ = proto.Marshal
var _ = &json.SyntaxError{}
var _ = math.Inf

type Identify struct {
	// protocolVersion determines compatibility between peers
	ProtocolVersion *string `protobuf:"bytes,5,opt,name=protocolVersion" json:"protocolVersion,omitempty"`
	// agentVersion is like a UserAgent string in browsers, or client version in bittorrent
	// includes the client name and client.
	AgentVersion *string `protobuf:"bytes,6,opt,name=agentVersion" json:"agentVersion,omitempty"`
	// publicKey is this node's public key (which also gives its node.ID)
	// - may not need to be sent, as secure channel implies it has been sent.
	// - then again, if we change / disable secure channel, may still want it.
	PublicKey []byte `protobuf:"bytes,1,opt,name=publicKey" json:"publicKey,omitempty"`
	// listenAddrs are the multiaddrs the sender node listens for open connections on
	ListenAddrs [][]byte `protobuf:"bytes,2,rep,name=listenAddrs" json:"listenAddrs,omitempty"`
	// oservedAddr is the multiaddr of the remote endpoint that the sender node perceives
	// this is useful information to convey to the other side, as it helps the remote endpoint
	// determine whether its connection to the local peer goes through NAT.
	ObservedAddr []byte `protobuf:"bytes,4,opt,name=observedAddr" json:"observedAddr,omitempty"`
	// protocols are the services this node is running
	Protocols        []string `protobuf:"bytes,3,rep,name=protocols" json:"protocols,omitempty"`
	XXX_unrecognized []byte   `json:"-"`
}

func (m *Identify) Reset()         { *m = Identify{} }
func (m *Identify) String() string { return proto.CompactTextString(m) }
func (*Identify) ProtoMessage()    {}

func (m *Identify) GetProtocolVersion() string {
	if m != nil && m.ProtocolVersion != nil {
		return *m.ProtocolVersion
	}
	return ""
}

func (m *Identify) GetAgentVersion() string {
	if m != nil && m.AgentVersion != nil {
		return *m.AgentVersion
	}
	return ""
}

func (m *Identify) GetPublicKey() []byte {
	if m != nil {
		return m.PublicKey
	}
	return nil
}

func (m *Identify) GetListenAddrs() [][]byte {
	if m != nil {
		return m.ListenAddrs
	}
	return nil
}

func (m *Identify) GetObservedAddr() []byte {
	if m != nil {
		return m.ObservedAddr
	}
	return nil
}

func (m *Identify) GetProtocols() []string {
	if m != nil {
		return m.Protocols
	}
	return nil
}

func init() {
}
<|endoftext|>