<commit_before>package main

import (
	"crypto/tls"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"strings"

	"github.com/lucas-clemente/quic-go/http3"
	"github.com/lucas-clemente/quic-go/internal/protocol"
	"github.com/lucas-clemente/quic-go/interop/http09"
	"golang.org/x/sync/errgroup"
)

var errUnsupported = errors.New("unsupported test case")

func main() {
	logFile, err := os.Create("/logs/log.txt")
	if err != nil {
		fmt.Printf("Could not create log file: %s\n", err.Error())
		os.Exit(1)
	}
	defer logFile.Close()
	log.SetOutput(logFile)

	testcase := os.Getenv("TESTCASE")
	if err := runTestcase(testcase); err != nil {
		if err == errUnsupported {
			fmt.Printf("unsupported test case: %s\n", testcase)
			os.Exit(127)
		}
		fmt.Printf("Downloading files failed: %s\n", err.Error())
		os.Exit(1)
	}
}

func runTestcase(testcase string) error {
	flag.Parse()
	urls := flag.Args()

	switch testcase {
	case "http3":
		r := &http3.RoundTripper{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
		defer r.Close()
		return downloadFiles(r, urls)
	case "handshake", "transfer", "retry":
	case "versionnegotiation":
		return runVersionNegotiationTest(urls)
	case "resumption":
		return runResumptionTest(urls)
	default:
		return errUnsupported
	}

	r := &http09.RoundTripper{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	defer r.Close()
	return downloadFiles(r, urls)
}

func runVersionNegotiationTest(urls []string) error {
	if len(urls) != 1 {
		return errors.New("expected at least 2 URLs")
	}
	protocol.SupportedVersions = []protocol.VersionNumber{0x1a2a3a4a}
	err := downloadFile(&http09.RoundTripper{}, urls[0])
	if err == nil {
		return errors.New("expected version negotiation to fail")
	}
	if !strings.Contains(err.Error(), "No compatible QUIC version found") {
		return fmt.Errorf("expect version negotiation error, got: %s", err.Error())
	}
	return nil
}

func runResumptionTest(urls []string) error {
	if len(urls) < 2 {
		return errors.New("expected at least 2 URLs")
	}
	csc := tls.NewLRUClientSessionCache(1)

	// do the first transfer
	r := &http09.RoundTripper{
		TLSClientConfig: &tls.Config{
			InsecureSkipVerify: true,
			ClientSessionCache: csc,
		},
	}
	if err := downloadFiles(r, urls[:1]); err != nil {
		return err
	}
	r.Close()

	// reestablish the connection, using the session ticket that the server (hopefully provided)
	r = &http09.RoundTripper{
		TLSClientConfig: &tls.Config{
			InsecureSkipVerify: true,
			ClientSessionCache: csc,
		},
	}
	defer r.Close()
	return downloadFiles(r, urls[1:])
}

func downloadFiles(cl http.RoundTripper, urls []string) error {
	var g errgroup.Group
	for _, u := range urls {
		url := u
		g.Go(func() error {
			return downloadFile(cl, url)
		})
	}
	return g.Wait()
}

func downloadFile(cl http.RoundTripper, url string) error {
	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		return err
	}
	rsp, err := cl.RoundTrip(req)
	if err != nil {
		return err
	}
	defer rsp.Body.Close()

	file, err := os.Create("/downloads" + req.URL.Path)
	if err != nil {
		return err
	}
	defer file.Close()
	_, err = io.Copy(file, rsp.Body)
	return err
}
<commit_msg>export a key log file from the client in the interop runner<commit_after>package main

import (
	"crypto/tls"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"strings"

	"github.com/lucas-clemente/quic-go/http3"
	"github.com/lucas-clemente/quic-go/internal/protocol"
	"github.com/lucas-clemente/quic-go/interop/http09"
	"golang.org/x/sync/errgroup"
)

var errUnsupported = errors.New("unsupported test case")

var tlsConf *tls.Config

func main() {
	logFile, err := os.Create("/logs/log.txt")
	if err != nil {
		fmt.Printf("Could not create log file: %s\n", err.Error())
		os.Exit(1)
	}
	defer logFile.Close()
	log.SetOutput(logFile)

	keyLog, err := os.Create("/logs/keylogfile.txt")
	if err != nil {
		fmt.Printf("Could not create key log file: %s\n", err.Error())
		os.Exit(1)
	}
	defer keyLog.Close()

	tlsConf = &tls.Config{
		InsecureSkipVerify: true,
		KeyLogWriter:       keyLog,
	}
	testcase := os.Getenv("TESTCASE")
	if err := runTestcase(testcase); err != nil {
		if err == errUnsupported {
			fmt.Printf("unsupported test case: %s\n", testcase)
			os.Exit(127)
		}
		fmt.Printf("Downloading files failed: %s\n", err.Error())
		os.Exit(1)
	}
}

func runTestcase(testcase string) error {
	flag.Parse()
	urls := flag.Args()

	switch testcase {
	case "http3":
		r := &http3.RoundTripper{TLSClientConfig: tlsConf}
		defer r.Close()
		return downloadFiles(r, urls)
	case "handshake", "transfer", "retry":
	case "versionnegotiation":
		return runVersionNegotiationTest(urls)
	case "resumption":
		return runResumptionTest(urls)
	default:
		return errUnsupported
	}

	r := &http09.RoundTripper{TLSClientConfig: tlsConf}
	defer r.Close()
	return downloadFiles(r, urls)
}

func runVersionNegotiationTest(urls []string) error {
	if len(urls) != 1 {
		return errors.New("expected at least 2 URLs")
	}
	protocol.SupportedVersions = []protocol.VersionNumber{0x1a2a3a4a}
	err := downloadFile(&http09.RoundTripper{}, urls[0])
	if err == nil {
		return errors.New("expected version negotiation to fail")
	}
	if !strings.Contains(err.Error(), "No compatible QUIC version found") {
		return fmt.Errorf("expect version negotiation error, got: %s", err.Error())
	}
	return nil
}

func runResumptionTest(urls []string) error {
	if len(urls) < 2 {
		return errors.New("expected at least 2 URLs")
	}

	tlsConf.ClientSessionCache = tls.NewLRUClientSessionCache(1)

	// do the first transfer
	r := &http09.RoundTripper{TLSClientConfig: tlsConf}
	if err := downloadFiles(r, urls[:1]); err != nil {
		return err
	}
	r.Close()

	// reestablish the connection, using the session ticket that the server (hopefully provided)
	r = &http09.RoundTripper{TLSClientConfig: tlsConf}
	defer r.Close()
	return downloadFiles(r, urls[1:])
}

func downloadFiles(cl http.RoundTripper, urls []string) error {
	var g errgroup.Group
	for _, u := range urls {
		url := u
		g.Go(func() error {
			return downloadFile(cl, url)
		})
	}
	return g.Wait()
}

func downloadFile(cl http.RoundTripper, url string) error {
	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		return err
	}
	rsp, err := cl.RoundTrip(req)
	if err != nil {
		return err
	}
	defer rsp.Body.Close()

	file, err := os.Create("/downloads" + req.URL.Path)
	if err != nil {
		return err
	}
	defer file.Close()
	_, err = io.Copy(file, rsp.Body)
	return err
}
<|endoftext|>


<commit_before>package ipfs

import (
	"context"
	"encoding/hex"
	"errors"

	//routinghelpers "gx/ipfs/QmRCrPXk2oUwpK1Cj2FXrUotRpddUxz56setkny2gz13Cx/go-libp2p-routing-helpers"
	dht "gx/ipfs/QmSY3nkMNLzh9GdbFKK5tT7YMfLpf52iUZ8ZRkr29MJaa5/go-libp2p-kad-dht"
	routing "gx/ipfs/QmYxUdYY9S6yg5tSPVin5GFTvtfsLauVcr7reHDD3dM8xf/go-libp2p-routing"
	ropts "gx/ipfs/QmYxUdYY9S6yg5tSPVin5GFTvtfsLauVcr7reHDD3dM8xf/go-libp2p-routing/options"
	record "gx/ipfs/QmbeHtaBy9nZsW4cHRcvgVY4CnDhXudE2Dr6qDxS7yg9rX/go-libp2p-record"
)

var (
	ErrCachingRouterIncorrectRoutingType = errors.New("Incorrect routing type")
)

type CachingRouter struct {
	apiRouter *APIRouter
	routing.IpfsRouting
	RecordValidator record.Validator
}

func NewCachingRouter(dht *dht.IpfsDHT, apiRouter *APIRouter) *CachingRouter {
	return &CachingRouter{
		apiRouter:       apiRouter,
		IpfsRouting:     dht,
		RecordValidator: dht.Validator,
	}
}

func (r *CachingRouter) DHT() (*dht.IpfsDHT, error) {
	dht, ok := r.IpfsRouting.(*dht.IpfsDHT)
	if !ok {
		return nil, ErrCachingRouterIncorrectRoutingType
	}
	return dht, nil
}

func (r *CachingRouter) APIRouter() *APIRouter {
	return r.apiRouter
}

func (r *CachingRouter) PutValue(ctx context.Context, key string, value []byte, opts ...ropts.Option) error {
	// Write to the tiered router in the background then write to the caching
	// router and return
	var err error
	if err = r.IpfsRouting.PutValue(ctx, key, value, opts...); err != nil {
		log.Errorf("ipfs dht put (%s): %s", hex.EncodeToString([]byte(key)), err)
		return err
	}
	if err = r.apiRouter.PutValue(ctx, key, value, opts...); err != nil {
		log.Errorf("api cache put (%s): %s", hex.EncodeToString([]byte(key)), err)
	}
	return err
}

func (r *CachingRouter) GetValue(ctx context.Context, key string, opts ...ropts.Option) ([]byte, error) {
	// First check the DHT router. If it's successful return the value otherwise
	// continue on to check the other routers.
	val, err := r.IpfsRouting.GetValue(ctx, key, opts...)
	if err != nil && len(val) == 0 {
		// No values from the DHT, check the API cache
		log.Warningf("ipfs dht lookup was empty: %s", err.Error())
		if val, err = r.apiRouter.GetValue(ctx, key, opts...); err != nil && len(val) == 0 {
			// No values still, report NotFound
			return nil, routing.ErrNotFound
		}
	}
	if err := r.apiRouter.PutValue(ctx, key, val, opts...); err != nil {
		log.Errorf("api cache put found dht value (%s): %s", hex.EncodeToString([]byte(key)), err.Error())
	}
	return val, nil
}

func (r *CachingRouter) SearchValue(ctx context.Context, key string, opts ...ropts.Option) (<-chan []byte, error) {
	// TODO: Restore parallel lookup once validation is properly applied to
	// the apiRouter results ensuring it doesn't return invalid records before the
	// IpfsRouting object can. For some reason the validation is not being considered
	// on returned results.
	return r.IpfsRouting.SearchValue(ctx, key, opts...)
	//return routinghelpers.Parallel{
	//Routers: []routing.IpfsRouting{
	//r.IpfsRouting,
	//r.apiRouter,
	//},
	//Validator: r.RecordValidator,
	//}.SearchValue(ctx, key, opts...)
}
<commit_msg>[#1557] Make CachingRouter implement routing.PubKeyFetcher<commit_after>package ipfs

import (
	"context"
	"encoding/hex"
	"errors"

	dht "gx/ipfs/QmSY3nkMNLzh9GdbFKK5tT7YMfLpf52iUZ8ZRkr29MJaa5/go-libp2p-kad-dht"
	ci "gx/ipfs/QmTW4SdgBWq9GjsBsHeUx8WuGxzhgzAf88UMH2w62PC8yK/go-libp2p-crypto"
	peer "gx/ipfs/QmYVXrKrKHDC9FobgmcmshCDyWwdrfwfanNQN4oxJ9Fk3h/go-libp2p-peer"
	routing "gx/ipfs/QmYxUdYY9S6yg5tSPVin5GFTvtfsLauVcr7reHDD3dM8xf/go-libp2p-routing"
	ropts "gx/ipfs/QmYxUdYY9S6yg5tSPVin5GFTvtfsLauVcr7reHDD3dM8xf/go-libp2p-routing/options"
	record "gx/ipfs/QmbeHtaBy9nZsW4cHRcvgVY4CnDhXudE2Dr6qDxS7yg9rX/go-libp2p-record"
)

var (
	ErrCachingRouterIncorrectRoutingType = errors.New("Incorrect routing type")
)

type CachingRouter struct {
	apiRouter *APIRouter
	routing.IpfsRouting
	RecordValidator record.Validator
}

func NewCachingRouter(dht *dht.IpfsDHT, apiRouter *APIRouter) *CachingRouter {
	return &CachingRouter{
		apiRouter:       apiRouter,
		IpfsRouting:     dht,
		RecordValidator: dht.Validator,
	}
}

func (r *CachingRouter) DHT() (*dht.IpfsDHT, error) {
	dht, ok := r.IpfsRouting.(*dht.IpfsDHT)
	if !ok {
		return nil, ErrCachingRouterIncorrectRoutingType
	}
	return dht, nil
}

func (r *CachingRouter) APIRouter() *APIRouter {
	return r.apiRouter
}

func (r *CachingRouter) PutValue(ctx context.Context, key string, value []byte, opts ...ropts.Option) error {
	// Write to the tiered router in the background then write to the caching
	// router and return
	var err error
	if err = r.IpfsRouting.PutValue(ctx, key, value, opts...); err != nil {
		log.Errorf("ipfs dht put (%s): %s", hex.EncodeToString([]byte(key)), err)
		return err
	}
	if err = r.apiRouter.PutValue(ctx, key, value, opts...); err != nil {
		log.Errorf("api cache put (%s): %s", hex.EncodeToString([]byte(key)), err)
	}
	return err
}

func (r *CachingRouter) GetValue(ctx context.Context, key string, opts ...ropts.Option) ([]byte, error) {
	// First check the DHT router. If it's successful return the value otherwise
	// continue on to check the other routers.
	val, err := r.IpfsRouting.GetValue(ctx, key, opts...)
	if err != nil && len(val) == 0 {
		// No values from the DHT, check the API cache
		log.Warningf("ipfs dht lookup was empty: %s", err.Error())
		if val, err = r.apiRouter.GetValue(ctx, key, opts...); err != nil && len(val) == 0 {
			// No values still, report NotFound
			return nil, routing.ErrNotFound
		}
	}
	if err := r.apiRouter.PutValue(ctx, key, val, opts...); err != nil {
		log.Errorf("api cache put found dht value (%s): %s", hex.EncodeToString([]byte(key)), err.Error())
	}
	return val, nil
}

func (r *CachingRouter) GetPublicKey(ctx context.Context, p peer.ID) (ci.PubKey, error) {
	if dht, ok := r.IpfsRouting.(routing.PubKeyFetcher); ok {
		return dht.GetPublicKey(ctx, p)
	}
	return nil, routing.ErrNotSupported
}

func (r *CachingRouter) SearchValue(ctx context.Context, key string, opts ...ropts.Option) (<-chan []byte, error) {
	// TODO: Restore parallel lookup once validation is properly applied to
	// the apiRouter results ensuring it doesn't return invalid records before the
	// IpfsRouting object can. For some reason the validation is not being considered
	// on returned results.
	return r.IpfsRouting.SearchValue(ctx, key, opts...)
	//return routinghelpers.Parallel{
	//Routers: []routing.IpfsRouting{
	//r.IpfsRouting,
	//r.apiRouter,
	//},
	//Validator: r.RecordValidator,
	//}.SearchValue(ctx, key, opts...)
}
<|endoftext|>


<commit_before>package scrypt

import (
	"bytes"
	"testing"
)

// TestSamePassphrase checks whether same passphrase returns different keys
// due to random salt everytime
func TestSamePassphrase(t *testing.T) {
	passphrase := "Hello there how are you doing"
	key1, err := DerivePassphrase(passphrase, 34)
	if err != nil {
		t.Errorf("Error returned: %s\n", err)
	}
	t.Logf("Returned key is - %v", key1)

	var key2 []byte
	key2, err = DerivePassphrase(passphrase, 34)
	if err != nil {
		t.Errorf("Error returned: %s\n", err)
	}
	t.Logf("Returned key is - %v", key2)

	if bytes.Equal(key1, key2) {
		t.Errorf("The 2 keys are the same for the same passphrase. Key1- %b, Key2- %b",
			key1, key2)
	}

}

// TestVerifyPassphrase checks whether the same passphrase passes the verify
// function or not
func TestVerifyPassphrase(t *testing.T) {
	passphrase_list := []string{
		"Hello there how are you doing",
		"this is bad",
		"oug84-3T[wZHcx*';k;=m",
	}

	for _, item := range passphrase_list {
		key, err := DerivePassphrase(item, 32)
		if err != nil {
			t.Errorf("Error returned: %s\n", err)
		}

		var result bool
		result, err = VerifyPassphrase(item, key)
		if err != nil {
			t.Errorf("Error returned: %s\n", err)
		}
		if !result {
			t.Errorf("Passphrase did not match\n")
		}
	}
}

// TestFailVerifyPassphrase derives one passphrase and tests with another
// passphrase to verify whether it fails or not
func TestFailVerifyPassphrase(t *testing.T) {
	passphrase := "Hello there how are you doing"

	key, err := DerivePassphrase(passphrase, 32)
	if err != nil {
		t.Errorf("Error returned: %s\n", err)
	}

	var result bool
	result, err = VerifyPassphrase("This should fail", key)
	if err != nil {
		t.Errorf("Error returned: %s\n", err)
	}
	if result {
		t.Errorf("The outputs matched whereas it shouldn't have\n")
	}
}
<commit_msg>Added Arabic and Chinese characters in test<commit_after>package scrypt

import (
	"bytes"
	"testing"
)

// TestSamePassphrase checks whether same passphrase returns different keys
// due to random salt everytime
func TestSamePassphrase(t *testing.T) {
	passphrase := "Hello there how are you doing"
	key1, err := DerivePassphrase(passphrase, 34)
	if err != nil {
		t.Errorf("Error returned: %s\n", err)
	}
	t.Logf("Returned key is - %v", key1)

	var key2 []byte
	key2, err = DerivePassphrase(passphrase, 34)
	if err != nil {
		t.Errorf("Error returned: %s\n", err)
	}
	t.Logf("Returned key is - %v", key2)

	if bytes.Equal(key1, key2) {
		t.Errorf("The 2 keys are the same for the same passphrase. Key1- %b, Key2- %b",
			key1, key2)
	}

}

// TestVerifyPassphrase checks whether the same passphrase passes the verify
// function or not
func TestVerifyPassphrase(t *testing.T) {
	passphrase_list := []string{
		"Hello there how are you doing",
		"this is bad",
		"oug84-3T[wZHcx*';k;=m",
		"指事字 zhǐshìzì",
		" الأَبْجَدِيَّة العَرَبِيَّة",
	}

	for _, item := range passphrase_list {
		key, err := DerivePassphrase(item, 32)
		if err != nil {
			t.Errorf("Error returned: %s\n", err)
		}

		var result bool
		result, err = VerifyPassphrase(item, key)
		if err != nil {
			t.Errorf("Error returned: %s\n", err)
		}
		if !result {
			t.Errorf("Passphrase did not match\n")
		}
	}
}

// TestFailVerifyPassphrase derives one passphrase and tests with another
// passphrase to verify whether it fails or not
func TestFailVerifyPassphrase(t *testing.T) {
	passphrase := "Hello there how are you doing"

	key, err := DerivePassphrase(passphrase, 32)
	if err != nil {
		t.Errorf("Error returned: %s\n", err)
	}

	var result bool
	result, err = VerifyPassphrase("This should fail", key)
	if err != nil {
		t.Errorf("Error returned: %s\n", err)
	}
	if result {
		t.Errorf("The outputs matched whereas it shouldn't have\n")
	}
}
<|endoftext|>


<commit_before>package lidar

import (
	"context"
	"sync"
	"time"

	"code.cloudfoundry.org/lager"
	"github.com/concourse/concourse/atc/creds"
	"github.com/concourse/concourse/atc/db"
	"github.com/concourse/concourse/atc/metric"
	"github.com/pkg/errors"
)

func NewScanner(
	logger lager.Logger,
	checkFactory db.CheckFactory,
	secrets creds.Secrets,
	defaultCheckTimeout time.Duration,
	defaultCheckInterval time.Duration,
	defaultWithWebhookCheckInterval time.Duration,
) *scanner {
	// In case that a user configures resource-checking-interval, but forgets to
	// configure resource-with-webhook-checking-interval, keep both checking-
	// intervals consistent. Even if both intervals are configured, there is no
	// reason webhooked resources take shorter checking interval than normal
	// resources.
	if defaultWithWebhookCheckInterval < defaultCheckInterval {
		logger.Info("update-default-with-webhook-check-interface",
			lager.Data{"oldValue": defaultWithWebhookCheckInterval, "newValue": defaultCheckInterval})
		defaultWithWebhookCheckInterval = defaultCheckInterval
	}

	return &scanner{
		logger:                          logger,
		checkFactory:                    checkFactory,
		secrets:                         secrets,
		defaultCheckTimeout:             defaultCheckTimeout,
		defaultCheckInterval:            defaultCheckInterval,
		defaultWithWebhookCheckInterval: defaultWithWebhookCheckInterval,
	}
}

type scanner struct {
	logger lager.Logger

	checkFactory                    db.CheckFactory
	secrets                         creds.Secrets
	defaultCheckTimeout             time.Duration
	defaultCheckInterval            time.Duration
	defaultWithWebhookCheckInterval time.Duration
}

func (s *scanner) Run(ctx context.Context) error {
	s.logger.Info("start")
	defer s.logger.Info("end")

	resources, err := s.checkFactory.Resources()
	if err != nil {
		s.logger.Error("failed-to-get-resources", err)
		return err
	}

	resourceTypes, err := s.checkFactory.ResourceTypes()
	if err != nil {
		s.logger.Error("failed-to-get-resources", err)
		return err
	}

	waitGroup := new(sync.WaitGroup)
	resourceTypesChecked := &sync.Map{}

	for _, resource := range resources {
		waitGroup.Add(1)

		go func(resource db.Resource, resourceTypes db.ResourceTypes) {
			defer waitGroup.Done()

			err := s.check(resource, resourceTypes, resourceTypesChecked)
			s.setCheckError(s.logger, resource, err)

		}(resource, resourceTypes)
	}

	waitGroup.Wait()

	return s.checkFactory.NotifyChecker()
}

func (s *scanner) check(checkable db.Checkable, resourceTypes db.ResourceTypes, resourceTypesChecked *sync.Map) error {

	var err error

	parentType, found := resourceTypes.Parent(checkable)
	if found {
		if _, exists := resourceTypesChecked.LoadOrStore(parentType.ID(), true); !exists {
			// only create a check for resource type if it has not been checked yet
			err = s.check(parentType, resourceTypes, resourceTypesChecked)
			s.setCheckError(s.logger, parentType, err)

			if err != nil {
				s.logger.Error("failed-to-create-type-check", err)
				return errors.Wrapf(err, "parent type '%v' error", parentType.Name())
			}
		}
	}

	interval := s.defaultCheckInterval
	if checkable.HasWebhook() {
		interval = s.defaultWithWebhookCheckInterval
	}
	if every := checkable.CheckEvery(); every != "" {
		interval, err = time.ParseDuration(every)
		if err != nil {
			s.logger.Error("failed-to-parse-check-every", err)
			return err
		}
	}

	if time.Now().Before(checkable.LastCheckEndTime().Add(interval)) {
		s.logger.Debug("interval-not-reached", lager.Data{"interval": interval})
		return nil
	}

	version := checkable.CurrentPinnedVersion()

	_, created, err := s.checkFactory.TryCreateCheck(s.logger, checkable, resourceTypes, version, false)
	if err != nil {
		s.logger.Error("failed-to-create-check", err)
		return err
	}

	if !created {
		s.logger.Debug("check-already-exists")
	}

	metric.ChecksEnqueued.Inc()

	return nil
}

func (s *scanner) setCheckError(logger lager.Logger, checkable db.Checkable, err error) {
	setErr := checkable.SetCheckSetupError(err)
	if setErr != nil {
		logger.Error("failed-to-set-check-error", setErr)
	}
}
<commit_msg>fixed a typo in log,<commit_after>package lidar

import (
	"context"
	"sync"
	"time"

	"code.cloudfoundry.org/lager"
	"github.com/concourse/concourse/atc/creds"
	"github.com/concourse/concourse/atc/db"
	"github.com/concourse/concourse/atc/metric"
	"github.com/pkg/errors"
)

func NewScanner(
	logger lager.Logger,
	checkFactory db.CheckFactory,
	secrets creds.Secrets,
	defaultCheckTimeout time.Duration,
	defaultCheckInterval time.Duration,
	defaultWithWebhookCheckInterval time.Duration,
) *scanner {
	// In case that a user configures resource-checking-interval, but forgets to
	// configure resource-with-webhook-checking-interval, keep both checking-
	// intervals consistent. Even if both intervals are configured, there is no
	// reason webhooked resources take shorter checking interval than normal
	// resources.
	if defaultWithWebhookCheckInterval < defaultCheckInterval {
		logger.Info("update-default-with-webhook-check-interval",
			lager.Data{"oldValue": defaultWithWebhookCheckInterval, "newValue": defaultCheckInterval})
		defaultWithWebhookCheckInterval = defaultCheckInterval
	}

	return &scanner{
		logger:                          logger,
		checkFactory:                    checkFactory,
		secrets:                         secrets,
		defaultCheckTimeout:             defaultCheckTimeout,
		defaultCheckInterval:            defaultCheckInterval,
		defaultWithWebhookCheckInterval: defaultWithWebhookCheckInterval,
	}
}

type scanner struct {
	logger lager.Logger

	checkFactory                    db.CheckFactory
	secrets                         creds.Secrets
	defaultCheckTimeout             time.Duration
	defaultCheckInterval            time.Duration
	defaultWithWebhookCheckInterval time.Duration
}

func (s *scanner) Run(ctx context.Context) error {
	s.logger.Info("start")
	defer s.logger.Info("end")

	resources, err := s.checkFactory.Resources()
	if err != nil {
		s.logger.Error("failed-to-get-resources", err)
		return err
	}

	resourceTypes, err := s.checkFactory.ResourceTypes()
	if err != nil {
		s.logger.Error("failed-to-get-resources", err)
		return err
	}

	waitGroup := new(sync.WaitGroup)
	resourceTypesChecked := &sync.Map{}

	for _, resource := range resources {
		waitGroup.Add(1)

		go func(resource db.Resource, resourceTypes db.ResourceTypes) {
			defer waitGroup.Done()

			err := s.check(resource, resourceTypes, resourceTypesChecked)
			s.setCheckError(s.logger, resource, err)

		}(resource, resourceTypes)
	}

	waitGroup.Wait()

	return s.checkFactory.NotifyChecker()
}

func (s *scanner) check(checkable db.Checkable, resourceTypes db.ResourceTypes, resourceTypesChecked *sync.Map) error {

	var err error

	parentType, found := resourceTypes.Parent(checkable)
	if found {
		if _, exists := resourceTypesChecked.LoadOrStore(parentType.ID(), true); !exists {
			// only create a check for resource type if it has not been checked yet
			err = s.check(parentType, resourceTypes, resourceTypesChecked)
			s.setCheckError(s.logger, parentType, err)

			if err != nil {
				s.logger.Error("failed-to-create-type-check", err)
				return errors.Wrapf(err, "parent type '%v' error", parentType.Name())
			}
		}
	}

	interval := s.defaultCheckInterval
	if checkable.HasWebhook() {
		interval = s.defaultWithWebhookCheckInterval
	}
	if every := checkable.CheckEvery(); every != "" {
		interval, err = time.ParseDuration(every)
		if err != nil {
			s.logger.Error("failed-to-parse-check-every", err)
			return err
		}
	}

	if time.Now().Before(checkable.LastCheckEndTime().Add(interval)) {
		s.logger.Debug("interval-not-reached", lager.Data{"interval": interval})
		return nil
	}

	version := checkable.CurrentPinnedVersion()

	_, created, err := s.checkFactory.TryCreateCheck(s.logger, checkable, resourceTypes, version, false)
	if err != nil {
		s.logger.Error("failed-to-create-check", err)
		return err
	}

	if !created {
		s.logger.Debug("check-already-exists")
	}

	metric.ChecksEnqueued.Inc()

	return nil
}

func (s *scanner) setCheckError(logger lager.Logger, checkable db.Checkable, err error) {
	setErr := checkable.SetCheckSetupError(err)
	if setErr != nil {
		logger.Error("failed-to-set-check-error", setErr)
	}
}
<|endoftext|>


<commit_before>package dbr

import (
	"reflect"
	"time"
)

// Unvetted thots:
// Given a query and given a structure (field list), there's 2 sets of fields.
// Take the intersection. We can fill those in. great.
// For fields in the structure that aren't in the query, we'll let that slide if db:"-"
// For fields in the structure that aren't in the query but without db:"-", return error
// For fields in the query that aren't in the structure, we'll ignore them.

// dest can be:
// - addr of a structure
// - addr of slice of pointers to structures
// - map of pointers to structures (addr of map also ok)
// If it's a single structure, only the first record returned will be set.
// If it's a slice or map, the slice/map won't be emptied first. New records will be allocated for each found record.
// If its a map, there is the potential to overwrite values (keys are 'id')
// Returns the number of items found (which is not necessarily the # of items set)
func (b *SelectBuilder) LoadAll(dest interface{}) (int, error) {
	//
	// Validate the dest, and extract the reflection values we need.
	//
	valueOfDest := reflect.ValueOf(dest) // We want this to eventually be a map or slice
	kindOfDest := valueOfDest.Kind()     // And this eventually needs to be a map or slice as well

	if kindOfDest == reflect.Ptr {
		valueOfDest = reflect.Indirect(valueOfDest)
		kindOfDest = valueOfDest.Kind()
	} else if kindOfDest == reflect.Map {
		// we're good
	} else {
		panic("invalid type passed to LoadAll. Need a map or addr of slice")
	}

	if !(kindOfDest == reflect.Map || kindOfDest == reflect.Slice) {
		panic("invalid type passed to LoadAll. Need a map or addr of slice")
	}

	recordType := valueOfDest.Type().Elem()
	if recordType.Kind() != reflect.Ptr {
		panic("Elements need to be pointers to structures")
	}

	recordType = recordType.Elem()
	if recordType.Kind() != reflect.Struct {
		panic("Elements need to be pointers to structures")
	}

	//
	// Get full SQL
	//
	fullSql, err := Interpolate(b.ToSql())
	if err != nil {
		return 0, b.EventErr("dbr.select.load_all.interpolate", err)
	}

	numberOfRowsReturned := 0

	// Start the timer:
	startTime := time.Now()
	defer func() { b.TimingKv("dbr.select", time.Since(startTime).Nanoseconds(), kvs{"sql": fullSql}) }()

	// Run the query:
	rows, err := b.runner.Query(fullSql)
	if err != nil {
		return 0, b.EventErrKv("dbr.select.load_all.query", err, kvs{"sql": fullSql})
	}
	defer rows.Close()

	// Get the columns returned
	columns, err := rows.Columns()
	if err != nil {
		return numberOfRowsReturned, b.EventErrKv("dbr.select.load_one.rows.Columns", err, kvs{"sql": fullSql})
	}

	// Create a map of this result set to the struct fields
	fieldMap, err := b.calculateFieldMap(recordType, columns, false)
	if err != nil {
		return numberOfRowsReturned, b.EventErrKv("dbr.select.load_all.calculateFieldMap", err, kvs{"sql": fullSql})
	}

	// Iterate over rows
	if kindOfDest == reflect.Slice {
		sliceValue := valueOfDest
		for rows.Next() {
			// Create a new record to store our row:
			pointerToNewRecord := reflect.New(recordType)
			newRecord := reflect.Indirect(pointerToNewRecord)

			// Build a 'holder', which is an []interface{}. Each value will be the address of the field corresponding to our newly made record:
			holder, err := b.holderFor(newRecord, fieldMap)
			if err != nil {
				return numberOfRowsReturned, b.EventErrKv("dbr.select.load_all.holderFor", err, kvs{"sql": fullSql})
			}

			// Load up our new structure with the row's values
			err = rows.Scan(holder...)
			if err != nil {
				return numberOfRowsReturned, b.EventErrKv("dbr.select.load_all.scan", err, kvs{"sql": fullSql})
			}

			// Append our new record to the slice:
			sliceValue = reflect.Append(sliceValue, pointerToNewRecord)

			numberOfRowsReturned += 1
		}
		valueOfDest.Set(sliceValue)
	} else { // Map

	}

	// Check for errors at the end. Supposedly these are error that can happen during iteration.
	if err = rows.Err(); err != nil {
		return numberOfRowsReturned, b.EventErrKv("dbr.select.load_all.rows_err", err, kvs{"sql": fullSql})
	}

	return numberOfRowsReturned, nil
}

// Returns ErrNotFound if nothing was found
func (b *SelectBuilder) LoadOne(dest interface{}) error {
	//
	// Validate the dest, and extract the reflection values we need.
	//
	valueOfDest := reflect.ValueOf(dest)
	indirectOfDest := reflect.Indirect(valueOfDest)
	kindOfDest := valueOfDest.Kind()

	if kindOfDest != reflect.Ptr || indirectOfDest.Kind() != reflect.Struct {
		panic("you need to pass in the address of a struct")
	}

	recordType := indirectOfDest.Type()

	//
	// Get full SQL
	//
	fullSql, err := Interpolate(b.ToSql())
	if err != nil {
		return err
	}

	// Start the timer:
	startTime := time.Now()
	defer func() { b.TimingKv("dbr.select", time.Since(startTime).Nanoseconds(), kvs{"sql": fullSql}) }()

	// Run the query:
	rows, err := b.runner.Query(fullSql)
	if err != nil {
		return b.EventErrKv("dbr.select.load_one.query", err, kvs{"sql": fullSql})
	}
	defer rows.Close()

	// Get the columns of this result set
	columns, err := rows.Columns()
	if err != nil {
		return b.EventErrKv("dbr.select.load_one.rows.Columns", err, kvs{"sql": fullSql})
	}

	// Create a map of this result set to the struct columns
	fieldMap, err := b.calculateFieldMap(recordType, columns, false)
	if err != nil {
		return b.EventErrKv("dbr.select.load_one.calculateFieldMap", err, kvs{"sql": fullSql})
	}

	if rows.Next() {
		// Build a 'holder', which is an []interface{}. Each value will be the address of the field corresponding to our newly made record:
		holder, err := b.holderFor(indirectOfDest, fieldMap)
		if err != nil {
			return b.EventErrKv("dbr.select.load_one.holderFor", err, kvs{"sql": fullSql})
		}

		// Load up our new structure with the row's values
		err = rows.Scan(holder...)
		if err != nil {
			return b.EventErrKv("dbr.select.load_one.scan", err, kvs{"sql": fullSql})
		}
		return nil
	}

	if err := rows.Err(); err != nil {
		return b.EventErrKv("dbr.select.load_one.rows_err", err, kvs{"sql": fullSql})
	}

	return ErrNotFound
}

// Returns ErrNotFound if no value was found, and it was therefore not set.
func (b *SelectBuilder) LoadValue(dest interface{}) error {
	// TODO: make sure dest is a ptr to something

	//
	// Get full SQL
	//
	fullSql, err := Interpolate(b.ToSql())
	if err != nil {
		return err
	}

	// Start the timer:
	startTime := time.Now()
	defer func() { b.TimingKv("dbr.select", time.Since(startTime).Nanoseconds(), kvs{"sql": fullSql}) }()

	// Run the query:
	rows, err := b.runner.Query(fullSql)
	if err != nil {
		return b.EventErrKv("dbr.select.load_value.query", err, kvs{"sql": fullSql})
	}
	defer rows.Close()

	if rows.Next() {
		err = rows.Scan(dest)
		if err != nil {
			return b.EventErrKv("dbr.select.load_value.scan", err, kvs{"sql": fullSql})
		}
		return nil
	}

	if err := rows.Err(); err != nil {
		return b.EventErrKv("dbr.select.load_value.rows_err", err, kvs{"sql": fullSql})
	}

	return ErrNotFound
}
<commit_msg>TWEAK: Validate destination interface{} is a pointer because it was an unfinished TODO. NOTESTS<commit_after>package dbr

import (
	"reflect"
	"time"
)

// Unvetted thots:
// Given a query and given a structure (field list), there's 2 sets of fields.
// Take the intersection. We can fill those in. great.
// For fields in the structure that aren't in the query, we'll let that slide if db:"-"
// For fields in the structure that aren't in the query but without db:"-", return error
// For fields in the query that aren't in the structure, we'll ignore them.

// dest can be:
// - addr of a structure
// - addr of slice of pointers to structures
// - map of pointers to structures (addr of map also ok)
// If it's a single structure, only the first record returned will be set.
// If it's a slice or map, the slice/map won't be emptied first. New records will be allocated for each found record.
// If its a map, there is the potential to overwrite values (keys are 'id')
// Returns the number of items found (which is not necessarily the # of items set)
func (b *SelectBuilder) LoadAll(dest interface{}) (int, error) {
	//
	// Validate the dest, and extract the reflection values we need.
	//
	valueOfDest := reflect.ValueOf(dest) // We want this to eventually be a map or slice
	kindOfDest := valueOfDest.Kind()     // And this eventually needs to be a map or slice as well

	if kindOfDest == reflect.Ptr {
		valueOfDest = reflect.Indirect(valueOfDest)
		kindOfDest = valueOfDest.Kind()
	} else if kindOfDest == reflect.Map {
		// we're good
	} else {
		panic("invalid type passed to LoadAll. Need a map or addr of slice")
	}

	if !(kindOfDest == reflect.Map || kindOfDest == reflect.Slice) {
		panic("invalid type passed to LoadAll. Need a map or addr of slice")
	}

	recordType := valueOfDest.Type().Elem()
	if recordType.Kind() != reflect.Ptr {
		panic("Elements need to be pointers to structures")
	}

	recordType = recordType.Elem()
	if recordType.Kind() != reflect.Struct {
		panic("Elements need to be pointers to structures")
	}

	//
	// Get full SQL
	//
	fullSql, err := Interpolate(b.ToSql())
	if err != nil {
		return 0, b.EventErr("dbr.select.load_all.interpolate", err)
	}

	numberOfRowsReturned := 0

	// Start the timer:
	startTime := time.Now()
	defer func() { b.TimingKv("dbr.select", time.Since(startTime).Nanoseconds(), kvs{"sql": fullSql}) }()

	// Run the query:
	rows, err := b.runner.Query(fullSql)
	if err != nil {
		return 0, b.EventErrKv("dbr.select.load_all.query", err, kvs{"sql": fullSql})
	}
	defer rows.Close()

	// Get the columns returned
	columns, err := rows.Columns()
	if err != nil {
		return numberOfRowsReturned, b.EventErrKv("dbr.select.load_one.rows.Columns", err, kvs{"sql": fullSql})
	}

	// Create a map of this result set to the struct fields
	fieldMap, err := b.calculateFieldMap(recordType, columns, false)
	if err != nil {
		return numberOfRowsReturned, b.EventErrKv("dbr.select.load_all.calculateFieldMap", err, kvs{"sql": fullSql})
	}

	// Iterate over rows
	if kindOfDest == reflect.Slice {
		sliceValue := valueOfDest
		for rows.Next() {
			// Create a new record to store our row:
			pointerToNewRecord := reflect.New(recordType)
			newRecord := reflect.Indirect(pointerToNewRecord)

			// Build a 'holder', which is an []interface{}. Each value will be the address of the field corresponding to our newly made record:
			holder, err := b.holderFor(newRecord, fieldMap)
			if err != nil {
				return numberOfRowsReturned, b.EventErrKv("dbr.select.load_all.holderFor", err, kvs{"sql": fullSql})
			}

			// Load up our new structure with the row's values
			err = rows.Scan(holder...)
			if err != nil {
				return numberOfRowsReturned, b.EventErrKv("dbr.select.load_all.scan", err, kvs{"sql": fullSql})
			}

			// Append our new record to the slice:
			sliceValue = reflect.Append(sliceValue, pointerToNewRecord)

			numberOfRowsReturned += 1
		}
		valueOfDest.Set(sliceValue)
	} else { // Map

	}

	// Check for errors at the end. Supposedly these are error that can happen during iteration.
	if err = rows.Err(); err != nil {
		return numberOfRowsReturned, b.EventErrKv("dbr.select.load_all.rows_err", err, kvs{"sql": fullSql})
	}

	return numberOfRowsReturned, nil
}

// Returns ErrNotFound if nothing was found
func (b *SelectBuilder) LoadOne(dest interface{}) error {
	//
	// Validate the dest, and extract the reflection values we need.
	//
	valueOfDest := reflect.ValueOf(dest)
	indirectOfDest := reflect.Indirect(valueOfDest)
	kindOfDest := valueOfDest.Kind()

	if kindOfDest != reflect.Ptr || indirectOfDest.Kind() != reflect.Struct {
		panic("you need to pass in the address of a struct")
	}

	recordType := indirectOfDest.Type()

	//
	// Get full SQL
	//
	fullSql, err := Interpolate(b.ToSql())
	if err != nil {
		return err
	}

	// Start the timer:
	startTime := time.Now()
	defer func() { b.TimingKv("dbr.select", time.Since(startTime).Nanoseconds(), kvs{"sql": fullSql}) }()

	// Run the query:
	rows, err := b.runner.Query(fullSql)
	if err != nil {
		return b.EventErrKv("dbr.select.load_one.query", err, kvs{"sql": fullSql})
	}
	defer rows.Close()

	// Get the columns of this result set
	columns, err := rows.Columns()
	if err != nil {
		return b.EventErrKv("dbr.select.load_one.rows.Columns", err, kvs{"sql": fullSql})
	}

	// Create a map of this result set to the struct columns
	fieldMap, err := b.calculateFieldMap(recordType, columns, false)
	if err != nil {
		return b.EventErrKv("dbr.select.load_one.calculateFieldMap", err, kvs{"sql": fullSql})
	}

	if rows.Next() {
		// Build a 'holder', which is an []interface{}. Each value will be the address of the field corresponding to our newly made record:
		holder, err := b.holderFor(indirectOfDest, fieldMap)
		if err != nil {
			return b.EventErrKv("dbr.select.load_one.holderFor", err, kvs{"sql": fullSql})
		}

		// Load up our new structure with the row's values
		err = rows.Scan(holder...)
		if err != nil {
			return b.EventErrKv("dbr.select.load_one.scan", err, kvs{"sql": fullSql})
		}
		return nil
	}

	if err := rows.Err(); err != nil {
		return b.EventErrKv("dbr.select.load_one.rows_err", err, kvs{"sql": fullSql})
	}

	return ErrNotFound
}

// Returns ErrNotFound if no value was found, and it was therefore not set.
func (b *SelectBuilder) LoadValue(dest interface{}) error {
	// Validate the dest
	valueOfDest := reflect.ValueOf(dest)
	kindOfDest := valueOfDest.Kind()

	if kindOfDest != reflect.Ptr {
		panic("Destination must be a pointer")
	}

	//
	// Get full SQL
	//
	fullSql, err := Interpolate(b.ToSql())
	if err != nil {
		return err
	}

	// Start the timer:
	startTime := time.Now()
	defer func() { b.TimingKv("dbr.select", time.Since(startTime).Nanoseconds(), kvs{"sql": fullSql}) }()

	// Run the query:
	rows, err := b.runner.Query(fullSql)
	if err != nil {
		return b.EventErrKv("dbr.select.load_value.query", err, kvs{"sql": fullSql})
	}
	defer rows.Close()

	if rows.Next() {
		err = rows.Scan(dest)
		if err != nil {
			return b.EventErrKv("dbr.select.load_value.scan", err, kvs{"sql": fullSql})
		}
		return nil
	}

	if err := rows.Err(); err != nil {
		return b.EventErrKv("dbr.select.load_value.rows_err", err, kvs{"sql": fullSql})
	}

	return ErrNotFound
}
<|endoftext|>


<commit_before>package docker

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ecr"
	"github.com/aws/aws-sdk-go/service/sts"
	"github.com/mweagle/Sparta/system"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

////////////////////////////////////////////////////////////////////////////////
// CONSTANTS
////////////////////////////////////////////////////////////////////////////////

const (
	// BinaryNameArgument is the argument provided to docker build that
	// supplies the local statically built Go binary
	BinaryNameArgument = "SPARTA_DOCKER_BINARY"
)

// BuildDockerImageWithFlags is an extended version of BuildDockerImage that includes
// support for build time tags
func BuildDockerImageWithFlags(serviceName string,
	dockerFilepath string,
	dockerTags map[string]string,
	buildTags string,
	linkFlags string,
	logger *logrus.Logger) error {

	// BEGIN DOCKER PRECONDITIONS
	// Ensure that serviceName and tags are lowercase to make Docker happy
	var dockerErrors []string
	for eachKey, eachValue := range dockerTags {
		if eachKey != strings.ToLower(eachKey) ||
			eachValue != strings.ToLower(eachValue) {
			dockerErrors = append(dockerErrors, fmt.Sprintf("--tag %s:%s MUST be lower case", eachKey, eachValue))
		}
	}

	if len(dockerErrors) > 0 {
		return errors.Errorf("Docker build errors: %s", strings.Join(dockerErrors[:], ", "))
	}
	// BEGIN Informational - output the docker version...
	dockerVersionCmd := exec.Command("docker", "-v")
	dockerVersionCmdErr := system.RunOSCommand(dockerVersionCmd, logger)
	if dockerVersionCmdErr != nil {
		return errors.Wrapf(dockerVersionCmdErr, "Attempting to get docker version")
	}
	// END Informational - output the docker version...

	// END DOCKER PRECONDITIONS

	// Compile this binary for minimal Docker size
	// https://blog.codeship.com/building-minimal-docker-containers-for-go-applications/
	currentTime := time.Now().UnixNano()
	executableOutput := fmt.Sprintf("%s-%d-docker.lambda.amd64", serviceName, currentTime)
	buildErr := system.BuildGoBinary(serviceName,
		executableOutput,
		false,
		fmt.Sprintf("%d", currentTime),
		buildTags,
		linkFlags,
		false,
		logger)
	if buildErr != nil {
		return errors.Wrapf(buildErr, "Attempting to build Docker binary")
	}
	defer func() {
		removeErr := os.Remove(executableOutput)
		if removeErr != nil {
			logger.WithFields(logrus.Fields{
				"Path":  executableOutput,
				"Error": removeErr,
			}).Warn("Failed to delete temporary Docker binary")
		}
	}()

	// ARG SPARTA_DOCKER_BINARY reference s.t. we can supply the binary
	// name to the build..
	// We need to build the static binary s.t. we can add it to the Docker container...
	// Build the image...
	var dockerArgs []string
	dockerArgs = append(dockerArgs,
		"build",
		"--build-arg",
		fmt.Sprintf("%s=%s", BinaryNameArgument, executableOutput))

	if dockerFilepath != "" {
		dockerArgs = append(dockerArgs, "--file", dockerFilepath)
	}
	// Add the latest tag
	// dockerArgs = append(dockerArgs, "--tag", fmt.Sprintf("sparta/%s:latest", serviceName))
	logger.WithFields(logrus.Fields{
		"Tags": dockerTags,
	}).Info("Creating Docker image")

	for eachKey, eachValue := range dockerTags {
		dockerArgs = append(dockerArgs, "--tag", fmt.Sprintf("%s:%s",
			strings.ToLower(eachKey),
			strings.ToLower(eachValue)))
	}

	dockerArgs = append(dockerArgs, ".")
	dockerCmd := exec.Command("docker", dockerArgs...)
	return system.RunOSCommand(dockerCmd, logger)
}

// BuildDockerImage creates the smallest docker image for this Golang binary
// using the serviceName as the image name and including the supplied tags
func BuildDockerImage(serviceName string,
	dockerFilepath string,
	tags map[string]string,
	logger *logrus.Logger) error {

	return BuildDockerImageWithFlags(serviceName,
		dockerFilepath,
		tags,
		"",
		"",
		logger)
}

// PushDockerImageToECR pushes a local Docker image to an ECR repository
func PushDockerImageToECR(localImageTag string,
	ecrRepoName string,
	awsSession *session.Session,
	logger *logrus.Logger) (string, error) {

	stsSvc := sts.New(awsSession)
	ecrSvc := ecr.New(awsSession)

	// 1. Get the caller identity s.t. we can get the ECR URL which includes the
	// account name
	stsIdentityOutput, stsIdentityErr := stsSvc.GetCallerIdentity(&sts.GetCallerIdentityInput{})
	if stsIdentityErr != nil {
		return "", errors.Wrapf(stsIdentityErr, "Attempting to get AWS caller identity")
	}

	// 2. Create the URL to which we're going to do the push
	localImageTagParts := strings.Split(localImageTag, ":")
	ecrTagValue := fmt.Sprintf("%s.dkr.ecr.%s.amazonaws.com/%s:%s",
		*stsIdentityOutput.Account,
		*awsSession.Config.Region,
		ecrRepoName,
		localImageTagParts[len(localImageTagParts)-1])

	// 3. Tag the local image with the ECR tag
	dockerTagCmd := exec.Command("docker", "tag", localImageTag, ecrTagValue)
	dockerTagCmdErr := system.RunOSCommand(dockerTagCmd, logger)
	if dockerTagCmdErr != nil {
		return "", errors.Wrapf(dockerTagCmdErr, "Attempting to tag Docker image")
	}

	// 4. Push the image - if that fails attempt to reauthorize with the docker
	// client and try again
	var pushError error
	dockerPushCmd := exec.Command("docker", "push", ecrTagValue)
	pushError = system.RunOSCommand(dockerPushCmd, logger)
	if pushError != nil {
		logger.WithFields(logrus.Fields{
			"Error": pushError,
		}).Info("ECR push failed - reauthorizing")
		ecrAuthTokenResult, ecrAuthTokenResultErr := ecrSvc.GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{})
		if ecrAuthTokenResultErr != nil {
			pushError = ecrAuthTokenResultErr
		} else {
			authData := ecrAuthTokenResult.AuthorizationData[0]
			authToken, authTokenErr := base64.StdEncoding.DecodeString(*authData.AuthorizationToken)
			if authTokenErr != nil {
				pushError = authTokenErr
			} else {
				authTokenString := string(authToken)
				authTokenParts := strings.Split(authTokenString, ":")
				dockerURL := fmt.Sprintf("https://%s.dkr.ecr.%s.amazonaws.com",
					*stsIdentityOutput.Account,
					*awsSession.Config.Region)
				dockerLoginCmd := exec.Command("docker",
					"login",
					"-u",
					authTokenParts[0],
					"--password-stdin",
					dockerURL)
				dockerLoginCmd.Stdout = os.Stdout
				dockerLoginCmd.Stdin = bytes.NewReader([]byte(fmt.Sprintf("%s\n", authTokenParts[1])))
				dockerLoginCmd.Stderr = os.Stderr
				dockerLoginCmdErr := system.RunOSCommand(dockerLoginCmd, logger)
				if dockerLoginCmdErr != nil {
					pushError = dockerLoginCmdErr
				} else {
					// Try it again...
					dockerRetryPushCmd := exec.Command("docker", "push", ecrTagValue)
					dockerRetryPushCmdErr := system.RunOSCommand(dockerRetryPushCmd, logger)
					pushError = dockerRetryPushCmdErr
				}
			}
		}
	}
	if pushError != nil {
		pushError = errors.Wrapf(pushError, "Attempting to push Docker image")
	}
	return ecrTagValue, pushError
}
<commit_msg>Patch prealloc<commit_after>package docker

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ecr"
	"github.com/aws/aws-sdk-go/service/sts"
	"github.com/mweagle/Sparta/system"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

////////////////////////////////////////////////////////////////////////////////
// CONSTANTS
////////////////////////////////////////////////////////////////////////////////

const (
	// BinaryNameArgument is the argument provided to docker build that
	// supplies the local statically built Go binary
	BinaryNameArgument = "SPARTA_DOCKER_BINARY"
)

// BuildDockerImageWithFlags is an extended version of BuildDockerImage that includes
// support for build time tags
func BuildDockerImageWithFlags(serviceName string,
	dockerFilepath string,
	dockerTags map[string]string,
	buildTags string,
	linkFlags string,
	logger *logrus.Logger) error {

	// BEGIN DOCKER PRECONDITIONS
	// Ensure that serviceName and tags are lowercase to make Docker happy
	var dockerErrors []string
	for eachKey, eachValue := range dockerTags {
		if eachKey != strings.ToLower(eachKey) ||
			eachValue != strings.ToLower(eachValue) {
			dockerErrors = append(dockerErrors, fmt.Sprintf("--tag %s:%s MUST be lower case", eachKey, eachValue))
		}
	}

	if len(dockerErrors) > 0 {
		return errors.Errorf("Docker build errors: %s", strings.Join(dockerErrors[:], ", "))
	}
	// BEGIN Informational - output the docker version...
	dockerVersionCmd := exec.Command("docker", "-v")
	dockerVersionCmdErr := system.RunOSCommand(dockerVersionCmd, logger)
	if dockerVersionCmdErr != nil {
		return errors.Wrapf(dockerVersionCmdErr, "Attempting to get docker version")
	}
	// END Informational - output the docker version...

	// END DOCKER PRECONDITIONS

	// Compile this binary for minimal Docker size
	// https://blog.codeship.com/building-minimal-docker-containers-for-go-applications/
	currentTime := time.Now().UnixNano()
	executableOutput := fmt.Sprintf("%s-%d-docker.lambda.amd64", serviceName, currentTime)
	buildErr := system.BuildGoBinary(serviceName,
		executableOutput,
		false,
		fmt.Sprintf("%d", currentTime),
		buildTags,
		linkFlags,
		false,
		logger)
	if buildErr != nil {
		return errors.Wrapf(buildErr, "Attempting to build Docker binary")
	}
	defer func() {
		removeErr := os.Remove(executableOutput)
		if removeErr != nil {
			logger.WithFields(logrus.Fields{
				"Path":  executableOutput,
				"Error": removeErr,
			}).Warn("Failed to delete temporary Docker binary")
		}
	}()

	// ARG SPARTA_DOCKER_BINARY reference s.t. we can supply the binary
	// name to the build..
	// We need to build the static binary s.t. we can add it to the Docker container...
	// Build the image...
	dockerArgs := []string{
		"build",
		"--build-arg",
		fmt.Sprintf("%s=%s", BinaryNameArgument, executableOutput),
	}

	if dockerFilepath != "" {
		dockerArgs = append(dockerArgs, "--file", dockerFilepath)
	}
	// Add the latest tag
	// dockerArgs = append(dockerArgs, "--tag", fmt.Sprintf("sparta/%s:latest", serviceName))
	logger.WithFields(logrus.Fields{
		"Tags": dockerTags,
	}).Info("Creating Docker image")

	for eachKey, eachValue := range dockerTags {
		dockerArgs = append(dockerArgs, "--tag", fmt.Sprintf("%s:%s",
			strings.ToLower(eachKey),
			strings.ToLower(eachValue)))
	}

	dockerArgs = append(dockerArgs, ".")
	dockerCmd := exec.Command("docker", dockerArgs...)
	return system.RunOSCommand(dockerCmd, logger)
}

// BuildDockerImage creates the smallest docker image for this Golang binary
// using the serviceName as the image name and including the supplied tags
func BuildDockerImage(serviceName string,
	dockerFilepath string,
	tags map[string]string,
	logger *logrus.Logger) error {

	return BuildDockerImageWithFlags(serviceName,
		dockerFilepath,
		tags,
		"",
		"",
		logger)
}

// PushDockerImageToECR pushes a local Docker image to an ECR repository
func PushDockerImageToECR(localImageTag string,
	ecrRepoName string,
	awsSession *session.Session,
	logger *logrus.Logger) (string, error) {

	stsSvc := sts.New(awsSession)
	ecrSvc := ecr.New(awsSession)

	// 1. Get the caller identity s.t. we can get the ECR URL which includes the
	// account name
	stsIdentityOutput, stsIdentityErr := stsSvc.GetCallerIdentity(&sts.GetCallerIdentityInput{})
	if stsIdentityErr != nil {
		return "", errors.Wrapf(stsIdentityErr, "Attempting to get AWS caller identity")
	}

	// 2. Create the URL to which we're going to do the push
	localImageTagParts := strings.Split(localImageTag, ":")
	ecrTagValue := fmt.Sprintf("%s.dkr.ecr.%s.amazonaws.com/%s:%s",
		*stsIdentityOutput.Account,
		*awsSession.Config.Region,
		ecrRepoName,
		localImageTagParts[len(localImageTagParts)-1])

	// 3. Tag the local image with the ECR tag
	dockerTagCmd := exec.Command("docker", "tag", localImageTag, ecrTagValue)
	dockerTagCmdErr := system.RunOSCommand(dockerTagCmd, logger)
	if dockerTagCmdErr != nil {
		return "", errors.Wrapf(dockerTagCmdErr, "Attempting to tag Docker image")
	}

	// 4. Push the image - if that fails attempt to reauthorize with the docker
	// client and try again
	var pushError error
	dockerPushCmd := exec.Command("docker", "push", ecrTagValue)
	pushError = system.RunOSCommand(dockerPushCmd, logger)
	if pushError != nil {
		logger.WithFields(logrus.Fields{
			"Error": pushError,
		}).Info("ECR push failed - reauthorizing")
		ecrAuthTokenResult, ecrAuthTokenResultErr := ecrSvc.GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{})
		if ecrAuthTokenResultErr != nil {
			pushError = ecrAuthTokenResultErr
		} else {
			authData := ecrAuthTokenResult.AuthorizationData[0]
			authToken, authTokenErr := base64.StdEncoding.DecodeString(*authData.AuthorizationToken)
			if authTokenErr != nil {
				pushError = authTokenErr
			} else {
				authTokenString := string(authToken)
				authTokenParts := strings.Split(authTokenString, ":")
				dockerURL := fmt.Sprintf("https://%s.dkr.ecr.%s.amazonaws.com",
					*stsIdentityOutput.Account,
					*awsSession.Config.Region)
				dockerLoginCmd := exec.Command("docker",
					"login",
					"-u",
					authTokenParts[0],
					"--password-stdin",
					dockerURL)
				dockerLoginCmd.Stdout = os.Stdout
				dockerLoginCmd.Stdin = bytes.NewReader([]byte(fmt.Sprintf("%s\n", authTokenParts[1])))
				dockerLoginCmd.Stderr = os.Stderr
				dockerLoginCmdErr := system.RunOSCommand(dockerLoginCmd, logger)
				if dockerLoginCmdErr != nil {
					pushError = dockerLoginCmdErr
				} else {
					// Try it again...
					dockerRetryPushCmd := exec.Command("docker", "push", ecrTagValue)
					dockerRetryPushCmdErr := system.RunOSCommand(dockerRetryPushCmd, logger)
					pushError = dockerRetryPushCmdErr
				}
			}
		}
	}
	if pushError != nil {
		pushError = errors.Wrapf(pushError, "Attempting to push Docker image")
	}
	return ecrTagValue, pushError
}
<|endoftext|>


<commit_before>package storage

import (
	"fmt"
	"os"
	"sync/atomic"

	. "github.com/chrislusf/seaweedfs/weed/storage/types"
	"github.com/willf/bloom"
)

type mapMetric struct {
	DeletionCounter     uint32 `json:"DeletionCounter"`
	FileCounter         uint32 `json:"FileCounter"`
	DeletionByteCounter uint64 `json:"DeletionByteCounter"`
	FileByteCounter     uint64 `json:"FileByteCounter"`
	MaximumFileKey      uint64 `json:"MaxFileKey"`
}

func (mm *mapMetric) logDelete(deletedByteCount uint32) {
	mm.LogDeletionCounter(deletedByteCount)
}

func (mm *mapMetric) logPut(key NeedleId, oldSize uint32, newSize uint32) {
	mm.MaybeSetMaxFileKey(key)
	mm.LogFileCounter(newSize)
	if oldSize > 0 && oldSize != TombstoneFileSize {
		mm.LogDeletionCounter(oldSize)
	}
}
func (mm mapMetric) LogFileCounter(newSize uint32) {
	atomic.AddUint32(&mm.FileCounter, 1)
	atomic.AddUint64(&mm.FileByteCounter, uint64(newSize))
}
func (mm mapMetric) LogDeletionCounter(oldSize uint32) {
	if oldSize > 0 {
		atomic.AddUint32(&mm.DeletionCounter, 1)
		atomic.AddUint64(&mm.DeletionByteCounter, uint64(oldSize))
	}
}
func (mm mapMetric) ContentSize() uint64 {
	return atomic.LoadUint64(&mm.FileByteCounter)
}
func (mm mapMetric) DeletedSize() uint64 {
	return atomic.LoadUint64(&mm.DeletionByteCounter)
}
func (mm mapMetric) FileCount() int {
	return int(atomic.LoadUint32(&mm.FileCounter))
}
func (mm mapMetric) DeletedCount() int {
	return int(atomic.LoadUint32(&mm.DeletionCounter))
}
func (mm mapMetric) MaxFileKey() NeedleId {
	t := uint64(mm.MaximumFileKey)
	return NeedleId(t)
}
func (mm mapMetric) MaybeSetMaxFileKey(key NeedleId) {
	if key > mm.MaxFileKey() {
		atomic.StoreUint64(&mm.MaximumFileKey, uint64(key))
	}
}

func newNeedleMapMetricFromIndexFile(r *os.File) (mm *mapMetric, err error) {
	mm = &mapMetric{}
	var bf *bloom.BloomFilter
	buf := make([]byte, NeedleIdSize)
	err = reverseWalkIndexFile(r, func(entryCount int64) {
		bf = bloom.NewWithEstimates(uint(entryCount), 0.001)
	}, func(key NeedleId, offset Offset, size uint32) error {

		mm.MaybeSetMaxFileKey(key)
		NeedleIdToBytes(buf, key)
		if size != TombstoneFileSize {
			mm.FileByteCounter += uint64(size)
		}

		if !bf.Test(buf) {
			mm.FileCounter++
			bf.Add(buf)
		} else {
			// deleted file
			mm.DeletionCounter++
			if size != TombstoneFileSize {
				// previously already deleted file
				mm.DeletionByteCounter += uint64(size)
			}
		}
		return nil
	})
	return
}

func reverseWalkIndexFile(r *os.File, initFn func(entryCount int64), fn func(key NeedleId, offset Offset, size uint32) error) error {
	fi, err := r.Stat()
	if err != nil {
		return fmt.Errorf("file %s stat error: %v", r.Name(), err)
	}
	fileSize := fi.Size()
	if fileSize%NeedleMapEntrySize != 0 {
		return fmt.Errorf("unexpected file %s size: %d", r.Name(), fileSize)
	}

	entryCount := fileSize / NeedleMapEntrySize
	initFn(entryCount)

	batchSize := int64(1024 * 4)

	bytes := make([]byte, NeedleMapEntrySize*batchSize)
	nextBatchSize := entryCount % batchSize
	if nextBatchSize == 0 {
		nextBatchSize = batchSize
	}
	remainingCount := entryCount - nextBatchSize

	for remainingCount >= 0 {
		_, e := r.ReadAt(bytes[:NeedleMapEntrySize*nextBatchSize], NeedleMapEntrySize*remainingCount)
		// glog.V(0).Infoln("file", r.Name(), "readerOffset", NeedleMapEntrySize*remainingCount, "count", count, "e", e)
		if e != nil {
			return e
		}
		for i := int(nextBatchSize) - 1; i >= 0; i-- {
			key, offset, size := IdxFileEntry(bytes[i*NeedleMapEntrySize : i*NeedleMapEntrySize+NeedleMapEntrySize])
			if e = fn(key, offset, size); e != nil {
				return e
			}
		}
		nextBatchSize = batchSize
		remainingCount -= nextBatchSize
	}
	return nil
}
<commit_msg>fix metrics<commit_after>package storage

import (
	"fmt"
	"os"
	"sync/atomic"

	. "github.com/chrislusf/seaweedfs/weed/storage/types"
	"github.com/willf/bloom"
)

type mapMetric struct {
	DeletionCounter     uint32 `json:"DeletionCounter"`
	FileCounter         uint32 `json:"FileCounter"`
	DeletionByteCounter uint64 `json:"DeletionByteCounter"`
	FileByteCounter     uint64 `json:"FileByteCounter"`
	MaximumFileKey      uint64 `json:"MaxFileKey"`
}

func (mm *mapMetric) logDelete(deletedByteCount uint32) {
	mm.LogDeletionCounter(deletedByteCount)
}

func (mm *mapMetric) logPut(key NeedleId, oldSize uint32, newSize uint32) {
	mm.MaybeSetMaxFileKey(key)
	mm.LogFileCounter(newSize)
	if oldSize > 0 && oldSize != TombstoneFileSize {
		mm.LogDeletionCounter(oldSize)
	}
}
func (mm *mapMetric) LogFileCounter(newSize uint32) {
	atomic.AddUint32(&mm.FileCounter, 1)
	atomic.AddUint64(&mm.FileByteCounter, uint64(newSize))
}
func (mm *mapMetric) LogDeletionCounter(oldSize uint32) {
	if oldSize > 0 {
		atomic.AddUint32(&mm.DeletionCounter, 1)
		atomic.AddUint64(&mm.DeletionByteCounter, uint64(oldSize))
	}
}
func (mm *mapMetric) ContentSize() uint64 {
	return atomic.LoadUint64(&mm.FileByteCounter)
}
func (mm *mapMetric) DeletedSize() uint64 {
	return atomic.LoadUint64(&mm.DeletionByteCounter)
}
func (mm *mapMetric) FileCount() int {
	return int(atomic.LoadUint32(&mm.FileCounter))
}
func (mm *mapMetric) DeletedCount() int {
	return int(atomic.LoadUint32(&mm.DeletionCounter))
}
func (mm *mapMetric) MaxFileKey() NeedleId {
	t := uint64(mm.MaximumFileKey)
	return NeedleId(t)
}
func (mm *mapMetric) MaybeSetMaxFileKey(key NeedleId) {
	if key > mm.MaxFileKey() {
		atomic.StoreUint64(&mm.MaximumFileKey, uint64(key))
	}
}

func newNeedleMapMetricFromIndexFile(r *os.File) (mm *mapMetric, err error) {
	mm = &mapMetric{}
	var bf *bloom.BloomFilter
	buf := make([]byte, NeedleIdSize)
	err = reverseWalkIndexFile(r, func(entryCount int64) {
		bf = bloom.NewWithEstimates(uint(entryCount), 0.001)
	}, func(key NeedleId, offset Offset, size uint32) error {

		mm.MaybeSetMaxFileKey(key)
		NeedleIdToBytes(buf, key)
		if size != TombstoneFileSize {
			mm.FileByteCounter += uint64(size)
		}

		if !bf.Test(buf) {
			mm.FileCounter++
			bf.Add(buf)
		} else {
			// deleted file
			mm.DeletionCounter++
			if size != TombstoneFileSize {
				// previously already deleted file
				mm.DeletionByteCounter += uint64(size)
			}
		}
		return nil
	})
	return
}

func reverseWalkIndexFile(r *os.File, initFn func(entryCount int64), fn func(key NeedleId, offset Offset, size uint32) error) error {
	fi, err := r.Stat()
	if err != nil {
		return fmt.Errorf("file %s stat error: %v", r.Name(), err)
	}
	fileSize := fi.Size()
	if fileSize%NeedleMapEntrySize != 0 {
		return fmt.Errorf("unexpected file %s size: %d", r.Name(), fileSize)
	}

	entryCount := fileSize / NeedleMapEntrySize
	initFn(entryCount)

	batchSize := int64(1024 * 4)

	bytes := make([]byte, NeedleMapEntrySize*batchSize)
	nextBatchSize := entryCount % batchSize
	if nextBatchSize == 0 {
		nextBatchSize = batchSize
	}
	remainingCount := entryCount - nextBatchSize

	for remainingCount >= 0 {
		_, e := r.ReadAt(bytes[:NeedleMapEntrySize*nextBatchSize], NeedleMapEntrySize*remainingCount)
		// glog.V(0).Infoln("file", r.Name(), "readerOffset", NeedleMapEntrySize*remainingCount, "count", count, "e", e)
		if e != nil {
			return e
		}
		for i := int(nextBatchSize) - 1; i >= 0; i-- {
			key, offset, size := IdxFileEntry(bytes[i*NeedleMapEntrySize : i*NeedleMapEntrySize+NeedleMapEntrySize])
			if e = fn(key, offset, size); e != nil {
				return e
			}
		}
		nextBatchSize = batchSize
		remainingCount -= nextBatchSize
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/rand"
	"crypto/rsa"
	"crypto/tls"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"io/ioutil"
	"math/big"
	"net"
	"os"
	"time"
)

func main() {
	app := cli.NewApp()
	app.Name = "tlsgen"
	app.Usage = "Generates web server TLS certificates"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "org",
			Value: "",
			Usage: "Organization name",
		},
		cli.StringSliceFlag{
			Name:  "host",
			Value: &cli.StringSlice{},
			Usage: "Server Host",
		},
	}
	app.Action = func(c *cli.Context) {
		org := c.String("org")
		if org == "" {
			logrus.Fatal("org shoule not be empty")
		}
		host := c.StringSlice("host")
		if len(host) == 0 {
			logrus.Fatal("host shoule not be empty")
		}
		GenerateCACertificate("ca.pem", "ca-key.pem", org, 2048)
		GenerateCert(host, "cert.pem", "key.pem", "ca.pem", "ca-key.pem", org, 2048)
	}

	app.Run(os.Args)
}

func getTLSConfig(caCert, cert, key []byte, allowInsecure bool) (*tls.Config, error) {
	// TLS config
	var tlsConfig tls.Config
	tlsConfig.InsecureSkipVerify = allowInsecure
	certPool := x509.NewCertPool()

	certPool.AppendCertsFromPEM(caCert)
	tlsConfig.RootCAs = certPool
	keypair, err := tls.X509KeyPair(cert, key)
	if err != nil {
		return &tlsConfig, err
	}
	tlsConfig.Certificates = []tls.Certificate{keypair}
	if allowInsecure {
		tlsConfig.InsecureSkipVerify = true
	}

	return &tlsConfig, nil
}

func newCertificate(org string) (*x509.Certificate, error) {
	now := time.Now()
	// need to set notBefore slightly in the past to account for time
	// skew in the VMs otherwise the certs sometimes are not yet valid
	notBefore := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute()-5, 0, 0, time.Local)
	notAfter := notBefore.Add(time.Hour * 24 * 1080)

	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
	if err != nil {
		return nil, err
	}

	return &x509.Certificate{
		SerialNumber: serialNumber,
		Subject: pkix.Name{
			Organization: []string{org},
		},
		NotBefore: notBefore,
		NotAfter:  notAfter,

		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		BasicConstraintsValid: true,
	}, nil

}

// GenerateCACertificate generates a new certificate authority from the specified org
// and bit size and stores the resulting certificate and key file
// in the arguments.
func GenerateCACertificate(certFile, keyFile, org string, bits int) error {
	template, err := newCertificate(org)
	if err != nil {
		return err
	}

	template.IsCA = true
	template.KeyUsage |= x509.KeyUsageCertSign

	priv, err := rsa.GenerateKey(rand.Reader, bits)
	if err != nil {
		return err
	}

	derBytes, err := x509.CreateCertificate(rand.Reader, template, template, &priv.PublicKey, priv)
	if err != nil {
		return err
	}

	certOut, err := os.Create(certFile)
	if err != nil {
		return err
	}

	pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
	certOut.Close()

	keyOut, err := os.OpenFile(keyFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return err

	}

	pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
	keyOut.Close()

	return nil
}

// GenerateCert generates a new certificate signed using the provided
// certificate authority files and stores the result in the certificate
// file and key provided.  The provided host names are set to the
// appropriate certificate fields.
func GenerateCert(hosts []string, certFile, keyFile, caFile, caKeyFile, org string, bits int) error {
	template, err := newCertificate(org)
	if err != nil {
		return err
	}
	// client
	if len(hosts) == 1 && hosts[0] == "" {
		template.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}
		template.KeyUsage = x509.KeyUsageDigitalSignature
	} else { // server
		template.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}
		for _, h := range hosts {
			if ip := net.ParseIP(h); ip != nil {
				template.IPAddresses = append(template.IPAddresses, ip)

			} else {
				template.DNSNames = append(template.DNSNames, h)
			}
		}
	}

	tlsCert, err := tls.LoadX509KeyPair(caFile, caKeyFile)
	if err != nil {
		return err

	}

	priv, err := rsa.GenerateKey(rand.Reader, bits)
	if err != nil {
		return err

	}

	x509Cert, err := x509.ParseCertificate(tlsCert.Certificate[0])
	if err != nil {
		return err
	}

	derBytes, err := x509.CreateCertificate(rand.Reader, template, x509Cert, &priv.PublicKey, tlsCert.PrivateKey)
	if err != nil {
		return err
	}

	certOut, err := os.Create(certFile)
	if err != nil {
		return err

	}

	pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
	certOut.Close()

	keyOut, err := os.OpenFile(keyFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return err

	}

	pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
	keyOut.Close()

	return nil
}

func ValidateCertificate(addr, caCertPath, serverCertPath, serverKeyPath string) (bool, error) {
	caCert, err := ioutil.ReadFile(caCertPath)
	if err != nil {
		return false, err
	}

	serverCert, err := ioutil.ReadFile(serverCertPath)
	if err != nil {
		return false, err
	}

	serverKey, err := ioutil.ReadFile(serverKeyPath)
	if err != nil {
		return false, err
	}

	tlsConfig, err := getTLSConfig(caCert, serverCert, serverKey, false)
	if err != nil {
		return false, err
	}

	dialer := &net.Dialer{
		Timeout: time.Second * 2,
	}

	_, err = tls.DialWithDialer(dialer, "tcp", addr, tlsConfig)
	if err != nil {
		return false, nil
	}

	return true, nil
}
<commit_msg>add client cert<commit_after>package main

import (
	"crypto/rand"
	"crypto/rsa"
	"crypto/tls"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"io/ioutil"
	"math/big"
	"net"
	"os"
	"time"
)

func main() {
	app := cli.NewApp()
	app.Name = "tlsgen"
	app.Usage = "Generates web server TLS certificates"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "org",
			Value: "",
			Usage: "Organization name",
		},
		cli.StringSliceFlag{
			Name:  "host",
			Value: &cli.StringSlice{},
			Usage: "Server Host",
		},
	}
	app.Action = func(c *cli.Context) {
		org := c.String("org")
		if org == "" {
			logrus.Fatal("org shoule not be empty")
		}
		host := c.StringSlice("host")
		if len(host) == 0 {
			logrus.Fatal("host shoule not be empty")
		}
		GenerateCACertificate("ca.pem", "ca-key.pem", org, 2048)
		//client
		GenerateCert([]string{""}, "cert.pem", "key.pem", "ca.pem", "ca-key.pem", org, 2048)
		//server
		GenerateCert(host, "server.pem", "server-key.pem", "ca.pem", "ca-key.pem", org, 2048)
	}

	app.Run(os.Args)
}

func getTLSConfig(caCert, cert, key []byte, allowInsecure bool) (*tls.Config, error) {
	// TLS config
	var tlsConfig tls.Config
	tlsConfig.InsecureSkipVerify = allowInsecure
	certPool := x509.NewCertPool()

	certPool.AppendCertsFromPEM(caCert)
	tlsConfig.RootCAs = certPool
	keypair, err := tls.X509KeyPair(cert, key)
	if err != nil {
		return &tlsConfig, err
	}
	tlsConfig.Certificates = []tls.Certificate{keypair}
	if allowInsecure {
		tlsConfig.InsecureSkipVerify = true
	}

	return &tlsConfig, nil
}

func newCertificate(org string) (*x509.Certificate, error) {
	now := time.Now()
	// need to set notBefore slightly in the past to account for time
	// skew in the VMs otherwise the certs sometimes are not yet valid
	notBefore := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute()-5, 0, 0, time.Local)
	notAfter := notBefore.Add(time.Hour * 24 * 1080)

	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
	if err != nil {
		return nil, err
	}

	return &x509.Certificate{
		SerialNumber: serialNumber,
		Subject: pkix.Name{
			Organization: []string{org},
		},
		NotBefore: notBefore,
		NotAfter:  notAfter,

		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		BasicConstraintsValid: true,
	}, nil

}

// GenerateCACertificate generates a new certificate authority from the specified org
// and bit size and stores the resulting certificate and key file
// in the arguments.
func GenerateCACertificate(certFile, keyFile, org string, bits int) error {
	template, err := newCertificate(org)
	if err != nil {
		return err
	}

	template.IsCA = true
	template.KeyUsage |= x509.KeyUsageCertSign

	priv, err := rsa.GenerateKey(rand.Reader, bits)
	if err != nil {
		return err
	}

	derBytes, err := x509.CreateCertificate(rand.Reader, template, template, &priv.PublicKey, priv)
	if err != nil {
		return err
	}

	certOut, err := os.Create(certFile)
	if err != nil {
		return err
	}

	pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
	certOut.Close()

	keyOut, err := os.OpenFile(keyFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return err

	}

	pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
	keyOut.Close()

	return nil
}

// GenerateCert generates a new certificate signed using the provided
// certificate authority files and stores the result in the certificate
// file and key provided.  The provided host names are set to the
// appropriate certificate fields.
func GenerateCert(hosts []string, certFile, keyFile, caFile, caKeyFile, org string, bits int) error {
	template, err := newCertificate(org)
	if err != nil {
		return err
	}
	// client
	if len(hosts) == 1 && hosts[0] == "" {
		template.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}
		template.KeyUsage = x509.KeyUsageDigitalSignature
	} else { // server
		template.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}
		for _, h := range hosts {
			if ip := net.ParseIP(h); ip != nil {
				template.IPAddresses = append(template.IPAddresses, ip)

			} else {
				template.DNSNames = append(template.DNSNames, h)
			}
		}
	}

	tlsCert, err := tls.LoadX509KeyPair(caFile, caKeyFile)
	if err != nil {
		return err

	}

	priv, err := rsa.GenerateKey(rand.Reader, bits)
	if err != nil {
		return err

	}

	x509Cert, err := x509.ParseCertificate(tlsCert.Certificate[0])
	if err != nil {
		return err
	}

	derBytes, err := x509.CreateCertificate(rand.Reader, template, x509Cert, &priv.PublicKey, tlsCert.PrivateKey)
	if err != nil {
		return err
	}

	certOut, err := os.Create(certFile)
	if err != nil {
		return err

	}

	pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
	certOut.Close()

	keyOut, err := os.OpenFile(keyFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return err

	}

	pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
	keyOut.Close()

	return nil
}

func ValidateCertificate(addr, caCertPath, serverCertPath, serverKeyPath string) (bool, error) {
	caCert, err := ioutil.ReadFile(caCertPath)
	if err != nil {
		return false, err
	}

	serverCert, err := ioutil.ReadFile(serverCertPath)
	if err != nil {
		return false, err
	}

	serverKey, err := ioutil.ReadFile(serverKeyPath)
	if err != nil {
		return false, err
	}

	tlsConfig, err := getTLSConfig(caCert, serverCert, serverKey, false)
	if err != nil {
		return false, err
	}

	dialer := &net.Dialer{
		Timeout: time.Second * 2,
	}

	_, err = tls.DialWithDialer(dialer, "tcp", addr, tlsConfig)
	if err != nil {
		return false, nil
	}

	return true, nil
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"
	"os"
	"regexp"
	"strings"
	"testing"

	"github.com/aws/aws-sdk-go/service/organizations"

	"github.com/aws/aws-sdk-go/aws/arn"
	"github.com/aws/aws-sdk-go/aws/endpoints"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/terraform"
	"github.com/terraform-providers/terraform-provider-template/template"
	"github.com/terraform-providers/terraform-provider-tls/tls"
)

var testAccProviders map[string]terraform.ResourceProvider
var testAccProvidersWithTLS map[string]terraform.ResourceProvider
var testAccProviderFactories func(providers *[]*schema.Provider) map[string]terraform.ResourceProviderFactory
var testAccProvider *schema.Provider
var testAccTemplateProvider *schema.Provider

func init() {
	testAccProvider = Provider().(*schema.Provider)
	testAccTemplateProvider = template.Provider().(*schema.Provider)
	testAccProviders = map[string]terraform.ResourceProvider{
		"aws":      testAccProvider,
		"template": testAccTemplateProvider,
	}
	testAccProviderFactories = func(providers *[]*schema.Provider) map[string]terraform.ResourceProviderFactory {
		return map[string]terraform.ResourceProviderFactory{
			"aws": func() (terraform.ResourceProvider, error) {
				p := Provider()
				*providers = append(*providers, p.(*schema.Provider))
				return p, nil
			},
		}
	}
	testAccProvidersWithTLS = map[string]terraform.ResourceProvider{
		"tls": tls.Provider(),
	}

	for k, v := range testAccProviders {
		testAccProvidersWithTLS[k] = v
	}
}

func TestProvider(t *testing.T) {
	if err := Provider().(*schema.Provider).InternalValidate(); err != nil {
		t.Fatalf("err: %s", err)
	}
}

func TestProvider_impl(t *testing.T) {
	var _ terraform.ResourceProvider = Provider()
}

func testAccPreCheck(t *testing.T) {
	if v := os.Getenv("AWS_PROFILE"); v == "" {
		if v := os.Getenv("AWS_ACCESS_KEY_ID"); v == "" {
			t.Fatal("AWS_ACCESS_KEY_ID must be set for acceptance tests")
		}
		if v := os.Getenv("AWS_SECRET_ACCESS_KEY"); v == "" {
			t.Fatal("AWS_SECRET_ACCESS_KEY must be set for acceptance tests")
		}
	}

	region := testAccGetRegion()
	log.Printf("[INFO] Test: Using %s as test region", region)
	os.Setenv("AWS_DEFAULT_REGION", region)

	err := testAccProvider.Configure(terraform.NewResourceConfig(nil))
	if err != nil {
		t.Fatal(err)
	}
}

// testAccAwsProviderAccountID returns the account ID of an AWS provider
func testAccAwsProviderAccountID(provider *schema.Provider) string {
	if provider == nil {
		log.Print("[DEBUG] Unable to read account ID from test provider: empty provider")
		return ""
	}
	if provider.Meta() == nil {
		log.Print("[DEBUG] Unable to read account ID from test provider: unconfigured provider")
		return ""
	}
	client, ok := provider.Meta().(*AWSClient)
	if !ok {
		log.Print("[DEBUG] Unable to read account ID from test provider: non-AWS or unconfigured AWS provider")
		return ""
	}
	return client.accountid
}

// testAccCheckResourceAttrAccountID ensures the Terraform state exactly matches the account ID
func testAccCheckResourceAttrAccountID(resourceName, attributeName string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		return resource.TestCheckResourceAttr(resourceName, attributeName, testAccGetAccountID())(s)
	}
}

// testAccCheckResourceAttrRegionalARN ensures the Terraform state exactly matches a formatted ARN with region
func testAccCheckResourceAttrRegionalARN(resourceName, attributeName, arnService, arnResource string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		attributeValue := arn.ARN{
			AccountID: testAccGetAccountID(),
			Partition: testAccGetPartition(),
			Region:    testAccGetRegion(),
			Resource:  arnResource,
			Service:   arnService,
		}.String()
		return resource.TestCheckResourceAttr(resourceName, attributeName, attributeValue)(s)
	}
}

// testAccMatchResourceAttrRegionalARN ensures the Terraform state regexp matches a formatted ARN with region
func testAccMatchResourceAttrRegionalARN(resourceName, attributeName, arnService string, arnResourceRegexp *regexp.Regexp) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		arnRegexp := arn.ARN{
			AccountID: testAccGetAccountID(),
			Partition: testAccGetPartition(),
			Region:    testAccGetRegion(),
			Resource:  arnResourceRegexp.String(),
			Service:   arnService,
		}.String()

		attributeMatch, err := regexp.Compile(arnRegexp)

		if err != nil {
			return fmt.Errorf("Unable to compile ARN regexp (%s): %s", arnRegexp, err)
		}

		return resource.TestMatchResourceAttr(resourceName, attributeName, attributeMatch)(s)
	}
}

// testAccCheckResourceAttrGlobalARN ensures the Terraform state exactly matches a formatted ARN without region
func testAccCheckResourceAttrGlobalARN(resourceName, attributeName, arnService, arnResource string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		attributeValue := arn.ARN{
			AccountID: testAccGetAccountID(),
			Partition: testAccGetPartition(),
			Resource:  arnResource,
			Service:   arnService,
		}.String()
		return resource.TestCheckResourceAttr(resourceName, attributeName, attributeValue)(s)
	}
}

// testAccMatchResourceAttrGlobalARN ensures the Terraform state regexp matches a formatted ARN without region
func testAccMatchResourceAttrGlobalARN(resourceName, attributeName, arnService string, arnResourceRegexp *regexp.Regexp) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		arnRegexp := arn.ARN{
			AccountID: testAccGetAccountID(),
			Partition: testAccGetPartition(),
			Resource:  arnResourceRegexp.String(),
			Service:   arnService,
		}.String()

		attributeMatch, err := regexp.Compile(arnRegexp)

		if err != nil {
			return fmt.Errorf("Unable to compile ARN regexp (%s): %s", arnRegexp, err)
		}

		return resource.TestMatchResourceAttr(resourceName, attributeName, attributeMatch)(s)
	}
}

// testAccGetAccountID returns the account ID of testAccProvider
// Must be used returned within a resource.TestCheckFunc
func testAccGetAccountID() string {
	return testAccAwsProviderAccountID(testAccProvider)
}

func testAccGetRegion() string {
	v := os.Getenv("AWS_DEFAULT_REGION")
	if v == "" {
		return "us-west-2"
	}
	return v
}

func testAccGetPartition() string {
	if partition, ok := endpoints.PartitionForRegion(endpoints.DefaultPartitions(), testAccGetRegion()); ok {
		return partition.ID()
	}
	return "aws"
}

func testAccGetServiceEndpoint(service string) string {
	endpoint, err := endpoints.DefaultResolver().EndpointFor(service, testAccGetRegion())
	if err != nil {
		return ""
	}
	return strings.TrimPrefix(endpoint.URL, "https://")
}

func testAccEC2ClassicPreCheck(t *testing.T) {
	client := testAccProvider.Meta().(*AWSClient)
	platforms := client.supportedplatforms
	region := client.region
	if !hasEc2Classic(platforms) {
		t.Skipf("This test can only run in EC2 Classic, platforms available in %s: %q",
			region, platforms)
	}
}

func testAccHasServicePreCheck(service string, t *testing.T) {
	if partition, ok := endpoints.PartitionForRegion(endpoints.DefaultPartitions(), testAccGetRegion()); ok {
		if _, ok := partition.Services()[service]; !ok {
			t.Skip(fmt.Sprintf("skipping tests; partition does not support %s service", service))
		}
	}
}

func testAccMultipleRegionsPreCheck(t *testing.T) {
	if partition, ok := endpoints.PartitionForRegion(endpoints.DefaultPartitions(), testAccGetRegion()); ok {
		if len(partition.Regions()) < 2 {
			t.Skip("skipping tests; partition only includes a single region")
		}
	}
}

func testAccOrganizationsAccountPreCheck(t *testing.T) {
	conn := testAccProvider.Meta().(*AWSClient).organizationsconn
	input := &organizations.DescribeOrganizationInput{}
	_, err := conn.DescribeOrganization(input)
	if isAWSErr(err, organizations.ErrCodeAWSOrganizationsNotInUseException, "") {
		return
	}
	if err != nil {
		t.Fatalf("error describing AWS Organization: %s", err)
	}
	t.Skip("skipping tests; this AWS account must not be an existing member of an AWS Organization")
}

func testAccAwsRegionProviderFunc(region string, providers *[]*schema.Provider) func() *schema.Provider {
	return func() *schema.Provider {
		if region == "" {
			log.Println("[DEBUG] No region given")
			return nil
		}
		if providers == nil {
			log.Println("[DEBUG] No providers given")
			return nil
		}

		log.Printf("[DEBUG] Checking providers for AWS region: %s", region)
		for _, provider := range *providers {
			// Ignore if Meta is empty, this can happen for validation providers
			if provider == nil || provider.Meta() == nil {
				log.Printf("[DEBUG] Skipping empty provider")
				continue
			}

			// Ignore if Meta is not AWSClient, this will happen for other providers
			client, ok := provider.Meta().(*AWSClient)
			if !ok {
				log.Printf("[DEBUG] Skipping non-AWS provider")
				continue
			}

			clientRegion := client.region
			log.Printf("[DEBUG] Checking AWS provider region %q against %q", clientRegion, region)
			if clientRegion == region {
				log.Printf("[DEBUG] Found AWS provider with region: %s", region)
				return provider
			}
		}

		log.Printf("[DEBUG] No suitable provider found for %q in %d providers", region, len(*providers))
		return nil
	}
}

func testAccCheckWithProviders(f func(*terraform.State, *schema.Provider) error, providers *[]*schema.Provider) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		numberOfProviders := len(*providers)
		for i, provider := range *providers {
			if provider.Meta() == nil {
				log.Printf("[DEBUG] Skipping empty provider %d (total: %d)", i, numberOfProviders)
				continue
			}
			log.Printf("[DEBUG] Calling check with provider %d (total: %d)", i, numberOfProviders)
			if err := f(s, provider); err != nil {
				return err
			}
		}
		return nil
	}
}

// Check sweeper API call error for reasons to skip sweeping
// These include missing API endpoints and unsupported API calls
func testSweepSkipSweepError(err error) bool {
	// Ignore missing API endpoints
	if isAWSErr(err, "RequestError", "send request failed") {
		return true
	}
	// Ignore unsupported API calls
	if isAWSErr(err, "UnsupportedOperation", "") {
		return true
	}
	// Ignore more unsupported API calls
	// InvalidParameterValue: Use of cache security groups is not permitted in this API version for your account.
	if isAWSErr(err, "InvalidParameterValue", "not permitted in this API version for your account") {
		return true
	}
	// GovCloud has endpoints that respond with (no message provided):
	// AccessDeniedException:
	// Since acceptance test sweepers are best effort and this response is very common,
	// we allow bypassing this error globally instead of individual test sweeper fixes.
	if isAWSErr(err, "AccessDeniedException", "") {
		return true
	}
	return false
}
<commit_msg>tests/provider: Add testAccPreCheckSkipError() function as base checking for implementing resource specific PreCheck functions<commit_after>package aws

import (
	"fmt"
	"log"
	"os"
	"regexp"
	"strings"
	"testing"

	"github.com/aws/aws-sdk-go/service/organizations"

	"github.com/aws/aws-sdk-go/aws/arn"
	"github.com/aws/aws-sdk-go/aws/endpoints"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/terraform"
	"github.com/terraform-providers/terraform-provider-template/template"
	"github.com/terraform-providers/terraform-provider-tls/tls"
)

var testAccProviders map[string]terraform.ResourceProvider
var testAccProvidersWithTLS map[string]terraform.ResourceProvider
var testAccProviderFactories func(providers *[]*schema.Provider) map[string]terraform.ResourceProviderFactory
var testAccProvider *schema.Provider
var testAccTemplateProvider *schema.Provider

func init() {
	testAccProvider = Provider().(*schema.Provider)
	testAccTemplateProvider = template.Provider().(*schema.Provider)
	testAccProviders = map[string]terraform.ResourceProvider{
		"aws":      testAccProvider,
		"template": testAccTemplateProvider,
	}
	testAccProviderFactories = func(providers *[]*schema.Provider) map[string]terraform.ResourceProviderFactory {
		return map[string]terraform.ResourceProviderFactory{
			"aws": func() (terraform.ResourceProvider, error) {
				p := Provider()
				*providers = append(*providers, p.(*schema.Provider))
				return p, nil
			},
		}
	}
	testAccProvidersWithTLS = map[string]terraform.ResourceProvider{
		"tls": tls.Provider(),
	}

	for k, v := range testAccProviders {
		testAccProvidersWithTLS[k] = v
	}
}

func TestProvider(t *testing.T) {
	if err := Provider().(*schema.Provider).InternalValidate(); err != nil {
		t.Fatalf("err: %s", err)
	}
}

func TestProvider_impl(t *testing.T) {
	var _ terraform.ResourceProvider = Provider()
}

func testAccPreCheck(t *testing.T) {
	if v := os.Getenv("AWS_PROFILE"); v == "" {
		if v := os.Getenv("AWS_ACCESS_KEY_ID"); v == "" {
			t.Fatal("AWS_ACCESS_KEY_ID must be set for acceptance tests")
		}
		if v := os.Getenv("AWS_SECRET_ACCESS_KEY"); v == "" {
			t.Fatal("AWS_SECRET_ACCESS_KEY must be set for acceptance tests")
		}
	}

	region := testAccGetRegion()
	log.Printf("[INFO] Test: Using %s as test region", region)
	os.Setenv("AWS_DEFAULT_REGION", region)

	err := testAccProvider.Configure(terraform.NewResourceConfig(nil))
	if err != nil {
		t.Fatal(err)
	}
}

// testAccAwsProviderAccountID returns the account ID of an AWS provider
func testAccAwsProviderAccountID(provider *schema.Provider) string {
	if provider == nil {
		log.Print("[DEBUG] Unable to read account ID from test provider: empty provider")
		return ""
	}
	if provider.Meta() == nil {
		log.Print("[DEBUG] Unable to read account ID from test provider: unconfigured provider")
		return ""
	}
	client, ok := provider.Meta().(*AWSClient)
	if !ok {
		log.Print("[DEBUG] Unable to read account ID from test provider: non-AWS or unconfigured AWS provider")
		return ""
	}
	return client.accountid
}

// testAccCheckResourceAttrAccountID ensures the Terraform state exactly matches the account ID
func testAccCheckResourceAttrAccountID(resourceName, attributeName string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		return resource.TestCheckResourceAttr(resourceName, attributeName, testAccGetAccountID())(s)
	}
}

// testAccCheckResourceAttrRegionalARN ensures the Terraform state exactly matches a formatted ARN with region
func testAccCheckResourceAttrRegionalARN(resourceName, attributeName, arnService, arnResource string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		attributeValue := arn.ARN{
			AccountID: testAccGetAccountID(),
			Partition: testAccGetPartition(),
			Region:    testAccGetRegion(),
			Resource:  arnResource,
			Service:   arnService,
		}.String()
		return resource.TestCheckResourceAttr(resourceName, attributeName, attributeValue)(s)
	}
}

// testAccMatchResourceAttrRegionalARN ensures the Terraform state regexp matches a formatted ARN with region
func testAccMatchResourceAttrRegionalARN(resourceName, attributeName, arnService string, arnResourceRegexp *regexp.Regexp) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		arnRegexp := arn.ARN{
			AccountID: testAccGetAccountID(),
			Partition: testAccGetPartition(),
			Region:    testAccGetRegion(),
			Resource:  arnResourceRegexp.String(),
			Service:   arnService,
		}.String()

		attributeMatch, err := regexp.Compile(arnRegexp)

		if err != nil {
			return fmt.Errorf("Unable to compile ARN regexp (%s): %s", arnRegexp, err)
		}

		return resource.TestMatchResourceAttr(resourceName, attributeName, attributeMatch)(s)
	}
}

// testAccCheckResourceAttrGlobalARN ensures the Terraform state exactly matches a formatted ARN without region
func testAccCheckResourceAttrGlobalARN(resourceName, attributeName, arnService, arnResource string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		attributeValue := arn.ARN{
			AccountID: testAccGetAccountID(),
			Partition: testAccGetPartition(),
			Resource:  arnResource,
			Service:   arnService,
		}.String()
		return resource.TestCheckResourceAttr(resourceName, attributeName, attributeValue)(s)
	}
}

// testAccMatchResourceAttrGlobalARN ensures the Terraform state regexp matches a formatted ARN without region
func testAccMatchResourceAttrGlobalARN(resourceName, attributeName, arnService string, arnResourceRegexp *regexp.Regexp) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		arnRegexp := arn.ARN{
			AccountID: testAccGetAccountID(),
			Partition: testAccGetPartition(),
			Resource:  arnResourceRegexp.String(),
			Service:   arnService,
		}.String()

		attributeMatch, err := regexp.Compile(arnRegexp)

		if err != nil {
			return fmt.Errorf("Unable to compile ARN regexp (%s): %s", arnRegexp, err)
		}

		return resource.TestMatchResourceAttr(resourceName, attributeName, attributeMatch)(s)
	}
}

// testAccGetAccountID returns the account ID of testAccProvider
// Must be used returned within a resource.TestCheckFunc
func testAccGetAccountID() string {
	return testAccAwsProviderAccountID(testAccProvider)
}

func testAccGetRegion() string {
	v := os.Getenv("AWS_DEFAULT_REGION")
	if v == "" {
		return "us-west-2"
	}
	return v
}

func testAccGetPartition() string {
	if partition, ok := endpoints.PartitionForRegion(endpoints.DefaultPartitions(), testAccGetRegion()); ok {
		return partition.ID()
	}
	return "aws"
}

func testAccGetServiceEndpoint(service string) string {
	endpoint, err := endpoints.DefaultResolver().EndpointFor(service, testAccGetRegion())
	if err != nil {
		return ""
	}
	return strings.TrimPrefix(endpoint.URL, "https://")
}

func testAccEC2ClassicPreCheck(t *testing.T) {
	client := testAccProvider.Meta().(*AWSClient)
	platforms := client.supportedplatforms
	region := client.region
	if !hasEc2Classic(platforms) {
		t.Skipf("This test can only run in EC2 Classic, platforms available in %s: %q",
			region, platforms)
	}
}

func testAccHasServicePreCheck(service string, t *testing.T) {
	if partition, ok := endpoints.PartitionForRegion(endpoints.DefaultPartitions(), testAccGetRegion()); ok {
		if _, ok := partition.Services()[service]; !ok {
			t.Skip(fmt.Sprintf("skipping tests; partition does not support %s service", service))
		}
	}
}

func testAccMultipleRegionsPreCheck(t *testing.T) {
	if partition, ok := endpoints.PartitionForRegion(endpoints.DefaultPartitions(), testAccGetRegion()); ok {
		if len(partition.Regions()) < 2 {
			t.Skip("skipping tests; partition only includes a single region")
		}
	}
}

func testAccOrganizationsAccountPreCheck(t *testing.T) {
	conn := testAccProvider.Meta().(*AWSClient).organizationsconn
	input := &organizations.DescribeOrganizationInput{}
	_, err := conn.DescribeOrganization(input)
	if isAWSErr(err, organizations.ErrCodeAWSOrganizationsNotInUseException, "") {
		return
	}
	if err != nil {
		t.Fatalf("error describing AWS Organization: %s", err)
	}
	t.Skip("skipping tests; this AWS account must not be an existing member of an AWS Organization")
}

func testAccAwsRegionProviderFunc(region string, providers *[]*schema.Provider) func() *schema.Provider {
	return func() *schema.Provider {
		if region == "" {
			log.Println("[DEBUG] No region given")
			return nil
		}
		if providers == nil {
			log.Println("[DEBUG] No providers given")
			return nil
		}

		log.Printf("[DEBUG] Checking providers for AWS region: %s", region)
		for _, provider := range *providers {
			// Ignore if Meta is empty, this can happen for validation providers
			if provider == nil || provider.Meta() == nil {
				log.Printf("[DEBUG] Skipping empty provider")
				continue
			}

			// Ignore if Meta is not AWSClient, this will happen for other providers
			client, ok := provider.Meta().(*AWSClient)
			if !ok {
				log.Printf("[DEBUG] Skipping non-AWS provider")
				continue
			}

			clientRegion := client.region
			log.Printf("[DEBUG] Checking AWS provider region %q against %q", clientRegion, region)
			if clientRegion == region {
				log.Printf("[DEBUG] Found AWS provider with region: %s", region)
				return provider
			}
		}

		log.Printf("[DEBUG] No suitable provider found for %q in %d providers", region, len(*providers))
		return nil
	}
}

func testAccCheckWithProviders(f func(*terraform.State, *schema.Provider) error, providers *[]*schema.Provider) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		numberOfProviders := len(*providers)
		for i, provider := range *providers {
			if provider.Meta() == nil {
				log.Printf("[DEBUG] Skipping empty provider %d (total: %d)", i, numberOfProviders)
				continue
			}
			log.Printf("[DEBUG] Calling check with provider %d (total: %d)", i, numberOfProviders)
			if err := f(s, provider); err != nil {
				return err
			}
		}
		return nil
	}
}

// Check service API call error for reasons to skip acceptance testing
// These include missing API endpoints and unsupported API calls
func testAccPreCheckSkipError(err error) bool {
	// Ignore missing API endpoints
	if isAWSErr(err, "RequestError", "send request failed") {
		return true
	}
	// Ignore unsupported API calls
	if isAWSErr(err, "UnsupportedOperation", "") {
		return true
	}
	return false
}

// Check sweeper API call error for reasons to skip sweeping
// These include missing API endpoints and unsupported API calls
func testSweepSkipSweepError(err error) bool {
	// Ignore missing API endpoints
	if isAWSErr(err, "RequestError", "send request failed") {
		return true
	}
	// Ignore unsupported API calls
	if isAWSErr(err, "UnsupportedOperation", "") {
		return true
	}
	// Ignore more unsupported API calls
	// InvalidParameterValue: Use of cache security groups is not permitted in this API version for your account.
	if isAWSErr(err, "InvalidParameterValue", "not permitted in this API version for your account") {
		return true
	}
	// GovCloud has endpoints that respond with (no message provided):
	// AccessDeniedException:
	// Since acceptance test sweepers are best effort and this response is very common,
	// we allow bypassing this error globally instead of individual test sweeper fixes.
	if isAWSErr(err, "AccessDeniedException", "") {
		return true
	}
	return false
}
<|endoftext|>